diff --git a/src/insexpand.c b/src/insexpand.c
index c22eb6d..f176d75 100644
--- a/src/insexpand.c
+++ b/src/insexpand.c
@@ -2687,16 +2687,17 @@
 {
     int	retval;
 
-    if (*curbuf->b_p_tsrfu != NUL)
-    {
+    if (args->os_flags & OPT_LOCAL)
 	// buffer-local option set
 	retval = option_set_callback_func(curbuf->b_p_tsrfu,
 							&curbuf->b_tsrfu_cb);
-    }
     else
     {
 	// global option set
 	retval = option_set_callback_func(p_tsrfu, &tsrfu_cb);
+	// when using :set, free the local callback
+	if (!(args->os_flags & OPT_GLOBAL))
+	    free_callback(&curbuf->b_tsrfu_cb);
     }
 
     return retval == FAIL ? e_invalid_argument : NULL;