diff --git a/src/buffer.c b/src/buffer.c
index 3bfd975..55e200a 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -580,7 +580,11 @@
 
     /* When closing the current buffer stop Visual mode before freeing
      * anything. */
-    if (buf == curbuf)
+    if (buf == curbuf
+#if defined(EXITFREE)
+	    && !entered_free_all_mem
+#endif
+	    )
 	end_visual_mode();
 
     /*