diff --git a/src/buffer.c b/src/buffer.c
index 55e200a..b013295 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -580,7 +580,7 @@
 
     /* When closing the current buffer stop Visual mode before freeing
      * anything. */
-    if (buf == curbuf
+    if (buf == curbuf && VIsual_active
 #if defined(EXITFREE)
 	    && !entered_free_all_mem
 #endif
@@ -1389,7 +1389,7 @@
 	}
 
 	/* When closing the current buffer stop Visual mode. */
-	if (buf == curbuf)
+	if (buf == curbuf && VIsual_active)
 	    end_visual_mode();
 
 	/*