diff --git a/src/mouse.c b/src/mouse.c
index f1c1aab..c17e26a 100644
--- a/src/mouse.c
+++ b/src/mouse.c
@@ -477,7 +477,7 @@
 		if ((mod_mask & MOD_MASK_MULTI_CLICK) == MOD_MASK_2CLICK)
 		{
 		    // double click opens new page
-		    end_visual_mode();
+		    end_visual_mode_keep_button();
 		    tabpage_new();
 		    tabpage_move(c1 == 0 ? 9999 : c1 - 1);
 		}
@@ -489,7 +489,7 @@
 
 		    // It's like clicking on the status line of a window.
 		    if (curwin != old_curwin)
-			end_visual_mode();
+			end_visual_mode_keep_button();
 		}
 	    }
 	    else
@@ -1568,7 +1568,7 @@
 #endif
 	if (flags & MOUSE_MAY_STOP_VIS)
 	{
-	    end_visual_mode();
+	    end_visual_mode_keep_button();
 	    redraw_curbuf_later(INVERTED);	// delete the inversion
 	}
 #if defined(FEAT_CMDWIN) && defined(FEAT_CLIPBOARD)
@@ -1717,7 +1717,7 @@
 #endif
 			&& (flags & MOUSE_MAY_STOP_VIS))))
 	{
-	    end_visual_mode();
+	    end_visual_mode_keep_button();
 	    redraw_curbuf_later(INVERTED);	// delete the inversion
 	}
 #ifdef FEAT_CMDWIN
@@ -1821,7 +1821,7 @@
 	// before moving the cursor for a left click, stop Visual mode
 	if (flags & MOUSE_MAY_STOP_VIS)
 	{
-	    end_visual_mode();
+	    end_visual_mode_keep_button();
 	    redraw_curbuf_later(INVERTED);	// delete the inversion
 	}