diff --git a/src/search.c b/src/search.c
index 36410e5..6e62184 100644
--- a/src/search.c
+++ b/src/search.c
@@ -4241,7 +4241,11 @@
 	 * line, we get stuck there.  Trap this here. */
 	if (VIsual_mode == 'V' && start_lnum == curwin->w_cursor.lnum)
 	    goto extend;
-	VIsual.lnum = start_lnum;
+	if (VIsual.lnum != start_lnum)
+	{
+	    VIsual.lnum = start_lnum;
+	    VIsual.col = 0;
+	}
 	VIsual_mode = 'V';
 	redraw_curbuf_later(INVERTED);	/* update the inversion */
 	showmode();