diff --git a/src/mouse.c b/src/mouse.c
index 98e0595..32407eb 100644
--- a/src/mouse.c
+++ b/src/mouse.c
@@ -3034,14 +3034,29 @@
 		    row -= win->w_topfill;
 		else
 		    row -= diff_check_fill(win, lnum);
-		count = plines_win_nofill(win, lnum, TRUE);
+		count = plines_win_nofill(win, lnum, FALSE);
 	    }
 	    else
 #endif
-		count = plines_win(win, lnum, TRUE);
+		count = plines_win(win, lnum, FALSE);
 	    if (plines_cache != NULL && cache_idx < Rows)
 		plines_cache[cache_idx] = count;
 	}
+
+	if (win->w_skipcol > 0 && lnum == win->w_topline)
+	{
+	    // Adjust for 'smoothscroll' clipping the top screen lines.
+	    // A similar formula is used in curs_columns().
+	    int width1 = win->w_width - win_col_off(win);
+	    int skip_lines = 0;
+	    if (win->w_skipcol > width1)
+		skip_lines = (win->w_skipcol - width1)
+					    / (width1 + win_col_off2(win)) + 1;
+	    else if (win->w_skipcol > 0)
+		skip_lines = 1;
+	    count -= skip_lines;
+	}
+
 	if (count > row)
 	    break;	// Position is in this buffer line.
 #ifdef FEAT_FOLDING
@@ -3063,8 +3078,10 @@
 	if (col < off)
 	    col = off;
 	col += row * (win->w_width - off);
-	// add skip column (for long wrapping line)
-	col += win->w_skipcol;
+
+	// Add skip column for the topline.
+	if (lnum == win->w_topline)
+	    col += win->w_skipcol;
     }
 
     if (!win->w_p_wrap)