diff --git a/src/diff.c b/src/diff.c
index b59a125..92a4d9f 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -656,7 +656,8 @@
 {
     win_T	*wp;
     win_T	*wp_other = NULL;
-    int		used_max_fill = FALSE;
+    int		used_max_fill_other = FALSE;
+    int		used_max_fill_curwin = FALSE;
     int		n;
 
     need_diff_redraw = FALSE;
@@ -681,16 +682,25 @@
 		{
 		    wp->w_topfill = n;
 		    if (wp == curwin)
-			used_max_fill = TRUE;
+			used_max_fill_curwin = TRUE;
+		    else if (wp_other != NULL)
+			used_max_fill_other = TRUE;
 		}
 		check_topfill(wp, FALSE);
 	    }
 	}
 
-    if (wp_other != NULL && used_max_fill && curwin->w_p_scb)
-	// The current window was set to used the maximum number of filler
-	// lines, may need to reduce them.
-	diff_set_topline(wp_other, curwin);
+    if (wp_other != NULL && curwin->w_p_scb)
+    {
+	if (used_max_fill_curwin)
+	    // The current window was set to used the maximum number of filler
+	    // lines, may need to reduce them.
+	    diff_set_topline(wp_other, curwin);
+	else if (used_max_fill_other)
+	    // The other window was set to used the maximum number of filler
+	    // lines, may need to reduce them.
+	    diff_set_topline(curwin, wp_other);
+    }
 }
 
     static void