diff --git a/src/diff.c b/src/diff.c
index d9aaec8..864e609 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -615,11 +615,13 @@
 #endif
 	    /* A change may have made filler lines invalid, need to take care
 	     * of that for other windows. */
-	    if (wp != curwin && wp->w_topfill > 0)
+	    n = diff_check(wp, wp->w_topline);
+	    if ((wp != curwin && wp->w_topfill > 0) || n > 0)
 	    {
-		n = diff_check(wp, wp->w_topline);
 		if (wp->w_topfill > n)
 		    wp->w_topfill = (n < 0 ? 0 : n);
+		else if (n > 0 && n > wp->w_topfill)
+		    wp->w_topfill = n;
 	    }
 	}
 }