diff --git a/src/diff.c b/src/diff.c
index b59a125..92a4d9f 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -656,7 +656,8 @@
 {
     win_T	*wp;
     win_T	*wp_other = NULL;
-    int		used_max_fill = FALSE;
+    int		used_max_fill_other = FALSE;
+    int		used_max_fill_curwin = FALSE;
     int		n;
 
     need_diff_redraw = FALSE;
@@ -681,16 +682,25 @@
 		{
 		    wp->w_topfill = n;
 		    if (wp == curwin)
-			used_max_fill = TRUE;
+			used_max_fill_curwin = TRUE;
+		    else if (wp_other != NULL)
+			used_max_fill_other = TRUE;
 		}
 		check_topfill(wp, FALSE);
 	    }
 	}
 
-    if (wp_other != NULL && used_max_fill && curwin->w_p_scb)
-	// The current window was set to used the maximum number of filler
-	// lines, may need to reduce them.
-	diff_set_topline(wp_other, curwin);
+    if (wp_other != NULL && curwin->w_p_scb)
+    {
+	if (used_max_fill_curwin)
+	    // The current window was set to used the maximum number of filler
+	    // lines, may need to reduce them.
+	    diff_set_topline(wp_other, curwin);
+	else if (used_max_fill_other)
+	    // The other window was set to used the maximum number of filler
+	    // lines, may need to reduce them.
+	    diff_set_topline(curwin, wp_other);
+    }
 }
 
     static void
diff --git a/src/testdir/dumps/Test_diff_scroll_change_03.dump b/src/testdir/dumps/Test_diff_scroll_change_03.dump
new file mode 100644
index 0000000..6d0c860
--- /dev/null
+++ b/src/testdir/dumps/Test_diff_scroll_change_03.dump
@@ -0,0 +1,20 @@
+| +0#0000e05#a8a8a8255@1|-+0#4040ff13#afffff255@34||+1#0000000#ffffff0| +0#0000e05#a8a8a8255@1|6+0#0000000#5fd7ff255| @33
+| +0#0000e05#a8a8a8255@1|-+0#4040ff13#afffff255@34||+1#0000000#ffffff0| +0#0000e05#a8a8a8255@1|7+0#0000000#5fd7ff255| @33
+| +0#0000e05#a8a8a8255@1|-+0#4040ff13#afffff255@34||+1#0000000#ffffff0| +0#0000e05#a8a8a8255@1|8+0#0000000#5fd7ff255| @33
+| +0#0000e05#a8a8a8255@1|9+0#0000000#ffffff0| @33||+1&&| +0#0000e05#a8a8a8255@1|9+0#0000000#ffffff0| @33
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|0| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|0| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0@1| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0@1| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|2| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|2| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|3| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|3| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|4| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|4| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffd7ff255|5|x+2&#ff404010| +0&#ffd7ff255@31||+1&#ffffff0| +0#0000e05#a8a8a8255@1|1+0#0000000#ffd7ff255|5>y+2&#ff404010| +0&#ffd7ff255@31
+|~+0#4040ff13#ffffff0| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|[+1#0000000&|N|o| |N|a|m|e|]| |[|+|]| @5|7|,|3| @11|A|l@1| |[+3&&|N|o| |N|a|m|e|]| |[|+|]| @5|1|5|,|3| @10|B|o|t
+| +0&&@74
diff --git a/src/testdir/test_diffmode.vim b/src/testdir/test_diffmode.vim
index 08ec71b..e8bd087 100644
--- a/src/testdir/test_diffmode.vim
+++ b/src/testdir/test_diffmode.vim
@@ -1013,6 +1013,9 @@
   call term_sendkeys(buf, "ax\<Esc>")
   call VerifyScreenDump(buf, 'Test_diff_scroll_change_02', {})
 
+  call term_sendkeys(buf, "\<C-W>lay\<Esc>")
+  call VerifyScreenDump(buf, 'Test_diff_scroll_change_03', {})
+
   " clean up
   call StopVimInTerminal(buf)
   call delete('Xtest_scroll_change')
diff --git a/src/version.c b/src/version.c
index 1c2231f..362129e 100644
--- a/src/version.c
+++ b/src/version.c
@@ -758,6 +758,8 @@
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    3556,
+/**/
     3555,
 /**/
     3554,