diff --git a/src/diff.c b/src/diff.c
index 7bdfa48..b59a125 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -655,6 +655,8 @@
     int		dofold)	    // also recompute the folds
 {
     win_T	*wp;
+    win_T	*wp_other = NULL;
+    int		used_max_fill = FALSE;
     int		n;
 
     need_diff_redraw = FALSE;
@@ -662,6 +664,8 @@
 	if (wp->w_p_diff)
 	{
 	    redraw_win_later(wp, SOME_VALID);
+	    if (wp != curwin)
+		wp_other = wp;
 #ifdef FEAT_FOLDING
 	    if (dofold && foldmethodIsDiff(wp))
 		foldUpdateAll(wp);
@@ -674,10 +678,19 @@
 		if (wp->w_topfill > n)
 		    wp->w_topfill = (n < 0 ? 0 : n);
 		else if (n > 0 && n > wp->w_topfill)
+		{
 		    wp->w_topfill = n;
+		    if (wp == curwin)
+			used_max_fill = TRUE;
+		}
 		check_topfill(wp, FALSE);
 	    }
 	}
+
+    if (wp_other != NULL && used_max_fill && curwin->w_p_scb)
+	// The current window was set to used the maximum number of filler
+	// lines, may need to reduce them.
+	diff_set_topline(wp_other, curwin);
 }
 
     static void
diff --git a/src/move.c b/src/move.c
index 20a1a25..ea86b88 100644
--- a/src/move.c
+++ b/src/move.c
@@ -535,6 +535,10 @@
     void
 set_topline(win_T *wp, linenr_T lnum)
 {
+#ifdef FEAT_DIFF
+    linenr_T prev_topline = wp->w_topline;
+#endif
+
 #ifdef FEAT_FOLDING
     // go to first of folded lines
     (void)hasFoldingWin(wp, lnum, &lnum, NULL, TRUE, NULL);
@@ -546,7 +550,9 @@
     wp->w_topline = lnum;
     wp->w_topline_was_set = TRUE;
 #ifdef FEAT_DIFF
-    wp->w_topfill = 0;
+    if (lnum != prev_topline)
+	// Keep the filler lines when the topline didn't change.
+	wp->w_topfill = 0;
 #endif
     wp->w_valid &= ~(VALID_WROW|VALID_CROW|VALID_BOTLINE|VALID_TOPLINE);
     // Don't set VALID_TOPLINE here, 'scrolloff' needs to be checked.
diff --git a/src/testdir/dumps/Test_diff_scroll_change_01.dump b/src/testdir/dumps/Test_diff_scroll_change_01.dump
new file mode 100644
index 0000000..886fe60
--- /dev/null
+++ b/src/testdir/dumps/Test_diff_scroll_change_01.dump
@@ -0,0 +1,20 @@
+| +0#0000e05#a8a8a8255@1|-+0#4040ff13#afffff255@34||+1#0000000#ffffff0| +0#0000e05#a8a8a8255@1|6+0#0000000#5fd7ff255| @33
+| +0#0000e05#a8a8a8255@1|-+0#4040ff13#afffff255@34||+1#0000000#ffffff0| +0#0000e05#a8a8a8255@1|7+0#0000000#5fd7ff255| @33
+| +0#0000e05#a8a8a8255@1|-+0#4040ff13#afffff255@34||+1#0000000#ffffff0| +0#0000e05#a8a8a8255@1|8+0#0000000#5fd7ff255| @33
+| +0#0000e05#a8a8a8255@1|9+0#0000000#ffffff0| @33||+1&&| +0#0000e05#a8a8a8255@1|9+0#0000000#ffffff0| @33
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|0| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|0| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0@1| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0@1| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|2| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|2| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|3| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|3| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|4| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|4| @32
+|-+0#0000e05#a8a8a8255| |1+0#0000000#ffffff0>5| @32||+1&&|-+0#0000e05#a8a8a8255| |1+0#0000000#ffffff0|5| @32
+|~+0#4040ff13&| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|[+3#0000000&|N|o| |N|a|m|e|]| |[|+|]| @5|7|,|2| @11|A|l@1| |[+1&&|N|o| |N|a|m|e|]| |[|+|]| @5|1|5|,|2| @10|B|o|t
+| +0&&@74
diff --git a/src/testdir/dumps/Test_diff_scroll_change_02.dump b/src/testdir/dumps/Test_diff_scroll_change_02.dump
new file mode 100644
index 0000000..d699ded
--- /dev/null
+++ b/src/testdir/dumps/Test_diff_scroll_change_02.dump
@@ -0,0 +1,20 @@
+| +0#0000e05#a8a8a8255@1|-+0#4040ff13#afffff255@34||+1#0000000#ffffff0| +0#0000e05#a8a8a8255@1|6+0#0000000#5fd7ff255| @33
+| +0#0000e05#a8a8a8255@1|-+0#4040ff13#afffff255@34||+1#0000000#ffffff0| +0#0000e05#a8a8a8255@1|7+0#0000000#5fd7ff255| @33
+| +0#0000e05#a8a8a8255@1|-+0#4040ff13#afffff255@34||+1#0000000#ffffff0| +0#0000e05#a8a8a8255@1|8+0#0000000#5fd7ff255| @33
+| +0#0000e05#a8a8a8255@1|9+0#0000000#ffffff0| @33||+1&&| +0#0000e05#a8a8a8255@1|9+0#0000000#ffffff0| @33
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|0| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|0| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0@1| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0@1| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|2| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|2| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|3| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|3| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|4| @32||+1&&| +0#0000e05#a8a8a8255@1|1+0#0000000#ffffff0|4| @32
+| +0#0000e05#a8a8a8255@1|1+0#0000000#ffd7ff255|5>x+2&#ff404010| +0&#ffd7ff255@31||+1&#ffffff0| +0#0000e05#a8a8a8255@1|1+0#0000000#ffd7ff255|5| @32
+|~+0#4040ff13#ffffff0| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|~| @35||+1#0000000&|~+0#4040ff13&| @35
+|[+3#0000000&|N|o| |N|a|m|e|]| |[|+|]| @5|7|,|3| @11|A|l@1| |[+1&&|N|o| |N|a|m|e|]| |[|+|]| @5|1|5|,|3| @10|B|o|t
+| +0&&@74
diff --git a/src/testdir/test_diffmode.vim b/src/testdir/test_diffmode.vim
index 5400de6..268c3e9 100644
--- a/src/testdir/test_diffmode.vim
+++ b/src/testdir/test_diffmode.vim
@@ -993,6 +993,30 @@
   call delete('XdiffSetup')
 endfunc
 
+func Test_diff_with_scroll_and_change()
+  CheckScreendump
+
+  let lines =<< trim END
+	call setline(1, range(1, 15))
+	vnew
+	call setline(1, range(9, 15))
+	windo diffthis
+	wincmd h
+	exe "normal Gl5\<C-E>"
+  END
+  call writefile(lines, 'Xtest_scroll_change')
+  let buf = RunVimInTerminal('-S Xtest_scroll_change', {})
+
+  call VerifyScreenDump(buf, 'Test_diff_scroll_change_01', {})
+
+  call term_sendkeys(buf, "ax\<Esc>")
+  call VerifyScreenDump(buf, 'Test_diff_scroll_change_02', {})
+
+  " clean up
+  call StopVimInTerminal(buf)
+  call delete('Xtest_scroll_change')
+endfunc
+
 func Test_diff_with_cursorline()
   CheckScreendump
 
diff --git a/src/version.c b/src/version.c
index 3a16dda..8811ee1 100644
--- a/src/version.c
+++ b/src/version.c
@@ -756,6 +756,8 @@
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    3394,
+/**/
     3393,
 /**/
     3392,