diff --git a/src/diff.c b/src/diff.c
index df3dea0..eddf331 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -119,7 +119,12 @@
 	    tp->tp_diffbuf[i] = NULL;
 	    tp->tp_diff_invalid = TRUE;
 	    if (tp == curtab)
-		diff_redraw(TRUE);
+	    {
+		// don't redraw right away, more might change or buffer state
+		// is invalid right now
+		need_diff_redraw = TRUE;
+		redraw_later(VALID);
+	    }
 	}
     }
 }
@@ -670,7 +675,8 @@
 
     need_diff_redraw = FALSE;
     FOR_ALL_WINDOWS(wp)
-	if (wp->w_p_diff)
+	// when closing windows or wiping buffers skip invalid window
+	if (wp->w_p_diff && buf_valid(wp->w_buffer))
 	{
 	    redraw_win_later(wp, SOME_VALID);
 	    if (wp != curwin)
diff --git a/src/testdir/test_diffmode.vim b/src/testdir/test_diffmode.vim
index 03c6c79..afa8f89 100644
--- a/src/testdir/test_diffmode.vim
+++ b/src/testdir/test_diffmode.vim
@@ -1603,5 +1603,17 @@
   call delete('Xright')
 endfunc
 
+" This was trying to update diffs for a buffer being closed
+func Test_diff_only()
+  silent! lfile
+  set diff
+  lopen
+  norm o
+  silent! norm o
+
+  set nodiff
+  %bwipe!
+endfunc
+
 
 " vim: shiftwidth=2 sts=2 expandtab
diff --git a/src/version.c b/src/version.c
index cf2e77d..c847d59 100644
--- a/src/version.c
+++ b/src/version.c
@@ -736,6 +736,8 @@
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    5163,
+/**/
     5162,
 /**/
     5161,