diff --git a/src/change.c b/src/change.c
index 54ed684..afc32b7 100644
--- a/src/change.c
+++ b/src/change.c
@@ -641,7 +641,7 @@
 		set_topline(wp, wp->w_topline);
 #endif
 	    // Relative numbering may require updating more.
-	    if (wp->w_p_rnu)
+	    if (wp->w_p_rnu && xtra != 0)
 		redraw_win_later(wp, SOME_VALID);
 #ifdef FEAT_SYN_HL
 	    // Cursor line highlighting probably need to be updated with
diff --git a/src/drawscreen.c b/src/drawscreen.c
index cb47577..9f9cb09 100644
--- a/src/drawscreen.c
+++ b/src/drawscreen.c
@@ -2507,11 +2507,11 @@
 	}
 	else
 	{
-	    if (wp->w_p_rnu)
+	    if (wp->w_p_rnu && wp->w_last_cursor_lnum_rnu != wp->w_cursor.lnum)
 	    {
 #ifdef FEAT_FOLDING
-		// 'relativenumber' set: The text doesn't need to be drawn, but
-		// the number column nearly always does.
+		// 'relativenumber' set and the cursor moved vertically: The
+		// text doesn't need to be drawn, but the number column does.
 		fold_count = foldedCount(wp, lnum, &win_foldinfo);
 		if (fold_count != 0)
 		    fold_line(wp, fold_count, &win_foldinfo, lnum, row);
@@ -2553,6 +2553,7 @@
     // update w_last_cursorline.
     wp->w_last_cursorline = wp->w_p_cul ? wp->w_cursor.lnum : 0;
 #endif
+    wp->w_last_cursor_lnum_rnu = wp->w_p_rnu ? wp->w_cursor.lnum : 0;
 
 #ifdef FEAT_VTP
     // Rewrite the character at the end of the screen line.
diff --git a/src/structs.h b/src/structs.h
index a153336..192693b 100644
--- a/src/structs.h
+++ b/src/structs.h
@@ -3465,6 +3465,9 @@
     colnr_T	w_old_visual_col;   // last known start of visual part
     colnr_T	w_old_curswant;	    // last known value of Curswant
 
+    linenr_T    w_last_cursor_lnum_rnu;  // cursor lnum when 'rnu' was last
+					 // redrawn
+
     lcs_chars_T	w_lcs_chars;	    // 'listchars' characters
 
     /*
diff --git a/src/version.c b/src/version.c
index fe7f299..90b30ab 100644
--- a/src/version.c
+++ b/src/version.c
@@ -751,6 +751,8 @@
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    4644,
+/**/
     4643,
 /**/
     4642,