diff --git a/src/drawline.c b/src/drawline.c
index 43604c8..ea7764e 100644
--- a/src/drawline.c
+++ b/src/drawline.c
@@ -1113,11 +1113,11 @@
 		      // the line number itself.
 		      // TODO: Can we use CursorLine instead of CursorLineNr
 		      // when CursorLineNr isn't set?
-		      if ((wp->w_p_cul || wp->w_p_rnu)
+		      if (wp->w_p_cul
+			      && lnum == wp->w_cursor.lnum
 			      && (wp->w_p_culopt_flags & CULOPT_NBR)
 			      && (row == startrow
-				  || wp->w_p_culopt_flags & CULOPT_LINE)
-			      && lnum == wp->w_cursor.lnum)
+				  || wp->w_p_culopt_flags & CULOPT_LINE))
 			char_attr = hl_combine_attr(wcr_attr, HL_ATTR(HLF_CLN));
 #endif
 		    }