diff --git a/src/highlight.c b/src/highlight.c
index 9ce6102..6291f24 100644
--- a/src/highlight.c
+++ b/src/highlight.c
@@ -2929,7 +2929,12 @@
 #  endif
 	at_en.ae_u.cterm.fg_rgb = GUI_MCH_GET_RGB2(sgp->sg_gui_fg);
 	at_en.ae_u.cterm.bg_rgb = GUI_MCH_GET_RGB2(sgp->sg_gui_bg);
-	at_en.ae_u.cterm.ul_rgb = GUI_MCH_GET_RGB2(sgp->sg_gui_sp);
+	// Only use the underline/undercurl color when used, it may clear the
+	// background color if not supported.
+	if (sgp->sg_cterm & (HL_UNDERLINE | HL_UNDERCURL))
+	    at_en.ae_u.cterm.ul_rgb = GUI_MCH_GET_RGB2(sgp->sg_gui_sp);
+	else
+	    at_en.ae_u.cterm.ul_rgb = INVALCOLOR;
 	if (at_en.ae_u.cterm.fg_rgb == INVALCOLOR
 		&& at_en.ae_u.cterm.bg_rgb == INVALCOLOR)
 	{