diff --git a/src/libvterm/include/vterm.h b/src/libvterm/include/vterm.h
index b6491f1..3a17404 100644
--- a/src/libvterm/include/vterm.h
+++ b/src/libvterm/include/vterm.h
@@ -85,18 +85,107 @@
 }
 #endif
 
-// The ansi_index is used for the lower 16 colors, which can be set to any
-// color.
-#define VTERM_ANSI_INDEX_DEFAULT 0	// color cleared
-#define VTERM_ANSI_INDEX_MIN 1
-#define VTERM_ANSI_INDEX_MAX 16
-#define VTERM_ANSI_INDEX_NONE 255	// non-ANSI color, use red/green/blue
+/**
+ * Bit-field describing the value of VTermColor.type
+ */
+typedef enum {
+  /**
+   * If the lower bit of `type` is not set, the colour is 24-bit RGB.
+   */
+  VTERM_COLOR_RGB = 0x00,
+
+  /**
+   * The colour is an index into a palette of 256 colours.
+   */
+  VTERM_COLOR_INDEXED = 0x01,
+
+  /**
+   * Mask that can be used to extract the RGB/Indexed bit.
+   */
+  VTERM_COLOR_TYPE_MASK = 0x01,
+
+  /**
+   * If set, indicates that this colour should be the default foreground
+   * color, i.e. there was no SGR request for another colour. When
+   * rendering this colour it is possible to ignore "idx" and just use a
+   * colour that is not in the palette.
+   */
+  VTERM_COLOR_DEFAULT_FG = 0x02,
+
+  /**
+   * If set, indicates that this colour should be the default background
+   * color, i.e. there was no SGR request for another colour. A common
+   * option when rendering this colour is to not render a background at
+   * all, for example by rendering the window transparently at this spot.
+   */
+  VTERM_COLOR_DEFAULT_BG = 0x04,
+
+  /**
+   * Mask that can be used to extract the default foreground/background bit.
+   */
+  VTERM_COLOR_DEFAULT_MASK = 0x06
+} VTermColorType;
+
+/**
+ * Returns true if the VTERM_COLOR_RGB `type` flag is set, indicating that the
+ * given VTermColor instance is an indexed colour.
+ */
+#define VTERM_COLOR_IS_INDEXED(col) \
+  (((col)->type & VTERM_COLOR_TYPE_MASK) == VTERM_COLOR_INDEXED)
+
+/**
+ * Returns true if the VTERM_COLOR_INDEXED `type` flag is set, indicating that
+ * the given VTermColor instance is an rgb colour.
+ */
+#define VTERM_COLOR_IS_RGB(col) \
+  (((col)->type & VTERM_COLOR_TYPE_MASK) == VTERM_COLOR_RGB)
+
+/**
+ * Returns true if the VTERM_COLOR_DEFAULT_FG `type` flag is set, indicating
+ * that the given VTermColor instance corresponds to the default foreground
+ * color.
+ */
+#define VTERM_COLOR_IS_DEFAULT_FG(col) \
+  (!!((col)->type & VTERM_COLOR_DEFAULT_FG))
+
+/**
+ * Returns true if the VTERM_COLOR_DEFAULT_BG `type` flag is set, indicating
+ * that the given VTermColor instance corresponds to the default background
+ * color.
+ */
+#define VTERM_COLOR_IS_DEFAULT_BG(col) \
+  (!!((col)->type & VTERM_COLOR_DEFAULT_BG))
 
 typedef struct {
+  /**
+   * Tag indicating which member is actually valid.
+   * Please use the `VTERM_COLOR_IS_*` test macros to check whether a
+   * particular type flag is set.
+   */
+  uint8_t type;
+
   uint8_t red, green, blue;
-  uint8_t ansi_index;
+
+  uint8_t index;
 } VTermColor;
 
+/**
+ * Constructs a new VTermColor instance representing the given RGB values.
+ */
+void vterm_color_rgb(VTermColor *col, uint8_t red, uint8_t green, uint8_t blue);
+
+/**
+ * Construct a new VTermColor instance representing an indexed color with the
+ * given index.
+ */
+void vterm_color_indexed(VTermColor *col, uint8_t idx);
+
+/**
+ * Compares two colours. Returns true if the colors are equal, false otherwise.
+ */
+int vterm_color_is_equal(const VTermColor *a, const VTermColor *b);
+
+
 typedef enum {
   // VTERM_VALUETYPE_NONE = 0
   VTERM_VALUETYPE_BOOL = 1,
@@ -346,6 +435,18 @@
 void vterm_state_focus_out(VTermState *state);
 const VTermLineInfo *vterm_state_get_lineinfo(const VTermState *state, int row);
 
+/**
+ * Makes sure that the given color `col` is indeed an RGB colour. After this
+ * function returns, VTERM_COLOR_IS_RGB(col) will return true, while all other
+ * flags stored in `col->type` will have been reset.
+ *
+ * @param state is the VTermState instance from which the colour palette should
+ * be extracted.
+ * @param col is a pointer at the VTermColor instance that should be converted
+ * to an RGB colour.
+ */
+void vterm_state_convert_color_to_rgb(const VTermState *state, VTermColor *col);
+
 // ------------
 // Screen layer
 // ------------
@@ -456,6 +557,12 @@
 
 int vterm_screen_is_eol(const VTermScreen *screen, VTermPos pos);
 
+/**
+ * Same as vterm_state_convert_color_to_rgb(), but takes a `screen` instead of a `state`
+ * instance.
+ */
+void vterm_screen_convert_color_to_rgb(const VTermScreen *screen, VTermColor *col);
+
 // ---------
 // Utilities
 // ---------
diff --git a/src/libvterm/src/pen.c b/src/libvterm/src/pen.c
index 831d180..98c5af9 100644
--- a/src/libvterm/src/pen.c
+++ b/src/libvterm/src/pen.c
@@ -2,26 +2,34 @@
 
 #include <stdio.h>
 
-static const VTermColor ansi_colors[] = {
-  // R    G    B   index
-  {   0,   0,   0,  1 }, // black
-  { 224,   0,   0,  2 }, // red
-  {   0, 224,   0,  3 }, // green
-  { 224, 224,   0,  4 }, // yellow
-  {   0,   0, 224,  5 }, // blue
-  { 224,   0, 224,  6 }, // magenta
-  {   0, 224, 224,  7 }, // cyan
-  { 224, 224, 224,  8 }, // white == light grey
+/**
+ * Structure used to store RGB triples without the additional metadata stored in
+ * VTermColor.
+ */
+typedef struct {
+  uint8_t red, green, blue;
+} VTermRGB;
+
+static const VTermRGB ansi_colors[] = {
+  // R    G    B
+  {   0,   0,   0 }, // black
+  { 224,   0,   0 }, // red
+  {   0, 224,   0 }, // green
+  { 224, 224,   0 }, // yellow
+  {   0,   0, 224 }, // blue
+  { 224,   0, 224 }, // magenta
+  {   0, 224, 224 }, // cyan
+  { 224, 224, 224 }, // white == light grey
 
   // high intensity
-  { 128, 128, 128,  9 }, // black
-  { 255,  64,  64, 10 }, // red
-  {  64, 255,  64, 11 }, // green
-  { 255, 255,  64, 12 }, // yellow
-  {  64,  64, 255, 13 }, // blue
-  { 255,  64, 255, 14 }, // magenta
-  {  64, 255, 255, 15 }, // cyan
-  { 255, 255, 255, 16 }, // white for real
+  { 128, 128, 128 }, // black
+  { 255,  64,  64 }, // red
+  {  64, 255,  64 }, // green
+  { 255, 255,  64 }, // yellow
+  {  64,  64, 255 }, // blue
+  { 255,  64, 255 }, // magenta
+  {  64, 255, 255 }, // cyan
+  { 255, 255, 255 }, // white for real
 };
 
 static int ramp6[] = {
@@ -34,6 +42,15 @@
   0x81, 0x8A, 0x94, 0x9E, 0xA8, 0xB2, 0xBC, 0xC6, 0xD0, 0xDA, 0xE4, 0xEE,
 };
 
+static void lookup_default_colour_ansi(long idx, VTermColor *col)
+{
+  vterm_color_rgb(
+      col,
+      ansi_colors[idx].red, ansi_colors[idx].green, ansi_colors[idx].blue);
+  col->index = idx;
+  col->type = VTERM_COLOR_INDEXED;
+}
+
 static int lookup_colour_ansi(const VTermState *state, long index, VTermColor *col)
 {
   if(index >= 0 && index < 16) {
@@ -54,10 +71,9 @@
     // 216-colour cube
     index -= 16;
 
-    col->blue  = ramp6[index     % 6];
-    col->green = ramp6[index/6   % 6];
-    col->red   = ramp6[index/6/6 % 6];
-    col->ansi_index = VTERM_ANSI_INDEX_NONE;
+    vterm_color_rgb(col, ramp6[index/6/6 % 6],
+                         ramp6[index/6   % 6],
+                         ramp6[index     % 6]);
 
     return TRUE;
   }
@@ -65,10 +81,7 @@
     // 24 greyscales
     index -= 232;
 
-    col->blue  = ramp24[index];
-    col->green = ramp24[index];
-    col->red   = ramp24[index];
-    col->ansi_index = VTERM_ANSI_INDEX_NONE;
+    vterm_color_rgb(col, ramp24[index], ramp24[index], ramp24[index]);
 
     return TRUE;
   }
@@ -76,25 +89,23 @@
   return FALSE;
 }
 
-static int lookup_colour(const VTermState *state, int palette, const long args[], int argcount, VTermColor *col, int *index)
+static int lookup_colour(const VTermState *state, int palette, const long args[], int argcount, VTermColor *col)
 {
   switch(palette) {
   case 2: // RGB mode - 3 args contain colour values directly
     if(argcount < 3)
       return argcount;
 
-    col->red   = (uint8_t)CSI_ARG(args[0]);
-    col->green = (uint8_t)CSI_ARG(args[1]);
-    col->blue  = (uint8_t)CSI_ARG(args[2]);
-    col->ansi_index = VTERM_ANSI_INDEX_NONE;
+    vterm_color_rgb(col, CSI_ARG(args[0]), CSI_ARG(args[1]), CSI_ARG(args[2]));
 
     return 3;
 
   case 5: // XTerm 256-colour mode
-    if(index)
-      *index = CSI_ARG_OR(args[0], -1);
+    if (!argcount || CSI_ARG_IS_MISSING(args[0])) {
+      return argcount ? 1 : 0;
+    }
 
-    lookup_colour_palette(state, argcount ? CSI_ARG_OR(args[0], -1) : -1, col);
+    lookup_colour_palette(state, args[0], col);
 
     return argcount ? 1 : 0;
 
@@ -154,13 +165,12 @@
   int col;
 
   // 90% grey so that pure white is brighter
-  state->default_fg.red = state->default_fg.green = state->default_fg.blue = 240;
-  state->default_fg.ansi_index = VTERM_ANSI_INDEX_DEFAULT;
-  state->default_bg.red = state->default_bg.green = state->default_bg.blue = 0;
-  state->default_bg.ansi_index = VTERM_ANSI_INDEX_DEFAULT;
+  vterm_color_rgb(&state->default_fg, 240, 240, 240);
+  vterm_color_rgb(&state->default_bg, 0, 0, 0);
+  vterm_state_set_default_colors(state, &state->default_fg, &state->default_bg);
 
   for(col = 0; col < 16; col++)
-    state->colors[col] = ansi_colors[col];
+    lookup_default_colour_ansi(col, &state->colors[col]);
 }
 
 INTERNAL void vterm_state_resetpen(VTermState *state)
@@ -174,8 +184,6 @@
   state->pen.strike = 0;    setpenattr_bool(state, VTERM_ATTR_STRIKE, 0);
   state->pen.font = 0;      setpenattr_int( state, VTERM_ATTR_FONT, 0);
 
-  state->fg_index = -1;
-  state->bg_index = -1;
   state->pen.fg = state->default_fg;  setpenattr_col(state, VTERM_ATTR_FOREGROUND, state->default_fg);
   state->pen.bg = state->default_bg;  setpenattr_col(state, VTERM_ATTR_BACKGROUND, state->default_bg);
 }
@@ -201,6 +209,40 @@
   }
 }
 
+void vterm_color_rgb(VTermColor *col, uint8_t red, uint8_t green, uint8_t blue)
+{
+  col->type = VTERM_COLOR_RGB;
+  col->red   = red;
+  col->green = green;
+  col->blue  = blue;
+}
+
+void vterm_color_indexed(VTermColor *col, uint8_t idx)
+{
+  col->type = VTERM_COLOR_INDEXED;
+  col->index = idx;
+}
+
+int vterm_color_is_equal(const VTermColor *a, const VTermColor *b)
+{
+  /* First make sure that the two colours are of the same type (RGB/Indexed) */
+  if (a->type != b->type) {
+    return FALSE;
+  }
+
+  /* Depending on the type inspect the corresponding members */
+  if (VTERM_COLOR_IS_INDEXED(a)) {
+    return a->index == b->index;
+  }
+  else if (VTERM_COLOR_IS_RGB(a)) {
+    return    (a->red   == b->red)
+           && (a->green == b->green)
+           && (a->blue  == b->blue);
+  }
+
+  return 0;
+}
+
 void vterm_state_get_default_colors(const VTermState *state, VTermColor *default_fg, VTermColor *default_bg)
 {
   *default_fg = state->default_fg;
@@ -214,8 +256,15 @@
 
 void vterm_state_set_default_colors(VTermState *state, const VTermColor *default_fg, const VTermColor *default_bg)
 {
+  /* Copy the given colors */
   state->default_fg = *default_fg;
   state->default_bg = *default_bg;
+
+  /* Make sure the correct type flags are set */
+  state->default_fg.type = (state->default_fg.type & ~VTERM_COLOR_DEFAULT_MASK)
+                         | VTERM_COLOR_DEFAULT_FG;
+  state->default_bg.type = (state->default_bg.type & ~VTERM_COLOR_DEFAULT_MASK)
+                         | VTERM_COLOR_DEFAULT_BG;
 }
 
 void vterm_state_set_palette_color(VTermState *state, int index, const VTermColor *col)
@@ -223,10 +272,18 @@
   if(index >= 0 && index < 16)
   {
     state->colors[index] = *col;
-    state->colors[index].ansi_index = index + VTERM_ANSI_INDEX_MIN;
+    state->colors[index].index = index + 1;
   }
 }
 
+void vterm_state_convert_color_to_rgb(const VTermState *state, VTermColor *col)
+{
+  if (VTERM_COLOR_IS_INDEXED(col)) { /* Convert indexed colors to RGB */
+    lookup_colour_palette(state, col->index, col);
+  }
+  col->type &= VTERM_COLOR_TYPE_MASK; /* Reset any metadata but the type */
+}
+
 void vterm_state_set_bold_highbright(VTermState *state, int bold_is_highbright)
 {
   state->bold_is_highbright = bold_is_highbright;
@@ -251,12 +308,14 @@
       vterm_state_resetpen(state);
       break;
 
-    case 1: // Bold on
+    case 1: { // Bold on
+      const VTermColor *fg = &state->pen.fg;
       state->pen.bold = 1;
       setpenattr_bool(state, VTERM_ATTR_BOLD, 1);
-      if(state->fg_index > -1 && state->fg_index < 8 && state->bold_is_highbright)
-        set_pen_col_ansi(state, VTERM_ATTR_FOREGROUND, state->fg_index + (state->pen.bold ? 8 : 0));
+      if(!VTERM_COLOR_IS_DEFAULT_FG(fg) && VTERM_COLOR_IS_INDEXED(fg) && fg->index < 8 && state->bold_is_highbright)
+        set_pen_col_ansi(state, VTERM_ATTR_FOREGROUND, fg->index + (state->pen.bold ? 8 : 0));
       break;
+    }
 
     case 3: // Italic on
       state->pen.italic = 1;
@@ -354,22 +413,19 @@
     case 30: case 31: case 32: case 33:
     case 34: case 35: case 36: case 37: // Foreground colour palette
       value = CSI_ARG(args[argi]) - 30;
-      state->fg_index = value;
       if(state->pen.bold && state->bold_is_highbright)
         value += 8;
       set_pen_col_ansi(state, VTERM_ATTR_FOREGROUND, value);
       break;
 
     case 38: // Foreground colour alternative palette
-      state->fg_index = -1;
       if(argcount - argi < 1)
         return;
-      argi += 1 + lookup_colour(state, CSI_ARG(args[argi+1]), args+argi+2, argcount-argi-2, &state->pen.fg, &state->fg_index);
+      argi += 1 + lookup_colour(state, CSI_ARG(args[argi+1]), args+argi+2, argcount-argi-2, &state->pen.fg);
       setpenattr_col(state, VTERM_ATTR_FOREGROUND, state->pen.fg);
       break;
 
     case 39: // Foreground colour default
-      state->fg_index = -1;
       state->pen.fg = state->default_fg;
       setpenattr_col(state, VTERM_ATTR_FOREGROUND, state->pen.fg);
       break;
@@ -377,20 +433,17 @@
     case 40: case 41: case 42: case 43:
     case 44: case 45: case 46: case 47: // Background colour palette
       value = CSI_ARG(args[argi]) - 40;
-      state->bg_index = value;
       set_pen_col_ansi(state, VTERM_ATTR_BACKGROUND, value);
       break;
 
     case 48: // Background colour alternative palette
-      state->bg_index = -1;
       if(argcount - argi < 1)
         return;
-      argi += 1 + lookup_colour(state, CSI_ARG(args[argi+1]), args+argi+2, argcount-argi-2, &state->pen.bg, &state->bg_index);
+      argi += 1 + lookup_colour(state, CSI_ARG(args[argi+1]), args+argi+2, argcount-argi-2, &state->pen.bg);
       setpenattr_col(state, VTERM_ATTR_BACKGROUND, state->pen.bg);
       break;
 
     case 49: // Default background
-      state->bg_index = -1;
       state->pen.bg = state->default_bg;
       setpenattr_col(state, VTERM_ATTR_BACKGROUND, state->pen.bg);
       break;
@@ -398,14 +451,12 @@
     case 90: case 91: case 92: case 93:
     case 94: case 95: case 96: case 97: // Foreground colour high-intensity palette
       value = CSI_ARG(args[argi]) - 90 + 8;
-      state->fg_index = value;
       set_pen_col_ansi(state, VTERM_ATTR_FOREGROUND, value);
       break;
 
     case 100: case 101: case 102: case 103:
     case 104: case 105: case 106: case 107: // Background colour high-intensity palette
       value = CSI_ARG(args[argi]) - 100 + 8;
-      state->bg_index = value;
       set_pen_col_ansi(state, VTERM_ATTR_BACKGROUND, value);
       break;
 
@@ -424,6 +475,39 @@
   }
 }
 
+static int vterm_state_getpen_color(const VTermColor *col, int argi, long args[], int fg)
+{
+    /* Do nothing if the given color is the default color */
+    if (( fg && VTERM_COLOR_IS_DEFAULT_FG(col)) ||
+        (!fg && VTERM_COLOR_IS_DEFAULT_BG(col))) {
+        return argi;
+    }
+
+    /* Decide whether to send an indexed color or an RGB color */
+    if (VTERM_COLOR_IS_INDEXED(col)) {
+        const uint8_t idx = col->index;
+        if (idx < 8) {
+            args[argi++] = (idx + (fg ? 30 : 40));
+        }
+        else if (idx < 16) {
+            args[argi++] = (idx - 8 + (fg ? 90 : 100));
+        }
+        else {
+            args[argi++] = CSI_ARG_FLAG_MORE | (fg ? 38 : 48);
+            args[argi++] = CSI_ARG_FLAG_MORE | 5;
+            args[argi++] = idx;
+        }
+    }
+    else if (VTERM_COLOR_IS_RGB(col)) {
+        args[argi++] = CSI_ARG_FLAG_MORE | (fg ? 38 : 48);
+        args[argi++] = CSI_ARG_FLAG_MORE | 2;
+        args[argi++] = CSI_ARG_FLAG_MORE | col->red;
+        args[argi++] = CSI_ARG_FLAG_MORE | col->green;
+        args[argi++] = col->blue;
+    }
+    return argi;
+}
+
 INTERNAL int vterm_state_getpen(VTermState *state, long args[], int argcount UNUSED)
 {
   int argi = 0;
@@ -457,49 +541,9 @@
   if(state->pen.underline == VTERM_UNDERLINE_DOUBLE)
     args[argi++] = 21;
 
-  if(state->fg_index >= 0 && state->fg_index < 8)
-    args[argi++] = 30 + state->fg_index;
-  else if(state->fg_index >= 8 && state->fg_index < 16)
-    args[argi++] = 90 + state->fg_index - 8;
-  else if(state->fg_index >= 16 && state->fg_index < 256) {
-    args[argi++] = CSI_ARG_FLAG_MORE|38;
-    args[argi++] = CSI_ARG_FLAG_MORE|5;
-    args[argi++] = state->fg_index;
-  }
-  else if(state->fg_index == -1) {
-    // Send palette 2 if the actual FG colour is not default
-    if(state->pen.fg.red   != state->default_fg.red   ||
-       state->pen.fg.green != state->default_fg.green ||
-       state->pen.fg.blue  != state->default_fg.blue  ) {
-      args[argi++] = CSI_ARG_FLAG_MORE|38;
-      args[argi++] = CSI_ARG_FLAG_MORE|2;
-      args[argi++] = CSI_ARG_FLAG_MORE | state->pen.fg.red;
-      args[argi++] = CSI_ARG_FLAG_MORE | state->pen.fg.green;
-      args[argi++] = state->pen.fg.blue;
-    }
-  }
+  argi = vterm_state_getpen_color(&state->pen.fg, argi, args, TRUE);
 
-  if(state->bg_index >= 0 && state->bg_index < 8)
-    args[argi++] = 40 + state->bg_index;
-  else if(state->bg_index >= 8 && state->bg_index < 16)
-    args[argi++] = 100 + state->bg_index - 8;
-  else if(state->bg_index >= 16 && state->bg_index < 256) {
-    args[argi++] = CSI_ARG_FLAG_MORE|48;
-    args[argi++] = CSI_ARG_FLAG_MORE|5;
-    args[argi++] = state->bg_index;
-  }
-  else if(state->bg_index == -1) {
-    // Send palette 2 if the actual BG colour is not default
-    if(state->pen.bg.red   != state->default_bg.red   ||
-       state->pen.bg.green != state->default_bg.green ||
-       state->pen.bg.blue  != state->default_bg.blue  ) {
-      args[argi++] = CSI_ARG_FLAG_MORE|48;
-      args[argi++] = CSI_ARG_FLAG_MORE|2;
-      args[argi++] = CSI_ARG_FLAG_MORE | state->pen.bg.red;
-      args[argi++] = CSI_ARG_FLAG_MORE | state->pen.bg.green;
-      args[argi++] = state->pen.bg.blue;
-    }
-  }
+  argi = vterm_state_getpen_color(&state->pen.bg, argi, args, FALSE);
 
   return argi;
 }
diff --git a/src/libvterm/src/screen.c b/src/libvterm/src/screen.c
index 1e2439c..ea551f9 100644
--- a/src/libvterm/src/screen.c
+++ b/src/libvterm/src/screen.c
@@ -949,9 +949,9 @@
     return 1;
   if((attrs & VTERM_ATTR_FONT_MASK)       && (a->pen.font != b->pen.font))
     return 1;
-  if((attrs & VTERM_ATTR_FOREGROUND_MASK) && !vterm_color_equal(a->pen.fg, b->pen.fg))
+  if((attrs & VTERM_ATTR_FOREGROUND_MASK) && !vterm_color_is_equal(&a->pen.fg, &b->pen.fg))
     return 1;
-  if((attrs & VTERM_ATTR_BACKGROUND_MASK) && !vterm_color_equal(a->pen.bg, b->pen.bg))
+  if((attrs & VTERM_ATTR_BACKGROUND_MASK) && !vterm_color_is_equal(&a->pen.bg, &b->pen.bg))
     return 1;
 
   return 0;
@@ -984,3 +984,8 @@
 
   return 1;
 }
+
+void vterm_screen_convert_color_to_rgb(const VTermScreen *screen, VTermColor *col)
+{
+  vterm_state_convert_color_to_rgb(screen->state, col);
+}
diff --git a/src/libvterm/src/state.c b/src/libvterm/src/state.c
index 15579d2..1fff43c 100644
--- a/src/libvterm/src/state.c
+++ b/src/libvterm/src/state.c
@@ -1680,8 +1680,6 @@
   if(!frag.final)
     return;
 
-  fprintf(stderr, "DECRQSS on <%s>\n", tmp);
-
   switch(tmp[0] | tmp[1]<<8 | tmp[2]<<16) {
     case 'm': {
       // Query SGR
diff --git a/src/libvterm/src/vterm_internal.h b/src/libvterm/src/vterm_internal.h
index a65e576..942315d 100644
--- a/src/libvterm/src/vterm_internal.h
+++ b/src/libvterm/src/vterm_internal.h
@@ -58,15 +58,6 @@
   unsigned int font:4; // To store 0-9
 };
 
-int vterm_color_equal(VTermColor a, VTermColor b);
-
-#if defined(DEFINE_INLINES) || USE_INLINE
-INLINE int vterm_color_equal(VTermColor a, VTermColor b)
-{
-  return a.red == b.red && a.green == b.green && a.blue == b.blue;
-}
-#endif
-
 struct VTermState
 {
   VTerm *vt;
@@ -144,8 +135,6 @@
   VTermColor default_bg;
   VTermColor colors[16]; // Store the 8 ANSI and the 8 ANSI high-brights only
 
-  int fg_index;
-  int bg_index;
   int bold_is_highbright;
 
   unsigned int protected_cell : 1;
diff --git a/src/libvterm/t/26state_query.test b/src/libvterm/t/26state_query.test
index c6be90b..c543b92 100644
--- a/src/libvterm/t/26state_query.test
+++ b/src/libvterm/t/26state_query.test
@@ -41,10 +41,10 @@
 PUSH "\eP\$qm\e\\"
   output "\eP1\$r93;104m\e\\"
 
-!DECRQSS on SGR 256-palette colours
-PUSH "\e[0;38:5:56;48:5:78m"
-PUSH "\eP\$qm\e\\"
-  output "\eP1\$r38:5:56;48:5:78m\e\\"
+##!DECRQSS on SGR 256-palette colours
+#PUSH "\e[0;38:5:56;48:5:78m"
+#PUSH "\eP\$qm\e\\"
+#  output "\eP1\$r38:5:56;48:5:78m\e\\"
 
 !DECRQSS on SGR RGB8 colours
 PUSH "\e[0;38:2:24:68:112;48:2:13:57:101m"
diff --git a/src/libvterm/t/30state_pen.test b/src/libvterm/t/30state_pen.test
index d1e0d14..915baec 100644
--- a/src/libvterm/t/30state_pen.test
+++ b/src/libvterm/t/30state_pen.test
@@ -10,8 +10,8 @@
   ?pen blink = off
   ?pen reverse = off
   ?pen font = 0
-  ?pen foreground = rgb(240,240,240)
-  ?pen background = rgb(0,0,0)
+  ?pen foreground = rgb(240,240,240,is_default_fg)
+  ?pen background = rgb(0,0,0,is_default_bg)
 
 !Bold
 PUSH "\e[1m"
@@ -75,40 +75,40 @@
 
 !Foreground
 PUSH "\e[31m"
-  ?pen foreground = rgb(224,0,0)
+  ?pen foreground = idx(1)
 PUSH "\e[32m"
-  ?pen foreground = rgb(0,224,0)
+  ?pen foreground = idx(2)
 PUSH "\e[34m"
-  ?pen foreground = rgb(0,0,224)
+  ?pen foreground = idx(4)
 PUSH "\e[91m"
-  ?pen foreground = rgb(255,64,64)
+  ?pen foreground = idx(9)
 PUSH "\e[38:2:10:20:30m"
   ?pen foreground = rgb(10,20,30)
 PUSH "\e[38:5:1m"
-  ?pen foreground = rgb(224,0,0)
+  ?pen foreground = idx(1)
 PUSH "\e[39m"
-  ?pen foreground = rgb(240,240,240)
+  ?pen foreground = rgb(240,240,240,is_default_fg)
 
 !Background
 PUSH "\e[41m"
-  ?pen background = rgb(224,0,0)
+  ?pen background = idx(1)
 PUSH "\e[42m"
-  ?pen background = rgb(0,224,0)
+  ?pen background = idx(2)
 PUSH "\e[44m"
-  ?pen background = rgb(0,0,224)
+  ?pen background = idx(4)
 PUSH "\e[101m"
-  ?pen background = rgb(255,64,64)
+  ?pen background = idx(9)
 PUSH "\e[48:2:10:20:30m"
   ?pen background = rgb(10,20,30)
 PUSH "\e[48:5:1m"
-  ?pen background = rgb(224,0,0)
+  ?pen background = idx(1)
 PUSH "\e[49m"
-  ?pen background = rgb(0,0,0)
+  ?pen background = rgb(0,0,0,is_default_bg)
 
 !Bold+ANSI colour == highbright
 PUSH "\e[m\e[1;37m"
   ?pen bold = on
-  ?pen foreground = rgb(255,255,255)
+  ?pen foreground = idx(15)
 PUSH "\e[m\e[37;1m"
   ?pen bold = on
-  ?pen foreground = rgb(255,255,255)
+  ?pen foreground = idx(15)
diff --git a/src/libvterm/t/64screen_pen.test b/src/libvterm/t/64screen_pen.test
index f1ee639..3a78140 100644
--- a/src/libvterm/t/64screen_pen.test
+++ b/src/libvterm/t/64screen_pen.test
@@ -29,27 +29,27 @@
 
 !Foreground
 PUSH "\e[31mG\e[m"
-  ?screen_cell 0,6 = {0x47} width=1 attrs={} fg=rgb(224,0,0) bg=rgb(0,0,0)
+  ?screen_cell 0,6 = {0x47} width=1 attrs={} fg=idx(1) bg=rgb(0,0,0)
 
 !Background
 PUSH "\e[42mH\e[m"
-  ?screen_cell 0,7 = {0x48} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,224,0)
+  ?screen_cell 0,7 = {0x48} width=1 attrs={} fg=rgb(240,240,240) bg=idx(2)
 
 !EL sets reverse and colours to end of line
 PUSH "\e[H\e[7;33;44m\e[K"
-  ?screen_cell 0,0  = {} width=1 attrs={R} fg=rgb(224,224,0) bg=rgb(0,0,224)
-  ?screen_cell 0,79 = {} width=1 attrs={R} fg=rgb(224,224,0) bg=rgb(0,0,224)
+  ?screen_cell 0,0  = {} width=1 attrs={R} fg=idx(3) bg=idx(4)
+  ?screen_cell 0,79 = {} width=1 attrs={R} fg=idx(3) bg=idx(4)
 
 !DECSCNM xors reverse for entire screen
 PUSH "\e[?5h"
-  ?screen_cell 0,0  = {} width=1 attrs={} fg=rgb(224,224,0) bg=rgb(0,0,224)
-  ?screen_cell 0,79 = {} width=1 attrs={} fg=rgb(224,224,0) bg=rgb(0,0,224)
+  ?screen_cell 0,0  = {} width=1 attrs={} fg=idx(3) bg=idx(4)
+  ?screen_cell 0,79 = {} width=1 attrs={} fg=idx(3) bg=idx(4)
   ?screen_cell 1,0  = {} width=1 attrs={R} fg=rgb(240,240,240) bg=rgb(0,0,0)
 PUSH "\e[?5\$p"
   output "\e[?5;1\$y"
 PUSH "\e[?5l"
-  ?screen_cell 0,0  = {} width=1 attrs={R} fg=rgb(224,224,0) bg=rgb(0,0,224)
-  ?screen_cell 0,79 = {} width=1 attrs={R} fg=rgb(224,224,0) bg=rgb(0,0,224)
+  ?screen_cell 0,0  = {} width=1 attrs={R} fg=idx(3) bg=idx(4)
+  ?screen_cell 0,79 = {} width=1 attrs={R} fg=idx(3) bg=idx(4)
   ?screen_cell 1,0  = {} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)
 PUSH "\e[?5\$p"
   output "\e[?5;2\$y"
diff --git a/src/libvterm/t/harness.c b/src/libvterm/t/harness.c
index fd129b7..10afbfd 100644
--- a/src/libvterm/t/harness.c
+++ b/src/libvterm/t/harness.c
@@ -60,6 +60,26 @@
   return VTERM_KEY_NONE;
 }
 
+static void print_color(const VTermColor *col)
+{
+  if (VTERM_COLOR_IS_RGB(col)) {
+    printf("rgb(%d,%d,%d", col->red, col->green, col->blue);
+  }
+  else if (VTERM_COLOR_IS_INDEXED(col)) {
+    printf("idx(%d", col->index);
+  }
+  else {
+    printf("invalid(%d", col->type);
+  }
+  if (VTERM_COLOR_IS_DEFAULT_FG(col)) {
+    printf(",is_default_fg");
+  }
+  if (VTERM_COLOR_IS_DEFAULT_BG(col)) {
+    printf(",is_default_bg");
+  }
+  printf(")");
+}
+
 static VTerm *vt;
 static VTermState *state;
 static VTermScreen *screen;
@@ -273,7 +293,9 @@
         val->string.initial ? "[" : "", val->string.len, val->string.str, val->string.final ? "]" : "");
     return 1;
   case VTERM_VALUETYPE_COLOR:
-    printf("settermprop %d rgb(%d,%d,%d)\n", prop, val->color.red, val->color.green, val->color.blue);
+    printf("settermprop %d ", prop);
+    print_color(&val->color);
+    printf("\n");
     return 1;
 
   case VTERM_N_VALUETYPES:
@@ -834,10 +856,12 @@
             printf("%d\n", state_pen.font);
         }
         else if(streq(linep, "foreground")) {
-          printf("rgb(%d,%d,%d)\n", state_pen.foreground.red, state_pen.foreground.green, state_pen.foreground.blue);
+          print_color(&state_pen.foreground);
+          printf("\n");
         }
         else if(streq(linep, "background")) {
-          printf("rgb(%d,%d,%d)\n", state_pen.background.red, state_pen.background.green, state_pen.background.blue);
+          print_color(&state_pen.background);
+          printf("\n");
         }
         else
           printf("?\n");
@@ -952,8 +976,13 @@
         printf("} ");
         if(cell.attrs.dwl)       printf("dwl ");
         if(cell.attrs.dhl)       printf("dhl-%s ", cell.attrs.dhl == 2 ? "bottom" : "top");
-        printf("fg=rgb(%d,%d,%d) ",  cell.fg.red, cell.fg.green, cell.fg.blue);
-        printf("bg=rgb(%d,%d,%d)\n", cell.bg.red, cell.bg.green, cell.bg.blue);
+        printf("fg=");
+        vterm_screen_convert_color_to_rgb(screen, &cell.fg);
+        print_color(&cell.fg);
+        printf(" bg=");
+        vterm_screen_convert_color_to_rgb(screen, &cell.bg);
+        print_color(&cell.bg);
+        printf("\n");
       }
       else if(strstartswith(line, "?screen_eol ")) {
         VTermPos pos;
diff --git a/src/term.c b/src/term.c
index 11972a1..e79fa5e 100644
--- a/src/term.c
+++ b/src/term.c
@@ -6357,12 +6357,6 @@
     0x80, 0x8A, 0x94, 0x9E, 0xA8, 0xB2, 0xBC, 0xC6, 0xD0, 0xDA, 0xE4, 0xEE
 };
 
-# ifdef FEAT_TERMINAL
-#  include "libvterm/include/vterm.h"  // for VTERM_ANSI_INDEX_NONE
-# else
-#  define VTERM_ANSI_INDEX_NONE 0
-# endif
-
 static char_u ansi_table[16][4] = {
 //   R    G    B   idx
   {  0,   0,   0,  1}, // black
@@ -6384,6 +6378,8 @@
   {255, 255, 255, 16}, // white
 };
 
+#define ANSI_INDEX_NONE 0
+
     void
 cterm_color2rgb(int nr, char_u *r, char_u *g, char_u *b, char_u *ansi_idx)
 {
@@ -6403,7 +6399,7 @@
 	*r = cube_value[idx / 36 % 6];
 	*g = cube_value[idx / 6  % 6];
 	*b = cube_value[idx      % 6];
-	*ansi_idx = VTERM_ANSI_INDEX_NONE;
+	*ansi_idx = ANSI_INDEX_NONE;
     }
     else if (nr < 256)
     {
@@ -6412,14 +6408,14 @@
 	*r = grey_ramp[idx];
 	*g = grey_ramp[idx];
 	*b = grey_ramp[idx];
-	*ansi_idx = VTERM_ANSI_INDEX_NONE;
+	*ansi_idx = ANSI_INDEX_NONE;
     }
     else
     {
 	*r = 0;
 	*g = 0;
 	*b = 0;
-	*ansi_idx = 0;
+	*ansi_idx = ANSI_INDEX_NONE;
     }
 }
 #endif
diff --git a/src/terminal.c b/src/terminal.c
index c41b2f1..3fcd659 100644
--- a/src/terminal.c
+++ b/src/terminal.c
@@ -1626,7 +1626,8 @@
     static int
 equal_celattr(cellattr_T *a, cellattr_T *b)
 {
-    // Comparing the colors should be sufficient.
+    // We only compare the RGB colors, ignoring the ANSI index and type.
+    // Thus black set explicitly is equal the background black.
     return a->fg.red == b->fg.red
 	&& a->fg.green == b->fg.green
 	&& a->fg.blue == b->fg.blue
@@ -2692,10 +2693,13 @@
     int blue = color->blue;
     int green = color->green;
 
-    if (color->ansi_index != VTERM_ANSI_INDEX_NONE)
+    if (VTERM_COLOR_IS_DEFAULT_FG(color)
+	    || VTERM_COLOR_IS_DEFAULT_BG(color))
+	return 0;
+    if (VTERM_COLOR_IS_INDEXED(color))
     {
 	// The first 16 colors and default: use the ANSI index.
-	switch (color->ansi_index)
+	switch (color->index + 1)
 	{
 	    case  0: return 0;
 	    case  1: return lookup_color( 0, fg, boldp) + 1; // black
@@ -3832,7 +3836,14 @@
     static void
 cterm_color2vterm(int nr, VTermColor *rgb)
 {
-    cterm_color2rgb(nr, &rgb->red, &rgb->green, &rgb->blue, &rgb->ansi_index);
+    cterm_color2rgb(nr, &rgb->red, &rgb->green, &rgb->blue, &rgb->index);
+    if (rgb->index == 0)
+	rgb->type = VTERM_COLOR_RGB;
+    else
+    {
+	rgb->type = VTERM_COLOR_INDEXED;
+	--rgb->index;
+    }
 }
 
 /*
@@ -3864,7 +3875,8 @@
     }
     fg->red = fg->green = fg->blue = fgval;
     bg->red = bg->green = bg->blue = bgval;
-    fg->ansi_index = bg->ansi_index = VTERM_ANSI_INDEX_DEFAULT;
+    fg->type = VTERM_COLOR_RGB | VTERM_COLOR_DEFAULT_FG;
+    bg->type = VTERM_COLOR_RGB | VTERM_COLOR_DEFAULT_BG;
 
     // The 'wincolor' or the highlight group overrules the defaults.
     if (wp != NULL && *wp->w_p_wcr != NUL)
@@ -4509,21 +4521,30 @@
     return buf;
 }
 
-    static int
-same_color(VTermColor *a, VTermColor *b)
+    static void
+clear_cell(VTermScreenCell *cell)
 {
-    return a->red == b->red
-	&& a->green == b->green
-	&& a->blue == b->blue
-	&& a->ansi_index == b->ansi_index;
+    CLEAR_FIELD(*cell);
+    cell->fg.type = VTERM_COLOR_DEFAULT_FG;
+    cell->bg.type = VTERM_COLOR_DEFAULT_BG;
 }
 
     static void
 dump_term_color(FILE *fd, VTermColor *color)
 {
+    int index;
+
+    if (VTERM_COLOR_IS_INDEXED(color))
+	index = color->index + 1;
+    else if (color->type == 0)
+	// use RGB values
+	index = 255;
+    else
+	// default color
+	index = 0;
     fprintf(fd, "%02x%02x%02x%d",
 	    (int)color->red, (int)color->green, (int)color->blue,
-	    (int)color->ansi_index);
+	    index);
 }
 
 /*
@@ -4607,7 +4628,7 @@
 	return;
     }
 
-    CLEAR_FIELD(prev_cell);
+    clear_cell(&prev_cell);
 
     screen = vterm_obtain_screen(term->tl_vterm);
     state = vterm_obtain_state(term->tl_vterm);
@@ -4629,7 +4650,7 @@
 						 && pos.row == cursor_pos.row);
 
 	    if (vterm_screen_get_cell(screen, pos, &cell) == 0)
-		CLEAR_FIELD(cell);
+		clear_cell(&cell);
 
 	    for (i = 0; i < VTERM_MAX_CHARS_PER_CELL; ++i)
 	    {
@@ -4649,8 +4670,8 @@
 	    }
 	    same_attr = vtermAttr2hl(cell.attrs)
 					       == vtermAttr2hl(prev_cell.attrs)
-			&& same_color(&cell.fg, &prev_cell.fg)
-			&& same_color(&cell.bg, &prev_cell.bg);
+			&& vterm_color_is_equal(&cell.fg, &prev_cell.fg)
+			&& vterm_color_is_equal(&cell.bg, &prev_cell.bg);
 	    if (same_chars && cell.width == prev_cell.width && same_attr
 							     && !is_cursor_pos)
 	    {
@@ -4697,14 +4718,14 @@
 		    else
 		    {
 			fprintf(fd, "%d", vtermAttr2hl(cell.attrs));
-			if (same_color(&cell.fg, &prev_cell.fg))
+			if (vterm_color_is_equal(&cell.fg, &prev_cell.fg))
 			    fputs("&", fd);
 			else
 			{
 			    fputs("#", fd);
 			    dump_term_color(fd, &cell.fg);
 			}
-			if (same_color(&cell.bg, &prev_cell.bg))
+			if (vterm_color_is_equal(&cell.bg, &prev_cell.bg))
 			    fputs("&", fd);
 			else
 			{
@@ -4747,6 +4768,14 @@
     }
 }
 
+    static void
+clear_cellattr(cellattr_T *cell)
+{
+    CLEAR_FIELD(*cell);
+    cell->fg.type = VTERM_COLOR_DEFAULT_FG;
+    cell->bg.type = VTERM_COLOR_DEFAULT_BG;
+}
+
 /*
  * Read the dump file from "fd" and append lines to the current buffer.
  * Return the cell width of the longest line.
@@ -4767,8 +4796,8 @@
 
     ga_init2(&ga_text, 1, 90);
     ga_init2(&ga_cell, sizeof(cellattr_T), 90);
-    CLEAR_FIELD(cell);
-    CLEAR_FIELD(empty_cell);
+    clear_cellattr(&cell);
+    clear_cellattr(&empty_cell);
     cursor_pos->row = -1;
     cursor_pos->col = -1;
 
@@ -4878,7 +4907,7 @@
 			}
 			else if (c == '#')
 			{
-			    int red, green, blue, index = 0;
+			    int red, green, blue, index = 0, type;
 
 			    c = fgetc(fd);
 			    red = hex2nr(c);
@@ -4900,20 +4929,37 @@
 				index = index * 10 + (c - '0');
 				c = fgetc(fd);
 			    }
-
-			    if (is_bg)
+			    if (index == 0 || index == 255)
 			    {
-				cell.bg.red = red;
-				cell.bg.green = green;
-				cell.bg.blue = blue;
-				cell.bg.ansi_index = index;
+				type = VTERM_COLOR_RGB;
+				if (index == 0)
+				{
+				    if (is_bg)
+					type |= VTERM_COLOR_DEFAULT_BG;
+				    else
+					type |= VTERM_COLOR_DEFAULT_FG;
+				}
 			    }
 			    else
 			    {
+				type = VTERM_COLOR_INDEXED;
+				index -= 1;
+			    }
+			    if (is_bg)
+			    {
+				cell.bg.type = type;
+				cell.bg.red = red;
+				cell.bg.green = green;
+				cell.bg.blue = blue;
+				cell.bg.index = index;
+			    }
+			    else
+			    {
+				cell.fg.type = type;
 				cell.fg.red = red;
 				cell.fg.green = green;
 				cell.fg.blue = blue;
-				cell.fg.ansi_index = index;
+				cell.fg.index = index;
 			    }
 			}
 			else
@@ -5226,10 +5272,10 @@
 			if ((cellattr1 + col)->width
 						   != (cellattr2 + col)->width)
 			    textline[col] = 'w';
-			else if (!same_color(&(cellattr1 + col)->fg,
+			else if (!vterm_color_is_equal(&(cellattr1 + col)->fg,
 						   &(cellattr2 + col)->fg))
 			    textline[col] = 'f';
-			else if (!same_color(&(cellattr1 + col)->bg,
+			else if (!vterm_color_is_equal(&(cellattr1 + col)->bg,
 						   &(cellattr2 + col)->bg))
 			    textline[col] = 'b';
 			else if (vtermAttr2hl((cellattr1 + col)->attrs)
@@ -5808,6 +5854,7 @@
 	else
 	{
 	    VTermScreenCell cell;
+
 	    if (vterm_screen_get_cell(screen, pos, &cell) == 0)
 		break;
 	    for (i = 0; i < VTERM_MAX_CHARS_PER_CELL; ++i)
diff --git a/src/version.c b/src/version.c
index 31aa395..a710bc5 100644
--- a/src/version.c
+++ b/src/version.c
@@ -747,6 +747,8 @@
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    804,
+/**/
     803,
 /**/
     802,