commit 6655d96dc475942f91af38a879627fd254a37c2e
author Pierre Ossman <ossman@cendio.se> Mon Jan 20 14:50:19 2014 +0100
committer Pierre Ossman <ossman@cendio.se> Thu Jul 03 16:22:33 2014 +0200
tree f32ff7ddc494f8035abc220ac62c8446b68340f6
parent 1338c0dc2d79235cd0380a1f11a7558548929a82

Make sure we check that PixelFormats we get are actually valid.

common/rfb/PixelFormat.cxx

diff --git a/common/rfb/PixelFormat.cxx b/common/rfb/PixelFormat.cxx
index a2e9496..8df5140 100644
--- a/common/rfb/PixelFormat.cxx
+++ b/common/rfb/PixelFormat.cxx
@@ -1,6 +1,6 @@
 /* Copyright (C) 2002-2005 RealVNC Ltd.  All Rights Reserved.
- * Copyright 2009 Pierre Ossman for Cendio AB
  * Copyright (C) 2011 D. R. Commander.  All Rights Reserved.
+ * Copyright 2009-2014 Pierre Ossman for Cendio AB
  * 
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
 #include <string.h>
 #include <rdr/InStream.h>
 #include <rdr/OutStream.h>
+#include <rfb/Exception.h>
 #include <rfb/PixelFormat.h>
 #include <rfb/util.h>
 
@@ -37,11 +38,7 @@
     redMax(rm), greenMax(gm), blueMax(bm),
     redShift(rs), greenShift(gs), blueShift(bs)
 {
-  assert((bpp == 8) || (bpp == 16) || (bpp == 32));
-  assert(depth <= bpp);
-  assert((redMax & (redMax + 1)) == 0);
-  assert((greenMax & (greenMax + 1)) == 0);
-  assert((blueMax & (blueMax + 1)) == 0);
+  assert(isSane());
 
   updateState();
 }
@@ -82,6 +79,9 @@
   blueShift = is->readU8();
   is->skip(3);
 
+  if (!isSane())
+    throw Exception("invalid pixel format");
+
   updateState();
 }
 
@@ -531,6 +531,8 @@
     return false;
   }
 
+  assert(isSane());
+
   updateState();
 
   return true;
@@ -581,3 +583,38 @@
   else
     endianMismatch = false;
 }
+
+bool PixelFormat::isSane(void)
+{
+  int totalBits;
+
+  if ((bpp != 8) && (bpp != 16) && (bpp != 32))
+    return false;
+  if (depth > bpp)
+    return false;
+
+  if (!trueColour && (depth != 8))
+    return false;
+
+  if (trueColour) {
+    if ((redMax & (redMax + 1)) != 0)
+      return false;
+    if ((greenMax & (greenMax + 1)) != 0)
+      return false;
+    if ((blueMax & (blueMax + 1)) != 0)
+      return false;
+
+    totalBits = bits(redMax) + bits(greenMax) + bits(blueMax);
+    if (totalBits > bpp)
+      return false;
+
+    if (((redMax << redShift) & (greenMax << greenShift)) != 0)
+      return false;
+    if (((redMax << redShift) & (blueMax << blueShift)) != 0)
+      return false;
+    if (((greenMax << greenShift) & (blueMax << blueShift)) != 0)
+      return false;
+  }
+
+  return true;
+}