commit 19dbca236cfc807ffd7c90322dcda158781c289b
author Pierre Ossman <ossman@cendio.se> Tue Apr 21 17:30:45 2009 +0000
committer Pierre Ossman <ossman@cendio.se> Tue Apr 21 17:30:45 2009 +0000
tree 0992de2b012a78aa446d00051e3d4b4cde8c6946
parent 07893cbb127acfdb4879fbc596aea9c89df3f753

Make sure we handle endian problems in the conversion code.


git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@3784 3789f03b-4d11-0410-bbf8-ca57d06f2519

common/rfb/PixelFormat.cxx

diff --git a/common/rfb/PixelFormat.cxx b/common/rfb/PixelFormat.cxx
index a106cf3..afcf429 100644
--- a/common/rfb/PixelFormat.cxx
+++ b/common/rfb/PixelFormat.cxx
@@ -42,7 +42,7 @@
   assert((greenMax & (greenMax + 1)) == 0);
   assert((blueMax & (blueMax + 1)) == 0);
 
-  updateShifts();
+  updateState();
 }
 
 PixelFormat::PixelFormat()
@@ -50,7 +50,7 @@
     redMax(7), greenMax(7), blueMax(3),
     redShift(0), greenShift(3), blueShift(6)
 {
-  updateShifts();
+  updateState();
 }
 
 bool PixelFormat::equal(const PixelFormat& other) const
@@ -81,7 +81,7 @@
   blueShift = is->readU8();
   is->skip(3);
 
-  updateShifts();
+  updateState();
 }
 
 void PixelFormat::write(rdr::OutStream* os) const
@@ -387,7 +387,7 @@
     return false;
   }
 
-  updateShifts();
+  updateState();
 
   return true;
 }
@@ -419,9 +419,10 @@
   return bits;
 }
 
-void PixelFormat::updateShifts(void)
+void PixelFormat::updateState(void)
 {
   int redBits, greenBits, blueBits;
+  int endianTest = 1;
 
   redBits = bits(redMax);
   greenBits = bits(greenMax);
@@ -430,4 +431,9 @@
   redConvShift = 16 - redBits;
   greenConvShift = 16 - greenBits;
   blueConvShift = 16 - blueBits;
+
+  if (((*(char*)&endianTest) == 0) != bigEndian)
+    endianMismatch = true;
+  else
+    endianMismatch = false;
 }