commit 19dbca236cfc807ffd7c90322dcda158781c289b
author Pierre Ossman <ossman@cendio.se> Tue Apr 21 17:30:45 2009 +0000
committer Pierre Ossman <ossman@cendio.se> Tue Apr 21 17:30:45 2009 +0000
tree 0992de2b012a78aa446d00051e3d4b4cde8c6946
parent 07893cbb127acfdb4879fbc596aea9c89df3f753

Make sure we handle endian problems in the conversion code.


git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@3784 3789f03b-4d11-0410-bbf8-ca57d06f2519

common/rfb/PixelFormat.cxx

diff --git a/common/rfb/PixelFormat.cxx b/common/rfb/PixelFormat.cxx
index a106cf3..afcf429 100644
--- a/common/rfb/PixelFormat.cxx
+++ b/common/rfb/PixelFormat.cxx
@@ -42,7 +42,7 @@
   assert((greenMax & (greenMax + 1)) == 0);
   assert((blueMax & (blueMax + 1)) == 0);
 
-  updateShifts();
+  updateState();
 }
 
 PixelFormat::PixelFormat()
@@ -50,7 +50,7 @@
     redMax(7), greenMax(7), blueMax(3),
     redShift(0), greenShift(3), blueShift(6)
 {
-  updateShifts();
+  updateState();
 }
 
 bool PixelFormat::equal(const PixelFormat& other) const
@@ -81,7 +81,7 @@
   blueShift = is->readU8();
   is->skip(3);
 
-  updateShifts();
+  updateState();
 }
 
 void PixelFormat::write(rdr::OutStream* os) const
@@ -387,7 +387,7 @@
     return false;
   }
 
-  updateShifts();
+  updateState();
 
   return true;
 }
@@ -419,9 +419,10 @@
   return bits;
 }
 
-void PixelFormat::updateShifts(void)
+void PixelFormat::updateState(void)
 {
   int redBits, greenBits, blueBits;
+  int endianTest = 1;
 
   redBits = bits(redMax);
   greenBits = bits(greenMax);
@@ -430,4 +431,9 @@
   redConvShift = 16 - redBits;
   greenConvShift = 16 - greenBits;
   blueConvShift = 16 - blueBits;
+
+  if (((*(char*)&endianTest) == 0) != bigEndian)
+    endianMismatch = true;
+  else
+    endianMismatch = false;
 }

common/rfb/PixelFormat.h

diff --git a/common/rfb/PixelFormat.h b/common/rfb/PixelFormat.h
index 8adbf91..c1de09a 100644
--- a/common/rfb/PixelFormat.h
+++ b/common/rfb/PixelFormat.h
@@ -64,7 +64,7 @@
     bool parse(const char* str);
 
   protected:
-    void updateShifts(void);
+    void updateState(void);
 
   public:
     int bpp;
@@ -85,6 +85,7 @@
     int redConvShift;
     int greenConvShift;
     int blueConvShift;
+    bool endianMismatch;
   };
 }
 

common/rfb/PixelFormat.inl

diff --git a/common/rfb/PixelFormat.inl b/common/rfb/PixelFormat.inl
index 90d8e6d..ca83a30 100644
--- a/common/rfb/PixelFormat.inl
+++ b/common/rfb/PixelFormat.inl
@@ -75,9 +75,16 @@
 }
 
 
+#define _rfb_bswap_32(x) \
+        ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) | \
+         (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
+
+
 inline void PixelFormat::rgbFromPixel(Pixel p, ColourMap* cm, rdr::U16 *r, rdr::U16 *g, rdr::U16 *b) const
 {
   if (trueColour) {
+    if (endianMismatch)
+      p = _rfb_bswap_32(p);
     /* We don't need to mask since we shift out unwanted bits */
     *r = (p >> redShift) << redConvShift;
     *g = (p >> greenShift) << greenConvShift;
@@ -100,6 +107,8 @@
 inline void PixelFormat::rgbFromPixel(Pixel p, ColourMap* cm, rdr::U8 *r, rdr::U8 *g, rdr::U8 *b) const
 {
   if (trueColour) {
+    if (endianMismatch)
+      p = _rfb_bswap_32(p);
     *r = (p >> redShift) << (redConvShift - 8);
     *g = (p >> greenShift) << (greenConvShift - 8);
     *b = (p >> blueShift) << (blueConvShift - 8);