Fix rendering on big endian system
Our XRender code assumes a certain pixel layout which was not
guaranteed on big endian systems. The previous workaround only worked
for some cases, so fix this properly now.
diff --git a/vncviewer/PlatformPixelBuffer.cxx b/vncviewer/PlatformPixelBuffer.cxx
index c79b5c1..e6a054a 100644
--- a/vncviewer/PlatformPixelBuffer.cxx
+++ b/vncviewer/PlatformPixelBuffer.cxx
@@ -34,13 +34,8 @@
static rfb::LogWriter vlog("PlatformPixelBuffer");
PlatformPixelBuffer::PlatformPixelBuffer(int width, int height) :
- FullFramePixelBuffer(rfb::PixelFormat(32, 24,
-#if !defined(WIN32) && !defined(__APPLE__)
- ImageByteOrder(fl_display) == MSBFirst,
-#else
- false,
-#endif
- true, 255, 255, 255, 16, 8, 0),
+ FullFramePixelBuffer(rfb::PixelFormat(32, 24, false, true,
+ 255, 255, 255, 16, 8, 0),
width, height, 0, stride),
Surface(width, height)
#if !defined(WIN32) && !defined(__APPLE__)
diff --git a/vncviewer/Surface_X11.cxx b/vncviewer/Surface_X11.cxx
index 3523da3..6562634 100644
--- a/vncviewer/Surface_X11.cxx
+++ b/vncviewer/Surface_X11.cxx
@@ -109,6 +109,7 @@
void Surface::alloc()
{
+ XRenderPictFormat templ;
XRenderPictFormat* format;
// Might not be open at this point
@@ -117,7 +118,37 @@
pixmap = XCreatePixmap(fl_display, XDefaultRootWindow(fl_display),
width(), height(), 32);
- format = XRenderFindStandardFormat(fl_display, PictStandardARGB32);
+ // Our code assumes a BGRA byte order, regardless of what the endian
+ // of the machine is or the native byte order of XImage, so make sure
+ // we find such a format
+ templ.type = PictTypeDirect;
+ templ.depth = 32;
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+ templ.direct.alpha = 0;
+ templ.direct.red = 8;
+ templ.direct.green = 16;
+ templ.direct.blue = 24;
+#else
+ templ.direct.alpha = 24;
+ templ.direct.red = 16;
+ templ.direct.green = 8;
+ templ.direct.blue = 0;
+#endif
+ templ.direct.alphaMask = 0xff;
+ templ.direct.redMask = 0xff;
+ templ.direct.greenMask = 0xff;
+ templ.direct.blueMask = 0xff;
+
+ format = XRenderFindFormat(fl_display, PictFormatType | PictFormatDepth |
+ PictFormatRed | PictFormatRedMask |
+ PictFormatGreen | PictFormatGreenMask |
+ PictFormatBlue | PictFormatBlueMask |
+ PictFormatAlpha | PictFormatAlphaMask,
+ &templ, 0);
+
+ if (!format)
+ throw rdr::Exception("XRenderFindFormat");
+
picture = XRenderCreatePicture(fl_display, pixmap, format, 0, NULL);
visFormat = XRenderFindVisualFormat(fl_display, fl_visual->visual);