Fix some offenders that poke around in the PixelFormat internals
diff --git a/common/rfb/JpegCompressor.cxx b/common/rfb/JpegCompressor.cxx
index c1ef3c7..960bd37 100644
--- a/common/rfb/JpegCompressor.cxx
+++ b/common/rfb/JpegCompressor.cxx
@@ -1,5 +1,6 @@
/* Copyright (C) 2002-2005 RealVNC Ltd. All Rights Reserved.
* Copyright (C) 2011 D. R. Commander. All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -33,6 +34,15 @@
using namespace rfb;
//
+// Special formats that libjpeg can have optimised code paths for
+//
+
+static const PixelFormat pfRGBX(32, 24, false, true, 255, 255, 255, 0, 8, 16);
+static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0);
+static const PixelFormat pfXRGB(32, 24, false, true, 255, 255, 255, 8, 16, 24);
+static const PixelFormat pfXBGR(32, 24, false, true, 255, 255, 255, 24, 16, 8);
+
+//
// Error manager implmentation for the JPEG library
//
@@ -166,33 +176,20 @@
pixelsize = 3;
#ifdef JCS_EXTENSIONS
- // Try to have libjpeg read directly from our native format
- if(pf.is888()) {
- int redShift, greenShift, blueShift;
+ // Try to have libjpeg output directly to our native format
+ // libjpeg can only handle some "standard" formats
+ if (pfRGBX.equal(pf))
+ cinfo->in_color_space = JCS_EXT_RGBX;
+ else if (pfBGRX.equal(pf))
+ cinfo->in_color_space = JCS_EXT_BGRX;
+ else if (pfXRGB.equal(pf))
+ cinfo->in_color_space = JCS_EXT_XRGB;
+ else if (pfXBGR.equal(pf))
+ cinfo->in_color_space = JCS_EXT_XBGR;
- if(pf.bigEndian) {
- redShift = 24 - pf.redShift;
- greenShift = 24 - pf.greenShift;
- blueShift = 24 - pf.blueShift;
- } else {
- redShift = pf.redShift;
- greenShift = pf.greenShift;
- blueShift = pf.blueShift;
- }
-
- if(redShift == 0 && greenShift == 8 && blueShift == 16)
- cinfo->in_color_space = JCS_EXT_RGBX;
- if(redShift == 16 && greenShift == 8 && blueShift == 0)
- cinfo->in_color_space = JCS_EXT_BGRX;
- if(redShift == 24 && greenShift == 16 && blueShift == 8)
- cinfo->in_color_space = JCS_EXT_XBGR;
- if(redShift == 8 && greenShift == 16 && blueShift == 24)
- cinfo->in_color_space = JCS_EXT_XRGB;
-
- if (cinfo->in_color_space != JCS_RGB) {
- srcBuf = (rdr::U8 *)buf;
- pixelsize = 4;
- }
+ if (cinfo->in_color_space != JCS_RGB) {
+ srcBuf = (rdr::U8 *)buf;
+ pixelsize = 4;
}
#endif
diff --git a/common/rfb/JpegDecompressor.cxx b/common/rfb/JpegDecompressor.cxx
index 4d230eb..3f4d2d0 100644
--- a/common/rfb/JpegDecompressor.cxx
+++ b/common/rfb/JpegDecompressor.cxx
@@ -1,6 +1,7 @@
/* Copyright (C) 2000-2003 Constantin Kaplinsky. All Rights Reserved.
* Copyright (C) 2004-2005 Cendio AB. All rights reserved.
* Copyright (C) 2011 D. R. Commander. All Rights Reserved.
+ * Copyright 2014 Pierre Ossman for Cendio AB
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -33,6 +34,14 @@
using namespace rfb;
+//
+// Special formats that libjpeg can have optimised code paths for
+//
+
+static const PixelFormat pfRGBX(32, 24, false, true, 255, 255, 255, 0, 8, 16);
+static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0);
+static const PixelFormat pfXRGB(32, 24, false, true, 255, 255, 255, 8, 16, 24);
+static const PixelFormat pfXBGR(32, 24, false, true, 255, 255, 255, 24, 16, 8);
//
// Error manager implmentation for the JPEG library
@@ -169,33 +178,19 @@
#ifdef JCS_EXTENSIONS
// Try to have libjpeg output directly to our native format
- if (pf.is888()) {
- int redShift, greenShift, blueShift;
+ // libjpeg can only handle some "standard" formats
+ if (pfRGBX.equal(pf))
+ dinfo->out_color_space = JCS_EXT_RGBX;
+ else if (pfBGRX.equal(pf))
+ dinfo->out_color_space = JCS_EXT_BGRX;
+ else if (pfXRGB.equal(pf))
+ dinfo->out_color_space = JCS_EXT_XRGB;
+ else if (pfXBGR.equal(pf))
+ dinfo->out_color_space = JCS_EXT_XBGR;
- if(pf.bigEndian) {
- redShift = 24 - pf.redShift;
- greenShift = 24 - pf.greenShift;
- blueShift = 24 - pf.blueShift;
- } else {
- redShift = pf.redShift;
- greenShift = pf.greenShift;
- blueShift = pf.blueShift;
- }
-
- // libjpeg can only handle some "standard" formats
- if(redShift == 0 && greenShift == 8 && blueShift == 16)
- dinfo->out_color_space = JCS_EXT_RGBX;
- if(redShift == 16 && greenShift == 8 && blueShift == 0)
- dinfo->out_color_space = JCS_EXT_BGRX;
- if(redShift == 24 && greenShift == 16 && blueShift == 8)
- dinfo->out_color_space = JCS_EXT_XBGR;
- if(redShift == 8 && greenShift == 16 && blueShift == 24)
- dinfo->out_color_space = JCS_EXT_XRGB;
-
- if (dinfo->out_color_space != JCS_RGB) {
- dstBuf = (rdr::U8 *)buf;
- pixelsize = 4;
- }
+ if (dinfo->out_color_space != JCS_RGB) {
+ dstBuf = (rdr::U8 *)buf;
+ pixelsize = 4;
}
#endif
diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h
index d5b2c66..c121b7a 100644
--- a/common/rfb/tightEncode.h
+++ b/common/rfb/tightEncode.h
@@ -540,12 +540,7 @@
*dataend = &data[stride * h];
bool willTransform = ig->willTransform();
- if (willTransform) {
- mask = serverpf.redMax << serverpf.redShift;
- mask |= serverpf.greenMax << serverpf.greenShift;
- mask |= serverpf.blueMax << serverpf.blueShift;
- }
- else mask = ~0;
+ serverpf.bufferFromPixel((rdr::U8*)&mask, ~0);
c0 = data[0] & mask;
n0 = 0;