Fix rounding error in pixel down conversion
Simple shifting can give noticable rounding errors if there is a large
difference in the number of bits between the formats. Do the proper
thing via a lookup table, the same way things are done for up conversion.
diff --git a/common/rfb/PixelFormat.inl b/common/rfb/PixelFormat.inl
index f9fb125..5a40379 100644
--- a/common/rfb/PixelFormat.inl
+++ b/common/rfb/PixelFormat.inl
@@ -79,10 +79,9 @@
{
Pixel p;
- /* We don't need to mask since we shift out unwanted bits */
- p = ((Pixel)red >> (16 - redBits)) << redShift;
- p |= ((Pixel)green >> (16 - greenBits)) << greenShift;
- p |= ((Pixel)blue >> (16 - blueBits)) << blueShift;
+ p = (Pixel)downconvTable[(redBits-1)*256 + (red >> 8)] << redShift;
+ p |= (Pixel)downconvTable[(greenBits-1)*256 + (green >> 8)] << greenShift;
+ p |= (Pixel)downconvTable[(blueBits-1)*256 + (blue >> 8)] << blueShift;
return p;
}
@@ -92,9 +91,9 @@
{
Pixel p;
- p = ((Pixel)red >> (8 - redBits)) << redShift;
- p |= ((Pixel)green >> (8 - greenBits)) << greenShift;
- p |= ((Pixel)blue >> (8 - blueBits)) << blueShift;
+ p = (Pixel)downconvTable[(redBits-1)*256 + red] << redShift;
+ p |= (Pixel)downconvTable[(greenBits-1)*256 + green] << greenShift;
+ p |= (Pixel)downconvTable[(blueBits-1)*256 + blue] << blueShift;
return p;
}