Fix regression caused by r4841. That patch assumed that JPEG encoding always uses the raw buffer, which is not true. If pixel translation is necessary, then JPEG images will sometimes be encoded from the translated (intermediate) buffer instead.
git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4852 3789f03b-4d11-0410-bbf8-ca57d06f2519
diff --git a/common/rfb/TightEncoder.cxx b/common/rfb/TightEncoder.cxx
index fcd9d9b..9be4581 100644
--- a/common/rfb/TightEncoder.cxx
+++ b/common/rfb/TightEncoder.cxx
@@ -418,19 +418,3 @@
os->writeBytes(mos.data(), mos.length());
writer->endRect();
}
-
-void TightEncoder::encodeJpegRect(const Rect& r, rdr::OutStream *os)
-{
- const rdr::U8 *buf;
- int stride;
-
- buf = ig->getRawPixelsR(r, &stride);
-
- jc.clear();
- jc.compress(buf, stride * serverpf.bpp / 8, r, serverpf,
- jpegQuality, jpegSubsampling);
-
- os->writeU8(0x09 << 4);
- os->writeCompactLength(jc.length());
- os->writeBytes(jc.data(), jc.length());
-}
diff --git a/common/rfb/TightEncoder.h b/common/rfb/TightEncoder.h
index 10ca761..4fff083 100644
--- a/common/rfb/TightEncoder.h
+++ b/common/rfb/TightEncoder.h
@@ -100,9 +100,9 @@
int paletteInsert(rdr::U32 rgb, int numPixels, int bpp);
void paletteReset(void);
- void fastFillPalette8(const rdr::U8 *buffer, int stride, const Rect &r);
- void fastFillPalette16(const rdr::U8 *buffer, int stride, const Rect &r);
- void fastFillPalette32(const rdr::U8 *buffer, int stride, const Rect &r);
+ void fastFillPalette8(const rdr::U8 *data, int stride, const Rect &r);
+ void fastFillPalette16(const rdr::U16 *data, int stride, const Rect &r);
+ void fastFillPalette32(const rdr::U32 *data, int stride, const Rect &r);
void fillPalette8(rdr::U8 *data, int count);
void fillPalette16(rdr::U16 *data, int count);
@@ -135,7 +135,10 @@
void encodeIndexedRect16(rdr::U16 *buf, const Rect& r, rdr::OutStream *os);
void encodeIndexedRect32(rdr::U32 *buf, const Rect& r, rdr::OutStream *os);
- void encodeJpegRect(const Rect& r, rdr::OutStream *os);
+ void encodeJpegRect16(rdr::U16 *buf, int stride, const Rect& r,
+ rdr::OutStream *os);
+ void encodeJpegRect32(rdr::U32 *buf, int stride, const Rect& r,
+ rdr::OutStream *os);
SMsgWriter* writer;
rdr::MemOutStream mos;
diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h
index 446d45e..8f900b5 100644
--- a/common/rfb/tightEncode.h
+++ b/common/rfb/tightEncode.h
@@ -191,8 +191,9 @@
{
int stride;
rdr::U32 solidColor;
- const rdr::U8 *rawPixels;
- PIXEL_T *pixels;
+ const PIXEL_T *rawPixels = (const PIXEL_T *)ig->getRawPixelsR(r, &stride);
+ PIXEL_T *pixels = NULL;
+ bool grayScaleJPEG = (jpegSubsampling == SUBSAMP_GRAY && jpegQuality != -1);
#if (BPP == 32)
// Check if it's necessary to pack 24-bit pixels, and
@@ -200,45 +201,41 @@
pack24 = clientpf.is888();
#endif
- rawPixels = ig->getRawPixelsR(r, &stride);
- pixels = NULL;
-
if (forceSolid) {
- // Forced solid block
+ // Subrectangle has already been determined to be solid.
palNumColors = 1;
ig->translatePixels(rawPixels, &solidColor, 1);
pixels = (PIXEL_T *)&solidColor;
- } else if (jpegSubsampling == SUBSAMP_GRAY && jpegQuality != -1) {
- // Forced gray scale (JPEG)
- palNumColors = 0;
} else {
- // Normal analysis
+ // Analyze subrectangle's colors to determine best encoding method.
palMaxColors = r.area() / pconf->idxMaxColorsDivisor;
-
if (jpegQuality != -1)
palMaxColors = pconf->palMaxColorsWithJPEG;
-
if (palMaxColors < 2 && r.area() >= pconf->monoMinRectSize)
palMaxColors = 2;
if (clientpf.equal(serverpf) && clientpf.bpp >= 16) {
- // No conversion, so just analyse the raw buffer
- FAST_FILL_PALETTE(rawPixels, stride, r);
+ // Count the colors in the raw buffer, so we can avoid unnecessary pixel
+ // translation when encoding with JPEG.
+ if (grayScaleJPEG) palNumColors = 0;
+ else FAST_FILL_PALETTE(rawPixels, stride, r);
- // JPEG reads from the raw buffer and has its own output buffer,
- // but the other encodings need some help.
+ // JPEG can read from the raw buffer, but for the other methods, we need
+ // to translate the raw pixels into an intermediate buffer.
if(palNumColors != 0 || jpegQuality == -1) {
pixels = (PIXEL_T *)writer->getImageBuf(r.area());
stride = r.width();
ig->getImage(pixels, r);
}
} else {
- // Need to do the conversion first so we have something to analyse
+ // Pixel translation will be required, so create an intermediate buffer,
+ // translate the raw pixels into it, and count its colors.
pixels = (PIXEL_T *)writer->getImageBuf(r.area());
stride = r.width();
ig->getImage(pixels, r);
- FILL_PALETTE(pixels, r.area());
+ if (grayScaleJPEG) palNumColors = 0;
+ else FILL_PALETTE(pixels, r.area());
}
}
@@ -247,7 +244,10 @@
// Truecolor image
#if (BPP != 8)
if (jpegQuality != -1) {
- encodeJpegRect(r, os);
+ if (pixels)
+ ENCODE_JPEG_RECT(pixels, stride, r, os);
+ else
+ ENCODE_JPEG_RECT((PIXEL_T *)rawPixels, stride, r, os);
break;
}
#endif
@@ -404,6 +404,23 @@
#endif // #if (BPP != 8)
//
+// JPEG compression.
+//
+
+#if (BPP != 8)
+void ENCODE_JPEG_RECT (PIXEL_T *buf, int stride, const Rect& r,
+ rdr::OutStream *os)
+{
+ jc.clear();
+ jc.compress((rdr::U8 *)buf, stride * clientpf.bpp / 8, r, clientpf,
+ jpegQuality, jpegSubsampling);
+ os->writeU8(0x09 << 4);
+ os->writeCompactLength(jc.length());
+ os->writeBytes(jc.data(), jc.length());
+}
+#endif // #if (BPP != 8)
+
+//
// Determine the number of colors in the rectangle, and fill in the palette.
//
@@ -449,7 +466,7 @@
}
}
-void FAST_FILL_PALETTE (const rdr::U8 *buffer, int stride, const Rect& r)
+void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r)
{
}
@@ -514,17 +531,15 @@
paletteInsert (ci, (rdr::U32)ni, BPP);
}
-void FAST_FILL_PALETTE (const rdr::U8 *buffer, int stride, const Rect& r)
+void FAST_FILL_PALETTE (const PIXEL_T *data, int stride, const Rect& r)
{
PIXEL_T c0, c1, ci = 0, mask, c0t, c1t, cit;
int n0, n1, ni;
int w = r.width(), h = r.height();
- const PIXEL_T *data, *rowptr, *colptr, *rowptr2, *colptr2, *dataend;
+ const PIXEL_T *rowptr, *colptr, *rowptr2, *colptr2,
+ *dataend = &data[stride * h];
bool willTransform = ig->willTransform();
- data = (const PIXEL_T*)buffer;
- dataend = &data[stride * h];
-
if (willTransform) {
mask = serverpf.redMax << serverpf.redShift;
mask |= serverpf.greenMax << serverpf.greenShift;