blob: 5dfa5d2d9b91b0bfb5edfc35f42977b6f7d91abb [file] [log] [blame]
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +00001/* Copyright (C) 2000-2003 Constantin Kaplinsky. All Rights Reserved.
2 * Copyright (C) 2004-2005 Cendio AB. All rights reserved.
3 *
4 * This is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This software is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this software; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
17 * USA.
18 */
19
20//
21// Tight decoding functions.
22//
23// This file is #included after having set the following macros:
24// BPP - 8, 16 or 32
25// EXTRA_ARGS - optional extra arguments
26// FILL_RECT - fill a rectangle with a single color
27// IMAGE_RECT - draw a rectangle of pixel data from a buffer
28
29#include <rdr/InStream.h>
30#include <rdr/ZlibInStream.h>
31#include <rfb/Exception.h>
32#include <assert.h>
33
34namespace rfb {
35
36// CONCAT2E concatenates its arguments, expanding them if they are macros
37
38#ifndef CONCAT2E
39#define CONCAT2(a,b) a##b
40#define CONCAT2E(a,b) CONCAT2(a,b)
41#endif
42
43#define PIXEL_T rdr::CONCAT2E(U,BPP)
44#define READ_PIXEL CONCAT2E(readOpaque,BPP)
45#define TIGHT_DECODE CONCAT2E(tightDecode,BPP)
46
47#define TIGHT_MIN_TO_COMPRESS 12
48static bool DecompressJpegRect(const Rect& r, rdr::InStream* is,
49 PIXEL_T* buf, CMsgHandler* handler);
50static void FilterGradient(const Rect& r, rdr::InStream* is, int dataSize,
51 PIXEL_T* buf, CMsgHandler* handler);
52#if BPP == 32
53static void FilterGradient24(const Rect& r, rdr::InStream* is, int dataSize,
54 PIXEL_T* buf, CMsgHandler* handler);
55#endif
56
57// Main function implementing Tight decoder
58
59void TIGHT_DECODE (const Rect& r, rdr::InStream* is,
60 rdr::ZlibInStream zis[], PIXEL_T* buf
61#ifdef EXTRA_ARGS
62 , EXTRA_ARGS
63#endif
64 )
65{
66 rdr::U8 *bytebuf = (rdr::U8*) buf;
67 bool cutZeros = false;
68 const rfb::PixelFormat& myFormat = handler->cp.pf();
69#if BPP == 32
Pierre Ossman67b2b2f2009-03-06 10:12:55 +000070 if (myFormat.is888()) {
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000071 cutZeros = true;
72 }
73#endif
74
75 rdr::U8 comp_ctl = is->readU8();
76
77 // Flush zlib streams if we are told by the server to do so.
78 for (int i = 0; i < 4; i++) {
79 if (comp_ctl & 1) {
80 zis[i].reset();
81 }
82 comp_ctl >>= 1;
83 }
84
85 // "Fill" compression type.
86 if (comp_ctl == rfbTightFill) {
87 PIXEL_T pix;
88 if (cutZeros) {
89 is->readBytes(bytebuf, 3);
Pierre Ossman67b2b2f2009-03-06 10:12:55 +000090 pix = myFormat.pixelFromRGB(bytebuf[0], bytebuf[1], bytebuf[2]);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000091 } else {
92 pix = is->READ_PIXEL();
93 }
94 FILL_RECT(r, pix);
95 return;
96 }
97
98 // "JPEG" compression type.
99 if (comp_ctl == rfbTightJpeg) {
100 DecompressJpegRect(r, is, buf, handler);
101 return;
102 }
103
104 // Quit on unsupported compression type.
105 if (comp_ctl > rfbTightMaxSubencoding) {
106 throw Exception("TightDecoder: bad subencoding value received");
107 return;
108 }
109
110 // "Basic" compression type.
111 int palSize = 0;
112 static PIXEL_T palette[256];
113 bool useGradient = false;
114
115 if ((comp_ctl & rfbTightExplicitFilter) != 0) {
116 rdr::U8 filterId = is->readU8();
117
118 switch (filterId) {
119 case rfbTightFilterPalette:
120 palSize = is->readU8() + 1;
121 if (cutZeros) {
122 rdr::U8 *tightPalette = (rdr::U8*) palette;
123 is->readBytes(tightPalette, palSize*3);
124 for (int i = palSize - 1; i >= 0; i--) {
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000125 palette[i] = myFormat.pixelFromRGB(tightPalette[i*3],
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000126 tightPalette[i*3+1],
127 tightPalette[i*3+2]);
128 }
129 } else {
130 for (int i = 0; i < palSize; i++)
131 palette[i] = is->READ_PIXEL();
132 }
133 break;
134 case rfbTightFilterGradient:
135 useGradient = true;
136 break;
137 case rfbTightFilterCopy:
138 break;
139 default:
140 throw Exception("TightDecoder: unknown filter code received");
141 return;
142 }
143 }
144
145 int bppp = BPP;
146 if (palSize != 0) {
147 bppp = (palSize <= 2) ? 1 : 8;
148 } else if (cutZeros) {
149 bppp = 24;
150 }
151
152 // Determine if the data should be decompressed or just copied.
153 int rowSize = (r.width() * bppp + 7) / 8;
154 int dataSize = r.height() * rowSize;
155 int streamId = -1;
156 rdr::InStream *input;
157 if (dataSize < TIGHT_MIN_TO_COMPRESS) {
158 input = is;
159 } else {
160 int length = is->readCompactLength();
161 streamId = comp_ctl & 0x03;
162 zis[streamId].setUnderlying(is, length);
163 input = &zis[streamId];
164 }
165
166 if (palSize == 0) {
167 // Truecolor data
168 if (useGradient) {
169#if BPP == 32
170 if (cutZeros) {
171 FilterGradient24(r, input, dataSize, buf, handler);
172 } else
173#endif
174 {
175 FilterGradient(r, input, dataSize, buf, handler);
176 }
177 } else {
178 input->readBytes(buf, dataSize);
179 if (cutZeros) {
180 for (int p = r.height() * r.width() - 1; p >= 0; p--) {
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000181 buf[p] = myFormat.pixelFromRGB(bytebuf[p*3],
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000182 bytebuf[p*3+1],
183 bytebuf[p*3+2]);
184 }
185 }
186 }
187 } else {
188 int x, y, b, w;
189 PIXEL_T *ptr = buf;
190 rdr::U8 bits;
191 if (palSize <= 2) {
192 // 2-color palette
193 w = (r.width() + 7) / 8;
194 for (y = 0; y < r.height(); y++) {
195 for (x = 0; x < r.width() / 8; x++) {
196 bits = input->readU8();
197 for (b = 7; b >= 0; b--) {
198 *ptr++ = palette[bits >> b & 1];
199 }
200 }
201 if (r.width() % 8 != 0) {
202 bits = input->readU8();
203 for (b = 7; b >= 8 - r.width() % 8; b--) {
204 *ptr++ = palette[bits >> b & 1];
205 }
206 }
207 }
208 } else {
209 // 256-color palette
210 for (y = 0; y < r.height(); y++) {
211 for (x = 0; x < r.width(); x++) {
212 *ptr++ = palette[input->readU8()];
213 }
214 }
215 }
216 }
217
218 IMAGE_RECT(r, buf);
219
220 if (streamId != -1) {
221 zis[streamId].reset();
222 }
223}
224
225static bool
226DecompressJpegRect(const Rect& r, rdr::InStream* is,
227 PIXEL_T* buf, CMsgHandler* handler)
228{
229 struct jpeg_decompress_struct cinfo;
230 struct jpeg_error_mgr jerr;
DRCe420dcb2009-04-06 07:20:34 +0000231 int w = r.width();
232 int h = r.height();
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000233 int pixelsize;
DRCe420dcb2009-04-06 07:20:34 +0000234 rdr::U8 *dstBuf = NULL;
235 bool dstBufIsTemp = false;
236 const rfb::PixelFormat& pf = handler->cp.pf();
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000237
238 // Read length
239 int compressedLen = is->readCompactLength();
240 if (compressedLen <= 0) {
241 throw Exception("Incorrect data received from the server.\n");
242 }
243
244 // Allocate netbuf and read in data
245 rdr::U8* netbuf = new rdr::U8[compressedLen];
246 if (!netbuf) {
247 throw Exception("rfb::tightDecode unable to allocate buffer");
248 }
249 is->readBytes(netbuf, compressedLen);
250
251 // Set up JPEG decompression
252 cinfo.err = jpeg_std_error(&jerr);
253 jpeg_create_decompress(&cinfo);
254 JpegSetSrcManager(&cinfo, (char*)netbuf, compressedLen);
255 jpeg_read_header(&cinfo, TRUE);
DRCe420dcb2009-04-06 07:20:34 +0000256
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000257 cinfo.out_color_space = JCS_RGB;
258 pixelsize = 3;
259
260#ifdef JCS_EXTENSIONS
261 // Try to have libjpeg output directly to our native format
262 if (pf.is888()) {
DRCe420dcb2009-04-06 07:20:34 +0000263 int redShift, greenShift, blueShift;
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000264
DRCe420dcb2009-04-06 07:20:34 +0000265 if(pf.bigEndian) {
266 redShift = 24 - pf.redShift;
267 greenShift = 24 - pf.greenShift;
268 blueShift = 24 - pf.blueShift;
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000269 } else {
DRCe420dcb2009-04-06 07:20:34 +0000270 redShift = pf.redShift;
271 greenShift = pf.greenShift;
272 blueShift = pf.blueShift;
273 }
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000274
275 // libjpeg can only handle some "standard" formats
276 if(redShift == 0 && greenShift == 8 && blueShift == 16)
DRCe420dcb2009-04-06 07:20:34 +0000277 cinfo.out_color_space = JCS_EXT_RGBX;
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000278 if(redShift == 16 && greenShift == 8 && blueShift == 0)
DRCe420dcb2009-04-06 07:20:34 +0000279 cinfo.out_color_space = JCS_EXT_BGRX;
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000280 if(redShift == 24 && greenShift == 16 && blueShift == 8)
DRCe420dcb2009-04-06 07:20:34 +0000281 cinfo.out_color_space = JCS_EXT_XBGR;
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000282 if(redShift == 8 && greenShift == 16 && blueShift == 24)
DRCe420dcb2009-04-06 07:20:34 +0000283 cinfo.out_color_space = JCS_EXT_XRGB;
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000284
285 if (cinfo.out_color_space != JCS_RGB) {
DRCe420dcb2009-04-06 07:20:34 +0000286 dstBuf = (rdr::U8 *)buf;
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000287 pixelsize = 4;
288 }
DRCe420dcb2009-04-06 07:20:34 +0000289 }
Pierre Ossman6f0ce582009-04-30 11:41:03 +0000290#endif
291
292 if (cinfo.out_color_space == JCS_RGB) {
DRCe420dcb2009-04-06 07:20:34 +0000293 dstBuf = new rdr::U8[w * h * pixelsize];
294 dstBufIsTemp = true;
DRCe420dcb2009-04-06 07:20:34 +0000295 }
296
297 JSAMPROW *rowPointer = new JSAMPROW[h];
298 for (int dy = 0; dy < h; dy++)
299 rowPointer[dy] = (JSAMPROW)(&dstBuf[dy * w * pixelsize]);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000300
301 jpeg_start_decompress(&cinfo);
302 if (cinfo.output_width != (unsigned)r.width() || cinfo.output_height != (unsigned)r.height() ||
DRCe420dcb2009-04-06 07:20:34 +0000303 cinfo.output_components != pixelsize) {
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000304 jpeg_destroy_decompress(&cinfo);
305 throw Exception("Tight Encoding: Wrong JPEG data received.\n");
306 }
307
308 // Decompress
309 const rfb::PixelFormat& myFormat = handler->cp.pf();
310 while (cinfo.output_scanline < cinfo.output_height) {
DRCe420dcb2009-04-06 07:20:34 +0000311 jpeg_read_scanlines(&cinfo, &rowPointer[cinfo.output_scanline],
312 cinfo.output_height - cinfo.output_scanline);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000313 if (jpegError) {
314 break;
315 }
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000316 }
317
Adam Tkac43d68232010-01-27 12:37:38 +0000318 delete [] rowPointer;
319
DRCe420dcb2009-04-06 07:20:34 +0000320 if (cinfo.out_color_space == JCS_RGB)
321 myFormat.bufferFromRGB((rdr::U8*)buf, dstBuf, w * h);
322
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000323 IMAGE_RECT(r, buf);
324
325 if (!jpegError) {
326 jpeg_finish_decompress(&cinfo);
327 }
328
329 jpeg_destroy_decompress(&cinfo);
330
DRCe420dcb2009-04-06 07:20:34 +0000331 if (dstBufIsTemp) delete [] dstBuf;
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000332 delete [] netbuf;
333
334 return !jpegError;
335}
336
337#if BPP == 32
338
339static void
340FilterGradient24(const Rect& r, rdr::InStream* is, int dataSize,
341 PIXEL_T* buf, CMsgHandler* handler)
342{
343 int x, y, c;
344 static rdr::U8 prevRow[TIGHT_MAX_WIDTH*3];
345 static rdr::U8 thisRow[TIGHT_MAX_WIDTH*3];
346 rdr::U8 pix[3];
347 int est[3];
348
349 memset(prevRow, 0, sizeof(prevRow));
350
351 // Allocate netbuf and read in data
352 rdr::U8 *netbuf = new rdr::U8[dataSize];
353 if (!netbuf) {
354 throw Exception("rfb::tightDecode unable to allocate buffer");
355 }
356 is->readBytes(netbuf, dataSize);
357
358 // Set up shortcut variables
359 const rfb::PixelFormat& myFormat = handler->cp.pf();
360 int rectHeight = r.height();
361 int rectWidth = r.width();
362
363 for (y = 0; y < rectHeight; y++) {
364 /* First pixel in a row */
365 for (c = 0; c < 3; c++) {
366 pix[c] = netbuf[y*rectWidth*3+c] + prevRow[c];
367 thisRow[c] = pix[c];
368 }
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000369 buf[y*rectWidth] = myFormat.pixelFromRGB(pix[0], pix[1], pix[2]);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000370
371 /* Remaining pixels of a row */
372 for (x = 1; x < rectWidth; x++) {
373 for (c = 0; c < 3; c++) {
374 est[c] = prevRow[x*3+c] + pix[c] - prevRow[(x-1)*3+c];
375 if (est[c] > 0xff) {
376 est[c] = 0xff;
377 } else if (est[c] < 0) {
378 est[c] = 0;
379 }
380 pix[c] = netbuf[(y*rectWidth+x)*3+c] + est[c];
381 thisRow[x*3+c] = pix[c];
382 }
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000383 buf[y*rectWidth+x] = myFormat.pixelFromRGB(pix[0], pix[1], pix[2]);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000384 }
385
386 memcpy(prevRow, thisRow, sizeof(prevRow));
387 }
388
389 delete [] netbuf;
390}
391
392#endif
393
394static void
395FilterGradient(const Rect& r, rdr::InStream* is, int dataSize,
396 PIXEL_T* buf, CMsgHandler* handler)
397{
398 int x, y, c;
399 static rdr::U8 prevRow[TIGHT_MAX_WIDTH*sizeof(PIXEL_T)];
400 static rdr::U8 thisRow[TIGHT_MAX_WIDTH*sizeof(PIXEL_T)];
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000401 rdr::U8 pix[3];
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000402 int est[3];
403
404 memset(prevRow, 0, sizeof(prevRow));
405
406 // Allocate netbuf and read in data
407 PIXEL_T *netbuf = (PIXEL_T*)new rdr::U8[dataSize];
408 if (!netbuf) {
409 throw Exception("rfb::tightDecode unable to allocate buffer");
410 }
411 is->readBytes(netbuf, dataSize);
412
413 // Set up shortcut variables
414 const rfb::PixelFormat& myFormat = handler->cp.pf();
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000415 int rectHeight = r.height();
416 int rectWidth = r.width();
417
418 for (y = 0; y < rectHeight; y++) {
419 /* First pixel in a row */
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000420 myFormat.rgbFromPixel(netbuf[y*rectWidth], NULL,
421 &pix[0], &pix[1], &pix[2]);
422 for (c = 0; c < 3; c++)
423 pix[c] += prevRow[c];
424
425 memcpy(thisRow, pix, sizeof(pix));
426
427 buf[y*rectWidth] = myFormat.pixelFromRGB(pix[0], pix[1], pix[2]);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000428
429 /* Remaining pixels of a row */
430 for (x = 1; x < rectWidth; x++) {
431 for (c = 0; c < 3; c++) {
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000432 est[c] = prevRow[x*3+c] + pix[c] - prevRow[(x-1)*3+c];
433 if (est[c] > 255) {
434 est[c] = 255;
435 } else if (est[c] < 0) {
436 est[c] = 0;
437 }
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000438 }
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000439
440 myFormat.rgbFromPixel(netbuf[y*rectWidth+x], NULL,
441 &pix[0], &pix[1], &pix[2]);
442 for (c = 0; c < 3; c++)
443 pix[c] += est[c];
444
445 memcpy(&thisRow[x*3], pix, sizeof(pix));
446
447 buf[y*rectWidth+x] = myFormat.pixelFromRGB(pix[0], pix[1], pix[2]);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000448 }
449
450 memcpy(prevRow, thisRow, sizeof(prevRow));
451 }
452
453 delete [] netbuf;
454}
455
456#undef TIGHT_MIN_TO_COMPRESS
457#undef TIGHT_DECODE
458#undef READ_PIXEL
459#undef PIXEL_T
460}