blob: 0089f453c52bdc8844e21c66870d5a1a4013633a [file] [log] [blame]
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +00001/* Copyright (C) 2000-2003 Constantin Kaplinsky. All Rights Reserved.
Peter Åstrandd69bcc42011-09-28 12:52:53 +00002 * Copyright 2004-2005 Cendio AB.
DRC33c15e32011-11-03 18:49:21 +00003 * Copyright (C) 2011 D. R. Commander. All Rights Reserved.
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +00004 *
5 * This is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This software is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this software; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
18 * USA.
19 */
20
21//
22// Tight decoding functions.
23//
Pierre Ossmanbcc295e2014-02-12 13:16:43 +010024// This file is #included after having set the following macro:
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000025// BPP - 8, 16 or 32
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000026
27#include <rdr/InStream.h>
28#include <rdr/ZlibInStream.h>
29#include <rfb/Exception.h>
Pierre Ossmanc0397262014-03-14 15:59:46 +010030#include <rfb/TightConstants.h>
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000031#include <assert.h>
32
33namespace rfb {
34
35// CONCAT2E concatenates its arguments, expanding them if they are macros
36
37#ifndef CONCAT2E
38#define CONCAT2(a,b) a##b
39#define CONCAT2E(a,b) CONCAT2(a,b)
40#endif
41
42#define PIXEL_T rdr::CONCAT2E(U,BPP)
43#define READ_PIXEL CONCAT2E(readOpaque,BPP)
DRC33c15e32011-11-03 18:49:21 +000044#define TIGHT_DECODE TightDecoder::CONCAT2E(tightDecode,BPP)
45#define DECOMPRESS_JPEG_RECT TightDecoder::CONCAT2E(DecompressJpegRect,BPP)
46#define FILTER_GRADIENT TightDecoder::CONCAT2E(FilterGradient,BPP)
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000047
48#define TIGHT_MIN_TO_COMPRESS 12
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000049
50// Main function implementing Tight decoder
51
DRC33c15e32011-11-03 18:49:21 +000052void TIGHT_DECODE (const Rect& r)
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000053{
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000054 bool cutZeros = false;
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000055#if BPP == 32
DRC33c15e32011-11-03 18:49:21 +000056 if (serverpf.is888()) {
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000057 cutZeros = true;
58 }
59#endif
60
61 rdr::U8 comp_ctl = is->readU8();
62
63 // Flush zlib streams if we are told by the server to do so.
64 for (int i = 0; i < 4; i++) {
65 if (comp_ctl & 1) {
66 zis[i].reset();
67 }
68 comp_ctl >>= 1;
69 }
70
71 // "Fill" compression type.
Pierre Ossmanc0397262014-03-14 15:59:46 +010072 if (comp_ctl == tightFill) {
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000073 PIXEL_T pix;
74 if (cutZeros) {
DRC33c15e32011-11-03 18:49:21 +000075 rdr::U8 bytebuf[3];
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000076 is->readBytes(bytebuf, 3);
Pierre Ossmanb6b4dc62014-01-20 15:05:21 +010077 serverpf.bufferFromRGB((rdr::U8*)&pix, bytebuf, 1);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000078 } else {
79 pix = is->READ_PIXEL();
80 }
Pierre Ossman0c9bd4b2014-07-09 16:44:11 +020081 pb->fillRect(serverpf, r, pix);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000082 return;
83 }
84
85 // "JPEG" compression type.
Pierre Ossmanc0397262014-03-14 15:59:46 +010086 if (comp_ctl == tightJpeg) {
DRC33c15e32011-11-03 18:49:21 +000087 DECOMPRESS_JPEG_RECT(r);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000088 return;
89 }
90
91 // Quit on unsupported compression type.
Pierre Ossmanc0397262014-03-14 15:59:46 +010092 if (comp_ctl > tightMaxSubencoding) {
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +000093 throw Exception("TightDecoder: bad subencoding value received");
94 return;
95 }
96
97 // "Basic" compression type.
98 int palSize = 0;
99 static PIXEL_T palette[256];
100 bool useGradient = false;
101
Pierre Ossmanc0397262014-03-14 15:59:46 +0100102 if ((comp_ctl & tightExplicitFilter) != 0) {
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000103 rdr::U8 filterId = is->readU8();
104
105 switch (filterId) {
Pierre Ossmanc0397262014-03-14 15:59:46 +0100106 case tightFilterPalette:
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000107 palSize = is->readU8() + 1;
108 if (cutZeros) {
DRCa5004a32011-11-04 04:51:17 +0000109 rdr::U8 tightPalette[256 * 3];
110 is->readBytes(tightPalette, palSize * 3);
Pierre Ossmanb6b4dc62014-01-20 15:05:21 +0100111 serverpf.bufferFromRGB((rdr::U8*)palette, tightPalette, palSize);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000112 } else {
DRCa5004a32011-11-04 04:51:17 +0000113 is->readBytes(palette, palSize * sizeof(PIXEL_T));
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000114 }
115 break;
Pierre Ossmanc0397262014-03-14 15:59:46 +0100116 case tightFilterGradient:
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000117 useGradient = true;
118 break;
Pierre Ossmanc0397262014-03-14 15:59:46 +0100119 case tightFilterCopy:
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000120 break;
121 default:
122 throw Exception("TightDecoder: unknown filter code received");
123 return;
124 }
125 }
126
127 int bppp = BPP;
128 if (palSize != 0) {
129 bppp = (palSize <= 2) ? 1 : 8;
130 } else if (cutZeros) {
131 bppp = 24;
132 }
133
134 // Determine if the data should be decompressed or just copied.
135 int rowSize = (r.width() * bppp + 7) / 8;
136 int dataSize = r.height() * rowSize;
137 int streamId = -1;
138 rdr::InStream *input;
139 if (dataSize < TIGHT_MIN_TO_COMPRESS) {
140 input = is;
141 } else {
Pierre Ossman7b5c0692014-03-17 14:35:51 +0100142 int length = readCompact(is);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000143 streamId = comp_ctl & 0x03;
144 zis[streamId].setUnderlying(is, length);
145 input = &zis[streamId];
146 }
147
DRCa5004a32011-11-04 04:51:17 +0000148 // Allocate netbuf and read in data
149 rdr::U8 *netbuf = new rdr::U8[dataSize];
150 if (!netbuf) {
151 throw Exception("rfb::TightDecoder::tightDecode unable to allocate buffer");
152 }
153 input->readBytes(netbuf, dataSize);
154
DRC33c15e32011-11-03 18:49:21 +0000155 PIXEL_T *buf;
156 int stride = r.width();
Pierre Ossman0c9bd4b2014-07-09 16:44:11 +0200157 if (directDecode) buf = (PIXEL_T *)pb->getBufferRW(r, &stride);
Pierre Ossman668468b2014-01-31 12:37:32 +0100158 else buf = (PIXEL_T *)conn->reader()->getImageBuf(r.area());
DRC33c15e32011-11-03 18:49:21 +0000159
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000160 if (palSize == 0) {
161 // Truecolor data
162 if (useGradient) {
163#if BPP == 32
164 if (cutZeros) {
DRCa5004a32011-11-04 04:51:17 +0000165 FilterGradient24(netbuf, buf, stride, r);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000166 } else
167#endif
DRC33c15e32011-11-03 18:49:21 +0000168 {
DRCa5004a32011-11-04 04:51:17 +0000169 FILTER_GRADIENT(netbuf, buf, stride, r);
DRC33c15e32011-11-03 18:49:21 +0000170 }
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000171 } else {
DRC33c15e32011-11-03 18:49:21 +0000172 // Copy
173 int h = r.height();
174 PIXEL_T *ptr = buf;
DRCa5004a32011-11-04 04:51:17 +0000175 rdr::U8 *srcPtr = netbuf;
176 int w = r.width();
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000177 if (cutZeros) {
DRC33c15e32011-11-03 18:49:21 +0000178 while (h > 0) {
Pierre Ossmanb6b4dc62014-01-20 15:05:21 +0100179 serverpf.bufferFromRGB((rdr::U8*)ptr, srcPtr, w);
DRCa5004a32011-11-04 04:51:17 +0000180 ptr += stride;
Pierre Ossmane3cb4a22011-11-08 13:52:33 +0000181 srcPtr += w * 3;
DRC33c15e32011-11-03 18:49:21 +0000182 h--;
Pierre Ossman7c4efd72010-09-30 11:30:20 +0000183 }
184 } else {
DRC33c15e32011-11-03 18:49:21 +0000185 while (h > 0) {
DRC77b50282011-11-09 18:18:11 +0000186 memcpy(ptr, srcPtr, w * sizeof(PIXEL_T));
DRC33c15e32011-11-03 18:49:21 +0000187 ptr += stride;
DRCa5004a32011-11-04 04:51:17 +0000188 srcPtr += w * sizeof(PIXEL_T);
DRC33c15e32011-11-03 18:49:21 +0000189 h--;
190 }
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000191 }
192 }
193 } else {
DRC33c15e32011-11-03 18:49:21 +0000194 // Indexed color
195 int x, h = r.height(), w = r.width(), b, pad = stride - w;
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000196 PIXEL_T *ptr = buf;
DRCa5004a32011-11-04 04:51:17 +0000197 rdr::U8 bits, *srcPtr = netbuf;
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000198 if (palSize <= 2) {
199 // 2-color palette
DRC33c15e32011-11-03 18:49:21 +0000200 while (h > 0) {
201 for (x = 0; x < w / 8; x++) {
DRCa5004a32011-11-04 04:51:17 +0000202 bits = *srcPtr++;
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000203 for (b = 7; b >= 0; b--) {
204 *ptr++ = palette[bits >> b & 1];
DRC33c15e32011-11-03 18:49:21 +0000205 }
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000206 }
DRC33c15e32011-11-03 18:49:21 +0000207 if (w % 8 != 0) {
DRCa5004a32011-11-04 04:51:17 +0000208 bits = *srcPtr++;
DRC33c15e32011-11-03 18:49:21 +0000209 for (b = 7; b >= 8 - w % 8; b--) {
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000210 *ptr++ = palette[bits >> b & 1];
211 }
212 }
DRC33c15e32011-11-03 18:49:21 +0000213 ptr += pad;
214 h--;
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000215 }
216 } else {
217 // 256-color palette
DRC33c15e32011-11-03 18:49:21 +0000218 while (h > 0) {
219 PIXEL_T *endOfRow = ptr + w;
220 while (ptr < endOfRow) {
DRCa5004a32011-11-04 04:51:17 +0000221 *ptr++ = palette[*srcPtr++];
DRC33c15e32011-11-03 18:49:21 +0000222 }
223 ptr += pad;
224 h--;
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000225 }
226 }
227 }
228
Pierre Ossman0c9bd4b2014-07-09 16:44:11 +0200229 if (directDecode) pb->commitBufferRW(r);
230 else pb->imageRect(serverpf, r, buf);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000231
DRCa5004a32011-11-04 04:51:17 +0000232 delete [] netbuf;
233
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000234 if (streamId != -1) {
235 zis[streamId].reset();
236 }
237}
238
DRC33c15e32011-11-03 18:49:21 +0000239void
240DECOMPRESS_JPEG_RECT(const Rect& r)
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000241{
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000242 // Read length
Pierre Ossman7b5c0692014-03-17 14:35:51 +0100243 int compressedLen = readCompact(is);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000244 if (compressedLen <= 0) {
245 throw Exception("Incorrect data received from the server.\n");
246 }
247
248 // Allocate netbuf and read in data
249 rdr::U8* netbuf = new rdr::U8[compressedLen];
250 if (!netbuf) {
DRCa5004a32011-11-04 04:51:17 +0000251 throw Exception("rfb::TightDecoder::DecompressJpegRect unable to allocate buffer");
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000252 }
253 is->readBytes(netbuf, compressedLen);
254
DRC33c15e32011-11-03 18:49:21 +0000255 // We always use direct decoding with JPEG images
256 int stride;
Pierre Ossman0c9bd4b2014-07-09 16:44:11 +0200257 rdr::U8 *buf = pb->getBufferRW(r, &stride);
Pierre Ossmana10d8fe2014-01-22 11:28:05 +0100258 jd.decompress(netbuf, compressedLen, buf, stride, r, clientpf);
Pierre Ossman0c9bd4b2014-07-09 16:44:11 +0200259 pb->commitBufferRW(r);
DRCe420dcb2009-04-06 07:20:34 +0000260
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000261 delete [] netbuf;
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000262}
263
264#if BPP == 32
265
DRC33c15e32011-11-03 18:49:21 +0000266void
DRCa5004a32011-11-04 04:51:17 +0000267TightDecoder::FilterGradient24(rdr::U8 *netbuf, PIXEL_T* buf, int stride,
268 const Rect& r)
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000269{
270 int x, y, c;
271 static rdr::U8 prevRow[TIGHT_MAX_WIDTH*3];
272 static rdr::U8 thisRow[TIGHT_MAX_WIDTH*3];
273 rdr::U8 pix[3];
274 int est[3];
275
276 memset(prevRow, 0, sizeof(prevRow));
277
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000278 // Set up shortcut variables
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000279 int rectHeight = r.height();
280 int rectWidth = r.width();
281
282 for (y = 0; y < rectHeight; y++) {
283 /* First pixel in a row */
284 for (c = 0; c < 3; c++) {
285 pix[c] = netbuf[y*rectWidth*3+c] + prevRow[c];
286 thisRow[c] = pix[c];
287 }
Pierre Ossmanb6b4dc62014-01-20 15:05:21 +0100288 serverpf.bufferFromRGB((rdr::U8*)&buf[y*stride], pix, 1);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000289
290 /* Remaining pixels of a row */
291 for (x = 1; x < rectWidth; x++) {
292 for (c = 0; c < 3; c++) {
DRC33c15e32011-11-03 18:49:21 +0000293 est[c] = prevRow[x*3+c] + pix[c] - prevRow[(x-1)*3+c];
294 if (est[c] > 0xff) {
295 est[c] = 0xff;
296 } else if (est[c] < 0) {
297 est[c] = 0;
298 }
299 pix[c] = netbuf[(y*rectWidth+x)*3+c] + est[c];
300 thisRow[x*3+c] = pix[c];
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000301 }
Pierre Ossmanb6b4dc62014-01-20 15:05:21 +0100302 serverpf.bufferFromRGB((rdr::U8*)&buf[y*stride+x], pix, 1);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000303 }
304
305 memcpy(prevRow, thisRow, sizeof(prevRow));
306 }
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000307}
308
309#endif
310
DRC33c15e32011-11-03 18:49:21 +0000311void
DRCa5004a32011-11-04 04:51:17 +0000312FILTER_GRADIENT(rdr::U8 *netbuf, PIXEL_T* buf, int stride, const Rect& r)
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000313{
314 int x, y, c;
315 static rdr::U8 prevRow[TIGHT_MAX_WIDTH*sizeof(PIXEL_T)];
316 static rdr::U8 thisRow[TIGHT_MAX_WIDTH*sizeof(PIXEL_T)];
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000317 rdr::U8 pix[3];
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000318 int est[3];
319
320 memset(prevRow, 0, sizeof(prevRow));
321
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000322 // Set up shortcut variables
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000323 int rectHeight = r.height();
324 int rectWidth = r.width();
325
326 for (y = 0; y < rectHeight; y++) {
327 /* First pixel in a row */
Pierre Ossmanb6b4dc62014-01-20 15:05:21 +0100328 serverpf.rgbFromBuffer(pix, (rdr::U8*)&netbuf[y*rectWidth], 1);
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000329 for (c = 0; c < 3; c++)
330 pix[c] += prevRow[c];
331
332 memcpy(thisRow, pix, sizeof(pix));
333
Pierre Ossmanb6b4dc62014-01-20 15:05:21 +0100334 serverpf.bufferFromRGB((rdr::U8*)&buf[y*stride], pix, 1);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000335
336 /* Remaining pixels of a row */
337 for (x = 1; x < rectWidth; x++) {
338 for (c = 0; c < 3; c++) {
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000339 est[c] = prevRow[x*3+c] + pix[c] - prevRow[(x-1)*3+c];
340 if (est[c] > 255) {
341 est[c] = 255;
342 } else if (est[c] < 0) {
343 est[c] = 0;
344 }
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000345 }
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000346
Pierre Ossmanb6b4dc62014-01-20 15:05:21 +0100347 serverpf.rgbFromBuffer(pix, (rdr::U8*)&netbuf[y*rectWidth+x], 1);
Pierre Ossman67b2b2f2009-03-06 10:12:55 +0000348 for (c = 0; c < 3; c++)
349 pix[c] += est[c];
350
351 memcpy(&thisRow[x*3], pix, sizeof(pix));
352
Pierre Ossmanb6b4dc62014-01-20 15:05:21 +0100353 serverpf.bufferFromRGB((rdr::U8*)&buf[y*stride+x], pix, 1);
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000354 }
355
356 memcpy(prevRow, thisRow, sizeof(prevRow));
357 }
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000358}
359
360#undef TIGHT_MIN_TO_COMPRESS
DRC33c15e32011-11-03 18:49:21 +0000361#undef FILTER_GRADIENT
362#undef DECOMPRESS_JPEG_RECT
Constantin Kaplinskya2adc8d2006-05-25 05:01:55 +0000363#undef TIGHT_DECODE
364#undef READ_PIXEL
365#undef PIXEL_T
366}