Fix temporary decoder buffer sizes
Some of these were excessively large because of redundant factors
in the size calculation.
diff --git a/common/rfb/ZRLEDecoder.cxx b/common/rfb/ZRLEDecoder.cxx
index c13f286..b891ba5 100644
--- a/common/rfb/ZRLEDecoder.cxx
+++ b/common/rfb/ZRLEDecoder.cxx
@@ -86,10 +86,9 @@
{
rdr::MemInStream is(buffer, buflen);
const rfb::PixelFormat& pf = cp.pf();
- rdr::U8* buf[64 * 64 * 4 * pf.bpp/8];
switch (pf.bpp) {
- case 8: zrleDecode8 (r, &is, &zis, (rdr::U8*) buf, pf, pb); break;
- case 16: zrleDecode16(r, &is, &zis, (rdr::U16*)buf, pf, pb); break;
+ case 8: zrleDecode8 (r, &is, &zis, pf, pb); break;
+ case 16: zrleDecode16(r, &is, &zis, pf, pb); break;
case 32:
{
Pixel maxPixel = pf.pixelFromRGB((rdr::U16)-1, (rdr::U16)-1, (rdr::U16)-1);
@@ -99,16 +98,16 @@
if ((fitsInLS3Bytes && pf.isLittleEndian()) ||
(fitsInMS3Bytes && pf.isBigEndian()))
{
- zrleDecode24A(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+ zrleDecode24A(r, &is, &zis, pf, pb);
}
else if ((fitsInLS3Bytes && pf.isBigEndian()) ||
(fitsInMS3Bytes && pf.isLittleEndian()))
{
- zrleDecode24B(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+ zrleDecode24B(r, &is, &zis, pf, pb);
}
else
{
- zrleDecode32(r, &is, &zis, (rdr::U32*)buf, pf, pb);
+ zrleDecode32(r, &is, &zis, pf, pb);
}
break;
}
diff --git a/common/rfb/hextileDecode.h b/common/rfb/hextileDecode.h
index 7affa15..47006a0 100644
--- a/common/rfb/hextileDecode.h
+++ b/common/rfb/hextileDecode.h
@@ -44,7 +44,7 @@
Rect t;
PIXEL_T bg = 0;
PIXEL_T fg = 0;
- PIXEL_T buf[16 * 16 * 4];
+ PIXEL_T buf[16 * 16];
for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 16) {
diff --git a/common/rfb/zrleDecode.h b/common/rfb/zrleDecode.h
index 07d6795..0bfbbe1 100644
--- a/common/rfb/zrleDecode.h
+++ b/common/rfb/zrleDecode.h
@@ -47,12 +47,13 @@
#endif
void ZRLE_DECODE (const Rect& r, rdr::InStream* is,
- rdr::ZlibInStream* zis, PIXEL_T* buf,
+ rdr::ZlibInStream* zis,
const PixelFormat& pf, ModifiablePixelBuffer* pb)
{
int length = is->readU32();
zis->setUnderlying(is, length);
Rect t;
+ PIXEL_T buf[64 * 64];
for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 64) {