Avoid integer overflows in pixel size calculations
diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx
index 0ce611e..53e0365 100644
--- a/common/rfb/EncodeManager.cxx
+++ b/common/rfb/EncodeManager.cxx
@@ -519,7 +519,7 @@
stats[klass][activeType].rects++;
stats[klass][activeType].pixels += rect.area();
- equiv = 12 + rect.area() * conn->cp.pf().bpp/8;
+ equiv = 12 + rect.area() * (conn->cp.pf().bpp/8);
stats[klass][activeType].equivalent += equiv;
encoder = encoders[klass];
@@ -561,7 +561,7 @@
copyStats.rects++;
copyStats.pixels += rect->area();
- equiv = 12 + rect->area() * conn->cp.pf().bpp/8;
+ equiv = 12 + rect->area() * (conn->cp.pf().bpp/8);
copyStats.equivalent += equiv;
conn->writer()->writeCopyRect(*rect, rect->tl.x - delta.x,
diff --git a/common/rfb/PixelBuffer.cxx b/common/rfb/PixelBuffer.cxx
index 007b6c8..7f4c1ad 100644
--- a/common/rfb/PixelBuffer.cxx
+++ b/common/rfb/PixelBuffer.cxx
@@ -204,6 +204,7 @@
const Point &move_by_delta)
{
int srcStride, dstStride;
+ int bytesPerPixel;
const U8* srcData;
U8* dstData;
@@ -221,6 +222,8 @@
srect.width(), srect.height(),
srect.tl.x, srect.tl.y, width_, height_);
+ bytesPerPixel = format.bpp/8;
+
srcData = getBuffer(srect, &srcStride);
dstData = getBufferRW(drect, &dstStride);
@@ -228,27 +231,27 @@
// Possible overlap. Be careful and use memmove().
int h = drect.height();
while (h--) {
- memmove(dstData, srcData, drect.width() * format.bpp/8);
- dstData += dstStride * format.bpp/8;
- srcData += srcStride * format.bpp/8;
+ memmove(dstData, srcData, drect.width() * bytesPerPixel);
+ dstData += dstStride * bytesPerPixel;
+ srcData += srcStride * bytesPerPixel;
}
} else if (move_by_delta.y < 0) {
// The data shifted upwards. Copy from top to bottom.
int h = drect.height();
while (h--) {
- memcpy(dstData, srcData, drect.width() * format.bpp/8);
- dstData += dstStride * format.bpp/8;
- srcData += srcStride * format.bpp/8;
+ memcpy(dstData, srcData, drect.width() * bytesPerPixel);
+ dstData += dstStride * bytesPerPixel;
+ srcData += srcStride * bytesPerPixel;
}
} else {
// The data shifted downwards. Copy from bottom to top.
int h = drect.height();
- dstData += (h-1) * dstStride * format.bpp/8;
- srcData += (h-1) * srcStride * format.bpp/8;
+ dstData += (h-1) * dstStride * bytesPerPixel;
+ srcData += (h-1) * srcStride * bytesPerPixel;
while (h--) {
- memcpy(dstData, srcData, drect.width() * format.bpp/8);
- dstData -= dstStride * format.bpp/8;
- srcData -= srcStride * format.bpp/8;
+ memcpy(dstData, srcData, drect.width() * bytesPerPixel);
+ dstData -= dstStride * bytesPerPixel;
+ srcData -= srcStride * bytesPerPixel;
}
}
@@ -304,7 +307,7 @@
r.tl.x, r.tl.y, width_, height_);
*stride_ = stride;
- return &data[(r.tl.x + (r.tl.y * stride)) * format.bpp/8];
+ return &data[(r.tl.x + (r.tl.y * stride)) * (format.bpp/8)];
}
void FullFramePixelBuffer::commitBufferRW(const Rect& r)
@@ -319,7 +322,7 @@
r.tl.x, r.tl.y, width_, height_);
*stride_ = stride;
- return &data[(r.tl.x + (r.tl.y * stride)) * format.bpp/8];
+ return &data[(r.tl.x + (r.tl.y * stride)) * (format.bpp/8)];
}
// -=- Managed pixel buffer class
diff --git a/common/rfb/RawDecoder.cxx b/common/rfb/RawDecoder.cxx
index 786f154..ec0c68e 100644
--- a/common/rfb/RawDecoder.cxx
+++ b/common/rfb/RawDecoder.cxx
@@ -36,13 +36,13 @@
void RawDecoder::readRect(const Rect& r, rdr::InStream* is,
const ConnParams& cp, rdr::OutStream* os)
{
- os->copyBytes(is, r.area() * cp.pf().bpp/8);
+ os->copyBytes(is, r.area() * (cp.pf().bpp/8));
}
void RawDecoder::decodeRect(const Rect& r, const void* buffer,
size_t buflen, const ConnParams& cp,
ModifiablePixelBuffer* pb)
{
- assert(buflen >= (size_t)r.area() * cp.pf().bpp/8);
+ assert(buflen >= (size_t)r.area() * (cp.pf().bpp/8));
pb->imageRect(cp.pf(), r, buffer);
}
diff --git a/common/rfb/SMsgWriter.cxx b/common/rfb/SMsgWriter.cxx
index 6ef7692..3da9413 100644
--- a/common/rfb/SMsgWriter.cxx
+++ b/common/rfb/SMsgWriter.cxx
@@ -350,7 +350,7 @@
if (needSetCursor) {
const Cursor& cursor = cp->cursor();
- rdr::U8Array data(cursor.width()*cursor.height() * cp->pf().bpp/8);
+ rdr::U8Array data(cursor.width()*cursor.height() * (cp->pf().bpp/8));
rdr::U8Array mask(cursor.getMask());
const rdr::U8* in;
diff --git a/common/rfb/TightDecoder.cxx b/common/rfb/TightDecoder.cxx
index 3a1254a..cc786f5 100644
--- a/common/rfb/TightDecoder.cxx
+++ b/common/rfb/TightDecoder.cxx
@@ -364,7 +364,7 @@
if (directDecode)
outbuf = pb->getBufferRW(r, &stride);
else {
- outbuf = new rdr::U8[r.area() * pf.bpp/8];
+ outbuf = new rdr::U8[r.area() * (pf.bpp/8)];
stride = r.width();
}
diff --git a/common/rfb/ZRLEEncoder.cxx b/common/rfb/ZRLEEncoder.cxx
index d3afe74..8917d8f 100644
--- a/common/rfb/ZRLEEncoder.cxx
+++ b/common/rfb/ZRLEEncoder.cxx
@@ -223,7 +223,7 @@
pf.bufferFromPixel(pixBuf, maxPixel);
if ((pf.bpp != 32) || ((pixBuf[0] != 0) && (pixBuf[3] != 0))) {
- zos.writeBytes(buffer, count * pf.bpp/8);
+ zos.writeBytes(buffer, count * (pf.bpp/8));
return;
}