commit f81148c43a25d4c69e635b6ad13eab674b473aca
author Pierre Ossman <ossman@cendio.se> Wed Jul 25 20:44:32 2018 +0200
committer Pierre Ossman <ossman@cendio.se> Wed Jul 25 20:44:32 2018 +0200
tree 608c1605e931b1fe2456ec7c236aec3c18207fbc
parent d3a3fd692f9ee83164590338118034422308579c

Avoid integer overflows in pixel size calculations

common/rfb/PixelBuffer.cxx

diff --git a/common/rfb/PixelBuffer.cxx b/common/rfb/PixelBuffer.cxx
index 007b6c8..7f4c1ad 100644
--- a/common/rfb/PixelBuffer.cxx
+++ b/common/rfb/PixelBuffer.cxx
@@ -204,6 +204,7 @@
                                      const Point &move_by_delta)
 {
   int srcStride, dstStride;
+  int bytesPerPixel;
   const U8* srcData;
   U8* dstData;
 
@@ -221,6 +222,8 @@
                          srect.width(), srect.height(),
                          srect.tl.x, srect.tl.y, width_, height_);
 
+  bytesPerPixel = format.bpp/8;
+
   srcData = getBuffer(srect, &srcStride);
   dstData = getBufferRW(drect, &dstStride);
 
@@ -228,27 +231,27 @@
     // Possible overlap. Be careful and use memmove().
     int h = drect.height();
     while (h--) {
-      memmove(dstData, srcData, drect.width() * format.bpp/8);
-      dstData += dstStride * format.bpp/8;
-      srcData += srcStride * format.bpp/8;
+      memmove(dstData, srcData, drect.width() * bytesPerPixel);
+      dstData += dstStride * bytesPerPixel;
+      srcData += srcStride * bytesPerPixel;
     }
   } else if (move_by_delta.y < 0) {
     // The data shifted upwards. Copy from top to bottom.
     int h = drect.height();
     while (h--) {
-      memcpy(dstData, srcData, drect.width() * format.bpp/8);
-      dstData += dstStride * format.bpp/8;
-      srcData += srcStride * format.bpp/8;
+      memcpy(dstData, srcData, drect.width() * bytesPerPixel);
+      dstData += dstStride * bytesPerPixel;
+      srcData += srcStride * bytesPerPixel;
     }
   } else {
     // The data shifted downwards. Copy from bottom to top.
     int h = drect.height();
-    dstData += (h-1) * dstStride * format.bpp/8;
-    srcData += (h-1) * srcStride * format.bpp/8;
+    dstData += (h-1) * dstStride * bytesPerPixel;
+    srcData += (h-1) * srcStride * bytesPerPixel;
     while (h--) {
-      memcpy(dstData, srcData, drect.width() * format.bpp/8);
-      dstData -= dstStride * format.bpp/8;
-      srcData -= srcStride * format.bpp/8;
+      memcpy(dstData, srcData, drect.width() * bytesPerPixel);
+      dstData -= dstStride * bytesPerPixel;
+      srcData -= srcStride * bytesPerPixel;
     }
   }
 
@@ -304,7 +307,7 @@
                          r.tl.x, r.tl.y, width_, height_);
 
   *stride_ = stride;
-  return &data[(r.tl.x + (r.tl.y * stride)) * format.bpp/8];
+  return &data[(r.tl.x + (r.tl.y * stride)) * (format.bpp/8)];
 }
 
 void FullFramePixelBuffer::commitBufferRW(const Rect& r)
@@ -319,7 +322,7 @@
                          r.tl.x, r.tl.y, width_, height_);
 
   *stride_ = stride;
-  return &data[(r.tl.x + (r.tl.y * stride)) * format.bpp/8];
+  return &data[(r.tl.x + (r.tl.y * stride)) * (format.bpp/8)];
 }
 
 // -=- Managed pixel buffer class