libstagefright: Check overflow during VideoFrame creation
During VideoFrame creation, prevent overflow of size calculation or
displayWidth calculation.
Bug: 180357299
Change-Id: I076f957846e12e68a3ba72495d48c0103ccf4adf
diff --git a/media/libstagefright/FrameDecoder.cpp b/media/libstagefright/FrameDecoder.cpp
index d11408d..a78e6d2 100644
--- a/media/libstagefright/FrameDecoder.cpp
+++ b/media/libstagefright/FrameDecoder.cpp
@@ -67,6 +67,12 @@
if (trackMeta->findInt32(kKeySARWidth, &sarWidth)
&& trackMeta->findInt32(kKeySARHeight, &sarHeight)
&& sarHeight != 0) {
+ int32_t multVal;
+ if (width < 0 || sarWidth < 0 ||
+ __builtin_mul_overflow(width, sarWidth, &multVal)) {
+ ALOGE("displayWidth overflow %dx%d", width, sarWidth);
+ return NULL;
+ }
displayWidth = (width * sarWidth) / sarHeight;
displayHeight = height;
} else if (trackMeta->findInt32(kKeyDisplayWidth, &displayWidth)
@@ -87,6 +93,16 @@
rotationAngle = 0;
}
+ if (!metaOnly) {
+ int32_t multVal;
+ if (width < 0 || height < 0 || dstBpp < 0 ||
+ __builtin_mul_overflow(dstBpp, width, &multVal) ||
+ __builtin_mul_overflow(multVal, height, &multVal)) {
+ ALOGE("Frame size overflow %dx%d bpp %d", width, height, dstBpp);
+ return NULL;
+ }
+ }
+
VideoFrame frame(width, height, displayWidth, displayHeight,
tileWidth, tileHeight, rotationAngle, dstBpp, !metaOnly, iccSize);