h264bsdActivateParamSets: Prevent multiplication overflow. am: 87277aac64 am: 2a68d5279d am: 5dfa5f268e am: a0bb560e3b am: 7c22e59819 am: 548439e243 am: 4d027a6bf5 am: 177aee1b6b am: e9bb4f9b51
am: 60ab0df366
* commit '60ab0df3666877a6d34acc74a07f38ec13927a1c':
h264bsdActivateParamSets: Prevent multiplication overflow.
Change-Id: I714f77542c419fbc58533d3c49e20f593d836607
diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c
index 3234754..ff7a42a 100644
--- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c
+++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c
@@ -58,6 +58,10 @@
3. Module defines
------------------------------------------------------------------------------*/
+#ifndef UINT32_MAX
+#define UINT32_MAX (4294967295U)
+#endif
+
/*------------------------------------------------------------------------------
4. Local function prototypes
------------------------------------------------------------------------------*/
@@ -326,9 +330,23 @@
pStorage->activePps = pStorage->pps[ppsId];
pStorage->activeSpsId = pStorage->activePps->seqParameterSetId;
pStorage->activeSps = pStorage->sps[pStorage->activeSpsId];
- pStorage->picSizeInMbs =
- pStorage->activeSps->picWidthInMbs *
- pStorage->activeSps->picHeightInMbs;
+
+ /* report error before multiplication to prevent integer overflow */
+ if (pStorage->activeSps->picWidthInMbs == 0)
+ {
+ pStorage->picSizeInMbs = 0;
+ }
+ else if (pStorage->activeSps->picHeightInMbs >
+ UINT32_MAX / pStorage->activeSps->picWidthInMbs)
+ {
+ return(MEMORY_ALLOCATION_ERROR);
+ }
+ else
+ {
+ pStorage->picSizeInMbs =
+ pStorage->activeSps->picWidthInMbs *
+ pStorage->activeSps->picHeightInMbs;
+ }
pStorage->currImage->width = pStorage->activeSps->picWidthInMbs;
pStorage->currImage->height = pStorage->activeSps->picHeightInMbs;