commit 91af834b1326ceff4a96724fd77696842dde432c
author Ray Essick <essick@google.com> Fri May 12 18:02:01 2017 +0000
committer android-build-merger <android-build-merger@google.com> Fri May 12 18:02:01 2017 +0000
tree 52e870d01d58dbf67e79b05eac2c9c4076bdadb7
parent e6678c5af9b491b8d37d8a211feb63ef9f267730
parent 9eb9b1b849379bbdcc5a3c648ef66bda2e5b621d

Prevent OOB write in soft_avc encoder am: 463452a949 am: 97db64baf6 am: b6bf46944f am: cd35c846f0 am: a43be7d42d am: 3c375611d6 am: 96b4e70f44
am: 9eb9b1b849

Change-Id: I603637d9ca99cea3aa899a3883ebe21d928ed3c3

media/libstagefright/codecs/avcenc/SoftAVCEnc.cpp

diff --git a/media/libstagefright/codecs/avcenc/SoftAVCEnc.cpp b/media/libstagefright/codecs/avcenc/SoftAVCEnc.cpp
index 9e7a3be..5b06722 100644
--- a/media/libstagefright/codecs/avcenc/SoftAVCEnc.cpp
+++ b/media/libstagefright/codecs/avcenc/SoftAVCEnc.cpp
@@ -614,6 +614,7 @@
     IV_STATUS_T status;
     WORD32 level;
     uint32_t displaySizeY;
+
     CHECK(!mStarted);
 
     OMX_ERRORTYPE errType = OMX_ErrorNone;
@@ -917,6 +918,9 @@
         }
     }
 
+    // clear other pointers into the space being free()d
+    mCodecCtx = NULL;
+
     mStarted = false;
 
     return OMX_ErrorNone;
@@ -1509,6 +1513,14 @@
     return;
 }
 
+void SoftAVC::onReset() {
+    SoftVideoEncoderOMXComponent::onReset();
+
+    if (releaseEncoder() != OMX_ErrorNone) {
+        ALOGW("releaseEncoder failed");
+    }
+}
+
 }  // namespace android
 
 android::SoftOMXComponent *createSoftOMXComponent(

media/libstagefright/codecs/avcenc/SoftAVCEnc.h

diff --git a/media/libstagefright/codecs/avcenc/SoftAVCEnc.h b/media/libstagefright/codecs/avcenc/SoftAVCEnc.h
index cf6f899..8b24b62 100644
--- a/media/libstagefright/codecs/avcenc/SoftAVCEnc.h
+++ b/media/libstagefright/codecs/avcenc/SoftAVCEnc.h
@@ -136,6 +136,8 @@
 protected:
     virtual ~SoftAVC();
 
+    virtual void onReset();
+
 private:
     enum {
         kNumBuffers = 2,