heif: fixes for image sequences and dual-function files

Adding support for two new sets of APIs on MediaMetadataRetriever:

- getImageAtIndex() and getPrimaryImage()
- getFrameAtIndex() and getFramesAtIndex()

Outline of changes:

- Proper indexing of all displayable still images, so that they
  can be retrieved by getImageAtIndex()

- Exposing still images as "image/x.android.heic" tracks in
  MediaExtractor with necessary descriptive keys (such as "grid-*")

- Support to retrieve video frames by absolute index instead
  of timestamps, as image use cases mostly are interested in
  getting the images and care less about timing.

- Support to retrieve video frames in batches because retrieving
  one frame at a time is inefficient.

- Refactor image / frame decoding code into FrameDecoder, and split
  still image decoding and video sequence decoding into to sub
  classes to facilite future development.

bug: 63633199

test:
cts-tradefed run cts-dev --module CtsMediaTestCases --compatibility:module-arg CtsMediaTestCases:include-annotation:android.platform.test.annotations.RequiresDevice

Change-Id: I2fe8519fb6907f315a8b513921fc1cc7f436e28d
diff --git a/media/libmedia/IMediaMetadataRetriever.cpp b/media/libmedia/IMediaMetadataRetriever.cpp
index 5ea2e8b..f725c97 100644
--- a/media/libmedia/IMediaMetadataRetriever.cpp
+++ b/media/libmedia/IMediaMetadataRetriever.cpp
@@ -68,6 +68,8 @@
     SET_DATA_SOURCE_FD,
     SET_DATA_SOURCE_CALLBACK,
     GET_FRAME_AT_TIME,
+    GET_IMAGE_AT_INDEX,
+    GET_FRAME_AT_INDEX,
     EXTRACT_ALBUM_ART,
     EXTRACT_METADATA,
 };
@@ -164,6 +166,55 @@
         return interface_cast<IMemory>(reply.readStrongBinder());
     }
 
+    sp<IMemory> getImageAtIndex(int index, int colorFormat, bool metaOnly)
+    {
+        ALOGV("getImageAtIndex: index %d, colorFormat(%d) metaOnly(%d)",
+                index, colorFormat, metaOnly);
+        Parcel data, reply;
+        data.writeInterfaceToken(IMediaMetadataRetriever::getInterfaceDescriptor());
+        data.writeInt32(index);
+        data.writeInt32(colorFormat);
+        data.writeInt32(metaOnly);
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+        sendSchedPolicy(data);
+#endif
+        remote()->transact(GET_IMAGE_AT_INDEX, data, &reply);
+        status_t ret = reply.readInt32();
+        if (ret != NO_ERROR) {
+            return NULL;
+        }
+        return interface_cast<IMemory>(reply.readStrongBinder());
+    }
+
+    status_t getFrameAtIndex(std::vector<sp<IMemory> > *frames,
+            int frameIndex, int numFrames, int colorFormat, bool metaOnly)
+    {
+        ALOGV("getFrameAtIndex: frameIndex(%d), numFrames(%d), colorFormat(%d) metaOnly(%d)",
+                frameIndex, numFrames, colorFormat, metaOnly);
+        Parcel data, reply;
+        data.writeInterfaceToken(IMediaMetadataRetriever::getInterfaceDescriptor());
+        data.writeInt32(frameIndex);
+        data.writeInt32(numFrames);
+        data.writeInt32(colorFormat);
+        data.writeInt32(metaOnly);
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+        sendSchedPolicy(data);
+#endif
+        remote()->transact(GET_FRAME_AT_INDEX, data, &reply);
+        status_t ret = reply.readInt32();
+        if (ret != NO_ERROR) {
+            return ret;
+        }
+        int retNumFrames = reply.readInt32();
+        if (retNumFrames < numFrames) {
+            numFrames = retNumFrames;
+        }
+        for (int i = 0; i < numFrames; i++) {
+            frames->push_back(interface_cast<IMemory>(reply.readStrongBinder()));
+        }
+        return OK;
+    }
+
     sp<IMemory> extractAlbumArt()
     {
         Parcel data, reply;
@@ -300,6 +351,54 @@
 #endif
             return NO_ERROR;
         } break;
+        case GET_IMAGE_AT_INDEX: {
+            CHECK_INTERFACE(IMediaMetadataRetriever, data, reply);
+            int index = data.readInt32();
+            int colorFormat = data.readInt32();
+            bool metaOnly = (data.readInt32() != 0);
+            ALOGV("getImageAtIndex: index(%d), colorFormat(%d), metaOnly(%d)",
+                    index, colorFormat, metaOnly);
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+            setSchedPolicy(data);
+#endif
+            sp<IMemory> bitmap = getImageAtIndex(index, colorFormat, metaOnly);
+            if (bitmap != 0) {  // Don't send NULL across the binder interface
+                reply->writeInt32(NO_ERROR);
+                reply->writeStrongBinder(IInterface::asBinder(bitmap));
+            } else {
+                reply->writeInt32(UNKNOWN_ERROR);
+            }
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+            restoreSchedPolicy();
+#endif
+            return NO_ERROR;
+        } break;
+        case GET_FRAME_AT_INDEX: {
+            CHECK_INTERFACE(IMediaMetadataRetriever, data, reply);
+            int frameIndex = data.readInt32();
+            int numFrames = data.readInt32();
+            int colorFormat = data.readInt32();
+            bool metaOnly = (data.readInt32() != 0);
+            ALOGV("getFrameAtIndex: frameIndex(%d), numFrames(%d), colorFormat(%d), metaOnly(%d)",
+                    frameIndex, numFrames, colorFormat, metaOnly);
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+            setSchedPolicy(data);
+#endif
+            std::vector<sp<IMemory> > frames;
+            status_t err = getFrameAtIndex(
+                    &frames, frameIndex, numFrames, colorFormat, metaOnly);
+            reply->writeInt32(err);
+            if (OK == err) {
+                reply->writeInt32(frames.size());
+                for (size_t i = 0; i < frames.size(); i++) {
+                    reply->writeStrongBinder(IInterface::asBinder(frames[i]));
+                }
+            }
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+            restoreSchedPolicy();
+#endif
+            return NO_ERROR;
+        } break;
         case EXTRACT_ALBUM_ART: {
             CHECK_INTERFACE(IMediaMetadataRetriever, data, reply);
 #ifndef DISABLE_GROUP_SCHEDULE_HACK
diff --git a/media/libmedia/include/media/IMediaMetadataRetriever.h b/media/libmedia/include/media/IMediaMetadataRetriever.h
index ea95161..5491535 100644
--- a/media/libmedia/include/media/IMediaMetadataRetriever.h
+++ b/media/libmedia/include/media/IMediaMetadataRetriever.h
@@ -44,6 +44,11 @@
             const sp<IDataSource>& dataSource, const char *mime) = 0;
     virtual sp<IMemory>     getFrameAtTime(
             int64_t timeUs, int option, int colorFormat, bool metaOnly) = 0;
+    virtual sp<IMemory>     getImageAtIndex(
+            int index, int colorFormat, bool metaOnly) = 0;
+    virtual status_t        getFrameAtIndex(
+            std::vector<sp<IMemory> > *frames,
+            int frameIndex, int numFrames, int colorFormat, bool metaOnly) = 0;
     virtual sp<IMemory>     extractAlbumArt() = 0;
     virtual const char*     extractMetadata(int keyCode) = 0;
 };
diff --git a/media/libmedia/include/media/IMediaSource.h b/media/libmedia/include/media/IMediaSource.h
index 1e36ab7..7aea90c 100644
--- a/media/libmedia/include/media/IMediaSource.h
+++ b/media/libmedia/include/media/IMediaSource.h
@@ -55,7 +55,6 @@
     // Returns the format of the data output by this media source.
     virtual sp<MetaData> getFormat() = 0;
 
-
     // Returns a new buffer of data. Call blocks until a
     // buffer is available, an error is encountered or the end of the stream
     // is reached.
diff --git a/media/libmedia/include/media/MediaMetadataRetrieverInterface.h b/media/libmedia/include/media/MediaMetadataRetrieverInterface.h
index 257002d..fc9e53c 100644
--- a/media/libmedia/include/media/MediaMetadataRetrieverInterface.h
+++ b/media/libmedia/include/media/MediaMetadataRetrieverInterface.h
@@ -22,6 +22,7 @@
 #include <media/mediametadataretriever.h>
 #include <media/mediascanner.h>
 #include <private/media/VideoFrame.h>
+#include <media/stagefright/MediaErrors.h>
 
 namespace android {
 
@@ -41,9 +42,14 @@
             const KeyedVector<String8, String8> *headers = NULL) = 0;
 
     virtual status_t    setDataSource(int fd, int64_t offset, int64_t length) = 0;
-    virtual status_t setDataSource(const sp<DataSource>& source, const char *mime) = 0;
+    virtual status_t    setDataSource(const sp<DataSource>& source, const char *mime) = 0;
     virtual VideoFrame* getFrameAtTime(
             int64_t timeUs, int option, int colorFormat, bool metaOnly) = 0;
+    virtual VideoFrame* getImageAtIndex(
+            int index, int colorFormat, bool metaOnly) = 0;
+    virtual status_t getFrameAtIndex(
+            std::vector<VideoFrame*>* frames,
+            int frameIndex, int numFrames, int colorFormat, bool metaOnly);
     virtual MediaAlbumArt* extractAlbumArt() = 0;
     virtual const char* extractMetadata(int keyCode) = 0;
 };
@@ -58,6 +64,13 @@
     virtual VideoFrame* getFrameAtTime(
             int64_t /*timeUs*/, int /*option*/, int /*colorFormat*/, bool /*metaOnly*/)
     { return NULL; }
+    virtual VideoFrame* getImageAtIndex(
+            int /*index*/, int /*colorFormat*/, bool /*metaOnly*/)
+    { return NULL; }
+    virtual status_t getFrameAtIndex(
+            std::vector<VideoFrame*>* /*frames*/,
+            int /*frameIndex*/, int /*numFrames*/, int /*colorFormat*/, bool /*metaOnly*/)
+    { return ERROR_UNSUPPORTED; }
     virtual MediaAlbumArt* extractAlbumArt() { return NULL; }
     virtual const char* extractMetadata(int /*keyCode*/) { return NULL; }
 };
diff --git a/media/libmedia/include/media/mediametadataretriever.h b/media/libmedia/include/media/mediametadataretriever.h
index 65c266b..3511253 100644
--- a/media/libmedia/include/media/mediametadataretriever.h
+++ b/media/libmedia/include/media/mediametadataretriever.h
@@ -59,6 +59,13 @@
     METADATA_KEY_LOCATION        = 23,
     METADATA_KEY_VIDEO_ROTATION  = 24,
     METADATA_KEY_CAPTURE_FRAMERATE = 25,
+    METADATA_KEY_HAS_IMAGE       = 26,
+    METADATA_KEY_IMAGE_COUNT     = 27,
+    METADATA_KEY_IMAGE_PRIMARY   = 28,
+    METADATA_KEY_IMAGE_WIDTH     = 29,
+    METADATA_KEY_IMAGE_HEIGHT    = 30,
+    METADATA_KEY_IMAGE_ROTATION  = 31,
+    METADATA_KEY_VIDEO_FRAME_COUNT  = 32,
 
     // Add more here...
 };
@@ -80,6 +87,11 @@
             const sp<IDataSource>& dataSource, const char *mime = NULL);
     sp<IMemory> getFrameAtTime(int64_t timeUs, int option,
             int colorFormat = HAL_PIXEL_FORMAT_RGB_565, bool metaOnly = false);
+    sp<IMemory> getImageAtIndex(int index,
+            int colorFormat = HAL_PIXEL_FORMAT_RGB_565, bool metaOnly = false);
+    status_t getFrameAtIndex(
+            std::vector<sp<IMemory> > *frames, int frameIndex, int numFrames = 1,
+            int colorFormat = HAL_PIXEL_FORMAT_RGB_565, bool metaOnly = false);
     sp<IMemory> extractAlbumArt();
     const char* extractMetadata(int keyCode);
 
diff --git a/media/libmedia/mediametadataretriever.cpp b/media/libmedia/mediametadataretriever.cpp
index 7d27d57..6a4204b 100644
--- a/media/libmedia/mediametadataretriever.cpp
+++ b/media/libmedia/mediametadataretriever.cpp
@@ -154,6 +154,32 @@
     return mRetriever->getFrameAtTime(timeUs, option, colorFormat, metaOnly);
 }
 
+sp<IMemory> MediaMetadataRetriever::getImageAtIndex(
+        int index, int colorFormat, bool metaOnly) {
+    ALOGV("getImageAtIndex: index(%d) colorFormat(%d) metaOnly(%d)",
+            index, colorFormat, metaOnly);
+    Mutex::Autolock _l(mLock);
+    if (mRetriever == 0) {
+        ALOGE("retriever is not initialized");
+        return NULL;
+    }
+    return mRetriever->getImageAtIndex(index, colorFormat, metaOnly);
+}
+
+status_t MediaMetadataRetriever::getFrameAtIndex(
+        std::vector<sp<IMemory> > *frames,
+        int frameIndex, int numFrames, int colorFormat, bool metaOnly) {
+    ALOGV("getFrameAtIndex: frameIndex(%d), numFrames(%d), colorFormat(%d) metaOnly(%d)",
+            frameIndex, numFrames, colorFormat, metaOnly);
+    Mutex::Autolock _l(mLock);
+    if (mRetriever == 0) {
+        ALOGE("retriever is not initialized");
+        return INVALID_OPERATION;
+    }
+    return mRetriever->getFrameAtIndex(
+            frames, frameIndex, numFrames, colorFormat, metaOnly);
+}
+
 const char* MediaMetadataRetriever::extractMetadata(int keyCode)
 {
     ALOGV("extractMetadata(%d)", keyCode);