heif: fixes for image sequences and dual-function files
Adding support for two new sets of APIs on MediaMetadataRetriever:
- getImageAtIndex() and getPrimaryImage()
- getFrameAtIndex() and getFramesAtIndex()
Outline of changes:
- Proper indexing of all displayable still images, so that they
can be retrieved by getImageAtIndex()
- Exposing still images as "image/x.android.heic" tracks in
MediaExtractor with necessary descriptive keys (such as "grid-*")
- Support to retrieve video frames by absolute index instead
of timestamps, as image use cases mostly are interested in
getting the images and care less about timing.
- Support to retrieve video frames in batches because retrieving
one frame at a time is inefficient.
- Refactor image / frame decoding code into FrameDecoder, and split
still image decoding and video sequence decoding into to sub
classes to facilite future development.
bug: 63633199
test:
cts-tradefed run cts-dev --module CtsMediaTestCases --compatibility:module-arg CtsMediaTestCases:include-annotation:android.platform.test.annotations.RequiresDevice
Change-Id: I2fe8519fb6907f315a8b513921fc1cc7f436e28d
diff --git a/media/libmedia/IMediaMetadataRetriever.cpp b/media/libmedia/IMediaMetadataRetriever.cpp
index 5ea2e8b..f725c97 100644
--- a/media/libmedia/IMediaMetadataRetriever.cpp
+++ b/media/libmedia/IMediaMetadataRetriever.cpp
@@ -68,6 +68,8 @@
SET_DATA_SOURCE_FD,
SET_DATA_SOURCE_CALLBACK,
GET_FRAME_AT_TIME,
+ GET_IMAGE_AT_INDEX,
+ GET_FRAME_AT_INDEX,
EXTRACT_ALBUM_ART,
EXTRACT_METADATA,
};
@@ -164,6 +166,55 @@
return interface_cast<IMemory>(reply.readStrongBinder());
}
+ sp<IMemory> getImageAtIndex(int index, int colorFormat, bool metaOnly)
+ {
+ ALOGV("getImageAtIndex: index %d, colorFormat(%d) metaOnly(%d)",
+ index, colorFormat, metaOnly);
+ Parcel data, reply;
+ data.writeInterfaceToken(IMediaMetadataRetriever::getInterfaceDescriptor());
+ data.writeInt32(index);
+ data.writeInt32(colorFormat);
+ data.writeInt32(metaOnly);
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+ sendSchedPolicy(data);
+#endif
+ remote()->transact(GET_IMAGE_AT_INDEX, data, &reply);
+ status_t ret = reply.readInt32();
+ if (ret != NO_ERROR) {
+ return NULL;
+ }
+ return interface_cast<IMemory>(reply.readStrongBinder());
+ }
+
+ status_t getFrameAtIndex(std::vector<sp<IMemory> > *frames,
+ int frameIndex, int numFrames, int colorFormat, bool metaOnly)
+ {
+ ALOGV("getFrameAtIndex: frameIndex(%d), numFrames(%d), colorFormat(%d) metaOnly(%d)",
+ frameIndex, numFrames, colorFormat, metaOnly);
+ Parcel data, reply;
+ data.writeInterfaceToken(IMediaMetadataRetriever::getInterfaceDescriptor());
+ data.writeInt32(frameIndex);
+ data.writeInt32(numFrames);
+ data.writeInt32(colorFormat);
+ data.writeInt32(metaOnly);
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+ sendSchedPolicy(data);
+#endif
+ remote()->transact(GET_FRAME_AT_INDEX, data, &reply);
+ status_t ret = reply.readInt32();
+ if (ret != NO_ERROR) {
+ return ret;
+ }
+ int retNumFrames = reply.readInt32();
+ if (retNumFrames < numFrames) {
+ numFrames = retNumFrames;
+ }
+ for (int i = 0; i < numFrames; i++) {
+ frames->push_back(interface_cast<IMemory>(reply.readStrongBinder()));
+ }
+ return OK;
+ }
+
sp<IMemory> extractAlbumArt()
{
Parcel data, reply;
@@ -300,6 +351,54 @@
#endif
return NO_ERROR;
} break;
+ case GET_IMAGE_AT_INDEX: {
+ CHECK_INTERFACE(IMediaMetadataRetriever, data, reply);
+ int index = data.readInt32();
+ int colorFormat = data.readInt32();
+ bool metaOnly = (data.readInt32() != 0);
+ ALOGV("getImageAtIndex: index(%d), colorFormat(%d), metaOnly(%d)",
+ index, colorFormat, metaOnly);
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+ setSchedPolicy(data);
+#endif
+ sp<IMemory> bitmap = getImageAtIndex(index, colorFormat, metaOnly);
+ if (bitmap != 0) { // Don't send NULL across the binder interface
+ reply->writeInt32(NO_ERROR);
+ reply->writeStrongBinder(IInterface::asBinder(bitmap));
+ } else {
+ reply->writeInt32(UNKNOWN_ERROR);
+ }
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+ restoreSchedPolicy();
+#endif
+ return NO_ERROR;
+ } break;
+ case GET_FRAME_AT_INDEX: {
+ CHECK_INTERFACE(IMediaMetadataRetriever, data, reply);
+ int frameIndex = data.readInt32();
+ int numFrames = data.readInt32();
+ int colorFormat = data.readInt32();
+ bool metaOnly = (data.readInt32() != 0);
+ ALOGV("getFrameAtIndex: frameIndex(%d), numFrames(%d), colorFormat(%d), metaOnly(%d)",
+ frameIndex, numFrames, colorFormat, metaOnly);
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+ setSchedPolicy(data);
+#endif
+ std::vector<sp<IMemory> > frames;
+ status_t err = getFrameAtIndex(
+ &frames, frameIndex, numFrames, colorFormat, metaOnly);
+ reply->writeInt32(err);
+ if (OK == err) {
+ reply->writeInt32(frames.size());
+ for (size_t i = 0; i < frames.size(); i++) {
+ reply->writeStrongBinder(IInterface::asBinder(frames[i]));
+ }
+ }
+#ifndef DISABLE_GROUP_SCHEDULE_HACK
+ restoreSchedPolicy();
+#endif
+ return NO_ERROR;
+ } break;
case EXTRACT_ALBUM_ART: {
CHECK_INTERFACE(IMediaMetadataRetriever, data, reply);
#ifndef DISABLE_GROUP_SCHEDULE_HACK
diff --git a/media/libmedia/include/media/IMediaMetadataRetriever.h b/media/libmedia/include/media/IMediaMetadataRetriever.h
index ea95161..5491535 100644
--- a/media/libmedia/include/media/IMediaMetadataRetriever.h
+++ b/media/libmedia/include/media/IMediaMetadataRetriever.h
@@ -44,6 +44,11 @@
const sp<IDataSource>& dataSource, const char *mime) = 0;
virtual sp<IMemory> getFrameAtTime(
int64_t timeUs, int option, int colorFormat, bool metaOnly) = 0;
+ virtual sp<IMemory> getImageAtIndex(
+ int index, int colorFormat, bool metaOnly) = 0;
+ virtual status_t getFrameAtIndex(
+ std::vector<sp<IMemory> > *frames,
+ int frameIndex, int numFrames, int colorFormat, bool metaOnly) = 0;
virtual sp<IMemory> extractAlbumArt() = 0;
virtual const char* extractMetadata(int keyCode) = 0;
};
diff --git a/media/libmedia/include/media/IMediaSource.h b/media/libmedia/include/media/IMediaSource.h
index 1e36ab7..7aea90c 100644
--- a/media/libmedia/include/media/IMediaSource.h
+++ b/media/libmedia/include/media/IMediaSource.h
@@ -55,7 +55,6 @@
// Returns the format of the data output by this media source.
virtual sp<MetaData> getFormat() = 0;
-
// Returns a new buffer of data. Call blocks until a
// buffer is available, an error is encountered or the end of the stream
// is reached.
diff --git a/media/libmedia/include/media/MediaMetadataRetrieverInterface.h b/media/libmedia/include/media/MediaMetadataRetrieverInterface.h
index 257002d..fc9e53c 100644
--- a/media/libmedia/include/media/MediaMetadataRetrieverInterface.h
+++ b/media/libmedia/include/media/MediaMetadataRetrieverInterface.h
@@ -22,6 +22,7 @@
#include <media/mediametadataretriever.h>
#include <media/mediascanner.h>
#include <private/media/VideoFrame.h>
+#include <media/stagefright/MediaErrors.h>
namespace android {
@@ -41,9 +42,14 @@
const KeyedVector<String8, String8> *headers = NULL) = 0;
virtual status_t setDataSource(int fd, int64_t offset, int64_t length) = 0;
- virtual status_t setDataSource(const sp<DataSource>& source, const char *mime) = 0;
+ virtual status_t setDataSource(const sp<DataSource>& source, const char *mime) = 0;
virtual VideoFrame* getFrameAtTime(
int64_t timeUs, int option, int colorFormat, bool metaOnly) = 0;
+ virtual VideoFrame* getImageAtIndex(
+ int index, int colorFormat, bool metaOnly) = 0;
+ virtual status_t getFrameAtIndex(
+ std::vector<VideoFrame*>* frames,
+ int frameIndex, int numFrames, int colorFormat, bool metaOnly);
virtual MediaAlbumArt* extractAlbumArt() = 0;
virtual const char* extractMetadata(int keyCode) = 0;
};
@@ -58,6 +64,13 @@
virtual VideoFrame* getFrameAtTime(
int64_t /*timeUs*/, int /*option*/, int /*colorFormat*/, bool /*metaOnly*/)
{ return NULL; }
+ virtual VideoFrame* getImageAtIndex(
+ int /*index*/, int /*colorFormat*/, bool /*metaOnly*/)
+ { return NULL; }
+ virtual status_t getFrameAtIndex(
+ std::vector<VideoFrame*>* /*frames*/,
+ int /*frameIndex*/, int /*numFrames*/, int /*colorFormat*/, bool /*metaOnly*/)
+ { return ERROR_UNSUPPORTED; }
virtual MediaAlbumArt* extractAlbumArt() { return NULL; }
virtual const char* extractMetadata(int /*keyCode*/) { return NULL; }
};
diff --git a/media/libmedia/include/media/mediametadataretriever.h b/media/libmedia/include/media/mediametadataretriever.h
index 65c266b..3511253 100644
--- a/media/libmedia/include/media/mediametadataretriever.h
+++ b/media/libmedia/include/media/mediametadataretriever.h
@@ -59,6 +59,13 @@
METADATA_KEY_LOCATION = 23,
METADATA_KEY_VIDEO_ROTATION = 24,
METADATA_KEY_CAPTURE_FRAMERATE = 25,
+ METADATA_KEY_HAS_IMAGE = 26,
+ METADATA_KEY_IMAGE_COUNT = 27,
+ METADATA_KEY_IMAGE_PRIMARY = 28,
+ METADATA_KEY_IMAGE_WIDTH = 29,
+ METADATA_KEY_IMAGE_HEIGHT = 30,
+ METADATA_KEY_IMAGE_ROTATION = 31,
+ METADATA_KEY_VIDEO_FRAME_COUNT = 32,
// Add more here...
};
@@ -80,6 +87,11 @@
const sp<IDataSource>& dataSource, const char *mime = NULL);
sp<IMemory> getFrameAtTime(int64_t timeUs, int option,
int colorFormat = HAL_PIXEL_FORMAT_RGB_565, bool metaOnly = false);
+ sp<IMemory> getImageAtIndex(int index,
+ int colorFormat = HAL_PIXEL_FORMAT_RGB_565, bool metaOnly = false);
+ status_t getFrameAtIndex(
+ std::vector<sp<IMemory> > *frames, int frameIndex, int numFrames = 1,
+ int colorFormat = HAL_PIXEL_FORMAT_RGB_565, bool metaOnly = false);
sp<IMemory> extractAlbumArt();
const char* extractMetadata(int keyCode);
diff --git a/media/libmedia/mediametadataretriever.cpp b/media/libmedia/mediametadataretriever.cpp
index 7d27d57..6a4204b 100644
--- a/media/libmedia/mediametadataretriever.cpp
+++ b/media/libmedia/mediametadataretriever.cpp
@@ -154,6 +154,32 @@
return mRetriever->getFrameAtTime(timeUs, option, colorFormat, metaOnly);
}
+sp<IMemory> MediaMetadataRetriever::getImageAtIndex(
+ int index, int colorFormat, bool metaOnly) {
+ ALOGV("getImageAtIndex: index(%d) colorFormat(%d) metaOnly(%d)",
+ index, colorFormat, metaOnly);
+ Mutex::Autolock _l(mLock);
+ if (mRetriever == 0) {
+ ALOGE("retriever is not initialized");
+ return NULL;
+ }
+ return mRetriever->getImageAtIndex(index, colorFormat, metaOnly);
+}
+
+status_t MediaMetadataRetriever::getFrameAtIndex(
+ std::vector<sp<IMemory> > *frames,
+ int frameIndex, int numFrames, int colorFormat, bool metaOnly) {
+ ALOGV("getFrameAtIndex: frameIndex(%d), numFrames(%d), colorFormat(%d) metaOnly(%d)",
+ frameIndex, numFrames, colorFormat, metaOnly);
+ Mutex::Autolock _l(mLock);
+ if (mRetriever == 0) {
+ ALOGE("retriever is not initialized");
+ return INVALID_OPERATION;
+ }
+ return mRetriever->getFrameAtIndex(
+ frames, frameIndex, numFrames, colorFormat, metaOnly);
+}
+
const char* MediaMetadataRetriever::extractMetadata(int keyCode)
{
ALOGV("extractMetadata(%d)", keyCode);