Support video render metrics for OMX
Currently, no devices on Android 14 use OMX codecs, however it is
expected that TV devices using OMX will upgrade to Android 14
eventually. Add support for video render metrics for these device
upgrades so that we have universal video render metrics on all Android
14 devices.
Bug: 286919416
Bug: 234833109
Test: atest DecoderRenderTest on ADT-3 on Android 12
Test: Play youtube videos on ADT-3 on Android 12
Change-Id: Ia22d976bc2ba423fd8cd71110b64b4715326fe55
diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp
index 86fd8ab..a75ce70 100644
--- a/media/codec2/sfplugin/CCodec.cpp
+++ b/media/codec2/sfplugin/CCodec.cpp
@@ -46,6 +46,7 @@
#include <media/stagefright/BufferProducerWrapper.h>
#include <media/stagefright/MediaCodecConstants.h>
#include <media/stagefright/PersistentSurface.h>
+#include <media/stagefright/RenderedFrameInfo.h>
#include <utils/NativeHandle.h>
#include "C2OMXNode.h"
@@ -672,8 +673,7 @@
}
void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) override {
- mCodec->mCallback->onOutputFramesRendered(
- {RenderedFrameInfo(mediaTimeUs, renderTimeNs)});
+ mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, renderTimeNs)});
}
void onOutputBuffersChanged() override {
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index 505775b..2bc8c96 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -43,6 +43,7 @@
#include <media/stagefright/MediaDefs.h>
#include <media/stagefright/OMXClient.h>
#include <media/stagefright/PersistentSurface.h>
+#include <media/stagefright/RenderedFrameInfo.h>
#include <media/stagefright/SurfaceUtils.h>
#include <media/hardware/HardwareAPI.h>
#include <media/MediaBufferHolder.h>
@@ -632,7 +633,8 @@
if (!mBufferChannel) {
mBufferChannel = std::make_shared<ACodecBufferChannel>(
new AMessage(kWhatInputBufferFilled, this),
- new AMessage(kWhatOutputBufferDrained, this));
+ new AMessage(kWhatOutputBufferDrained, this),
+ new AMessage(kWhatPollForRenderedBuffers, this));
}
return mBufferChannel;
}
@@ -742,6 +744,7 @@
// if we have not yet started the codec, we can simply set the native window
if (mBuffers[kPortIndexInput].size() == 0) {
mNativeWindow = surface;
+ initializeFrameTracking();
return OK;
}
@@ -850,6 +853,7 @@
mNativeWindow = nativeWindow;
mNativeWindowUsageBits = usageBits;
+ initializeFrameTracking();
return OK;
}
@@ -960,7 +964,6 @@
BufferInfo info;
info.mStatus = BufferInfo::OWNED_BY_US;
info.mFenceFd = -1;
- info.mRenderInfo = NULL;
info.mGraphicBuffer = NULL;
info.mNewGraphicBuffer = false;
@@ -1228,6 +1231,7 @@
*bufferCount = def.nBufferCountActual;
*bufferSize = def.nBufferSize;
+ initializeFrameTracking();
return err;
}
@@ -1266,7 +1270,6 @@
info.mStatus = BufferInfo::OWNED_BY_US;
info.mFenceFd = fenceFd;
info.mIsReadFence = false;
- info.mRenderInfo = NULL;
info.mGraphicBuffer = graphicBuffer;
info.mNewGraphicBuffer = false;
info.mDequeuedAt = mDequeueCounter;
@@ -1343,7 +1346,6 @@
BufferInfo info;
info.mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;
info.mFenceFd = -1;
- info.mRenderInfo = NULL;
info.mGraphicBuffer = NULL;
info.mNewGraphicBuffer = false;
info.mDequeuedAt = mDequeueCounter;
@@ -1439,42 +1441,6 @@
return err;
}
-void ACodec::updateRenderInfoForDequeuedBuffer(
- ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info) {
-
- info->mRenderInfo =
- mRenderTracker.updateInfoForDequeuedBuffer(
- buf, fenceFd, info - &mBuffers[kPortIndexOutput][0]);
-
- // check for any fences already signaled
- notifyOfRenderedFrames(false /* dropIncomplete */, info->mRenderInfo);
-}
-
-void ACodec::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
- if (mRenderTracker.onFrameRendered(mediaTimeUs, systemNano) != OK) {
- mRenderTracker.dumpRenderQueue();
- }
-}
-
-void ACodec::notifyOfRenderedFrames(bool dropIncomplete, FrameRenderTracker::Info *until) {
- std::list<FrameRenderTracker::Info> done =
- mRenderTracker.checkFencesAndGetRenderedFrames(until, dropIncomplete);
-
- // unlink untracked frames
- for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
- it != done.cend(); ++it) {
- ssize_t index = it->getIndex();
- if (index >= 0 && (size_t)index < mBuffers[kPortIndexOutput].size()) {
- mBuffers[kPortIndexOutput][index].mRenderInfo = NULL;
- } else if (index >= 0) {
- // THIS SHOULD NEVER HAPPEN
- ALOGE("invalid index %zd in %zu", index, mBuffers[kPortIndexOutput].size());
- }
- }
-
- mCallback->onOutputFramesRendered(done);
-}
-
void ACodec::onFirstTunnelFrameReady() {
mCallback->onFirstTunnelFrameReady();
}
@@ -1529,7 +1495,6 @@
info->mStatus = BufferInfo::OWNED_BY_US;
info->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow");
- updateRenderInfoForDequeuedBuffer(buf, fenceFd, info);
return info;
}
}
@@ -1574,18 +1539,96 @@
oldest->mNewGraphicBuffer = true;
oldest->mStatus = BufferInfo::OWNED_BY_US;
oldest->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow for oldest");
- mRenderTracker.untrackFrame(oldest->mRenderInfo);
- oldest->mRenderInfo = NULL;
ALOGV("replaced oldest buffer #%u with age %u, graphicBuffer %p",
(unsigned)(oldest - &mBuffers[kPortIndexOutput][0]),
mDequeueCounter - oldest->mDequeuedAt,
oldest->mGraphicBuffer->handle);
-
- updateRenderInfoForDequeuedBuffer(buf, fenceFd, oldest);
return oldest;
}
+void ACodec::initializeFrameTracking() {
+ mTrackedFrames.clear();
+
+ int isWindowToDisplay = 0;
+ mNativeWindow->query(mNativeWindow.get(), NATIVE_WINDOW_QUEUES_TO_WINDOW_COMPOSER,
+ &isWindowToDisplay);
+ mIsWindowToDisplay = isWindowToDisplay == 1;
+ // No frame tracking is needed if we're not sending frames to the display
+ if (!mIsWindowToDisplay) {
+ // Return early so we don't call into SurfaceFlinger (requiring permissions)
+ return;
+ }
+
+ int hasPresentFenceTimes = 0;
+ mNativeWindow->query(mNativeWindow.get(), NATIVE_WINDOW_FRAME_TIMESTAMPS_SUPPORTS_PRESENT,
+ &hasPresentFenceTimes);
+ mHasPresentFenceTimes = hasPresentFenceTimes == 1;
+ if (!mHasPresentFenceTimes) {
+ ALOGI("Using latch times for frame rendered signals - present fences not supported");
+ }
+
+ status_t err = native_window_enable_frame_timestamps(mNativeWindow.get(), true);
+ if (err) {
+ ALOGE("Failed to enable frame timestamps (%d)", err);
+ }
+}
+
+void ACodec::trackReleasedFrame(int64_t frameId, int64_t mediaTimeUs, int64_t desiredRenderTimeNs) {
+ // If the render time is earlier than now, then we're suggesting it should be rendered ASAP,
+ // so track the frame as if the desired render time is now.
+ int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
+ if (desiredRenderTimeNs < nowNs) {
+ desiredRenderTimeNs = nowNs;
+ }
+ // We've just queued a frame to the surface, so keep track of it and later check to see if it is
+ // actually rendered.
+ TrackedFrame frame;
+ frame.id = frameId;
+ frame.mediaTimeUs = mediaTimeUs;
+ frame.desiredRenderTimeNs = desiredRenderTimeNs;
+ mTrackedFrames.push_back(frame);
+}
+
+void ACodec::pollForRenderedFrames() {
+ std::list<RenderedFrameInfo> renderedFrameInfos;
+ // Scan all frames and check to see if the frames that SHOULD have been rendered by now, have,
+ // in fact, been rendered.
+ int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
+ while (!mTrackedFrames.empty()) {
+ TrackedFrame & frame = mTrackedFrames.front();
+ // Frames that should have been rendered at least 100ms in the past are checked
+ if (frame.desiredRenderTimeNs > nowNs - 100*1000*1000LL) {
+ break;
+ }
+
+ status_t err;
+ nsecs_t latchOrPresentTimeNs = NATIVE_WINDOW_TIMESTAMP_INVALID;
+ err = native_window_get_frame_timestamps(mNativeWindow.get(), frame.id,
+ /* outRequestedPresentTime */ nullptr, /* outAcquireTime */ nullptr,
+ mHasPresentFenceTimes ? nullptr : &latchOrPresentTimeNs, // latch time
+ /* outFirstRefreshStartTime */ nullptr, /* outLastRefreshStartTime */ nullptr,
+ /* outGpuCompositionDoneTime */ nullptr,
+ mHasPresentFenceTimes ? &latchOrPresentTimeNs : nullptr, // display present time,
+ /* outDequeueReadyTime */ nullptr, /* outReleaseTime */ nullptr);
+ if (err) {
+ ALOGE("Failed to get frame timestamps for %lld: %d", (long long) frame.id, err);
+ }
+ // If we don't have a render time by now, then consider the frame as dropped
+ if (latchOrPresentTimeNs != NATIVE_WINDOW_TIMESTAMP_PENDING &&
+ latchOrPresentTimeNs != NATIVE_WINDOW_TIMESTAMP_INVALID) {
+ renderedFrameInfos.push_back(RenderedFrameInfo(frame.mediaTimeUs,
+ latchOrPresentTimeNs));
+ }
+
+ mTrackedFrames.pop_front();
+ }
+
+ if (!renderedFrameInfos.empty()) {
+ mCallback->onOutputFramesRendered(renderedFrameInfos);
+ }
+}
+
status_t ACodec::freeBuffersOnPort(OMX_U32 portIndex) {
if (portIndex == kPortIndexInput) {
mBufferChannel->setInputBufferArray({});
@@ -1661,11 +1704,6 @@
::close(info->mFenceFd);
}
- if (portIndex == kPortIndexOutput) {
- mRenderTracker.untrackFrame(info->mRenderInfo, i);
- info->mRenderInfo = NULL;
- }
-
// remove buffer even if mOMXNode->freeBuffer fails
mBuffers[portIndex].erase(mBuffers[portIndex].begin() + i);
return err;
@@ -6030,22 +6068,10 @@
sp<RefBase> obj;
CHECK(msg->findObject("messages", &obj));
sp<MessageList> msgList = static_cast<MessageList *>(obj.get());
-
- bool receivedRenderedEvents = false;
for (std::list<sp<AMessage>>::const_iterator it = msgList->getList().cbegin();
it != msgList->getList().cend(); ++it) {
(*it)->setWhat(ACodec::kWhatOMXMessageItem);
mCodec->handleMessage(*it);
- int32_t type;
- CHECK((*it)->findInt32("type", &type));
- if (type == omx_message::FRAME_RENDERED) {
- receivedRenderedEvents = true;
- }
- }
-
- if (receivedRenderedEvents) {
- // NOTE: all buffers are rendered in this case
- mCodec->notifyOfRenderedFrames();
}
return true;
}
@@ -6554,15 +6580,6 @@
info->mDequeuedAt = ++mCodec->mDequeueCounter;
info->mStatus = BufferInfo::OWNED_BY_US;
- if (info->mRenderInfo != NULL) {
- // The fence for an emptied buffer must have signaled, but there still could be queued
- // or out-of-order dequeued buffers in the render queue prior to this buffer. Drop these,
- // as we will soon requeue this buffer to the surface. While in theory we could still keep
- // track of buffers that are requeued to the surface, it is better to add support to the
- // buffer-queue to notify us of released buffers and their fences (in the future).
- mCodec->notifyOfRenderedFrames(true /* dropIncomplete */);
- }
-
// byte buffers cannot take fences, so wait for any fence now
if (mCodec->mNativeWindow == NULL) {
(void)mCodec->waitForFence(fenceFd, "onOMXFillBufferDone");
@@ -6769,14 +6786,6 @@
mCodec->mLastHdr10PlusBuffer = hdr10PlusInfo;
}
- // save buffers sent to the surface so we can get render time when they return
- int64_t mediaTimeUs = -1;
- buffer->meta()->findInt64("timeUs", &mediaTimeUs);
- if (mediaTimeUs >= 0) {
- mCodec->mRenderTracker.onFrameQueued(
- mediaTimeUs, info->mGraphicBuffer, new Fence(::dup(info->mFenceFd)));
- }
-
int64_t timestampNs = 0;
if (!msg->findInt64("timestampNs", ×tampNs)) {
// use media timestamp if client did not request a specific render timestamp
@@ -6790,11 +6799,25 @@
err = native_window_set_buffers_timestamp(mCodec->mNativeWindow.get(), timestampNs);
ALOGW_IF(err != NO_ERROR, "failed to set buffer timestamp: %d", err);
+ uint64_t frameId;
+ err = native_window_get_next_frame_id(mCodec->mNativeWindow.get(), &frameId);
+
info->checkReadFence("onOutputBufferDrained before queueBuffer");
err = mCodec->mNativeWindow->queueBuffer(
mCodec->mNativeWindow.get(), info->mGraphicBuffer.get(), info->mFenceFd);
- // TODO(b/266211548): Poll the native window for rendered buffers, since when queueing
- // buffers, the frame event history delta is retrieved.
+
+ int64_t mediaTimeUs = -1;
+ buffer->meta()->findInt64("timeUs", &mediaTimeUs);
+ if (mCodec->mIsWindowToDisplay) {
+ mCodec->trackReleasedFrame(frameId, mediaTimeUs, timestampNs);
+ mCodec->pollForRenderedFrames();
+ } else {
+ // When the surface is an intermediate surface, onFrameRendered is triggered immediately
+ // when the frame is queued to the non-display surface
+ mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs,
+ timestampNs)});
+ }
+
info->mFenceFd = -1;
if (err == OK) {
info->mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;
@@ -7021,7 +7044,6 @@
++mCodec->mNodeGeneration;
mCodec->mComponentName = componentName;
- mCodec->mRenderTracker.setComponentName(componentName);
mCodec->mFlags = 0;
if (componentName.endsWith(".secure")) {
@@ -7644,7 +7666,6 @@
void ACodec::ExecutingState::stateEntered() {
ALOGV("[%s] Now Executing", mCodec->mComponentName.c_str());
- mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));
mCodec->processDeferredMessages();
}
@@ -7755,7 +7776,15 @@
mCodec->signalSubmitOutputMetadataBufferIfEOS_workaround();
}
}
- return true;
+ handled = true;
+ break;
+ }
+
+ case kWhatPollForRenderedBuffers:
+ {
+ mCodec->pollForRenderedFrames();
+ handled = true;
+ break;
}
default:
@@ -8439,7 +8468,7 @@
}
bool ACodec::ExecutingState::onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
- mCodec->onFrameRendered(mediaTimeUs, systemNano);
+ mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, systemNano)});
return true;
}
@@ -8613,7 +8642,7 @@
bool ACodec::OutputPortSettingsChangedState::onOMXFrameRendered(
int64_t mediaTimeUs, nsecs_t systemNano) {
- mCodec->onFrameRendered(mediaTimeUs, systemNano);
+ mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, systemNano)});
return true;
}
@@ -8644,10 +8673,6 @@
OMX_CommandPortEnable, kPortIndexOutput);
}
- // Clear the RenderQueue in which queued GraphicBuffers hold the
- // actual buffer references in order to free them early.
- mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));
-
if (err == OK) {
err = mCodec->allocateBuffersOnPort(kPortIndexOutput);
ALOGE_IF(err != OK, "Failed to allocate output port buffers after port "
@@ -9031,8 +9056,6 @@
// the native window for rendering. Let's get those back as well.
mCodec->waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs();
- mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));
-
mCodec->mCallback->onFlushCompleted();
mCodec->mPortEOS[kPortIndexInput] =
diff --git a/media/libstagefright/ACodecBufferChannel.cpp b/media/libstagefright/ACodecBufferChannel.cpp
index 8f2bed2..ad42813 100644
--- a/media/libstagefright/ACodecBufferChannel.cpp
+++ b/media/libstagefright/ACodecBufferChannel.cpp
@@ -32,6 +32,7 @@
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/AMessage.h>
#include <media/stagefright/foundation/AUtils.h>
+#include <media/stagefright/ACodec.h>
#include <media/stagefright/MediaCodec.h>
#include <media/MediaCodecBuffer.h>
#include <system/window.h>
@@ -87,9 +88,11 @@
}
ACodecBufferChannel::ACodecBufferChannel(
- const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained)
+ const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained,
+ const sp<AMessage> &pollForRenderedBuffers)
: mInputBufferFilled(inputBufferFilled),
mOutputBufferDrained(outputBufferDrained),
+ mPollForRenderedBuffers(pollForRenderedBuffers),
mHeapSeqNum(-1) {
}
@@ -488,7 +491,7 @@
}
void ACodecBufferChannel::pollForRenderedBuffers() {
- // TODO(b/266211548): Poll the native window for rendered buffers.
+ mPollForRenderedBuffers->post();
}
status_t ACodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 1a8597f..463f212 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -79,6 +79,7 @@
#include <media/stagefright/MediaErrors.h>
#include <media/stagefright/OMXClient.h>
#include <media/stagefright/PersistentSurface.h>
+#include <media/stagefright/RenderedFrameInfo.h>
#include <media/stagefright/SurfaceUtils.h>
#include <nativeloader/dlext_namespaces.h>
#include <private/android_filesystem_config.h>
@@ -750,7 +751,7 @@
const sp<AMessage> &outputFormat) override;
virtual void onInputSurfaceDeclined(status_t err) override;
virtual void onSignaledInputEOS(status_t err) override;
- virtual void onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) override;
+ virtual void onOutputFramesRendered(const std::list<RenderedFrameInfo> &done) override;
virtual void onOutputBuffersChanged() override;
virtual void onFirstTunnelFrameReady() override;
private:
@@ -859,7 +860,7 @@
notify->post();
}
-void CodecCallback::onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) {
+void CodecCallback::onOutputFramesRendered(const std::list<RenderedFrameInfo> &done) {
sp<AMessage> notify(mNotify->dup());
notify->setInt32("what", kWhatOutputFramesRendered);
if (MediaCodec::CreateFramesRenderedMessage(done, notify)) {
@@ -5953,12 +5954,10 @@
return onQueueInputBuffer(msg);
}
-//static
-size_t MediaCodec::CreateFramesRenderedMessage(
- const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) {
+template<typename T>
+static size_t CreateFramesRenderedMessageInternal(const std::list<T> &done, sp<AMessage> &msg) {
size_t index = 0;
- for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
- it != done.cend(); ++it) {
+ for (typename std::list<T>::const_iterator it = done.cbegin(); it != done.cend(); ++it) {
if (it->getRenderTimeNs() < 0) {
continue; // dropped frame from tracking
}
@@ -5969,6 +5968,18 @@
return index;
}
+//static
+size_t MediaCodec::CreateFramesRenderedMessage(
+ const std::list<RenderedFrameInfo> &done, sp<AMessage> &msg) {
+ return CreateFramesRenderedMessageInternal(done, msg);
+}
+
+//static
+size_t MediaCodec::CreateFramesRenderedMessage(
+ const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) {
+ return CreateFramesRenderedMessageInternal(done, msg);
+}
+
status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
size_t index;
CHECK(msg->findSize("index", &index));
diff --git a/media/libstagefright/include/ACodecBufferChannel.h b/media/libstagefright/include/ACodecBufferChannel.h
index 903280f..a464504 100644
--- a/media/libstagefright/include/ACodecBufferChannel.h
+++ b/media/libstagefright/include/ACodecBufferChannel.h
@@ -29,6 +29,7 @@
#include <media/IOMX.h>
namespace android {
+ struct ACodec;
namespace hardware {
class HidlMemory;
};
@@ -63,7 +64,8 @@
};
ACodecBufferChannel(
- const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained);
+ const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained,
+ const sp<AMessage> &pollForRenderedBuffers);
virtual ~ACodecBufferChannel();
// BufferChannelBase interface
@@ -138,6 +140,7 @@
const sp<AMessage> mInputBufferFilled;
const sp<AMessage> mOutputBufferDrained;
+ const sp<AMessage> mPollForRenderedBuffers;
sp<MemoryDealer> mDealer;
sp<IMemory> mDecryptDestination;
diff --git a/media/libstagefright/include/media/stagefright/ACodec.h b/media/libstagefright/include/media/stagefright/ACodec.h
index 08c7917..88cff23 100644
--- a/media/libstagefright/include/media/stagefright/ACodec.h
+++ b/media/libstagefright/include/media/stagefright/ACodec.h
@@ -19,7 +19,7 @@
#include <set>
#include <stdint.h>
-#include <list>
+#include <deque>
#include <vector>
#include <android/native_window.h>
#include <media/hardware/MetadataBufferType.h>
@@ -27,9 +27,9 @@
#include <media/IOMX.h>
#include <media/stagefright/AHierarchicalStateMachine.h>
#include <media/stagefright/CodecBase.h>
-#include <media/stagefright/FrameRenderTracker.h>
#include <media/stagefright/MediaDefs.h>
#include <media/stagefright/SkipCutBuffer.h>
+#include <ui/GraphicBuffer.h>
#include <utils/NativeHandle.h>
#include <OMX_Audio.h>
#include <hardware/gralloc.h>
@@ -156,6 +156,7 @@
kWhatForceStateTransition = 'fstt',
kWhatCheckIfStuck = 'Cstk',
kWhatSubmitExtraOutputMetadataBuffer = 'sbxo',
+ kWhatPollForRenderedBuffers = 'pfrb',
};
enum {
@@ -177,6 +178,13 @@
| static_cast<uint64_t>(BufferUsage::VIDEO_DECODER),
};
+ struct TrackedFrame {
+ int64_t id;
+ int64_t mediaTimeUs;
+ int64_t desiredRenderTimeNs;
+ nsecs_t renderTimeNs;
+ };
+
struct BufferInfo {
enum Status {
OWNED_BY_US,
@@ -204,7 +212,6 @@
sp<GraphicBuffer> mGraphicBuffer;
bool mNewGraphicBuffer;
int mFenceFd;
- FrameRenderTracker::Info *mRenderInfo;
// The following field and 4 methods are used for debugging only
bool mIsReadFence;
@@ -253,6 +260,8 @@
bool mUsingNativeWindow;
sp<ANativeWindow> mNativeWindow;
+ bool mIsWindowToDisplay;
+ bool mHasPresentFenceTimes;
int mNativeWindowUsageBits;
android_native_rect_t mLastNativeWindowCrop;
int32_t mLastNativeWindowDataSpace;
@@ -267,7 +276,7 @@
// format updates. This will equal to mOutputFormat until the first actual frame is received.
sp<AMessage> mBaseOutputFormat;
- FrameRenderTracker mRenderTracker; // render information for buffers rendered by ACodec
+ std::deque<TrackedFrame> mTrackedFrames; // render information for buffers sent to a window
std::vector<BufferInfo> mBuffers[2];
bool mPortEOS[2];
status_t mInputEOSResult;
@@ -349,6 +358,10 @@
status_t freeOutputBuffersNotOwnedByComponent();
BufferInfo *dequeueBufferFromNativeWindow();
+ void initializeFrameTracking();
+ void trackReleasedFrame(int64_t frameId, int64_t mediaTimeUs, int64_t desiredRenderTimeNs);
+ void pollForRenderedFrames();
+
inline bool storingMetadataInDecodedBuffers() {
return (mPortMode[kPortIndexOutput] == IOMX::kPortModeDynamicANWBuffer) && !mIsEncoder;
}
@@ -571,21 +584,6 @@
void processDeferredMessages();
void onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
- // called when we have dequeued a buffer |buf| from the native window to track render info.
- // |fenceFd| is the dequeue fence, and |info| points to the buffer info where this buffer is
- // stored.
- void updateRenderInfoForDequeuedBuffer(
- ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info);
-
- // Checks to see if any frames have rendered up until |until|, and to notify client
- // (MediaCodec) of rendered frames up-until the frame pointed to by |until| or the first
- // unrendered frame. These frames are removed from the render queue.
- // If |dropIncomplete| is true, unrendered frames up-until |until| will be dropped from the
- // queue, allowing all rendered framed up till then to be notified of.
- // (This will effectively clear the render queue up-until (and including) |until|.)
- // If |until| is NULL, or is not in the rendered queue, this method will check all frames.
- void notifyOfRenderedFrames(
- bool dropIncomplete = false, FrameRenderTracker::Info *until = NULL);
void onFirstTunnelFrameReady();
diff --git a/media/libstagefright/include/media/stagefright/CodecBase.h b/media/libstagefright/include/media/stagefright/CodecBase.h
index 916d41e..90347f9 100644
--- a/media/libstagefright/include/media/stagefright/CodecBase.h
+++ b/media/libstagefright/include/media/stagefright/CodecBase.h
@@ -41,7 +41,7 @@
struct BufferProducerWrapper;
class MediaCodecBuffer;
struct PersistentSurface;
-struct RenderedFrameInfo;
+class RenderedFrameInfo;
class Surface;
struct ICrypto;
class IMemory;
diff --git a/media/libstagefright/include/media/stagefright/FrameRenderTracker.h b/media/libstagefright/include/media/stagefright/FrameRenderTracker.h
index c14755a..cab7ecc 100644
--- a/media/libstagefright/include/media/stagefright/FrameRenderTracker.h
+++ b/media/libstagefright/include/media/stagefright/FrameRenderTracker.h
@@ -32,61 +32,59 @@
namespace android {
-// Tracks the render information about a frame. Frames go through several states while
-// the render information is tracked:
-//
-// 1. queued frame: mMediaTime and mGraphicBuffer are set for the frame. mFence is the
-// queue fence (read fence). mIndex is negative, and mRenderTimeNs is invalid.
-// Key characteristics: mFence is not NULL and mIndex is negative.
-//
-// 2. dequeued frame: mFence is updated with the dequeue fence (write fence). mIndex is set.
-// Key characteristics: mFence is not NULL and mIndex is non-negative. mRenderTime is still
-// invalid.
-//
-// 3. rendered frame or frame: mFence is cleared, mRenderTimeNs is set.
-// Key characteristics: mFence is NULL.
-//
-struct RenderedFrameInfo {
- // set by client during onFrameQueued or onFrameRendered
- int64_t getMediaTimeUs() const { return mMediaTimeUs; }
-
- // -1 if frame is not yet rendered
- nsecs_t getRenderTimeNs() const { return mRenderTimeNs; }
-
- // set by client during updateRenderInfoForDequeuedBuffer; -1 otherwise
- ssize_t getIndex() const { return mIndex; }
-
- // creates information for a queued frame
- RenderedFrameInfo(int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer,
- const sp<Fence> &fence)
- : mMediaTimeUs(mediaTimeUs),
- mRenderTimeNs(-1),
- mIndex(-1),
- mGraphicBuffer(graphicBuffer),
- mFence(fence) {
- }
-
- // creates information for a frame rendered on a tunneled surface
- RenderedFrameInfo(int64_t mediaTimeUs, nsecs_t renderTimeNs)
- : mMediaTimeUs(mediaTimeUs),
- mRenderTimeNs(renderTimeNs),
- mIndex(-1),
- mGraphicBuffer(NULL),
- mFence(NULL) {
- }
-
-private:
- int64_t mMediaTimeUs;
- nsecs_t mRenderTimeNs;
- ssize_t mIndex; // to be used by client
- sp<GraphicBuffer> mGraphicBuffer;
- sp<Fence> mFence;
-
- friend struct FrameRenderTracker;
-};
-
struct FrameRenderTracker {
- typedef RenderedFrameInfo Info;
+ // Tracks the render information about a frame. Frames go through several states while
+ // the render information is tracked:
+ //
+ // 1. queued frame: mMediaTime and mGraphicBuffer are set for the frame. mFence is the
+ // queue fence (read fence). mIndex is negative, and mRenderTimeNs is invalid.
+ // Key characteristics: mFence is not NULL and mIndex is negative.
+ //
+ // 2. dequeued frame: mFence is updated with the dequeue fence (write fence). mIndex is set.
+ // Key characteristics: mFence is not NULL and mIndex is non-negative. mRenderTime is still
+ // invalid.
+ //
+ // 3. rendered frame or frame: mFence is cleared, mRenderTimeNs is set.
+ // Key characteristics: mFence is NULL.
+ //
+ struct Info {
+ // set by client during onFrameQueued or onFrameRendered
+ int64_t getMediaTimeUs() const { return mMediaTimeUs; }
+
+ // -1 if frame is not yet rendered
+ nsecs_t getRenderTimeNs() const { return mRenderTimeNs; }
+
+ // set by client during updateRenderInfoForDequeuedBuffer; -1 otherwise
+ ssize_t getIndex() const { return mIndex; }
+
+ // creates information for a queued frame
+ Info(int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer,
+ const sp<Fence> &fence)
+ : mMediaTimeUs(mediaTimeUs),
+ mRenderTimeNs(-1),
+ mIndex(-1),
+ mGraphicBuffer(graphicBuffer),
+ mFence(fence) {
+ }
+
+ // creates information for a frame rendered on a tunneled surface
+ Info(int64_t mediaTimeUs, nsecs_t renderTimeNs)
+ : mMediaTimeUs(mediaTimeUs),
+ mRenderTimeNs(renderTimeNs),
+ mIndex(-1),
+ mGraphicBuffer(NULL),
+ mFence(NULL) {
+ }
+
+ private:
+ int64_t mMediaTimeUs;
+ nsecs_t mRenderTimeNs;
+ ssize_t mIndex; // to be used by client
+ sp<GraphicBuffer> mGraphicBuffer;
+ sp<Fence> mFence;
+
+ friend struct FrameRenderTracker;
+ };
FrameRenderTracker();
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index 52d7d3d..6c2a096 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -64,6 +64,7 @@
class MediaCodecBuffer;
class IMemory;
struct PersistentSurface;
+class RenderedFrameInfo;
class SoftwareRenderer;
class Surface;
namespace hardware {
@@ -281,6 +282,8 @@
// by adding rendered frame information to a base notification message. Returns the number
// of frames that were rendered.
static size_t CreateFramesRenderedMessage(
+ const std::list<RenderedFrameInfo> &done, sp<AMessage> &msg);
+ static size_t CreateFramesRenderedMessage(
const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg);
static status_t CanFetchLinearBlock(
diff --git a/media/libstagefright/include/media/stagefright/RenderedFrameInfo.h b/media/libstagefright/include/media/stagefright/RenderedFrameInfo.h
new file mode 100644
index 0000000..4b8a58d
--- /dev/null
+++ b/media/libstagefright/include/media/stagefright/RenderedFrameInfo.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RENDERED_FRAME_INFO_H
+#define RENDERED_FRAME_INFO_H
+
+namespace android {
+
+class RenderedFrameInfo {
+public:
+ RenderedFrameInfo(int64_t mediaTimeUs, int64_t renderTimeNs)
+ : mMediaTimeUs(mediaTimeUs), mRenderTimeNs(renderTimeNs) {}
+
+ int64_t getMediaTimeUs() const { return mMediaTimeUs; }
+ nsecs_t getRenderTimeNs() const { return mRenderTimeNs;}
+
+private:
+ int64_t mMediaTimeUs;
+ nsecs_t mRenderTimeNs;
+};
+
+} // android
+
+#endif // RENDERED_FRAME_INFO_H
\ No newline at end of file