stagefright: add support for output frame rendered callback
- Added FRAME_RENDERED event in OMX, used by tunneled video decoders
to signal rendered event timing
- Track buffers sent for rendering in ACodec and in SoftwareRenderer, and
determine when they have rendered
- Propagate render times to MediaCodec
Bug: 20503131
Change-Id: Idf0a8714d5368b237c2285dd39fa82db847c232f
diff --git a/include/media/IOMX.h b/include/media/IOMX.h
index 7023453..3d29e4a 100644
--- a/include/media/IOMX.h
+++ b/include/media/IOMX.h
@@ -198,7 +198,7 @@
EVENT,
EMPTY_BUFFER_DONE,
FILL_BUFFER_DONE,
-
+ FRAME_RENDERED,
} type;
IOMX::node_id node;
@@ -226,6 +226,11 @@
OMX_TICKS timestamp;
} extended_buffer_data;
+ // if type == FRAME_RENDERED
+ struct {
+ OMX_TICKS timestamp;
+ OMX_S64 nanoTime;
+ } render_data;
} u;
};
diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h
index d4891a1..f9ea38e 100644
--- a/include/media/stagefright/ACodec.h
+++ b/include/media/stagefright/ACodec.h
@@ -24,6 +24,7 @@
#include <media/IOMX.h>
#include <media/stagefright/foundation/AHierarchicalStateMachine.h>
#include <media/stagefright/CodecBase.h>
+#include <media/stagefright/FrameRenderTracker.h>
#include <media/stagefright/SkipCutBuffer.h>
#include <OMX_Audio.h>
@@ -162,6 +163,7 @@
sp<ABuffer> mData;
sp<GraphicBuffer> mGraphicBuffer;
int mFenceFd;
+ FrameRenderTracker::Info *mRenderInfo;
// The following field and 4 methods are used for debugging only
bool mIsReadFence;
@@ -214,6 +216,7 @@
sp<AMessage> mOutputFormat;
sp<AMessage> mBaseOutputFormat;
+ FrameRenderTracker mRenderTracker; // render information for buffers rendered by ACodec
Vector<BufferInfo> mBuffers[2];
bool mPortEOS[2];
status_t mInputEOSResult;
@@ -375,6 +378,23 @@
void deferMessage(const sp<AMessage> &msg);
void processDeferredMessages();
+ void onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
+ // called when we have dequeued a buffer |buf| from the native window to track render info.
+ // |fenceFd| is the dequeue fence, and |info| points to the buffer info where this buffer is
+ // stored.
+ void updateRenderInfoForDequeuedBuffer(
+ ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info);
+
+ // Checks to see if any frames have rendered up until |until|, and to notify client
+ // (MediaCodec) of rendered frames up-until the frame pointed to by |until| or the first
+ // unrendered frame. These frames are removed from the render queue.
+ // If |dropIncomplete| is true, unrendered frames up-until |until| will be dropped from the
+ // queue, allowing all rendered framed up till then to be notified of.
+ // (This will effectively clear the render queue up-until (and including) |until|.)
+ // If |until| is NULL, or is not in the rendered queue, this method will check all frames.
+ void notifyOfRenderedFrames(
+ bool dropIncomplete = false, FrameRenderTracker::Info *until = NULL);
+
void sendFormatChange(const sp<AMessage> &reply);
status_t getPortFormat(OMX_U32 portIndex, sp<AMessage> ¬ify);
diff --git a/include/media/stagefright/CodecBase.h b/include/media/stagefright/CodecBase.h
index 989df4f..bb36052 100644
--- a/include/media/stagefright/CodecBase.h
+++ b/include/media/stagefright/CodecBase.h
@@ -43,6 +43,7 @@
kWhatInputSurfaceAccepted = 'isfa',
kWhatSignaledInputEOS = 'seos',
kWhatBuffersAllocated = 'allc',
+ kWhatOutputFramesRendered = 'outR',
};
virtual void setNotificationMessage(const sp<AMessage> &msg) = 0;
diff --git a/include/media/stagefright/FrameRenderTracker.h b/include/media/stagefright/FrameRenderTracker.h
new file mode 100644
index 0000000..3b0db5a
--- /dev/null
+++ b/include/media/stagefright/FrameRenderTracker.h
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FRAME_RENDER_TRACKER_H_
+
+#define FRAME_RENDER_TRACKER_H_
+
+#include <utils/RefBase.h>
+#include <utils/Timers.h>
+#include <system/window.h>
+
+#include <media/stagefright/foundation/ADebug.h>
+#include <media/stagefright/foundation/AString.h>
+
+#include <list>
+
+namespace android {
+
+class Fence;
+class GraphicBuffer;
+
+struct FrameRenderTracker : public RefBase {
+ // Tracks the render information about a frame. Frames go through several states while
+ // the render information is tracked:
+ //
+ // 1. queued frame: mMediaTime and mGraphicBuffer are set for the frame. mFence is the
+ // queue fence (read fence). mIndex is negative, and mRenderTimeNs is invalid.
+ // Key characteristics: mFence is not NULL and mIndex is negative.
+ //
+ // 2. dequeued frame: mFence is updated with the dequeue fence (write fence). mIndex is set.
+ // Key characteristics: mFence is not NULL and mIndex is non-negative. mRenderTime is still
+ // invalid.
+ //
+ // 3. rendered frame or frame: mFence is cleared, mRenderTimeNs is set.
+ // Key characteristics: mFence is NULL.
+ //
+ struct Info {
+ // set by client during onFrameQueued or onFrameRendered
+ int64_t getMediaTimeUs() const { return mMediaTimeUs; }
+
+ // -1 if frame is not yet rendered
+ nsecs_t getRenderTimeNs() const { return mRenderTimeNs; }
+
+ // set by client during updateRenderInfoForDequeuedBuffer; -1 otherwise
+ ssize_t getIndex() const { return mIndex; }
+
+ // creates information for a queued frame
+ Info(int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer, const sp<Fence> &fence)
+ : mMediaTimeUs(mediaTimeUs),
+ mRenderTimeNs(-1),
+ mIndex(-1),
+ mGraphicBuffer(graphicBuffer),
+ mFence(fence) {
+ }
+
+ // creates information for a frame rendered on a tunneled surface
+ Info(int64_t mediaTimeUs, nsecs_t renderTimeNs)
+ : mMediaTimeUs(mediaTimeUs),
+ mRenderTimeNs(renderTimeNs),
+ mIndex(-1),
+ mGraphicBuffer(NULL),
+ mFence(NULL) {
+ }
+
+ private:
+ int64_t mMediaTimeUs;
+ nsecs_t mRenderTimeNs;
+ ssize_t mIndex; // to be used by client
+ sp<GraphicBuffer> mGraphicBuffer;
+ sp<Fence> mFence;
+
+ friend class FrameRenderTracker;
+ };
+
+ FrameRenderTracker();
+
+ void setComponentName(const AString &componentName);
+
+ // clears all tracked frames, and resets last render time
+ void clear(nsecs_t lastRenderTimeNs);
+
+ // called when |graphicBuffer| corresponding to |mediaTimeUs| is
+ // queued to the output surface using |fence|.
+ void onFrameQueued(
+ int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer, const sp<Fence> &fence);
+
+ // Called when we have dequeued a buffer |buf| from the native window to track render info.
+ // |fenceFd| is the dequeue fence, and |index| is a positive buffer ID to be usable by the
+ // client to track this render info among the dequeued buffers.
+ // Returns pointer to the tracked info, or NULL if buffer is not tracked or if |index|
+ // is negative.
+ Info *updateInfoForDequeuedBuffer(ANativeWindowBuffer *buf, int fenceFd, int index);
+
+ // called when tunneled codec signals frame rendered event
+ // returns BAD_VALUE if systemNano is not monotonic. Otherwise, returns OK.
+ status_t onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
+
+ // Checks to see if any frames have rendered up until |until|. If |until| is NULL or not a
+ // tracked info, this method searches the entire render queue.
+ // Returns list of rendered frames up-until the frame pointed to by |until| or the first
+ // unrendered frame, as well as any dropped frames (those with invalid fence) up-until |until|.
+ // These frames are removed from the render queue.
+ // If |dropIncomplete| is true, unrendered frames up-until |until| will also be dropped from the
+ // queue, allowing all rendered framed up till then to be notified of.
+ // (This will effectively clear the render queue up-until (and including) |until|.)
+ std::list<Info> checkFencesAndGetRenderedFrames(const Info *until, bool dropIncomplete);
+
+ // Stop tracking a queued frame (e.g. if the frame has been discarded). If |info| is NULL or is
+ // not tracked, this method is a no-op.
+ void untrackFrame(const Info *info);
+
+ void dumpRenderQueue() const;
+
+ virtual ~FrameRenderTracker();
+
+private:
+
+ // Render information for buffers. Regular surface buffers are queued in the order of
+ // rendering. Tunneled buffers are queued in the order of receipt.
+ std::list<Info> mRenderQueue;
+ nsecs_t mLastRenderTimeNs;
+ AString mComponentName;
+
+ DISALLOW_EVIL_CONSTRUCTORS(FrameRenderTracker);
+};
+
+} // namespace android
+
+#endif // FRAME_RENDER_TRACKER_H_
diff --git a/include/media/stagefright/MediaCodec.h b/include/media/stagefright/MediaCodec.h
index d62b35d..09cbe8f 100644
--- a/include/media/stagefright/MediaCodec.h
+++ b/include/media/stagefright/MediaCodec.h
@@ -22,6 +22,7 @@
#include <media/hardware/CryptoAPI.h>
#include <media/MediaResource.h>
#include <media/stagefright/foundation/AHandler.h>
+#include <media/stagefright/FrameRenderTracker.h>
#include <utils/Vector.h>
namespace android {
@@ -76,6 +77,8 @@
status_t setCallback(const sp<AMessage> &callback);
+ status_t setOnFrameRenderedNotification(const sp<AMessage> ¬ify);
+
status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
status_t setInputSurface(const sp<PersistentSurface> &surface);
@@ -157,6 +160,12 @@
status_t setParameters(const sp<AMessage> ¶ms);
+ // Create a MediaCodec notification message from a list of rendered or dropped render infos
+ // by adding rendered frame information to a base notification message. Returns the number
+ // of frames that were rendered.
+ static size_t CreateFramesRenderedMessage(
+ std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg);
+
protected:
virtual ~MediaCodec();
virtual void onMessageReceived(const sp<AMessage> &msg);
@@ -212,6 +221,7 @@
kWhatGetName = 'getN',
kWhatSetParameters = 'setP',
kWhatSetCallback = 'setC',
+ kWhatSetNotification = 'setN',
};
enum {
@@ -275,9 +285,11 @@
status_t mStickyError;
sp<Surface> mSurface;
SoftwareRenderer *mSoftRenderer;
+
sp<AMessage> mOutputFormat;
sp<AMessage> mInputFormat;
sp<AMessage> mCallback;
+ sp<AMessage> mOnFrameRenderedNotification;
sp<MemoryDealer> mDealer;
sp<IResourceManagerClient> mResourceManagerClient;
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index 172e19c..71cb194 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -37,6 +37,7 @@
#include <media/stagefright/foundation/AUtils.h>
#include <media/stagefright/BufferProducerWrapper.h>
+#include <media/stagefright/MediaCodec.h>
#include <media/stagefright/MediaCodecList.h>
#include <media/stagefright/MediaDefs.h>
#include <media/stagefright/OMXClient.h>
@@ -175,6 +176,15 @@
break;
}
+ case omx_message::FRAME_RENDERED:
+ {
+ msg->setInt64(
+ "media_time_us", omx_msg.u.render_data.timestamp);
+ msg->setInt64(
+ "system_nano", omx_msg.u.render_data.nanoTime);
+ break;
+ }
+
default:
ALOGE("Unrecognized message type: %d", omx_msg.type);
break;
@@ -238,6 +248,8 @@
int64_t timeUs,
int fenceFd);
+ virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
+
void getMoreInputDataIfPossible();
DISALLOW_EVIL_CONSTRUCTORS(BaseState);
@@ -354,6 +366,7 @@
virtual void stateEntered();
virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
+ virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
private:
bool mActive;
@@ -372,6 +385,7 @@
virtual void stateEntered();
virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
+ virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
private:
DISALLOW_EVIL_CONSTRUCTORS(OutputPortSettingsChangedState);
@@ -786,6 +800,7 @@
BufferInfo info;
info.mStatus = BufferInfo::OWNED_BY_US;
info.mFenceFd = -1;
+ info.mRenderInfo = NULL;
uint32_t requiresAllocateBufferBit =
(portIndex == kPortIndexInput)
@@ -1002,6 +1017,7 @@
info.mStatus = BufferInfo::OWNED_BY_US;
info.mFenceFd = fenceFd;
info.mIsReadFence = false;
+ info.mRenderInfo = NULL;
info.mData = new ABuffer(NULL /* data */, bufferSize /* capacity */);
info.mGraphicBuffer = graphicBuffer;
mBuffers[kPortIndexOutput].push(info);
@@ -1073,6 +1089,7 @@
BufferInfo info;
info.mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;
info.mFenceFd = -1;
+ info.mRenderInfo = NULL;
info.mGraphicBuffer = NULL;
info.mDequeuedAt = mDequeueCounter;
@@ -1219,6 +1236,42 @@
return err;
}
+void ACodec::updateRenderInfoForDequeuedBuffer(
+ ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info) {
+
+ info->mRenderInfo =
+ mRenderTracker.updateInfoForDequeuedBuffer(
+ buf, fenceFd, info - &mBuffers[kPortIndexOutput][0]);
+
+ // check for any fences already signaled
+ notifyOfRenderedFrames(false /* dropIncomplete */, info->mRenderInfo);
+}
+
+void ACodec::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
+ if (mRenderTracker.onFrameRendered(mediaTimeUs, systemNano) != OK) {
+ mRenderTracker.dumpRenderQueue();
+ }
+}
+
+void ACodec::notifyOfRenderedFrames(bool dropIncomplete, FrameRenderTracker::Info *until) {
+ sp<AMessage> msg = mNotify->dup();
+ msg->setInt32("what", CodecBase::kWhatOutputFramesRendered);
+ std::list<FrameRenderTracker::Info> done =
+ mRenderTracker.checkFencesAndGetRenderedFrames(until, dropIncomplete);
+
+ // unlink untracked frames
+ for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
+ it != done.cend(); ++it) {
+ if (it->getIndex() >= 0) {
+ mBuffers[kPortIndexOutput].editItemAt(it->getIndex()).mRenderInfo = NULL;
+ }
+ }
+
+ if (MediaCodec::CreateFramesRenderedMessage(done, msg)) {
+ msg->post();
+ }
+}
+
ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() {
ANativeWindowBuffer *buf;
CHECK(mNativeWindow.get() != NULL);
@@ -1242,7 +1295,7 @@
BufferInfo *info = &mBuffers[kPortIndexOutput].editItemAt(i);
if (info->mGraphicBuffer != NULL &&
- info->mGraphicBuffer->handle == buf->handle) {
+ info->mGraphicBuffer->handle == buf->handle) {
// Since consumers can attach buffers to BufferQueues, it is possible
// that a known yet stale buffer can return from a surface that we
// once used. We can simply ignore this as we have already dequeued
@@ -1255,9 +1308,11 @@
stale = true;
break;
}
+
ALOGV("dequeued buffer %p", info->mGraphicBuffer->getNativeBuffer());
info->mStatus = BufferInfo::OWNED_BY_US;
info->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow");
+ updateRenderInfoForDequeuedBuffer(buf, fenceFd, info);
return info;
}
}
@@ -1301,6 +1356,8 @@
oldest->mGraphicBuffer = new GraphicBuffer(buf, false);
oldest->mStatus = BufferInfo::OWNED_BY_US;
oldest->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow for oldest");
+ mRenderTracker.untrackFrame(oldest->mRenderInfo);
+ oldest->mRenderInfo = NULL;
mOMX->updateGraphicBufferInMeta(
mNode, kPortIndexOutput, oldest->mGraphicBuffer,
@@ -1324,6 +1381,7 @@
oldest->mGraphicBuffer->getNativeBuffer(), oldest->mData->base());
}
+ updateRenderInfoForDequeuedBuffer(buf, fenceFd, oldest);
return oldest;
}
@@ -1398,6 +1456,9 @@
::close(info->mFenceFd);
}
+ mRenderTracker.untrackFrame(info->mRenderInfo);
+ info->mRenderInfo = NULL;
+
// remove buffer even if mOMX->freeBuffer fails
mBuffers[portIndex].removeAt(i);
@@ -4518,9 +4579,20 @@
CHECK(msg->findObject("messages", &obj));
sp<MessageList> msgList = static_cast<MessageList *>(obj.get());
+ bool receivedRenderedEvents = false;
for (std::list<sp<AMessage>>::const_iterator it = msgList->getList().cbegin();
it != msgList->getList().cend(); ++it) {
onOMXMessage(*it);
+ int32_t type;
+ CHECK((*it)->findInt32("type", &type));
+ if (type == omx_message::FRAME_RENDERED) {
+ receivedRenderedEvents = true;
+ }
+ }
+
+ if (receivedRenderedEvents) {
+ // NOTE: all buffers are rendered in this case
+ mCodec->notifyOfRenderedFrames();
}
return true;
}
@@ -4587,12 +4659,29 @@
fenceFd);
}
+ case omx_message::FRAME_RENDERED:
+ {
+ int64_t mediaTimeUs, systemNano;
+
+ CHECK(msg->findInt64("media_time_us", &mediaTimeUs));
+ CHECK(msg->findInt64("system_nano", &systemNano));
+
+ return onOMXFrameRendered(
+ mediaTimeUs, systemNano);
+ }
+
default:
ALOGE("Unexpected message type: %d", type);
return false;
}
}
+bool ACodec::BaseState::onOMXFrameRendered(
+ int64_t mediaTimeUs __unused, nsecs_t systemNano __unused) {
+ // ignore outside of Executing and PortSettingsChanged states
+ return true;
+}
+
bool ACodec::BaseState::onOMXEvent(
OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
if (event != OMX_EventError) {
@@ -4949,6 +5038,15 @@
info->mDequeuedAt = ++mCodec->mDequeueCounter;
info->mStatus = BufferInfo::OWNED_BY_US;
+ if (info->mRenderInfo != NULL) {
+ // The fence for an emptied buffer must have signaled, but there still could be queued
+ // or out-of-order dequeued buffers in the render queue prior to this buffer. Drop these,
+ // as we will soon requeue this buffer to the surface. While in theory we could still keep
+ // track of buffers that are requeued to the surface, it is better to add support to the
+ // buffer-queue to notify us of released buffers and their fences (in the future).
+ mCodec->notifyOfRenderedFrames(true /* dropIncomplete */);
+ }
+
// byte buffers cannot take fences, so wait for any fence now
if (mCodec->mNativeWindow == NULL) {
(void)mCodec->waitForFence(fenceFd, "onOMXFillBufferDone");
@@ -5085,6 +5183,14 @@
ATRACE_NAME("render");
// The client wants this buffer to be rendered.
+ // save buffers sent to the surface so we can get render time when they return
+ int64_t mediaTimeUs = -1;
+ info->mData->meta()->findInt64("timeUs", &mediaTimeUs);
+ if (mediaTimeUs >= 0) {
+ mCodec->mRenderTracker.onFrameQueued(
+ mediaTimeUs, info->mGraphicBuffer, new Fence(::dup(info->mFenceFd)));
+ }
+
int64_t timestampNs = 0;
if (!msg->findInt64("timestampNs", ×tampNs)) {
// TODO: it seems like we should use the timestamp
@@ -5367,6 +5473,7 @@
observer->setNotificationMessage(notify);
mCodec->mComponentName = componentName;
+ mCodec->mRenderTracker.setComponentName(componentName);
mCodec->mFlags = 0;
if (componentName.endsWith(".secure")) {
@@ -5969,6 +6076,7 @@
void ACodec::ExecutingState::stateEntered() {
ALOGV("[%s] Now Executing", mCodec->mComponentName.c_str());
+ mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));
mCodec->processDeferredMessages();
}
@@ -6179,6 +6287,11 @@
notify->post();
}
+bool ACodec::ExecutingState::onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
+ mCodec->onFrameRendered(mediaTimeUs, systemNano);
+ return true;
+}
+
bool ACodec::ExecutingState::onOMXEvent(
OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
switch (event) {
@@ -6266,6 +6379,12 @@
mCodec->mComponentName.c_str());
}
+bool ACodec::OutputPortSettingsChangedState::onOMXFrameRendered(
+ int64_t mediaTimeUs, nsecs_t systemNano) {
+ mCodec->onFrameRendered(mediaTimeUs, systemNano);
+ return true;
+}
+
bool ACodec::OutputPortSettingsChangedState::onOMXEvent(
OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
switch (event) {
@@ -6644,6 +6763,8 @@
// the native window for rendering. Let's get those back as well.
mCodec->waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs();
+ mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));
+
sp<AMessage> notify = mCodec->mNotify->dup();
notify->setInt32("what", CodecBase::kWhatFlushCompleted);
notify->post();
diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk
index 0dfa300..69128bd 100644
--- a/media/libstagefright/Android.mk
+++ b/media/libstagefright/Android.mk
@@ -23,6 +23,7 @@
ESDS.cpp \
FileSource.cpp \
FLACExtractor.cpp \
+ FrameRenderTracker.cpp \
HTTPBase.cpp \
JPEGSource.cpp \
MP3Extractor.cpp \
diff --git a/media/libstagefright/AwesomePlayer.cpp b/media/libstagefright/AwesomePlayer.cpp
index df01e7c..4e6c2a6 100644
--- a/media/libstagefright/AwesomePlayer.cpp
+++ b/media/libstagefright/AwesomePlayer.cpp
@@ -113,11 +113,11 @@
CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
render((const uint8_t *)buffer->data() + buffer->range_offset(),
- buffer->range_length(), timeUs * 1000);
+ buffer->range_length(), timeUs, timeUs * 1000);
}
- void render(const void *data, size_t size, int64_t timestampNs) {
- mTarget->render(data, size, timestampNs, NULL, mFormat);
+ void render(const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs) {
+ (void)mTarget->render(data, size, mediaTimeUs, renderTimeNs, NULL, mFormat);
}
protected:
diff --git a/media/libstagefright/FrameRenderTracker.cpp b/media/libstagefright/FrameRenderTracker.cpp
new file mode 100644
index 0000000..ebd2197
--- /dev/null
+++ b/media/libstagefright/FrameRenderTracker.cpp
@@ -0,0 +1,184 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "FrameRenderTracker"
+
+#include <inttypes.h>
+#include <gui/Surface.h>
+
+#include <media/stagefright/foundation/ADebug.h>
+#include <media/stagefright/FrameRenderTracker.h>
+
+namespace android {
+
+FrameRenderTracker::FrameRenderTracker()
+ : mLastRenderTimeNs(-1),
+ mComponentName("unknown component") {
+}
+
+FrameRenderTracker::~FrameRenderTracker() {
+}
+
+void FrameRenderTracker::setComponentName(const AString &componentName) {
+ mComponentName = componentName;
+}
+
+void FrameRenderTracker::clear(nsecs_t lastRenderTimeNs) {
+ mRenderQueue.clear();
+ mLastRenderTimeNs = lastRenderTimeNs;
+}
+
+void FrameRenderTracker::onFrameQueued(
+ int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer, const sp<Fence> &fence) {
+ mRenderQueue.emplace_back(mediaTimeUs, graphicBuffer, fence);
+}
+
+FrameRenderTracker::Info *FrameRenderTracker::updateInfoForDequeuedBuffer(
+ ANativeWindowBuffer *buf, int fenceFd, int index) {
+ if (index < 0) {
+ return NULL;
+ }
+
+ // see if this is a buffer that was to be rendered
+ std::list<Info>::iterator renderInfo = mRenderQueue.end();
+ for (std::list<Info>::iterator it = mRenderQueue.begin();
+ it != mRenderQueue.end(); ++it) {
+ if (it->mGraphicBuffer->handle == buf->handle) {
+ renderInfo = it;
+ break;
+ }
+ }
+ if (renderInfo == mRenderQueue.end()) {
+ // could have been canceled after fence has signaled
+ return NULL;
+ }
+
+ if (renderInfo->mIndex >= 0) {
+ // buffer has been dequeued before, so there is nothing to do
+ return NULL;
+ }
+
+ // was this frame dropped (we could also infer this if the fence is invalid or a dup of
+ // the queued fence; however, there is no way to figure that out.)
+ if (fenceFd < 0) {
+ // frame is new or was dropped
+ mRenderQueue.erase(renderInfo);
+ return NULL;
+ }
+
+ // store dequeue fence and buffer index
+ renderInfo->mFence = new Fence(::dup(fenceFd));
+ renderInfo->mIndex = index;
+ return &*renderInfo;
+}
+
+status_t FrameRenderTracker::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
+ // ensure monotonic timestamps
+ if (mLastRenderTimeNs >= systemNano) {
+ ALOGW("[%s] Ignoring out of order/stale system nano %lld for media time %lld from codec.",
+ mComponentName.c_str(), (long long)systemNano, (long long)mediaTimeUs);
+ return BAD_VALUE;
+ }
+
+ nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC);
+ if (systemNano > now) {
+ ALOGW("[%s] Ignoring system nano %lld in the future for media time %lld from codec.",
+ mComponentName.c_str(), (long long)systemNano, (long long)mediaTimeUs);
+ return OK;
+ }
+
+ mRenderQueue.emplace_back(mediaTimeUs, systemNano);
+ mLastRenderTimeNs = systemNano;
+ return OK;
+}
+
+std::list<FrameRenderTracker::Info> FrameRenderTracker::checkFencesAndGetRenderedFrames(
+ const FrameRenderTracker::Info *until, bool dropIncomplete) {
+ std::list<Info> done;
+
+ // complete any frames queued prior to this and drop any incomplete ones if requested
+ for (std::list<Info>::iterator it = mRenderQueue.begin();
+ it != mRenderQueue.end(); ) {
+ bool drop = false; // whether to drop each frame
+ if (it->mIndex < 0) {
+ // frame not yet dequeued (or already rendered on a tunneled surface)
+ drop = dropIncomplete;
+ } else if (it->mFence != NULL) {
+ // check if fence signaled
+ nsecs_t signalTime = it->mFence->getSignalTime();
+ if (signalTime < 0) { // invalid fence
+ drop = true;
+ } else if (signalTime == INT64_MAX) { // unsignaled fence
+ drop = dropIncomplete;
+ } else { // signaled
+ // save render time
+ it->mFence.clear();
+ it->mRenderTimeNs = signalTime;
+ }
+ }
+ bool foundFrame = (Info *)&*it == until;
+
+ // Return frames with signaled fences at the start of the queue, as they are
+ // in submit order, and we don't have to wait for any in-between frames.
+ // Also return any dropped frames.
+ if (drop || (it->mFence == NULL && it == mRenderQueue.begin())) {
+ // (unrendered) dropped frames have their mRenderTimeNs still set to -1
+ done.splice(done.end(), mRenderQueue, it++);
+ } else {
+ ++it;
+ }
+ if (foundFrame) {
+ break;
+ }
+ }
+
+ return done;
+}
+
+void FrameRenderTracker::untrackFrame(const FrameRenderTracker::Info *info) {
+ if (info != NULL) {
+ for (std::list<Info>::iterator it = mRenderQueue.begin();
+ it != mRenderQueue.end(); ++it) {
+ if (&*it == info) {
+ mRenderQueue.erase(it);
+ return;
+ }
+ }
+ }
+}
+
+void FrameRenderTracker::dumpRenderQueue() const {
+ ALOGI("[%s] Render Queue: (last render time: %lldns)",
+ mComponentName.c_str(), (long long)mLastRenderTimeNs);
+ for (std::list<Info>::const_iterator it = mRenderQueue.cbegin();
+ it != mRenderQueue.cend(); ++it) {
+ if (it->mFence == NULL) {
+ ALOGI(" RENDERED: handle: %p, media time: %lldus, index: %zd, render time: %lldns",
+ it->mGraphicBuffer == NULL ? NULL : it->mGraphicBuffer->handle,
+ (long long)it->mMediaTimeUs, it->mIndex, (long long)it->mRenderTimeNs);
+ } else if (it->mIndex < 0) {
+ ALOGI(" QUEUED: handle: %p, media time: %lldus, fence: %s",
+ it->mGraphicBuffer->handle, (long long)it->mMediaTimeUs,
+ it->mFence->isValid() ? "YES" : "NO");
+ } else {
+ ALOGI(" DEQUEUED: handle: %p, media time: %lldus, index: %zd",
+ it->mGraphicBuffer->handle, (long long)it->mMediaTimeUs, it->mIndex);
+ }
+ }
+}
+
+} // namespace android
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index f918d2d..7ee84a8 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -397,6 +397,12 @@
return PostAndAwaitResponse(msg, &response);
}
+status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> ¬ify) {
+ sp<AMessage> msg = new AMessage(kWhatSetNotification, this);
+ msg->setMessage("on-frame-rendered", notify);
+ return msg->post();
+}
+
status_t MediaCodec::configure(
const sp<AMessage> &format,
const sp<Surface> &surface,
@@ -1333,6 +1339,18 @@
break;
}
+ case CodecBase::kWhatOutputFramesRendered:
+ {
+ // ignore these in all states except running, and check that we have a
+ // notification set
+ if (mState == STARTED && mOnFrameRenderedNotification != NULL) {
+ sp<AMessage> notify = mOnFrameRenderedNotification->dup();
+ notify->setMessage("data", msg);
+ notify->post();
+ }
+ break;
+ }
+
case CodecBase::kWhatFillThisBuffer:
{
/* size_t index = */updateBuffers(kPortIndexInput, msg);
@@ -1530,6 +1548,15 @@
break;
}
+ case kWhatSetNotification:
+ {
+ sp<AMessage> notify;
+ if (msg->findMessage("on-frame-rendered", ¬ify)) {
+ mOnFrameRenderedNotification = notify;
+ }
+ break;
+ }
+
case kWhatSetCallback:
{
sp<AReplyToken> replyID;
@@ -2372,6 +2399,23 @@
return OK;
}
+//static
+size_t MediaCodec::CreateFramesRenderedMessage(
+ std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg) {
+ size_t index = 0;
+
+ for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
+ it != done.cend(); ++it) {
+ if (it->getRenderTimeNs() < 0) {
+ continue; // dropped frame from tracking
+ }
+ msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs());
+ msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs());
+ ++index;
+ }
+ return index;
+}
+
status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
size_t index;
CHECK(msg->findSize("index", &index));
@@ -2404,26 +2448,37 @@
if (render && info->mData != NULL && info->mData->size() != 0) {
info->mNotify->setInt32("render", true);
- int64_t timestampNs = 0;
- if (msg->findInt64("timestampNs", ×tampNs)) {
- info->mNotify->setInt64("timestampNs", timestampNs);
+ int64_t mediaTimeUs = -1;
+ info->mData->meta()->findInt64("timeUs", &mediaTimeUs);
+
+ int64_t renderTimeNs = 0;
+ if (msg->findInt64("timestampNs", &renderTimeNs)) {
+ info->mNotify->setInt64("timestampNs", renderTimeNs);
} else {
// TODO: it seems like we should use the timestamp
// in the (media)buffer as it potentially came from
// an input surface, but we did not propagate it prior to
// API 20. Perhaps check for target SDK version.
#if 0
- if (info->mData->meta()->findInt64("timeUs", ×tampNs)) {
- ALOGV("using buffer PTS of %" PRId64, timestampNs);
- timestampNs *= 1000;
- }
+ ALOGV("using buffer PTS of %" PRId64, timestampNs);
+ renderTimeNs = mediaTimeUs * 1000;
#endif
}
if (mSoftRenderer != NULL) {
- mSoftRenderer->render(
+ std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render(
info->mData->data(), info->mData->size(),
- timestampNs, NULL, info->mFormat);
+ mediaTimeUs, renderTimeNs, NULL, info->mFormat);
+
+ // if we are running, notify rendered frames
+ if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) {
+ sp<AMessage> notify = mOnFrameRenderedNotification->dup();
+ sp<AMessage> data = new AMessage;
+ if (CreateFramesRenderedMessage(doneFrames, data)) {
+ notify->setMessage("data", data);
+ notify->post();
+ }
+ }
}
}
diff --git a/media/libstagefright/colorconversion/SoftwareRenderer.cpp b/media/libstagefright/colorconversion/SoftwareRenderer.cpp
index 695cfc8..d22451b 100644
--- a/media/libstagefright/colorconversion/SoftwareRenderer.cpp
+++ b/media/libstagefright/colorconversion/SoftwareRenderer.cpp
@@ -196,17 +196,29 @@
mNativeWindow.get(), transform));
}
-void SoftwareRenderer::render(
- const void *data, size_t size, int64_t timestampNs,
+void SoftwareRenderer::clearTracker() {
+ mRenderTracker.clear(-1 /* lastRenderTimeNs */);
+}
+
+std::list<FrameRenderTracker::Info> SoftwareRenderer::render(
+ const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs,
void* /*platformPrivate*/, const sp<AMessage>& format) {
resetFormatIfChanged(format);
+ FrameRenderTracker::Info *info = NULL;
ANativeWindowBuffer *buf;
- int err;
- if ((err = native_window_dequeue_buffer_and_wait(mNativeWindow.get(),
- &buf)) != 0) {
+ int fenceFd = -1;
+ int err = mNativeWindow->dequeueBuffer(mNativeWindow.get(), &buf, &fenceFd);
+ if (err == 0 && fenceFd >= 0) {
+ info = mRenderTracker.updateInfoForDequeuedBuffer(buf, fenceFd, 0);
+ sp<Fence> fence = new Fence(fenceFd);
+ err = fence->waitForever("SoftwareRenderer::render");
+ }
+ if (err != 0) {
ALOGW("Surface::dequeueBuffer returned error %d", err);
- return;
+ // complete (drop) dequeued frame if fence wait failed; otherwise,
+ // this returns an empty list as no frames should have rendered and not yet returned.
+ return mRenderTracker.checkFencesAndGetRenderedFrames(info, false /* dropIncomplete */);
}
GraphicBufferMapper &mapper = GraphicBufferMapper::get();
@@ -342,16 +354,21 @@
skip_copying:
CHECK_EQ(0, mapper.unlock(buf->handle));
- if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(),
- timestampNs)) != 0) {
- ALOGW("Surface::set_buffers_timestamp returned error %d", err);
+ if (renderTimeNs >= 0) {
+ if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(),
+ renderTimeNs)) != 0) {
+ ALOGW("Surface::set_buffers_timestamp returned error %d", err);
+ }
}
- if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf,
- -1)) != 0) {
+ if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf, -1)) != 0) {
ALOGW("Surface::queueBuffer returned error %d", err);
+ } else {
+ mRenderTracker.onFrameQueued(mediaTimeUs, (GraphicBuffer *)buf, Fence::NO_FENCE);
}
+
buf = NULL;
+ return mRenderTracker.checkFencesAndGetRenderedFrames(info, info != NULL /* dropIncomplete */);
}
} // namespace android
diff --git a/media/libstagefright/include/SoftwareRenderer.h b/media/libstagefright/include/SoftwareRenderer.h
index fa3ea89..9e652d5 100644
--- a/media/libstagefright/include/SoftwareRenderer.h
+++ b/media/libstagefright/include/SoftwareRenderer.h
@@ -19,9 +19,12 @@
#define SOFTWARE_RENDERER_H_
#include <media/stagefright/ColorConverter.h>
+#include <media/stagefright/FrameRenderTracker.h>
#include <utils/RefBase.h>
#include <system/window.h>
+#include <list>
+
namespace android {
struct AMessage;
@@ -32,9 +35,10 @@
~SoftwareRenderer();
- void render(
- const void *data, size_t size, int64_t timestampNs,
+ std::list<FrameRenderTracker::Info> render(
+ const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs,
void *platformPrivate, const sp<AMessage> &format);
+ void clearTracker();
private:
enum YUVMode {
@@ -48,6 +52,7 @@
int32_t mWidth, mHeight;
int32_t mCropLeft, mCropTop, mCropRight, mCropBottom;
int32_t mCropWidth, mCropHeight;
+ FrameRenderTracker mRenderTracker;
SoftwareRenderer(const SoftwareRenderer &);
SoftwareRenderer &operator=(const SoftwareRenderer &);
diff --git a/media/libstagefright/omx/OMX.cpp b/media/libstagefright/omx/OMX.cpp
index e94adbd..cb7ab5e 100644
--- a/media/libstagefright/omx/OMX.cpp
+++ b/media/libstagefright/omx/OMX.cpp
@@ -34,6 +34,7 @@
#include <OMX_AsString.h>
#include <OMX_Component.h>
+#include <OMX_VideoExt.h>
namespace android {
@@ -455,12 +456,35 @@
OMX_IN OMX_EVENTTYPE eEvent,
OMX_IN OMX_U32 nData1,
OMX_IN OMX_U32 nData2,
- OMX_IN OMX_PTR /* pEventData */) {
+ OMX_IN OMX_PTR pEventData) {
ALOGV("OnEvent(%d, %" PRIu32", %" PRIu32 ")", eEvent, nData1, nData2);
// Forward to OMXNodeInstance.
findInstance(node)->onEvent(eEvent, nData1, nData2);
+ sp<OMX::CallbackDispatcher> dispatcher = findDispatcher(node);
+
+ // output rendered events are not processed as regular events until they hit the observer
+ if (eEvent == OMX_EventOutputRendered) {
+ if (pEventData == NULL) {
+ return OMX_ErrorBadParameter;
+ }
+
+ // process data from array
+ OMX_VIDEO_RENDEREVENTTYPE *renderData = (OMX_VIDEO_RENDEREVENTTYPE *)pEventData;
+ for (size_t i = 0; i < nData1; ++i) {
+ omx_message msg;
+ msg.type = omx_message::FRAME_RENDERED;
+ msg.node = node;
+ msg.fenceFd = -1;
+ msg.u.render_data.timestamp = renderData[i].nMediaTimeUs;
+ msg.u.render_data.nanoTime = renderData[i].nSystemTimeNs;
+
+ dispatcher->post(msg, false /* realTime */);
+ }
+ return OMX_ErrorNone;
+ }
+
omx_message msg;
msg.type = omx_message::EVENT;
msg.node = node;
@@ -469,7 +493,7 @@
msg.u.event_data.data1 = nData1;
msg.u.event_data.data2 = nData2;
- findDispatcher(node)->post(msg);
+ dispatcher->post(msg, true /* realTime */);
return OMX_ErrorNone;
}