Add support for video peek in Codec2
Video peek is a feature that enables displaying the first video frame
of a video stream decoded by a codec configured in tunnel mode.
This commit adds support for video peek to Codec2-based codecs.
Test: atest android.media.cts.DecoderTest
Merged-In: I74f0b6de2161f563a80b33f1bb3d6ff961de09b6
Merged-In: If8bc83f93d5cf12c1138c467e3152eafc3e92015
Merged-In: I298e27b1024440ea353f8a1db2afdabdb695f702
Change-Id: I533917908a7050c760374c7b607dd736ec173339
diff --git a/media/codec2/core/include/C2Config.h b/media/codec2/core/include/C2Config.h
index bdf2027..7eaed08 100644
--- a/media/codec2/core/include/C2Config.h
+++ b/media/codec2/core/include/C2Config.h
@@ -262,6 +262,8 @@
kParamIndexTunneledMode, // struct
kParamIndexTunnelHandle, // int32[]
kParamIndexTunnelSystemTime, // int64
+ kParamIndexTunnelHoldRender, // bool
+ kParamIndexTunnelStartRender, // bool
// dmabuf allocator
kParamIndexStoreDmaBufUsage, // store, struct
@@ -2365,6 +2367,30 @@
C2PortTunnelSystemTime;
constexpr char C2_PARAMKEY_OUTPUT_RENDER_TIME[] = "output.render-time";
+/**
+ * Tunneled mode video peek signaling flag.
+ *
+ * When a video frame is pushed to the decoder with this parameter set to true,
+ * the decoder must decode the frame, signal partial completion, and hold on the
+ * frame until C2StreamTunnelStartRender is set to true (which resets this
+ * flag). Flush will also result in the frames being returned back to the
+ * client (but not rendered).
+ */
+typedef C2StreamParam<C2Info, C2EasyBoolValue, kParamIndexTunnelHoldRender>
+ C2StreamTunnelHoldRender;
+constexpr char C2_PARAMKEY_TUNNEL_HOLD_RENDER[] = "output.tunnel-hold-render";
+
+/**
+ * Tunneled mode video peek signaling flag.
+ *
+ * Upon receiving this flag, the decoder shall set C2StreamTunnelHoldRender to
+ * false, which shall cause any frames held for rendering to be immediately
+ * displayed, regardless of their timestamps.
+*/
+typedef C2StreamParam<C2Info, C2EasyBoolValue, kParamIndexTunnelStartRender>
+ C2StreamTunnelStartRender;
+constexpr char C2_PARAMKEY_TUNNEL_START_RENDER[] = "output.tunnel-start-render";
+
C2ENUM(C2PlatformConfig::encoding_quality_level_t, uint32_t,
NONE,
S_HANDHELD,
diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp
index d942606..10a6896 100644
--- a/media/codec2/sfplugin/CCodec.cpp
+++ b/media/codec2/sfplugin/CCodec.cpp
@@ -665,6 +665,10 @@
mCodec->mCallback->onOutputBuffersChanged();
}
+ void onFirstTunnelFrameReady() override {
+ mCodec->mCallback->onFirstTunnelFrameReady();
+ }
+
private:
CCodec *mCodec;
};
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp
index 5a58fd8..2dc63f3 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.cpp
+++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp
@@ -205,6 +205,7 @@
int32_t flags = 0;
int32_t tmp = 0;
bool eos = false;
+ bool tunnelFirstFrame = false;
if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
eos = true;
mInputMetEos = true;
@@ -213,6 +214,9 @@
if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
flags |= C2FrameData::FLAG_CODEC_CONFIG;
}
+ if (buffer->meta()->findInt32("tunnel-first-frame", &tmp) && tmp) {
+ tunnelFirstFrame = true;
+ }
ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
std::list<std::unique_ptr<C2Work>> items;
std::unique_ptr<C2Work> work(new C2Work);
@@ -284,6 +288,13 @@
// TODO: fill info's
work->input.configUpdate = std::move(mParamsToBeSet);
+ if (tunnelFirstFrame) {
+ C2StreamTunnelHoldRender::input tunnelHoldRender{
+ 0u /* stream */,
+ C2_TRUE /* value */
+ };
+ work->input.configUpdate.push_back(C2Param::Copy(tunnelHoldRender));
+ }
work->worklets.clear();
work->worklets.emplace_back(new C2Worklet);
@@ -1706,6 +1717,15 @@
}
break;
}
+ case C2StreamTunnelHoldRender::CORE_INDEX: {
+ C2StreamTunnelHoldRender::output firstTunnelFrameHoldRender;
+ if (!(worklet->output.flags & C2FrameData::FLAG_INCOMPLETE)) break;
+ if (!firstTunnelFrameHoldRender.updateFrom(*param)) break;
+ if (firstTunnelFrameHoldRender.value != C2_TRUE) break;
+ ALOGV("[%s] onWorkDone: first tunnel frame ready", mName);
+ mCCodecCallback->onFirstTunnelFrameReady();
+ break;
+ }
default:
ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
mName, param->index());
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.h b/media/codec2/sfplugin/CCodecBufferChannel.h
index 45da003..5a2aca2 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.h
+++ b/media/codec2/sfplugin/CCodecBufferChannel.h
@@ -45,6 +45,7 @@
virtual void onError(status_t err, enum ActionCode actionCode) = 0;
virtual void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) = 0;
virtual void onOutputBuffersChanged() = 0;
+ virtual void onFirstTunnelFrameReady() = 0;
};
/**
diff --git a/media/codec2/sfplugin/CCodecConfig.cpp b/media/codec2/sfplugin/CCodecConfig.cpp
index 27e87e6..5646095 100644
--- a/media/codec2/sfplugin/CCodecConfig.cpp
+++ b/media/codec2/sfplugin/CCodecConfig.cpp
@@ -938,6 +938,14 @@
return value == 0 ? C2_FALSE : C2_TRUE;
}));
+ add(ConfigMapper("android._trigger-tunnel-peek", C2_PARAMKEY_TUNNEL_START_RENDER, "value")
+ .limitTo(D::PARAM & D::VIDEO & D::DECODER)
+ .withMapper([](C2Value v) -> C2Value {
+ int32_t value = 0;
+ (void)v.get(&value);
+ return value == 0 ? C2_FALSE : C2_TRUE;
+ }));
+
/* still to do
constexpr char KEY_PUSH_BLANK_BUFFERS_ON_STOP[] = "push-blank-buffers-on-shutdown";
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 6bf7612..061bb85 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -410,6 +410,7 @@
kWhatSignaledInputEOS = 'seos',
kWhatOutputFramesRendered = 'outR',
kWhatOutputBuffersChanged = 'outC',
+ kWhatFirstTunnelFrameReady = 'ftfR',
};
class BufferCallback : public CodecBase::BufferCallback {
@@ -472,6 +473,7 @@
virtual void onSignaledInputEOS(status_t err) override;
virtual void onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) override;
virtual void onOutputBuffersChanged() override;
+ virtual void onFirstTunnelFrameReady() override;
private:
const sp<AMessage> mNotify;
};
@@ -592,6 +594,12 @@
notify->post();
}
+void CodecCallback::onFirstTunnelFrameReady() {
+ sp<AMessage> notify(mNotify->dup());
+ notify->setInt32("what", kWhatFirstTunnelFrameReady);
+ notify->post();
+}
+
} // namespace
////////////////////////////////////////////////////////////////////////////////
@@ -690,6 +698,7 @@
mTunneledInputWidth(0),
mTunneledInputHeight(0),
mTunneled(false),
+ mTunnelPeekState(TunnelPeekState::kEnabledNoBuffer),
mHaveInputSurface(false),
mHavePendingInputBuffers(false),
mCpuBoostRequested(false),
@@ -879,6 +888,65 @@
}
}
+constexpr const char *MediaCodec::asString(TunnelPeekState state, const char *default_string){
+ switch(state) {
+ case TunnelPeekState::kEnabledNoBuffer:
+ return "EnabledNoBuffer";
+ case TunnelPeekState::kDisabledNoBuffer:
+ return "DisabledNoBuffer";
+ case TunnelPeekState::kBufferDecoded:
+ return "BufferDecoded";
+ case TunnelPeekState::kBufferRendered:
+ return "BufferRendered";
+ case TunnelPeekState::kDisabledQueued:
+ return "DisabledQueued";
+ case TunnelPeekState::kEnabledQueued:
+ return "EnabledQueued";
+ default:
+ return default_string;
+ }
+}
+
+void MediaCodec::updateTunnelPeek(const sp<AMessage> &msg) {
+ int32_t tunnelPeek = 0;
+ if (!msg->findInt32("tunnel-peek", &tunnelPeek)){
+ return;
+ }
+
+ TunnelPeekState previousState = mTunnelPeekState;
+ if(tunnelPeek == 0){
+ switch (mTunnelPeekState) {
+ case TunnelPeekState::kEnabledNoBuffer:
+ mTunnelPeekState = TunnelPeekState::kDisabledNoBuffer;
+ break;
+ case TunnelPeekState::kEnabledQueued:
+ mTunnelPeekState = TunnelPeekState::kDisabledQueued;
+ break;
+ default:
+ ALOGV("Ignoring tunnel-peek=%d for %s", tunnelPeek, asString(mTunnelPeekState));
+ return;
+ }
+ } else {
+ switch (mTunnelPeekState) {
+ case TunnelPeekState::kDisabledNoBuffer:
+ mTunnelPeekState = TunnelPeekState::kEnabledNoBuffer;
+ break;
+ case TunnelPeekState::kDisabledQueued:
+ mTunnelPeekState = TunnelPeekState::kEnabledQueued;
+ break;
+ case TunnelPeekState::kBufferDecoded:
+ msg->setInt32("android._trigger-tunnel-peek", 1);
+ mTunnelPeekState = TunnelPeekState::kBufferRendered;
+ break;
+ default:
+ ALOGV("Ignoring tunnel-peek=%d for %s", tunnelPeek, asString(mTunnelPeekState));
+ return;
+ }
+ }
+
+ ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(mTunnelPeekState));
+}
+
bool MediaCodec::Histogram::setup(int nbuckets, int64_t width, int64_t floor)
{
if (nbuckets <= 0 || width <= 0) {
@@ -1280,6 +1348,12 @@
return msg->post();
}
+status_t MediaCodec::setOnFirstTunnelFrameReadyNotification(const sp<AMessage> ¬ify) {
+ sp<AMessage> msg = new AMessage(kWhatSetNotification, this);
+ msg->setMessage("first-tunnel-frame-ready", notify);
+ return msg->post();
+}
+
status_t MediaCodec::configure(
const sp<AMessage> &format,
const sp<Surface> &nativeWindow,
@@ -2618,9 +2692,17 @@
case kWhatOutputFramesRendered:
{
- // ignore these in all states except running, and check that we have a
- // notification set
- if (mState == STARTED && mOnFrameRenderedNotification != NULL) {
+ // ignore these in all states except running
+ if (mState != STARTED) {
+ break;
+ }
+ TunnelPeekState previousState = mTunnelPeekState;
+ mTunnelPeekState = TunnelPeekState::kBufferRendered;
+ ALOGV("TunnelPeekState: %s -> %s",
+ asString(previousState),
+ asString(TunnelPeekState::kBufferRendered));
+ // check that we have a notification set
+ if (mOnFrameRenderedNotification != NULL) {
sp<AMessage> notify = mOnFrameRenderedNotification->dup();
notify->setMessage("data", msg);
notify->post();
@@ -2628,6 +2710,48 @@
break;
}
+ case kWhatFirstTunnelFrameReady:
+ {
+ if (mState != STARTED) {
+ break;
+ }
+ TunnelPeekState previousState = mTunnelPeekState;
+ switch(mTunnelPeekState) {
+ case TunnelPeekState::kDisabledNoBuffer:
+ case TunnelPeekState::kDisabledQueued:
+ mTunnelPeekState = TunnelPeekState::kBufferDecoded;
+ ALOGV("First tunnel frame ready");
+ ALOGV("TunnelPeekState: %s -> %s",
+ asString(previousState),
+ asString(mTunnelPeekState));
+ break;
+ case TunnelPeekState::kEnabledNoBuffer:
+ case TunnelPeekState::kEnabledQueued:
+ {
+ sp<AMessage> parameters = new AMessage();
+ parameters->setInt32("android._trigger-tunnel-peek", 1);
+ mCodec->signalSetParameters(parameters);
+ }
+ mTunnelPeekState = TunnelPeekState::kBufferRendered;
+ ALOGV("First tunnel frame ready");
+ ALOGV("TunnelPeekState: %s -> %s",
+ asString(previousState),
+ asString(mTunnelPeekState));
+ break;
+ default:
+ ALOGV("Ignoring first tunnel frame ready, TunnelPeekState: %s",
+ asString(mTunnelPeekState));
+ break;
+ }
+
+ if (mOnFirstTunnelFrameReadyNotification != nullptr) {
+ sp<AMessage> notify = mOnFirstTunnelFrameReadyNotification->dup();
+ notify->setMessage("data", msg);
+ notify->post();
+ }
+ break;
+ }
+
case kWhatFillThisBuffer:
{
/* size_t index = */updateBuffers(kPortIndexInput, msg);
@@ -2844,6 +2968,9 @@
if (msg->findMessage("on-frame-rendered", ¬ify)) {
mOnFrameRenderedNotification = notify;
}
+ if (msg->findMessage("first-tunnel-frame-ready", ¬ify)) {
+ mOnFirstTunnelFrameReadyNotification = notify;
+ }
break;
}
@@ -3085,6 +3212,11 @@
}
sp<AReplyToken> replyID;
CHECK(msg->senderAwaitsResponse(&replyID));
+ TunnelPeekState previousState = mTunnelPeekState;
+ mTunnelPeekState = TunnelPeekState::kEnabledNoBuffer;
+ ALOGV("TunnelPeekState: %s -> %s",
+ asString(previousState),
+ asString(TunnelPeekState::kEnabledNoBuffer));
mReplyID = replyID;
setState(STARTING);
@@ -3517,6 +3649,11 @@
mCodec->signalFlush();
returnBuffersToCodec();
+ TunnelPeekState previousState = mTunnelPeekState;
+ mTunnelPeekState = TunnelPeekState::kEnabledNoBuffer;
+ ALOGV("TunnelPeekState: %s -> %s",
+ asString(previousState),
+ asString(TunnelPeekState::kEnabledNoBuffer));
break;
}
@@ -4058,6 +4195,28 @@
buffer->meta()->setInt32("csd", true);
}
+ if (mTunneled) {
+ TunnelPeekState previousState = mTunnelPeekState;
+ switch(mTunnelPeekState){
+ case TunnelPeekState::kEnabledNoBuffer:
+ buffer->meta()->setInt32("tunnel-first-frame", 1);
+ mTunnelPeekState = TunnelPeekState::kEnabledQueued;
+ ALOGV("TunnelPeekState: %s -> %s",
+ asString(previousState),
+ asString(mTunnelPeekState));
+ break;
+ case TunnelPeekState::kDisabledNoBuffer:
+ buffer->meta()->setInt32("tunnel-first-frame", 1);
+ mTunnelPeekState = TunnelPeekState::kDisabledQueued;
+ ALOGV("TunnelPeekState: %s -> %s",
+ asString(previousState),
+ asString(mTunnelPeekState));
+ break;
+ default:
+ break;
+ }
+ }
+
status_t err = OK;
if (hasCryptoOrDescrambler() && !c2Buffer && !memory) {
AString *errorDetailMsg;
@@ -4426,6 +4585,7 @@
status_t MediaCodec::onSetParameters(const sp<AMessage> ¶ms) {
updateLowLatency(params);
+ updateTunnelPeek(params);
mCodec->signalSetParameters(params);
return OK;
diff --git a/media/libstagefright/include/media/stagefright/CodecBase.h b/media/libstagefright/include/media/stagefright/CodecBase.h
index 2e98fec..efb2f86 100644
--- a/media/libstagefright/include/media/stagefright/CodecBase.h
+++ b/media/libstagefright/include/media/stagefright/CodecBase.h
@@ -178,6 +178,10 @@
* Notify MediaCodec that output buffers are changed.
*/
virtual void onOutputBuffersChanged() = 0;
+ /**
+ * Notify MediaCodec that the first tunnel frame is ready.
+ */
+ virtual void onFirstTunnelFrameReady() = 0;
};
/**
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index 8952376..0660ccf 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -129,6 +129,8 @@
status_t setOnFrameRenderedNotification(const sp<AMessage> ¬ify);
+ status_t setOnFirstTunnelFrameReadyNotification(const sp<AMessage> ¬ify);
+
status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
status_t setInputSurface(const sp<PersistentSurface> &surface);
@@ -367,6 +369,30 @@
bool mOwnedByClient;
};
+ // This type is used to track the tunnel mode video peek state machine:
+ //
+ // DisabledNoBuffer -> EnabledNoBuffer when tunnel-peek = true
+ // DisabledQueued -> EnabledQueued when tunnel-peek = true
+ // DisabledNoBuffer -> DisabledQueued when first frame queued
+ // EnabledNoBuffer -> DisabledNoBuffer when tunnel-peek = false
+ // EnabledQueued -> DisabledQueued when tunnel-peek = false
+ // EnabledNoBuffer -> EnabledQueued when first frame queued
+ // DisabledNoBuffer -> BufferDecoded when kWhatFirstTunnelFrameReady
+ // DisabledQueued -> BufferDecoded when kWhatFirstTunnelFrameReady
+ // EnabledNoBuffer -> BufferDecoded when kWhatFirstTunnelFrameReady
+ // EnabledQueued -> BufferDecoded when kWhatFirstTunnelFrameReady
+ // BufferDecoded -> BufferRendered when kWhatFrameRendered
+ // <all states> -> EnabledNoBuffer when flush
+ // <all states> -> EnabledNoBuffer when stop then configure then start
+ enum struct TunnelPeekState {
+ kDisabledNoBuffer,
+ kEnabledNoBuffer,
+ kDisabledQueued,
+ kEnabledQueued,
+ kBufferDecoded,
+ kBufferRendered,
+ };
+
struct ResourceManagerServiceProxy;
State mState;
@@ -393,12 +419,15 @@
void flushMediametrics();
void updateEphemeralMediametrics(mediametrics_handle_t item);
void updateLowLatency(const sp<AMessage> &msg);
+ constexpr const char *asString(TunnelPeekState state, const char *default_string="?");
+ void updateTunnelPeek(const sp<AMessage> &msg);
sp<AMessage> mOutputFormat;
sp<AMessage> mInputFormat;
sp<AMessage> mCallback;
sp<AMessage> mOnFrameRenderedNotification;
sp<AMessage> mAsyncReleaseCompleteNotification;
+ sp<AMessage> mOnFirstTunnelFrameReadyNotification;
sp<ResourceManagerServiceProxy> mResourceManagerProxy;
@@ -434,6 +463,7 @@
int32_t mTunneledInputWidth;
int32_t mTunneledInputHeight;
bool mTunneled;
+ TunnelPeekState mTunnelPeekState;
sp<IDescrambler> mDescrambler;