Implement video peek for Codec2 based codecs
Bug: 184030186
Test: atest android.media.cts.DecoderTest
Change-Id: I74f0b6de2161f563a80b33f1bb3d6ff961de09b6
diff --git a/media/codec2/core/include/C2Config.h b/media/codec2/core/include/C2Config.h
index 9d9ed70..7caa457 100644
--- a/media/codec2/core/include/C2Config.h
+++ b/media/codec2/core/include/C2Config.h
@@ -262,6 +262,8 @@
kParamIndexTunneledMode, // struct
kParamIndexTunnelHandle, // int32[]
kParamIndexTunnelSystemTime, // int64
+ kParamIndexTunnelHoldRender, // bool
+ kParamIndexTunnelStartRender, // bool
// dmabuf allocator
kParamIndexStoreDmaBufUsage, // store, struct
@@ -2366,6 +2368,31 @@
C2PortTunnelSystemTime;
constexpr char C2_PARAMKEY_OUTPUT_RENDER_TIME[] = "output.render-time";
+
+/**
+ * Tunneled mode video peek signaling flag.
+ *
+ * When a video frame is pushed to the decoder with this parameter set to true,
+ * the decoder must decode the frame, signal partial completion, and hold on the
+ * frame until C2StreamTunnelStartRender is set to true (which resets this
+ * flag). Flush will also result in the frames being returned back to the
+ * client (but not rendered).
+ */
+typedef C2StreamParam<C2Info, C2EasyBoolValue, kParamIndexTunnelHoldRender>
+ C2StreamTunnelHoldRender;
+constexpr char C2_PARAMKEY_TUNNEL_HOLD_RENDER[] = "output.tunnel-hold-render";
+
+/**
+ * Tunneled mode video peek signaling flag.
+ *
+ * Upon receiving this flag, the decoder shall set C2StreamTunnelHoldRender to
+ * false, which shall cause any frames held for rendering to be immediately
+ * displayed, regardless of their timestamps.
+*/
+typedef C2StreamParam<C2Info, C2EasyBoolValue, kParamIndexTunnelStartRender>
+ C2StreamTunnelStartRender;
+constexpr char C2_PARAMKEY_TUNNEL_START_RENDER[] = "output.tunnel-start-render";
+
C2ENUM(C2PlatformConfig::encoding_quality_level_t, uint32_t,
NONE,
S_HANDHELD,
diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp
index ce15a30..16398a4 100644
--- a/media/codec2/sfplugin/CCodec.cpp
+++ b/media/codec2/sfplugin/CCodec.cpp
@@ -673,6 +673,10 @@
mCodec->mCallback->onOutputBuffersChanged();
}
+ void onFirstTunnelFrameReady() override {
+ mCodec->mCallback->onFirstTunnelFrameReady();
+ }
+
private:
CCodec *mCodec;
};
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp
index 3c3b41d..f88408e 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.cpp
+++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp
@@ -209,6 +209,7 @@
int32_t flags = 0;
int32_t tmp = 0;
bool eos = false;
+ bool tunnelFirstFrame = false;
if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
eos = true;
mInputMetEos = true;
@@ -217,6 +218,9 @@
if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
flags |= C2FrameData::FLAG_CODEC_CONFIG;
}
+ if (buffer->meta()->findInt32("tunnel-first-frame", &tmp) && tmp) {
+ tunnelFirstFrame = true;
+ }
ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
std::list<std::unique_ptr<C2Work>> items;
std::unique_ptr<C2Work> work(new C2Work);
@@ -288,6 +292,13 @@
// TODO: fill info's
work->input.configUpdate = std::move(mParamsToBeSet);
+ if (tunnelFirstFrame) {
+ C2StreamTunnelHoldRender::input tunnelHoldRender{
+ 0u /* stream */,
+ C2_TRUE /* value */
+ };
+ work->input.configUpdate.push_back(C2Param::Copy(tunnelHoldRender));
+ }
work->worklets.clear();
work->worklets.emplace_back(new C2Worklet);
@@ -1724,6 +1735,15 @@
}
break;
}
+ case C2StreamTunnelHoldRender::CORE_INDEX: {
+ C2StreamTunnelHoldRender::output firstTunnelFrameHoldRender;
+ if (!(worklet->output.flags & C2FrameData::FLAG_INCOMPLETE)) break;
+ if (!firstTunnelFrameHoldRender.updateFrom(*param)) break;
+ if (firstTunnelFrameHoldRender.value != C2_TRUE) break;
+ ALOGV("[%s] onWorkDone: first tunnel frame ready", mName);
+ mCCodecCallback->onFirstTunnelFrameReady();
+ break;
+ }
default:
ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
mName, param->index());
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.h b/media/codec2/sfplugin/CCodecBufferChannel.h
index 45da003..5a2aca2 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.h
+++ b/media/codec2/sfplugin/CCodecBufferChannel.h
@@ -45,6 +45,7 @@
virtual void onError(status_t err, enum ActionCode actionCode) = 0;
virtual void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) = 0;
virtual void onOutputBuffersChanged() = 0;
+ virtual void onFirstTunnelFrameReady() = 0;
};
/**
diff --git a/media/codec2/sfplugin/CCodecConfig.cpp b/media/codec2/sfplugin/CCodecConfig.cpp
index 27e87e6..2df0ba2 100644
--- a/media/codec2/sfplugin/CCodecConfig.cpp
+++ b/media/codec2/sfplugin/CCodecConfig.cpp
@@ -938,6 +938,14 @@
return value == 0 ? C2_FALSE : C2_TRUE;
}));
+ add(ConfigMapper("android._trigger-tunnel-peek", C2_PARAMKEY_TUNNEL_START_RENDER, "value")
+ .limitTo(D::PARAM & D::VIDEO & D::DECODER)
+ .withMapper([](C2Value v) -> C2Value {
+ int32_t value = 0;
+ (void)v.get(&value);
+ return value == 0 ? C2_FALSE : C2_TRUE;
+ }));
+
/* still to do
constexpr char KEY_PUSH_BLANK_BUFFERS_ON_STOP[] = "push-blank-buffers-on-shutdown";
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 8fa7463..1986272 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -976,6 +976,10 @@
return "BufferDecoded";
case TunnelPeekState::kBufferRendered:
return "BufferRendered";
+ case TunnelPeekState::kDisabledQueued:
+ return "DisabledQueued";
+ case TunnelPeekState::kEnabledQueued:
+ return "EnabledQueued";
default:
return default_string;
}
@@ -986,25 +990,39 @@
if (!msg->findInt32("tunnel-peek", &tunnelPeek)){
return;
}
+
+ TunnelPeekState previousState = mTunnelPeekState;
if(tunnelPeek == 0){
- if (mTunnelPeekState == TunnelPeekState::kEnabledNoBuffer) {
- mTunnelPeekState = TunnelPeekState::kDisabledNoBuffer;
- ALOGV("TunnelPeekState: %s -> %s",
- asString(TunnelPeekState::kEnabledNoBuffer),
- asString(TunnelPeekState::kDisabledNoBuffer));
- return;
+ switch (mTunnelPeekState) {
+ case TunnelPeekState::kEnabledNoBuffer:
+ mTunnelPeekState = TunnelPeekState::kDisabledNoBuffer;
+ break;
+ case TunnelPeekState::kEnabledQueued:
+ mTunnelPeekState = TunnelPeekState::kDisabledQueued;
+ break;
+ default:
+ ALOGV("Ignoring tunnel-peek=%d for %s", tunnelPeek, asString(mTunnelPeekState));
+ return;
}
} else {
- if (mTunnelPeekState == TunnelPeekState::kDisabledNoBuffer) {
- mTunnelPeekState = TunnelPeekState::kEnabledNoBuffer;
- ALOGV("TunnelPeekState: %s -> %s",
- asString(TunnelPeekState::kDisabledNoBuffer),
- asString(TunnelPeekState::kEnabledNoBuffer));
- return;
+ switch (mTunnelPeekState) {
+ case TunnelPeekState::kDisabledNoBuffer:
+ mTunnelPeekState = TunnelPeekState::kEnabledNoBuffer;
+ break;
+ case TunnelPeekState::kDisabledQueued:
+ mTunnelPeekState = TunnelPeekState::kEnabledQueued;
+ break;
+ case TunnelPeekState::kBufferDecoded:
+ msg->setInt32("android._trigger-tunnel-peek", 1);
+ mTunnelPeekState = TunnelPeekState::kBufferRendered;
+ break;
+ default:
+ ALOGV("Ignoring tunnel-peek=%d for %s", tunnelPeek, asString(mTunnelPeekState));
+ return;
}
}
- ALOGV("Ignoring tunnel-peek=%d for %s", tunnelPeek, asString(mTunnelPeekState));
+ ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(mTunnelPeekState));
}
void MediaCodec::updatePlaybackDuration(const sp<AMessage> &msg) {
@@ -3294,25 +3312,32 @@
if (mState != STARTED) {
break;
}
+ TunnelPeekState previousState = mTunnelPeekState;
switch(mTunnelPeekState) {
case TunnelPeekState::kDisabledNoBuffer:
+ case TunnelPeekState::kDisabledQueued:
mTunnelPeekState = TunnelPeekState::kBufferDecoded;
+ ALOGV("First tunnel frame ready");
ALOGV("TunnelPeekState: %s -> %s",
- asString(TunnelPeekState::kDisabledNoBuffer),
- asString(TunnelPeekState::kBufferDecoded));
+ asString(previousState),
+ asString(mTunnelPeekState));
break;
case TunnelPeekState::kEnabledNoBuffer:
- mTunnelPeekState = TunnelPeekState::kBufferDecoded;
- ALOGV("TunnelPeekState: %s -> %s",
- asString(TunnelPeekState::kEnabledNoBuffer),
- asString(TunnelPeekState::kBufferDecoded));
+ case TunnelPeekState::kEnabledQueued:
{
sp<AMessage> parameters = new AMessage();
parameters->setInt32("android._trigger-tunnel-peek", 1);
mCodec->signalSetParameters(parameters);
}
+ mTunnelPeekState = TunnelPeekState::kBufferRendered;
+ ALOGV("First tunnel frame ready");
+ ALOGV("TunnelPeekState: %s -> %s",
+ asString(previousState),
+ asString(mTunnelPeekState));
break;
default:
+ ALOGV("Ignoring first tunnel frame ready, TunnelPeekState: %s",
+ asString(mTunnelPeekState));
break;
}
@@ -4777,6 +4802,28 @@
buffer->meta()->setInt32("csd", true);
}
+ if (mTunneled) {
+ TunnelPeekState previousState = mTunnelPeekState;
+ switch(mTunnelPeekState){
+ case TunnelPeekState::kEnabledNoBuffer:
+ buffer->meta()->setInt32("tunnel-first-frame", 1);
+ mTunnelPeekState = TunnelPeekState::kEnabledQueued;
+ ALOGV("TunnelPeekState: %s -> %s",
+ asString(previousState),
+ asString(mTunnelPeekState));
+ break;
+ case TunnelPeekState::kDisabledNoBuffer:
+ buffer->meta()->setInt32("tunnel-first-frame", 1);
+ mTunnelPeekState = TunnelPeekState::kDisabledQueued;
+ ALOGV("TunnelPeekState: %s -> %s",
+ asString(previousState),
+ asString(mTunnelPeekState));
+ break;
+ default:
+ break;
+ }
+ }
+
status_t err = OK;
if (hasCryptoOrDescrambler() && !c2Buffer && !memory) {
AString *errorDetailMsg;
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index 0e6f0b3..d372140 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -377,15 +377,23 @@
// This type is used to track the tunnel mode video peek state machine:
//
// DisabledNoBuffer -> EnabledNoBuffer when tunnel-peek = true
+ // DisabledQueued -> EnabledQueued when tunnel-peek = true
+ // DisabledNoBuffer -> DisabledQueued when first frame queued
// EnabledNoBuffer -> DisabledNoBuffer when tunnel-peek = false
+ // EnabledQueued -> DisabledQueued when tunnel-peek = false
+ // EnabledNoBuffer -> EnabledQueued when first frame queued
// DisabledNoBuffer -> BufferDecoded when kWhatFirstTunnelFrameReady
+ // DisabledQueued -> BufferDecoded when kWhatFirstTunnelFrameReady
// EnabledNoBuffer -> BufferDecoded when kWhatFirstTunnelFrameReady
+ // EnabledQueued -> BufferDecoded when kWhatFirstTunnelFrameReady
// BufferDecoded -> BufferRendered when kWhatFrameRendered
// <all states> -> EnabledNoBuffer when flush
// <all states> -> EnabledNoBuffer when stop then configure then start
enum struct TunnelPeekState {
kDisabledNoBuffer,
kEnabledNoBuffer,
+ kDisabledQueued,
+ kEnabledQueued,
kBufferDecoded,
kBufferRendered,
};