MediaCodec: Use IProducerListener and MediaCodec's generation number
Use MediaCodec's generation number. And use IProducerListener when
connect to the surface if Codec implementation provides the listener.
Bug: 254050314
Test: atest android.media.decoder.cts.AdaptivePlaybackTest
Change-Id: If04c1b7be1cc7db39099c57a8905a702113833c5
diff --git a/media/codec2/hal/client/client.cpp b/media/codec2/hal/client/client.cpp
index ce10109..597852a 100644
--- a/media/codec2/hal/client/client.cpp
+++ b/media/codec2/hal/client/client.cpp
@@ -2370,6 +2370,11 @@
mOutputBufferQueue->expireOldWaiters();
}
+void Codec2Client::Component::onBufferReleasedFromOutputSurface(
+ uint32_t generation) {
+ (void) generation;
+}
+
c2_status_t Codec2Client::Component::connectToInputSurface(
const std::shared_ptr<InputSurface>& inputSurface,
std::shared_ptr<InputSurfaceConnection>* connection) {
diff --git a/media/codec2/hal/client/include/codec2/hidl/client.h b/media/codec2/hal/client/include/codec2/hidl/client.h
index 0c7dd77..a8dd0bf 100644
--- a/media/codec2/hal/client/include/codec2/hidl/client.h
+++ b/media/codec2/hal/client/include/codec2/hidl/client.h
@@ -474,6 +474,10 @@
void stopUsingOutputSurface(
C2BlockPool::local_id_t blockPoolId);
+ // Notify a buffer is released from output surface.
+ void onBufferReleasedFromOutputSurface(
+ uint32_t generation);
+
// Connect to a given InputSurface.
c2_status_t connectToInputSurface(
const std::shared_ptr<InputSurface>& inputSurface,
diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp
index cd3a80f..c102c4b 100644
--- a/media/codec2/sfplugin/CCodec.cpp
+++ b/media/codec2/sfplugin/CCodec.cpp
@@ -862,6 +862,8 @@
sp<Surface> surface;
if (msg->findObject("native-window", &obj)) {
surface = static_cast<Surface *>(obj.get());
+ int32_t generation;
+ (void)msg->findInt32("native-window-generation", &generation);
// setup tunneled playback
if (surface != nullptr) {
Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
@@ -896,7 +898,7 @@
}
}
}
- setSurface(surface);
+ setSurface(surface, (uint32_t)generation);
}
Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
@@ -2033,7 +2035,7 @@
}
}
-status_t CCodec::setSurface(const sp<Surface> &surface) {
+status_t CCodec::setSurface(const sp<Surface> &surface, uint32_t generation) {
bool pushBlankBuffer = false;
{
Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
@@ -2062,7 +2064,7 @@
}
pushBlankBuffer = config->mPushBlankBuffersOnStop;
}
- return mChannel->setSurface(surface, pushBlankBuffer);
+ return mChannel->setSurface(surface, generation, pushBlankBuffer);
}
void CCodec::signalFlush() {
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp
index bddf3cb..f89e889 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.cpp
+++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp
@@ -1121,6 +1121,10 @@
processRenderedFrames(delta);
}
+void CCodecBufferChannel::onBufferReleasedFromOutputSurface(uint32_t generation) {
+ mComponent->onBufferReleasedFromOutputSurface(generation);
+}
+
status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
bool released = false;
@@ -2265,12 +2269,8 @@
}
}
-status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface, bool pushBlankBuffer) {
- static std::atomic_uint32_t surfaceGeneration{0};
- uint32_t generation = (getpid() << 10) |
- ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
- & ((1 << 10) - 1));
-
+status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface,
+ uint32_t generation, bool pushBlankBuffer) {
sp<IGraphicBufferProducer> producer;
int maxDequeueCount;
sp<Surface> oldSurface;
@@ -2284,7 +2284,6 @@
newSurface->setDequeueTimeout(kDequeueTimeoutNs);
newSurface->setMaxDequeuedBufferCount(maxDequeueCount);
producer = newSurface->getIGraphicBufferProducer();
- producer->setGenerationNumber(generation);
} else {
ALOGE("[%s] setting output surface to null", mName);
return INVALID_OPERATION;
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.h b/media/codec2/sfplugin/CCodecBufferChannel.h
index 2cdf59d..f23cb83 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.h
+++ b/media/codec2/sfplugin/CCodecBufferChannel.h
@@ -91,6 +91,7 @@
status_t renderOutputBuffer(
const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) override;
void pollForRenderedBuffers() override;
+ void onBufferReleasedFromOutputSurface(uint32_t generation) override;
status_t discardBuffer(const sp<MediaCodecBuffer> &buffer) override;
void getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) override;
void getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) override;
@@ -105,7 +106,7 @@
/**
* Set output graphic surface for rendering.
*/
- status_t setSurface(const sp<Surface> &surface, bool pushBlankBuffer);
+ status_t setSurface(const sp<Surface> &surface, uint32_t generation, bool pushBlankBuffer);
/**
* Set GraphicBufferSource object from which the component extracts input
diff --git a/media/codec2/sfplugin/include/media/stagefright/CCodec.h b/media/codec2/sfplugin/include/media/stagefright/CCodec.h
index 13713bc..ffae08d 100644
--- a/media/codec2/sfplugin/include/media/stagefright/CCodec.h
+++ b/media/codec2/sfplugin/include/media/stagefright/CCodec.h
@@ -56,7 +56,7 @@
virtual void initiateStart() override;
virtual void initiateShutdown(bool keepComponentAllocated = false) override;
- virtual status_t setSurface(const sp<Surface> &surface) override;
+ virtual status_t setSurface(const sp<Surface> &surface, uint32_t generation) override;
virtual void signalFlush() override;
virtual void signalResume() override;
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index a91b24a..75a1ac9 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -657,7 +657,7 @@
msg->post();
}
-status_t ACodec::setSurface(const sp<Surface> &surface) {
+status_t ACodec::setSurface(const sp<Surface> &surface, uint32_t /*generation*/) {
sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
msg->setObject("surface", surface);
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index a18d1d3..617a2cf 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -822,6 +822,37 @@
const sp<AMessage> mNotify;
};
+class OnBufferReleasedListener : public ::android::BnProducerListener{
+private:
+ uint32_t mGeneration;
+ std::weak_ptr<BufferChannelBase> mBufferChannel;
+
+ void notifyBufferReleased() {
+ auto p = mBufferChannel.lock();
+ if (p) {
+ p->onBufferReleasedFromOutputSurface(mGeneration);
+ }
+ }
+
+public:
+ explicit OnBufferReleasedListener(
+ uint32_t generation,
+ const std::shared_ptr<BufferChannelBase> &bufferChannel)
+ : mGeneration(generation), mBufferChannel(bufferChannel) {}
+
+ virtual ~OnBufferReleasedListener() = default;
+
+ void onBufferReleased() override {
+ notifyBufferReleased();
+ }
+
+ void onBufferDetached([[maybe_unused]] int slot) override {
+ notifyBufferReleased();
+ }
+
+ bool needsReleaseNotify() override { return true; }
+};
+
class BufferCallback : public CodecBase::BufferCallback {
public:
explicit BufferCallback(const sp<AMessage> ¬ify);
@@ -4681,6 +4712,8 @@
PostReplyWithError(replyID, err);
break;
}
+ uint32_t generation = mSurfaceGeneration;
+ format->setInt32("native-window-generation", generation);
} else {
// we are not using surface so this variable is not used, but initialize sensibly anyway
mAllowFrameDroppingBySurface = false;
@@ -4809,7 +4842,8 @@
mErrorLog.log(LOG_TAG, "Unsetting surface is not supported");
err = BAD_VALUE;
} else {
- err = connectToSurface(surface);
+ uint32_t generation;
+ err = connectToSurface(surface, &generation);
if (err == ALREADY_EXISTS) {
// reconnecting to same surface
err = OK;
@@ -4824,12 +4858,13 @@
mSoftRenderer = new SoftwareRenderer(surface);
// TODO: check if this was successful
} else {
- err = mCodec->setSurface(surface);
+ err = mCodec->setSurface(surface, generation);
}
}
if (err == OK) {
(void)disconnectFromSurface();
mSurface = surface;
+ mSurfaceGeneration = generation;
}
mReliabilityContextMetrics.setOutputSurfaceCount++;
}
@@ -5067,15 +5102,17 @@
mReleaseSurface.reset(new ReleaseSurface(usage));
}
if (mSurface != mReleaseSurface->getSurface()) {
- status_t err = connectToSurface(mReleaseSurface->getSurface());
+ uint32_t generation;
+ status_t err = connectToSurface(mReleaseSurface->getSurface(), &generation);
ALOGW_IF(err != OK, "error connecting to release surface: err = %d", err);
if (err == OK && !(mFlags & kFlagUsesSoftwareRenderer)) {
- err = mCodec->setSurface(mReleaseSurface->getSurface());
+ err = mCodec->setSurface(mReleaseSurface->getSurface(), generation);
ALOGW_IF(err != OK, "error setting release surface: err = %d", err);
}
if (err == OK) {
(void)disconnectFromSurface();
mSurface = mReleaseSurface->getSurface();
+ mSurfaceGeneration = generation;
} else {
// We were not able to switch the surface, so force
// synchronous release.
@@ -6333,7 +6370,7 @@
return index;
}
-status_t MediaCodec::connectToSurface(const sp<Surface> &surface) {
+status_t MediaCodec::connectToSurface(const sp<Surface> &surface, uint32_t *generation) {
status_t err = OK;
if (surface != NULL) {
uint64_t oldId, newId;
@@ -6355,21 +6392,26 @@
// number. Rely on the fact that max supported process id by Linux is 2^22.
// PID is never 0 so we don't have to worry that we use the default generation of 0.
// TODO: come up with a unique scheme if other producers also set the generation number.
- static uint32_t mSurfaceGeneration = 0;
- uint32_t generation = (getpid() << 10) | (++mSurfaceGeneration & ((1 << 10) - 1));
- surface->setGenerationNumber(generation);
- ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), generation);
+ static uint32_t sSurfaceGeneration = 0;
+ *generation = (getpid() << 10) | (++sSurfaceGeneration & ((1 << 10) - 1));
+ surface->setGenerationNumber(*generation);
+ ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), *generation);
// HACK: clear any free buffers. Remove when connect will automatically do this.
// This is needed as the consumer may be holding onto stale frames that it can reattach
// to this surface after disconnect/connect, and those free frames would inherit the new
// generation number. Disconnecting after setting a unique generation prevents this.
nativeWindowDisconnect(surface.get(), "connectToSurface(reconnect)");
- err = nativeWindowConnect(surface.get(), "connectToSurface(reconnect)");
+ sp<IProducerListener> listener =
+ new OnBufferReleasedListener(*generation, mBufferChannel);
+ err = surfaceConnectWithListener(
+ surface, listener, "connectToSurface(reconnect-with-listener)");
}
if (err != OK) {
- ALOGE("nativeWindowConnect returned an error: %s (%d)", strerror(-err), err);
+ *generation = 0;
+ ALOGE("nativeWindowConnect/surfaceConnectWithListener returned an error: %s (%d)",
+ strerror(-err), err);
} else {
if (!mAllowFrameDroppingBySurface) {
disableLegacyBufferDropPostQ(surface);
@@ -6395,6 +6437,7 @@
}
// assume disconnected even on error
mSurface.clear();
+ mSurfaceGeneration = 0;
mIsSurfaceToDisplay = false;
}
return err;
@@ -6406,9 +6449,11 @@
(void)disconnectFromSurface();
}
if (surface != NULL) {
- err = connectToSurface(surface);
+ uint32_t generation;
+ err = connectToSurface(surface, &generation);
if (err == OK) {
mSurface = surface;
+ mSurfaceGeneration = generation;
}
}
return err;
diff --git a/media/libstagefright/SurfaceUtils.cpp b/media/libstagefright/SurfaceUtils.cpp
index 827052d..f16e635 100644
--- a/media/libstagefright/SurfaceUtils.cpp
+++ b/media/libstagefright/SurfaceUtils.cpp
@@ -328,6 +328,16 @@
return err;
}
+status_t surfaceConnectWithListener(
+ const sp<Surface> &surface, sp<IProducerListener> listener, const char *reason) {
+ ALOGD("connecting to surface %p, reason %s", surface.get(), reason);
+
+ status_t err = surface->connect(NATIVE_WINDOW_API_MEDIA, listener);
+ ALOGE_IF(err != OK, "Failed to connect from surface %p, err %d", surface.get(), err);
+
+ return err;
+}
+
status_t nativeWindowDisconnect(ANativeWindow *surface, const char *reason) {
ALOGD("disconnecting from surface %p, reason %s", surface, reason);
diff --git a/media/libstagefright/include/media/stagefright/ACodec.h b/media/libstagefright/include/media/stagefright/ACodec.h
index e535d5d..232cf6b 100644
--- a/media/libstagefright/include/media/stagefright/ACodec.h
+++ b/media/libstagefright/include/media/stagefright/ACodec.h
@@ -83,7 +83,7 @@
const char* mime, bool isEncoder,
MediaCodecInfo::CapabilitiesWriter* caps);
- virtual status_t setSurface(const sp<Surface> &surface);
+ virtual status_t setSurface(const sp<Surface> &surface, uint32_t /*generation*/);
virtual void signalFlush();
virtual void signalResume();
diff --git a/media/libstagefright/include/media/stagefright/CodecBase.h b/media/libstagefright/include/media/stagefright/CodecBase.h
index 2a5989f..9b5b93c 100644
--- a/media/libstagefright/include/media/stagefright/CodecBase.h
+++ b/media/libstagefright/include/media/stagefright/CodecBase.h
@@ -239,7 +239,9 @@
// require an explicit message handler
virtual void onMessageReceived(const sp<AMessage> &msg) = 0;
- virtual status_t setSurface(const sp<Surface>& /*surface*/) { return INVALID_OPERATION; }
+ virtual status_t setSurface(const sp<Surface>& /*surface*/, uint32_t /*generation*/) {
+ return INVALID_OPERATION;
+ }
virtual void signalFlush() = 0;
virtual void signalResume() = 0;
@@ -424,6 +426,15 @@
virtual void pollForRenderedBuffers() = 0;
/**
+ * Notify a buffer is released from output surface.
+ *
+ * @param generation MediaCodec's surface specifier
+ */
+ virtual void onBufferReleasedFromOutputSurface(uint32_t /*generation*/) {
+ // default: no-op
+ };
+
+ /**
* Discard a buffer to the underlying CodecBase object.
*
* TODO: remove once this operation can be handled by just clearing the
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index c1f9bbf..009680a 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -446,6 +446,7 @@
int64_t mPresentationTimeUs = 0;
status_t mStickyError;
sp<Surface> mSurface;
+ uint32_t mSurfaceGeneration = 0;
SoftwareRenderer *mSoftRenderer;
Mutex mMetricsLock;
@@ -614,7 +615,7 @@
status_t queueCSDInputBuffer(size_t bufferIndex);
status_t handleSetSurface(const sp<Surface> &surface);
- status_t connectToSurface(const sp<Surface> &surface);
+ status_t connectToSurface(const sp<Surface> &surface, uint32_t *generation);
status_t disconnectFromSurface();
bool hasCryptoOrDescrambler() {
diff --git a/media/libstagefright/include/media/stagefright/SurfaceUtils.h b/media/libstagefright/include/media/stagefright/SurfaceUtils.h
index 35b3fa2..eccb413 100644
--- a/media/libstagefright/include/media/stagefright/SurfaceUtils.h
+++ b/media/libstagefright/include/media/stagefright/SurfaceUtils.h
@@ -27,6 +27,7 @@
namespace android {
struct HDRStaticInfo;
+class IProducerListener;
/**
* Configures |nativeWindow| for given |width|x|height|, pixel |format|, |rotation| and |usage|.
@@ -43,6 +44,8 @@
status_t pushBlankBuffersToNativeWindow(ANativeWindow *nativeWindow /* nonnull */);
status_t nativeWindowConnect(ANativeWindow *surface, const char *reason);
status_t nativeWindowDisconnect(ANativeWindow *surface, const char *reason);
+status_t surfaceConnectWithListener(const sp<Surface> &surface,
+ sp<IProducerListener> listener, const char *reason);
/**
* Disable buffer dropping behavior of BufferQueue if target sdk of application
diff --git a/media/libstagefright/tests/mediacodec/MediaCodecTest.cpp b/media/libstagefright/tests/mediacodec/MediaCodecTest.cpp
index 71ddbe5..ed01e36 100644
--- a/media/libstagefright/tests/mediacodec/MediaCodecTest.cpp
+++ b/media/libstagefright/tests/mediacodec/MediaCodecTest.cpp
@@ -89,7 +89,8 @@
MOCK_METHOD(void, initiateStart, (), (override));
MOCK_METHOD(void, initiateShutdown, (bool keepComponentAllocated), (override));
MOCK_METHOD(void, onMessageReceived, (const sp<AMessage> &msg), (override));
- MOCK_METHOD(status_t, setSurface, (const sp<Surface> &surface), (override));
+ MOCK_METHOD(
+ status_t, setSurface, (const sp<Surface> &surface, uint32_t generation), (override));
MOCK_METHOD(void, signalFlush, (), (override));
MOCK_METHOD(void, signalResume, (), (override));
MOCK_METHOD(void, signalRequestIDRFrame, (), (override));