CCodec: fix input surface throttling
There were no throttling in place for video encoders set up with
input surfaces. Throttle based on the number of output buffers
notified to the client.
Bug: 310807188
Test: atest android.hardware.camera2.cts.RecordingTest
Test: atest android.mediav2.cts.CodecEncoderSurfaceTest
Change-Id: Ifec1d16ed25f8100ae1f1eb06a490c955c0c4961
diff --git a/media/aconfig/codec_fwk.aconfig b/media/aconfig/codec_fwk.aconfig
index a2b6a82..f6c9d0b 100644
--- a/media/aconfig/codec_fwk.aconfig
+++ b/media/aconfig/codec_fwk.aconfig
@@ -44,6 +44,16 @@
}
flag {
+ name: "input_surface_throttle"
+ namespace: "codec_fwk"
+ description: "Bugfix flag for input surface throttle"
+ bug: "342269852"
+ metadata {
+ purpose: PURPOSE_BUGFIX
+ }
+}
+
+flag {
name: "large_audio_frame_finish"
namespace: "codec_fwk"
description: "Implementation flag for large audio frame finishing tasks"
diff --git a/media/codec2/sfplugin/C2AidlNode.cpp b/media/codec2/sfplugin/C2AidlNode.cpp
index 93c9d8b..0f23205 100644
--- a/media/codec2/sfplugin/C2AidlNode.cpp
+++ b/media/codec2/sfplugin/C2AidlNode.cpp
@@ -105,6 +105,10 @@
return mImpl->onInputBufferDone(index);
}
+void C2AidlNode::onInputBufferEmptied() {
+ return mImpl->onInputBufferEmptied();
+}
+
android_dataspace C2AidlNode::getDataspace() {
return mImpl->getDataspace();
}
diff --git a/media/codec2/sfplugin/C2AidlNode.h b/media/codec2/sfplugin/C2AidlNode.h
index 365a41d..9dd3504 100644
--- a/media/codec2/sfplugin/C2AidlNode.h
+++ b/media/codec2/sfplugin/C2AidlNode.h
@@ -68,13 +68,19 @@
void setFrameSize(uint32_t width, uint32_t height);
/**
- * Clean up work item reference.
+ * Notify that the input buffer reference is no longer needed by the component.
+ * Clean up if necessary.
*
* \param index input work index
*/
void onInputBufferDone(c2_cntr64_t index);
/**
+ * Notify input buffer is emptied.
+ */
+ void onInputBufferEmptied();
+
+ /**
* Returns dataspace information from GraphicBufferSource.
*/
android_dataspace getDataspace();
diff --git a/media/codec2/sfplugin/C2NodeImpl.cpp b/media/codec2/sfplugin/C2NodeImpl.cpp
index 6f53e0f..585072d 100644
--- a/media/codec2/sfplugin/C2NodeImpl.cpp
+++ b/media/codec2/sfplugin/C2NodeImpl.cpp
@@ -25,6 +25,7 @@
#include <C2Debug.h>
#include <C2PlatformSupport.h>
+#include <android_media_codec.h>
#include <android/fdsan.h>
#include <media/stagefright/foundation/ColorUtils.h>
#include <ui/Fence.h>
@@ -373,7 +374,10 @@
}
work->worklets.clear();
work->worklets.emplace_back(new C2Worklet);
- mBufferIdsInUse.lock()->emplace(work->input.ordinal.frameIndex.peeku(), buffer);
+ {
+ Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
+ buffers->mIdsInUse.emplace(work->input.ordinal.frameIndex.peeku(), buffer);
+ }
mQueueThread->queue(comp, fenceFd, std::move(work), std::move(fd0), std::move(fd1));
return OK;
@@ -405,29 +409,74 @@
}
void C2NodeImpl::onInputBufferDone(c2_cntr64_t index) {
- if (mAidlHal) {
- if (!mAidlBufferSource) {
- ALOGD("Buffer source not set (index=%llu)", index.peekull());
- return;
- }
- } else {
- if (!mBufferSource) {
- ALOGD("Buffer source not set (index=%llu)", index.peekull());
- return;
- }
- }
-
- int32_t bufferId = 0;
- {
- decltype(mBufferIdsInUse)::Locked bufferIds(mBufferIdsInUse);
- auto it = bufferIds->find(index.peeku());
- if (it == bufferIds->end()) {
+ if (android::media::codec::provider_->input_surface_throttle()) {
+ Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
+ auto it = buffers->mIdsInUse.find(index.peeku());
+ if (it == buffers->mIdsInUse.end()) {
ALOGV("Untracked input index %llu (maybe already removed)", index.peekull());
return;
}
- bufferId = it->second;
- (void)bufferIds->erase(it);
+ int32_t bufferId = it->second;
+ (void)buffers->mIdsInUse.erase(it);
+ buffers->mAvailableIds.push_back(bufferId);
+ } else {
+ if (!hasBufferSource()) {
+ return;
+ }
+ int32_t bufferId = 0;
+ {
+ Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
+ auto it = buffers->mIdsInUse.find(index.peeku());
+ if (it == buffers->mIdsInUse.end()) {
+ ALOGV("Untracked input index %llu (maybe already removed)", index.peekull());
+ return;
+ }
+ bufferId = it->second;
+ (void)buffers->mIdsInUse.erase(it);
+ }
+ notifyInputBufferEmptied(bufferId);
}
+}
+
+void C2NodeImpl::onInputBufferEmptied() {
+ if (!android::media::codec::provider_->input_surface_throttle()) {
+ ALOGE("onInputBufferEmptied should not be called "
+ "when input_surface_throttle is false");
+ return;
+ }
+ if (!hasBufferSource()) {
+ return;
+ }
+ int32_t bufferId = 0;
+ {
+ Mutexed<BuffersTracker>::Locked buffers(mBuffersTracker);
+ if (buffers->mAvailableIds.empty()) {
+ ALOGV("The codec is ready to take more input buffers "
+ "but no input buffers are ready yet.");
+ return;
+ }
+ bufferId = buffers->mAvailableIds.front();
+ buffers->mAvailableIds.pop_front();
+ }
+ notifyInputBufferEmptied(bufferId);
+}
+
+bool C2NodeImpl::hasBufferSource() {
+ if (mAidlHal) {
+ if (!mAidlBufferSource) {
+ ALOGD("Buffer source not set");
+ return false;
+ }
+ } else {
+ if (!mBufferSource) {
+ ALOGD("Buffer source not set");
+ return false;
+ }
+ }
+ return true;
+}
+
+void C2NodeImpl::notifyInputBufferEmptied(int32_t bufferId) {
if (mAidlHal) {
::ndk::ScopedFileDescriptor nullFence;
(void)mAidlBufferSource->onInputBufferEmptied(bufferId, nullFence);
diff --git a/media/codec2/sfplugin/C2NodeImpl.h b/media/codec2/sfplugin/C2NodeImpl.h
index e060fd8..cc826b4 100644
--- a/media/codec2/sfplugin/C2NodeImpl.h
+++ b/media/codec2/sfplugin/C2NodeImpl.h
@@ -73,13 +73,19 @@
void setFrameSize(uint32_t width, uint32_t height);
/**
- * Clean up work item reference.
+ * Notify that the input buffer reference is no longer needed by the component.
+ * Clean up if necessary.
*
* \param index input work index
*/
void onInputBufferDone(c2_cntr64_t index);
/**
+ * Notify input buffer is emptied.
+ */
+ void onInputBufferEmptied();
+
+ /**
* Returns dataspace information from GraphicBufferSource.
*/
android_dataspace getDataspace();
@@ -118,12 +124,24 @@
c2_cntr64_t mPrevInputTimestamp; // input timestamp for previous frame
c2_cntr64_t mPrevCodecTimestamp; // adjusted (codec) timestamp for previous frame
- Mutexed<std::map<uint64_t, uint32_t>> mBufferIdsInUse;
+ // Tracks the status of buffers
+ struct BuffersTracker {
+ BuffersTracker() = default;
+
+ // Keeps track of buffers that are used by the component. Maps timestamp -> ID
+ std::map<uint64_t, uint32_t> mIdsInUse;
+ // Keeps track of the buffer IDs that are available after being released from the component.
+ std::list<uint32_t> mAvailableIds;
+ };
+ Mutexed<BuffersTracker> mBuffersTracker;
class QueueThread;
sp<QueueThread> mQueueThread;
bool mAidlHal;
+
+ bool hasBufferSource();
+ void notifyInputBufferEmptied(int32_t bufferId);
};
} // namespace android
diff --git a/media/codec2/sfplugin/C2OMXNode.cpp b/media/codec2/sfplugin/C2OMXNode.cpp
index ce02c88..98e25e2 100644
--- a/media/codec2/sfplugin/C2OMXNode.cpp
+++ b/media/codec2/sfplugin/C2OMXNode.cpp
@@ -291,6 +291,10 @@
return mImpl->onInputBufferDone(index);
}
+void C2OMXNode::onInputBufferEmptied() {
+ return mImpl->onInputBufferEmptied();
+}
+
android_dataspace C2OMXNode::getDataspace() {
return mImpl->getDataspace();
}
diff --git a/media/codec2/sfplugin/C2OMXNode.h b/media/codec2/sfplugin/C2OMXNode.h
index d077202..5549b88 100644
--- a/media/codec2/sfplugin/C2OMXNode.h
+++ b/media/codec2/sfplugin/C2OMXNode.h
@@ -86,13 +86,19 @@
void setFrameSize(uint32_t width, uint32_t height);
/**
- * Clean up work item reference.
+ * Notify that the input buffer reference is no longer needed by the component.
+ * Clean up if necessary.
*
* \param index input work index
*/
void onInputBufferDone(c2_cntr64_t index);
/**
+ * Notify input buffer is emptied.
+ */
+ void onInputBufferEmptied();
+
+ /**
* Returns dataspace information from GraphicBufferSource.
*/
android_dataspace getDataspace();
diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp
index 463b63f..bc69688 100644
--- a/media/codec2/sfplugin/CCodec.cpp
+++ b/media/codec2/sfplugin/CCodec.cpp
@@ -444,6 +444,10 @@
mNode->onInputBufferDone(index);
}
+ void onInputBufferEmptied() override {
+ mNode->onInputBufferEmptied();
+ }
+
android_dataspace getDataspace() override {
return mNode->getDataspace();
}
@@ -663,6 +667,10 @@
mNode->onInputBufferDone(index);
}
+ void onInputBufferEmptied() override {
+ mNode->onInputBufferEmptied();
+ }
+
android_dataspace getDataspace() override {
return mNode->getDataspace();
}
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp
index 3984b83..10c51d1 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.cpp
+++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp
@@ -1069,6 +1069,10 @@
return;
}
}
+ if (android::media::codec::provider_->input_surface_throttle()
+ && mInputSurface != nullptr) {
+ mInputSurface->onInputBufferEmptied();
+ }
size_t numActiveSlots = 0;
while (!mPipelineWatcher.lock()->pipelineFull()) {
sp<MediaCodecBuffer> inBuffer;
diff --git a/media/codec2/sfplugin/InputSurfaceWrapper.h b/media/codec2/sfplugin/InputSurfaceWrapper.h
index 4bf6cd0..c158c5b 100644
--- a/media/codec2/sfplugin/InputSurfaceWrapper.h
+++ b/media/codec2/sfplugin/InputSurfaceWrapper.h
@@ -102,6 +102,7 @@
}
/**
+ * Notify that the input buffer reference is no longer needed.
* Clean up C2Work related references if necessary. No-op by default.
*
* \param index index of input work.
@@ -109,6 +110,12 @@
virtual void onInputBufferDone(c2_cntr64_t /* index */) {}
/**
+ * Signal one input buffer as emptied.
+ * No-op by default.
+ */
+ virtual void onInputBufferEmptied() {}
+
+ /**
* Returns dataspace information from GraphicBufferSource.
*/
virtual android_dataspace getDataspace() { return mDataSpace; }