Trigger texture update on each submission of input buffer.
When input queue is full (which will happen if there are
no capture requests coming from the camera client), it blocks
until there's buffer freed up.
This is not desired (for example, it causes video decoder
attached to the virtual camera surface to drop frames), and
causes that the subsequent capture request received by camera
won't get the most recent image from the virtual camera owner.
This cl fixes this behavior, by consuming the buffer from
the input queue immediatelly after it's submitted.
Bug: 344572859
Test: atest VirtualCameraCaptureTest
Change-Id: Ic7556921630d56f73bbd861d35c5218351217ee6
diff --git a/services/camera/virtualcamera/VirtualCameraRenderThread.cc b/services/camera/virtualcamera/VirtualCameraRenderThread.cc
index 9a5bd1e..ff09256 100644
--- a/services/camera/virtualcamera/VirtualCameraRenderThread.cc
+++ b/services/camera/virtualcamera/VirtualCameraRenderThread.cc
@@ -24,6 +24,7 @@
#include <memory>
#include <mutex>
#include <thread>
+#include <utility>
#include <vector>
#include "Exif.h"
@@ -78,6 +79,15 @@
namespace {
+// helper type for the visitor
+template <class... Ts>
+struct overloaded : Ts... {
+ using Ts::operator()...;
+};
+// explicit deduction guide (not needed as of C++20)
+template <class... Ts>
+overloaded(Ts...) -> overloaded<Ts...>;
+
using namespace std::chrono_literals;
static constexpr std::chrono::milliseconds kAcquireFenceTimeout = 500ms;
@@ -89,6 +99,8 @@
static constexpr size_t kJpegThumbnailBufferSize = 32 * 1024; // 32 KiB
+static constexpr UpdateTextureTask kUpdateTextureTask;
+
CameraMetadata createCaptureResultMetadata(
const std::chrono::nanoseconds timestamp,
const RequestSettings& requestSettings,
@@ -287,6 +299,21 @@
static_cast<uint64_t>(1e9 / VirtualCameraDevice::kMinFps));
}
+class FrameAvailableListenerProxy : public ConsumerBase::FrameAvailableListener {
+ public:
+ FrameAvailableListenerProxy(std::function<void()> callback)
+ : mOnFrameAvailableCallback(callback) {
+ }
+
+ virtual void onFrameAvailable(const BufferItem&) override {
+ ALOGV("%s: onFrameAvailable", __func__);
+ mOnFrameAvailableCallback();
+ }
+
+ private:
+ std::function<void()> mOnFrameAvailableCallback;
+};
+
} // namespace
CaptureRequestBuffer::CaptureRequestBuffer(int streamId, int bufferId,
@@ -345,9 +372,25 @@
return mRequestSettings;
}
+void VirtualCameraRenderThread::requestTextureUpdate() {
+ std::lock_guard<std::mutex> lock(mLock);
+ // If queue is not empty, we don't need to set the mTextureUpdateRequested
+ // flag, since the texture will be updated during ProcessCaptureRequestTask
+ // processing anyway.
+ if (mQueue.empty()) {
+ mTextureUpdateRequested = true;
+ mCondVar.notify_one();
+ }
+}
+
void VirtualCameraRenderThread::enqueueTask(
std::unique_ptr<ProcessCaptureRequestTask> task) {
std::lock_guard<std::mutex> lock(mLock);
+ // When enqueving process capture request task, clear the
+ // mTextureUpdateRequested flag. If this flag is set, the texture was not yet
+ // updated and it will be updated when processing ProcessCaptureRequestTask
+ // anyway.
+ mTextureUpdateRequested = false;
mQueue.emplace_back(std::move(task));
mCondVar.notify_one();
}
@@ -377,8 +420,7 @@
return mInputSurfaceFuture.get();
}
-std::unique_ptr<ProcessCaptureRequestTask>
-VirtualCameraRenderThread::dequeueTask() {
+RenderThreadTask VirtualCameraRenderThread::dequeueTask() {
std::unique_lock<std::mutex> lock(mLock);
// Clang's thread safety analysis doesn't perform alias analysis,
// so it doesn't support moveable std::unique_lock.
@@ -389,12 +431,20 @@
ScopedLockAssertion lockAssertion(mLock);
mCondVar.wait(lock, [this]() REQUIRES(mLock) {
- return mPendingExit || !mQueue.empty();
+ return mPendingExit || mTextureUpdateRequested || !mQueue.empty();
});
if (mPendingExit) {
- return nullptr;
+ // Render thread task with null task signals render thread to terminate.
+ return RenderThreadTask(nullptr);
}
- std::unique_ptr<ProcessCaptureRequestTask> task = std::move(mQueue.front());
+ if (mTextureUpdateRequested) {
+ // If mTextureUpdateRequested, it's guaranteed the queue is empty, return
+ // kUpdateTextureTask to signal we want render thread to update the texture
+ // (consume buffer from the queue).
+ mTextureUpdateRequested = false;
+ return RenderThreadTask(kUpdateTextureTask);
+ }
+ RenderThreadTask task(std::move(mQueue.front()));
mQueue.pop_front();
return task;
}
@@ -409,11 +459,23 @@
EglTextureProgram::TextureFormat::RGBA);
mEglSurfaceTexture = std::make_unique<EglSurfaceTexture>(
mInputSurfaceSize.width, mInputSurfaceSize.height);
+ sp<FrameAvailableListenerProxy> frameAvailableListener =
+ sp<FrameAvailableListenerProxy>::make(
+ [this]() { requestTextureUpdate(); });
+ mEglSurfaceTexture->setFrameAvailableListener(frameAvailableListener);
mInputSurfacePromise.set_value(mEglSurfaceTexture->getSurface());
- while (std::unique_ptr<ProcessCaptureRequestTask> task = dequeueTask()) {
- processCaptureRequest(*task);
+ while (RenderThreadTask task = dequeueTask()) {
+ std::visit(
+ overloaded{[this](const std::unique_ptr<ProcessCaptureRequestTask>& t) {
+ processTask(*t);
+ },
+ [this](const UpdateTextureTask&) {
+ ALOGV("Idle update of the texture");
+ mEglSurfaceTexture->updateTexture();
+ }},
+ task);
}
// Destroy EGL utilities still on the render thread.
@@ -425,7 +487,7 @@
ALOGV("Render thread exiting");
}
-void VirtualCameraRenderThread::processCaptureRequest(
+void VirtualCameraRenderThread::processTask(
const ProcessCaptureRequestTask& request) {
std::chrono::nanoseconds timestamp =
std::chrono::duration_cast<std::chrono::nanoseconds>(
diff --git a/services/camera/virtualcamera/VirtualCameraRenderThread.h b/services/camera/virtualcamera/VirtualCameraRenderThread.h
index b23c30c..5a5966b 100644
--- a/services/camera/virtualcamera/VirtualCameraRenderThread.h
+++ b/services/camera/virtualcamera/VirtualCameraRenderThread.h
@@ -23,6 +23,7 @@
#include <future>
#include <memory>
#include <thread>
+#include <variant>
#include <vector>
#include "VirtualCameraDevice.h"
@@ -34,7 +35,6 @@
#include "util/EglFramebuffer.h"
#include "util/EglProgram.h"
#include "util/EglSurfaceTexture.h"
-#include "util/MetadataUtil.h"
#include "util/Util.h"
namespace android {
@@ -94,6 +94,24 @@
const RequestSettings mRequestSettings;
};
+struct UpdateTextureTask {};
+
+struct RenderThreadTask
+ : public std::variant<std::unique_ptr<ProcessCaptureRequestTask>,
+ UpdateTextureTask> {
+ // Allow implicit conversion to bool.
+ //
+ // Returns false, if the RenderThreadTask consist of null
+ // ProcessCaptureRequestTask, which signals that the thread should terminate.
+ operator bool() const {
+ const bool isExitSignal =
+ std::holds_alternative<std::unique_ptr<ProcessCaptureRequestTask>>(
+ *this) &&
+ std::get<std::unique_ptr<ProcessCaptureRequestTask>>(*this) == nullptr;
+ return !isExitSignal;
+ }
+};
+
// Wraps dedicated rendering thread and rendering business with corresponding
// input surface.
class VirtualCameraRenderThread {
@@ -120,6 +138,12 @@
// Stop rendering thread.
void stop();
+ // Send request to render thread to update the texture.
+ // Currently queued buffers in the input surface will be consumed and the most
+ // recent buffer in the input surface will be attached to the texture), all
+ // other buffers will be returned to the buffer queue.
+ void requestTextureUpdate() EXCLUDES(mLock);
+
// Equeue capture task for processing on render thread.
void enqueueTask(std::unique_ptr<ProcessCaptureRequestTask> task)
EXCLUDES(mLock);
@@ -131,13 +155,13 @@
sp<Surface> getInputSurface();
private:
- std::unique_ptr<ProcessCaptureRequestTask> dequeueTask() EXCLUDES(mLock);
+ RenderThreadTask dequeueTask() EXCLUDES(mLock);
// Rendering thread entry point.
void threadLoop();
// Process single capture request task (always called on render thread).
- void processCaptureRequest(const ProcessCaptureRequestTask& captureRequestTask);
+ void processTask(const ProcessCaptureRequestTask& captureRequestTask);
// Flush single capture request task returning the error status immediately.
void flushCaptureRequest(const ProcessCaptureRequestTask& captureRequestTask);
@@ -192,6 +216,7 @@
std::mutex mLock;
std::deque<std::unique_ptr<ProcessCaptureRequestTask>> mQueue GUARDED_BY(mLock);
std::condition_variable mCondVar;
+ volatile bool mTextureUpdateRequested GUARDED_BY(mLock);
volatile bool mPendingExit GUARDED_BY(mLock);
// Acquisition timestamp of last frame.
diff --git a/services/camera/virtualcamera/util/EglSurfaceTexture.cc b/services/camera/virtualcamera/util/EglSurfaceTexture.cc
index 7de5020..c81d36d 100644
--- a/services/camera/virtualcamera/util/EglSurfaceTexture.cc
+++ b/services/camera/virtualcamera/util/EglSurfaceTexture.cc
@@ -64,6 +64,11 @@
return mGlConsumer->getCurrentBuffer();
}
+void EglSurfaceTexture::setFrameAvailableListener(
+ const wp<ConsumerBase::FrameAvailableListener>& listener) {
+ mGlConsumer->setFrameAvailableListener(listener);
+}
+
bool EglSurfaceTexture::waitForNextFrame(const std::chrono::nanoseconds timeout) {
return mSurface->waitForNextFrame(mGlConsumer->getFrameNumber(),
static_cast<nsecs_t>(timeout.count()));
diff --git a/services/camera/virtualcamera/util/EglSurfaceTexture.h b/services/camera/virtualcamera/util/EglSurfaceTexture.h
index b9c5126..ac3cf7d 100644
--- a/services/camera/virtualcamera/util/EglSurfaceTexture.h
+++ b/services/camera/virtualcamera/util/EglSurfaceTexture.h
@@ -21,6 +21,7 @@
#include <cstdint>
#include "GLES/gl.h"
+#include "gui/ConsumerBase.h"
#include "gui/Surface.h"
#include "utils/RefBase.h"
@@ -58,6 +59,9 @@
// Returns false on timeout, true if new frame was received before timeout.
bool waitForNextFrame(std::chrono::nanoseconds timeout);
+ void setFrameAvailableListener(
+ const wp<ConsumerBase::FrameAvailableListener>& listener);
+
// Update the texture with the most recent submitted buffer.
// Most be called on thread with EGL context.
//