Pass surface timestamp to the capture result
Get the timestamp from the Surface, if any, and add it to the capture
restult and output surface.
Bug: 351341245
Test: android.virtualdevice.cts.camera.VirtualCameraCaptureTest#virtualCamera_captureWithTimestamp_mediaCodec
Test: android.virtualdevice.cts.camera.VirtualCameraCaptureTest#virtualCamera_captureWithTimestamp_imageWriter
Flag: FLAG_CAMERA_TIMESTAMP_FROM_SURFACE
Change-Id: Id4f3c4bf6247c8c6ba7b16be79d8a6d33c4c1416
diff --git a/services/camera/virtualcamera/Android.bp b/services/camera/virtualcamera/Android.bp
index 7ece0cb..dd64daa 100644
--- a/services/camera/virtualcamera/Android.bp
+++ b/services/camera/virtualcamera/Android.bp
@@ -25,6 +25,7 @@
"libEGL",
"libGLESv2",
"libGLESv3",
+ "android.companion.virtualdevice.flags-aconfig-cc",
],
static_libs: [
"android.hardware.camera.common@1.0-helper",
diff --git a/services/camera/virtualcamera/VirtualCameraRenderThread.cc b/services/camera/virtualcamera/VirtualCameraRenderThread.cc
index 40a96e4..bf4a45d 100644
--- a/services/camera/virtualcamera/VirtualCameraRenderThread.cc
+++ b/services/camera/virtualcamera/VirtualCameraRenderThread.cc
@@ -14,9 +14,12 @@
* limitations under the License.
*/
+// #define LOG_NDEBUG 0
#define LOG_TAG "VirtualCameraRenderThread"
#include "VirtualCameraRenderThread.h"
+#include <android_companion_virtualdevice_flags.h>
+
#include <chrono>
#include <cstdint>
#include <cstring>
@@ -46,13 +49,11 @@
#include "android-base/thread_annotations.h"
#include "android/binder_auto_utils.h"
#include "android/hardware_buffer.h"
-#include "hardware/gralloc.h"
#include "system/camera_metadata.h"
#include "ui/GraphicBuffer.h"
#include "ui/Rect.h"
#include "util/EglFramebuffer.h"
#include "util/JpegUtil.h"
-#include "util/MetadataUtil.h"
#include "util/Util.h"
#include "utils/Errors.h"
@@ -91,6 +92,8 @@
using namespace std::chrono_literals;
+namespace flags = ::android::companion::virtualdevice::flags;
+
static constexpr std::chrono::milliseconds kAcquireFenceTimeout = 500ms;
static constexpr size_t kJpegThumbnailBufferSize = 32 * 1024; // 32 KiB
@@ -117,12 +120,12 @@
NotifyMsg createRequestErrorNotifyMsg(int frameNumber) {
NotifyMsg msg;
- msg.set<NotifyMsg::Tag::error>(ErrorMsg{
- .frameNumber = frameNumber,
- // errorStreamId needs to be set to -1 for ERROR_REQUEST
- // (not tied to specific stream).
- .errorStreamId = -1,
- .errorCode = ErrorCode::ERROR_REQUEST});
+ msg.set<NotifyMsg::Tag::error>(
+ ErrorMsg{.frameNumber = frameNumber,
+ // errorStreamId needs to be set to -1 for ERROR_REQUEST
+ // (not tied to specific stream).
+ .errorStreamId = -1,
+ .errorCode = ErrorCode::ERROR_REQUEST});
return msg;
}
@@ -413,29 +416,8 @@
std::memory_order_relaxed));
if (request.getRequestSettings().fpsRange) {
- const int maxFps =
- std::max(1, request.getRequestSettings().fpsRange->maxFps);
- const std::chrono::nanoseconds minFrameDuration(
- static_cast<uint64_t>(1e9 / maxFps));
- const std::chrono::nanoseconds frameDuration =
- timestamp - lastAcquisitionTimestamp;
- if (frameDuration < minFrameDuration) {
- // We're too fast for the configured maxFps, let's wait a bit.
- const std::chrono::nanoseconds sleepTime =
- minFrameDuration - frameDuration;
- ALOGV("Current frame duration would be %" PRIu64
- " ns corresponding to, "
- "sleeping for %" PRIu64
- " ns before updating texture to match maxFps %d",
- static_cast<uint64_t>(frameDuration.count()),
- static_cast<uint64_t>(sleepTime.count()), maxFps);
-
- std::this_thread::sleep_for(sleepTime);
- timestamp = std::chrono::duration_cast<std::chrono::nanoseconds>(
- std::chrono::steady_clock::now().time_since_epoch());
- mLastAcquisitionTimestampNanoseconds.store(timestamp.count(),
- std::memory_order_relaxed);
- }
+ int maxFps = std::max(1, request.getRequestSettings().fpsRange->maxFps);
+ timestamp = throttleRendering(maxFps, lastAcquisitionTimestamp, timestamp);
}
// Calculate the maximal amount of time we can afford to wait for next frame.
@@ -463,6 +445,17 @@
}
// Acquire new (most recent) image from the Surface.
mEglSurfaceTexture->updateTexture();
+ std::chrono::nanoseconds captureTimestamp = timestamp;
+
+ if (flags::camera_timestamp_from_surface()) {
+ std::chrono::nanoseconds surfaceTimestamp =
+ getSurfaceTimestamp(elapsedDuration);
+ if (surfaceTimestamp.count() > 0) {
+ captureTimestamp = surfaceTimestamp;
+ }
+ ALOGV("%s captureTimestamp:%lld timestamp:%lld", __func__,
+ captureTimestamp.count(), timestamp.count());
+ }
CaptureResult captureResult;
captureResult.fmqResultSize = 0;
@@ -472,7 +465,7 @@
captureResult.inputBuffer.streamId = -1;
captureResult.physicalCameraMetadata.resize(0);
captureResult.result = createCaptureResultMetadata(
- timestamp, request.getRequestSettings(), mReportedSensorSize);
+ captureTimestamp, request.getRequestSettings(), mReportedSensorSize);
const std::vector<CaptureRequestBuffer>& buffers = request.getBuffers();
captureResult.outputBuffers.resize(buffers.size());
@@ -506,7 +499,7 @@
}
std::vector<NotifyMsg> notifyMsg{
- createShutterNotifyMsg(request.getFrameNumber(), timestamp)};
+ createShutterNotifyMsg(request.getFrameNumber(), captureTimestamp)};
for (const StreamBuffer& resBuffer : captureResult.outputBuffers) {
if (resBuffer.status != BufferStatus::OK) {
notifyMsg.push_back(createBufferErrorNotifyMsg(request.getFrameNumber(),
@@ -535,6 +528,51 @@
ALOGV("%s: Successfully called processCaptureResult", __func__);
}
+std::chrono::nanoseconds VirtualCameraRenderThread::throttleRendering(
+ int maxFps, std::chrono::nanoseconds lastAcquisitionTimestamp,
+ std::chrono::nanoseconds timestamp) {
+ const std::chrono::nanoseconds minFrameDuration(
+ static_cast<uint64_t>(1e9 / maxFps));
+ const std::chrono::nanoseconds frameDuration =
+ timestamp - lastAcquisitionTimestamp;
+ if (frameDuration < minFrameDuration) {
+ // We're too fast for the configured maxFps, let's wait a bit.
+ const std::chrono::nanoseconds sleepTime = minFrameDuration - frameDuration;
+ ALOGV("Current frame duration would be %" PRIu64
+ " ns corresponding to, "
+ "sleeping for %" PRIu64
+ " ns before updating texture to match maxFps %d",
+ static_cast<uint64_t>(frameDuration.count()),
+ static_cast<uint64_t>(sleepTime.count()), maxFps);
+
+ std::this_thread::sleep_for(sleepTime);
+ timestamp = std::chrono::duration_cast<std::chrono::nanoseconds>(
+ std::chrono::steady_clock::now().time_since_epoch());
+ mLastAcquisitionTimestampNanoseconds.store(timestamp.count(),
+ std::memory_order_relaxed);
+ }
+ return timestamp;
+}
+
+std::chrono::nanoseconds VirtualCameraRenderThread::getSurfaceTimestamp(
+ std::chrono::nanoseconds timeSinceLastFrame) {
+ std::chrono::nanoseconds surfaceTimestamp = mEglSurfaceTexture->getTimestamp();
+ if (surfaceTimestamp.count() < 0) {
+ uint64_t lastSurfaceTimestamp = mLastSurfaceTimestampNanoseconds.load();
+ if (lastSurfaceTimestamp > 0) {
+ // The timestamps were provided by the producer but we are
+ // repeating the last frame, so we increase the previous timestamp by
+ // the elapsed time sinced its capture, otherwise the camera framework
+ // will discard the frame.
+ surfaceTimestamp = std::chrono::nanoseconds(lastSurfaceTimestamp +
+ timeSinceLastFrame.count());
+ }
+ }
+ mLastSurfaceTimestampNanoseconds.store(surfaceTimestamp.count(),
+ std::memory_order_relaxed);
+ return surfaceTimestamp;
+}
+
void VirtualCameraRenderThread::flushCaptureRequest(
const ProcessCaptureRequestTask& request) {
CaptureResult captureResult;
diff --git a/services/camera/virtualcamera/VirtualCameraRenderThread.h b/services/camera/virtualcamera/VirtualCameraRenderThread.h
index aafed44..a35eea1 100644
--- a/services/camera/virtualcamera/VirtualCameraRenderThread.h
+++ b/services/camera/virtualcamera/VirtualCameraRenderThread.h
@@ -18,6 +18,7 @@
#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H
#include <atomic>
+#include <chrono>
#include <cstdint>
#include <deque>
#include <future>
@@ -188,6 +189,22 @@
EglFrameBuffer& framebuffer, sp<Fence> fence = nullptr,
std::optional<Rect> viewport = std::nullopt);
+ // Throttle the current thread to ensure that we are not rendering faster than
+ // the provided maxFps.
+ // maxFps: The maximum fps in the capture request
+ // lastAcquisitionTimestamp: timestamp of the previous frame
+ // timestamp: the current capture time
+ // Returns the time at which the capture has happened after throttling.
+ std::chrono::nanoseconds throttleRendering(
+ int maxFps, std::chrono::nanoseconds lastAcquisitionTimestamp,
+ std::chrono::nanoseconds timestamp);
+
+ // Fetch the timestamp of the latest buffer from the EGL Surface
+ // timeSinceLastFrame: The elapsed time since the last captured frame.
+ // Return 0 if no timestamp has been associated to this surface by the producer.
+ std::chrono::nanoseconds getSurfaceTimestamp(
+ std::chrono::nanoseconds timeSinceLastFrame);
+
// Camera callback
const std::shared_ptr<
::aidl::android::hardware::camera::device::ICameraDeviceCallback>
@@ -209,6 +226,7 @@
// Acquisition timestamp of last frame.
std::atomic<uint64_t> mLastAcquisitionTimestampNanoseconds;
+ std::atomic<uint64_t> mLastSurfaceTimestampNanoseconds;
// EGL helpers - constructed and accessed only from rendering thread.
std::unique_ptr<EglDisplayContext> mEglDisplayContext;
diff --git a/services/camera/virtualcamera/util/EglSurfaceTexture.cc b/services/camera/virtualcamera/util/EglSurfaceTexture.cc
index 98bf62a..be36ec4 100644
--- a/services/camera/virtualcamera/util/EglSurfaceTexture.cc
+++ b/services/camera/virtualcamera/util/EglSurfaceTexture.cc
@@ -15,6 +15,8 @@
*/
// #define LOG_NDEBUG 0
+#include <chrono>
+
#include "utils/Timers.h"
#define LOG_TAG "EglSurfaceTexture"
@@ -99,6 +101,10 @@
static_cast<nsecs_t>(timeout.count()));
}
+std::chrono::nanoseconds EglSurfaceTexture::getTimestamp() {
+ return std::chrono::nanoseconds(mGlConsumer->getTimestamp());
+}
+
GLuint EglSurfaceTexture::updateTexture() {
int previousFrameId;
int framesAdvance = 0;
diff --git a/services/camera/virtualcamera/util/EglSurfaceTexture.h b/services/camera/virtualcamera/util/EglSurfaceTexture.h
index a46af8f..c1f1169 100644
--- a/services/camera/virtualcamera/util/EglSurfaceTexture.h
+++ b/services/camera/virtualcamera/util/EglSurfaceTexture.h
@@ -82,6 +82,10 @@
// See SurfaceTexture.getTransformMatrix for more details.
std::array<float, 16> getTransformMatrix();
+ // Retrieves the timestamp associated with the texture image
+ // set by the most recent call to updateTexture.
+ std::chrono::nanoseconds getTimestamp();
+
private:
#if !COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ)
sp<IGraphicBufferProducer> mBufferProducer;