Camera: re-space camera output for hardware texture target
2 issues with overriding presentation timestamp for SurfaceTexture:
- We cannot override timestamp for SurfaceTexture, because the timestamp
may be used for AV sync.
- TextureView always picks the latest buffer and display it right away
without using the presentation timestamp.
Due to these reasons, the cameraservice re-spaces queuing of the buffer
based on capture time intervals rather than overriding timestamps.
The buffer timestamp is left untouched.
Test: Run Instagram, Snapchat, and Tiktok and observe improved preview
Test: Camera CTS
Bug: 195025014
Change-Id: I28b23d9f2b39b6a289b6d9f87968e4bc667870a4
diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp
index 1e2dccb..47594c3 100644
--- a/services/camera/libcameraservice/Android.bp
+++ b/services/camera/libcameraservice/Android.bp
@@ -81,6 +81,7 @@
"device3/Camera3OutputUtils.cpp",
"device3/Camera3DeviceInjectionMethods.cpp",
"device3/UHRCropAndMeteringRegionMapper.cpp",
+ "device3/PreviewFrameSpacer.cpp",
"device3/hidl/HidlCamera3Device.cpp",
"device3/hidl/HidlCamera3OfflineSession.cpp",
"device3/hidl/HidlCamera3OutputUtils.cpp",
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
index 37f9227..c5bdd00 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
+++ b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
@@ -376,24 +376,35 @@
dumpImageToDisk(timestamp, anwBuffer, anwReleaseFence);
}
- nsecs_t captureTime = (mSyncToDisplay ? readoutTimestamp : timestamp) - mTimestampOffset;
- nsecs_t presentTime = mSyncToDisplay ?
- syncTimestampToDisplayLocked(captureTime) : captureTime;
+ if (mPreviewFrameSpacer != nullptr) {
+ res = mPreviewFrameSpacer->queuePreviewBuffer(timestamp - mTimestampOffset, transform,
+ anwBuffer, anwReleaseFence);
+ if (res != OK) {
+ ALOGE("%s: Stream %d: Error queuing buffer to preview buffer spacer: %s (%d)",
+ __FUNCTION__, mId, strerror(-res), res);
+ return res;
+ }
+ } else {
+ nsecs_t captureTime = (mSyncToDisplay ? readoutTimestamp : timestamp)
+ - mTimestampOffset;
+ nsecs_t presentTime = mSyncToDisplay ?
+ syncTimestampToDisplayLocked(captureTime) : captureTime;
- setTransform(transform, true/*mayChangeMirror*/);
- res = native_window_set_buffers_timestamp(mConsumer.get(), presentTime);
- if (res != OK) {
- ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
- __FUNCTION__, mId, strerror(-res), res);
- return res;
- }
+ setTransform(transform, true/*mayChangeMirror*/);
+ res = native_window_set_buffers_timestamp(mConsumer.get(), presentTime);
+ if (res != OK) {
+ ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
+ __FUNCTION__, mId, strerror(-res), res);
+ return res;
+ }
- queueHDRMetadata(anwBuffer->handle, currentConsumer, dynamic_range_profile);
+ queueHDRMetadata(anwBuffer->handle, currentConsumer, dynamic_range_profile);
- res = queueBufferToConsumer(currentConsumer, anwBuffer, anwReleaseFence, surface_ids);
- if (shouldLogError(res, state)) {
- ALOGE("%s: Stream %d: Error queueing buffer to native window:"
- " %s (%d)", __FUNCTION__, mId, strerror(-res), res);
+ res = queueBufferToConsumer(currentConsumer, anwBuffer, anwReleaseFence, surface_ids);
+ if (shouldLogError(res, state)) {
+ ALOGE("%s: Stream %d: Error queueing buffer to native window:"
+ " %s (%d)", __FUNCTION__, mId, strerror(-res), res);
+ }
}
}
mLock.lock();
@@ -468,7 +479,7 @@
return res;
}
- if ((res = configureConsumerQueueLocked(true /*allowDisplaySync*/)) != OK) {
+ if ((res = configureConsumerQueueLocked(true /*allowPreviewRespace*/)) != OK) {
return res;
}
@@ -492,7 +503,7 @@
return OK;
}
-status_t Camera3OutputStream::configureConsumerQueueLocked(bool allowDisplaySync) {
+status_t Camera3OutputStream::configureConsumerQueueLocked(bool allowPreviewRespace) {
status_t res;
mTraceFirstBuffer = true;
@@ -582,20 +593,25 @@
int timestampBase = getTimestampBase();
bool isDefaultTimeBase = (timestampBase ==
OutputConfiguration::TIMESTAMP_BASE_DEFAULT);
- if (allowDisplaySync) {
- // We cannot distinguish between a SurfaceView and an ImageReader of
- // preview buffer format. Frames are synchronized to display in both
- // cases.
+ if (allowPreviewRespace) {
bool forceChoreographer = (timestampBase ==
OutputConfiguration::TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED);
- bool defaultToChoreographer = (isDefaultTimeBase && isConsumedByHWComposer() &&
+ bool defaultToChoreographer = (isDefaultTimeBase &&
+ isConsumedByHWComposer() &&
!property_get_bool("camera.disable_preview_scheduler", false));
if (forceChoreographer || defaultToChoreographer) {
mSyncToDisplay = true;
mTotalBufferCount += kDisplaySyncExtraBuffer;
+ } else if (isConsumedByHWTexture() && !isVideoStream()) {
+ mPreviewFrameSpacer = new PreviewFrameSpacer(*this, mConsumer);
+ mTotalBufferCount ++;
+ res = mPreviewFrameSpacer->run(String8::format("PreviewSpacer-%d", mId).string());
+ if (res != OK) {
+ ALOGE("%s: Unable to start preview spacer", __FUNCTION__);
+ return res;
+ }
}
}
-
mHandoutTotalBufferCount = 0;
mFrameCount = 0;
mLastTimestamp = 0;
@@ -1316,6 +1332,11 @@
return expectedPresentT - vsyncEventData.frameInterval/3;
}
+bool Camera3OutputStream::shouldLogError(status_t res) {
+ Mutex::Autolock l(mLock);
+ return shouldLogError(res, mState);
+}
+
}; // namespace camera3
}; // namespace android
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.h b/services/camera/libcameraservice/device3/Camera3OutputStream.h
index 6ea7ef7..e68bc67 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputStream.h
+++ b/services/camera/libcameraservice/device3/Camera3OutputStream.h
@@ -28,6 +28,7 @@
#include "Camera3IOStreamBase.h"
#include "Camera3OutputStreamInterface.h"
#include "Camera3BufferManager.h"
+#include "PreviewFrameSpacer.h"
namespace android {
@@ -250,6 +251,7 @@
static void applyZSLUsageQuirk(int format, uint64_t *consumerUsage /*inout*/);
void setImageDumpMask(int mask) { mImageDumpMask = mask; }
+ bool shouldLogError(status_t res);
protected:
Camera3OutputStream(int id, camera_stream_type_t type,
@@ -282,7 +284,7 @@
status_t getEndpointUsageForSurface(uint64_t *usage,
const sp<Surface>& surface) const;
- status_t configureConsumerQueueLocked(bool allowDisplaySync);
+ status_t configureConsumerQueueLocked(bool allowPreviewRespace);
// Consumer as the output of camera HAL
sp<Surface> mConsumer;
@@ -396,15 +398,14 @@
void returnPrefetchedBuffersLocked();
- // Synchronize camera timestamp to display, and the return value
- // can be used as presentation timestamp
- nsecs_t syncTimestampToDisplayLocked(nsecs_t t);
static const int32_t kDequeueLatencyBinSize = 5; // in ms
CameraLatencyHistogram mDequeueBufferLatency;
int mImageDumpMask = 0;
+ // Re-space frames by overriding timestamp to align with display Vsync.
+ // Default is on for SurfaceView bound streams.
nsecs_t mMinExpectedDuration = 0;
bool mSyncToDisplay = false;
DisplayEventReceiver mDisplayEventReceiver;
@@ -414,6 +415,12 @@
static constexpr size_t kDisplaySyncExtraBuffer = 2;
static constexpr nsecs_t kSpacingResetIntervalNs = 1000000000LL; // 1 second
static constexpr nsecs_t kTimelineThresholdNs = 1000000LL; // 1 millisecond
+ nsecs_t syncTimestampToDisplayLocked(nsecs_t t);
+
+ // Re-space frames by delaying queueBuffer so that frame delivery has
+ // the same cadence as capture. Default is on for SurfaceTexture bound
+ // streams.
+ sp<PreviewFrameSpacer> mPreviewFrameSpacer;
}; // class Camera3OutputStream
} // namespace camera3
diff --git a/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp b/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp
index d24b527..82958a2 100644
--- a/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp
+++ b/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp
@@ -251,7 +251,7 @@
return res;
}
- res = configureConsumerQueueLocked(false/*allowDisplaySync*/);
+ res = configureConsumerQueueLocked(false/*allowPreviewRespace*/);
if (res != OK) {
ALOGE("Failed to configureConsumerQueueLocked: %s(%d)", strerror(-res), res);
return res;
diff --git a/services/camera/libcameraservice/device3/PreviewFrameSpacer.cpp b/services/camera/libcameraservice/device3/PreviewFrameSpacer.cpp
new file mode 100644
index 0000000..9112b93
--- /dev/null
+++ b/services/camera/libcameraservice/device3/PreviewFrameSpacer.cpp
@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "Camera3-PreviewFrameSpacer"
+#define ATRACE_TAG ATRACE_TAG_CAMERA
+//#define LOG_NDEBUG 0
+
+#include <utils/Log.h>
+
+#include "PreviewFrameSpacer.h"
+#include "Camera3OutputStream.h"
+
+namespace android {
+
+namespace camera3 {
+
+PreviewFrameSpacer::PreviewFrameSpacer(Camera3OutputStream& parent, sp<Surface> consumer) :
+ mParent(parent),
+ mConsumer(consumer) {
+}
+
+PreviewFrameSpacer::~PreviewFrameSpacer() {
+ Thread::requestExitAndWait();
+}
+
+status_t PreviewFrameSpacer::queuePreviewBuffer(nsecs_t timestamp, int32_t transform,
+ ANativeWindowBuffer* anwBuffer, int releaseFence) {
+ Mutex::Autolock l(mLock);
+ mPendingBuffers.emplace(timestamp, transform, anwBuffer, releaseFence);
+ ALOGV("%s: mPendingBuffers size %zu, timestamp %" PRId64, __FUNCTION__,
+ mPendingBuffers.size(), timestamp);
+
+ mBufferCond.signal();
+ return OK;
+}
+
+bool PreviewFrameSpacer::threadLoop() {
+ Mutex::Autolock l(mLock);
+ if (mPendingBuffers.size() == 0) {
+ mBufferCond.waitRelative(mLock, kWaitDuration);
+ return true;
+ }
+
+ nsecs_t currentTime = systemTime();
+ auto buffer = mPendingBuffers.front();
+ nsecs_t captureInterval = buffer.timestamp - mLastCameraCaptureTime;
+ // If the capture interval exceeds threshold, directly queue
+ // cached buffer.
+ if (captureInterval >= kFrameIntervalThreshold) {
+ mPendingBuffers.pop();
+ queueBufferToClientLocked(buffer, currentTime);
+ return true;
+ }
+
+ // Cache the frame to match capture time interval, for up to 33ms
+ nsecs_t expectedQueueTime = mLastCameraPresentTime + captureInterval;
+ nsecs_t frameWaitTime = std::min(kMaxFrameWaitTime, expectedQueueTime - currentTime);
+ if (frameWaitTime > 0 && mPendingBuffers.size() < 2) {
+ mBufferCond.waitRelative(mLock, frameWaitTime);
+ if (exitPending()) {
+ return true;
+ }
+ currentTime = systemTime();
+ }
+ ALOGV("%s: captureInterval %" PRId64 ", queueInterval %" PRId64 ", waited for %" PRId64
+ ", timestamp %" PRId64, __FUNCTION__, captureInterval,
+ currentTime - mLastCameraPresentTime, frameWaitTime, buffer.timestamp);
+ mPendingBuffers.pop();
+ queueBufferToClientLocked(buffer, currentTime);
+ return true;
+}
+
+void PreviewFrameSpacer::requestExit() {
+ // Call parent to set up shutdown
+ Thread::requestExit();
+ // Exit from other possible wait
+ mBufferCond.signal();
+}
+
+void PreviewFrameSpacer::queueBufferToClientLocked(
+ const BufferHolder& bufferHolder, nsecs_t currentTime) {
+ mParent.setTransform(bufferHolder.transform, true/*mayChangeMirror*/);
+
+ status_t res = native_window_set_buffers_timestamp(mConsumer.get(), bufferHolder.timestamp);
+ if (res != OK) {
+ ALOGE("%s: Preview Stream: Error setting timestamp: %s (%d)",
+ __FUNCTION__, strerror(-res), res);
+ }
+
+ Camera3Stream::queueHDRMetadata(bufferHolder.anwBuffer.get()->handle, mConsumer,
+ mParent.getDynamicRangeProfile());
+
+ res = mConsumer->queueBuffer(mConsumer.get(), bufferHolder.anwBuffer.get(),
+ bufferHolder.releaseFence);
+ if (res != OK) {
+ close(bufferHolder.releaseFence);
+ if (mParent.shouldLogError(res)) {
+ ALOGE("%s: Failed to queue buffer to client: %s(%d)", __FUNCTION__,
+ strerror(-res), res);
+ }
+ }
+
+ mLastCameraPresentTime = currentTime;
+ mLastCameraCaptureTime = bufferHolder.timestamp;
+}
+
+}; // namespace camera3
+
+}; // namespace android
diff --git a/services/camera/libcameraservice/device3/PreviewFrameSpacer.h b/services/camera/libcameraservice/device3/PreviewFrameSpacer.h
new file mode 100644
index 0000000..5062553
--- /dev/null
+++ b/services/camera/libcameraservice/device3/PreviewFrameSpacer.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_SERVERS_CAMERA_CAMERA3_PREVIEWFRAMESPACER_H
+#define ANDROID_SERVERS_CAMERA_CAMERA3_PREVIEWFRAMESPACER_H
+
+#include <queue>
+
+#include <gui/Surface.h>
+#include <utils/Condition.h>
+#include <utils/Mutex.h>
+#include <utils/Thread.h>
+#include <utils/Timers.h>
+
+namespace android {
+
+namespace camera3 {
+
+class Camera3OutputStream;
+
+/***
+ * Preview stream spacer for better frame spacing
+ *
+ * The ideal viewfinder user experience is that frames are presented to the
+ * user in the same cadence as outputed by the camera sensor. However, the
+ * processing latency between frames could vary, due to factors such
+ * as CPU load, differences in request settings, etc. This frame processing
+ * latency results in variation in presentation of frames to the user.
+ *
+ * The PreviewFrameSpacer improves the viewfinder user experience by:
+ * - Cache the frame buffers if the intervals between queueBuffer is shorter
+ * than the camera capture intervals.
+ * - Queue frame buffers in the same cadence as the camera capture time.
+ * - Maintain at most 1 queue-able buffer. If the 2nd preview buffer becomes
+ * available, queue the oldest cached buffer to the buffer queue.
+ */
+class PreviewFrameSpacer : public Thread {
+ public:
+ explicit PreviewFrameSpacer(Camera3OutputStream& parent, sp<Surface> consumer);
+ virtual ~PreviewFrameSpacer();
+
+ // Queue preview buffer locally
+ status_t queuePreviewBuffer(nsecs_t timestamp, int32_t transform,
+ ANativeWindowBuffer* anwBuffer, int releaseFence);
+
+ bool threadLoop() override;
+ void requestExit() override;
+
+ private:
+ // structure holding cached preview buffer info
+ struct BufferHolder {
+ nsecs_t timestamp;
+ int32_t transform;
+ sp<ANativeWindowBuffer> anwBuffer;
+ int releaseFence;
+
+ BufferHolder(nsecs_t t, int32_t tr, ANativeWindowBuffer* anwb, int rf) :
+ timestamp(t), transform(tr), anwBuffer(anwb), releaseFence(rf) {}
+ };
+
+ void queueBufferToClientLocked(const BufferHolder& bufferHolder, nsecs_t currentTime);
+
+
+ Camera3OutputStream& mParent;
+ sp<ANativeWindow> mConsumer;
+ mutable Mutex mLock;
+ Condition mBufferCond;
+
+ std::queue<BufferHolder> mPendingBuffers;
+ nsecs_t mLastCameraCaptureTime = 0;
+ nsecs_t mLastCameraPresentTime = 0;
+ static constexpr nsecs_t kWaitDuration = 5000000LL; // 50ms
+ static constexpr nsecs_t kFrameIntervalThreshold = 80000000LL; // 80ms
+ static constexpr nsecs_t kMaxFrameWaitTime = 33333333LL; // 33ms
+};
+
+}; //namespace camera3
+}; //namespace android
+
+#endif