Camera: Add PreviewFrameScheduler to address preview jitter
The ideal viewfinder user experience is that frames are presented to the
user in the same cadence as outputed by the camera sensor. However, the
processing latency between frames could vary, due to factors such
as CPU load, difference in request settings, etc. This frame processing
latency results in variation in presentation of frames to the user.
Improve the user experience by:
1. Cache preview buffers in PreviewFrameScheduler.
2. For each choreographer callback, queue the oldest preview buffer,
with the best matching presentation timestamp. Frame N's
presentation timestamp is the choreographer timeline timestamp closest to
(Frame N-1's presentation time + capture interval between frame N-1 and N).
3. Maintain at most 2 queue-able buffers. If a 3rd preview buffer becomes
available, queue it to the buffer queue right away.
Test: Run GoogleCamera video mode and observe smoother viewfinder
Test: Observe surfaceflinger trace when running viewfinder
Test: Camera CTS
Bug: 200306379
Change-Id: I791c841aaded2acd112de8f7e99a131443b21e11
diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp
index 26562e0..a24b35d 100644
--- a/services/camera/libcameraservice/Android.bp
+++ b/services/camera/libcameraservice/Android.bp
@@ -84,6 +84,7 @@
"device3/Camera3OutputUtils.cpp",
"device3/Camera3DeviceInjectionMethods.cpp",
"device3/UHRCropAndMeteringRegionMapper.cpp",
+ "device3/PreviewFrameScheduler.cpp",
"gui/RingBufferConsumer.cpp",
"hidl/AidlCameraDeviceCallbacks.cpp",
"hidl/AidlCameraServiceListener.cpp",
@@ -107,6 +108,7 @@
],
shared_libs: [
+ "libandroid",
"libbase",
"libdl",
"libexif",
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
index 3738d01..d8bcc8a 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
+++ b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
@@ -22,6 +22,7 @@
#include <fstream>
#include <android-base/unique_fd.h>
+#include <cutils/properties.h>
#include <ui/GraphicBuffer.h>
#include <utils/Log.h>
#include <utils/Trace.h>
@@ -347,20 +348,6 @@
mTraceFirstBuffer = false;
}
- if (transform != -1) {
- setTransformLocked(transform);
- }
-
- /* Certain consumers (such as AudioSource or HardwareComposer) use
- * MONOTONIC time, causing time misalignment if camera timestamp is
- * in BOOTTIME. Do the conversion if necessary. */
- res = native_window_set_buffers_timestamp(mConsumer.get(),
- mUseMonoTimestamp ? timestamp - mTimestampOffset : timestamp);
- if (res != OK) {
- ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
- __FUNCTION__, mId, strerror(-res), res);
- return res;
- }
// If this is a JPEG output, and image dump mask is set, save image to
// disk.
if (getFormat() == HAL_PIXEL_FORMAT_BLOB && getDataSpace() == HAL_DATASPACE_V0_JFIF &&
@@ -368,10 +355,31 @@
dumpImageToDisk(timestamp, anwBuffer, anwReleaseFence);
}
- res = queueBufferToConsumer(currentConsumer, anwBuffer, anwReleaseFence, surface_ids);
- if (shouldLogError(res, state)) {
- ALOGE("%s: Stream %d: Error queueing buffer to native window:"
- " %s (%d)", __FUNCTION__, mId, strerror(-res), res);
+ /* Certain consumers (such as AudioSource or HardwareComposer) use
+ * MONOTONIC time, causing time misalignment if camera timestamp is
+ * in BOOTTIME. Do the conversion if necessary. */
+ nsecs_t adjustedTs = mUseMonoTimestamp ? timestamp - mTimestampOffset : timestamp;
+ if (mPreviewFrameScheduler != nullptr) {
+ res = mPreviewFrameScheduler->queuePreviewBuffer(adjustedTs, transform,
+ anwBuffer, anwReleaseFence);
+ if (res != OK) {
+ ALOGE("%s: Stream %d: Error queuing buffer to preview buffer scheduler: %s (%d)",
+ __FUNCTION__, mId, strerror(-res), res);
+ return res;
+ }
+ } else {
+ setTransform(transform);
+ res = native_window_set_buffers_timestamp(mConsumer.get(), adjustedTs);
+ if (res != OK) {
+ ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
+ __FUNCTION__, mId, strerror(-res), res);
+ return res;
+ }
+ res = queueBufferToConsumer(currentConsumer, anwBuffer, anwReleaseFence, surface_ids);
+ if (shouldLogError(res, state)) {
+ ALOGE("%s: Stream %d: Error queueing buffer to native window:"
+ " %s (%d)", __FUNCTION__, mId, strerror(-res), res);
+ }
}
}
mLock.lock();
@@ -412,6 +420,9 @@
status_t Camera3OutputStream::setTransformLocked(int transform) {
status_t res = OK;
+
+ if (transform == -1) return res;
+
if (mState == STATE_ERROR) {
ALOGE("%s: Stream in error state", __FUNCTION__);
return INVALID_OPERATION;
@@ -437,7 +448,7 @@
return res;
}
- if ((res = configureConsumerQueueLocked()) != OK) {
+ if ((res = configureConsumerQueueLocked(true /*allowPreviewScheduler*/)) != OK) {
return res;
}
@@ -461,7 +472,7 @@
return OK;
}
-status_t Camera3OutputStream::configureConsumerQueueLocked() {
+status_t Camera3OutputStream::configureConsumerQueueLocked(bool allowPreviewScheduler) {
status_t res;
mTraceFirstBuffer = true;
@@ -547,6 +558,15 @@
}
mTotalBufferCount = maxConsumerBuffers + camera_stream::max_buffers;
+ if (allowPreviewScheduler && isConsumedByHWComposer()) {
+ // We cannot distinguish between a SurfaceView and an ImageReader of
+ // preview buffer format. The PreviewFrameScheduler needs to handle both.
+ if (!property_get_bool("camera.disable_preview_scheduler", false)) {
+ mPreviewFrameScheduler = std::make_unique<PreviewFrameScheduler>(*this, mConsumer);
+ mTotalBufferCount += PreviewFrameScheduler::kQueueDepthWatermark;
+ }
+ }
+
mHandoutTotalBufferCount = 0;
mFrameCount = 0;
mLastTimestamp = 0;
@@ -1185,6 +1205,11 @@
}
}
+bool Camera3OutputStream::shouldLogError(status_t res) {
+ Mutex::Autolock l(mLock);
+ return shouldLogError(res, mState);
+}
+
}; // namespace camera3
}; // namespace android
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.h b/services/camera/libcameraservice/device3/Camera3OutputStream.h
index 0872687..b2b574a 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputStream.h
+++ b/services/camera/libcameraservice/device3/Camera3OutputStream.h
@@ -27,6 +27,7 @@
#include "Camera3IOStreamBase.h"
#include "Camera3OutputStreamInterface.h"
#include "Camera3BufferManager.h"
+#include "PreviewFrameScheduler.h"
namespace android {
@@ -229,6 +230,7 @@
static void applyZSLUsageQuirk(int format, uint64_t *consumerUsage /*inout*/);
void setImageDumpMask(int mask) { mImageDumpMask = mask; }
+ bool shouldLogError(status_t res);
protected:
Camera3OutputStream(int id, camera_stream_type_t type,
@@ -255,7 +257,7 @@
status_t getEndpointUsageForSurface(uint64_t *usage,
const sp<Surface>& surface) const;
- status_t configureConsumerQueueLocked();
+ status_t configureConsumerQueueLocked(bool allowPreviewScheduler);
// Consumer as the output of camera HAL
sp<Surface> mConsumer;
@@ -370,6 +372,8 @@
int mImageDumpMask = 0;
+ // The preview stream scheduler for re-timing frames
+ std::unique_ptr<PreviewFrameScheduler> mPreviewFrameScheduler;
}; // class Camera3OutputStream
} // namespace camera3
diff --git a/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp b/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp
index 15cf7f4..9e0c8f3 100644
--- a/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp
+++ b/services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp
@@ -247,7 +247,7 @@
return res;
}
- res = configureConsumerQueueLocked();
+ res = configureConsumerQueueLocked(false/*allowPreviewScheduler*/);
if (res != OK) {
ALOGE("Failed to configureConsumerQueueLocked: %s(%d)", strerror(-res), res);
return res;
diff --git a/services/camera/libcameraservice/device3/PreviewFrameScheduler.cpp b/services/camera/libcameraservice/device3/PreviewFrameScheduler.cpp
new file mode 100644
index 0000000..1fbdb18
--- /dev/null
+++ b/services/camera/libcameraservice/device3/PreviewFrameScheduler.cpp
@@ -0,0 +1,210 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "Camera3-PreviewFrameScheduler"
+#define ATRACE_TAG ATRACE_TAG_CAMERA
+//#define LOG_NDEBUG 0
+
+#include <utils/Log.h>
+#include <utils/Trace.h>
+
+#include <android/looper.h>
+#include "PreviewFrameScheduler.h"
+#include "Camera3OutputStream.h"
+
+namespace android {
+
+namespace camera3 {
+
+/**
+ * Internal Choreographer thread implementation for polling and handling callbacks
+ */
+
+// Callback function for Choreographer
+static void frameCallback(const AChoreographerFrameCallbackData* callbackData, void* data) {
+ PreviewFrameScheduler* parent = static_cast<PreviewFrameScheduler*>(data);
+ if (parent == nullptr) {
+ ALOGE("%s: Invalid data for Choreographer callback!", __FUNCTION__);
+ return;
+ }
+
+ size_t length = AChoreographerFrameCallbackData_getFrameTimelinesLength(callbackData);
+ std::vector<nsecs_t> timeline(length);
+ for (size_t i = 0; i < length; i++) {
+ nsecs_t timestamp = AChoreographerFrameCallbackData_getFrameTimelineExpectedPresentTime(
+ callbackData, i);
+ timeline[i] = timestamp;
+ }
+
+ parent->onNewPresentationTime(timeline);
+
+ AChoreographer_postExtendedFrameCallback(AChoreographer_getInstance(), frameCallback, data);
+}
+
+struct ChoreographerThread : public Thread {
+ ChoreographerThread();
+ status_t start(PreviewFrameScheduler* parent);
+ virtual status_t readyToRun() override;
+ virtual bool threadLoop() override;
+
+protected:
+ virtual ~ChoreographerThread() {}
+
+private:
+ ChoreographerThread &operator=(const ChoreographerThread &);
+
+ // This only impacts the shutdown time. It won't impact the choreographer
+ // callback frequency.
+ static constexpr nsecs_t kPollingTimeoutMs = 5;
+ PreviewFrameScheduler* mParent = nullptr;
+};
+
+ChoreographerThread::ChoreographerThread() : Thread(false /*canCallJava*/) {
+}
+
+status_t ChoreographerThread::start(PreviewFrameScheduler* parent) {
+ mParent = parent;
+ return run("PreviewChoreographer");
+}
+
+status_t ChoreographerThread::readyToRun() {
+ ALooper_prepare(ALOOPER_PREPARE_ALLOW_NON_CALLBACKS);
+ if (AChoreographer_getInstance() == NULL) {
+ return NO_INIT;
+ }
+
+ AChoreographer_postExtendedFrameCallback(
+ AChoreographer_getInstance(), frameCallback, mParent);
+ return OK;
+}
+
+bool ChoreographerThread::threadLoop() {
+ if (exitPending()) {
+ return false;
+ }
+ ALooper_pollOnce(kPollingTimeoutMs, nullptr, nullptr, nullptr);
+ return true;
+}
+
+/**
+ * PreviewFrameScheduler implementation
+ */
+
+PreviewFrameScheduler::PreviewFrameScheduler(Camera3OutputStream& parent, sp<Surface> consumer) :
+ mParent(parent),
+ mConsumer(consumer),
+ mChoreographerThread(new ChoreographerThread()) {
+}
+
+PreviewFrameScheduler::~PreviewFrameScheduler() {
+ {
+ Mutex::Autolock l(mLock);
+ mChoreographerThread->requestExit();
+ }
+ mChoreographerThread->join();
+}
+
+status_t PreviewFrameScheduler::queuePreviewBuffer(nsecs_t timestamp, int32_t transform,
+ ANativeWindowBuffer* anwBuffer, int releaseFence) {
+ // Start choreographer thread if it's not already running.
+ if (!mChoreographerThread->isRunning()) {
+ status_t res = mChoreographerThread->start(this);
+ if (res != OK) {
+ ALOGE("%s: Failed to init choreographer thread!", __FUNCTION__);
+ return res;
+ }
+ }
+
+ {
+ Mutex::Autolock l(mLock);
+ mPendingBuffers.emplace(timestamp, transform, anwBuffer, releaseFence);
+
+ // Queue buffer to client right away if pending buffers are more than
+ // the queue depth watermark.
+ if (mPendingBuffers.size() > kQueueDepthWatermark) {
+ auto oldBuffer = mPendingBuffers.front();
+ mPendingBuffers.pop();
+
+ status_t res = queueBufferToClientLocked(oldBuffer, oldBuffer.timestamp);
+ if (res != OK) {
+ return res;
+ }
+
+ // Reset the last capture and presentation time
+ mLastCameraCaptureTime = 0;
+ mLastCameraPresentTime = 0;
+ } else {
+ ATRACE_INT(kPendingBufferTraceName, mPendingBuffers.size());
+ }
+ }
+ return OK;
+}
+
+void PreviewFrameScheduler::onNewPresentationTime(const std::vector<nsecs_t>& timeline) {
+ ATRACE_CALL();
+ Mutex::Autolock l(mLock);
+ if (mPendingBuffers.size() > 0) {
+ auto nextBuffer = mPendingBuffers.front();
+ mPendingBuffers.pop();
+
+ // Find the best presentation time by finding the element in the
+ // choreographer timeline that's closest to the ideal presentation time.
+ // The ideal presentation time is the last presentation time + frame
+ // interval.
+ nsecs_t cameraInterval = nextBuffer.timestamp - mLastCameraCaptureTime;
+ nsecs_t idealPresentTime = (cameraInterval < kSpacingResetIntervalNs) ?
+ (mLastCameraPresentTime + cameraInterval) : nextBuffer.timestamp;
+ nsecs_t presentTime = *std::min_element(timeline.begin(), timeline.end(),
+ [idealPresentTime](nsecs_t p1, nsecs_t p2) {
+ return std::abs(p1 - idealPresentTime) < std::abs(p2 - idealPresentTime);
+ });
+
+ status_t res = queueBufferToClientLocked(nextBuffer, presentTime);
+ ATRACE_INT(kPendingBufferTraceName, mPendingBuffers.size());
+
+ if (mParent.shouldLogError(res)) {
+ ALOGE("%s: Preview Stream: Error queueing buffer to native window:"
+ " %s (%d)", __FUNCTION__, strerror(-res), res);
+ }
+
+ mLastCameraCaptureTime = nextBuffer.timestamp;
+ mLastCameraPresentTime = presentTime;
+ }
+}
+
+status_t PreviewFrameScheduler::queueBufferToClientLocked(
+ const BufferHolder& bufferHolder, nsecs_t timestamp) {
+ mParent.setTransform(bufferHolder.transform);
+
+ status_t res = native_window_set_buffers_timestamp(mConsumer.get(), timestamp);
+ if (res != OK) {
+ ALOGE("%s: Preview Stream: Error setting timestamp: %s (%d)",
+ __FUNCTION__, strerror(-res), res);
+ return res;
+ }
+
+ res = mConsumer->queueBuffer(mConsumer.get(), bufferHolder.anwBuffer.get(),
+ bufferHolder.releaseFence);
+ if (res != OK) {
+ close(bufferHolder.releaseFence);
+ }
+
+ return res;
+}
+
+}; // namespace camera3
+
+}; // namespace android
diff --git a/services/camera/libcameraservice/device3/PreviewFrameScheduler.h b/services/camera/libcameraservice/device3/PreviewFrameScheduler.h
new file mode 100644
index 0000000..c0574fd
--- /dev/null
+++ b/services/camera/libcameraservice/device3/PreviewFrameScheduler.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_SERVERS_CAMERA_CAMERA3_PREVIEWFRAMESCHEDULER_H
+#define ANDROID_SERVERS_CAMERA_CAMERA3_PREVIEWFRAMESCHEDULER_H
+
+#include <queue>
+
+#include <android/choreographer.h>
+#include <gui/Surface.h>
+#include <gui/ISurfaceComposer.h>
+#include <utils/Condition.h>
+#include <utils/Mutex.h>
+#include <utils/Looper.h>
+#include <utils/Thread.h>
+#include <utils/Timers.h>
+
+namespace android {
+
+namespace camera3 {
+
+class Camera3OutputStream;
+struct ChoreographerThread;
+
+/***
+ * Preview stream scheduler for better preview display synchronization
+ *
+ * The ideal viewfinder user experience is that frames are presented to the
+ * user in the same cadence as outputed by the camera sensor. However, the
+ * processing latency between frames could vary, due to factors such
+ * as CPU load, differences in request settings, etc. This frame processing
+ * latency results in variation in presentation of frames to the user.
+ *
+ * The PreviewFrameScheduler improves the viewfinder user experience by:
+ * 1. Cache preview buffers in the scheduler
+ * 2. For each choreographer callback, queue the oldest cached buffer with
+ * the best matching presentation timestamp. Frame N's presentation timestamp
+ * is the choreographer timeline timestamp closest to (Frame N-1's
+ * presentation time + camera capture interval between frame N-1 and frame N).
+ * 3. Maintain at most 2 queue-able buffers. If the 3rd preview buffer becomes
+ * available, queue the oldest cached buffer to the buffer queue.
+ */
+class PreviewFrameScheduler {
+ public:
+ explicit PreviewFrameScheduler(Camera3OutputStream& parent, sp<Surface> consumer);
+ virtual ~PreviewFrameScheduler();
+
+ // Queue preview buffer locally
+ status_t queuePreviewBuffer(nsecs_t timestamp, int32_t transform,
+ ANativeWindowBuffer* anwBuffer, int releaseFence);
+
+ // Callback function with a new presentation timeline from choreographer. This
+ // will trigger a locally queued buffer be sent to the buffer queue.
+ void onNewPresentationTime(const std::vector<nsecs_t>& presentationTimeline);
+
+ // Maintain at most 2 queue-able buffers
+ static constexpr int32_t kQueueDepthWatermark = 2;
+
+ private:
+ // structure holding cached preview buffer info
+ struct BufferHolder {
+ nsecs_t timestamp;
+ int32_t transform;
+ sp<ANativeWindowBuffer> anwBuffer;
+ int releaseFence;
+
+ BufferHolder(nsecs_t t, int32_t tr, ANativeWindowBuffer* anwb, int rf) :
+ timestamp(t), transform(tr), anwBuffer(anwb), releaseFence(rf) {}
+ };
+
+ status_t queueBufferToClientLocked(const BufferHolder& bufferHolder,
+ nsecs_t presentTime);
+
+ static constexpr char kPendingBufferTraceName[] = "pending_preview_buffers";
+
+ // Camera capture interval for resetting frame spacing between preview sessions
+ static constexpr nsecs_t kSpacingResetIntervalNs = 1000000000L; // 1 second
+
+ Camera3OutputStream& mParent;
+ sp<ANativeWindow> mConsumer;
+ mutable Mutex mLock;
+
+ std::queue<BufferHolder> mPendingBuffers;
+ nsecs_t mLastCameraCaptureTime = 0;
+ nsecs_t mLastCameraPresentTime = 0;
+
+ // Choreographer related
+ sp<Looper> mLooper;
+ sp<ChoreographerThread> mChoreographerThread;
+};
+
+}; //namespace camera3
+}; //namespace android
+
+#endif