Camera: Add support for readout timestamp

The readout timestamp is added to CaptureResultExtras used by
onCaptureStarted(). If the HAL supports readoutTimestamp, the
readout timestamp is passed in onCaptureStarted().

Test: atest Experimental2022SystemTest
Bug: 229097448
Bug: 229773853
Change-Id: I1ac822b14a2a4198e7c8ef17a39212f504c2d719
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
index c5bdd00..111c053 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
+++ b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp
@@ -59,6 +59,7 @@
         mTraceFirstBuffer(true),
         mUseBufferManager(false),
         mTimestampOffset(timestampOffset),
+        mUseReadoutTime(false),
         mConsumerUsage(0),
         mDropBuffers(false),
         mMirrorMode(mirrorMode),
@@ -91,6 +92,7 @@
         mTraceFirstBuffer(true),
         mUseBufferManager(false),
         mTimestampOffset(timestampOffset),
+        mUseReadoutTime(false),
         mConsumerUsage(0),
         mDropBuffers(false),
         mMirrorMode(mirrorMode),
@@ -130,6 +132,7 @@
         mTraceFirstBuffer(true),
         mUseBufferManager(false),
         mTimestampOffset(timestampOffset),
+        mUseReadoutTime(false),
         mConsumerUsage(consumerUsage),
         mDropBuffers(false),
         mMirrorMode(mirrorMode),
@@ -176,6 +179,7 @@
         mTraceFirstBuffer(true),
         mUseBufferManager(false),
         mTimestampOffset(timestampOffset),
+        mUseReadoutTime(false),
         mConsumerUsage(consumerUsage),
         mDropBuffers(false),
         mMirrorMode(mirrorMode),
@@ -376,8 +380,10 @@
             dumpImageToDisk(timestamp, anwBuffer, anwReleaseFence);
         }
 
+        nsecs_t captureTime = (mUseReadoutTime && readoutTimestamp != 0 ?
+                readoutTimestamp : timestamp) - mTimestampOffset;
         if (mPreviewFrameSpacer != nullptr) {
-            res = mPreviewFrameSpacer->queuePreviewBuffer(timestamp - mTimestampOffset, transform,
+            res = mPreviewFrameSpacer->queuePreviewBuffer(captureTime, transform,
                     anwBuffer, anwReleaseFence);
             if (res != OK) {
                 ALOGE("%s: Stream %d: Error queuing buffer to preview buffer spacer: %s (%d)",
@@ -385,8 +391,6 @@
                 return res;
             }
         } else {
-            nsecs_t captureTime = (mSyncToDisplay ? readoutTimestamp : timestamp)
-                    - mTimestampOffset;
             nsecs_t presentTime = mSyncToDisplay ?
                     syncTimestampToDisplayLocked(captureTime) : captureTime;
 
@@ -616,12 +620,16 @@
     mFrameCount = 0;
     mLastTimestamp = 0;
 
+    mUseReadoutTime =
+            (timestampBase == OutputConfiguration::TIMESTAMP_BASE_READOUT_SENSOR || mSyncToDisplay);
+
     if (isDeviceTimeBaseRealtime()) {
         if (isDefaultTimeBase && !isConsumedByHWComposer() && !isVideoStream()) {
             // Default time base, but not hardware composer or video encoder
             mTimestampOffset = 0;
         } else if (timestampBase == OutputConfiguration::TIMESTAMP_BASE_REALTIME ||
-                timestampBase == OutputConfiguration::TIMESTAMP_BASE_SENSOR) {
+                timestampBase == OutputConfiguration::TIMESTAMP_BASE_SENSOR ||
+                timestampBase == OutputConfiguration::TIMESTAMP_BASE_READOUT_SENSOR) {
             mTimestampOffset = 0;
         }
         // If timestampBase is CHOREOGRAPHER SYNCED or MONOTONIC, leave
@@ -631,7 +639,7 @@
             // Reverse offset for monotonicTime -> bootTime
             mTimestampOffset = -mTimestampOffset;
         } else {
-            // If timestampBase is DEFAULT, MONOTONIC, SENSOR, or
+            // If timestampBase is DEFAULT, MONOTONIC, SENSOR, READOUT_SENSOR or
             // CHOREOGRAPHER_SYNCED, timestamp offset is 0.
             mTimestampOffset = 0;
         }
diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.h b/services/camera/libcameraservice/device3/Camera3OutputStream.h
index e68bc67..b6d1631 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputStream.h
+++ b/services/camera/libcameraservice/device3/Camera3OutputStream.h
@@ -334,6 +334,11 @@
     nsecs_t mTimestampOffset;
 
     /**
+     * If camera readout time is used rather than the start-of-exposure time.
+     */
+    bool mUseReadoutTime;
+
+    /**
      * Consumer end point usage flag set by the constructor for the deferred
      * consumer case.
      */
diff --git a/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp b/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp
index ed66df0..f4e3fad 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp
+++ b/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp
@@ -787,10 +787,12 @@
         SessionStatsBuilder& sessionStatsBuilder) {
     bool timestampIncreasing =
             !((request.zslCapture && request.stillCapture) || request.hasInputBuffer);
+    nsecs_t readoutTimestamp = request.resultExtras.hasReadoutTimestamp ?
+            request.resultExtras.readoutTimestamp : 0;
     returnOutputBuffers(useHalBufManager, listener,
             request.pendingOutputBuffers.array(),
             request.pendingOutputBuffers.size(),
-            request.shutterTimestamp, request.shutterReadoutTimestamp,
+            request.shutterTimestamp, readoutTimestamp,
             /*requested*/true, request.requestTimeNs, sessionStatsBuilder, timestampIncreasing,
             request.outputSurfaces, request.resultExtras,
             request.errorBufStrategy, request.transform);
@@ -852,7 +854,10 @@
             }
 
             r.shutterTimestamp = msg.timestamp;
-            r.shutterReadoutTimestamp = msg.readout_timestamp;
+            if (msg.readout_timestamp_valid) {
+                r.resultExtras.hasReadoutTimestamp = true;
+                r.resultExtras.readoutTimestamp = msg.readout_timestamp;
+            }
             if (r.minExpectedDuration != states.minFrameDuration) {
                 for (size_t i = 0; i < states.outputStreams.size(); i++) {
                     auto outputStream = states.outputStreams[i];
diff --git a/services/camera/libcameraservice/device3/InFlightRequest.h b/services/camera/libcameraservice/device3/InFlightRequest.h
index 493a9e2..fa00495 100644
--- a/services/camera/libcameraservice/device3/InFlightRequest.h
+++ b/services/camera/libcameraservice/device3/InFlightRequest.h
@@ -65,6 +65,7 @@
 typedef struct camera_shutter_msg {
     uint32_t frame_number;
     uint64_t timestamp;
+    bool readout_timestamp_valid;
     uint64_t readout_timestamp;
 } camera_shutter_msg_t;
 
@@ -104,8 +105,6 @@
 struct InFlightRequest {
     // Set by notify() SHUTTER call.
     nsecs_t shutterTimestamp;
-    // Set by notify() SHUTTER call with readout time.
-    nsecs_t shutterReadoutTimestamp;
     // Set by process_capture_result().
     nsecs_t sensorTimestamp;
     int     requestStatus;
diff --git a/services/camera/libcameraservice/device3/aidl/AidlCamera3OutputUtils.cpp b/services/camera/libcameraservice/device3/aidl/AidlCamera3OutputUtils.cpp
index 3809f37..b462d44 100644
--- a/services/camera/libcameraservice/device3/aidl/AidlCamera3OutputUtils.cpp
+++ b/services/camera/libcameraservice/device3/aidl/AidlCamera3OutputUtils.cpp
@@ -110,6 +110,7 @@
             m.type = CAMERA_MSG_SHUTTER;
             m.message.shutter.frame_number = msg.get<Tag::shutter>().frameNumber;
             m.message.shutter.timestamp = msg.get<Tag::shutter>().timestamp;
+            m.message.shutter.readout_timestamp_valid = true;
             m.message.shutter.readout_timestamp = msg.get<Tag::shutter>().readoutTimestamp;
             break;
     }
diff --git a/services/camera/libcameraservice/device3/hidl/HidlCamera3OutputUtils.cpp b/services/camera/libcameraservice/device3/hidl/HidlCamera3OutputUtils.cpp
index 60e4e42..a9a6c66 100644
--- a/services/camera/libcameraservice/device3/hidl/HidlCamera3OutputUtils.cpp
+++ b/services/camera/libcameraservice/device3/hidl/HidlCamera3OutputUtils.cpp
@@ -127,8 +127,8 @@
             m.type = CAMERA_MSG_SHUTTER;
             m.message.shutter.frame_number = msg.msg.shutter.frameNumber;
             m.message.shutter.timestamp = msg.msg.shutter.timestamp;
-            m.message.shutter.readout_timestamp = hasReadoutTime ?
-                    readoutTime : m.message.shutter.timestamp;
+            m.message.shutter.readout_timestamp_valid = hasReadoutTime;
+            m.message.shutter.readout_timestamp = readoutTime;
             break;
     }
     notify(states, &m);