blob: b261b52b79980dd1a49b66bda05b1e221ae8dd7a [file] [log] [blame]
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001/*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "Camera3-OutputUtils"
18#define ATRACE_TAG ATRACE_TAG_CAMERA
19//#define LOG_NDEBUG 0
20//#define LOG_NNDEBUG 0 // Per-frame verbose logging
21
22#ifdef LOG_NNDEBUG
23#define ALOGVV(...) ALOGV(__VA_ARGS__)
24#else
25#define ALOGVV(...) ((void)0)
26#endif
27
28// Convenience macros for transitioning to the error state
29#define SET_ERR(fmt, ...) states.setErrIntf.setErrorState( \
30 "%s: " fmt, __FUNCTION__, \
31 ##__VA_ARGS__)
32
33#include <inttypes.h>
34
35#include <utils/Log.h>
36#include <utils/SortedVector.h>
37#include <utils/Trace.h>
38
39#include <android/hardware/camera2/ICameraDeviceCallbacks.h>
40
41#include <android/hardware/camera/device/3.4/ICameraDeviceCallback.h>
42#include <android/hardware/camera/device/3.5/ICameraDeviceCallback.h>
43#include <android/hardware/camera/device/3.5/ICameraDeviceSession.h>
44
45#include <camera_metadata_hidden.h>
46
47#include "device3/Camera3OutputUtils.h"
48
49using namespace android::camera3;
50using namespace android::hardware::camera;
51
52namespace android {
53namespace camera3 {
54
55status_t fixupMonochromeTags(
56 CaptureOutputStates& states,
57 const CameraMetadata& deviceInfo,
58 CameraMetadata& resultMetadata) {
59 status_t res = OK;
60 if (!states.needFixupMonoChrome) {
61 return res;
62 }
63
64 // Remove tags that are not applicable to monochrome camera.
65 int32_t tagsToRemove[] = {
66 ANDROID_SENSOR_GREEN_SPLIT,
67 ANDROID_SENSOR_NEUTRAL_COLOR_POINT,
68 ANDROID_COLOR_CORRECTION_MODE,
69 ANDROID_COLOR_CORRECTION_TRANSFORM,
70 ANDROID_COLOR_CORRECTION_GAINS,
71 };
72 for (auto tag : tagsToRemove) {
73 res = resultMetadata.erase(tag);
74 if (res != OK) {
75 ALOGE("%s: Failed to remove tag %d for monochrome camera", __FUNCTION__, tag);
76 return res;
77 }
78 }
79
80 // ANDROID_SENSOR_DYNAMIC_BLACK_LEVEL
81 camera_metadata_entry blEntry = resultMetadata.find(ANDROID_SENSOR_DYNAMIC_BLACK_LEVEL);
82 for (size_t i = 1; i < blEntry.count; i++) {
83 blEntry.data.f[i] = blEntry.data.f[0];
84 }
85
86 // ANDROID_SENSOR_NOISE_PROFILE
87 camera_metadata_entry npEntry = resultMetadata.find(ANDROID_SENSOR_NOISE_PROFILE);
88 if (npEntry.count > 0 && npEntry.count % 2 == 0) {
89 double np[] = {npEntry.data.d[0], npEntry.data.d[1]};
90 res = resultMetadata.update(ANDROID_SENSOR_NOISE_PROFILE, np, 2);
91 if (res != OK) {
92 ALOGE("%s: Failed to update SENSOR_NOISE_PROFILE: %s (%d)",
93 __FUNCTION__, strerror(-res), res);
94 return res;
95 }
96 }
97
98 // ANDROID_STATISTICS_LENS_SHADING_MAP
99 camera_metadata_ro_entry lsSizeEntry = deviceInfo.find(ANDROID_LENS_INFO_SHADING_MAP_SIZE);
100 camera_metadata_entry lsEntry = resultMetadata.find(ANDROID_STATISTICS_LENS_SHADING_MAP);
101 if (lsSizeEntry.count == 2 && lsEntry.count > 0
102 && (int32_t)lsEntry.count == 4 * lsSizeEntry.data.i32[0] * lsSizeEntry.data.i32[1]) {
103 for (int32_t i = 0; i < lsSizeEntry.data.i32[0] * lsSizeEntry.data.i32[1]; i++) {
104 lsEntry.data.f[4*i+1] = lsEntry.data.f[4*i];
105 lsEntry.data.f[4*i+2] = lsEntry.data.f[4*i];
106 lsEntry.data.f[4*i+3] = lsEntry.data.f[4*i];
107 }
108 }
109
110 // ANDROID_TONEMAP_CURVE_BLUE
111 // ANDROID_TONEMAP_CURVE_GREEN
112 // ANDROID_TONEMAP_CURVE_RED
113 camera_metadata_entry tcbEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_BLUE);
114 camera_metadata_entry tcgEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_GREEN);
115 camera_metadata_entry tcrEntry = resultMetadata.find(ANDROID_TONEMAP_CURVE_RED);
116 if (tcbEntry.count > 0
117 && tcbEntry.count == tcgEntry.count
118 && tcbEntry.count == tcrEntry.count) {
119 for (size_t i = 0; i < tcbEntry.count; i++) {
120 tcbEntry.data.f[i] = tcrEntry.data.f[i];
121 tcgEntry.data.f[i] = tcrEntry.data.f[i];
122 }
123 }
124
125 return res;
126}
127
128void insertResultLocked(CaptureOutputStates& states, CaptureResult *result, uint32_t frameNumber) {
129 if (result == nullptr) return;
130
131 camera_metadata_t *meta = const_cast<camera_metadata_t *>(
132 result->mMetadata.getAndLock());
133 set_camera_metadata_vendor_id(meta, states.vendorTagId);
134 result->mMetadata.unlock(meta);
135
136 if (result->mMetadata.update(ANDROID_REQUEST_FRAME_COUNT,
137 (int32_t*)&frameNumber, 1) != OK) {
138 SET_ERR("Failed to set frame number %d in metadata", frameNumber);
139 return;
140 }
141
142 if (result->mMetadata.update(ANDROID_REQUEST_ID, &result->mResultExtras.requestId, 1) != OK) {
143 SET_ERR("Failed to set request ID in metadata for frame %d", frameNumber);
144 return;
145 }
146
147 // Update vendor tag id for physical metadata
148 for (auto& physicalMetadata : result->mPhysicalMetadatas) {
149 camera_metadata_t *pmeta = const_cast<camera_metadata_t *>(
150 physicalMetadata.mPhysicalCameraMetadata.getAndLock());
151 set_camera_metadata_vendor_id(pmeta, states.vendorTagId);
152 physicalMetadata.mPhysicalCameraMetadata.unlock(pmeta);
153 }
154
155 // Valid result, insert into queue
Jayant Chowdhary8a0be292020-01-08 13:10:38 -0800156 std::list<CaptureResult>::iterator queuedResult =
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800157 states.resultQueue.insert(states.resultQueue.end(), CaptureResult(*result));
Jayant Chowdhary8a0be292020-01-08 13:10:38 -0800158 ALOGV("%s: result requestId = %" PRId32 ", frameNumber = %" PRId64
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800159 ", burstId = %" PRId32, __FUNCTION__,
160 queuedResult->mResultExtras.requestId,
161 queuedResult->mResultExtras.frameNumber,
162 queuedResult->mResultExtras.burstId);
163
164 states.resultSignal.notify_one();
165}
166
167
168void sendPartialCaptureResult(CaptureOutputStates& states,
169 const camera_metadata_t * partialResult,
170 const CaptureResultExtras &resultExtras, uint32_t frameNumber) {
171 ATRACE_CALL();
172 std::lock_guard<std::mutex> l(states.outputLock);
173
174 CaptureResult captureResult;
175 captureResult.mResultExtras = resultExtras;
176 captureResult.mMetadata = partialResult;
177
178 // Fix up result metadata for monochrome camera.
179 status_t res = fixupMonochromeTags(states, states.deviceInfo, captureResult.mMetadata);
180 if (res != OK) {
181 SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res);
182 return;
183 }
184
Shuzhen Wang8c75a642020-10-29 21:58:53 -0700185 // Update partial result by removing keys remapped by DistortionCorrection, ZoomRatio,
186 // and RotationAndCrop mappers.
187 std::set<uint32_t> keysToRemove;
188
189 auto iter = states.distortionMappers.find(states.cameraId.c_str());
190 if (iter != states.distortionMappers.end()) {
191 const auto& remappedKeys = iter->second.getRemappedKeys();
192 keysToRemove.insert(remappedKeys.begin(), remappedKeys.end());
193 }
194
195 const auto& remappedKeys = states.zoomRatioMappers[states.cameraId.c_str()].getRemappedKeys();
196 keysToRemove.insert(remappedKeys.begin(), remappedKeys.end());
197
198 auto mapper = states.rotateAndCropMappers.find(states.cameraId.c_str());
199 if (mapper != states.rotateAndCropMappers.end()) {
200 const auto& remappedKeys = iter->second.getRemappedKeys();
201 keysToRemove.insert(remappedKeys.begin(), remappedKeys.end());
202 }
203
204 for (uint32_t key : keysToRemove) {
205 captureResult.mMetadata.erase(key);
206 }
207
208 // Send partial result
209 if (captureResult.mMetadata.entryCount() > 0) {
210 insertResultLocked(states, &captureResult, frameNumber);
211 }
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800212}
213
214void sendCaptureResult(
215 CaptureOutputStates& states,
216 CameraMetadata &pendingMetadata,
217 CaptureResultExtras &resultExtras,
218 CameraMetadata &collectedPartialResult,
219 uint32_t frameNumber,
Eino-Ville Talvalaf2e37092020-01-07 15:32:32 -0800220 bool reprocess, bool zslStillCapture, bool rotateAndCropAuto,
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800221 const std::set<std::string>& cameraIdsWithZoom,
222 const std::vector<PhysicalCaptureResultInfo>& physicalMetadatas) {
223 ATRACE_CALL();
224 if (pendingMetadata.isEmpty())
225 return;
226
227 std::lock_guard<std::mutex> l(states.outputLock);
228
229 // TODO: need to track errors for tighter bounds on expected frame number
230 if (reprocess) {
231 if (frameNumber < states.nextReprocResultFrameNum) {
232 SET_ERR("Out-of-order reprocess capture result metadata submitted! "
233 "(got frame number %d, expecting %d)",
234 frameNumber, states.nextReprocResultFrameNum);
235 return;
236 }
237 states.nextReprocResultFrameNum = frameNumber + 1;
238 } else if (zslStillCapture) {
239 if (frameNumber < states.nextZslResultFrameNum) {
240 SET_ERR("Out-of-order ZSL still capture result metadata submitted! "
241 "(got frame number %d, expecting %d)",
242 frameNumber, states.nextZslResultFrameNum);
243 return;
244 }
245 states.nextZslResultFrameNum = frameNumber + 1;
246 } else {
247 if (frameNumber < states.nextResultFrameNum) {
248 SET_ERR("Out-of-order capture result metadata submitted! "
249 "(got frame number %d, expecting %d)",
250 frameNumber, states.nextResultFrameNum);
251 return;
252 }
253 states.nextResultFrameNum = frameNumber + 1;
254 }
255
256 CaptureResult captureResult;
257 captureResult.mResultExtras = resultExtras;
258 captureResult.mMetadata = pendingMetadata;
259 captureResult.mPhysicalMetadatas = physicalMetadatas;
260
261 // Append any previous partials to form a complete result
262 if (states.usePartialResult && !collectedPartialResult.isEmpty()) {
263 captureResult.mMetadata.append(collectedPartialResult);
264 }
265
266 captureResult.mMetadata.sort();
267
268 // Check that there's a timestamp in the result metadata
269 camera_metadata_entry timestamp = captureResult.mMetadata.find(ANDROID_SENSOR_TIMESTAMP);
270 if (timestamp.count == 0) {
271 SET_ERR("No timestamp provided by HAL for frame %d!",
272 frameNumber);
273 return;
274 }
Yin-Chia Yehc530c592020-03-09 14:50:36 -0700275 nsecs_t sensorTimestamp = timestamp.data.i64[0];
276
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800277 for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) {
278 camera_metadata_entry timestamp =
279 physicalMetadata.mPhysicalCameraMetadata.find(ANDROID_SENSOR_TIMESTAMP);
280 if (timestamp.count == 0) {
281 SET_ERR("No timestamp provided by HAL for physical camera %s frame %d!",
282 String8(physicalMetadata.mPhysicalCameraId).c_str(), frameNumber);
283 return;
284 }
285 }
286
287 // Fix up some result metadata to account for HAL-level distortion correction
Yin-Chia Yeh7ffbd982020-04-13 10:04:23 -0700288 status_t res = OK;
289 auto iter = states.distortionMappers.find(states.cameraId.c_str());
290 if (iter != states.distortionMappers.end()) {
291 res = iter->second.correctCaptureResult(&captureResult.mMetadata);
292 if (res != OK) {
293 SET_ERR("Unable to correct capture result metadata for frame %d: %s (%d)",
294 frameNumber, strerror(-res), res);
295 return;
296 }
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800297 }
298
299 // Fix up result metadata to account for zoom ratio availabilities between
300 // HAL and app.
301 bool zoomRatioIs1 = cameraIdsWithZoom.find(states.cameraId.c_str()) == cameraIdsWithZoom.end();
302 res = states.zoomRatioMappers[states.cameraId.c_str()].updateCaptureResult(
303 &captureResult.mMetadata, zoomRatioIs1);
304 if (res != OK) {
305 SET_ERR("Failed to update capture result zoom ratio metadata for frame %d: %s (%d)",
306 frameNumber, strerror(-res), res);
307 return;
308 }
309
Eino-Ville Talvalaf2e37092020-01-07 15:32:32 -0800310 // Fix up result metadata to account for rotateAndCrop in AUTO mode
311 if (rotateAndCropAuto) {
312 auto mapper = states.rotateAndCropMappers.find(states.cameraId.c_str());
313 if (mapper != states.rotateAndCropMappers.end()) {
314 res = mapper->second.updateCaptureResult(
315 &captureResult.mMetadata);
316 if (res != OK) {
317 SET_ERR("Unable to correct capture result rotate-and-crop for frame %d: %s (%d)",
318 frameNumber, strerror(-res), res);
319 return;
320 }
321 }
322 }
323
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800324 for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) {
325 String8 cameraId8(physicalMetadata.mPhysicalCameraId);
Eino-Ville Talvalaf2e37092020-01-07 15:32:32 -0800326 auto mapper = states.distortionMappers.find(cameraId8.c_str());
327 if (mapper != states.distortionMappers.end()) {
328 res = mapper->second.correctCaptureResult(
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800329 &physicalMetadata.mPhysicalCameraMetadata);
330 if (res != OK) {
331 SET_ERR("Unable to correct physical capture result metadata for frame %d: %s (%d)",
332 frameNumber, strerror(-res), res);
333 return;
334 }
335 }
336
337 zoomRatioIs1 = cameraIdsWithZoom.find(cameraId8.c_str()) == cameraIdsWithZoom.end();
338 res = states.zoomRatioMappers[cameraId8.c_str()].updateCaptureResult(
339 &physicalMetadata.mPhysicalCameraMetadata, zoomRatioIs1);
340 if (res != OK) {
341 SET_ERR("Failed to update camera %s's physical zoom ratio metadata for "
342 "frame %d: %s(%d)", cameraId8.c_str(), frameNumber, strerror(-res), res);
343 return;
344 }
345 }
346
347 // Fix up result metadata for monochrome camera.
348 res = fixupMonochromeTags(states, states.deviceInfo, captureResult.mMetadata);
349 if (res != OK) {
350 SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res);
351 return;
352 }
353 for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) {
354 String8 cameraId8(physicalMetadata.mPhysicalCameraId);
355 res = fixupMonochromeTags(states,
356 states.physicalDeviceInfoMap.at(cameraId8.c_str()),
357 physicalMetadata.mPhysicalCameraMetadata);
358 if (res != OK) {
359 SET_ERR("Failed to override result metadata: %s (%d)", strerror(-res), res);
360 return;
361 }
362 }
363
364 std::unordered_map<std::string, CameraMetadata> monitoredPhysicalMetadata;
365 for (auto& m : physicalMetadatas) {
366 monitoredPhysicalMetadata.emplace(String8(m.mPhysicalCameraId).string(),
367 CameraMetadata(m.mPhysicalCameraMetadata));
368 }
369 states.tagMonitor.monitorMetadata(TagMonitor::RESULT,
Yin-Chia Yehc530c592020-03-09 14:50:36 -0700370 frameNumber, sensorTimestamp, captureResult.mMetadata,
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800371 monitoredPhysicalMetadata);
372
373 insertResultLocked(states, &captureResult, frameNumber);
374}
375
376// Reading one camera metadata from result argument via fmq or from the result
377// Assuming the fmq is protected by a lock already
378status_t readOneCameraMetadataLocked(
379 std::unique_ptr<ResultMetadataQueue>& fmq,
380 uint64_t fmqResultSize,
381 hardware::camera::device::V3_2::CameraMetadata& resultMetadata,
382 const hardware::camera::device::V3_2::CameraMetadata& result) {
383 if (fmqResultSize > 0) {
384 resultMetadata.resize(fmqResultSize);
385 if (fmq == nullptr) {
386 return NO_MEMORY; // logged in initialize()
387 }
388 if (!fmq->read(resultMetadata.data(), fmqResultSize)) {
389 ALOGE("%s: Cannot read camera metadata from fmq, size = %" PRIu64,
390 __FUNCTION__, fmqResultSize);
391 return INVALID_OPERATION;
392 }
393 } else {
394 resultMetadata.setToExternal(const_cast<uint8_t *>(result.data()),
395 result.size());
396 }
397
398 if (resultMetadata.size() != 0) {
399 status_t res;
400 const camera_metadata_t* metadata =
401 reinterpret_cast<const camera_metadata_t*>(resultMetadata.data());
402 size_t expected_metadata_size = resultMetadata.size();
403 if ((res = validate_camera_metadata_structure(metadata, &expected_metadata_size)) != OK) {
404 ALOGE("%s: Invalid camera metadata received by camera service from HAL: %s (%d)",
405 __FUNCTION__, strerror(-res), res);
406 return INVALID_OPERATION;
407 }
408 }
409
410 return OK;
411}
412
413void removeInFlightMapEntryLocked(CaptureOutputStates& states, int idx) {
414 ATRACE_CALL();
415 InFlightRequestMap& inflightMap = states.inflightMap;
416 nsecs_t duration = inflightMap.valueAt(idx).maxExpectedDuration;
417 inflightMap.removeItemsAt(idx, 1);
418
419 states.inflightIntf.onInflightEntryRemovedLocked(duration);
420}
421
422void removeInFlightRequestIfReadyLocked(CaptureOutputStates& states, int idx) {
423 InFlightRequestMap& inflightMap = states.inflightMap;
Greg Kaiser51b882c2020-06-10 05:41:44 +0000424 const InFlightRequest &request = inflightMap.valueAt(idx);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800425 const uint32_t frameNumber = inflightMap.keyAt(idx);
Shuzhen Wang316781a2020-08-18 18:11:01 -0700426 SessionStatsBuilder& sessionStatsBuilder = states.sessionStatsBuilder;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800427
428 nsecs_t sensorTimestamp = request.sensorTimestamp;
429 nsecs_t shutterTimestamp = request.shutterTimestamp;
430
431 // Check if it's okay to remove the request from InFlightMap:
432 // In the case of a successful request:
433 // all input and output buffers, all result metadata, shutter callback
434 // arrived.
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700435 // In the case of an unsuccessful request:
436 // all input and output buffers, as well as request/result error notifications, arrived.
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800437 if (request.numBuffersLeft == 0 &&
438 (request.skipResultMetadata ||
439 (request.haveResultMetadata && shutterTimestamp != 0))) {
440 if (request.stillCapture) {
441 ATRACE_ASYNC_END("still capture", frameNumber);
442 }
443
444 ATRACE_ASYNC_END("frame capture", frameNumber);
445
Ivan Lozanoc0ad82f2020-07-30 09:32:57 -0400446 // Validation check - if sensor timestamp matches shutter timestamp in the
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800447 // case of request having callback.
448 if (request.hasCallback && request.requestStatus == OK &&
449 sensorTimestamp != shutterTimestamp) {
450 SET_ERR("sensor timestamp (%" PRId64
451 ") for frame %d doesn't match shutter timestamp (%" PRId64 ")",
452 sensorTimestamp, frameNumber, shutterTimestamp);
453 }
454
455 // for an unsuccessful request, it may have pending output buffers to
456 // return.
457 assert(request.requestStatus != OK ||
458 request.pendingOutputBuffers.size() == 0);
459
Greg Kaiser51b882c2020-06-10 05:41:44 +0000460 returnOutputBuffers(
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800461 states.useHalBufManager, states.listener,
462 request.pendingOutputBuffers.array(),
Shuzhen Wang316781a2020-08-18 18:11:01 -0700463 request.pendingOutputBuffers.size(), 0,
464 /*requested*/true, request.requestTimeNs, states.sessionStatsBuilder,
465 /*timestampIncreasing*/true,
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700466 request.outputSurfaces, request.resultExtras,
467 request.errorBufStrategy);
468
469 // Note down the just completed frame number
470 if (request.hasInputBuffer) {
471 states.lastCompletedReprocessFrameNumber = frameNumber;
Shuzhen Wang828ade32022-01-10 17:19:02 -0800472 } else if (request.zslCapture && request.stillCapture) {
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700473 states.lastCompletedZslFrameNumber = frameNumber;
474 } else {
475 states.lastCompletedRegularFrameNumber = frameNumber;
476 }
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800477
Shuzhen Wang316781a2020-08-18 18:11:01 -0700478 sessionStatsBuilder.incResultCounter(request.skipResultMetadata);
479
Greg Kaiser51b882c2020-06-10 05:41:44 +0000480 removeInFlightMapEntryLocked(states, idx);
481 ALOGVV("%s: removed frame %d from InFlightMap", __FUNCTION__, frameNumber);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800482 }
483
484 states.inflightIntf.checkInflightMapLengthLocked();
485}
486
Shuzhen Wang99080502021-03-07 21:08:20 -0800487// Erase the subset of physicalCameraIds that contains id
488bool erasePhysicalCameraIdSet(
489 std::set<std::set<String8>>& physicalCameraIds, const String8& id) {
490 bool found = false;
491 for (auto iter = physicalCameraIds.begin(); iter != physicalCameraIds.end(); iter++) {
492 if (iter->count(id) == 1) {
493 physicalCameraIds.erase(iter);
494 found = true;
495 break;
496 }
497 }
498 return found;
499}
500
Emilian Peevf4816702020-04-03 15:44:51 -0700501void processCaptureResult(CaptureOutputStates& states, const camera_capture_result *result) {
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800502 ATRACE_CALL();
503
504 status_t res;
505
506 uint32_t frameNumber = result->frame_number;
507 if (result->result == NULL && result->num_output_buffers == 0 &&
508 result->input_buffer == NULL) {
509 SET_ERR("No result data provided by HAL for frame %d",
510 frameNumber);
511 return;
512 }
513
514 if (!states.usePartialResult &&
515 result->result != NULL &&
516 result->partial_result != 1) {
517 SET_ERR("Result is malformed for frame %d: partial_result %u must be 1"
518 " if partial result is not supported",
519 frameNumber, result->partial_result);
520 return;
521 }
522
523 bool isPartialResult = false;
524 CameraMetadata collectedPartialResult;
525 bool hasInputBufferInRequest = false;
526
527 // Get shutter timestamp and resultExtras from list of in-flight requests,
528 // where it was added by the shutter notification for this frame. If the
529 // shutter timestamp isn't received yet, append the output buffers to the
530 // in-flight request and they will be returned when the shutter timestamp
531 // arrives. Update the in-flight status and remove the in-flight entry if
532 // all result data and shutter timestamp have been received.
533 nsecs_t shutterTimestamp = 0;
534 {
535 std::lock_guard<std::mutex> l(states.inflightLock);
536 ssize_t idx = states.inflightMap.indexOfKey(frameNumber);
537 if (idx == NAME_NOT_FOUND) {
538 SET_ERR("Unknown frame number for capture result: %d",
539 frameNumber);
540 return;
541 }
542 InFlightRequest &request = states.inflightMap.editValueAt(idx);
543 ALOGVV("%s: got InFlightRequest requestId = %" PRId32
544 ", frameNumber = %" PRId64 ", burstId = %" PRId32
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700545 ", partialResultCount = %d/%d, hasCallback = %d, num_output_buffers %d"
546 ", usePartialResult = %d",
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800547 __FUNCTION__, request.resultExtras.requestId,
548 request.resultExtras.frameNumber, request.resultExtras.burstId,
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700549 result->partial_result, states.numPartialResults,
550 request.hasCallback, result->num_output_buffers,
551 states.usePartialResult);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800552 // Always update the partial count to the latest one if it's not 0
553 // (buffers only). When framework aggregates adjacent partial results
554 // into one, the latest partial count will be used.
555 if (result->partial_result != 0)
556 request.resultExtras.partialResultCount = result->partial_result;
557
558 // Check if this result carries only partial metadata
559 if (states.usePartialResult && result->result != NULL) {
560 if (result->partial_result > states.numPartialResults || result->partial_result < 1) {
561 SET_ERR("Result is malformed for frame %d: partial_result %u must be in"
562 " the range of [1, %d] when metadata is included in the result",
563 frameNumber, result->partial_result, states.numPartialResults);
564 return;
565 }
566 isPartialResult = (result->partial_result < states.numPartialResults);
567 if (isPartialResult && result->num_physcam_metadata) {
568 SET_ERR("Result is malformed for frame %d: partial_result not allowed for"
569 " physical camera result", frameNumber);
570 return;
571 }
572 if (isPartialResult) {
573 request.collectedPartialResult.append(result->result);
574 }
575
576 if (isPartialResult && request.hasCallback) {
577 // Send partial capture result
578 sendPartialCaptureResult(states, result->result, request.resultExtras,
579 frameNumber);
580 }
581 }
582
583 shutterTimestamp = request.shutterTimestamp;
584 hasInputBufferInRequest = request.hasInputBuffer;
585
586 // Did we get the (final) result metadata for this capture?
587 if (result->result != NULL && !isPartialResult) {
588 if (request.physicalCameraIds.size() != result->num_physcam_metadata) {
589 SET_ERR("Expected physical Camera metadata count %d not equal to actual count %d",
590 request.physicalCameraIds.size(), result->num_physcam_metadata);
591 return;
592 }
593 if (request.haveResultMetadata) {
594 SET_ERR("Called multiple times with metadata for frame %d",
595 frameNumber);
596 return;
597 }
598 for (uint32_t i = 0; i < result->num_physcam_metadata; i++) {
599 String8 physicalId(result->physcam_ids[i]);
Shuzhen Wang99080502021-03-07 21:08:20 -0800600 bool validPhysicalCameraMetadata =
601 erasePhysicalCameraIdSet(request.physicalCameraIds, physicalId);
602 if (!validPhysicalCameraMetadata) {
603 SET_ERR("Unexpected total result for frame %d camera %s",
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800604 frameNumber, physicalId.c_str());
605 return;
606 }
607 }
608 if (states.usePartialResult &&
609 !request.collectedPartialResult.isEmpty()) {
610 collectedPartialResult.acquire(
611 request.collectedPartialResult);
612 }
613 request.haveResultMetadata = true;
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700614 request.errorBufStrategy = ERROR_BUF_RETURN_NOTIFY;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800615 }
616
617 uint32_t numBuffersReturned = result->num_output_buffers;
618 if (result->input_buffer != NULL) {
619 if (hasInputBufferInRequest) {
620 numBuffersReturned += 1;
621 } else {
622 ALOGW("%s: Input buffer should be NULL if there is no input"
623 " buffer sent in the request",
624 __FUNCTION__);
625 }
626 }
627 request.numBuffersLeft -= numBuffersReturned;
628 if (request.numBuffersLeft < 0) {
629 SET_ERR("Too many buffers returned for frame %d",
630 frameNumber);
631 return;
632 }
633
634 camera_metadata_ro_entry_t entry;
635 res = find_camera_metadata_ro_entry(result->result,
636 ANDROID_SENSOR_TIMESTAMP, &entry);
637 if (res == OK && entry.count == 1) {
638 request.sensorTimestamp = entry.data.i64[0];
639 }
640
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700641 // If shutter event isn't received yet, do not return the pending output
642 // buffers.
643 request.pendingOutputBuffers.appendArray(result->output_buffers,
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800644 result->num_output_buffers);
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700645 if (shutterTimestamp != 0) {
646 returnAndRemovePendingOutputBuffers(
647 states.useHalBufManager, states.listener,
Shuzhen Wang316781a2020-08-18 18:11:01 -0700648 request, states.sessionStatsBuilder);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800649 }
650
651 if (result->result != NULL && !isPartialResult) {
652 for (uint32_t i = 0; i < result->num_physcam_metadata; i++) {
653 CameraMetadata physicalMetadata;
654 physicalMetadata.append(result->physcam_metadata[i]);
655 request.physicalMetadatas.push_back({String16(result->physcam_ids[i]),
656 physicalMetadata});
657 }
658 if (shutterTimestamp == 0) {
659 request.pendingMetadata = result->result;
660 request.collectedPartialResult = collectedPartialResult;
661 } else if (request.hasCallback) {
662 CameraMetadata metadata;
663 metadata = result->result;
664 sendCaptureResult(states, metadata, request.resultExtras,
665 collectedPartialResult, frameNumber,
666 hasInputBufferInRequest, request.zslCapture && request.stillCapture,
Eino-Ville Talvalaf2e37092020-01-07 15:32:32 -0800667 request.rotateAndCropAuto, request.cameraIdsWithZoom,
668 request.physicalMetadatas);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800669 }
670 }
671 removeInFlightRequestIfReadyLocked(states, idx);
672 } // scope for states.inFlightLock
673
674 if (result->input_buffer != NULL) {
675 if (hasInputBufferInRequest) {
676 Camera3Stream *stream =
677 Camera3Stream::cast(result->input_buffer->stream);
678 res = stream->returnInputBuffer(*(result->input_buffer));
679 // Note: stream may be deallocated at this point, if this buffer was the
680 // last reference to it.
681 if (res != OK) {
682 ALOGE("%s: RequestThread: Can't return input buffer for frame %d to"
683 " its stream:%s (%d)", __FUNCTION__,
684 frameNumber, strerror(-res), res);
685 }
686 } else {
687 ALOGW("%s: Input buffer should be NULL if there is no input"
688 " buffer sent in the request, skipping input buffer return.",
689 __FUNCTION__);
690 }
691 }
692}
693
694void processOneCaptureResultLocked(
695 CaptureOutputStates& states,
696 const hardware::camera::device::V3_2::CaptureResult& result,
697 const hardware::hidl_vec<
698 hardware::camera::device::V3_4::PhysicalCameraMetadata> physicalCameraMetadata) {
699 using hardware::camera::device::V3_2::StreamBuffer;
700 using hardware::camera::device::V3_2::BufferStatus;
701 std::unique_ptr<ResultMetadataQueue>& fmq = states.fmq;
702 BufferRecordsInterface& bufferRecords = states.bufferRecordsIntf;
Emilian Peevf4816702020-04-03 15:44:51 -0700703 camera_capture_result r;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800704 status_t res;
705 r.frame_number = result.frameNumber;
706
707 // Read and validate the result metadata.
708 hardware::camera::device::V3_2::CameraMetadata resultMetadata;
709 res = readOneCameraMetadataLocked(
710 fmq, result.fmqResultSize,
711 resultMetadata, result.result);
712 if (res != OK) {
713 ALOGE("%s: Frame %d: Failed to read capture result metadata",
714 __FUNCTION__, result.frameNumber);
715 return;
716 }
717 r.result = reinterpret_cast<const camera_metadata_t*>(resultMetadata.data());
718
719 // Read and validate physical camera metadata
720 size_t physResultCount = physicalCameraMetadata.size();
721 std::vector<const char*> physCamIds(physResultCount);
722 std::vector<const camera_metadata_t *> phyCamMetadatas(physResultCount);
723 std::vector<hardware::camera::device::V3_2::CameraMetadata> physResultMetadata;
724 physResultMetadata.resize(physResultCount);
725 for (size_t i = 0; i < physicalCameraMetadata.size(); i++) {
726 res = readOneCameraMetadataLocked(fmq, physicalCameraMetadata[i].fmqMetadataSize,
727 physResultMetadata[i], physicalCameraMetadata[i].metadata);
728 if (res != OK) {
729 ALOGE("%s: Frame %d: Failed to read capture result metadata for camera %s",
730 __FUNCTION__, result.frameNumber,
731 physicalCameraMetadata[i].physicalCameraId.c_str());
732 return;
733 }
734 physCamIds[i] = physicalCameraMetadata[i].physicalCameraId.c_str();
735 phyCamMetadatas[i] = reinterpret_cast<const camera_metadata_t*>(
736 physResultMetadata[i].data());
737 }
738 r.num_physcam_metadata = physResultCount;
739 r.physcam_ids = physCamIds.data();
740 r.physcam_metadata = phyCamMetadatas.data();
741
Emilian Peevf4816702020-04-03 15:44:51 -0700742 std::vector<camera_stream_buffer_t> outputBuffers(result.outputBuffers.size());
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800743 std::vector<buffer_handle_t> outputBufferHandles(result.outputBuffers.size());
744 for (size_t i = 0; i < result.outputBuffers.size(); i++) {
745 auto& bDst = outputBuffers[i];
746 const StreamBuffer &bSrc = result.outputBuffers[i];
747
748 sp<Camera3StreamInterface> stream = states.outputStreams.get(bSrc.streamId);
749 if (stream == nullptr) {
750 ALOGE("%s: Frame %d: Buffer %zu: Invalid output stream id %d",
751 __FUNCTION__, result.frameNumber, i, bSrc.streamId);
752 return;
753 }
754 bDst.stream = stream->asHalStream();
755
756 bool noBufferReturned = false;
757 buffer_handle_t *buffer = nullptr;
758 if (states.useHalBufManager) {
759 // This is suspicious most of the time but can be correct during flush where HAL
760 // has to return capture result before a buffer is requested
761 if (bSrc.bufferId == BUFFER_ID_NO_BUFFER) {
762 if (bSrc.status == BufferStatus::OK) {
763 ALOGE("%s: Frame %d: Buffer %zu: No bufferId for stream %d",
764 __FUNCTION__, result.frameNumber, i, bSrc.streamId);
765 // Still proceeds so other buffers can be returned
766 }
767 noBufferReturned = true;
768 }
769 if (noBufferReturned) {
770 res = OK;
771 } else {
772 res = bufferRecords.popInflightRequestBuffer(bSrc.bufferId, &buffer);
773 }
774 } else {
775 res = bufferRecords.popInflightBuffer(result.frameNumber, bSrc.streamId, &buffer);
776 }
777
778 if (res != OK) {
779 ALOGE("%s: Frame %d: Buffer %zu: No in-flight buffer for stream %d",
780 __FUNCTION__, result.frameNumber, i, bSrc.streamId);
781 return;
782 }
783
784 bDst.buffer = buffer;
785 bDst.status = mapHidlBufferStatus(bSrc.status);
786 bDst.acquire_fence = -1;
787 if (bSrc.releaseFence == nullptr) {
788 bDst.release_fence = -1;
789 } else if (bSrc.releaseFence->numFds == 1) {
790 if (noBufferReturned) {
791 ALOGE("%s: got releaseFence without output buffer!", __FUNCTION__);
792 }
793 bDst.release_fence = dup(bSrc.releaseFence->data[0]);
794 } else {
795 ALOGE("%s: Frame %d: Invalid release fence for buffer %zu, fd count is %d, not 1",
796 __FUNCTION__, result.frameNumber, i, bSrc.releaseFence->numFds);
797 return;
798 }
799 }
800 r.num_output_buffers = outputBuffers.size();
801 r.output_buffers = outputBuffers.data();
802
Emilian Peevf4816702020-04-03 15:44:51 -0700803 camera_stream_buffer_t inputBuffer;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800804 if (result.inputBuffer.streamId == -1) {
805 r.input_buffer = nullptr;
806 } else {
807 if (states.inputStream->getId() != result.inputBuffer.streamId) {
808 ALOGE("%s: Frame %d: Invalid input stream id %d", __FUNCTION__,
809 result.frameNumber, result.inputBuffer.streamId);
810 return;
811 }
812 inputBuffer.stream = states.inputStream->asHalStream();
813 buffer_handle_t *buffer;
814 res = bufferRecords.popInflightBuffer(result.frameNumber, result.inputBuffer.streamId,
815 &buffer);
816 if (res != OK) {
817 ALOGE("%s: Frame %d: Input buffer: No in-flight buffer for stream %d",
818 __FUNCTION__, result.frameNumber, result.inputBuffer.streamId);
819 return;
820 }
821 inputBuffer.buffer = buffer;
822 inputBuffer.status = mapHidlBufferStatus(result.inputBuffer.status);
823 inputBuffer.acquire_fence = -1;
824 if (result.inputBuffer.releaseFence == nullptr) {
825 inputBuffer.release_fence = -1;
826 } else if (result.inputBuffer.releaseFence->numFds == 1) {
827 inputBuffer.release_fence = dup(result.inputBuffer.releaseFence->data[0]);
828 } else {
829 ALOGE("%s: Frame %d: Invalid release fence for input buffer, fd count is %d, not 1",
830 __FUNCTION__, result.frameNumber, result.inputBuffer.releaseFence->numFds);
831 return;
832 }
833 r.input_buffer = &inputBuffer;
834 }
835
836 r.partial_result = result.partialResult;
837
838 processCaptureResult(states, &r);
839}
840
Greg Kaiser51b882c2020-06-10 05:41:44 +0000841void returnOutputBuffers(
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800842 bool useHalBufManager,
843 sp<NotificationListener> listener,
Emilian Peevf4816702020-04-03 15:44:51 -0700844 const camera_stream_buffer_t *outputBuffers, size_t numBuffers,
Shuzhen Wang316781a2020-08-18 18:11:01 -0700845 nsecs_t timestamp, bool requested, nsecs_t requestTimeNs,
846 SessionStatsBuilder& sessionStatsBuilder, bool timestampIncreasing,
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800847 const SurfaceMap& outputSurfaces,
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700848 const CaptureResultExtras &inResultExtras,
849 ERROR_BUF_STRATEGY errorBufStrategy) {
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800850
851 for (size_t i = 0; i < numBuffers; i++)
852 {
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700853 Camera3StreamInterface *stream = Camera3Stream::cast(outputBuffers[i].stream);
854 int streamId = stream->getId();
855
856 // Call notify(ERROR_BUFFER) if necessary.
Emilian Peevf4816702020-04-03 15:44:51 -0700857 if (outputBuffers[i].status == CAMERA_BUFFER_STATUS_ERROR &&
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700858 errorBufStrategy == ERROR_BUF_RETURN_NOTIFY) {
859 if (listener != nullptr) {
860 CaptureResultExtras extras = inResultExtras;
861 extras.errorStreamId = streamId;
862 listener->notifyError(
863 hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER,
864 extras);
865 }
866 }
867
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800868 if (outputBuffers[i].buffer == nullptr) {
869 if (!useHalBufManager) {
870 // With HAL buffer management API, HAL sometimes will have to return buffers that
871 // has not got a output buffer handle filled yet. This is though illegal if HAL
872 // buffer management API is not being used.
873 ALOGE("%s: cannot return a null buffer!", __FUNCTION__);
Shuzhen Wang316781a2020-08-18 18:11:01 -0700874 } else {
875 if (requested) {
876 sessionStatsBuilder.incCounter(streamId, /*dropped*/true, 0);
877 }
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800878 }
879 continue;
880 }
881
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800882 const auto& it = outputSurfaces.find(streamId);
883 status_t res = OK;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800884
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700885 // Do not return the buffer if the buffer status is error, and the error
886 // buffer strategy is CACHE.
Emilian Peevf4816702020-04-03 15:44:51 -0700887 if (outputBuffers[i].status != CAMERA_BUFFER_STATUS_ERROR ||
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700888 errorBufStrategy != ERROR_BUF_CACHE) {
889 if (it != outputSurfaces.end()) {
890 res = stream->returnBuffer(
891 outputBuffers[i], timestamp, timestampIncreasing, it->second,
892 inResultExtras.frameNumber);
893 } else {
894 res = stream->returnBuffer(
895 outputBuffers[i], timestamp, timestampIncreasing, std::vector<size_t> (),
896 inResultExtras.frameNumber);
897 }
898 }
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800899 // Note: stream may be deallocated at this point, if this buffer was
900 // the last reference to it.
Shuzhen Wang316781a2020-08-18 18:11:01 -0700901 bool dropped = false;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800902 if (res == NO_INIT || res == DEAD_OBJECT) {
903 ALOGV("Can't return buffer to its stream: %s (%d)", strerror(-res), res);
Shuzhen Wang316781a2020-08-18 18:11:01 -0700904 sessionStatsBuilder.stopCounter(streamId);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800905 } else if (res != OK) {
906 ALOGE("Can't return buffer to its stream: %s (%d)", strerror(-res), res);
Shuzhen Wang316781a2020-08-18 18:11:01 -0700907 dropped = true;
908 } else {
Emilian Peevf4816702020-04-03 15:44:51 -0700909 if (outputBuffers[i].status == CAMERA_BUFFER_STATUS_ERROR || timestamp == 0) {
Shuzhen Wang316781a2020-08-18 18:11:01 -0700910 dropped = true;
911 }
912 }
913 if (requested) {
914 nsecs_t bufferTimeNs = systemTime();
915 int32_t captureLatencyMs = ns2ms(bufferTimeNs - requestTimeNs);
916 sessionStatsBuilder.incCounter(streamId, dropped, captureLatencyMs);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800917 }
918
919 // Long processing consumers can cause returnBuffer timeout for shared stream
920 // If that happens, cancel the buffer and send a buffer error to client
921 if (it != outputSurfaces.end() && res == TIMED_OUT &&
Emilian Peevf4816702020-04-03 15:44:51 -0700922 outputBuffers[i].status == CAMERA_BUFFER_STATUS_OK) {
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800923 // cancel the buffer
Emilian Peevf4816702020-04-03 15:44:51 -0700924 camera_stream_buffer_t sb = outputBuffers[i];
925 sb.status = CAMERA_BUFFER_STATUS_ERROR;
Shuzhen Wang316781a2020-08-18 18:11:01 -0700926 stream->returnBuffer(sb, /*timestamp*/0,
927 timestampIncreasing, std::vector<size_t> (),
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800928 inResultExtras.frameNumber);
929
930 if (listener != nullptr) {
931 CaptureResultExtras extras = inResultExtras;
932 extras.errorStreamId = streamId;
933 listener->notifyError(
934 hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER,
935 extras);
936 }
937 }
938 }
939}
940
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700941void returnAndRemovePendingOutputBuffers(bool useHalBufManager,
Shuzhen Wang316781a2020-08-18 18:11:01 -0700942 sp<NotificationListener> listener, InFlightRequest& request,
943 SessionStatsBuilder& sessionStatsBuilder) {
Shuzhen Wang828ade32022-01-10 17:19:02 -0800944 bool timestampIncreasing =
945 !((request.zslCapture && request.stillCapture) || request.hasInputBuffer);
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700946 returnOutputBuffers(useHalBufManager, listener,
947 request.pendingOutputBuffers.array(),
948 request.pendingOutputBuffers.size(),
Shuzhen Wang316781a2020-08-18 18:11:01 -0700949 request.shutterTimestamp, /*requested*/true,
950 request.requestTimeNs, sessionStatsBuilder, timestampIncreasing,
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700951 request.outputSurfaces, request.resultExtras,
952 request.errorBufStrategy);
953
954 // Remove error buffers that are not cached.
955 for (auto iter = request.pendingOutputBuffers.begin();
956 iter != request.pendingOutputBuffers.end(); ) {
957 if (request.errorBufStrategy != ERROR_BUF_CACHE ||
Emilian Peevf4816702020-04-03 15:44:51 -0700958 iter->status != CAMERA_BUFFER_STATUS_ERROR) {
Shuzhen Wangb7b42652020-05-07 11:59:02 -0700959 iter = request.pendingOutputBuffers.erase(iter);
960 } else {
961 iter++;
962 }
963 }
964}
965
Emilian Peevf4816702020-04-03 15:44:51 -0700966void notifyShutter(CaptureOutputStates& states, const camera_shutter_msg_t &msg) {
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -0800967 ATRACE_CALL();
968 ssize_t idx;
969
970 // Set timestamp for the request in the in-flight tracking
971 // and get the request ID to send upstream
972 {
973 std::lock_guard<std::mutex> l(states.inflightLock);
974 InFlightRequestMap& inflightMap = states.inflightMap;
975 idx = inflightMap.indexOfKey(msg.frame_number);
976 if (idx >= 0) {
977 InFlightRequest &r = inflightMap.editValueAt(idx);
978
979 // Verify ordering of shutter notifications
980 {
981 std::lock_guard<std::mutex> l(states.outputLock);
982 // TODO: need to track errors for tighter bounds on expected frame number.
983 if (r.hasInputBuffer) {
984 if (msg.frame_number < states.nextReprocShutterFrameNum) {
985 SET_ERR("Reprocess shutter notification out-of-order. Expected "
986 "notification for frame %d, got frame %d",
987 states.nextReprocShutterFrameNum, msg.frame_number);
988 return;
989 }
990 states.nextReprocShutterFrameNum = msg.frame_number + 1;
991 } else if (r.zslCapture && r.stillCapture) {
992 if (msg.frame_number < states.nextZslShutterFrameNum) {
993 SET_ERR("ZSL still capture shutter notification out-of-order. Expected "
994 "notification for frame %d, got frame %d",
995 states.nextZslShutterFrameNum, msg.frame_number);
996 return;
997 }
998 states.nextZslShutterFrameNum = msg.frame_number + 1;
999 } else {
1000 if (msg.frame_number < states.nextShutterFrameNum) {
1001 SET_ERR("Shutter notification out-of-order. Expected "
1002 "notification for frame %d, got frame %d",
1003 states.nextShutterFrameNum, msg.frame_number);
1004 return;
1005 }
1006 states.nextShutterFrameNum = msg.frame_number + 1;
1007 }
1008 }
1009
1010 r.shutterTimestamp = msg.timestamp;
1011 if (r.hasCallback) {
1012 ALOGVV("Camera %s: %s: Shutter fired for frame %d (id %d) at %" PRId64,
1013 states.cameraId.string(), __FUNCTION__,
1014 msg.frame_number, r.resultExtras.requestId, msg.timestamp);
1015 // Call listener, if any
1016 if (states.listener != nullptr) {
Shuzhen Wangb7b42652020-05-07 11:59:02 -07001017 r.resultExtras.lastCompletedRegularFrameNumber =
1018 states.lastCompletedRegularFrameNumber;
1019 r.resultExtras.lastCompletedReprocessFrameNumber =
1020 states.lastCompletedReprocessFrameNumber;
1021 r.resultExtras.lastCompletedZslFrameNumber =
1022 states.lastCompletedZslFrameNumber;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001023 states.listener->notifyShutter(r.resultExtras, msg.timestamp);
1024 }
1025 // send pending result and buffers
1026 sendCaptureResult(states,
1027 r.pendingMetadata, r.resultExtras,
1028 r.collectedPartialResult, msg.frame_number,
1029 r.hasInputBuffer, r.zslCapture && r.stillCapture,
Eino-Ville Talvalaf2e37092020-01-07 15:32:32 -08001030 r.rotateAndCropAuto, r.cameraIdsWithZoom, r.physicalMetadatas);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001031 }
Shuzhen Wangb7b42652020-05-07 11:59:02 -07001032 returnAndRemovePendingOutputBuffers(
Shuzhen Wang316781a2020-08-18 18:11:01 -07001033 states.useHalBufManager, states.listener, r, states.sessionStatsBuilder);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001034
1035 removeInFlightRequestIfReadyLocked(states, idx);
1036 }
1037 }
1038 if (idx < 0) {
1039 SET_ERR("Shutter notification for non-existent frame number %d",
1040 msg.frame_number);
1041 }
1042}
1043
Emilian Peevf4816702020-04-03 15:44:51 -07001044void notifyError(CaptureOutputStates& states, const camera_error_msg_t &msg) {
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001045 ATRACE_CALL();
1046 // Map camera HAL error codes to ICameraDeviceCallback error codes
1047 // Index into this with the HAL error code
Emilian Peevf4816702020-04-03 15:44:51 -07001048 static const int32_t halErrorMap[CAMERA_MSG_NUM_ERRORS] = {
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001049 // 0 = Unused error code
1050 hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR,
Emilian Peevf4816702020-04-03 15:44:51 -07001051 // 1 = CAMERA_MSG_ERROR_DEVICE
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001052 hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE,
Emilian Peevf4816702020-04-03 15:44:51 -07001053 // 2 = CAMERA_MSG_ERROR_REQUEST
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001054 hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST,
Emilian Peevf4816702020-04-03 15:44:51 -07001055 // 3 = CAMERA_MSG_ERROR_RESULT
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001056 hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT,
Emilian Peevf4816702020-04-03 15:44:51 -07001057 // 4 = CAMERA_MSG_ERROR_BUFFER
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001058 hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER
1059 };
1060
1061 int32_t errorCode =
1062 ((msg.error_code >= 0) &&
Emilian Peevf4816702020-04-03 15:44:51 -07001063 (msg.error_code < CAMERA_MSG_NUM_ERRORS)) ?
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001064 halErrorMap[msg.error_code] :
1065 hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR;
1066
1067 int streamId = 0;
1068 String16 physicalCameraId;
1069 if (msg.error_stream != nullptr) {
1070 Camera3Stream *stream =
1071 Camera3Stream::cast(msg.error_stream);
1072 streamId = stream->getId();
1073 physicalCameraId = String16(stream->physicalCameraId());
1074 }
1075 ALOGV("Camera %s: %s: HAL error, frame %d, stream %d: %d",
1076 states.cameraId.string(), __FUNCTION__, msg.frame_number,
1077 streamId, msg.error_code);
1078
1079 CaptureResultExtras resultExtras;
1080 switch (errorCode) {
1081 case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE:
1082 // SET_ERR calls into listener to notify application
1083 SET_ERR("Camera HAL reported serious device error");
1084 break;
1085 case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST:
1086 case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT:
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001087 {
1088 std::lock_guard<std::mutex> l(states.inflightLock);
1089 ssize_t idx = states.inflightMap.indexOfKey(msg.frame_number);
1090 if (idx >= 0) {
1091 InFlightRequest &r = states.inflightMap.editValueAt(idx);
1092 r.requestStatus = msg.error_code;
1093 resultExtras = r.resultExtras;
Shuzhen Wangb7b42652020-05-07 11:59:02 -07001094 bool physicalDeviceResultError = false;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001095 if (hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT ==
1096 errorCode) {
1097 if (physicalCameraId.size() > 0) {
1098 String8 cameraId(physicalCameraId);
Shuzhen Wang99080502021-03-07 21:08:20 -08001099 bool validPhysicalCameraId =
1100 erasePhysicalCameraIdSet(r.physicalCameraIds, cameraId);
1101 if (!validPhysicalCameraId) {
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001102 ALOGE("%s: Reported result failure for physical camera device: %s "
1103 " which is not part of the respective request!",
1104 __FUNCTION__, cameraId.string());
1105 break;
1106 }
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001107 resultExtras.errorPhysicalCameraId = physicalCameraId;
Shuzhen Wangb7b42652020-05-07 11:59:02 -07001108 physicalDeviceResultError = true;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001109 }
1110 }
1111
Shuzhen Wangb7b42652020-05-07 11:59:02 -07001112 if (!physicalDeviceResultError) {
Greg Kaiser51b882c2020-06-10 05:41:44 +00001113 r.skipResultMetadata = true;
Shuzhen Wangb7b42652020-05-07 11:59:02 -07001114 if (hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT
1115 == errorCode) {
1116 r.errorBufStrategy = ERROR_BUF_RETURN_NOTIFY;
1117 } else {
1118 // errorCode is ERROR_CAMERA_REQUEST
1119 r.errorBufStrategy = ERROR_BUF_RETURN;
1120 }
1121
1122 // Check whether the buffers returned. If they returned,
1123 // remove inflight request.
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001124 removeInFlightRequestIfReadyLocked(states, idx);
1125 }
1126 } else {
1127 resultExtras.frameNumber = msg.frame_number;
1128 ALOGE("Camera %s: %s: cannot find in-flight request on "
1129 "frame %" PRId64 " error", states.cameraId.string(), __FUNCTION__,
1130 resultExtras.frameNumber);
1131 }
1132 }
1133 resultExtras.errorStreamId = streamId;
1134 if (states.listener != nullptr) {
1135 states.listener->notifyError(errorCode, resultExtras);
1136 } else {
1137 ALOGE("Camera %s: %s: no listener available",
1138 states.cameraId.string(), __FUNCTION__);
1139 }
1140 break;
Shuzhen Wangb7b42652020-05-07 11:59:02 -07001141 case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER:
1142 // Do not depend on HAL ERROR_CAMERA_BUFFER to send buffer error
1143 // callback to the app. Rather, use STATUS_ERROR of image buffers.
1144 break;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001145 default:
1146 // SET_ERR calls notifyError
1147 SET_ERR("Unknown error message from HAL: %d", msg.error_code);
1148 break;
1149 }
1150}
1151
Emilian Peevf4816702020-04-03 15:44:51 -07001152void notify(CaptureOutputStates& states, const camera_notify_msg *msg) {
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001153 switch (msg->type) {
Emilian Peevf4816702020-04-03 15:44:51 -07001154 case CAMERA_MSG_ERROR: {
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001155 notifyError(states, msg->message.error);
1156 break;
1157 }
Emilian Peevf4816702020-04-03 15:44:51 -07001158 case CAMERA_MSG_SHUTTER: {
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001159 notifyShutter(states, msg->message.shutter);
1160 break;
1161 }
1162 default:
1163 SET_ERR("Unknown notify message from HAL: %d",
1164 msg->type);
1165 }
1166}
1167
1168void notify(CaptureOutputStates& states,
1169 const hardware::camera::device::V3_2::NotifyMsg& msg) {
1170 using android::hardware::camera::device::V3_2::MsgType;
1171 using android::hardware::camera::device::V3_2::ErrorCode;
1172
1173 ATRACE_CALL();
Emilian Peevf4816702020-04-03 15:44:51 -07001174 camera_notify_msg m;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001175 switch (msg.type) {
1176 case MsgType::ERROR:
Emilian Peevf4816702020-04-03 15:44:51 -07001177 m.type = CAMERA_MSG_ERROR;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001178 m.message.error.frame_number = msg.msg.error.frameNumber;
1179 if (msg.msg.error.errorStreamId >= 0) {
1180 sp<Camera3StreamInterface> stream =
1181 states.outputStreams.get(msg.msg.error.errorStreamId);
1182 if (stream == nullptr) {
1183 ALOGE("%s: Frame %d: Invalid error stream id %d", __FUNCTION__,
1184 m.message.error.frame_number, msg.msg.error.errorStreamId);
1185 return;
1186 }
1187 m.message.error.error_stream = stream->asHalStream();
1188 } else {
1189 m.message.error.error_stream = nullptr;
1190 }
1191 switch (msg.msg.error.errorCode) {
1192 case ErrorCode::ERROR_DEVICE:
Emilian Peevf4816702020-04-03 15:44:51 -07001193 m.message.error.error_code = CAMERA_MSG_ERROR_DEVICE;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001194 break;
1195 case ErrorCode::ERROR_REQUEST:
Emilian Peevf4816702020-04-03 15:44:51 -07001196 m.message.error.error_code = CAMERA_MSG_ERROR_REQUEST;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001197 break;
1198 case ErrorCode::ERROR_RESULT:
Emilian Peevf4816702020-04-03 15:44:51 -07001199 m.message.error.error_code = CAMERA_MSG_ERROR_RESULT;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001200 break;
1201 case ErrorCode::ERROR_BUFFER:
Emilian Peevf4816702020-04-03 15:44:51 -07001202 m.message.error.error_code = CAMERA_MSG_ERROR_BUFFER;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001203 break;
1204 }
1205 break;
1206 case MsgType::SHUTTER:
Emilian Peevf4816702020-04-03 15:44:51 -07001207 m.type = CAMERA_MSG_SHUTTER;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001208 m.message.shutter.frame_number = msg.msg.shutter.frameNumber;
1209 m.message.shutter.timestamp = msg.msg.shutter.timestamp;
1210 break;
1211 }
1212 notify(states, &m);
1213}
1214
1215void requestStreamBuffers(RequestBufferStates& states,
1216 const hardware::hidl_vec<hardware::camera::device::V3_5::BufferRequest>& bufReqs,
1217 hardware::camera::device::V3_5::ICameraDeviceCallback::requestStreamBuffers_cb _hidl_cb) {
1218 using android::hardware::camera::device::V3_2::BufferStatus;
1219 using android::hardware::camera::device::V3_2::StreamBuffer;
1220 using android::hardware::camera::device::V3_5::BufferRequestStatus;
1221 using android::hardware::camera::device::V3_5::StreamBufferRet;
1222 using android::hardware::camera::device::V3_5::StreamBufferRequestError;
1223
1224 std::lock_guard<std::mutex> lock(states.reqBufferLock);
1225
1226 hardware::hidl_vec<StreamBufferRet> bufRets;
1227 if (!states.useHalBufManager) {
1228 ALOGE("%s: Camera %s does not support HAL buffer management",
1229 __FUNCTION__, states.cameraId.string());
1230 _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets);
1231 return;
1232 }
1233
1234 SortedVector<int32_t> streamIds;
1235 ssize_t sz = streamIds.setCapacity(bufReqs.size());
1236 if (sz < 0 || static_cast<size_t>(sz) != bufReqs.size()) {
1237 ALOGE("%s: failed to allocate memory for %zu buffer requests",
1238 __FUNCTION__, bufReqs.size());
1239 _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets);
1240 return;
1241 }
1242
1243 if (bufReqs.size() > states.outputStreams.size()) {
1244 ALOGE("%s: too many buffer requests (%zu > # of output streams %zu)",
1245 __FUNCTION__, bufReqs.size(), states.outputStreams.size());
1246 _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets);
1247 return;
1248 }
1249
1250 // Check for repeated streamId
1251 for (const auto& bufReq : bufReqs) {
1252 if (streamIds.indexOf(bufReq.streamId) != NAME_NOT_FOUND) {
1253 ALOGE("%s: Stream %d appear multiple times in buffer requests",
1254 __FUNCTION__, bufReq.streamId);
1255 _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, bufRets);
1256 return;
1257 }
1258 streamIds.add(bufReq.streamId);
1259 }
1260
1261 if (!states.reqBufferIntf.startRequestBuffer()) {
1262 ALOGE("%s: request buffer disallowed while camera service is configuring",
1263 __FUNCTION__);
1264 _hidl_cb(BufferRequestStatus::FAILED_CONFIGURING, bufRets);
1265 return;
1266 }
1267
1268 bufRets.resize(bufReqs.size());
1269
1270 bool allReqsSucceeds = true;
1271 bool oneReqSucceeds = false;
1272 for (size_t i = 0; i < bufReqs.size(); i++) {
1273 const auto& bufReq = bufReqs[i];
1274 auto& bufRet = bufRets[i];
1275 int32_t streamId = bufReq.streamId;
1276 sp<Camera3OutputStreamInterface> outputStream = states.outputStreams.get(streamId);
1277 if (outputStream == nullptr) {
1278 ALOGE("%s: Output stream id %d not found!", __FUNCTION__, streamId);
1279 hardware::hidl_vec<StreamBufferRet> emptyBufRets;
1280 _hidl_cb(BufferRequestStatus::FAILED_ILLEGAL_ARGUMENTS, emptyBufRets);
1281 states.reqBufferIntf.endRequestBuffer();
1282 return;
1283 }
1284
Yin-Chia Yeh20126532020-07-14 11:35:48 -07001285 bufRet.streamId = streamId;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001286 if (outputStream->isAbandoned()) {
1287 bufRet.val.error(StreamBufferRequestError::STREAM_DISCONNECTED);
1288 allReqsSucceeds = false;
1289 continue;
1290 }
1291
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001292 size_t handOutBufferCount = outputStream->getOutstandingBuffersCount();
1293 uint32_t numBuffersRequested = bufReq.numBuffersRequested;
1294 size_t totalHandout = handOutBufferCount + numBuffersRequested;
1295 uint32_t maxBuffers = outputStream->asHalStream()->max_buffers;
1296 if (totalHandout > maxBuffers) {
1297 // Not able to allocate enough buffer. Exit early for this stream
1298 ALOGE("%s: request too much buffers for stream %d: at HAL: %zu + requesting: %d"
1299 " > max: %d", __FUNCTION__, streamId, handOutBufferCount,
1300 numBuffersRequested, maxBuffers);
1301 bufRet.val.error(StreamBufferRequestError::MAX_BUFFER_EXCEEDED);
1302 allReqsSucceeds = false;
1303 continue;
1304 }
1305
1306 hardware::hidl_vec<StreamBuffer> tmpRetBuffers(numBuffersRequested);
1307 bool currentReqSucceeds = true;
Emilian Peevf4816702020-04-03 15:44:51 -07001308 std::vector<camera_stream_buffer_t> streamBuffers(numBuffersRequested);
Shuzhen Wangcd5b1822021-09-07 11:52:48 -07001309 std::vector<buffer_handle_t> newBuffers;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001310 size_t numAllocatedBuffers = 0;
1311 size_t numPushedInflightBuffers = 0;
1312 for (size_t b = 0; b < numBuffersRequested; b++) {
Emilian Peevf4816702020-04-03 15:44:51 -07001313 camera_stream_buffer_t& sb = streamBuffers[b];
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001314 // Since this method can run concurrently with request thread
1315 // We need to update the wait duration everytime we call getbuffer
1316 nsecs_t waitDuration = states.reqBufferIntf.getWaitDuration();
1317 status_t res = outputStream->getBuffer(&sb, waitDuration);
1318 if (res != OK) {
1319 if (res == NO_INIT || res == DEAD_OBJECT) {
1320 ALOGV("%s: Can't get output buffer for stream %d: %s (%d)",
1321 __FUNCTION__, streamId, strerror(-res), res);
1322 bufRet.val.error(StreamBufferRequestError::STREAM_DISCONNECTED);
Shuzhen Wang316781a2020-08-18 18:11:01 -07001323 states.sessionStatsBuilder.stopCounter(streamId);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001324 } else {
1325 ALOGE("%s: Can't get output buffer for stream %d: %s (%d)",
1326 __FUNCTION__, streamId, strerror(-res), res);
1327 if (res == TIMED_OUT || res == NO_MEMORY) {
1328 bufRet.val.error(StreamBufferRequestError::NO_BUFFER_AVAILABLE);
1329 } else {
1330 bufRet.val.error(StreamBufferRequestError::UNKNOWN_ERROR);
1331 }
1332 }
1333 currentReqSucceeds = false;
1334 break;
1335 }
1336 numAllocatedBuffers++;
1337
1338 buffer_handle_t *buffer = sb.buffer;
1339 auto pair = states.bufferRecordsIntf.getBufferId(*buffer, streamId);
1340 bool isNewBuffer = pair.first;
1341 uint64_t bufferId = pair.second;
1342 StreamBuffer& hBuf = tmpRetBuffers[b];
1343
1344 hBuf.streamId = streamId;
1345 hBuf.bufferId = bufferId;
1346 hBuf.buffer = (isNewBuffer) ? *buffer : nullptr;
1347 hBuf.status = BufferStatus::OK;
1348 hBuf.releaseFence = nullptr;
Shuzhen Wangcd5b1822021-09-07 11:52:48 -07001349 if (isNewBuffer) {
1350 newBuffers.push_back(*buffer);
1351 }
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001352
1353 native_handle_t *acquireFence = nullptr;
1354 if (sb.acquire_fence != -1) {
1355 acquireFence = native_handle_create(1,0);
1356 acquireFence->data[0] = sb.acquire_fence;
1357 }
1358 hBuf.acquireFence.setTo(acquireFence, /*shouldOwn*/true);
1359 hBuf.releaseFence = nullptr;
1360
1361 res = states.bufferRecordsIntf.pushInflightRequestBuffer(bufferId, buffer, streamId);
1362 if (res != OK) {
1363 ALOGE("%s: Can't get register request buffers for stream %d: %s (%d)",
1364 __FUNCTION__, streamId, strerror(-res), res);
1365 bufRet.val.error(StreamBufferRequestError::UNKNOWN_ERROR);
1366 currentReqSucceeds = false;
1367 break;
1368 }
1369 numPushedInflightBuffers++;
1370 }
1371 if (currentReqSucceeds) {
1372 bufRet.val.buffers(std::move(tmpRetBuffers));
1373 oneReqSucceeds = true;
1374 } else {
1375 allReqsSucceeds = false;
1376 for (size_t b = 0; b < numPushedInflightBuffers; b++) {
1377 StreamBuffer& hBuf = tmpRetBuffers[b];
1378 buffer_handle_t* buffer;
1379 status_t res = states.bufferRecordsIntf.popInflightRequestBuffer(
1380 hBuf.bufferId, &buffer);
1381 if (res != OK) {
1382 SET_ERR("%s: popInflightRequestBuffer failed for stream %d: %s (%d)",
1383 __FUNCTION__, streamId, strerror(-res), res);
1384 }
1385 }
1386 for (size_t b = 0; b < numAllocatedBuffers; b++) {
Emilian Peevf4816702020-04-03 15:44:51 -07001387 camera_stream_buffer_t& sb = streamBuffers[b];
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001388 sb.acquire_fence = -1;
Emilian Peevf4816702020-04-03 15:44:51 -07001389 sb.status = CAMERA_BUFFER_STATUS_ERROR;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001390 }
1391 returnOutputBuffers(states.useHalBufManager, /*listener*/nullptr,
Shuzhen Wang316781a2020-08-18 18:11:01 -07001392 streamBuffers.data(), numAllocatedBuffers, 0, /*requested*/false,
1393 /*requestTimeNs*/0, states.sessionStatsBuilder);
Shuzhen Wangcd5b1822021-09-07 11:52:48 -07001394 for (auto buf : newBuffers) {
1395 states.bufferRecordsIntf.removeOneBufferCache(streamId, buf);
1396 }
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001397 }
1398 }
1399
1400 _hidl_cb(allReqsSucceeds ? BufferRequestStatus::OK :
1401 oneReqSucceeds ? BufferRequestStatus::FAILED_PARTIAL :
1402 BufferRequestStatus::FAILED_UNKNOWN,
1403 bufRets);
1404 states.reqBufferIntf.endRequestBuffer();
1405}
1406
1407void returnStreamBuffers(ReturnBufferStates& states,
1408 const hardware::hidl_vec<hardware::camera::device::V3_2::StreamBuffer>& buffers) {
1409 if (!states.useHalBufManager) {
1410 ALOGE("%s: Camera %s does not support HAL buffer managerment",
1411 __FUNCTION__, states.cameraId.string());
1412 return;
1413 }
1414
1415 for (const auto& buf : buffers) {
1416 if (buf.bufferId == BUFFER_ID_NO_BUFFER) {
1417 ALOGE("%s: cannot return a buffer without bufferId", __FUNCTION__);
1418 continue;
1419 }
1420
1421 buffer_handle_t* buffer;
1422 status_t res = states.bufferRecordsIntf.popInflightRequestBuffer(buf.bufferId, &buffer);
1423
1424 if (res != OK) {
1425 ALOGE("%s: cannot find in-flight buffer %" PRIu64 " for stream %d",
1426 __FUNCTION__, buf.bufferId, buf.streamId);
1427 continue;
1428 }
1429
Emilian Peevf4816702020-04-03 15:44:51 -07001430 camera_stream_buffer_t streamBuffer;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001431 streamBuffer.buffer = buffer;
Emilian Peevf4816702020-04-03 15:44:51 -07001432 streamBuffer.status = CAMERA_BUFFER_STATUS_ERROR;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001433 streamBuffer.acquire_fence = -1;
1434 streamBuffer.release_fence = -1;
1435
1436 if (buf.releaseFence == nullptr) {
1437 streamBuffer.release_fence = -1;
1438 } else if (buf.releaseFence->numFds == 1) {
1439 streamBuffer.release_fence = dup(buf.releaseFence->data[0]);
1440 } else {
1441 ALOGE("%s: Invalid release fence, fd count is %d, not 1",
1442 __FUNCTION__, buf.releaseFence->numFds);
1443 continue;
1444 }
1445
1446 sp<Camera3StreamInterface> stream = states.outputStreams.get(buf.streamId);
1447 if (stream == nullptr) {
1448 ALOGE("%s: Output stream id %d not found!", __FUNCTION__, buf.streamId);
1449 continue;
1450 }
1451 streamBuffer.stream = stream->asHalStream();
1452 returnOutputBuffers(states.useHalBufManager, /*listener*/nullptr,
Shuzhen Wang316781a2020-08-18 18:11:01 -07001453 &streamBuffer, /*size*/1, /*timestamp*/ 0, /*requested*/false,
1454 /*requestTimeNs*/0, states.sessionStatsBuilder);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001455 }
1456}
1457
1458void flushInflightRequests(FlushInflightReqStates& states) {
1459 ATRACE_CALL();
Shuzhen Wang316781a2020-08-18 18:11:01 -07001460 { // First return buffers cached in inFlightMap
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001461 std::lock_guard<std::mutex> l(states.inflightLock);
1462 for (size_t idx = 0; idx < states.inflightMap.size(); idx++) {
Greg Kaiser51b882c2020-06-10 05:41:44 +00001463 const InFlightRequest &request = states.inflightMap.valueAt(idx);
1464 returnOutputBuffers(
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001465 states.useHalBufManager, states.listener,
1466 request.pendingOutputBuffers.array(),
Shuzhen Wang316781a2020-08-18 18:11:01 -07001467 request.pendingOutputBuffers.size(), 0, /*requested*/true,
1468 request.requestTimeNs, states.sessionStatsBuilder, /*timestampIncreasing*/true,
1469 request.outputSurfaces, request.resultExtras, request.errorBufStrategy);
Shuzhen Wangb7b42652020-05-07 11:59:02 -07001470 ALOGW("%s: Frame %d | Timestamp: %" PRId64 ", metadata"
1471 " arrived: %s, buffers left: %d.\n", __FUNCTION__,
1472 states.inflightMap.keyAt(idx), request.shutterTimestamp,
1473 request.haveResultMetadata ? "true" : "false",
1474 request.numBuffersLeft);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001475 }
Shuzhen Wangb7b42652020-05-07 11:59:02 -07001476
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001477 states.inflightMap.clear();
1478 states.inflightIntf.onInflightMapFlushedLocked();
1479 }
1480
1481 // Then return all inflight buffers not returned by HAL
1482 std::vector<std::pair<int32_t, int32_t>> inflightKeys;
1483 states.flushBufferIntf.getInflightBufferKeys(&inflightKeys);
1484
1485 // Inflight buffers for HAL buffer manager
1486 std::vector<uint64_t> inflightRequestBufferKeys;
1487 states.flushBufferIntf.getInflightRequestBufferKeys(&inflightRequestBufferKeys);
1488
1489 // (streamId, frameNumber, buffer_handle_t*) tuple for all inflight buffers.
1490 // frameNumber will be -1 for buffers from HAL buffer manager
1491 std::vector<std::tuple<int32_t, int32_t, buffer_handle_t*>> inflightBuffers;
1492 inflightBuffers.reserve(inflightKeys.size() + inflightRequestBufferKeys.size());
1493
1494 for (auto& pair : inflightKeys) {
1495 int32_t frameNumber = pair.first;
1496 int32_t streamId = pair.second;
1497 buffer_handle_t* buffer;
1498 status_t res = states.bufferRecordsIntf.popInflightBuffer(frameNumber, streamId, &buffer);
1499 if (res != OK) {
1500 ALOGE("%s: Frame %d: No in-flight buffer for stream %d",
1501 __FUNCTION__, frameNumber, streamId);
1502 continue;
1503 }
1504 inflightBuffers.push_back(std::make_tuple(streamId, frameNumber, buffer));
1505 }
1506
1507 for (auto& bufferId : inflightRequestBufferKeys) {
1508 int32_t streamId = -1;
1509 buffer_handle_t* buffer = nullptr;
1510 status_t res = states.bufferRecordsIntf.popInflightRequestBuffer(
1511 bufferId, &buffer, &streamId);
1512 if (res != OK) {
1513 ALOGE("%s: cannot find in-flight buffer %" PRIu64, __FUNCTION__, bufferId);
1514 continue;
1515 }
1516 inflightBuffers.push_back(std::make_tuple(streamId, /*frameNumber*/-1, buffer));
1517 }
1518
1519 std::vector<sp<Camera3StreamInterface>> streams = states.flushBufferIntf.getAllStreams();
1520
1521 for (auto& tuple : inflightBuffers) {
1522 status_t res = OK;
1523 int32_t streamId = std::get<0>(tuple);
1524 int32_t frameNumber = std::get<1>(tuple);
1525 buffer_handle_t* buffer = std::get<2>(tuple);
1526
Emilian Peevf4816702020-04-03 15:44:51 -07001527 camera_stream_buffer_t streamBuffer;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001528 streamBuffer.buffer = buffer;
Emilian Peevf4816702020-04-03 15:44:51 -07001529 streamBuffer.status = CAMERA_BUFFER_STATUS_ERROR;
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001530 streamBuffer.acquire_fence = -1;
1531 streamBuffer.release_fence = -1;
1532
1533 for (auto& stream : streams) {
1534 if (streamId == stream->getId()) {
1535 // Return buffer to deleted stream
Emilian Peevf4816702020-04-03 15:44:51 -07001536 camera_stream* halStream = stream->asHalStream();
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001537 streamBuffer.stream = halStream;
1538 switch (halStream->stream_type) {
Emilian Peevf4816702020-04-03 15:44:51 -07001539 case CAMERA_STREAM_OUTPUT:
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001540 res = stream->returnBuffer(streamBuffer, /*timestamp*/ 0,
Shuzhen Wang316781a2020-08-18 18:11:01 -07001541 /*timestampIncreasing*/true,
1542 std::vector<size_t> (), frameNumber);
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001543 if (res != OK) {
1544 ALOGE("%s: Can't return output buffer for frame %d to"
1545 " stream %d: %s (%d)", __FUNCTION__,
1546 frameNumber, streamId, strerror(-res), res);
1547 }
1548 break;
Emilian Peevf4816702020-04-03 15:44:51 -07001549 case CAMERA_STREAM_INPUT:
Yin-Chia Yeh5fd603e2019-11-20 11:22:27 -08001550 res = stream->returnInputBuffer(streamBuffer);
1551 if (res != OK) {
1552 ALOGE("%s: Can't return input buffer for frame %d to"
1553 " stream %d: %s (%d)", __FUNCTION__,
1554 frameNumber, streamId, strerror(-res), res);
1555 }
1556 break;
1557 default: // Bi-direcitonal stream is deprecated
1558 ALOGE("%s: stream %d has unknown stream type %d",
1559 __FUNCTION__, streamId, halStream->stream_type);
1560 break;
1561 }
1562 break;
1563 }
1564 }
1565 }
1566}
1567
1568} // camera3
1569} // namespace android