blob: 54cc27aebfa57ed33992112a5b6a2e79bbc379ad [file] [log] [blame]
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001/*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "Camera3-HeicCompositeStream"
18#define ATRACE_TAG ATRACE_TAG_CAMERA
Susmitha Gummallae911f2e2020-11-23 09:57:03 -080019#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -080020//#define LOG_NDEBUG 0
21
22#include <linux/memfd.h>
23#include <pthread.h>
24#include <sys/syscall.h>
25
Jayant Chowdharyc67af1b2022-04-07 18:05:04 +000026#include <aidl/android/hardware/camera/device/CameraBlob.h>
27#include <aidl/android/hardware/camera/device/CameraBlobId.h>
Shuzhen Wang219c2992019-02-15 17:24:28 -080028#include <libyuv.h>
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -080029#include <gui/Surface.h>
30#include <utils/Log.h>
31#include <utils/Trace.h>
32
Marco Nelissen13aa1a42019-09-27 10:21:55 -070033#include <mediadrm/ICrypto.h>
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -080034#include <media/MediaCodecBuffer.h>
35#include <media/stagefright/foundation/ABuffer.h>
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -080036#include <media/stagefright/foundation/MediaDefs.h>
37#include <media/stagefright/MediaCodecConstants.h>
38
39#include "common/CameraDeviceBase.h"
40#include "utils/ExifUtils.h"
Jayant Chowdhary13f9b2f2020-12-02 22:46:15 -080041#include "utils/SessionConfigurationUtils.h"
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -080042#include "HeicEncoderInfoManager.h"
43#include "HeicCompositeStream.h"
44
Jayant Chowdharyc67af1b2022-04-07 18:05:04 +000045using aidl::android::hardware::camera::device::CameraBlob;
46using aidl::android::hardware::camera::device::CameraBlobId;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -080047
48namespace android {
49namespace camera3 {
50
Shuzhen Wange8675782019-12-05 09:12:14 -080051HeicCompositeStream::HeicCompositeStream(sp<CameraDeviceBase> device,
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -080052 wp<hardware::camera2::ICameraDeviceCallbacks> cb) :
53 CompositeStream(device, cb),
54 mUseHeic(false),
55 mNumOutputTiles(1),
56 mOutputWidth(0),
57 mOutputHeight(0),
58 mMaxHeicBufferSize(0),
59 mGridWidth(HeicEncoderInfoManager::kGridWidth),
60 mGridHeight(HeicEncoderInfoManager::kGridHeight),
61 mGridRows(1),
62 mGridCols(1),
63 mUseGrid(false),
64 mAppSegmentStreamId(-1),
65 mAppSegmentSurfaceId(-1),
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -080066 mMainImageStreamId(-1),
67 mMainImageSurfaceId(-1),
68 mYuvBufferAcquired(false),
69 mProducerListener(new ProducerListener()),
Shuzhen Wang3d00ee52019-09-25 14:19:28 -070070 mDequeuedOutputBufferCnt(0),
71 mCodecOutputCounter(0),
Shuzhen Wang62f49ed2019-09-04 14:07:53 -070072 mQuality(-1),
Shuzhen Wange8675782019-12-05 09:12:14 -080073 mGridTimestampUs(0),
74 mStatusId(StatusTracker::NO_STATUS_ID) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -080075}
76
77HeicCompositeStream::~HeicCompositeStream() {
78 // Call deinitCodec in case stream hasn't been deleted yet to avoid any
79 // memory/resource leak.
80 deinitCodec();
81
82 mInputAppSegmentBuffers.clear();
83 mCodecOutputBuffers.clear();
84
85 mAppSegmentStreamId = -1;
86 mAppSegmentSurfaceId = -1;
87 mAppSegmentConsumer.clear();
88 mAppSegmentSurface.clear();
89
90 mMainImageStreamId = -1;
91 mMainImageSurfaceId = -1;
92 mMainImageConsumer.clear();
93 mMainImageSurface.clear();
94}
95
96bool HeicCompositeStream::isHeicCompositeStream(const sp<Surface> &surface) {
97 ANativeWindow *anw = surface.get();
98 status_t err;
99 int format;
100 if ((err = anw->query(anw, NATIVE_WINDOW_FORMAT, &format)) != OK) {
101 String8 msg = String8::format("Failed to query Surface format: %s (%d)", strerror(-err),
102 err);
103 ALOGE("%s: %s", __FUNCTION__, msg.string());
104 return false;
105 }
106
107 int dataspace;
108 if ((err = anw->query(anw, NATIVE_WINDOW_DEFAULT_DATASPACE, &dataspace)) != OK) {
109 String8 msg = String8::format("Failed to query Surface dataspace: %s (%d)", strerror(-err),
110 err);
111 ALOGE("%s: %s", __FUNCTION__, msg.string());
112 return false;
113 }
114
115 return ((format == HAL_PIXEL_FORMAT_BLOB) && (dataspace == HAL_DATASPACE_HEIF));
116}
117
118status_t HeicCompositeStream::createInternalStreams(const std::vector<sp<Surface>>& consumers,
119 bool /*hasDeferredConsumer*/, uint32_t width, uint32_t height, int format,
Emilian Peevf4816702020-04-03 15:44:51 -0700120 camera_stream_rotation_t rotation, int *id, const String8& physicalCameraId,
Jayant Chowdhary13f9b2f2020-12-02 22:46:15 -0800121 const std::unordered_set<int32_t> &sensorPixelModesUsed,
122 std::vector<int> *surfaceIds,
123 int /*streamSetId*/, bool /*isShared*/) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800124
125 sp<CameraDeviceBase> device = mDevice.promote();
126 if (!device.get()) {
127 ALOGE("%s: Invalid camera device!", __FUNCTION__);
128 return NO_INIT;
129 }
130
131 status_t res = initializeCodec(width, height, device);
132 if (res != OK) {
133 ALOGE("%s: Failed to initialize HEIC/HEVC codec: %s (%d)",
134 __FUNCTION__, strerror(-res), res);
135 return NO_INIT;
136 }
137
138 sp<IGraphicBufferProducer> producer;
139 sp<IGraphicBufferConsumer> consumer;
140 BufferQueue::createBufferQueue(&producer, &consumer);
Michael Gonzalezb5986a32019-10-09 15:38:17 -0700141 mAppSegmentConsumer = new CpuConsumer(consumer, kMaxAcquiredAppSegment);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800142 mAppSegmentConsumer->setFrameAvailableListener(this);
143 mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream"));
144 mAppSegmentSurface = new Surface(producer);
145
Shuzhen Wange7f4b462019-02-12 08:43:07 -0800146 mStaticInfo = device->info();
147
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800148 res = device->createStream(mAppSegmentSurface, mAppSegmentMaxSize, 1, format,
Jayant Chowdhary13f9b2f2020-12-02 22:46:15 -0800149 kAppSegmentDataSpace, rotation, &mAppSegmentStreamId, physicalCameraId,
150 sensorPixelModesUsed,surfaceIds);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800151 if (res == OK) {
152 mAppSegmentSurfaceId = (*surfaceIds)[0];
153 } else {
154 ALOGE("%s: Failed to create JPEG App segment stream: %s (%d)", __FUNCTION__,
155 strerror(-res), res);
156 return res;
157 }
158
159 if (!mUseGrid) {
160 res = mCodec->createInputSurface(&producer);
161 if (res != OK) {
162 ALOGE("%s: Failed to create input surface for Heic codec: %s (%d)",
163 __FUNCTION__, strerror(-res), res);
164 return res;
165 }
166 } else {
167 BufferQueue::createBufferQueue(&producer, &consumer);
168 mMainImageConsumer = new CpuConsumer(consumer, 1);
169 mMainImageConsumer->setFrameAvailableListener(this);
170 mMainImageConsumer->setName(String8("Camera3-HeicComposite-HevcInputYUVStream"));
171 }
172 mMainImageSurface = new Surface(producer);
173
174 res = mCodec->start();
175 if (res != OK) {
176 ALOGE("%s: Failed to start codec: %s (%d)", __FUNCTION__,
177 strerror(-res), res);
178 return res;
179 }
180
181 std::vector<int> sourceSurfaceId;
182 //Use YUV_888 format if framework tiling is needed.
183 int srcStreamFmt = mUseGrid ? HAL_PIXEL_FORMAT_YCbCr_420_888 :
184 HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
185 res = device->createStream(mMainImageSurface, width, height, srcStreamFmt, kHeifDataSpace,
Jayant Chowdhary13f9b2f2020-12-02 22:46:15 -0800186 rotation, id, physicalCameraId, sensorPixelModesUsed, &sourceSurfaceId);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800187 if (res == OK) {
188 mMainImageSurfaceId = sourceSurfaceId[0];
189 mMainImageStreamId = *id;
190 } else {
191 ALOGE("%s: Failed to create main image stream: %s (%d)", __FUNCTION__,
192 strerror(-res), res);
193 return res;
194 }
195
196 mOutputSurface = consumers[0];
Shuzhen Wange8675782019-12-05 09:12:14 -0800197 res = registerCompositeStreamListener(mMainImageStreamId);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800198 if (res != OK) {
Shuzhen Wange8675782019-12-05 09:12:14 -0800199 ALOGE("%s: Failed to register HAL main image stream: %s (%d)", __FUNCTION__,
200 strerror(-res), res);
201 return res;
202 }
203
204 res = registerCompositeStreamListener(mAppSegmentStreamId);
205 if (res != OK) {
206 ALOGE("%s: Failed to register HAL app segment stream: %s (%d)", __FUNCTION__,
207 strerror(-res), res);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800208 return res;
209 }
210
Shuzhen Wang219c2992019-02-15 17:24:28 -0800211 initCopyRowFunction(width);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800212 return res;
213}
214
215status_t HeicCompositeStream::deleteInternalStreams() {
216 requestExit();
217 auto res = join();
218 if (res != OK) {
219 ALOGE("%s: Failed to join with the main processing thread: %s (%d)", __FUNCTION__,
220 strerror(-res), res);
221 }
222
223 deinitCodec();
224
225 if (mAppSegmentStreamId >= 0) {
Emilian Peevc0fe54c2020-03-11 14:05:07 -0700226 // Camera devices may not be valid after switching to offline mode.
227 // In this case, all offline streams including internal composite streams
228 // are managed and released by the offline session.
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800229 sp<CameraDeviceBase> device = mDevice.promote();
Emilian Peevc0fe54c2020-03-11 14:05:07 -0700230 if (device.get() != nullptr) {
231 res = device->deleteStream(mAppSegmentStreamId);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800232 }
233
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800234 mAppSegmentStreamId = -1;
235 }
236
Shuzhen Wang2c545042019-02-07 10:27:35 -0800237 if (mOutputSurface != nullptr) {
238 mOutputSurface->disconnect(NATIVE_WINDOW_API_CAMERA);
239 mOutputSurface.clear();
240 }
Shuzhen Wange8675782019-12-05 09:12:14 -0800241
242 sp<StatusTracker> statusTracker = mStatusTracker.promote();
243 if (statusTracker != nullptr && mStatusId != StatusTracker::NO_STATUS_ID) {
244 statusTracker->removeComponent(mStatusId);
245 mStatusId = StatusTracker::NO_STATUS_ID;
246 }
247
248 if (mPendingInputFrames.size() > 0) {
249 ALOGW("%s: mPendingInputFrames has %zu stale entries",
250 __FUNCTION__, mPendingInputFrames.size());
251 mPendingInputFrames.clear();
252 }
253
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800254 return res;
255}
256
257void HeicCompositeStream::onBufferReleased(const BufferInfo& bufferInfo) {
258 Mutex::Autolock l(mMutex);
259
260 if (bufferInfo.mError) return;
261
Shuzhen Wange8675782019-12-05 09:12:14 -0800262 if (bufferInfo.mStreamId == mMainImageStreamId) {
263 mMainImageFrameNumbers.push(bufferInfo.mFrameNumber);
264 mCodecOutputBufferFrameNumbers.push(bufferInfo.mFrameNumber);
265 ALOGV("%s: [%" PRId64 "]: Adding main image frame number (%zu frame numbers in total)",
266 __FUNCTION__, bufferInfo.mFrameNumber, mMainImageFrameNumbers.size());
267 } else if (bufferInfo.mStreamId == mAppSegmentStreamId) {
268 mAppSegmentFrameNumbers.push(bufferInfo.mFrameNumber);
269 ALOGV("%s: [%" PRId64 "]: Adding app segment frame number (%zu frame numbers in total)",
270 __FUNCTION__, bufferInfo.mFrameNumber, mAppSegmentFrameNumbers.size());
271 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800272}
273
274// We need to get the settings early to handle the case where the codec output
275// arrives earlier than result metadata.
276void HeicCompositeStream::onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId,
277 const CameraMetadata& settings) {
278 ATRACE_ASYNC_BEGIN("HEIC capture", frameNumber);
279
280 Mutex::Autolock l(mMutex);
281 if (mErrorState || (streamId != getStreamId())) {
282 return;
283 }
284
285 mPendingCaptureResults.emplace(frameNumber, CameraMetadata());
286
287 camera_metadata_ro_entry entry;
288
289 int32_t orientation = 0;
290 entry = settings.find(ANDROID_JPEG_ORIENTATION);
291 if (entry.count == 1) {
292 orientation = entry.data.i32[0];
293 }
294
295 int32_t quality = kDefaultJpegQuality;
296 entry = settings.find(ANDROID_JPEG_QUALITY);
297 if (entry.count == 1) {
298 quality = entry.data.i32[0];
299 }
300
Shuzhen Wange8675782019-12-05 09:12:14 -0800301 mSettingsByFrameNumber[frameNumber] = {orientation, quality};
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800302}
303
304void HeicCompositeStream::onFrameAvailable(const BufferItem& item) {
305 if (item.mDataSpace == static_cast<android_dataspace>(kAppSegmentDataSpace)) {
306 ALOGV("%s: JPEG APP segments buffer with ts: %" PRIu64 " ms. arrived!",
307 __func__, ns2ms(item.mTimestamp));
308
309 Mutex::Autolock l(mMutex);
310 if (!mErrorState) {
311 mInputAppSegmentBuffers.push_back(item.mTimestamp);
312 mInputReadyCondition.signal();
313 }
314 } else if (item.mDataSpace == kHeifDataSpace) {
315 ALOGV("%s: YUV_888 buffer with ts: %" PRIu64 " ms. arrived!",
316 __func__, ns2ms(item.mTimestamp));
317
318 Mutex::Autolock l(mMutex);
319 if (!mUseGrid) {
320 ALOGE("%s: YUV_888 internal stream is only supported for HEVC tiling",
321 __FUNCTION__);
322 return;
323 }
324 if (!mErrorState) {
325 mInputYuvBuffers.push_back(item.mTimestamp);
326 mInputReadyCondition.signal();
327 }
328 } else {
329 ALOGE("%s: Unexpected data space: 0x%x", __FUNCTION__, item.mDataSpace);
330 }
331}
332
333status_t HeicCompositeStream::getCompositeStreamInfo(const OutputStreamInfo &streamInfo,
334 const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/) {
335 if (compositeOutput == nullptr) {
336 return BAD_VALUE;
337 }
338
339 compositeOutput->clear();
340
341 bool useGrid, useHeic;
342 bool isSizeSupported = isSizeSupportedByHeifEncoder(
343 streamInfo.width, streamInfo.height, &useHeic, &useGrid, nullptr);
344 if (!isSizeSupported) {
345 // Size is not supported by either encoder.
346 return OK;
347 }
348
349 compositeOutput->insert(compositeOutput->end(), 2, streamInfo);
350
351 // JPEG APPS segments Blob stream info
352 (*compositeOutput)[0].width = calcAppSegmentMaxSize(ch);
353 (*compositeOutput)[0].height = 1;
354 (*compositeOutput)[0].format = HAL_PIXEL_FORMAT_BLOB;
355 (*compositeOutput)[0].dataSpace = kAppSegmentDataSpace;
356 (*compositeOutput)[0].consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
357
358 // YUV/IMPLEMENTATION_DEFINED stream info
359 (*compositeOutput)[1].width = streamInfo.width;
360 (*compositeOutput)[1].height = streamInfo.height;
361 (*compositeOutput)[1].format = useGrid ? HAL_PIXEL_FORMAT_YCbCr_420_888 :
362 HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
363 (*compositeOutput)[1].dataSpace = kHeifDataSpace;
364 (*compositeOutput)[1].consumerUsage = useHeic ? GRALLOC_USAGE_HW_IMAGE_ENCODER :
365 useGrid ? GRALLOC_USAGE_SW_READ_OFTEN : GRALLOC_USAGE_HW_VIDEO_ENCODER;
366
367 return NO_ERROR;
368}
369
370bool HeicCompositeStream::isSizeSupportedByHeifEncoder(int32_t width, int32_t height,
Chong Zhang688abaa2019-05-17 16:32:23 -0700371 bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800372 static HeicEncoderInfoManager& heicManager = HeicEncoderInfoManager::getInstance();
Chong Zhang688abaa2019-05-17 16:32:23 -0700373 return heicManager.isSizeSupported(width, height, useHeic, useGrid, stall, hevcName);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800374}
375
376bool HeicCompositeStream::isInMemoryTempFileSupported() {
377 int memfd = syscall(__NR_memfd_create, "HEIF-try-memfd", MFD_CLOEXEC);
378 if (memfd == -1) {
379 if (errno != ENOSYS) {
380 ALOGE("%s: Failed to create tmpfs file. errno %d", __FUNCTION__, errno);
381 }
382 return false;
383 }
384 close(memfd);
385 return true;
386}
387
388void HeicCompositeStream::onHeicOutputFrameAvailable(
389 const CodecOutputBufferInfo& outputBufferInfo) {
390 Mutex::Autolock l(mMutex);
391
392 ALOGV("%s: index %d, offset %d, size %d, time %" PRId64 ", flags 0x%x",
393 __FUNCTION__, outputBufferInfo.index, outputBufferInfo.offset,
394 outputBufferInfo.size, outputBufferInfo.timeUs, outputBufferInfo.flags);
395
396 if (!mErrorState) {
397 if ((outputBufferInfo.size > 0) &&
398 ((outputBufferInfo.flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) == 0)) {
399 mCodecOutputBuffers.push_back(outputBufferInfo);
400 mInputReadyCondition.signal();
401 } else {
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700402 ALOGV("%s: Releasing output buffer: size %d flags: 0x%x ", __FUNCTION__,
403 outputBufferInfo.size, outputBufferInfo.flags);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800404 mCodec->releaseOutputBuffer(outputBufferInfo.index);
405 }
406 } else {
407 mCodec->releaseOutputBuffer(outputBufferInfo.index);
408 }
409}
410
411void HeicCompositeStream::onHeicInputFrameAvailable(int32_t index) {
412 Mutex::Autolock l(mMutex);
413
414 if (!mUseGrid) {
415 ALOGE("%s: Codec YUV input mode must only be used for Hevc tiling mode", __FUNCTION__);
416 return;
417 }
418
419 mCodecInputBuffers.push_back(index);
420 mInputReadyCondition.signal();
421}
422
423void HeicCompositeStream::onHeicFormatChanged(sp<AMessage>& newFormat) {
424 if (newFormat == nullptr) {
425 ALOGE("%s: newFormat must not be null!", __FUNCTION__);
426 return;
427 }
428
429 Mutex::Autolock l(mMutex);
430
431 AString mime;
432 AString mimeHeic(MIMETYPE_IMAGE_ANDROID_HEIC);
433 newFormat->findString(KEY_MIME, &mime);
434 if (mime != mimeHeic) {
435 // For HEVC codec, below keys need to be filled out or overwritten so that the
436 // muxer can handle them as HEIC output image.
437 newFormat->setString(KEY_MIME, mimeHeic);
438 newFormat->setInt32(KEY_WIDTH, mOutputWidth);
439 newFormat->setInt32(KEY_HEIGHT, mOutputHeight);
440 if (mUseGrid) {
441 newFormat->setInt32(KEY_TILE_WIDTH, mGridWidth);
442 newFormat->setInt32(KEY_TILE_HEIGHT, mGridHeight);
443 newFormat->setInt32(KEY_GRID_ROWS, mGridRows);
444 newFormat->setInt32(KEY_GRID_COLUMNS, mGridCols);
dan.shen60fa4b92022-04-12 18:57:43 +0800445 int32_t left, top, right, bottom;
446 if (newFormat->findRect("crop", &left, &top, &right, &bottom)) {
447 newFormat->setRect("crop", 0, 0, mOutputWidth - 1, mOutputHeight - 1);
448 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800449 }
450 }
451 newFormat->setInt32(KEY_IS_DEFAULT, 1 /*isPrimary*/);
452
453 int32_t gridRows, gridCols;
454 if (newFormat->findInt32(KEY_GRID_ROWS, &gridRows) &&
455 newFormat->findInt32(KEY_GRID_COLUMNS, &gridCols)) {
456 mNumOutputTiles = gridRows * gridCols;
457 } else {
458 mNumOutputTiles = 1;
459 }
460
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800461 mFormat = newFormat;
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700462
463 ALOGV("%s: mNumOutputTiles is %zu", __FUNCTION__, mNumOutputTiles);
464 mInputReadyCondition.signal();
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800465}
466
467void HeicCompositeStream::onHeicCodecError() {
468 Mutex::Autolock l(mMutex);
469 mErrorState = true;
470}
471
472status_t HeicCompositeStream::configureStream() {
473 if (isRunning()) {
474 // Processing thread is already running, nothing more to do.
475 return NO_ERROR;
476 }
477
478 if (mOutputSurface.get() == nullptr) {
479 ALOGE("%s: No valid output surface set!", __FUNCTION__);
480 return NO_INIT;
481 }
482
483 auto res = mOutputSurface->connect(NATIVE_WINDOW_API_CAMERA, mProducerListener);
484 if (res != OK) {
485 ALOGE("%s: Unable to connect to native window for stream %d",
486 __FUNCTION__, mMainImageStreamId);
487 return res;
488 }
489
490 if ((res = native_window_set_buffers_format(mOutputSurface.get(), HAL_PIXEL_FORMAT_BLOB))
491 != OK) {
492 ALOGE("%s: Unable to configure stream buffer format for stream %d", __FUNCTION__,
493 mMainImageStreamId);
494 return res;
495 }
496
497 ANativeWindow *anwConsumer = mOutputSurface.get();
498 int maxConsumerBuffers;
499 if ((res = anwConsumer->query(anwConsumer, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS,
500 &maxConsumerBuffers)) != OK) {
501 ALOGE("%s: Unable to query consumer undequeued"
502 " buffer count for stream %d", __FUNCTION__, mMainImageStreamId);
503 return res;
504 }
505
506 // Cannot use SourceSurface buffer count since it could be codec's 512*512 tile
507 // buffer count.
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800508 if ((res = native_window_set_buffer_count(
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700509 anwConsumer, kMaxOutputSurfaceProducerCount + maxConsumerBuffers)) != OK) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800510 ALOGE("%s: Unable to set buffer count for stream %d", __FUNCTION__, mMainImageStreamId);
511 return res;
512 }
513
514 if ((res = native_window_set_buffers_dimensions(anwConsumer, mMaxHeicBufferSize, 1)) != OK) {
515 ALOGE("%s: Unable to set buffer dimension %zu x 1 for stream %d: %s (%d)",
516 __FUNCTION__, mMaxHeicBufferSize, mMainImageStreamId, strerror(-res), res);
517 return res;
518 }
519
Shuzhen Wange8675782019-12-05 09:12:14 -0800520 sp<camera3::StatusTracker> statusTracker = mStatusTracker.promote();
521 if (statusTracker != nullptr) {
Yin-Chia Yeh87b3ec02019-06-03 10:44:39 -0700522 std::string name = std::string("HeicStream ") + std::to_string(getStreamId());
523 mStatusId = statusTracker->addComponent(name);
Shuzhen Wange8675782019-12-05 09:12:14 -0800524 }
525
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800526 run("HeicCompositeStreamProc");
527
528 return NO_ERROR;
529}
530
531status_t HeicCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap,
532 Vector<int32_t>* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) {
533 if (outSurfaceMap->find(mAppSegmentStreamId) == outSurfaceMap->end()) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800534 outputStreamIds->push_back(mAppSegmentStreamId);
535 }
536 (*outSurfaceMap)[mAppSegmentStreamId].push_back(mAppSegmentSurfaceId);
537
538 if (outSurfaceMap->find(mMainImageStreamId) == outSurfaceMap->end()) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800539 outputStreamIds->push_back(mMainImageStreamId);
540 }
541 (*outSurfaceMap)[mMainImageStreamId].push_back(mMainImageSurfaceId);
542
543 if (currentStreamId != nullptr) {
544 *currentStreamId = mMainImageStreamId;
545 }
546
547 return NO_ERROR;
548}
549
Emilian Peev4697b642019-11-19 17:11:14 -0800550status_t HeicCompositeStream::insertCompositeStreamIds(
551 std::vector<int32_t>* compositeStreamIds /*out*/) {
552 if (compositeStreamIds == nullptr) {
553 return BAD_VALUE;
554 }
555
556 compositeStreamIds->push_back(mAppSegmentStreamId);
557 compositeStreamIds->push_back(mMainImageStreamId);
558
559 return OK;
560}
561
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800562void HeicCompositeStream::onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) {
563 Mutex::Autolock l(mMutex);
564 if (mErrorState) {
565 return;
566 }
567
568 if (mSettingsByFrameNumber.find(resultExtras.frameNumber) != mSettingsByFrameNumber.end()) {
Shuzhen Wange8675782019-12-05 09:12:14 -0800569 ALOGV("%s: [%" PRId64 "]: timestamp %" PRId64 ", requestId %d", __FUNCTION__,
570 resultExtras.frameNumber, timestamp, resultExtras.requestId);
571 mSettingsByFrameNumber[resultExtras.frameNumber].shutterNotified = true;
572 mSettingsByFrameNumber[resultExtras.frameNumber].timestamp = timestamp;
573 mSettingsByFrameNumber[resultExtras.frameNumber].requestId = resultExtras.requestId;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800574 mInputReadyCondition.signal();
575 }
576}
577
578void HeicCompositeStream::compilePendingInputLocked() {
Shuzhen Wange8675782019-12-05 09:12:14 -0800579 auto i = mSettingsByFrameNumber.begin();
580 while (i != mSettingsByFrameNumber.end()) {
581 if (i->second.shutterNotified) {
582 mPendingInputFrames[i->first].orientation = i->second.orientation;
583 mPendingInputFrames[i->first].quality = i->second.quality;
584 mPendingInputFrames[i->first].timestamp = i->second.timestamp;
585 mPendingInputFrames[i->first].requestId = i->second.requestId;
586 ALOGV("%s: [%" PRId64 "]: timestamp is %" PRId64, __FUNCTION__,
587 i->first, i->second.timestamp);
588 i = mSettingsByFrameNumber.erase(i);
Shuzhen Wang62f49ed2019-09-04 14:07:53 -0700589
Shuzhen Wange8675782019-12-05 09:12:14 -0800590 // Set encoder quality if no inflight encoding
591 if (mPendingInputFrames.size() == 1) {
592 sp<StatusTracker> statusTracker = mStatusTracker.promote();
593 if (statusTracker != nullptr) {
594 statusTracker->markComponentActive(mStatusId);
595 ALOGV("%s: Mark component as active", __FUNCTION__);
596 }
597
598 int32_t newQuality = mPendingInputFrames.begin()->second.quality;
599 updateCodecQualityLocked(newQuality);
600 }
601 } else {
602 i++;
Shuzhen Wang62f49ed2019-09-04 14:07:53 -0700603 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800604 }
605
Shuzhen Wange8675782019-12-05 09:12:14 -0800606 while (!mInputAppSegmentBuffers.empty() && mAppSegmentFrameNumbers.size() > 0) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800607 CpuConsumer::LockedBuffer imgBuffer;
608 auto it = mInputAppSegmentBuffers.begin();
609 auto res = mAppSegmentConsumer->lockNextBuffer(&imgBuffer);
610 if (res == NOT_ENOUGH_DATA) {
Michael Gonzalezb5986a32019-10-09 15:38:17 -0700611 // Can not lock any more buffers.
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800612 break;
613 } else if ((res != OK) || (*it != imgBuffer.timestamp)) {
614 if (res != OK) {
615 ALOGE("%s: Error locking JPEG_APP_SEGMENTS image buffer: %s (%d)", __FUNCTION__,
616 strerror(-res), res);
617 } else {
618 ALOGE("%s: Expecting JPEG_APP_SEGMENTS buffer with time stamp: %" PRId64
619 " received buffer with time stamp: %" PRId64, __FUNCTION__,
620 *it, imgBuffer.timestamp);
Michael Gonzalezb5986a32019-10-09 15:38:17 -0700621 mAppSegmentConsumer->unlockBuffer(imgBuffer);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800622 }
623 mPendingInputFrames[*it].error = true;
624 mInputAppSegmentBuffers.erase(it);
625 continue;
626 }
627
Shuzhen Wange8675782019-12-05 09:12:14 -0800628 if (mPendingInputFrames.find(mAppSegmentFrameNumbers.front()) == mPendingInputFrames.end()) {
629 ALOGE("%s: mPendingInputFrames doesn't contain frameNumber %" PRId64, __FUNCTION__,
630 mAppSegmentFrameNumbers.front());
Shuzhen Wang991b7b62020-06-17 11:39:11 -0700631 mInputAppSegmentBuffers.erase(it);
632 mAppSegmentFrameNumbers.pop();
Shuzhen Wange8675782019-12-05 09:12:14 -0800633 continue;
634 }
635
636 int64_t frameNumber = mAppSegmentFrameNumbers.front();
637 // If mPendingInputFrames doesn't contain the expected frame number, the captured
638 // input app segment frame must have been dropped via a buffer error. Simply
639 // return the buffer to the buffer queue.
640 if ((mPendingInputFrames.find(frameNumber) == mPendingInputFrames.end()) ||
641 (mPendingInputFrames[frameNumber].error)) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800642 mAppSegmentConsumer->unlockBuffer(imgBuffer);
643 } else {
Shuzhen Wange8675782019-12-05 09:12:14 -0800644 mPendingInputFrames[frameNumber].appSegmentBuffer = imgBuffer;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800645 }
646 mInputAppSegmentBuffers.erase(it);
Shuzhen Wange8675782019-12-05 09:12:14 -0800647 mAppSegmentFrameNumbers.pop();
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800648 }
649
Shuzhen Wange8675782019-12-05 09:12:14 -0800650 while (!mInputYuvBuffers.empty() && !mYuvBufferAcquired && mMainImageFrameNumbers.size() > 0) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800651 CpuConsumer::LockedBuffer imgBuffer;
652 auto it = mInputYuvBuffers.begin();
653 auto res = mMainImageConsumer->lockNextBuffer(&imgBuffer);
654 if (res == NOT_ENOUGH_DATA) {
Michael Gonzalezb5986a32019-10-09 15:38:17 -0700655 // Can not lock any more buffers.
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800656 break;
657 } else if (res != OK) {
658 ALOGE("%s: Error locking YUV_888 image buffer: %s (%d)", __FUNCTION__,
659 strerror(-res), res);
660 mPendingInputFrames[*it].error = true;
661 mInputYuvBuffers.erase(it);
662 continue;
663 } else if (*it != imgBuffer.timestamp) {
664 ALOGW("%s: Expecting YUV_888 buffer with time stamp: %" PRId64 " received buffer with "
665 "time stamp: %" PRId64, __FUNCTION__, *it, imgBuffer.timestamp);
666 mPendingInputFrames[*it].error = true;
667 mInputYuvBuffers.erase(it);
668 continue;
669 }
670
Shuzhen Wange8675782019-12-05 09:12:14 -0800671 if (mPendingInputFrames.find(mMainImageFrameNumbers.front()) == mPendingInputFrames.end()) {
672 ALOGE("%s: mPendingInputFrames doesn't contain frameNumber %" PRId64, __FUNCTION__,
673 mMainImageFrameNumbers.front());
674 mInputYuvBuffers.erase(it);
Shuzhen Wang991b7b62020-06-17 11:39:11 -0700675 mMainImageFrameNumbers.pop();
Shuzhen Wange8675782019-12-05 09:12:14 -0800676 continue;
677 }
678
679 int64_t frameNumber = mMainImageFrameNumbers.front();
680 // If mPendingInputFrames doesn't contain the expected frame number, the captured
681 // input main image must have been dropped via a buffer error. Simply
682 // return the buffer to the buffer queue.
683 if ((mPendingInputFrames.find(frameNumber) == mPendingInputFrames.end()) ||
684 (mPendingInputFrames[frameNumber].error)) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800685 mMainImageConsumer->unlockBuffer(imgBuffer);
686 } else {
Shuzhen Wange8675782019-12-05 09:12:14 -0800687 mPendingInputFrames[frameNumber].yuvBuffer = imgBuffer;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800688 mYuvBufferAcquired = true;
689 }
690 mInputYuvBuffers.erase(it);
Shuzhen Wange8675782019-12-05 09:12:14 -0800691 mMainImageFrameNumbers.pop();
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800692 }
693
694 while (!mCodecOutputBuffers.empty()) {
695 auto it = mCodecOutputBuffers.begin();
Shuzhen Wange8675782019-12-05 09:12:14 -0800696 // Assume encoder input to output is FIFO, use a queue to look up
697 // frameNumber when handling codec outputs.
698 int64_t bufferFrameNumber = -1;
699 if (mCodecOutputBufferFrameNumbers.empty()) {
700 ALOGV("%s: Failed to find buffer frameNumber for codec output buffer!", __FUNCTION__);
Michael Gonzalez5c103f22019-10-08 14:30:32 -0700701 break;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800702 } else {
Shuzhen Wange8675782019-12-05 09:12:14 -0800703 // Direct mapping between camera frame number and codec timestamp (in us).
704 bufferFrameNumber = mCodecOutputBufferFrameNumbers.front();
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700705 mCodecOutputCounter++;
706 if (mCodecOutputCounter == mNumOutputTiles) {
Shuzhen Wange8675782019-12-05 09:12:14 -0800707 mCodecOutputBufferFrameNumbers.pop();
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700708 mCodecOutputCounter = 0;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800709 }
710
Shuzhen Wange8675782019-12-05 09:12:14 -0800711 mPendingInputFrames[bufferFrameNumber].codecOutputBuffers.push_back(*it);
712 ALOGV("%s: [%" PRId64 "]: Pushing codecOutputBuffers (frameNumber %" PRId64 ")",
713 __FUNCTION__, bufferFrameNumber, it->timeUs);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800714 }
715 mCodecOutputBuffers.erase(it);
716 }
717
Shuzhen Wange7f4b462019-02-12 08:43:07 -0800718 while (!mCaptureResults.empty()) {
719 auto it = mCaptureResults.begin();
Shuzhen Wange8675782019-12-05 09:12:14 -0800720 // Negative frame number indicates that something went wrong during the capture result
Shuzhen Wange7f4b462019-02-12 08:43:07 -0800721 // collection process.
Shuzhen Wange8675782019-12-05 09:12:14 -0800722 int64_t frameNumber = std::get<0>(it->second);
723 if (it->first >= 0 &&
724 mPendingInputFrames.find(frameNumber) != mPendingInputFrames.end()) {
725 if (mPendingInputFrames[frameNumber].timestamp == it->first) {
726 mPendingInputFrames[frameNumber].result =
Shuzhen Wange7f4b462019-02-12 08:43:07 -0800727 std::make_unique<CameraMetadata>(std::get<1>(it->second));
728 } else {
729 ALOGE("%s: Capture result frameNumber/timestamp mapping changed between "
Shuzhen Wange8675782019-12-05 09:12:14 -0800730 "shutter and capture result! before: %" PRId64 ", after: %" PRId64,
731 __FUNCTION__, mPendingInputFrames[frameNumber].timestamp,
732 it->first);
Shuzhen Wange7f4b462019-02-12 08:43:07 -0800733 }
734 }
735 mCaptureResults.erase(it);
736 }
737
738 // mErrorFrameNumbers stores frame number of dropped buffers.
739 auto it = mErrorFrameNumbers.begin();
740 while (it != mErrorFrameNumbers.end()) {
Shuzhen Wange8675782019-12-05 09:12:14 -0800741 if (mPendingInputFrames.find(*it) != mPendingInputFrames.end()) {
742 mPendingInputFrames[*it].error = true;
Shuzhen Wange7f4b462019-02-12 08:43:07 -0800743 } else {
Shuzhen Wange8675782019-12-05 09:12:14 -0800744 //Error callback is guaranteed to arrive after shutter notify, which
745 //results in mPendingInputFrames being populated.
Shuzhen Wange7f4b462019-02-12 08:43:07 -0800746 ALOGW("%s: Not able to find failing input with frame number: %" PRId64, __FUNCTION__,
747 *it);
Shuzhen Wange7f4b462019-02-12 08:43:07 -0800748 }
Shuzhen Wange8675782019-12-05 09:12:14 -0800749 it = mErrorFrameNumbers.erase(it);
750 }
751
752 // mExifErrorFrameNumbers stores the frame number of dropped APP_SEGMENT buffers
753 it = mExifErrorFrameNumbers.begin();
754 while (it != mExifErrorFrameNumbers.end()) {
755 if (mPendingInputFrames.find(*it) != mPendingInputFrames.end()) {
756 mPendingInputFrames[*it].exifError = true;
757 }
758 it = mExifErrorFrameNumbers.erase(it);
Shuzhen Wange7f4b462019-02-12 08:43:07 -0800759 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800760
761 // Distribute codec input buffers to be filled out from YUV output
762 for (auto it = mPendingInputFrames.begin();
763 it != mPendingInputFrames.end() && mCodecInputBuffers.size() > 0; it++) {
764 InputFrame& inputFrame(it->second);
765 if (inputFrame.codecInputCounter < mGridRows * mGridCols) {
766 // Available input tiles that are required for the current input
767 // image.
768 size_t newInputTiles = std::min(mCodecInputBuffers.size(),
769 mGridRows * mGridCols - inputFrame.codecInputCounter);
770 for (size_t i = 0; i < newInputTiles; i++) {
771 CodecInputBufferInfo inputInfo =
772 { mCodecInputBuffers[0], mGridTimestampUs++, inputFrame.codecInputCounter };
773 inputFrame.codecInputBuffers.push_back(inputInfo);
774
775 mCodecInputBuffers.erase(mCodecInputBuffers.begin());
776 inputFrame.codecInputCounter++;
777 }
778 break;
779 }
780 }
781}
782
Shuzhen Wange8675782019-12-05 09:12:14 -0800783bool HeicCompositeStream::getNextReadyInputLocked(int64_t *frameNumber /*out*/) {
784 if (frameNumber == nullptr) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800785 return false;
786 }
787
788 bool newInputAvailable = false;
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700789 for (auto& it : mPendingInputFrames) {
790 // New input is considered to be available only if:
791 // 1. input buffers are ready, or
792 // 2. App segment and muxer is created, or
793 // 3. A codec output tile is ready, and an output buffer is available.
794 // This makes sure that muxer gets created only when an output tile is
795 // generated, because right now we only handle 1 HEIC output buffer at a
796 // time (max dequeued buffer count is 1).
Shuzhen Wange8675782019-12-05 09:12:14 -0800797 bool appSegmentReady =
798 (it.second.appSegmentBuffer.data != nullptr || it.second.exifError) &&
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700799 !it.second.appSegmentWritten && it.second.result != nullptr &&
800 it.second.muxer != nullptr;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800801 bool codecOutputReady = !it.second.codecOutputBuffers.empty();
802 bool codecInputReady = (it.second.yuvBuffer.data != nullptr) &&
803 (!it.second.codecInputBuffers.empty());
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700804 bool hasOutputBuffer = it.second.muxer != nullptr ||
805 (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800806 if ((!it.second.error) &&
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700807 (appSegmentReady || (codecOutputReady && hasOutputBuffer) || codecInputReady)) {
Shuzhen Wange8675782019-12-05 09:12:14 -0800808 *frameNumber = it.first;
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700809 if (it.second.format == nullptr && mFormat != nullptr) {
810 it.second.format = mFormat->dup();
811 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800812 newInputAvailable = true;
813 break;
814 }
815 }
816
817 return newInputAvailable;
818}
819
Shuzhen Wange8675782019-12-05 09:12:14 -0800820int64_t HeicCompositeStream::getNextFailingInputLocked() {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800821 int64_t res = -1;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800822
823 for (const auto& it : mPendingInputFrames) {
Shuzhen Wange8675782019-12-05 09:12:14 -0800824 if (it.second.error) {
825 res = it.first;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800826 break;
827 }
828 }
829
830 return res;
831}
832
Shuzhen Wange8675782019-12-05 09:12:14 -0800833status_t HeicCompositeStream::processInputFrame(int64_t frameNumber,
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800834 InputFrame &inputFrame) {
835 ATRACE_CALL();
836 status_t res = OK;
837
Shuzhen Wange8675782019-12-05 09:12:14 -0800838 bool appSegmentReady =
839 (inputFrame.appSegmentBuffer.data != nullptr || inputFrame.exifError) &&
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700840 !inputFrame.appSegmentWritten && inputFrame.result != nullptr &&
841 inputFrame.muxer != nullptr;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800842 bool codecOutputReady = inputFrame.codecOutputBuffers.size() > 0;
843 bool codecInputReady = inputFrame.yuvBuffer.data != nullptr &&
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700844 !inputFrame.codecInputBuffers.empty();
845 bool hasOutputBuffer = inputFrame.muxer != nullptr ||
846 (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800847
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700848 ALOGV("%s: [%" PRId64 "]: appSegmentReady %d, codecOutputReady %d, codecInputReady %d,"
Shuzhen Wange8675782019-12-05 09:12:14 -0800849 " dequeuedOutputBuffer %d, timestamp %" PRId64, __FUNCTION__, frameNumber,
850 appSegmentReady, codecOutputReady, codecInputReady, mDequeuedOutputBufferCnt,
851 inputFrame.timestamp);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800852
853 // Handle inputs for Hevc tiling
854 if (codecInputReady) {
855 res = processCodecInputFrame(inputFrame);
856 if (res != OK) {
857 ALOGE("%s: Failed to process codec input frame: %s (%d)", __FUNCTION__,
858 strerror(-res), res);
859 return res;
860 }
861 }
862
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700863 if (!(codecOutputReady && hasOutputBuffer) && !appSegmentReady) {
864 return OK;
865 }
866
867 // Initialize and start muxer if not yet done so. In this case,
868 // codecOutputReady must be true. Otherwise, appSegmentReady is guaranteed
869 // to be false, and the function must have returned early.
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800870 if (inputFrame.muxer == nullptr) {
Shuzhen Wange8675782019-12-05 09:12:14 -0800871 res = startMuxerForInputFrame(frameNumber, inputFrame);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800872 if (res != OK) {
873 ALOGE("%s: Failed to create and start muxer: %s (%d)", __FUNCTION__,
874 strerror(-res), res);
875 return res;
876 }
877 }
878
879 // Write JPEG APP segments data to the muxer.
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700880 if (appSegmentReady) {
Shuzhen Wange8675782019-12-05 09:12:14 -0800881 res = processAppSegment(frameNumber, inputFrame);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800882 if (res != OK) {
883 ALOGE("%s: Failed to process JPEG APP segments: %s (%d)", __FUNCTION__,
884 strerror(-res), res);
885 return res;
886 }
887 }
888
889 // Write media codec bitstream buffers to muxer.
890 while (!inputFrame.codecOutputBuffers.empty()) {
Shuzhen Wange8675782019-12-05 09:12:14 -0800891 res = processOneCodecOutputFrame(frameNumber, inputFrame);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800892 if (res != OK) {
893 ALOGE("%s: Failed to process codec output frame: %s (%d)", __FUNCTION__,
894 strerror(-res), res);
895 return res;
896 }
897 }
898
Michael Gonzalezb5986a32019-10-09 15:38:17 -0700899 if (inputFrame.pendingOutputTiles == 0) {
900 if (inputFrame.appSegmentWritten) {
Shuzhen Wange8675782019-12-05 09:12:14 -0800901 res = processCompletedInputFrame(frameNumber, inputFrame);
Michael Gonzalezb5986a32019-10-09 15:38:17 -0700902 if (res != OK) {
903 ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__,
904 strerror(-res), res);
905 return res;
906 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800907 }
908 }
909
910 return res;
911}
912
Shuzhen Wange8675782019-12-05 09:12:14 -0800913status_t HeicCompositeStream::startMuxerForInputFrame(int64_t frameNumber, InputFrame &inputFrame) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800914 sp<ANativeWindow> outputANW = mOutputSurface;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800915
916 auto res = outputANW->dequeueBuffer(mOutputSurface.get(), &inputFrame.anb, &inputFrame.fenceFd);
917 if (res != OK) {
918 ALOGE("%s: Error retrieving output buffer: %s (%d)", __FUNCTION__, strerror(-res),
919 res);
920 return res;
921 }
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700922 mDequeuedOutputBufferCnt++;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800923
924 // Combine current thread id, stream id and timestamp to uniquely identify image.
925 std::ostringstream tempOutputFile;
926 tempOutputFile << "HEIF-" << pthread_self() << "-"
Shuzhen Wange8675782019-12-05 09:12:14 -0800927 << getStreamId() << "-" << frameNumber;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800928 inputFrame.fileFd = syscall(__NR_memfd_create, tempOutputFile.str().c_str(), MFD_CLOEXEC);
929 if (inputFrame.fileFd < 0) {
930 ALOGE("%s: Failed to create file %s. Error no is %d", __FUNCTION__,
931 tempOutputFile.str().c_str(), errno);
932 return NO_INIT;
933 }
934 inputFrame.muxer = new MediaMuxer(inputFrame.fileFd, MediaMuxer::OUTPUT_FORMAT_HEIF);
935 if (inputFrame.muxer == nullptr) {
936 ALOGE("%s: Failed to create MediaMuxer for file fd %d",
937 __FUNCTION__, inputFrame.fileFd);
938 return NO_INIT;
939 }
940
941 res = inputFrame.muxer->setOrientationHint(inputFrame.orientation);
942 if (res != OK) {
943 ALOGE("%s: Failed to setOrientationHint: %s (%d)", __FUNCTION__,
944 strerror(-res), res);
945 return res;
946 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800947
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700948 ssize_t trackId = inputFrame.muxer->addTrack(inputFrame.format);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800949 if (trackId < 0) {
950 ALOGE("%s: Failed to addTrack to the muxer: %zd", __FUNCTION__, trackId);
951 return NO_INIT;
952 }
953
954 inputFrame.trackIndex = trackId;
955 inputFrame.pendingOutputTiles = mNumOutputTiles;
956
957 res = inputFrame.muxer->start();
958 if (res != OK) {
959 ALOGE("%s: Failed to start MediaMuxer: %s (%d)",
960 __FUNCTION__, strerror(-res), res);
961 return res;
962 }
963
Shuzhen Wang3d00ee52019-09-25 14:19:28 -0700964 ALOGV("%s: [%" PRId64 "]: Muxer started for inputFrame", __FUNCTION__,
Shuzhen Wange8675782019-12-05 09:12:14 -0800965 frameNumber);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800966 return OK;
967}
968
Shuzhen Wange8675782019-12-05 09:12:14 -0800969status_t HeicCompositeStream::processAppSegment(int64_t frameNumber, InputFrame &inputFrame) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800970 size_t app1Size = 0;
Shuzhen Wange8675782019-12-05 09:12:14 -0800971 size_t appSegmentSize = 0;
972 if (!inputFrame.exifError) {
973 appSegmentSize = findAppSegmentsSize(inputFrame.appSegmentBuffer.data,
974 inputFrame.appSegmentBuffer.width * inputFrame.appSegmentBuffer.height,
975 &app1Size);
976 if (appSegmentSize == 0) {
977 ALOGE("%s: Failed to find JPEG APP segment size", __FUNCTION__);
978 return NO_INIT;
979 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800980 }
981
982 std::unique_ptr<ExifUtils> exifUtils(ExifUtils::create());
Shuzhen Wange8675782019-12-05 09:12:14 -0800983 auto exifRes = inputFrame.exifError ?
984 exifUtils->initializeEmpty() :
985 exifUtils->initialize(inputFrame.appSegmentBuffer.data, app1Size);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800986 if (!exifRes) {
987 ALOGE("%s: Failed to initialize ExifUtils object!", __FUNCTION__);
988 return BAD_VALUE;
989 }
Shuzhen Wange7f4b462019-02-12 08:43:07 -0800990 exifRes = exifUtils->setFromMetadata(*inputFrame.result, mStaticInfo,
991 mOutputWidth, mOutputHeight);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -0800992 if (!exifRes) {
993 ALOGE("%s: Failed to set Exif tags using metadata and main image sizes", __FUNCTION__);
994 return BAD_VALUE;
995 }
996 exifRes = exifUtils->setOrientation(inputFrame.orientation);
997 if (!exifRes) {
998 ALOGE("%s: ExifUtils failed to set orientation", __FUNCTION__);
999 return BAD_VALUE;
1000 }
1001 exifRes = exifUtils->generateApp1();
1002 if (!exifRes) {
1003 ALOGE("%s: ExifUtils failed to generate APP1 segment", __FUNCTION__);
1004 return BAD_VALUE;
1005 }
1006
1007 unsigned int newApp1Length = exifUtils->getApp1Length();
1008 const uint8_t *newApp1Segment = exifUtils->getApp1Buffer();
1009
1010 //Assemble the APP1 marker buffer required by MediaCodec
1011 uint8_t kExifApp1Marker[] = {'E', 'x', 'i', 'f', 0xFF, 0xE1, 0x00, 0x00};
1012 kExifApp1Marker[6] = static_cast<uint8_t>(newApp1Length >> 8);
1013 kExifApp1Marker[7] = static_cast<uint8_t>(newApp1Length & 0xFF);
1014 size_t appSegmentBufferSize = sizeof(kExifApp1Marker) +
1015 appSegmentSize - app1Size + newApp1Length;
1016 uint8_t* appSegmentBuffer = new uint8_t[appSegmentBufferSize];
1017 memcpy(appSegmentBuffer, kExifApp1Marker, sizeof(kExifApp1Marker));
1018 memcpy(appSegmentBuffer + sizeof(kExifApp1Marker), newApp1Segment, newApp1Length);
1019 if (appSegmentSize - app1Size > 0) {
1020 memcpy(appSegmentBuffer + sizeof(kExifApp1Marker) + newApp1Length,
1021 inputFrame.appSegmentBuffer.data + app1Size, appSegmentSize - app1Size);
1022 }
1023
1024 sp<ABuffer> aBuffer = new ABuffer(appSegmentBuffer, appSegmentBufferSize);
1025 auto res = inputFrame.muxer->writeSampleData(aBuffer, inputFrame.trackIndex,
Shuzhen Wange8675782019-12-05 09:12:14 -08001026 inputFrame.timestamp, MediaCodec::BUFFER_FLAG_MUXER_DATA);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001027 delete[] appSegmentBuffer;
1028
1029 if (res != OK) {
1030 ALOGE("%s: Failed to write JPEG APP segments to muxer: %s (%d)",
1031 __FUNCTION__, strerror(-res), res);
1032 return res;
1033 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001034
Shuzhen Wang3d00ee52019-09-25 14:19:28 -07001035 ALOGV("%s: [%" PRId64 "]: appSegmentSize is %zu, width %d, height %d, app1Size %zu",
Shuzhen Wange8675782019-12-05 09:12:14 -08001036 __FUNCTION__, frameNumber, appSegmentSize, inputFrame.appSegmentBuffer.width,
Shuzhen Wang3d00ee52019-09-25 14:19:28 -07001037 inputFrame.appSegmentBuffer.height, app1Size);
Michael Gonzalezb5986a32019-10-09 15:38:17 -07001038
1039 inputFrame.appSegmentWritten = true;
1040 // Release the buffer now so any pending input app segments can be processed
1041 mAppSegmentConsumer->unlockBuffer(inputFrame.appSegmentBuffer);
1042 inputFrame.appSegmentBuffer.data = nullptr;
Shuzhen Wange8675782019-12-05 09:12:14 -08001043 inputFrame.exifError = false;
Michael Gonzalezb5986a32019-10-09 15:38:17 -07001044
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001045 return OK;
1046}
1047
1048status_t HeicCompositeStream::processCodecInputFrame(InputFrame &inputFrame) {
1049 for (auto& inputBuffer : inputFrame.codecInputBuffers) {
1050 sp<MediaCodecBuffer> buffer;
1051 auto res = mCodec->getInputBuffer(inputBuffer.index, &buffer);
1052 if (res != OK) {
1053 ALOGE("%s: Error getting codec input buffer: %s (%d)", __FUNCTION__,
1054 strerror(-res), res);
1055 return res;
1056 }
1057
1058 // Copy one tile from source to destination.
1059 size_t tileX = inputBuffer.tileIndex % mGridCols;
1060 size_t tileY = inputBuffer.tileIndex / mGridCols;
1061 size_t top = mGridHeight * tileY;
1062 size_t left = mGridWidth * tileX;
1063 size_t width = (tileX == static_cast<size_t>(mGridCols) - 1) ?
1064 mOutputWidth - tileX * mGridWidth : mGridWidth;
1065 size_t height = (tileY == static_cast<size_t>(mGridRows) - 1) ?
1066 mOutputHeight - tileY * mGridHeight : mGridHeight;
Shuzhen Wang3d00ee52019-09-25 14:19:28 -07001067 ALOGV("%s: inputBuffer tileIndex [%zu, %zu], top %zu, left %zu, width %zu, height %zu,"
1068 " timeUs %" PRId64, __FUNCTION__, tileX, tileY, top, left, width, height,
1069 inputBuffer.timeUs);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001070
1071 res = copyOneYuvTile(buffer, inputFrame.yuvBuffer, top, left, width, height);
1072 if (res != OK) {
1073 ALOGE("%s: Failed to copy YUV tile %s (%d)", __FUNCTION__,
1074 strerror(-res), res);
1075 return res;
1076 }
1077
1078 res = mCodec->queueInputBuffer(inputBuffer.index, 0, buffer->capacity(),
1079 inputBuffer.timeUs, 0, nullptr /*errorDetailMsg*/);
1080 if (res != OK) {
1081 ALOGE("%s: Failed to queueInputBuffer to Codec: %s (%d)",
1082 __FUNCTION__, strerror(-res), res);
1083 return res;
1084 }
1085 }
1086
1087 inputFrame.codecInputBuffers.clear();
1088 return OK;
1089}
1090
Shuzhen Wange8675782019-12-05 09:12:14 -08001091status_t HeicCompositeStream::processOneCodecOutputFrame(int64_t frameNumber,
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001092 InputFrame &inputFrame) {
1093 auto it = inputFrame.codecOutputBuffers.begin();
1094 sp<MediaCodecBuffer> buffer;
1095 status_t res = mCodec->getOutputBuffer(it->index, &buffer);
1096 if (res != OK) {
1097 ALOGE("%s: Error getting Heic codec output buffer at index %d: %s (%d)",
1098 __FUNCTION__, it->index, strerror(-res), res);
1099 return res;
1100 }
1101 if (buffer == nullptr) {
1102 ALOGE("%s: Invalid Heic codec output buffer at index %d",
1103 __FUNCTION__, it->index);
1104 return BAD_VALUE;
1105 }
1106
1107 sp<ABuffer> aBuffer = new ABuffer(buffer->data(), buffer->size());
1108 res = inputFrame.muxer->writeSampleData(
Shuzhen Wange8675782019-12-05 09:12:14 -08001109 aBuffer, inputFrame.trackIndex, inputFrame.timestamp, 0 /*flags*/);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001110 if (res != OK) {
1111 ALOGE("%s: Failed to write buffer index %d to muxer: %s (%d)",
1112 __FUNCTION__, it->index, strerror(-res), res);
1113 return res;
1114 }
1115
1116 mCodec->releaseOutputBuffer(it->index);
1117 if (inputFrame.pendingOutputTiles == 0) {
1118 ALOGW("%s: Codec generated more tiles than expected!", __FUNCTION__);
1119 } else {
1120 inputFrame.pendingOutputTiles--;
1121 }
1122
1123 inputFrame.codecOutputBuffers.erase(inputFrame.codecOutputBuffers.begin());
Shuzhen Wang3d00ee52019-09-25 14:19:28 -07001124
1125 ALOGV("%s: [%" PRId64 "]: Output buffer index %d",
Shuzhen Wange8675782019-12-05 09:12:14 -08001126 __FUNCTION__, frameNumber, it->index);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001127 return OK;
1128}
1129
Shuzhen Wange8675782019-12-05 09:12:14 -08001130status_t HeicCompositeStream::processCompletedInputFrame(int64_t frameNumber,
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001131 InputFrame &inputFrame) {
1132 sp<ANativeWindow> outputANW = mOutputSurface;
1133 inputFrame.muxer->stop();
1134
1135 // Copy the content of the file to memory.
1136 sp<GraphicBuffer> gb = GraphicBuffer::from(inputFrame.anb);
1137 void* dstBuffer;
Shuzhen Wangc87315d2022-03-17 00:11:20 +00001138 GraphicBufferLocker gbLocker(gb);
1139 auto res = gbLocker.lockAsync(&dstBuffer, inputFrame.fenceFd);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001140 if (res != OK) {
1141 ALOGE("%s: Error trying to lock output buffer fence: %s (%d)", __FUNCTION__,
1142 strerror(-res), res);
1143 return res;
1144 }
1145
1146 off_t fSize = lseek(inputFrame.fileFd, 0, SEEK_END);
1147 if (static_cast<size_t>(fSize) > mMaxHeicBufferSize - sizeof(CameraBlob)) {
1148 ALOGE("%s: Error: MediaMuxer output size %ld is larger than buffer sizer %zu",
1149 __FUNCTION__, fSize, mMaxHeicBufferSize - sizeof(CameraBlob));
1150 return BAD_VALUE;
1151 }
1152
1153 lseek(inputFrame.fileFd, 0, SEEK_SET);
1154 ssize_t bytesRead = read(inputFrame.fileFd, dstBuffer, fSize);
1155 if (bytesRead < fSize) {
1156 ALOGE("%s: Only %zd of %ld bytes read", __FUNCTION__, bytesRead, fSize);
1157 return BAD_VALUE;
1158 }
1159
1160 close(inputFrame.fileFd);
1161 inputFrame.fileFd = -1;
1162
1163 // Fill in HEIC header
1164 uint8_t *header = static_cast<uint8_t*>(dstBuffer) + mMaxHeicBufferSize - sizeof(CameraBlob);
Jayant Chowdharyc67af1b2022-04-07 18:05:04 +00001165 CameraBlob *blobHeader = (CameraBlob *)header;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001166 // Must be in sync with CAMERA3_HEIC_BLOB_ID in android_media_Utils.cpp
1167 blobHeader->blobId = static_cast<CameraBlobId>(0x00FE);
Jayant Chowdharyc67af1b2022-04-07 18:05:04 +00001168 blobHeader->blobSizeBytes = fSize;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001169
Shuzhen Wange8675782019-12-05 09:12:14 -08001170 res = native_window_set_buffers_timestamp(mOutputSurface.get(), inputFrame.timestamp);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001171 if (res != OK) {
1172 ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
1173 __FUNCTION__, getStreamId(), strerror(-res), res);
1174 return res;
1175 }
1176
1177 res = outputANW->queueBuffer(mOutputSurface.get(), inputFrame.anb, /*fence*/ -1);
1178 if (res != OK) {
1179 ALOGE("%s: Failed to queueBuffer to Heic stream: %s (%d)", __FUNCTION__,
1180 strerror(-res), res);
1181 return res;
1182 }
1183 inputFrame.anb = nullptr;
Shuzhen Wang3d00ee52019-09-25 14:19:28 -07001184 mDequeuedOutputBufferCnt--;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001185
Shuzhen Wange8675782019-12-05 09:12:14 -08001186 ALOGV("%s: [%" PRId64 "]", __FUNCTION__, frameNumber);
1187 ATRACE_ASYNC_END("HEIC capture", frameNumber);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001188 return OK;
1189}
1190
1191
Shuzhen Wange8675782019-12-05 09:12:14 -08001192void HeicCompositeStream::releaseInputFrameLocked(int64_t frameNumber,
1193 InputFrame *inputFrame /*out*/) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001194 if (inputFrame == nullptr) {
1195 return;
1196 }
1197
1198 if (inputFrame->appSegmentBuffer.data != nullptr) {
1199 mAppSegmentConsumer->unlockBuffer(inputFrame->appSegmentBuffer);
1200 inputFrame->appSegmentBuffer.data = nullptr;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001201 }
1202
1203 while (!inputFrame->codecOutputBuffers.empty()) {
1204 auto it = inputFrame->codecOutputBuffers.begin();
1205 ALOGV("%s: releaseOutputBuffer index %d", __FUNCTION__, it->index);
1206 mCodec->releaseOutputBuffer(it->index);
1207 inputFrame->codecOutputBuffers.erase(it);
1208 }
1209
1210 if (inputFrame->yuvBuffer.data != nullptr) {
1211 mMainImageConsumer->unlockBuffer(inputFrame->yuvBuffer);
1212 inputFrame->yuvBuffer.data = nullptr;
1213 mYuvBufferAcquired = false;
1214 }
1215
1216 while (!inputFrame->codecInputBuffers.empty()) {
1217 auto it = inputFrame->codecInputBuffers.begin();
1218 inputFrame->codecInputBuffers.erase(it);
1219 }
1220
Shuzhen Wange8675782019-12-05 09:12:14 -08001221 if (inputFrame->error || mErrorState) {
1222 ALOGV("%s: notifyError called for frameNumber %" PRId64, __FUNCTION__, frameNumber);
1223 notifyError(frameNumber, inputFrame->requestId);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001224 }
1225
1226 if (inputFrame->fileFd >= 0) {
1227 close(inputFrame->fileFd);
1228 inputFrame->fileFd = -1;
1229 }
1230
1231 if (inputFrame->anb != nullptr) {
1232 sp<ANativeWindow> outputANW = mOutputSurface;
1233 outputANW->cancelBuffer(mOutputSurface.get(), inputFrame->anb, /*fence*/ -1);
1234 inputFrame->anb = nullptr;
Shuzhen Wange8675782019-12-05 09:12:14 -08001235
1236 mDequeuedOutputBufferCnt--;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001237 }
1238}
1239
Michael Gonzalezb5986a32019-10-09 15:38:17 -07001240void HeicCompositeStream::releaseInputFramesLocked() {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001241 auto it = mPendingInputFrames.begin();
Shuzhen Wang62f49ed2019-09-04 14:07:53 -07001242 bool inputFrameDone = false;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001243 while (it != mPendingInputFrames.end()) {
Michael Gonzalezb5986a32019-10-09 15:38:17 -07001244 auto& inputFrame = it->second;
1245 if (inputFrame.error ||
Shuzhen Wange8675782019-12-05 09:12:14 -08001246 (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0)) {
1247 releaseInputFrameLocked(it->first, &inputFrame);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001248 it = mPendingInputFrames.erase(it);
Shuzhen Wang62f49ed2019-09-04 14:07:53 -07001249 inputFrameDone = true;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001250 } else {
1251 it++;
1252 }
1253 }
Shuzhen Wang62f49ed2019-09-04 14:07:53 -07001254
1255 // Update codec quality based on first upcoming input frame.
1256 // Note that when encoding is in surface mode, currently there is no
1257 // way for camera service to synchronize quality setting on a per-frame
1258 // basis: we don't get notification when codec is ready to consume a new
1259 // input frame. So we update codec quality on a best-effort basis.
1260 if (inputFrameDone) {
1261 auto firstPendingFrame = mPendingInputFrames.begin();
1262 if (firstPendingFrame != mPendingInputFrames.end()) {
1263 updateCodecQualityLocked(firstPendingFrame->second.quality);
Shuzhen Wange8675782019-12-05 09:12:14 -08001264 } else {
1265 markTrackerIdle();
Shuzhen Wang62f49ed2019-09-04 14:07:53 -07001266 }
1267 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001268}
1269
1270status_t HeicCompositeStream::initializeCodec(uint32_t width, uint32_t height,
1271 const sp<CameraDeviceBase>& cameraDevice) {
1272 ALOGV("%s", __FUNCTION__);
1273
1274 bool useGrid = false;
Chong Zhang688abaa2019-05-17 16:32:23 -07001275 AString hevcName;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001276 bool isSizeSupported = isSizeSupportedByHeifEncoder(width, height,
Chong Zhang688abaa2019-05-17 16:32:23 -07001277 &mUseHeic, &useGrid, nullptr, &hevcName);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001278 if (!isSizeSupported) {
1279 ALOGE("%s: Encoder doesnt' support size %u x %u!",
1280 __FUNCTION__, width, height);
1281 return BAD_VALUE;
1282 }
1283
1284 // Create Looper for MediaCodec.
1285 auto desiredMime = mUseHeic ? MIMETYPE_IMAGE_ANDROID_HEIC : MIMETYPE_VIDEO_HEVC;
1286 mCodecLooper = new ALooper;
1287 mCodecLooper->setName("Camera3-HeicComposite-MediaCodecLooper");
1288 status_t res = mCodecLooper->start(
1289 false, // runOnCallingThread
1290 false, // canCallJava
1291 PRIORITY_AUDIO);
1292 if (res != OK) {
1293 ALOGE("%s: Failed to start codec looper: %s (%d)",
1294 __FUNCTION__, strerror(-res), res);
1295 return NO_INIT;
1296 }
1297
1298 // Create HEIC/HEVC codec.
Chong Zhang688abaa2019-05-17 16:32:23 -07001299 if (mUseHeic) {
1300 mCodec = MediaCodec::CreateByType(mCodecLooper, desiredMime, true /*encoder*/);
1301 } else {
1302 mCodec = MediaCodec::CreateByComponentName(mCodecLooper, hevcName);
1303 }
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001304 if (mCodec == nullptr) {
1305 ALOGE("%s: Failed to create codec for %s", __FUNCTION__, desiredMime);
1306 return NO_INIT;
1307 }
1308
1309 // Create Looper and handler for Codec callback.
1310 mCodecCallbackHandler = new CodecCallbackHandler(this);
1311 if (mCodecCallbackHandler == nullptr) {
1312 ALOGE("%s: Failed to create codec callback handler", __FUNCTION__);
1313 return NO_MEMORY;
1314 }
1315 mCallbackLooper = new ALooper;
1316 mCallbackLooper->setName("Camera3-HeicComposite-MediaCodecCallbackLooper");
1317 res = mCallbackLooper->start(
1318 false, // runOnCallingThread
1319 false, // canCallJava
1320 PRIORITY_AUDIO);
1321 if (res != OK) {
1322 ALOGE("%s: Failed to start media callback looper: %s (%d)",
1323 __FUNCTION__, strerror(-res), res);
1324 return NO_INIT;
1325 }
1326 mCallbackLooper->registerHandler(mCodecCallbackHandler);
1327
1328 mAsyncNotify = new AMessage(kWhatCallbackNotify, mCodecCallbackHandler);
1329 res = mCodec->setCallback(mAsyncNotify);
1330 if (res != OK) {
1331 ALOGE("%s: Failed to set MediaCodec callback: %s (%d)", __FUNCTION__,
1332 strerror(-res), res);
1333 return res;
1334 }
1335
1336 // Create output format and configure the Codec.
1337 sp<AMessage> outputFormat = new AMessage();
1338 outputFormat->setString(KEY_MIME, desiredMime);
1339 outputFormat->setInt32(KEY_BITRATE_MODE, BITRATE_MODE_CQ);
1340 outputFormat->setInt32(KEY_QUALITY, kDefaultJpegQuality);
1341 // Ask codec to skip timestamp check and encode all frames.
Chong Zhang70bfcec2019-03-18 12:52:28 -07001342 outputFormat->setInt64(KEY_MAX_PTS_GAP_TO_ENCODER, kNoFrameDropMaxPtsGap);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001343
1344 int32_t gridWidth, gridHeight, gridRows, gridCols;
1345 if (useGrid || mUseHeic) {
1346 gridWidth = HeicEncoderInfoManager::kGridWidth;
1347 gridHeight = HeicEncoderInfoManager::kGridHeight;
1348 gridRows = (height + gridHeight - 1)/gridHeight;
1349 gridCols = (width + gridWidth - 1)/gridWidth;
1350
1351 if (mUseHeic) {
1352 outputFormat->setInt32(KEY_TILE_WIDTH, gridWidth);
1353 outputFormat->setInt32(KEY_TILE_HEIGHT, gridHeight);
1354 outputFormat->setInt32(KEY_GRID_COLUMNS, gridCols);
1355 outputFormat->setInt32(KEY_GRID_ROWS, gridRows);
1356 }
1357
1358 } else {
1359 gridWidth = width;
1360 gridHeight = height;
1361 gridRows = 1;
1362 gridCols = 1;
1363 }
1364
1365 outputFormat->setInt32(KEY_WIDTH, !useGrid ? width : gridWidth);
1366 outputFormat->setInt32(KEY_HEIGHT, !useGrid ? height : gridHeight);
1367 outputFormat->setInt32(KEY_I_FRAME_INTERVAL, 0);
1368 outputFormat->setInt32(KEY_COLOR_FORMAT,
1369 useGrid ? COLOR_FormatYUV420Flexible : COLOR_FormatSurface);
Shuzhen Wang0ca81522019-08-30 14:15:16 -07001370 outputFormat->setInt32(KEY_FRAME_RATE, useGrid ? gridRows * gridCols : kNoGridOpRate);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001371 // This only serves as a hint to encoder when encoding is not real-time.
1372 outputFormat->setInt32(KEY_OPERATING_RATE, useGrid ? kGridOpRate : kNoGridOpRate);
1373
1374 res = mCodec->configure(outputFormat, nullptr /*nativeWindow*/,
1375 nullptr /*crypto*/, CONFIGURE_FLAG_ENCODE);
1376 if (res != OK) {
1377 ALOGE("%s: Failed to configure codec: %s (%d)", __FUNCTION__,
1378 strerror(-res), res);
1379 return res;
1380 }
1381
1382 mGridWidth = gridWidth;
1383 mGridHeight = gridHeight;
1384 mGridRows = gridRows;
1385 mGridCols = gridCols;
1386 mUseGrid = useGrid;
1387 mOutputWidth = width;
1388 mOutputHeight = height;
1389 mAppSegmentMaxSize = calcAppSegmentMaxSize(cameraDevice->info());
Susmitha Gummallae911f2e2020-11-23 09:57:03 -08001390 mMaxHeicBufferSize =
1391 ALIGN(mOutputWidth, HeicEncoderInfoManager::kGridWidth) *
1392 ALIGN(mOutputHeight, HeicEncoderInfoManager::kGridHeight) * 3 / 2 + mAppSegmentMaxSize;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001393
1394 return OK;
1395}
1396
1397void HeicCompositeStream::deinitCodec() {
1398 ALOGV("%s", __FUNCTION__);
1399 if (mCodec != nullptr) {
1400 mCodec->stop();
1401 mCodec->release();
1402 mCodec.clear();
1403 }
1404
1405 if (mCodecLooper != nullptr) {
1406 mCodecLooper->stop();
1407 mCodecLooper.clear();
1408 }
1409
1410 if (mCallbackLooper != nullptr) {
1411 mCallbackLooper->stop();
1412 mCallbackLooper.clear();
1413 }
1414
1415 mAsyncNotify.clear();
1416 mFormat.clear();
1417}
1418
1419// Return the size of the complete list of app segment, 0 indicates failure
1420size_t HeicCompositeStream::findAppSegmentsSize(const uint8_t* appSegmentBuffer,
1421 size_t maxSize, size_t *app1SegmentSize) {
1422 if (appSegmentBuffer == nullptr || app1SegmentSize == nullptr) {
1423 ALOGE("%s: Invalid input appSegmentBuffer %p, app1SegmentSize %p",
1424 __FUNCTION__, appSegmentBuffer, app1SegmentSize);
1425 return 0;
1426 }
1427
1428 size_t expectedSize = 0;
1429 // First check for EXIF transport header at the end of the buffer
Jayant Chowdharyc67af1b2022-04-07 18:05:04 +00001430 const uint8_t *header = appSegmentBuffer + (maxSize - sizeof(CameraBlob));
1431 const CameraBlob *blob = (const CameraBlob*)(header);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001432 if (blob->blobId != CameraBlobId::JPEG_APP_SEGMENTS) {
Jayant Chowdharyc67af1b2022-04-07 18:05:04 +00001433 ALOGE("%s: Invalid EXIF blobId %d", __FUNCTION__, blob->blobId);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001434 return 0;
1435 }
1436
Jayant Chowdharyc67af1b2022-04-07 18:05:04 +00001437 expectedSize = blob->blobSizeBytes;
1438 if (expectedSize == 0 || expectedSize > maxSize - sizeof(CameraBlob)) {
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001439 ALOGE("%s: Invalid blobSize %zu.", __FUNCTION__, expectedSize);
1440 return 0;
1441 }
1442
1443 uint32_t totalSize = 0;
1444
1445 // Verify APP1 marker (mandatory)
1446 uint8_t app1Marker[] = {0xFF, 0xE1};
1447 if (memcmp(appSegmentBuffer, app1Marker, sizeof(app1Marker))) {
1448 ALOGE("%s: Invalid APP1 marker: %x, %x", __FUNCTION__,
1449 appSegmentBuffer[0], appSegmentBuffer[1]);
1450 return 0;
1451 }
1452 totalSize += sizeof(app1Marker);
1453
1454 uint16_t app1Size = (static_cast<uint16_t>(appSegmentBuffer[totalSize]) << 8) +
1455 appSegmentBuffer[totalSize+1];
1456 totalSize += app1Size;
1457
1458 ALOGV("%s: Expected APP segments size %zu, APP1 segment size %u",
1459 __FUNCTION__, expectedSize, app1Size);
1460 while (totalSize < expectedSize) {
1461 if (appSegmentBuffer[totalSize] != 0xFF ||
1462 appSegmentBuffer[totalSize+1] <= 0xE1 ||
1463 appSegmentBuffer[totalSize+1] > 0xEF) {
1464 // Invalid APPn marker
1465 ALOGE("%s: Invalid APPn marker: %x, %x", __FUNCTION__,
1466 appSegmentBuffer[totalSize], appSegmentBuffer[totalSize+1]);
1467 return 0;
1468 }
1469 totalSize += 2;
1470
1471 uint16_t appnSize = (static_cast<uint16_t>(appSegmentBuffer[totalSize]) << 8) +
1472 appSegmentBuffer[totalSize+1];
1473 totalSize += appnSize;
1474 }
1475
1476 if (totalSize != expectedSize) {
1477 ALOGE("%s: Invalid JPEG APP segments: totalSize %u vs expected size %zu",
1478 __FUNCTION__, totalSize, expectedSize);
1479 return 0;
1480 }
1481
1482 *app1SegmentSize = app1Size + sizeof(app1Marker);
1483 return expectedSize;
1484}
1485
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001486status_t HeicCompositeStream::copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer,
1487 const CpuConsumer::LockedBuffer& yuvBuffer,
1488 size_t top, size_t left, size_t width, size_t height) {
1489 ATRACE_CALL();
1490
1491 // Get stride information for codecBuffer
1492 sp<ABuffer> imageData;
1493 if (!codecBuffer->meta()->findBuffer("image-data", &imageData)) {
1494 ALOGE("%s: Codec input buffer is not for image data!", __FUNCTION__);
1495 return BAD_VALUE;
1496 }
1497 if (imageData->size() != sizeof(MediaImage2)) {
1498 ALOGE("%s: Invalid codec input image size %zu, expected %zu",
1499 __FUNCTION__, imageData->size(), sizeof(MediaImage2));
1500 return BAD_VALUE;
1501 }
1502 MediaImage2* imageInfo = reinterpret_cast<MediaImage2*>(imageData->data());
1503 if (imageInfo->mType != MediaImage2::MEDIA_IMAGE_TYPE_YUV ||
1504 imageInfo->mBitDepth != 8 ||
1505 imageInfo->mBitDepthAllocated != 8 ||
1506 imageInfo->mNumPlanes != 3) {
1507 ALOGE("%s: Invalid codec input image info: mType %d, mBitDepth %d, "
1508 "mBitDepthAllocated %d, mNumPlanes %d!", __FUNCTION__,
1509 imageInfo->mType, imageInfo->mBitDepth,
1510 imageInfo->mBitDepthAllocated, imageInfo->mNumPlanes);
1511 return BAD_VALUE;
1512 }
1513
1514 ALOGV("%s: yuvBuffer chromaStep %d, chromaStride %d",
1515 __FUNCTION__, yuvBuffer.chromaStep, yuvBuffer.chromaStride);
1516 ALOGV("%s: U offset %u, V offset %u, U rowInc %d, V rowInc %d, U colInc %d, V colInc %d",
1517 __FUNCTION__, imageInfo->mPlane[MediaImage2::U].mOffset,
1518 imageInfo->mPlane[MediaImage2::V].mOffset,
1519 imageInfo->mPlane[MediaImage2::U].mRowInc,
1520 imageInfo->mPlane[MediaImage2::V].mRowInc,
1521 imageInfo->mPlane[MediaImage2::U].mColInc,
1522 imageInfo->mPlane[MediaImage2::V].mColInc);
1523
1524 // Y
1525 for (auto row = top; row < top+height; row++) {
1526 uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::Y].mOffset +
1527 imageInfo->mPlane[MediaImage2::Y].mRowInc * (row - top);
Shuzhen Wang219c2992019-02-15 17:24:28 -08001528 mFnCopyRow(yuvBuffer.data+row*yuvBuffer.stride+left, dst, width);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001529 }
1530
1531 // U is Cb, V is Cr
1532 bool codecUPlaneFirst = imageInfo->mPlane[MediaImage2::V].mOffset >
1533 imageInfo->mPlane[MediaImage2::U].mOffset;
1534 uint32_t codecUvOffsetDiff = codecUPlaneFirst ?
1535 imageInfo->mPlane[MediaImage2::V].mOffset - imageInfo->mPlane[MediaImage2::U].mOffset :
1536 imageInfo->mPlane[MediaImage2::U].mOffset - imageInfo->mPlane[MediaImage2::V].mOffset;
1537 bool isCodecUvSemiplannar = (codecUvOffsetDiff == 1) &&
1538 (imageInfo->mPlane[MediaImage2::U].mRowInc ==
1539 imageInfo->mPlane[MediaImage2::V].mRowInc) &&
1540 (imageInfo->mPlane[MediaImage2::U].mColInc == 2) &&
1541 (imageInfo->mPlane[MediaImage2::V].mColInc == 2);
1542 bool isCodecUvPlannar =
1543 ((codecUPlaneFirst && codecUvOffsetDiff >=
1544 imageInfo->mPlane[MediaImage2::U].mRowInc * imageInfo->mHeight/2) ||
1545 ((!codecUPlaneFirst && codecUvOffsetDiff >=
1546 imageInfo->mPlane[MediaImage2::V].mRowInc * imageInfo->mHeight/2))) &&
1547 imageInfo->mPlane[MediaImage2::U].mColInc == 1 &&
1548 imageInfo->mPlane[MediaImage2::V].mColInc == 1;
1549 bool cameraUPlaneFirst = yuvBuffer.dataCr > yuvBuffer.dataCb;
1550
1551 if (isCodecUvSemiplannar && yuvBuffer.chromaStep == 2 &&
1552 (codecUPlaneFirst == cameraUPlaneFirst)) {
1553 // UV semiplannar
1554 // The chrome plane could be either Cb first, or Cr first. Take the
1555 // smaller address.
1556 uint8_t *src = std::min(yuvBuffer.dataCb, yuvBuffer.dataCr);
1557 MediaImage2::PlaneIndex dstPlane = codecUvOffsetDiff > 0 ? MediaImage2::U : MediaImage2::V;
1558 for (auto row = top/2; row < (top+height)/2; row++) {
1559 uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[dstPlane].mOffset +
1560 imageInfo->mPlane[dstPlane].mRowInc * (row - top/2);
Shuzhen Wang219c2992019-02-15 17:24:28 -08001561 mFnCopyRow(src+row*yuvBuffer.chromaStride+left, dst, width);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001562 }
1563 } else if (isCodecUvPlannar && yuvBuffer.chromaStep == 1) {
1564 // U plane
1565 for (auto row = top/2; row < (top+height)/2; row++) {
1566 uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::U].mOffset +
1567 imageInfo->mPlane[MediaImage2::U].mRowInc * (row - top/2);
Shuzhen Wang219c2992019-02-15 17:24:28 -08001568 mFnCopyRow(yuvBuffer.dataCb+row*yuvBuffer.chromaStride+left/2, dst, width/2);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001569 }
1570
1571 // V plane
1572 for (auto row = top/2; row < (top+height)/2; row++) {
1573 uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::V].mOffset +
1574 imageInfo->mPlane[MediaImage2::V].mRowInc * (row - top/2);
Shuzhen Wang219c2992019-02-15 17:24:28 -08001575 mFnCopyRow(yuvBuffer.dataCr+row*yuvBuffer.chromaStride+left/2, dst, width/2);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001576 }
1577 } else {
Shuzhen Wang219c2992019-02-15 17:24:28 -08001578 // Convert between semiplannar and plannar, or when UV orders are
1579 // different.
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001580 uint8_t *dst = codecBuffer->data();
1581 for (auto row = top/2; row < (top+height)/2; row++) {
1582 for (auto col = left/2; col < (left+width)/2; col++) {
1583 // U/Cb
1584 int32_t dstIndex = imageInfo->mPlane[MediaImage2::U].mOffset +
1585 imageInfo->mPlane[MediaImage2::U].mRowInc * (row - top/2) +
1586 imageInfo->mPlane[MediaImage2::U].mColInc * (col - left/2);
1587 int32_t srcIndex = row * yuvBuffer.chromaStride + yuvBuffer.chromaStep * col;
1588 dst[dstIndex] = yuvBuffer.dataCb[srcIndex];
1589
1590 // V/Cr
1591 dstIndex = imageInfo->mPlane[MediaImage2::V].mOffset +
1592 imageInfo->mPlane[MediaImage2::V].mRowInc * (row - top/2) +
1593 imageInfo->mPlane[MediaImage2::V].mColInc * (col - left/2);
1594 srcIndex = row * yuvBuffer.chromaStride + yuvBuffer.chromaStep * col;
1595 dst[dstIndex] = yuvBuffer.dataCr[srcIndex];
1596 }
1597 }
1598 }
1599 return OK;
1600}
1601
Shuzhen Wang219c2992019-02-15 17:24:28 -08001602void HeicCompositeStream::initCopyRowFunction(int32_t width)
1603{
1604 using namespace libyuv;
1605
1606 mFnCopyRow = CopyRow_C;
1607#if defined(HAS_COPYROW_SSE2)
1608 if (TestCpuFlag(kCpuHasSSE2)) {
1609 mFnCopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2;
1610 }
1611#endif
1612#if defined(HAS_COPYROW_AVX)
1613 if (TestCpuFlag(kCpuHasAVX)) {
1614 mFnCopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
1615 }
1616#endif
1617#if defined(HAS_COPYROW_ERMS)
1618 if (TestCpuFlag(kCpuHasERMS)) {
1619 mFnCopyRow = CopyRow_ERMS;
1620 }
1621#endif
1622#if defined(HAS_COPYROW_NEON)
1623 if (TestCpuFlag(kCpuHasNEON)) {
1624 mFnCopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
1625 }
1626#endif
1627#if defined(HAS_COPYROW_MIPS)
1628 if (TestCpuFlag(kCpuHasMIPS)) {
1629 mFnCopyRow = CopyRow_MIPS;
1630 }
1631#endif
1632}
1633
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001634size_t HeicCompositeStream::calcAppSegmentMaxSize(const CameraMetadata& info) {
1635 camera_metadata_ro_entry_t entry = info.find(ANDROID_HEIC_INFO_MAX_JPEG_APP_SEGMENTS_COUNT);
1636 size_t maxAppsSegment = 1;
1637 if (entry.count > 0) {
1638 maxAppsSegment = entry.data.u8[0] < 1 ? 1 :
1639 entry.data.u8[0] > 16 ? 16 : entry.data.u8[0];
1640 }
Jayant Chowdharyc67af1b2022-04-07 18:05:04 +00001641 return maxAppsSegment * (2 + 0xFFFF) + sizeof(CameraBlob);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001642}
1643
Shuzhen Wang62f49ed2019-09-04 14:07:53 -07001644void HeicCompositeStream::updateCodecQualityLocked(int32_t quality) {
1645 if (quality != mQuality) {
1646 sp<AMessage> qualityParams = new AMessage;
1647 qualityParams->setInt32(PARAMETER_KEY_VIDEO_BITRATE, quality);
1648 status_t res = mCodec->setParameters(qualityParams);
1649 if (res != OK) {
1650 ALOGE("%s: Failed to set codec quality: %s (%d)",
1651 __FUNCTION__, strerror(-res), res);
1652 } else {
1653 mQuality = quality;
1654 }
1655 }
1656}
1657
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001658bool HeicCompositeStream::threadLoop() {
Shuzhen Wange8675782019-12-05 09:12:14 -08001659 int64_t frameNumber = -1;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001660 bool newInputAvailable = false;
1661
1662 {
1663 Mutex::Autolock l(mMutex);
1664 if (mErrorState) {
1665 // In case we landed in error state, return any pending buffers and
1666 // halt all further processing.
1667 compilePendingInputLocked();
Michael Gonzalezb5986a32019-10-09 15:38:17 -07001668 releaseInputFramesLocked();
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001669 return false;
1670 }
1671
1672
1673 while (!newInputAvailable) {
1674 compilePendingInputLocked();
Shuzhen Wange8675782019-12-05 09:12:14 -08001675 newInputAvailable = getNextReadyInputLocked(&frameNumber);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001676
1677 if (!newInputAvailable) {
Shuzhen Wange8675782019-12-05 09:12:14 -08001678 auto failingFrameNumber = getNextFailingInputLocked();
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001679 if (failingFrameNumber >= 0) {
Shuzhen Wange8675782019-12-05 09:12:14 -08001680 releaseInputFrameLocked(failingFrameNumber,
1681 &mPendingInputFrames[failingFrameNumber]);
1682
1683 // It's okay to remove the entry from mPendingInputFrames
1684 // because:
1685 // 1. Only one internal stream (main input) is critical in
1686 // backing the output stream.
1687 // 2. If captureResult/appSegment arrives after the entry is
1688 // removed, they are simply skipped.
1689 mPendingInputFrames.erase(failingFrameNumber);
1690 if (mPendingInputFrames.size() == 0) {
1691 markTrackerIdle();
1692 }
1693 return true;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001694 }
1695
1696 auto ret = mInputReadyCondition.waitRelative(mMutex, kWaitDuration);
1697 if (ret == TIMED_OUT) {
1698 return true;
1699 } else if (ret != OK) {
1700 ALOGE("%s: Timed wait on condition failed: %s (%d)", __FUNCTION__,
1701 strerror(-ret), ret);
1702 return false;
1703 }
1704 }
1705 }
1706 }
1707
Shuzhen Wange8675782019-12-05 09:12:14 -08001708 auto res = processInputFrame(frameNumber, mPendingInputFrames[frameNumber]);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001709 Mutex::Autolock l(mMutex);
1710 if (res != OK) {
Shuzhen Wange8675782019-12-05 09:12:14 -08001711 ALOGE("%s: Failed processing frame with timestamp: %" PRIu64 ", frameNumber: %"
1712 PRId64 ": %s (%d)", __FUNCTION__, mPendingInputFrames[frameNumber].timestamp,
1713 frameNumber, strerror(-res), res);
1714 mPendingInputFrames[frameNumber].error = true;
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001715 }
1716
Michael Gonzalezb5986a32019-10-09 15:38:17 -07001717 releaseInputFramesLocked();
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001718
1719 return true;
1720}
1721
Shuzhen Wange8675782019-12-05 09:12:14 -08001722void HeicCompositeStream::flagAnExifErrorFrameNumber(int64_t frameNumber) {
1723 Mutex::Autolock l(mMutex);
1724 mExifErrorFrameNumbers.emplace(frameNumber);
1725 mInputReadyCondition.signal();
1726}
1727
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001728bool HeicCompositeStream::onStreamBufferError(const CaptureResultExtras& resultExtras) {
1729 bool res = false;
Shuzhen Wange8675782019-12-05 09:12:14 -08001730 int64_t frameNumber = resultExtras.frameNumber;
1731
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001732 // Buffer errors concerning internal composite streams should not be directly visible to
1733 // camera clients. They must only receive a single buffer error with the public composite
1734 // stream id.
Shuzhen Wange8675782019-12-05 09:12:14 -08001735 if (resultExtras.errorStreamId == mAppSegmentStreamId) {
1736 ALOGV("%s: APP_SEGMENT frameNumber: %" PRId64, __FUNCTION__, frameNumber);
1737 flagAnExifErrorFrameNumber(frameNumber);
1738 res = true;
1739 } else if (resultExtras.errorStreamId == mMainImageStreamId) {
1740 ALOGV("%s: YUV frameNumber: %" PRId64, __FUNCTION__, frameNumber);
1741 flagAnErrorFrameNumber(frameNumber);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001742 res = true;
1743 }
1744
1745 return res;
1746}
1747
Shuzhen Wange7f4b462019-02-12 08:43:07 -08001748void HeicCompositeStream::onResultError(const CaptureResultExtras& resultExtras) {
1749 // For result error, since the APPS_SEGMENT buffer already contains EXIF,
1750 // simply skip using the capture result metadata to override EXIF.
1751 Mutex::Autolock l(mMutex);
1752
1753 int64_t timestamp = -1;
Shuzhen Wange8675782019-12-05 09:12:14 -08001754 for (const auto& fn : mSettingsByFrameNumber) {
Shuzhen Wange7f4b462019-02-12 08:43:07 -08001755 if (fn.first == resultExtras.frameNumber) {
Shuzhen Wange8675782019-12-05 09:12:14 -08001756 timestamp = fn.second.timestamp;
Shuzhen Wange7f4b462019-02-12 08:43:07 -08001757 break;
1758 }
1759 }
1760 if (timestamp == -1) {
1761 for (const auto& inputFrame : mPendingInputFrames) {
Shuzhen Wange8675782019-12-05 09:12:14 -08001762 if (inputFrame.first == resultExtras.frameNumber) {
1763 timestamp = inputFrame.second.timestamp;
Shuzhen Wange7f4b462019-02-12 08:43:07 -08001764 break;
1765 }
1766 }
1767 }
1768
1769 if (timestamp == -1) {
1770 ALOGE("%s: Failed to find shutter timestamp for result error!", __FUNCTION__);
1771 return;
1772 }
1773
1774 mCaptureResults.emplace(timestamp, std::make_tuple(resultExtras.frameNumber, CameraMetadata()));
Shuzhen Wange8675782019-12-05 09:12:14 -08001775 ALOGV("%s: timestamp %" PRId64 ", frameNumber %" PRId64, __FUNCTION__,
1776 timestamp, resultExtras.frameNumber);
Shuzhen Wange7f4b462019-02-12 08:43:07 -08001777 mInputReadyCondition.signal();
1778}
1779
Shuzhen Wange8675782019-12-05 09:12:14 -08001780void HeicCompositeStream::onRequestError(const CaptureResultExtras& resultExtras) {
1781 auto frameNumber = resultExtras.frameNumber;
1782 ALOGV("%s: frameNumber: %" PRId64, __FUNCTION__, frameNumber);
1783 Mutex::Autolock l(mMutex);
1784 auto numRequests = mSettingsByFrameNumber.erase(frameNumber);
1785 if (numRequests == 0) {
1786 // Pending request has been populated into mPendingInputFrames
1787 mErrorFrameNumbers.emplace(frameNumber);
1788 mInputReadyCondition.signal();
1789 } else {
1790 // REQUEST_ERROR was received without onShutter.
1791 }
1792}
1793
1794void HeicCompositeStream::markTrackerIdle() {
1795 sp<StatusTracker> statusTracker = mStatusTracker.promote();
1796 if (statusTracker != nullptr) {
1797 statusTracker->markComponentIdle(mStatusId, Fence::NO_FENCE);
1798 ALOGV("%s: Mark component as idle", __FUNCTION__);
1799 }
1800}
1801
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001802void HeicCompositeStream::CodecCallbackHandler::onMessageReceived(const sp<AMessage> &msg) {
1803 sp<HeicCompositeStream> parent = mParent.promote();
1804 if (parent == nullptr) return;
1805
1806 switch (msg->what()) {
1807 case kWhatCallbackNotify: {
1808 int32_t cbID;
1809 if (!msg->findInt32("callbackID", &cbID)) {
1810 ALOGE("kWhatCallbackNotify: callbackID is expected.");
1811 break;
1812 }
1813
1814 ALOGV("kWhatCallbackNotify: cbID = %d", cbID);
1815
1816 switch (cbID) {
1817 case MediaCodec::CB_INPUT_AVAILABLE: {
1818 int32_t index;
1819 if (!msg->findInt32("index", &index)) {
1820 ALOGE("CB_INPUT_AVAILABLE: index is expected.");
1821 break;
1822 }
1823 parent->onHeicInputFrameAvailable(index);
1824 break;
1825 }
1826
1827 case MediaCodec::CB_OUTPUT_AVAILABLE: {
1828 int32_t index;
1829 size_t offset;
1830 size_t size;
1831 int64_t timeUs;
1832 int32_t flags;
1833
1834 if (!msg->findInt32("index", &index)) {
1835 ALOGE("CB_OUTPUT_AVAILABLE: index is expected.");
1836 break;
1837 }
1838 if (!msg->findSize("offset", &offset)) {
1839 ALOGE("CB_OUTPUT_AVAILABLE: offset is expected.");
1840 break;
1841 }
1842 if (!msg->findSize("size", &size)) {
1843 ALOGE("CB_OUTPUT_AVAILABLE: size is expected.");
1844 break;
1845 }
1846 if (!msg->findInt64("timeUs", &timeUs)) {
1847 ALOGE("CB_OUTPUT_AVAILABLE: timeUs is expected.");
1848 break;
1849 }
1850 if (!msg->findInt32("flags", &flags)) {
1851 ALOGE("CB_OUTPUT_AVAILABLE: flags is expected.");
1852 break;
1853 }
1854
1855 CodecOutputBufferInfo bufferInfo = {
1856 index,
1857 (int32_t)offset,
1858 (int32_t)size,
1859 timeUs,
1860 (uint32_t)flags};
1861
1862 parent->onHeicOutputFrameAvailable(bufferInfo);
1863 break;
1864 }
1865
1866 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED: {
1867 sp<AMessage> format;
1868 if (!msg->findMessage("format", &format)) {
1869 ALOGE("CB_OUTPUT_FORMAT_CHANGED: format is expected.");
1870 break;
1871 }
Chong Zhang860eff12019-09-16 16:15:00 -07001872 // Here format is MediaCodec's internal copy of output format.
1873 // Make a copy since onHeicFormatChanged() might modify it.
1874 sp<AMessage> formatCopy;
1875 if (format != nullptr) {
1876 formatCopy = format->dup();
1877 }
1878 parent->onHeicFormatChanged(formatCopy);
Shuzhen Wang68ac7ad2019-01-30 14:03:28 -08001879 break;
1880 }
1881
1882 case MediaCodec::CB_ERROR: {
1883 status_t err;
1884 int32_t actionCode;
1885 AString detail;
1886 if (!msg->findInt32("err", &err)) {
1887 ALOGE("CB_ERROR: err is expected.");
1888 break;
1889 }
1890 if (!msg->findInt32("action", &actionCode)) {
1891 ALOGE("CB_ERROR: action is expected.");
1892 break;
1893 }
1894 msg->findString("detail", &detail);
1895 ALOGE("Codec reported error(0x%x), actionCode(%d), detail(%s)",
1896 err, actionCode, detail.c_str());
1897
1898 parent->onHeicCodecError();
1899 break;
1900 }
1901
1902 default: {
1903 ALOGE("kWhatCallbackNotify: callbackID(%d) is unexpected.", cbID);
1904 break;
1905 }
1906 }
1907 break;
1908 }
1909
1910 default:
1911 ALOGE("shouldn't be here");
1912 break;
1913 }
1914}
1915
1916}; // namespace camera3
1917}; // namespace android