blob: 041b4278363022b37c3de9b9fa26acc11cfb076d [file] [log] [blame]
Chong Zhangea280cb2017-08-02 11:10:10 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
Liz Kammer3ec5a0b2021-07-21 09:30:07 -040018#include "include/HeifDecoderAPI.h"
Chong Zhangea280cb2017-08-02 11:10:10 -070019#define LOG_TAG "HeifDecoderImpl"
20
21#include "HeifDecoderImpl.h"
22
23#include <stdio.h>
24
Marco Nelissendab79b32019-11-18 08:25:47 -080025#include <android/IDataSource.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070026#include <binder/IMemory.h>
Dongwon Kang49ce6712018-01-24 10:16:25 -080027#include <binder/MemoryDealer.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070028#include <drm/drm_framework_common.h>
Ray Essicke89e6322022-02-02 13:33:50 -080029#include <log/log.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070030#include <media/mediametadataretriever.h>
Marco Nelissen7291da62019-12-17 13:01:55 -080031#include <media/stagefright/MediaSource.h>
Chong Zhangd3e0d862017-10-03 13:17:13 -070032#include <media/stagefright/foundation/ADebug.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070033#include <private/media/VideoFrame.h>
34#include <utils/Log.h>
35#include <utils/RefBase.h>
Chong Zhang8541d302019-07-12 11:19:47 -070036#include <vector>
Chong Zhangea280cb2017-08-02 11:10:10 -070037
38HeifDecoder* createHeifDecoder() {
39 return new android::HeifDecoderImpl();
40}
41
42namespace android {
43
Chong Zhang8541d302019-07-12 11:19:47 -070044void initFrameInfo(HeifFrameInfo *info, const VideoFrame *videoFrame) {
45 info->mWidth = videoFrame->mWidth;
46 info->mHeight = videoFrame->mHeight;
47 info->mRotationAngle = videoFrame->mRotationAngle;
48 info->mBytesPerPixel = videoFrame->mBytesPerPixel;
Chong Zhang0af4db62019-08-23 15:34:58 -070049 info->mDurationUs = videoFrame->mDurationUs;
Chong Zhang8541d302019-07-12 11:19:47 -070050 if (videoFrame->mIccSize > 0) {
51 info->mIccData.assign(
52 videoFrame->getFlattenedIccData(),
53 videoFrame->getFlattenedIccData() + videoFrame->mIccSize);
54 } else {
55 // clear old Icc data if there is no Icc data.
56 info->mIccData.clear();
57 }
58}
59
Chong Zhangea280cb2017-08-02 11:10:10 -070060/*
61 * HeifDataSource
62 *
63 * Proxies data requests over IDataSource interface from MediaMetadataRetriever
64 * to the HeifStream interface we received from the heif decoder client.
65 */
66class HeifDataSource : public BnDataSource {
67public:
68 /*
69 * Constructs HeifDataSource; will take ownership of |stream|.
70 */
71 HeifDataSource(HeifStream* stream)
Chong Zhang3f3ffab2017-08-23 13:51:59 -070072 : mStream(stream), mEOS(false),
73 mCachedOffset(0), mCachedSize(0), mCacheBufferSize(0) {}
Chong Zhangea280cb2017-08-02 11:10:10 -070074
75 ~HeifDataSource() override {}
76
77 /*
78 * Initializes internal resources.
79 */
80 bool init();
81
82 sp<IMemory> getIMemory() override { return mMemory; }
83 ssize_t readAt(off64_t offset, size_t size) override;
84 status_t getSize(off64_t* size) override ;
85 void close() {}
86 uint32_t getFlags() override { return 0; }
87 String8 toString() override { return String8("HeifDataSource"); }
Chong Zhangea280cb2017-08-02 11:10:10 -070088
89private:
Chong Zhangea280cb2017-08-02 11:10:10 -070090 enum {
Chong Zhang3f3ffab2017-08-23 13:51:59 -070091 /*
92 * Buffer size for passing the read data to mediaserver. Set to 64K
93 * (which is what MediaDataSource Java API's jni implementation uses).
94 */
Chong Zhangea280cb2017-08-02 11:10:10 -070095 kBufferSize = 64 * 1024,
Chong Zhang3f3ffab2017-08-23 13:51:59 -070096 /*
97 * Initial and max cache buffer size.
98 */
99 kInitialCacheBufferSize = 4 * 1024 * 1024,
100 kMaxCacheBufferSize = 64 * 1024 * 1024,
Chong Zhangea280cb2017-08-02 11:10:10 -0700101 };
102 sp<IMemory> mMemory;
103 std::unique_ptr<HeifStream> mStream;
Chong Zhangea280cb2017-08-02 11:10:10 -0700104 bool mEOS;
Chong Zhangcd031922018-08-24 13:03:19 -0700105 std::unique_ptr<uint8_t[]> mCache;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700106 off64_t mCachedOffset;
107 size_t mCachedSize;
108 size_t mCacheBufferSize;
Chong Zhangea280cb2017-08-02 11:10:10 -0700109};
110
111bool HeifDataSource::init() {
112 sp<MemoryDealer> memoryDealer =
113 new MemoryDealer(kBufferSize, "HeifDataSource");
114 mMemory = memoryDealer->allocate(kBufferSize);
115 if (mMemory == nullptr) {
116 ALOGE("Failed to allocate shared memory!");
117 return false;
118 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700119 mCache.reset(new uint8_t[kInitialCacheBufferSize]);
120 if (mCache.get() == nullptr) {
121 ALOGE("mFailed to allocate cache!");
122 return false;
123 }
124 mCacheBufferSize = kInitialCacheBufferSize;
Chong Zhangea280cb2017-08-02 11:10:10 -0700125 return true;
126}
127
128ssize_t HeifDataSource::readAt(off64_t offset, size_t size) {
129 ALOGV("readAt: offset=%lld, size=%zu", (long long)offset, size);
130
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700131 if (offset < mCachedOffset) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700132 // try seek, then rewind/skip, fail if none worked
133 if (mStream->seek(offset)) {
134 ALOGV("readAt: seek to offset=%lld", (long long)offset);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700135 mCachedOffset = offset;
136 mCachedSize = 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700137 mEOS = false;
138 } else if (mStream->rewind()) {
139 ALOGV("readAt: rewind to offset=0");
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700140 mCachedOffset = 0;
141 mCachedSize = 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700142 mEOS = false;
143 } else {
144 ALOGE("readAt: couldn't seek or rewind!");
145 mEOS = true;
146 }
147 }
148
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700149 if (mEOS && (offset < mCachedOffset ||
150 offset >= (off64_t)(mCachedOffset + mCachedSize))) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700151 ALOGV("readAt: EOS");
152 return ERROR_END_OF_STREAM;
153 }
154
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700155 // at this point, offset must be >= mCachedOffset, other cases should
156 // have been caught above.
157 CHECK(offset >= mCachedOffset);
158
Sungtak Lee237f9032018-03-05 15:21:33 -0800159 off64_t resultOffset;
160 if (__builtin_add_overflow(offset, size, &resultOffset)) {
161 return ERROR_IO;
162 }
163
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700164 if (size == 0) {
165 return 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700166 }
167
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700168 // Can only read max of kBufferSize
Chong Zhangea280cb2017-08-02 11:10:10 -0700169 if (size > kBufferSize) {
170 size = kBufferSize;
171 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700172
173 // copy from cache if the request falls entirely in cache
174 if (offset + size <= mCachedOffset + mCachedSize) {
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700175 memcpy(mMemory->unsecurePointer(), mCache.get() + offset - mCachedOffset, size);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700176 return size;
177 }
178
179 // need to fetch more, check if we need to expand the cache buffer.
180 if ((off64_t)(offset + size) > mCachedOffset + kMaxCacheBufferSize) {
181 // it's reaching max cache buffer size, need to roll window, and possibly
182 // expand the cache buffer.
183 size_t newCacheBufferSize = mCacheBufferSize;
Chong Zhangcd031922018-08-24 13:03:19 -0700184 std::unique_ptr<uint8_t[]> newCache;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700185 uint8_t* dst = mCache.get();
186 if (newCacheBufferSize < kMaxCacheBufferSize) {
187 newCacheBufferSize = kMaxCacheBufferSize;
188 newCache.reset(new uint8_t[newCacheBufferSize]);
189 dst = newCache.get();
190 }
191
192 // when rolling the cache window, try to keep about half the old bytes
193 // in case that the client goes back.
194 off64_t newCachedOffset = offset - (off64_t)(newCacheBufferSize / 2);
195 if (newCachedOffset < mCachedOffset) {
196 newCachedOffset = mCachedOffset;
197 }
198
199 int64_t newCachedSize = (int64_t)(mCachedOffset + mCachedSize) - newCachedOffset;
200 if (newCachedSize > 0) {
201 // in this case, the new cache region partially overlop the old cache,
202 // move the portion of the cache we want to save to the beginning of
203 // the cache buffer.
204 memcpy(dst, mCache.get() + newCachedOffset - mCachedOffset, newCachedSize);
205 } else if (newCachedSize < 0){
206 // in this case, the new cache region is entirely out of the old cache,
207 // in order to guarantee sequential read, we need to skip a number of
208 // bytes before reading.
209 size_t bytesToSkip = -newCachedSize;
210 size_t bytesSkipped = mStream->read(nullptr, bytesToSkip);
211 if (bytesSkipped != bytesToSkip) {
212 // bytesSkipped is invalid, there is not enough bytes to reach
213 // the requested offset.
214 ALOGE("readAt: skip failed, EOS");
215
216 mEOS = true;
217 mCachedOffset = newCachedOffset;
218 mCachedSize = 0;
219 return ERROR_END_OF_STREAM;
220 }
221 // set cache size to 0, since we're not keeping any old cache
222 newCachedSize = 0;
223 }
224
225 if (newCache.get() != nullptr) {
226 mCache.reset(newCache.release());
227 mCacheBufferSize = newCacheBufferSize;
228 }
229 mCachedOffset = newCachedOffset;
230 mCachedSize = newCachedSize;
231
232 ALOGV("readAt: rolling cache window to (%lld, %zu), cache buffer size %zu",
233 (long long)mCachedOffset, mCachedSize, mCacheBufferSize);
234 } else {
235 // expand cache buffer, but no need to roll the window
236 size_t newCacheBufferSize = mCacheBufferSize;
237 while (offset + size > mCachedOffset + newCacheBufferSize) {
238 newCacheBufferSize *= 2;
239 }
240 CHECK(newCacheBufferSize <= kMaxCacheBufferSize);
241 if (mCacheBufferSize < newCacheBufferSize) {
242 uint8_t* newCache = new uint8_t[newCacheBufferSize];
243 memcpy(newCache, mCache.get(), mCachedSize);
244 mCache.reset(newCache);
245 mCacheBufferSize = newCacheBufferSize;
246
247 ALOGV("readAt: current cache window (%lld, %zu), new cache buffer size %zu",
248 (long long) mCachedOffset, mCachedSize, mCacheBufferSize);
249 }
250 }
251 size_t bytesToRead = offset + size - mCachedOffset - mCachedSize;
252 size_t bytesRead = mStream->read(mCache.get() + mCachedSize, bytesToRead);
253 if (bytesRead > bytesToRead || bytesRead == 0) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700254 // bytesRead is invalid
255 mEOS = true;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700256 bytesRead = 0;
257 } else if (bytesRead < bytesToRead) {
258 // read some bytes but not all, set EOS
Chong Zhangea280cb2017-08-02 11:10:10 -0700259 mEOS = true;
260 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700261 mCachedSize += bytesRead;
262 ALOGV("readAt: current cache window (%lld, %zu)",
263 (long long) mCachedOffset, mCachedSize);
264
265 // here bytesAvailable could be negative if offset jumped past EOS.
266 int64_t bytesAvailable = mCachedOffset + mCachedSize - offset;
267 if (bytesAvailable <= 0) {
268 return ERROR_END_OF_STREAM;
269 }
270 if (bytesAvailable < (int64_t)size) {
271 size = bytesAvailable;
272 }
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700273 memcpy(mMemory->unsecurePointer(), mCache.get() + offset - mCachedOffset, size);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700274 return size;
Chong Zhangea280cb2017-08-02 11:10:10 -0700275}
276
277status_t HeifDataSource::getSize(off64_t* size) {
278 if (!mStream->hasLength()) {
279 *size = -1;
280 ALOGE("getSize: not supported!");
281 return ERROR_UNSUPPORTED;
282 }
283 *size = mStream->getLength();
284 ALOGV("getSize: size=%lld", (long long)*size);
285 return OK;
286}
287
288/////////////////////////////////////////////////////////////////////////
289
Chong Zhang0c1407f2018-05-02 17:09:05 -0700290struct HeifDecoderImpl::DecodeThread : public Thread {
291 explicit DecodeThread(HeifDecoderImpl *decoder) : mDecoder(decoder) {}
292
293private:
294 HeifDecoderImpl* mDecoder;
295
296 bool threadLoop();
297
298 DISALLOW_EVIL_CONSTRUCTORS(DecodeThread);
299};
300
301bool HeifDecoderImpl::DecodeThread::threadLoop() {
302 return mDecoder->decodeAsync();
303}
304
305/////////////////////////////////////////////////////////////////////////
306
Chong Zhangea280cb2017-08-02 11:10:10 -0700307HeifDecoderImpl::HeifDecoderImpl() :
308 // output color format should always be set via setOutputColor(), in case
309 // it's not, default to HAL_PIXEL_FORMAT_RGB_565.
310 mOutputColor(HAL_PIXEL_FORMAT_RGB_565),
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700311 mCurScanline(0),
Chong Zhang8541d302019-07-12 11:19:47 -0700312 mTotalScanline(0),
Chong Zhangd3e0d862017-10-03 13:17:13 -0700313 mFrameDecoded(false),
314 mHasImage(false),
Chong Zhang0c1407f2018-05-02 17:09:05 -0700315 mHasVideo(false),
Chong Zhang8541d302019-07-12 11:19:47 -0700316 mSequenceLength(0),
Chong Zhang0c1407f2018-05-02 17:09:05 -0700317 mAvailableLines(0),
318 mNumSlices(1),
319 mSliceHeight(0),
320 mAsyncDecodeDone(false) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700321}
322
323HeifDecoderImpl::~HeifDecoderImpl() {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700324 if (mThread != nullptr) {
325 mThread->join();
326 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700327}
328
329bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) {
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700330 mFrameDecoded = false;
Chong Zhangd3e0d862017-10-03 13:17:13 -0700331 mFrameMemory.clear();
332
Chong Zhangea280cb2017-08-02 11:10:10 -0700333 sp<HeifDataSource> dataSource = new HeifDataSource(stream);
334 if (!dataSource->init()) {
335 return false;
336 }
337 mDataSource = dataSource;
338
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500339 return reinit(frameInfo);
340}
341
342bool HeifDecoderImpl::reinit(HeifFrameInfo* frameInfo) {
343 mFrameDecoded = false;
344 mFrameMemory.clear();
345
Chong Zhangea280cb2017-08-02 11:10:10 -0700346 mRetriever = new MediaMetadataRetriever();
Chong Zhangd3e0d862017-10-03 13:17:13 -0700347 status_t err = mRetriever->setDataSource(mDataSource, "image/heif");
Chong Zhangea280cb2017-08-02 11:10:10 -0700348 if (err != OK) {
349 ALOGE("failed to set data source!");
350
351 mRetriever.clear();
352 mDataSource.clear();
353 return false;
354 }
355 ALOGV("successfully set data source.");
356
Chong Zhangd3e0d862017-10-03 13:17:13 -0700357 const char* hasImage = mRetriever->extractMetadata(METADATA_KEY_HAS_IMAGE);
Chong Zhangea280cb2017-08-02 11:10:10 -0700358 const char* hasVideo = mRetriever->extractMetadata(METADATA_KEY_HAS_VIDEO);
Chong Zhangd3e0d862017-10-03 13:17:13 -0700359
360 mHasImage = hasImage && !strcasecmp(hasImage, "yes");
361 mHasVideo = hasVideo && !strcasecmp(hasVideo, "yes");
Chong Zhang8541d302019-07-12 11:19:47 -0700362
363 HeifFrameInfo* defaultInfo = nullptr;
Chong Zhangd3e0d862017-10-03 13:17:13 -0700364 if (mHasImage) {
365 // image index < 0 to retrieve primary image
Chong Zhang8541d302019-07-12 11:19:47 -0700366 sp<IMemory> sharedMem = mRetriever->getImageAtIndex(
Chong Zhangd3e0d862017-10-03 13:17:13 -0700367 -1, mOutputColor, true /*metaOnly*/);
Chong Zhang8541d302019-07-12 11:19:47 -0700368
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700369 if (sharedMem == nullptr || sharedMem->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700370 ALOGE("init: videoFrame is a nullptr");
371 return false;
372 }
373
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700374 // TODO: Using unsecurePointer() has some associated security pitfalls
375 // (see declaration for details).
376 // Either document why it is safe in this case or address the
377 // issue (e.g. by copying).
378 VideoFrame* videoFrame = static_cast<VideoFrame*>(sharedMem->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700379
380 ALOGV("Image dimension %dx%d, display %dx%d, angle %d, iccSize %d",
381 videoFrame->mWidth,
382 videoFrame->mHeight,
383 videoFrame->mDisplayWidth,
384 videoFrame->mDisplayHeight,
385 videoFrame->mRotationAngle,
386 videoFrame->mIccSize);
387
388 initFrameInfo(&mImageInfo, videoFrame);
389
390 if (videoFrame->mTileHeight >= 512) {
391 // Try decoding in slices only if the image has tiles and is big enough.
392 mSliceHeight = videoFrame->mTileHeight;
393 ALOGV("mSliceHeight %u", mSliceHeight);
394 }
395
396 defaultInfo = &mImageInfo;
Chong Zhangea280cb2017-08-02 11:10:10 -0700397 }
398
Chong Zhang8541d302019-07-12 11:19:47 -0700399 if (mHasVideo) {
400 sp<IMemory> sharedMem = mRetriever->getFrameAtTime(0,
401 MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC,
402 mOutputColor, true /*metaOnly*/);
403
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700404 if (sharedMem == nullptr || sharedMem->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700405 ALOGE("init: videoFrame is a nullptr");
406 return false;
407 }
408
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700409 // TODO: Using unsecurePointer() has some associated security pitfalls
410 // (see declaration for details).
411 // Either document why it is safe in this case or address the
412 // issue (e.g. by copying).
413 VideoFrame* videoFrame = static_cast<VideoFrame*>(
414 sharedMem->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700415
416 ALOGV("Sequence dimension %dx%d, display %dx%d, angle %d, iccSize %d",
417 videoFrame->mWidth,
418 videoFrame->mHeight,
419 videoFrame->mDisplayWidth,
420 videoFrame->mDisplayHeight,
421 videoFrame->mRotationAngle,
422 videoFrame->mIccSize);
423
424 initFrameInfo(&mSequenceInfo, videoFrame);
425
Ray Essicke89e6322022-02-02 13:33:50 -0800426 const char* frameCount = mRetriever->extractMetadata(METADATA_KEY_VIDEO_FRAME_COUNT);
427 if (frameCount == nullptr) {
428 android_errorWriteWithInfoLog(0x534e4554, "215002587", -1, NULL, 0);
429 ALOGD("No valid sequence information in metadata");
430 return false;
431 }
432 mSequenceLength = atoi(frameCount);
Chong Zhang8541d302019-07-12 11:19:47 -0700433
434 if (defaultInfo == nullptr) {
435 defaultInfo = &mSequenceInfo;
436 }
437 }
438
439 if (defaultInfo == nullptr) {
440 ALOGD("No valid image or sequence available");
Chong Zhangea280cb2017-08-02 11:10:10 -0700441 return false;
442 }
443
Chong Zhangea280cb2017-08-02 11:10:10 -0700444 if (frameInfo != nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700445 *frameInfo = *defaultInfo;
Chong Zhangea280cb2017-08-02 11:10:10 -0700446 }
Chong Zhang8541d302019-07-12 11:19:47 -0700447
448 // default total scanline, this might change if decodeSequence() is used
449 mTotalScanline = defaultInfo->mHeight;
450
451 return true;
452}
453
454bool HeifDecoderImpl::getSequenceInfo(
455 HeifFrameInfo* frameInfo, size_t *frameCount) {
456 ALOGV("%s", __FUNCTION__);
457 if (!mHasVideo) {
458 return false;
459 }
460 if (frameInfo != nullptr) {
461 *frameInfo = mSequenceInfo;
462 }
463 if (frameCount != nullptr) {
464 *frameCount = mSequenceLength;
Chong Zhang0c1407f2018-05-02 17:09:05 -0700465 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700466 return true;
467}
468
469bool HeifDecoderImpl::getEncodedColor(HeifEncodedColor* /*outColor*/) const {
470 ALOGW("getEncodedColor: not implemented!");
471 return false;
472}
473
474bool HeifDecoderImpl::setOutputColor(HeifColorFormat heifColor) {
Liz Kammer3ec5a0b2021-07-21 09:30:07 -0400475 if (heifColor == (HeifColorFormat)mOutputColor) {
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500476 return true;
477 }
478
Chong Zhangea280cb2017-08-02 11:10:10 -0700479 switch(heifColor) {
480 case kHeifColorFormat_RGB565:
481 {
482 mOutputColor = HAL_PIXEL_FORMAT_RGB_565;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500483 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700484 }
485 case kHeifColorFormat_RGBA_8888:
486 {
487 mOutputColor = HAL_PIXEL_FORMAT_RGBA_8888;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500488 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700489 }
490 case kHeifColorFormat_BGRA_8888:
491 {
492 mOutputColor = HAL_PIXEL_FORMAT_BGRA_8888;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500493 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700494 }
Dichen Zhang1b3441a2021-11-17 11:54:43 -0800495 case kHeifColorFormat_RGBA_1010102:
496 {
497 mOutputColor = HAL_PIXEL_FORMAT_RGBA_1010102;
498 break;
499 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700500 default:
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500501 ALOGE("Unsupported output color format %d", heifColor);
502 return false;
Chong Zhangea280cb2017-08-02 11:10:10 -0700503 }
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500504
505 if (mFrameDecoded) {
506 return reinit(nullptr);
507 }
508 return true;
Chong Zhangea280cb2017-08-02 11:10:10 -0700509}
510
Chong Zhang0c1407f2018-05-02 17:09:05 -0700511bool HeifDecoderImpl::decodeAsync() {
512 for (size_t i = 1; i < mNumSlices; i++) {
513 ALOGV("decodeAsync(): decoding slice %zu", i);
514 size_t top = i * mSliceHeight;
515 size_t bottom = (i + 1) * mSliceHeight;
Chong Zhang8541d302019-07-12 11:19:47 -0700516 if (bottom > mImageInfo.mHeight) {
517 bottom = mImageInfo.mHeight;
Chong Zhang0c1407f2018-05-02 17:09:05 -0700518 }
519 sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(
Chong Zhang8541d302019-07-12 11:19:47 -0700520 -1, mOutputColor, 0, top, mImageInfo.mWidth, bottom);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700521 {
522 Mutex::Autolock autolock(mLock);
523
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700524 if (frameMemory == nullptr || frameMemory->unsecurePointer() == nullptr) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700525 mAsyncDecodeDone = true;
526 mScanlineReady.signal();
527 break;
528 }
529 mFrameMemory = frameMemory;
530 mAvailableLines = bottom;
531 ALOGV("decodeAsync(): available lines %zu", mAvailableLines);
532 mScanlineReady.signal();
533 }
534 }
535 // Aggressive clear to avoid holding on to resources
536 mRetriever.clear();
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500537
538 // Hold on to mDataSource in case the client wants to redecode.
Chong Zhang0c1407f2018-05-02 17:09:05 -0700539 return false;
540}
541
Chong Zhangea280cb2017-08-02 11:10:10 -0700542bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700543 // reset scanline pointer
544 mCurScanline = 0;
545
546 if (mFrameDecoded) {
547 return true;
548 }
549
Chong Zhang0c1407f2018-05-02 17:09:05 -0700550 // See if we want to decode in slices to allow client to start
551 // scanline processing in parallel with decode. If this fails
552 // we fallback to decoding the full frame.
Chong Zhang8541d302019-07-12 11:19:47 -0700553 if (mHasImage) {
554 if (mSliceHeight >= 512 &&
555 mImageInfo.mWidth >= 3000 &&
556 mImageInfo.mHeight >= 2000 ) {
557 // Try decoding in slices only if the image has tiles and is big enough.
558 mNumSlices = (mImageInfo.mHeight + mSliceHeight - 1) / mSliceHeight;
559 ALOGV("mSliceHeight %u, mNumSlices %zu", mSliceHeight, mNumSlices);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700560 }
561
Chong Zhang8541d302019-07-12 11:19:47 -0700562 if (mNumSlices > 1) {
563 // get first slice and metadata
564 sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(
565 -1, mOutputColor, 0, 0, mImageInfo.mWidth, mSliceHeight);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700566
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700567 if (frameMemory == nullptr || frameMemory->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700568 ALOGE("decode: metadata is a nullptr");
569 return false;
570 }
571
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700572 // TODO: Using unsecurePointer() has some associated security pitfalls
573 // (see declaration for details).
574 // Either document why it is safe in this case or address the
575 // issue (e.g. by copying).
576 VideoFrame* videoFrame = static_cast<VideoFrame*>(frameMemory->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700577
578 if (frameInfo != nullptr) {
579 initFrameInfo(frameInfo, videoFrame);
580 }
581 mFrameMemory = frameMemory;
582 mAvailableLines = mSliceHeight;
583 mThread = new DecodeThread(this);
584 if (mThread->run("HeifDecode", ANDROID_PRIORITY_FOREGROUND) == OK) {
585 mFrameDecoded = true;
586 return true;
587 }
588 // Fallback to decode without slicing
589 mThread.clear();
590 mNumSlices = 1;
591 mSliceHeight = 0;
592 mAvailableLines = 0;
593 mFrameMemory.clear();
Chong Zhang0c1407f2018-05-02 17:09:05 -0700594 }
Chong Zhang0c1407f2018-05-02 17:09:05 -0700595 }
596
Chong Zhangd3e0d862017-10-03 13:17:13 -0700597 if (mHasImage) {
598 // image index < 0 to retrieve primary image
599 mFrameMemory = mRetriever->getImageAtIndex(-1, mOutputColor);
600 } else if (mHasVideo) {
601 mFrameMemory = mRetriever->getFrameAtTime(0,
602 MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC, mOutputColor);
603 }
604
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700605 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700606 ALOGE("decode: videoFrame is a nullptr");
Chong Zhangea280cb2017-08-02 11:10:10 -0700607 return false;
608 }
609
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700610 // TODO: Using unsecurePointer() has some associated security pitfalls
611 // (see declaration for details).
612 // Either document why it is safe in this case or address the
613 // issue (e.g. by copying).
614 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700615 if (videoFrame->mSize == 0 ||
616 mFrameMemory->size() < videoFrame->getFlattenedSize()) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700617 ALOGE("decode: videoFrame size is invalid");
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700618 return false;
619 }
620
Chong Zhangea280cb2017-08-02 11:10:10 -0700621 ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d",
622 videoFrame->mWidth,
623 videoFrame->mHeight,
624 videoFrame->mDisplayWidth,
625 videoFrame->mDisplayHeight,
626 videoFrame->mRotationAngle,
627 videoFrame->mRowBytes,
628 videoFrame->mSize);
629
630 if (frameInfo != nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700631 initFrameInfo(frameInfo, videoFrame);
632
Chong Zhangea280cb2017-08-02 11:10:10 -0700633 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700634 mFrameDecoded = true;
Chong Zhang4c6398b2017-09-07 17:43:19 -0700635
Chong Zhang0c1407f2018-05-02 17:09:05 -0700636 // Aggressively clear to avoid holding on to resources
Chong Zhang4c6398b2017-09-07 17:43:19 -0700637 mRetriever.clear();
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500638
639 // Hold on to mDataSource in case the client wants to redecode.
Chong Zhangea280cb2017-08-02 11:10:10 -0700640 return true;
641}
642
Chong Zhang8541d302019-07-12 11:19:47 -0700643bool HeifDecoderImpl::decodeSequence(int frameIndex, HeifFrameInfo* frameInfo) {
644 ALOGV("%s: frame index %d", __FUNCTION__, frameIndex);
645 if (!mHasVideo) {
646 return false;
647 }
648
649 if (frameIndex < 0 || frameIndex >= mSequenceLength) {
650 ALOGE("invalid frame index: %d, total frames %zu", frameIndex, mSequenceLength);
651 return false;
652 }
653
654 mCurScanline = 0;
655
656 // set total scanline to sequence height now
657 mTotalScanline = mSequenceInfo.mHeight;
658
659 mFrameMemory = mRetriever->getFrameAtIndex(frameIndex, mOutputColor);
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700660 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700661 ALOGE("decode: videoFrame is a nullptr");
662 return false;
663 }
664
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700665 // TODO: Using unsecurePointer() has some associated security pitfalls
666 // (see declaration for details).
667 // Either document why it is safe in this case or address the
668 // issue (e.g. by copying).
669 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700670 if (videoFrame->mSize == 0 ||
671 mFrameMemory->size() < videoFrame->getFlattenedSize()) {
672 ALOGE("decode: videoFrame size is invalid");
673 return false;
674 }
675
676 ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d",
677 videoFrame->mWidth,
678 videoFrame->mHeight,
679 videoFrame->mDisplayWidth,
680 videoFrame->mDisplayHeight,
681 videoFrame->mRotationAngle,
682 videoFrame->mRowBytes,
683 videoFrame->mSize);
684
685 if (frameInfo != nullptr) {
686 initFrameInfo(frameInfo, videoFrame);
687 }
688 return true;
689}
690
Chong Zhang0c1407f2018-05-02 17:09:05 -0700691bool HeifDecoderImpl::getScanlineInner(uint8_t* dst) {
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700692 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700693 return false;
694 }
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700695 // TODO: Using unsecurePointer() has some associated security pitfalls
696 // (see declaration for details).
697 // Either document why it is safe in this case or address the
698 // issue (e.g. by copying).
699 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhangea280cb2017-08-02 11:10:10 -0700700 uint8_t* src = videoFrame->getFlattenedData() + videoFrame->mRowBytes * mCurScanline++;
Chong Zhang70b0afd2018-02-27 12:43:06 -0800701 memcpy(dst, src, videoFrame->mBytesPerPixel * videoFrame->mWidth);
Chong Zhangea280cb2017-08-02 11:10:10 -0700702 return true;
703}
704
Chong Zhang0c1407f2018-05-02 17:09:05 -0700705bool HeifDecoderImpl::getScanline(uint8_t* dst) {
Chong Zhang8541d302019-07-12 11:19:47 -0700706 if (mCurScanline >= mTotalScanline) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700707 ALOGE("no more scanline available");
708 return false;
Chong Zhangea280cb2017-08-02 11:10:10 -0700709 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700710
Chong Zhang0c1407f2018-05-02 17:09:05 -0700711 if (mNumSlices > 1) {
712 Mutex::Autolock autolock(mLock);
713
714 while (!mAsyncDecodeDone && mCurScanline >= mAvailableLines) {
715 mScanlineReady.wait(mLock);
716 }
717 return (mCurScanline < mAvailableLines) ? getScanlineInner(dst) : false;
718 }
719
720 return getScanlineInner(dst);
721}
722
723size_t HeifDecoderImpl::skipScanlines(size_t count) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700724 uint32_t oldScanline = mCurScanline;
725 mCurScanline += count;
Chong Zhang8541d302019-07-12 11:19:47 -0700726 if (mCurScanline > mTotalScanline) {
727 mCurScanline = mTotalScanline;
Chong Zhangea280cb2017-08-02 11:10:10 -0700728 }
729 return (mCurScanline > oldScanline) ? (mCurScanline - oldScanline) : 0;
730}
731
732} // namespace android