blob: fcac551e452739dc57be057f72156bc647b930ae [file] [log] [blame]
Chong Zhangea280cb2017-08-02 11:10:10 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
Liz Kammer3ec5a0b2021-07-21 09:30:07 -040018#include "include/HeifDecoderAPI.h"
Chong Zhangea280cb2017-08-02 11:10:10 -070019#define LOG_TAG "HeifDecoderImpl"
20
21#include "HeifDecoderImpl.h"
22
23#include <stdio.h>
24
Marco Nelissendab79b32019-11-18 08:25:47 -080025#include <android/IDataSource.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070026#include <binder/IMemory.h>
Dongwon Kang49ce6712018-01-24 10:16:25 -080027#include <binder/MemoryDealer.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070028#include <drm/drm_framework_common.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070029#include <media/mediametadataretriever.h>
Marco Nelissen7291da62019-12-17 13:01:55 -080030#include <media/stagefright/MediaSource.h>
Chong Zhangd3e0d862017-10-03 13:17:13 -070031#include <media/stagefright/foundation/ADebug.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070032#include <private/media/VideoFrame.h>
33#include <utils/Log.h>
34#include <utils/RefBase.h>
Chong Zhang8541d302019-07-12 11:19:47 -070035#include <vector>
Chong Zhangea280cb2017-08-02 11:10:10 -070036
37HeifDecoder* createHeifDecoder() {
38 return new android::HeifDecoderImpl();
39}
40
41namespace android {
42
Chong Zhang8541d302019-07-12 11:19:47 -070043void initFrameInfo(HeifFrameInfo *info, const VideoFrame *videoFrame) {
44 info->mWidth = videoFrame->mWidth;
45 info->mHeight = videoFrame->mHeight;
46 info->mRotationAngle = videoFrame->mRotationAngle;
47 info->mBytesPerPixel = videoFrame->mBytesPerPixel;
Chong Zhang0af4db62019-08-23 15:34:58 -070048 info->mDurationUs = videoFrame->mDurationUs;
Chong Zhang8541d302019-07-12 11:19:47 -070049 if (videoFrame->mIccSize > 0) {
50 info->mIccData.assign(
51 videoFrame->getFlattenedIccData(),
52 videoFrame->getFlattenedIccData() + videoFrame->mIccSize);
53 } else {
54 // clear old Icc data if there is no Icc data.
55 info->mIccData.clear();
56 }
57}
58
Chong Zhangea280cb2017-08-02 11:10:10 -070059/*
60 * HeifDataSource
61 *
62 * Proxies data requests over IDataSource interface from MediaMetadataRetriever
63 * to the HeifStream interface we received from the heif decoder client.
64 */
65class HeifDataSource : public BnDataSource {
66public:
67 /*
68 * Constructs HeifDataSource; will take ownership of |stream|.
69 */
70 HeifDataSource(HeifStream* stream)
Chong Zhang3f3ffab2017-08-23 13:51:59 -070071 : mStream(stream), mEOS(false),
72 mCachedOffset(0), mCachedSize(0), mCacheBufferSize(0) {}
Chong Zhangea280cb2017-08-02 11:10:10 -070073
74 ~HeifDataSource() override {}
75
76 /*
77 * Initializes internal resources.
78 */
79 bool init();
80
81 sp<IMemory> getIMemory() override { return mMemory; }
82 ssize_t readAt(off64_t offset, size_t size) override;
83 status_t getSize(off64_t* size) override ;
84 void close() {}
85 uint32_t getFlags() override { return 0; }
86 String8 toString() override { return String8("HeifDataSource"); }
Chong Zhangea280cb2017-08-02 11:10:10 -070087
88private:
Chong Zhangea280cb2017-08-02 11:10:10 -070089 enum {
Chong Zhang3f3ffab2017-08-23 13:51:59 -070090 /*
91 * Buffer size for passing the read data to mediaserver. Set to 64K
92 * (which is what MediaDataSource Java API's jni implementation uses).
93 */
Chong Zhangea280cb2017-08-02 11:10:10 -070094 kBufferSize = 64 * 1024,
Chong Zhang3f3ffab2017-08-23 13:51:59 -070095 /*
96 * Initial and max cache buffer size.
97 */
98 kInitialCacheBufferSize = 4 * 1024 * 1024,
99 kMaxCacheBufferSize = 64 * 1024 * 1024,
Chong Zhangea280cb2017-08-02 11:10:10 -0700100 };
101 sp<IMemory> mMemory;
102 std::unique_ptr<HeifStream> mStream;
Chong Zhangea280cb2017-08-02 11:10:10 -0700103 bool mEOS;
Chong Zhangcd031922018-08-24 13:03:19 -0700104 std::unique_ptr<uint8_t[]> mCache;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700105 off64_t mCachedOffset;
106 size_t mCachedSize;
107 size_t mCacheBufferSize;
Chong Zhangea280cb2017-08-02 11:10:10 -0700108};
109
110bool HeifDataSource::init() {
111 sp<MemoryDealer> memoryDealer =
112 new MemoryDealer(kBufferSize, "HeifDataSource");
113 mMemory = memoryDealer->allocate(kBufferSize);
114 if (mMemory == nullptr) {
115 ALOGE("Failed to allocate shared memory!");
116 return false;
117 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700118 mCache.reset(new uint8_t[kInitialCacheBufferSize]);
119 if (mCache.get() == nullptr) {
120 ALOGE("mFailed to allocate cache!");
121 return false;
122 }
123 mCacheBufferSize = kInitialCacheBufferSize;
Chong Zhangea280cb2017-08-02 11:10:10 -0700124 return true;
125}
126
127ssize_t HeifDataSource::readAt(off64_t offset, size_t size) {
128 ALOGV("readAt: offset=%lld, size=%zu", (long long)offset, size);
129
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700130 if (offset < mCachedOffset) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700131 // try seek, then rewind/skip, fail if none worked
132 if (mStream->seek(offset)) {
133 ALOGV("readAt: seek to offset=%lld", (long long)offset);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700134 mCachedOffset = offset;
135 mCachedSize = 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700136 mEOS = false;
137 } else if (mStream->rewind()) {
138 ALOGV("readAt: rewind to offset=0");
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700139 mCachedOffset = 0;
140 mCachedSize = 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700141 mEOS = false;
142 } else {
143 ALOGE("readAt: couldn't seek or rewind!");
144 mEOS = true;
145 }
146 }
147
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700148 if (mEOS && (offset < mCachedOffset ||
149 offset >= (off64_t)(mCachedOffset + mCachedSize))) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700150 ALOGV("readAt: EOS");
151 return ERROR_END_OF_STREAM;
152 }
153
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700154 // at this point, offset must be >= mCachedOffset, other cases should
155 // have been caught above.
156 CHECK(offset >= mCachedOffset);
157
Sungtak Lee237f9032018-03-05 15:21:33 -0800158 off64_t resultOffset;
159 if (__builtin_add_overflow(offset, size, &resultOffset)) {
160 return ERROR_IO;
161 }
162
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700163 if (size == 0) {
164 return 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700165 }
166
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700167 // Can only read max of kBufferSize
Chong Zhangea280cb2017-08-02 11:10:10 -0700168 if (size > kBufferSize) {
169 size = kBufferSize;
170 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700171
172 // copy from cache if the request falls entirely in cache
173 if (offset + size <= mCachedOffset + mCachedSize) {
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700174 memcpy(mMemory->unsecurePointer(), mCache.get() + offset - mCachedOffset, size);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700175 return size;
176 }
177
178 // need to fetch more, check if we need to expand the cache buffer.
179 if ((off64_t)(offset + size) > mCachedOffset + kMaxCacheBufferSize) {
180 // it's reaching max cache buffer size, need to roll window, and possibly
181 // expand the cache buffer.
182 size_t newCacheBufferSize = mCacheBufferSize;
Chong Zhangcd031922018-08-24 13:03:19 -0700183 std::unique_ptr<uint8_t[]> newCache;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700184 uint8_t* dst = mCache.get();
185 if (newCacheBufferSize < kMaxCacheBufferSize) {
186 newCacheBufferSize = kMaxCacheBufferSize;
187 newCache.reset(new uint8_t[newCacheBufferSize]);
188 dst = newCache.get();
189 }
190
191 // when rolling the cache window, try to keep about half the old bytes
192 // in case that the client goes back.
193 off64_t newCachedOffset = offset - (off64_t)(newCacheBufferSize / 2);
194 if (newCachedOffset < mCachedOffset) {
195 newCachedOffset = mCachedOffset;
196 }
197
198 int64_t newCachedSize = (int64_t)(mCachedOffset + mCachedSize) - newCachedOffset;
199 if (newCachedSize > 0) {
200 // in this case, the new cache region partially overlop the old cache,
201 // move the portion of the cache we want to save to the beginning of
202 // the cache buffer.
203 memcpy(dst, mCache.get() + newCachedOffset - mCachedOffset, newCachedSize);
204 } else if (newCachedSize < 0){
205 // in this case, the new cache region is entirely out of the old cache,
206 // in order to guarantee sequential read, we need to skip a number of
207 // bytes before reading.
208 size_t bytesToSkip = -newCachedSize;
209 size_t bytesSkipped = mStream->read(nullptr, bytesToSkip);
210 if (bytesSkipped != bytesToSkip) {
211 // bytesSkipped is invalid, there is not enough bytes to reach
212 // the requested offset.
213 ALOGE("readAt: skip failed, EOS");
214
215 mEOS = true;
216 mCachedOffset = newCachedOffset;
217 mCachedSize = 0;
218 return ERROR_END_OF_STREAM;
219 }
220 // set cache size to 0, since we're not keeping any old cache
221 newCachedSize = 0;
222 }
223
224 if (newCache.get() != nullptr) {
225 mCache.reset(newCache.release());
226 mCacheBufferSize = newCacheBufferSize;
227 }
228 mCachedOffset = newCachedOffset;
229 mCachedSize = newCachedSize;
230
231 ALOGV("readAt: rolling cache window to (%lld, %zu), cache buffer size %zu",
232 (long long)mCachedOffset, mCachedSize, mCacheBufferSize);
233 } else {
234 // expand cache buffer, but no need to roll the window
235 size_t newCacheBufferSize = mCacheBufferSize;
236 while (offset + size > mCachedOffset + newCacheBufferSize) {
237 newCacheBufferSize *= 2;
238 }
239 CHECK(newCacheBufferSize <= kMaxCacheBufferSize);
240 if (mCacheBufferSize < newCacheBufferSize) {
241 uint8_t* newCache = new uint8_t[newCacheBufferSize];
242 memcpy(newCache, mCache.get(), mCachedSize);
243 mCache.reset(newCache);
244 mCacheBufferSize = newCacheBufferSize;
245
246 ALOGV("readAt: current cache window (%lld, %zu), new cache buffer size %zu",
247 (long long) mCachedOffset, mCachedSize, mCacheBufferSize);
248 }
249 }
250 size_t bytesToRead = offset + size - mCachedOffset - mCachedSize;
251 size_t bytesRead = mStream->read(mCache.get() + mCachedSize, bytesToRead);
252 if (bytesRead > bytesToRead || bytesRead == 0) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700253 // bytesRead is invalid
254 mEOS = true;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700255 bytesRead = 0;
256 } else if (bytesRead < bytesToRead) {
257 // read some bytes but not all, set EOS
Chong Zhangea280cb2017-08-02 11:10:10 -0700258 mEOS = true;
259 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700260 mCachedSize += bytesRead;
261 ALOGV("readAt: current cache window (%lld, %zu)",
262 (long long) mCachedOffset, mCachedSize);
263
264 // here bytesAvailable could be negative if offset jumped past EOS.
265 int64_t bytesAvailable = mCachedOffset + mCachedSize - offset;
266 if (bytesAvailable <= 0) {
267 return ERROR_END_OF_STREAM;
268 }
269 if (bytesAvailable < (int64_t)size) {
270 size = bytesAvailable;
271 }
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700272 memcpy(mMemory->unsecurePointer(), mCache.get() + offset - mCachedOffset, size);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700273 return size;
Chong Zhangea280cb2017-08-02 11:10:10 -0700274}
275
276status_t HeifDataSource::getSize(off64_t* size) {
277 if (!mStream->hasLength()) {
278 *size = -1;
279 ALOGE("getSize: not supported!");
280 return ERROR_UNSUPPORTED;
281 }
282 *size = mStream->getLength();
283 ALOGV("getSize: size=%lld", (long long)*size);
284 return OK;
285}
286
287/////////////////////////////////////////////////////////////////////////
288
Chong Zhang0c1407f2018-05-02 17:09:05 -0700289struct HeifDecoderImpl::DecodeThread : public Thread {
290 explicit DecodeThread(HeifDecoderImpl *decoder) : mDecoder(decoder) {}
291
292private:
293 HeifDecoderImpl* mDecoder;
294
295 bool threadLoop();
296
297 DISALLOW_EVIL_CONSTRUCTORS(DecodeThread);
298};
299
300bool HeifDecoderImpl::DecodeThread::threadLoop() {
301 return mDecoder->decodeAsync();
302}
303
304/////////////////////////////////////////////////////////////////////////
305
Chong Zhangea280cb2017-08-02 11:10:10 -0700306HeifDecoderImpl::HeifDecoderImpl() :
307 // output color format should always be set via setOutputColor(), in case
308 // it's not, default to HAL_PIXEL_FORMAT_RGB_565.
309 mOutputColor(HAL_PIXEL_FORMAT_RGB_565),
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700310 mCurScanline(0),
Chong Zhang8541d302019-07-12 11:19:47 -0700311 mTotalScanline(0),
Chong Zhangd3e0d862017-10-03 13:17:13 -0700312 mFrameDecoded(false),
313 mHasImage(false),
Chong Zhang0c1407f2018-05-02 17:09:05 -0700314 mHasVideo(false),
Chong Zhang8541d302019-07-12 11:19:47 -0700315 mSequenceLength(0),
Chong Zhang0c1407f2018-05-02 17:09:05 -0700316 mAvailableLines(0),
317 mNumSlices(1),
318 mSliceHeight(0),
319 mAsyncDecodeDone(false) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700320}
321
322HeifDecoderImpl::~HeifDecoderImpl() {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700323 if (mThread != nullptr) {
324 mThread->join();
325 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700326}
327
328bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) {
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700329 mFrameDecoded = false;
Chong Zhangd3e0d862017-10-03 13:17:13 -0700330 mFrameMemory.clear();
331
Chong Zhangea280cb2017-08-02 11:10:10 -0700332 sp<HeifDataSource> dataSource = new HeifDataSource(stream);
333 if (!dataSource->init()) {
334 return false;
335 }
336 mDataSource = dataSource;
337
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500338 return reinit(frameInfo);
339}
340
341bool HeifDecoderImpl::reinit(HeifFrameInfo* frameInfo) {
342 mFrameDecoded = false;
343 mFrameMemory.clear();
344
Chong Zhangea280cb2017-08-02 11:10:10 -0700345 mRetriever = new MediaMetadataRetriever();
Chong Zhangd3e0d862017-10-03 13:17:13 -0700346 status_t err = mRetriever->setDataSource(mDataSource, "image/heif");
Chong Zhangea280cb2017-08-02 11:10:10 -0700347 if (err != OK) {
348 ALOGE("failed to set data source!");
349
350 mRetriever.clear();
351 mDataSource.clear();
352 return false;
353 }
354 ALOGV("successfully set data source.");
355
Chong Zhangd3e0d862017-10-03 13:17:13 -0700356 const char* hasImage = mRetriever->extractMetadata(METADATA_KEY_HAS_IMAGE);
Chong Zhangea280cb2017-08-02 11:10:10 -0700357 const char* hasVideo = mRetriever->extractMetadata(METADATA_KEY_HAS_VIDEO);
Chong Zhangd3e0d862017-10-03 13:17:13 -0700358
359 mHasImage = hasImage && !strcasecmp(hasImage, "yes");
360 mHasVideo = hasVideo && !strcasecmp(hasVideo, "yes");
Chong Zhang8541d302019-07-12 11:19:47 -0700361
362 HeifFrameInfo* defaultInfo = nullptr;
Chong Zhangd3e0d862017-10-03 13:17:13 -0700363 if (mHasImage) {
364 // image index < 0 to retrieve primary image
Chong Zhang8541d302019-07-12 11:19:47 -0700365 sp<IMemory> sharedMem = mRetriever->getImageAtIndex(
Chong Zhangd3e0d862017-10-03 13:17:13 -0700366 -1, mOutputColor, true /*metaOnly*/);
Chong Zhang8541d302019-07-12 11:19:47 -0700367
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700368 if (sharedMem == nullptr || sharedMem->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700369 ALOGE("init: videoFrame is a nullptr");
370 return false;
371 }
372
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700373 // TODO: Using unsecurePointer() has some associated security pitfalls
374 // (see declaration for details).
375 // Either document why it is safe in this case or address the
376 // issue (e.g. by copying).
377 VideoFrame* videoFrame = static_cast<VideoFrame*>(sharedMem->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700378
379 ALOGV("Image dimension %dx%d, display %dx%d, angle %d, iccSize %d",
380 videoFrame->mWidth,
381 videoFrame->mHeight,
382 videoFrame->mDisplayWidth,
383 videoFrame->mDisplayHeight,
384 videoFrame->mRotationAngle,
385 videoFrame->mIccSize);
386
387 initFrameInfo(&mImageInfo, videoFrame);
388
389 if (videoFrame->mTileHeight >= 512) {
390 // Try decoding in slices only if the image has tiles and is big enough.
391 mSliceHeight = videoFrame->mTileHeight;
392 ALOGV("mSliceHeight %u", mSliceHeight);
393 }
394
395 defaultInfo = &mImageInfo;
Chong Zhangea280cb2017-08-02 11:10:10 -0700396 }
397
Chong Zhang8541d302019-07-12 11:19:47 -0700398 if (mHasVideo) {
399 sp<IMemory> sharedMem = mRetriever->getFrameAtTime(0,
400 MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC,
401 mOutputColor, true /*metaOnly*/);
402
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700403 if (sharedMem == nullptr || sharedMem->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700404 ALOGE("init: videoFrame is a nullptr");
405 return false;
406 }
407
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700408 // TODO: Using unsecurePointer() has some associated security pitfalls
409 // (see declaration for details).
410 // Either document why it is safe in this case or address the
411 // issue (e.g. by copying).
412 VideoFrame* videoFrame = static_cast<VideoFrame*>(
413 sharedMem->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700414
415 ALOGV("Sequence dimension %dx%d, display %dx%d, angle %d, iccSize %d",
416 videoFrame->mWidth,
417 videoFrame->mHeight,
418 videoFrame->mDisplayWidth,
419 videoFrame->mDisplayHeight,
420 videoFrame->mRotationAngle,
421 videoFrame->mIccSize);
422
423 initFrameInfo(&mSequenceInfo, videoFrame);
424
425 mSequenceLength = atoi(mRetriever->extractMetadata(METADATA_KEY_VIDEO_FRAME_COUNT));
426
427 if (defaultInfo == nullptr) {
428 defaultInfo = &mSequenceInfo;
429 }
430 }
431
432 if (defaultInfo == nullptr) {
433 ALOGD("No valid image or sequence available");
Chong Zhangea280cb2017-08-02 11:10:10 -0700434 return false;
435 }
436
Chong Zhangea280cb2017-08-02 11:10:10 -0700437 if (frameInfo != nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700438 *frameInfo = *defaultInfo;
Chong Zhangea280cb2017-08-02 11:10:10 -0700439 }
Chong Zhang8541d302019-07-12 11:19:47 -0700440
441 // default total scanline, this might change if decodeSequence() is used
442 mTotalScanline = defaultInfo->mHeight;
443
444 return true;
445}
446
447bool HeifDecoderImpl::getSequenceInfo(
448 HeifFrameInfo* frameInfo, size_t *frameCount) {
449 ALOGV("%s", __FUNCTION__);
450 if (!mHasVideo) {
451 return false;
452 }
453 if (frameInfo != nullptr) {
454 *frameInfo = mSequenceInfo;
455 }
456 if (frameCount != nullptr) {
457 *frameCount = mSequenceLength;
Chong Zhang0c1407f2018-05-02 17:09:05 -0700458 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700459 return true;
460}
461
462bool HeifDecoderImpl::getEncodedColor(HeifEncodedColor* /*outColor*/) const {
463 ALOGW("getEncodedColor: not implemented!");
464 return false;
465}
466
467bool HeifDecoderImpl::setOutputColor(HeifColorFormat heifColor) {
Liz Kammer3ec5a0b2021-07-21 09:30:07 -0400468 if (heifColor == (HeifColorFormat)mOutputColor) {
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500469 return true;
470 }
471
Chong Zhangea280cb2017-08-02 11:10:10 -0700472 switch(heifColor) {
473 case kHeifColorFormat_RGB565:
474 {
475 mOutputColor = HAL_PIXEL_FORMAT_RGB_565;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500476 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700477 }
478 case kHeifColorFormat_RGBA_8888:
479 {
480 mOutputColor = HAL_PIXEL_FORMAT_RGBA_8888;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500481 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700482 }
483 case kHeifColorFormat_BGRA_8888:
484 {
485 mOutputColor = HAL_PIXEL_FORMAT_BGRA_8888;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500486 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700487 }
488 default:
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500489 ALOGE("Unsupported output color format %d", heifColor);
490 return false;
Chong Zhangea280cb2017-08-02 11:10:10 -0700491 }
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500492
493 if (mFrameDecoded) {
494 return reinit(nullptr);
495 }
496 return true;
Chong Zhangea280cb2017-08-02 11:10:10 -0700497}
498
Chong Zhang0c1407f2018-05-02 17:09:05 -0700499bool HeifDecoderImpl::decodeAsync() {
500 for (size_t i = 1; i < mNumSlices; i++) {
501 ALOGV("decodeAsync(): decoding slice %zu", i);
502 size_t top = i * mSliceHeight;
503 size_t bottom = (i + 1) * mSliceHeight;
Chong Zhang8541d302019-07-12 11:19:47 -0700504 if (bottom > mImageInfo.mHeight) {
505 bottom = mImageInfo.mHeight;
Chong Zhang0c1407f2018-05-02 17:09:05 -0700506 }
507 sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(
Chong Zhang8541d302019-07-12 11:19:47 -0700508 -1, mOutputColor, 0, top, mImageInfo.mWidth, bottom);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700509 {
510 Mutex::Autolock autolock(mLock);
511
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700512 if (frameMemory == nullptr || frameMemory->unsecurePointer() == nullptr) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700513 mAsyncDecodeDone = true;
514 mScanlineReady.signal();
515 break;
516 }
517 mFrameMemory = frameMemory;
518 mAvailableLines = bottom;
519 ALOGV("decodeAsync(): available lines %zu", mAvailableLines);
520 mScanlineReady.signal();
521 }
522 }
523 // Aggressive clear to avoid holding on to resources
524 mRetriever.clear();
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500525
526 // Hold on to mDataSource in case the client wants to redecode.
Chong Zhang0c1407f2018-05-02 17:09:05 -0700527 return false;
528}
529
Chong Zhangea280cb2017-08-02 11:10:10 -0700530bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700531 // reset scanline pointer
532 mCurScanline = 0;
533
534 if (mFrameDecoded) {
535 return true;
536 }
537
Chong Zhang0c1407f2018-05-02 17:09:05 -0700538 // See if we want to decode in slices to allow client to start
539 // scanline processing in parallel with decode. If this fails
540 // we fallback to decoding the full frame.
Chong Zhang8541d302019-07-12 11:19:47 -0700541 if (mHasImage) {
542 if (mSliceHeight >= 512 &&
543 mImageInfo.mWidth >= 3000 &&
544 mImageInfo.mHeight >= 2000 ) {
545 // Try decoding in slices only if the image has tiles and is big enough.
546 mNumSlices = (mImageInfo.mHeight + mSliceHeight - 1) / mSliceHeight;
547 ALOGV("mSliceHeight %u, mNumSlices %zu", mSliceHeight, mNumSlices);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700548 }
549
Chong Zhang8541d302019-07-12 11:19:47 -0700550 if (mNumSlices > 1) {
551 // get first slice and metadata
552 sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(
553 -1, mOutputColor, 0, 0, mImageInfo.mWidth, mSliceHeight);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700554
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700555 if (frameMemory == nullptr || frameMemory->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700556 ALOGE("decode: metadata is a nullptr");
557 return false;
558 }
559
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700560 // TODO: Using unsecurePointer() has some associated security pitfalls
561 // (see declaration for details).
562 // Either document why it is safe in this case or address the
563 // issue (e.g. by copying).
564 VideoFrame* videoFrame = static_cast<VideoFrame*>(frameMemory->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700565
566 if (frameInfo != nullptr) {
567 initFrameInfo(frameInfo, videoFrame);
568 }
569 mFrameMemory = frameMemory;
570 mAvailableLines = mSliceHeight;
571 mThread = new DecodeThread(this);
572 if (mThread->run("HeifDecode", ANDROID_PRIORITY_FOREGROUND) == OK) {
573 mFrameDecoded = true;
574 return true;
575 }
576 // Fallback to decode without slicing
577 mThread.clear();
578 mNumSlices = 1;
579 mSliceHeight = 0;
580 mAvailableLines = 0;
581 mFrameMemory.clear();
Chong Zhang0c1407f2018-05-02 17:09:05 -0700582 }
Chong Zhang0c1407f2018-05-02 17:09:05 -0700583 }
584
Chong Zhangd3e0d862017-10-03 13:17:13 -0700585 if (mHasImage) {
586 // image index < 0 to retrieve primary image
587 mFrameMemory = mRetriever->getImageAtIndex(-1, mOutputColor);
588 } else if (mHasVideo) {
589 mFrameMemory = mRetriever->getFrameAtTime(0,
590 MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC, mOutputColor);
591 }
592
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700593 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700594 ALOGE("decode: videoFrame is a nullptr");
Chong Zhangea280cb2017-08-02 11:10:10 -0700595 return false;
596 }
597
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700598 // TODO: Using unsecurePointer() has some associated security pitfalls
599 // (see declaration for details).
600 // Either document why it is safe in this case or address the
601 // issue (e.g. by copying).
602 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700603 if (videoFrame->mSize == 0 ||
604 mFrameMemory->size() < videoFrame->getFlattenedSize()) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700605 ALOGE("decode: videoFrame size is invalid");
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700606 return false;
607 }
608
Chong Zhangea280cb2017-08-02 11:10:10 -0700609 ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d",
610 videoFrame->mWidth,
611 videoFrame->mHeight,
612 videoFrame->mDisplayWidth,
613 videoFrame->mDisplayHeight,
614 videoFrame->mRotationAngle,
615 videoFrame->mRowBytes,
616 videoFrame->mSize);
617
618 if (frameInfo != nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700619 initFrameInfo(frameInfo, videoFrame);
620
Chong Zhangea280cb2017-08-02 11:10:10 -0700621 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700622 mFrameDecoded = true;
Chong Zhang4c6398b2017-09-07 17:43:19 -0700623
Chong Zhang0c1407f2018-05-02 17:09:05 -0700624 // Aggressively clear to avoid holding on to resources
Chong Zhang4c6398b2017-09-07 17:43:19 -0700625 mRetriever.clear();
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500626
627 // Hold on to mDataSource in case the client wants to redecode.
Chong Zhangea280cb2017-08-02 11:10:10 -0700628 return true;
629}
630
Chong Zhang8541d302019-07-12 11:19:47 -0700631bool HeifDecoderImpl::decodeSequence(int frameIndex, HeifFrameInfo* frameInfo) {
632 ALOGV("%s: frame index %d", __FUNCTION__, frameIndex);
633 if (!mHasVideo) {
634 return false;
635 }
636
637 if (frameIndex < 0 || frameIndex >= mSequenceLength) {
638 ALOGE("invalid frame index: %d, total frames %zu", frameIndex, mSequenceLength);
639 return false;
640 }
641
642 mCurScanline = 0;
643
644 // set total scanline to sequence height now
645 mTotalScanline = mSequenceInfo.mHeight;
646
647 mFrameMemory = mRetriever->getFrameAtIndex(frameIndex, mOutputColor);
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700648 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700649 ALOGE("decode: videoFrame is a nullptr");
650 return false;
651 }
652
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700653 // TODO: Using unsecurePointer() has some associated security pitfalls
654 // (see declaration for details).
655 // Either document why it is safe in this case or address the
656 // issue (e.g. by copying).
657 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700658 if (videoFrame->mSize == 0 ||
659 mFrameMemory->size() < videoFrame->getFlattenedSize()) {
660 ALOGE("decode: videoFrame size is invalid");
661 return false;
662 }
663
664 ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d",
665 videoFrame->mWidth,
666 videoFrame->mHeight,
667 videoFrame->mDisplayWidth,
668 videoFrame->mDisplayHeight,
669 videoFrame->mRotationAngle,
670 videoFrame->mRowBytes,
671 videoFrame->mSize);
672
673 if (frameInfo != nullptr) {
674 initFrameInfo(frameInfo, videoFrame);
675 }
676 return true;
677}
678
Chong Zhang0c1407f2018-05-02 17:09:05 -0700679bool HeifDecoderImpl::getScanlineInner(uint8_t* dst) {
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700680 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700681 return false;
682 }
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700683 // TODO: Using unsecurePointer() has some associated security pitfalls
684 // (see declaration for details).
685 // Either document why it is safe in this case or address the
686 // issue (e.g. by copying).
687 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhangea280cb2017-08-02 11:10:10 -0700688 uint8_t* src = videoFrame->getFlattenedData() + videoFrame->mRowBytes * mCurScanline++;
Chong Zhang70b0afd2018-02-27 12:43:06 -0800689 memcpy(dst, src, videoFrame->mBytesPerPixel * videoFrame->mWidth);
Chong Zhangea280cb2017-08-02 11:10:10 -0700690 return true;
691}
692
Chong Zhang0c1407f2018-05-02 17:09:05 -0700693bool HeifDecoderImpl::getScanline(uint8_t* dst) {
Chong Zhang8541d302019-07-12 11:19:47 -0700694 if (mCurScanline >= mTotalScanline) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700695 ALOGE("no more scanline available");
696 return false;
Chong Zhangea280cb2017-08-02 11:10:10 -0700697 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700698
Chong Zhang0c1407f2018-05-02 17:09:05 -0700699 if (mNumSlices > 1) {
700 Mutex::Autolock autolock(mLock);
701
702 while (!mAsyncDecodeDone && mCurScanline >= mAvailableLines) {
703 mScanlineReady.wait(mLock);
704 }
705 return (mCurScanline < mAvailableLines) ? getScanlineInner(dst) : false;
706 }
707
708 return getScanlineInner(dst);
709}
710
711size_t HeifDecoderImpl::skipScanlines(size_t count) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700712 uint32_t oldScanline = mCurScanline;
713 mCurScanline += count;
Chong Zhang8541d302019-07-12 11:19:47 -0700714 if (mCurScanline > mTotalScanline) {
715 mCurScanline = mTotalScanline;
Chong Zhangea280cb2017-08-02 11:10:10 -0700716 }
717 return (mCurScanline > oldScanline) ? (mCurScanline - oldScanline) : 0;
718}
719
720} // namespace android