blob: 923f5c1f71852d0d529fa8cd6b4adffe18f017fa [file] [log] [blame]
Chong Zhangea280cb2017-08-02 11:10:10 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
Liz Kammer3ec5a0b2021-07-21 09:30:07 -040018#include "include/HeifDecoderAPI.h"
Chong Zhangea280cb2017-08-02 11:10:10 -070019#define LOG_TAG "HeifDecoderImpl"
20
21#include "HeifDecoderImpl.h"
22
23#include <stdio.h>
24
Marco Nelissendab79b32019-11-18 08:25:47 -080025#include <android/IDataSource.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070026#include <binder/IMemory.h>
Dongwon Kang49ce6712018-01-24 10:16:25 -080027#include <binder/MemoryDealer.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070028#include <drm/drm_framework_common.h>
Ray Essicke89e6322022-02-02 13:33:50 -080029#include <log/log.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070030#include <media/mediametadataretriever.h>
Marco Nelissen7291da62019-12-17 13:01:55 -080031#include <media/stagefright/MediaSource.h>
Chong Zhangd3e0d862017-10-03 13:17:13 -070032#include <media/stagefright/foundation/ADebug.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070033#include <private/media/VideoFrame.h>
34#include <utils/Log.h>
35#include <utils/RefBase.h>
Chong Zhang8541d302019-07-12 11:19:47 -070036#include <vector>
Chong Zhangea280cb2017-08-02 11:10:10 -070037
38HeifDecoder* createHeifDecoder() {
39 return new android::HeifDecoderImpl();
40}
41
42namespace android {
43
Chong Zhang8541d302019-07-12 11:19:47 -070044void initFrameInfo(HeifFrameInfo *info, const VideoFrame *videoFrame) {
45 info->mWidth = videoFrame->mWidth;
46 info->mHeight = videoFrame->mHeight;
47 info->mRotationAngle = videoFrame->mRotationAngle;
48 info->mBytesPerPixel = videoFrame->mBytesPerPixel;
Chong Zhang0af4db62019-08-23 15:34:58 -070049 info->mDurationUs = videoFrame->mDurationUs;
Dichen Zhangf4066492022-02-17 17:32:53 -080050 info->mBitDepth = videoFrame->mBitDepth;
Chong Zhang8541d302019-07-12 11:19:47 -070051 if (videoFrame->mIccSize > 0) {
52 info->mIccData.assign(
53 videoFrame->getFlattenedIccData(),
54 videoFrame->getFlattenedIccData() + videoFrame->mIccSize);
55 } else {
56 // clear old Icc data if there is no Icc data.
57 info->mIccData.clear();
58 }
59}
60
Chong Zhangea280cb2017-08-02 11:10:10 -070061/*
62 * HeifDataSource
63 *
64 * Proxies data requests over IDataSource interface from MediaMetadataRetriever
65 * to the HeifStream interface we received from the heif decoder client.
66 */
67class HeifDataSource : public BnDataSource {
68public:
69 /*
70 * Constructs HeifDataSource; will take ownership of |stream|.
71 */
72 HeifDataSource(HeifStream* stream)
Chong Zhang3f3ffab2017-08-23 13:51:59 -070073 : mStream(stream), mEOS(false),
74 mCachedOffset(0), mCachedSize(0), mCacheBufferSize(0) {}
Chong Zhangea280cb2017-08-02 11:10:10 -070075
76 ~HeifDataSource() override {}
77
78 /*
79 * Initializes internal resources.
80 */
81 bool init();
82
83 sp<IMemory> getIMemory() override { return mMemory; }
84 ssize_t readAt(off64_t offset, size_t size) override;
85 status_t getSize(off64_t* size) override ;
86 void close() {}
87 uint32_t getFlags() override { return 0; }
88 String8 toString() override { return String8("HeifDataSource"); }
Chong Zhangea280cb2017-08-02 11:10:10 -070089
90private:
Chong Zhangea280cb2017-08-02 11:10:10 -070091 enum {
Chong Zhang3f3ffab2017-08-23 13:51:59 -070092 /*
93 * Buffer size for passing the read data to mediaserver. Set to 64K
94 * (which is what MediaDataSource Java API's jni implementation uses).
95 */
Chong Zhangea280cb2017-08-02 11:10:10 -070096 kBufferSize = 64 * 1024,
Chong Zhang3f3ffab2017-08-23 13:51:59 -070097 /*
98 * Initial and max cache buffer size.
99 */
100 kInitialCacheBufferSize = 4 * 1024 * 1024,
101 kMaxCacheBufferSize = 64 * 1024 * 1024,
Chong Zhangea280cb2017-08-02 11:10:10 -0700102 };
103 sp<IMemory> mMemory;
104 std::unique_ptr<HeifStream> mStream;
Chong Zhangea280cb2017-08-02 11:10:10 -0700105 bool mEOS;
Chong Zhangcd031922018-08-24 13:03:19 -0700106 std::unique_ptr<uint8_t[]> mCache;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700107 off64_t mCachedOffset;
108 size_t mCachedSize;
109 size_t mCacheBufferSize;
Chong Zhangea280cb2017-08-02 11:10:10 -0700110};
111
112bool HeifDataSource::init() {
113 sp<MemoryDealer> memoryDealer =
114 new MemoryDealer(kBufferSize, "HeifDataSource");
115 mMemory = memoryDealer->allocate(kBufferSize);
116 if (mMemory == nullptr) {
117 ALOGE("Failed to allocate shared memory!");
118 return false;
119 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700120 mCache.reset(new uint8_t[kInitialCacheBufferSize]);
121 if (mCache.get() == nullptr) {
122 ALOGE("mFailed to allocate cache!");
123 return false;
124 }
125 mCacheBufferSize = kInitialCacheBufferSize;
Chong Zhangea280cb2017-08-02 11:10:10 -0700126 return true;
127}
128
129ssize_t HeifDataSource::readAt(off64_t offset, size_t size) {
130 ALOGV("readAt: offset=%lld, size=%zu", (long long)offset, size);
131
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700132 if (offset < mCachedOffset) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700133 // try seek, then rewind/skip, fail if none worked
134 if (mStream->seek(offset)) {
135 ALOGV("readAt: seek to offset=%lld", (long long)offset);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700136 mCachedOffset = offset;
137 mCachedSize = 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700138 mEOS = false;
139 } else if (mStream->rewind()) {
140 ALOGV("readAt: rewind to offset=0");
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700141 mCachedOffset = 0;
142 mCachedSize = 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700143 mEOS = false;
144 } else {
145 ALOGE("readAt: couldn't seek or rewind!");
146 mEOS = true;
147 }
148 }
149
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700150 if (mEOS && (offset < mCachedOffset ||
151 offset >= (off64_t)(mCachedOffset + mCachedSize))) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700152 ALOGV("readAt: EOS");
153 return ERROR_END_OF_STREAM;
154 }
155
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700156 // at this point, offset must be >= mCachedOffset, other cases should
157 // have been caught above.
158 CHECK(offset >= mCachedOffset);
159
Sungtak Lee237f9032018-03-05 15:21:33 -0800160 off64_t resultOffset;
161 if (__builtin_add_overflow(offset, size, &resultOffset)) {
162 return ERROR_IO;
163 }
164
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700165 if (size == 0) {
166 return 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700167 }
168
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700169 // Can only read max of kBufferSize
Chong Zhangea280cb2017-08-02 11:10:10 -0700170 if (size > kBufferSize) {
171 size = kBufferSize;
172 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700173
174 // copy from cache if the request falls entirely in cache
175 if (offset + size <= mCachedOffset + mCachedSize) {
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700176 memcpy(mMemory->unsecurePointer(), mCache.get() + offset - mCachedOffset, size);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700177 return size;
178 }
179
180 // need to fetch more, check if we need to expand the cache buffer.
181 if ((off64_t)(offset + size) > mCachedOffset + kMaxCacheBufferSize) {
182 // it's reaching max cache buffer size, need to roll window, and possibly
183 // expand the cache buffer.
184 size_t newCacheBufferSize = mCacheBufferSize;
Chong Zhangcd031922018-08-24 13:03:19 -0700185 std::unique_ptr<uint8_t[]> newCache;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700186 uint8_t* dst = mCache.get();
187 if (newCacheBufferSize < kMaxCacheBufferSize) {
188 newCacheBufferSize = kMaxCacheBufferSize;
189 newCache.reset(new uint8_t[newCacheBufferSize]);
190 dst = newCache.get();
191 }
192
193 // when rolling the cache window, try to keep about half the old bytes
194 // in case that the client goes back.
195 off64_t newCachedOffset = offset - (off64_t)(newCacheBufferSize / 2);
196 if (newCachedOffset < mCachedOffset) {
197 newCachedOffset = mCachedOffset;
198 }
199
200 int64_t newCachedSize = (int64_t)(mCachedOffset + mCachedSize) - newCachedOffset;
201 if (newCachedSize > 0) {
202 // in this case, the new cache region partially overlop the old cache,
203 // move the portion of the cache we want to save to the beginning of
204 // the cache buffer.
205 memcpy(dst, mCache.get() + newCachedOffset - mCachedOffset, newCachedSize);
206 } else if (newCachedSize < 0){
207 // in this case, the new cache region is entirely out of the old cache,
208 // in order to guarantee sequential read, we need to skip a number of
209 // bytes before reading.
210 size_t bytesToSkip = -newCachedSize;
211 size_t bytesSkipped = mStream->read(nullptr, bytesToSkip);
212 if (bytesSkipped != bytesToSkip) {
213 // bytesSkipped is invalid, there is not enough bytes to reach
214 // the requested offset.
215 ALOGE("readAt: skip failed, EOS");
216
217 mEOS = true;
218 mCachedOffset = newCachedOffset;
219 mCachedSize = 0;
220 return ERROR_END_OF_STREAM;
221 }
222 // set cache size to 0, since we're not keeping any old cache
223 newCachedSize = 0;
224 }
225
226 if (newCache.get() != nullptr) {
227 mCache.reset(newCache.release());
228 mCacheBufferSize = newCacheBufferSize;
229 }
230 mCachedOffset = newCachedOffset;
231 mCachedSize = newCachedSize;
232
233 ALOGV("readAt: rolling cache window to (%lld, %zu), cache buffer size %zu",
234 (long long)mCachedOffset, mCachedSize, mCacheBufferSize);
235 } else {
236 // expand cache buffer, but no need to roll the window
237 size_t newCacheBufferSize = mCacheBufferSize;
238 while (offset + size > mCachedOffset + newCacheBufferSize) {
239 newCacheBufferSize *= 2;
240 }
241 CHECK(newCacheBufferSize <= kMaxCacheBufferSize);
242 if (mCacheBufferSize < newCacheBufferSize) {
243 uint8_t* newCache = new uint8_t[newCacheBufferSize];
244 memcpy(newCache, mCache.get(), mCachedSize);
245 mCache.reset(newCache);
246 mCacheBufferSize = newCacheBufferSize;
247
248 ALOGV("readAt: current cache window (%lld, %zu), new cache buffer size %zu",
249 (long long) mCachedOffset, mCachedSize, mCacheBufferSize);
250 }
251 }
252 size_t bytesToRead = offset + size - mCachedOffset - mCachedSize;
253 size_t bytesRead = mStream->read(mCache.get() + mCachedSize, bytesToRead);
254 if (bytesRead > bytesToRead || bytesRead == 0) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700255 // bytesRead is invalid
256 mEOS = true;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700257 bytesRead = 0;
258 } else if (bytesRead < bytesToRead) {
259 // read some bytes but not all, set EOS
Chong Zhangea280cb2017-08-02 11:10:10 -0700260 mEOS = true;
261 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700262 mCachedSize += bytesRead;
263 ALOGV("readAt: current cache window (%lld, %zu)",
264 (long long) mCachedOffset, mCachedSize);
265
266 // here bytesAvailable could be negative if offset jumped past EOS.
267 int64_t bytesAvailable = mCachedOffset + mCachedSize - offset;
268 if (bytesAvailable <= 0) {
269 return ERROR_END_OF_STREAM;
270 }
271 if (bytesAvailable < (int64_t)size) {
272 size = bytesAvailable;
273 }
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700274 memcpy(mMemory->unsecurePointer(), mCache.get() + offset - mCachedOffset, size);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700275 return size;
Chong Zhangea280cb2017-08-02 11:10:10 -0700276}
277
278status_t HeifDataSource::getSize(off64_t* size) {
279 if (!mStream->hasLength()) {
280 *size = -1;
281 ALOGE("getSize: not supported!");
282 return ERROR_UNSUPPORTED;
283 }
284 *size = mStream->getLength();
285 ALOGV("getSize: size=%lld", (long long)*size);
286 return OK;
287}
288
289/////////////////////////////////////////////////////////////////////////
290
Chong Zhang0c1407f2018-05-02 17:09:05 -0700291struct HeifDecoderImpl::DecodeThread : public Thread {
292 explicit DecodeThread(HeifDecoderImpl *decoder) : mDecoder(decoder) {}
293
294private:
295 HeifDecoderImpl* mDecoder;
296
297 bool threadLoop();
298
299 DISALLOW_EVIL_CONSTRUCTORS(DecodeThread);
300};
301
302bool HeifDecoderImpl::DecodeThread::threadLoop() {
303 return mDecoder->decodeAsync();
304}
305
306/////////////////////////////////////////////////////////////////////////
307
Chong Zhangea280cb2017-08-02 11:10:10 -0700308HeifDecoderImpl::HeifDecoderImpl() :
309 // output color format should always be set via setOutputColor(), in case
310 // it's not, default to HAL_PIXEL_FORMAT_RGB_565.
311 mOutputColor(HAL_PIXEL_FORMAT_RGB_565),
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700312 mCurScanline(0),
Chong Zhang8541d302019-07-12 11:19:47 -0700313 mTotalScanline(0),
Chong Zhangd3e0d862017-10-03 13:17:13 -0700314 mFrameDecoded(false),
315 mHasImage(false),
Chong Zhang0c1407f2018-05-02 17:09:05 -0700316 mHasVideo(false),
Chong Zhang8541d302019-07-12 11:19:47 -0700317 mSequenceLength(0),
Chong Zhang0c1407f2018-05-02 17:09:05 -0700318 mAvailableLines(0),
319 mNumSlices(1),
320 mSliceHeight(0),
321 mAsyncDecodeDone(false) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700322}
323
324HeifDecoderImpl::~HeifDecoderImpl() {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700325 if (mThread != nullptr) {
326 mThread->join();
327 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700328}
329
330bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) {
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700331 mFrameDecoded = false;
Chong Zhangd3e0d862017-10-03 13:17:13 -0700332 mFrameMemory.clear();
333
Chong Zhangea280cb2017-08-02 11:10:10 -0700334 sp<HeifDataSource> dataSource = new HeifDataSource(stream);
335 if (!dataSource->init()) {
336 return false;
337 }
338 mDataSource = dataSource;
339
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500340 return reinit(frameInfo);
341}
342
343bool HeifDecoderImpl::reinit(HeifFrameInfo* frameInfo) {
344 mFrameDecoded = false;
345 mFrameMemory.clear();
346
Chong Zhangea280cb2017-08-02 11:10:10 -0700347 mRetriever = new MediaMetadataRetriever();
Chong Zhangd3e0d862017-10-03 13:17:13 -0700348 status_t err = mRetriever->setDataSource(mDataSource, "image/heif");
Chong Zhangea280cb2017-08-02 11:10:10 -0700349 if (err != OK) {
350 ALOGE("failed to set data source!");
351
352 mRetriever.clear();
353 mDataSource.clear();
354 return false;
355 }
356 ALOGV("successfully set data source.");
357
Chong Zhangd3e0d862017-10-03 13:17:13 -0700358 const char* hasImage = mRetriever->extractMetadata(METADATA_KEY_HAS_IMAGE);
Chong Zhangea280cb2017-08-02 11:10:10 -0700359 const char* hasVideo = mRetriever->extractMetadata(METADATA_KEY_HAS_VIDEO);
Chong Zhangd3e0d862017-10-03 13:17:13 -0700360
361 mHasImage = hasImage && !strcasecmp(hasImage, "yes");
362 mHasVideo = hasVideo && !strcasecmp(hasVideo, "yes");
Chong Zhang8541d302019-07-12 11:19:47 -0700363
364 HeifFrameInfo* defaultInfo = nullptr;
Chong Zhangd3e0d862017-10-03 13:17:13 -0700365 if (mHasImage) {
366 // image index < 0 to retrieve primary image
Chong Zhang8541d302019-07-12 11:19:47 -0700367 sp<IMemory> sharedMem = mRetriever->getImageAtIndex(
Chong Zhangd3e0d862017-10-03 13:17:13 -0700368 -1, mOutputColor, true /*metaOnly*/);
Chong Zhang8541d302019-07-12 11:19:47 -0700369
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700370 if (sharedMem == nullptr || sharedMem->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700371 ALOGE("init: videoFrame is a nullptr");
372 return false;
373 }
374
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700375 // TODO: Using unsecurePointer() has some associated security pitfalls
376 // (see declaration for details).
377 // Either document why it is safe in this case or address the
378 // issue (e.g. by copying).
379 VideoFrame* videoFrame = static_cast<VideoFrame*>(sharedMem->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700380
Dichen Zhangf4066492022-02-17 17:32:53 -0800381 ALOGV("Image dimension %dx%d, display %dx%d, angle %d, iccSize %d, bitDepth %d",
Chong Zhang8541d302019-07-12 11:19:47 -0700382 videoFrame->mWidth,
383 videoFrame->mHeight,
384 videoFrame->mDisplayWidth,
385 videoFrame->mDisplayHeight,
386 videoFrame->mRotationAngle,
Dichen Zhangf4066492022-02-17 17:32:53 -0800387 videoFrame->mIccSize,
388 videoFrame->mBitDepth);
Chong Zhang8541d302019-07-12 11:19:47 -0700389
390 initFrameInfo(&mImageInfo, videoFrame);
391
392 if (videoFrame->mTileHeight >= 512) {
393 // Try decoding in slices only if the image has tiles and is big enough.
394 mSliceHeight = videoFrame->mTileHeight;
395 ALOGV("mSliceHeight %u", mSliceHeight);
396 }
397
398 defaultInfo = &mImageInfo;
Chong Zhangea280cb2017-08-02 11:10:10 -0700399 }
400
Chong Zhang8541d302019-07-12 11:19:47 -0700401 if (mHasVideo) {
402 sp<IMemory> sharedMem = mRetriever->getFrameAtTime(0,
403 MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC,
404 mOutputColor, true /*metaOnly*/);
405
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700406 if (sharedMem == nullptr || sharedMem->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700407 ALOGE("init: videoFrame is a nullptr");
408 return false;
409 }
410
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700411 // TODO: Using unsecurePointer() has some associated security pitfalls
412 // (see declaration for details).
413 // Either document why it is safe in this case or address the
414 // issue (e.g. by copying).
415 VideoFrame* videoFrame = static_cast<VideoFrame*>(
416 sharedMem->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700417
418 ALOGV("Sequence dimension %dx%d, display %dx%d, angle %d, iccSize %d",
419 videoFrame->mWidth,
420 videoFrame->mHeight,
421 videoFrame->mDisplayWidth,
422 videoFrame->mDisplayHeight,
423 videoFrame->mRotationAngle,
424 videoFrame->mIccSize);
425
426 initFrameInfo(&mSequenceInfo, videoFrame);
427
Ray Essicke89e6322022-02-02 13:33:50 -0800428 const char* frameCount = mRetriever->extractMetadata(METADATA_KEY_VIDEO_FRAME_COUNT);
429 if (frameCount == nullptr) {
430 android_errorWriteWithInfoLog(0x534e4554, "215002587", -1, NULL, 0);
431 ALOGD("No valid sequence information in metadata");
432 return false;
433 }
434 mSequenceLength = atoi(frameCount);
Chong Zhang8541d302019-07-12 11:19:47 -0700435
436 if (defaultInfo == nullptr) {
437 defaultInfo = &mSequenceInfo;
438 }
439 }
440
441 if (defaultInfo == nullptr) {
442 ALOGD("No valid image or sequence available");
Chong Zhangea280cb2017-08-02 11:10:10 -0700443 return false;
444 }
445
Chong Zhangea280cb2017-08-02 11:10:10 -0700446 if (frameInfo != nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700447 *frameInfo = *defaultInfo;
Chong Zhangea280cb2017-08-02 11:10:10 -0700448 }
Chong Zhang8541d302019-07-12 11:19:47 -0700449
450 // default total scanline, this might change if decodeSequence() is used
451 mTotalScanline = defaultInfo->mHeight;
452
453 return true;
454}
455
456bool HeifDecoderImpl::getSequenceInfo(
457 HeifFrameInfo* frameInfo, size_t *frameCount) {
458 ALOGV("%s", __FUNCTION__);
459 if (!mHasVideo) {
460 return false;
461 }
462 if (frameInfo != nullptr) {
463 *frameInfo = mSequenceInfo;
464 }
465 if (frameCount != nullptr) {
466 *frameCount = mSequenceLength;
Chong Zhang0c1407f2018-05-02 17:09:05 -0700467 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700468 return true;
469}
470
471bool HeifDecoderImpl::getEncodedColor(HeifEncodedColor* /*outColor*/) const {
472 ALOGW("getEncodedColor: not implemented!");
473 return false;
474}
475
476bool HeifDecoderImpl::setOutputColor(HeifColorFormat heifColor) {
Liz Kammer3ec5a0b2021-07-21 09:30:07 -0400477 if (heifColor == (HeifColorFormat)mOutputColor) {
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500478 return true;
479 }
480
Chong Zhangea280cb2017-08-02 11:10:10 -0700481 switch(heifColor) {
482 case kHeifColorFormat_RGB565:
483 {
484 mOutputColor = HAL_PIXEL_FORMAT_RGB_565;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500485 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700486 }
487 case kHeifColorFormat_RGBA_8888:
488 {
489 mOutputColor = HAL_PIXEL_FORMAT_RGBA_8888;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500490 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700491 }
492 case kHeifColorFormat_BGRA_8888:
493 {
494 mOutputColor = HAL_PIXEL_FORMAT_BGRA_8888;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500495 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700496 }
Dichen Zhang1b3441a2021-11-17 11:54:43 -0800497 case kHeifColorFormat_RGBA_1010102:
498 {
499 mOutputColor = HAL_PIXEL_FORMAT_RGBA_1010102;
500 break;
501 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700502 default:
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500503 ALOGE("Unsupported output color format %d", heifColor);
504 return false;
Chong Zhangea280cb2017-08-02 11:10:10 -0700505 }
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500506
507 if (mFrameDecoded) {
508 return reinit(nullptr);
509 }
510 return true;
Chong Zhangea280cb2017-08-02 11:10:10 -0700511}
512
Chong Zhang0c1407f2018-05-02 17:09:05 -0700513bool HeifDecoderImpl::decodeAsync() {
514 for (size_t i = 1; i < mNumSlices; i++) {
515 ALOGV("decodeAsync(): decoding slice %zu", i);
516 size_t top = i * mSliceHeight;
517 size_t bottom = (i + 1) * mSliceHeight;
Chong Zhang8541d302019-07-12 11:19:47 -0700518 if (bottom > mImageInfo.mHeight) {
519 bottom = mImageInfo.mHeight;
Chong Zhang0c1407f2018-05-02 17:09:05 -0700520 }
521 sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(
Chong Zhang8541d302019-07-12 11:19:47 -0700522 -1, mOutputColor, 0, top, mImageInfo.mWidth, bottom);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700523 {
524 Mutex::Autolock autolock(mLock);
525
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700526 if (frameMemory == nullptr || frameMemory->unsecurePointer() == nullptr) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700527 mAsyncDecodeDone = true;
528 mScanlineReady.signal();
529 break;
530 }
531 mFrameMemory = frameMemory;
532 mAvailableLines = bottom;
533 ALOGV("decodeAsync(): available lines %zu", mAvailableLines);
534 mScanlineReady.signal();
535 }
536 }
537 // Aggressive clear to avoid holding on to resources
538 mRetriever.clear();
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500539
540 // Hold on to mDataSource in case the client wants to redecode.
Chong Zhang0c1407f2018-05-02 17:09:05 -0700541 return false;
542}
543
Chong Zhangea280cb2017-08-02 11:10:10 -0700544bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700545 // reset scanline pointer
546 mCurScanline = 0;
547
548 if (mFrameDecoded) {
549 return true;
550 }
551
Chong Zhang0c1407f2018-05-02 17:09:05 -0700552 // See if we want to decode in slices to allow client to start
553 // scanline processing in parallel with decode. If this fails
554 // we fallback to decoding the full frame.
Chong Zhang8541d302019-07-12 11:19:47 -0700555 if (mHasImage) {
556 if (mSliceHeight >= 512 &&
557 mImageInfo.mWidth >= 3000 &&
558 mImageInfo.mHeight >= 2000 ) {
559 // Try decoding in slices only if the image has tiles and is big enough.
560 mNumSlices = (mImageInfo.mHeight + mSliceHeight - 1) / mSliceHeight;
561 ALOGV("mSliceHeight %u, mNumSlices %zu", mSliceHeight, mNumSlices);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700562 }
563
Chong Zhang8541d302019-07-12 11:19:47 -0700564 if (mNumSlices > 1) {
565 // get first slice and metadata
566 sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(
567 -1, mOutputColor, 0, 0, mImageInfo.mWidth, mSliceHeight);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700568
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700569 if (frameMemory == nullptr || frameMemory->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700570 ALOGE("decode: metadata is a nullptr");
571 return false;
572 }
573
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700574 // TODO: Using unsecurePointer() has some associated security pitfalls
575 // (see declaration for details).
576 // Either document why it is safe in this case or address the
577 // issue (e.g. by copying).
578 VideoFrame* videoFrame = static_cast<VideoFrame*>(frameMemory->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700579
580 if (frameInfo != nullptr) {
581 initFrameInfo(frameInfo, videoFrame);
582 }
583 mFrameMemory = frameMemory;
584 mAvailableLines = mSliceHeight;
585 mThread = new DecodeThread(this);
586 if (mThread->run("HeifDecode", ANDROID_PRIORITY_FOREGROUND) == OK) {
587 mFrameDecoded = true;
588 return true;
589 }
590 // Fallback to decode without slicing
591 mThread.clear();
592 mNumSlices = 1;
593 mSliceHeight = 0;
594 mAvailableLines = 0;
595 mFrameMemory.clear();
Chong Zhang0c1407f2018-05-02 17:09:05 -0700596 }
Chong Zhang0c1407f2018-05-02 17:09:05 -0700597 }
598
Chong Zhangd3e0d862017-10-03 13:17:13 -0700599 if (mHasImage) {
600 // image index < 0 to retrieve primary image
601 mFrameMemory = mRetriever->getImageAtIndex(-1, mOutputColor);
602 } else if (mHasVideo) {
603 mFrameMemory = mRetriever->getFrameAtTime(0,
604 MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC, mOutputColor);
605 }
606
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700607 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700608 ALOGE("decode: videoFrame is a nullptr");
Chong Zhangea280cb2017-08-02 11:10:10 -0700609 return false;
610 }
611
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700612 // TODO: Using unsecurePointer() has some associated security pitfalls
613 // (see declaration for details).
614 // Either document why it is safe in this case or address the
615 // issue (e.g. by copying).
616 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700617 if (videoFrame->mSize == 0 ||
618 mFrameMemory->size() < videoFrame->getFlattenedSize()) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700619 ALOGE("decode: videoFrame size is invalid");
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700620 return false;
621 }
622
Chong Zhangea280cb2017-08-02 11:10:10 -0700623 ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d",
624 videoFrame->mWidth,
625 videoFrame->mHeight,
626 videoFrame->mDisplayWidth,
627 videoFrame->mDisplayHeight,
628 videoFrame->mRotationAngle,
629 videoFrame->mRowBytes,
630 videoFrame->mSize);
631
632 if (frameInfo != nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700633 initFrameInfo(frameInfo, videoFrame);
634
Chong Zhangea280cb2017-08-02 11:10:10 -0700635 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700636 mFrameDecoded = true;
Chong Zhang4c6398b2017-09-07 17:43:19 -0700637
Chong Zhang0c1407f2018-05-02 17:09:05 -0700638 // Aggressively clear to avoid holding on to resources
Chong Zhang4c6398b2017-09-07 17:43:19 -0700639 mRetriever.clear();
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500640
641 // Hold on to mDataSource in case the client wants to redecode.
Chong Zhangea280cb2017-08-02 11:10:10 -0700642 return true;
643}
644
Chong Zhang8541d302019-07-12 11:19:47 -0700645bool HeifDecoderImpl::decodeSequence(int frameIndex, HeifFrameInfo* frameInfo) {
646 ALOGV("%s: frame index %d", __FUNCTION__, frameIndex);
647 if (!mHasVideo) {
648 return false;
649 }
650
651 if (frameIndex < 0 || frameIndex >= mSequenceLength) {
652 ALOGE("invalid frame index: %d, total frames %zu", frameIndex, mSequenceLength);
653 return false;
654 }
655
656 mCurScanline = 0;
657
658 // set total scanline to sequence height now
659 mTotalScanline = mSequenceInfo.mHeight;
660
661 mFrameMemory = mRetriever->getFrameAtIndex(frameIndex, mOutputColor);
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700662 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700663 ALOGE("decode: videoFrame is a nullptr");
664 return false;
665 }
666
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700667 // TODO: Using unsecurePointer() has some associated security pitfalls
668 // (see declaration for details).
669 // Either document why it is safe in this case or address the
670 // issue (e.g. by copying).
671 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700672 if (videoFrame->mSize == 0 ||
673 mFrameMemory->size() < videoFrame->getFlattenedSize()) {
674 ALOGE("decode: videoFrame size is invalid");
675 return false;
676 }
677
678 ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d",
679 videoFrame->mWidth,
680 videoFrame->mHeight,
681 videoFrame->mDisplayWidth,
682 videoFrame->mDisplayHeight,
683 videoFrame->mRotationAngle,
684 videoFrame->mRowBytes,
685 videoFrame->mSize);
686
687 if (frameInfo != nullptr) {
688 initFrameInfo(frameInfo, videoFrame);
689 }
690 return true;
691}
692
Chong Zhang0c1407f2018-05-02 17:09:05 -0700693bool HeifDecoderImpl::getScanlineInner(uint8_t* dst) {
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700694 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700695 return false;
696 }
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700697 // TODO: Using unsecurePointer() has some associated security pitfalls
698 // (see declaration for details).
699 // Either document why it is safe in this case or address the
700 // issue (e.g. by copying).
701 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhangea280cb2017-08-02 11:10:10 -0700702 uint8_t* src = videoFrame->getFlattenedData() + videoFrame->mRowBytes * mCurScanline++;
Chong Zhang70b0afd2018-02-27 12:43:06 -0800703 memcpy(dst, src, videoFrame->mBytesPerPixel * videoFrame->mWidth);
Chong Zhangea280cb2017-08-02 11:10:10 -0700704 return true;
705}
706
Chong Zhang0c1407f2018-05-02 17:09:05 -0700707bool HeifDecoderImpl::getScanline(uint8_t* dst) {
Chong Zhang8541d302019-07-12 11:19:47 -0700708 if (mCurScanline >= mTotalScanline) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700709 ALOGE("no more scanline available");
710 return false;
Chong Zhangea280cb2017-08-02 11:10:10 -0700711 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700712
Chong Zhang0c1407f2018-05-02 17:09:05 -0700713 if (mNumSlices > 1) {
714 Mutex::Autolock autolock(mLock);
715
716 while (!mAsyncDecodeDone && mCurScanline >= mAvailableLines) {
717 mScanlineReady.wait(mLock);
718 }
719 return (mCurScanline < mAvailableLines) ? getScanlineInner(dst) : false;
720 }
721
722 return getScanlineInner(dst);
723}
724
725size_t HeifDecoderImpl::skipScanlines(size_t count) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700726 uint32_t oldScanline = mCurScanline;
727 mCurScanline += count;
Chong Zhang8541d302019-07-12 11:19:47 -0700728 if (mCurScanline > mTotalScanline) {
729 mCurScanline = mTotalScanline;
Chong Zhangea280cb2017-08-02 11:10:10 -0700730 }
731 return (mCurScanline > oldScanline) ? (mCurScanline - oldScanline) : 0;
732}
733
Dichen Zhangf4066492022-02-17 17:32:53 -0800734uint32_t HeifDecoderImpl::getColorDepth() {
735 HeifFrameInfo* info = &mImageInfo;
736 if (info != nullptr) {
737 return mImageInfo.mBitDepth;
738 } else {
739 return 0;
740 }
741}
742
Chong Zhangea280cb2017-08-02 11:10:10 -0700743} // namespace android