Implement AImageDecoder _advanceFrame and _rewind
Bug: 160984428
Test: Iae7d274b69999c471fd5610c6ef4d148cca81bec
Disallow AImageDecoder_set* methods after the first frame, since
changing the settings would interfere with blending and caching for
kRestorePrevious frames.
Add a cache (and a state machine) for handling kRestorePrevious frames.
Follow-on to Ib93b0ced09fa3cca4a6681745406355c48158fae - support using
a matrix for unpremul + orientation (the orientation was previously
handled by a matrix internally in SkAndroidCodec).
Change-Id: I7c32ede013fa83f1fe95c35778c33278ca6fe6a3
diff --git a/libs/hwui/hwui/ImageDecoder.cpp b/libs/hwui/hwui/ImageDecoder.cpp
index 6889134..b713f88 100644
--- a/libs/hwui/hwui/ImageDecoder.cpp
+++ b/libs/hwui/hwui/ImageDecoder.cpp
@@ -17,11 +17,19 @@
#include "ImageDecoder.h"
#include <hwui/Bitmap.h>
+#include <log/log.h>
#include <SkAndroidCodec.h>
+#include <SkBitmap.h>
+#include <SkBlendMode.h>
#include <SkCanvas.h>
+#include <SkEncodedOrigin.h>
+#include <SkFilterQuality.h>
#include <SkPaint.h>
+#undef LOG_TAG
+#define LOG_TAG "ImageDecoder"
+
using namespace android;
sk_sp<SkColorSpace> ImageDecoder::getDefaultColorSpace() const {
@@ -44,17 +52,29 @@
, mOutColorType(mCodec->computeOutputColorType(kN32_SkColorType))
, mUnpremultipliedRequired(false)
, mOutColorSpace(getDefaultColorSpace())
- , mSampleSize(1)
{
mTargetSize = swapWidthHeight() ? SkISize { mDecodeSize.height(), mDecodeSize.width() }
: mDecodeSize;
+ this->rewind();
}
+ImageDecoder::~ImageDecoder() = default;
+
SkAlphaType ImageDecoder::getOutAlphaType() const {
return opaque() ? kOpaque_SkAlphaType
: mUnpremultipliedRequired ? kUnpremul_SkAlphaType : kPremul_SkAlphaType;
}
+static SkISize swapped(const SkISize& size) {
+ return SkISize { size.height(), size.width() };
+}
+
+static bool requires_matrix_scaling(bool swapWidthHeight, const SkISize& decodeSize,
+ const SkISize& targetSize) {
+ return (swapWidthHeight && decodeSize != swapped(targetSize))
+ || (!swapWidthHeight && decodeSize != targetSize);
+}
+
bool ImageDecoder::setTargetSize(int width, int height) {
if (width <= 0 || height <= 0) {
return false;
@@ -78,17 +98,21 @@
}
}
- SkISize targetSize = { width, height };
- SkISize decodeSize = swapWidthHeight() ? SkISize { height, width } : targetSize;
+ const bool swap = swapWidthHeight();
+ const SkISize targetSize = { width, height };
+ SkISize decodeSize = swap ? SkISize { height, width } : targetSize;
int sampleSize = mCodec->computeSampleSize(&decodeSize);
- if (decodeSize != targetSize && mUnpremultipliedRequired && !opaque()) {
- return false;
+ if (mUnpremultipliedRequired && !opaque()) {
+ // Allow using a matrix to handle orientation, but not scaling.
+ if (requires_matrix_scaling(swap, decodeSize, targetSize)) {
+ return false;
+ }
}
mTargetSize = targetSize;
mDecodeSize = decodeSize;
- mSampleSize = sampleSize;
+ mOptions.fSampleSize = sampleSize;
return true;
}
@@ -137,8 +161,10 @@
}
bool ImageDecoder::setUnpremultipliedRequired(bool required) {
- if (required && !opaque() && mDecodeSize != mTargetSize) {
- return false;
+ if (required && !opaque()) {
+ if (requires_matrix_scaling(swapWidthHeight(), mDecodeSize, mTargetSize)) {
+ return false;
+ }
}
mUnpremultipliedRequired = required;
return true;
@@ -176,51 +202,150 @@
}
bool ImageDecoder::opaque() const {
- return mCodec->getInfo().alphaType() == kOpaque_SkAlphaType;
+ return mCurrentFrameIsOpaque;
}
bool ImageDecoder::gray() const {
return mCodec->getInfo().colorType() == kGray_8_SkColorType;
}
+bool ImageDecoder::isAnimated() {
+ return mCodec->codec()->getFrameCount() > 1;
+}
+
+int ImageDecoder::currentFrame() const {
+ return mOptions.fFrameIndex;
+}
+
+bool ImageDecoder::rewind() {
+ mOptions.fFrameIndex = 0;
+ mOptions.fPriorFrame = SkCodec::kNoFrame;
+ mCurrentFrameIsIndependent = true;
+ mCurrentFrameIsOpaque = mCodec->getInfo().isOpaque();
+ mRestoreState = RestoreState::kDoNothing;
+ mRestoreFrame = nullptr;
+
+ // TODO: Rewind the input now instead of in the next call to decode, and
+ // plumb through whether rewind succeeded.
+ return true;
+}
+
+bool ImageDecoder::advanceFrame() {
+ const int frameIndex = ++mOptions.fFrameIndex;
+ const int frameCount = mCodec->codec()->getFrameCount();
+ if (frameIndex >= frameCount) {
+ // Prevent overflow from repeated calls to advanceFrame.
+ mOptions.fFrameIndex = frameCount;
+ return false;
+ }
+
+ SkCodec::FrameInfo frameInfo;
+ if (!mCodec->codec()->getFrameInfo(frameIndex, &frameInfo)
+ || !frameInfo.fFullyReceived) {
+ // Mark the decoder as finished, requiring a rewind.
+ mOptions.fFrameIndex = frameCount;
+ return false;
+ }
+
+ mCurrentFrameIsIndependent = frameInfo.fRequiredFrame == SkCodec::kNoFrame;
+ mCurrentFrameIsOpaque = frameInfo.fAlphaType == kOpaque_SkAlphaType;
+
+ if (frameInfo.fDisposalMethod == SkCodecAnimation::DisposalMethod::kRestorePrevious) {
+ switch (mRestoreState) {
+ case RestoreState::kDoNothing:
+ case RestoreState::kNeedsRestore:
+ mRestoreState = RestoreState::kFirstRPFrame;
+ break;
+ case RestoreState::kFirstRPFrame:
+ mRestoreState = RestoreState::kRPFrame;
+ break;
+ case RestoreState::kRPFrame:
+ // Unchanged.
+ break;
+ }
+ } else { // New frame is not restore previous
+ switch (mRestoreState) {
+ case RestoreState::kFirstRPFrame:
+ case RestoreState::kRPFrame:
+ mRestoreState = RestoreState::kNeedsRestore;
+ break;
+ case RestoreState::kNeedsRestore:
+ mRestoreState = RestoreState::kDoNothing;
+ mRestoreFrame = nullptr;
+ [[fallthrough]];
+ case RestoreState::kDoNothing:
+ mOptions.fPriorFrame = frameIndex - 1;
+ break;
+ }
+ }
+
+ return true;
+}
+
+bool ImageDecoder::finished() const {
+ return mOptions.fFrameIndex >= mCodec->codec()->getFrameCount();
+}
+
SkCodec::Result ImageDecoder::decode(void* pixels, size_t rowBytes) {
+ // This was checked inside setTargetSize, but it's possible the first frame
+ // was opaque, so that method succeeded, but after calling advanceFrame, the
+ // current frame is not opaque.
+ if (mUnpremultipliedRequired && !opaque()) {
+ // Allow using a matrix to handle orientation, but not scaling.
+ if (requires_matrix_scaling(swapWidthHeight(), mDecodeSize, mTargetSize)) {
+ return SkCodec::kInvalidScale;
+ }
+ }
+
void* decodePixels = pixels;
size_t decodeRowBytes = rowBytes;
- auto decodeInfo = SkImageInfo::Make(mDecodeSize, mOutColorType, getOutAlphaType(),
- getOutputColorSpace());
+ const auto decodeInfo = SkImageInfo::Make(mDecodeSize, mOutColorType, getOutAlphaType(),
+ getOutputColorSpace());
+ const auto outputInfo = getOutputInfo();
+ switch (mRestoreState) {
+ case RestoreState::kFirstRPFrame:{
+ // This frame is marked kRestorePrevious. The prior frame should be in
+ // |pixels|, and it is what we'll restore after each consecutive
+ // kRestorePrevious frame. Cache it now.
+ if (!(mRestoreFrame = Bitmap::allocateHeapBitmap(outputInfo))) {
+ return SkCodec::kInternalError;
+ }
+
+ const uint8_t* srcRow = static_cast<uint8_t*>(pixels);
+ uint8_t* dstRow = static_cast<uint8_t*>(mRestoreFrame->pixels());
+ for (int y = 0; y < outputInfo.height(); y++) {
+ memcpy(dstRow, srcRow, outputInfo.minRowBytes());
+ srcRow += rowBytes;
+ dstRow += mRestoreFrame->rowBytes();
+ }
+ break;
+ }
+ case RestoreState::kRPFrame:
+ case RestoreState::kNeedsRestore:
+ // Restore the cached frame. It's possible that the client skipped decoding a frame, so
+ // we never cached it.
+ if (mRestoreFrame) {
+ const uint8_t* srcRow = static_cast<uint8_t*>(mRestoreFrame->pixels());
+ uint8_t* dstRow = static_cast<uint8_t*>(pixels);
+ for (int y = 0; y < outputInfo.height(); y++) {
+ memcpy(dstRow, srcRow, outputInfo.minRowBytes());
+ srcRow += mRestoreFrame->rowBytes();
+ dstRow += rowBytes;
+ }
+ }
+ break;
+ case RestoreState::kDoNothing:
+ break;
+ }
+
// Used if we need a temporary before scaling or subsetting.
// FIXME: Use scanline decoding on only a couple lines to save memory. b/70709380.
SkBitmap tmp;
const bool scale = mDecodeSize != mTargetSize;
const auto origin = mCodec->codec()->getOrigin();
const bool handleOrigin = origin != kDefault_SkEncodedOrigin;
+ SkMatrix outputMatrix;
if (scale || handleOrigin || mCropRect) {
- if (!tmp.setInfo(decodeInfo)) {
- return SkCodec::kInternalError;
- }
- if (!Bitmap::allocateHeapBitmap(&tmp)) {
- return SkCodec::kInternalError;
- }
- decodePixels = tmp.getPixels();
- decodeRowBytes = tmp.rowBytes();
- }
-
- SkAndroidCodec::AndroidOptions options;
- options.fSampleSize = mSampleSize;
- auto result = mCodec->getAndroidPixels(decodeInfo, decodePixels, decodeRowBytes, &options);
-
- if (scale || handleOrigin || mCropRect) {
- SkBitmap scaledBm;
- if (!scaledBm.installPixels(getOutputInfo(), pixels, rowBytes)) {
- return SkCodec::kInternalError;
- }
-
- SkPaint paint;
- paint.setBlendMode(SkBlendMode::kSrc);
- paint.setFilterQuality(kLow_SkFilterQuality); // bilinear filtering
-
- SkCanvas canvas(scaledBm, SkCanvas::ColorBehavior::kLegacy);
- SkMatrix outputMatrix;
if (mCropRect) {
outputMatrix.setTranslate(-mCropRect->fLeft, -mCropRect->fTop);
}
@@ -233,13 +358,54 @@
std::swap(targetWidth, targetHeight);
}
}
-
if (scale) {
float scaleX = (float) targetWidth / mDecodeSize.width();
float scaleY = (float) targetHeight / mDecodeSize.height();
outputMatrix.preScale(scaleX, scaleY);
}
+ // It's possible that this portion *does* have alpha, even if the
+ // composed frame does not. In that case, the SkBitmap needs to have
+ // alpha so it blends properly.
+ if (!tmp.setInfo(decodeInfo.makeAlphaType(mUnpremultipliedRequired ? kUnpremul_SkAlphaType
+ : kPremul_SkAlphaType)))
+ {
+ return SkCodec::kInternalError;
+ }
+ if (!Bitmap::allocateHeapBitmap(&tmp)) {
+ return SkCodec::kInternalError;
+ }
+ decodePixels = tmp.getPixels();
+ decodeRowBytes = tmp.rowBytes();
+ if (!mCurrentFrameIsIndependent) {
+ SkMatrix inverse;
+ if (outputMatrix.invert(&inverse)) {
+ SkCanvas canvas(tmp, SkCanvas::ColorBehavior::kLegacy);
+ canvas.setMatrix(inverse);
+ SkPaint paint;
+ paint.setFilterQuality(kLow_SkFilterQuality); // bilinear
+ SkBitmap priorFrame;
+ priorFrame.installPixels(outputInfo, pixels, rowBytes);
+ canvas.drawBitmap(priorFrame, 0, 0, &paint);
+ } else {
+ ALOGE("Failed to invert matrix!");
+ }
+ }
+ }
+
+ auto result = mCodec->getAndroidPixels(decodeInfo, decodePixels, decodeRowBytes, &mOptions);
+
+ if (scale || handleOrigin || mCropRect) {
+ SkBitmap scaledBm;
+ if (!scaledBm.installPixels(outputInfo, pixels, rowBytes)) {
+ return SkCodec::kInternalError;
+ }
+
+ SkPaint paint;
+ paint.setBlendMode(SkBlendMode::kSrc);
+ paint.setFilterQuality(kLow_SkFilterQuality); // bilinear filtering
+
+ SkCanvas canvas(scaledBm, SkCanvas::ColorBehavior::kLegacy);
canvas.setMatrix(outputMatrix);
canvas.drawBitmap(tmp, 0.0f, 0.0f, &paint);
}
diff --git a/libs/hwui/hwui/ImageDecoder.h b/libs/hwui/hwui/ImageDecoder.h
index a08e924..4985932 100644
--- a/libs/hwui/hwui/ImageDecoder.h
+++ b/libs/hwui/hwui/ImageDecoder.h
@@ -15,6 +15,7 @@
*/
#pragma once
+#include <SkAndroidCodec.h>
#include <SkCodec.h>
#include <SkImageInfo.h>
#include <SkPngChunkReader.h>
@@ -24,17 +25,18 @@
#include <optional>
-class SkAndroidCodec;
-
namespace android {
-class ANDROID_API ImageDecoder {
+class Bitmap;
+
+class ANDROID_API ImageDecoder final {
public:
std::unique_ptr<SkAndroidCodec> mCodec;
sk_sp<SkPngChunkReader> mPeeker;
ImageDecoder(std::unique_ptr<SkAndroidCodec> codec,
sk_sp<SkPngChunkReader> peeker = nullptr);
+ ~ImageDecoder();
bool setTargetSize(int width, int height);
bool setCropRect(const SkIRect*);
@@ -46,25 +48,63 @@
sk_sp<SkColorSpace> getDefaultColorSpace() const;
void setOutColorSpace(sk_sp<SkColorSpace> cs);
- // The size is the final size after scaling and cropping.
+ // The size is the final size after scaling, adjusting for the origin, and
+ // cropping.
SkImageInfo getOutputInfo() const;
int width() const;
int height() const;
+ // True if the current frame is opaque.
bool opaque() const;
bool gray() const;
SkCodec::Result decode(void* pixels, size_t rowBytes);
+ // Return true if the decoder has advanced beyond all frames.
+ bool finished() const;
+
+ bool advanceFrame();
+ bool rewind();
+
+ bool isAnimated();
+ int currentFrame() const;
+
private:
+ // State machine for keeping track of how to handle RestorePrevious (RP)
+ // frames in decode().
+ enum class RestoreState {
+ // Neither this frame nor the prior is RP, so there is no need to cache
+ // or restore.
+ kDoNothing,
+
+ // This is the first in a sequence of one or more RP frames. decode()
+ // needs to cache the provided pixels.
+ kFirstRPFrame,
+
+ // This is the second (or later) in a sequence of multiple RP frames.
+ // decode() needs to restore the cached frame that preceded the first RP
+ // frame in the sequence.
+ kRPFrame,
+
+ // This is the first non-RP frame after a sequence of one or more RP
+ // frames. decode() still needs to restore the cached frame. Separate
+ // from kRPFrame because if the following frame is RP the state will
+ // change to kFirstRPFrame.
+ kNeedsRestore,
+ };
+
SkISize mTargetSize;
SkISize mDecodeSize;
SkColorType mOutColorType;
bool mUnpremultipliedRequired;
sk_sp<SkColorSpace> mOutColorSpace;
- int mSampleSize;
+ SkAndroidCodec::AndroidOptions mOptions;
+ bool mCurrentFrameIsIndependent;
+ bool mCurrentFrameIsOpaque;
+ RestoreState mRestoreState;
+ sk_sp<Bitmap> mRestoreFrame;
std::optional<SkIRect> mCropRect;
ImageDecoder(const ImageDecoder&) = delete;
diff --git a/native/graphics/jni/imagedecoder.cpp b/native/graphics/jni/imagedecoder.cpp
index 4aeebe4..971ba37 100644
--- a/native/graphics/jni/imagedecoder.cpp
+++ b/native/graphics/jni/imagedecoder.cpp
@@ -173,7 +173,13 @@
|| format > ANDROID_BITMAP_FORMAT_RGBA_F16) {
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
- return toDecoder(decoder)->setOutColorType(getColorType((AndroidBitmapFormat) format))
+
+ auto* imageDecoder = toDecoder(decoder);
+ if (imageDecoder->currentFrame() != 0) {
+ return ANDROID_IMAGE_DECODER_INVALID_STATE;
+ }
+
+ return imageDecoder->setOutColorType(getColorType((AndroidBitmapFormat) format))
? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_CONVERSION;
}
@@ -185,6 +191,10 @@
}
ImageDecoder* imageDecoder = toDecoder(decoder);
+ if (imageDecoder->currentFrame() != 0) {
+ return ANDROID_IMAGE_DECODER_INVALID_STATE;
+ }
+
imageDecoder->setOutColorSpace(std::move(cs));
return ANDROID_IMAGE_DECODER_SUCCESS;
}
@@ -279,7 +289,12 @@
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
- return toDecoder(decoder)->setUnpremultipliedRequired(required)
+ auto* imageDecoder = toDecoder(decoder);
+ if (imageDecoder->currentFrame() != 0) {
+ return ANDROID_IMAGE_DECODER_INVALID_STATE;
+ }
+
+ return imageDecoder->setUnpremultipliedRequired(required)
? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_CONVERSION;
}
@@ -288,7 +303,12 @@
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
- return toDecoder(decoder)->setTargetSize(width, height)
+ auto* imageDecoder = toDecoder(decoder);
+ if (imageDecoder->currentFrame() != 0) {
+ return ANDROID_IMAGE_DECODER_INVALID_STATE;
+ }
+
+ return imageDecoder->setTargetSize(width, height)
? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_INVALID_SCALE;
}
@@ -309,10 +329,15 @@
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
+ auto* imageDecoder = toDecoder(decoder);
+ if (imageDecoder->currentFrame() != 0) {
+ return ANDROID_IMAGE_DECODER_INVALID_STATE;
+ }
+
SkIRect cropIRect;
cropIRect.setLTRB(crop.left, crop.top, crop.right, crop.bottom);
SkIRect* cropPtr = cropIRect == SkIRect::MakeEmpty() ? nullptr : &cropIRect;
- return toDecoder(decoder)->setCropRect(cropPtr)
+ return imageDecoder->setCropRect(cropPtr)
? ANDROID_IMAGE_DECODER_SUCCESS : ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
@@ -341,6 +366,10 @@
return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
}
+ if (imageDecoder->finished()) {
+ return ANDROID_IMAGE_DECODER_FINISHED;
+ }
+
return ResultToErrorCode(imageDecoder->decode(pixels, stride));
}
@@ -352,7 +381,7 @@
if (!decoder) return false;
ImageDecoder* imageDecoder = toDecoder(decoder);
- return imageDecoder->mCodec->codec()->getFrameCount() > 1;
+ return imageDecoder->isAnimated();
}
int32_t AImageDecoder_getRepeatCount(AImageDecoder* decoder) {
@@ -369,3 +398,34 @@
}
return count;
}
+
+int AImageDecoder_advanceFrame(AImageDecoder* decoder) {
+ if (!decoder) return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
+
+ ImageDecoder* imageDecoder = toDecoder(decoder);
+ if (!imageDecoder->isAnimated()) {
+ return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
+ }
+
+ if (imageDecoder->advanceFrame()) {
+ return ANDROID_IMAGE_DECODER_SUCCESS;
+ }
+
+ if (imageDecoder->finished()) {
+ return ANDROID_IMAGE_DECODER_FINISHED;
+ }
+
+ return ANDROID_IMAGE_DECODER_INCOMPLETE;
+}
+
+int AImageDecoder_rewind(AImageDecoder* decoder) {
+ if (!decoder) return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
+
+ ImageDecoder* imageDecoder = toDecoder(decoder);
+ if (!imageDecoder->isAnimated()) {
+ return ANDROID_IMAGE_DECODER_BAD_PARAMETER;
+ }
+
+ return imageDecoder->rewind() ? ANDROID_IMAGE_DECODER_SUCCESS
+ : ANDROID_IMAGE_DECODER_SEEK_ERROR;
+}
diff --git a/native/graphics/jni/libjnigraphics.map.txt b/native/graphics/jni/libjnigraphics.map.txt
index a184ab9..bd659e0 100644
--- a/native/graphics/jni/libjnigraphics.map.txt
+++ b/native/graphics/jni/libjnigraphics.map.txt
@@ -15,6 +15,8 @@
AImageDecoder_setCrop; # introduced=30
AImageDecoder_isAnimated; # introduced=31
AImageDecoder_getRepeatCount; # introduced=31
+ AImageDecoder_advanceFrame; # introduced=31
+ AImageDecoder_rewind; # introduced=31
AImageDecoderHeaderInfo_getWidth; # introduced=30
AImageDecoderHeaderInfo_getHeight; # introduced=30
AImageDecoderHeaderInfo_getMimeType; # introduced=30