Implement initial version of tone mapping in Skia RenderEngine.
Normally Skia does not perform the same type of polynomial tone mapping
that RenderEngine is expected to do. As such, HDR playback presents far
too darkly when presented alongside SDR content on an SDR target buffer.
Furthermore, precompose HAL is still under development and is not
guaranteed to support tone mapping or be present on all devices, so Skia
RenderEngine needs to support this so that we can test internally.
Works with Youtube HDR - other formats like HLG haven't been implemented
yet and will need some testing.
This structure is also expected to support color transforms applied in
linear space in an upcoming patch.
Bug: 164223050
Test: Peru HDR
Change-Id: Ie95b58e9dad4e4f1936d49e24c426cc884d2de4d
diff --git a/libs/renderengine/Android.bp b/libs/renderengine/Android.bp
index eb967ce..cd7f37b 100644
--- a/libs/renderengine/Android.bp
+++ b/libs/renderengine/Android.bp
@@ -78,6 +78,7 @@
"skia/SkiaRenderEngine.cpp",
"skia/SkiaGLRenderEngine.cpp",
"skia/filters/BlurFilter.cpp",
+ "skia/filters/LinearEffect.cpp",
],
}
diff --git a/libs/renderengine/skia/SkiaGLRenderEngine.cpp b/libs/renderengine/skia/SkiaGLRenderEngine.cpp
index 69ad189..6bf14e2 100644
--- a/libs/renderengine/skia/SkiaGLRenderEngine.cpp
+++ b/libs/renderengine/skia/SkiaGLRenderEngine.cpp
@@ -16,6 +16,9 @@
//#define LOG_NDEBUG 0
#include <cstdint>
+
+#include "SkImageInfo.h"
+#include "system/graphics-base-v1.0.h"
#undef LOG_TAG
#define LOG_TAG "RenderEngine"
#define ATRACE_TAG ATRACE_TAG_GRAPHICS
@@ -42,6 +45,7 @@
#include "../gl/GLExtensions.h"
#include "SkiaGLRenderEngine.h"
#include "filters/BlurFilter.h"
+#include "filters/LinearEffect.h"
extern "C" EGLAPI const char* eglQueryStringImplementationANDROID(EGLDisplay dpy, EGLint name);
@@ -403,6 +407,32 @@
matrix[3][3], 0);
}
+static bool needsToneMapping(ui::Dataspace sourceDataspace, ui::Dataspace destinationDataspace) {
+ int64_t sourceTransfer = sourceDataspace & HAL_DATASPACE_TRANSFER_MASK;
+ int64_t destTransfer = destinationDataspace & HAL_DATASPACE_TRANSFER_MASK;
+
+ // Treat unsupported dataspaces as srgb
+ if (destTransfer != HAL_DATASPACE_TRANSFER_LINEAR &&
+ destTransfer != HAL_DATASPACE_TRANSFER_HLG &&
+ destTransfer != HAL_DATASPACE_TRANSFER_ST2084) {
+ destTransfer = HAL_DATASPACE_TRANSFER_SRGB;
+ }
+
+ if (sourceTransfer != HAL_DATASPACE_TRANSFER_LINEAR &&
+ sourceTransfer != HAL_DATASPACE_TRANSFER_HLG &&
+ sourceTransfer != HAL_DATASPACE_TRANSFER_ST2084) {
+ sourceTransfer = HAL_DATASPACE_TRANSFER_SRGB;
+ }
+
+ const bool isSourceLinear = sourceTransfer == HAL_DATASPACE_TRANSFER_LINEAR;
+ const bool isSourceSRGB = sourceTransfer == HAL_DATASPACE_TRANSFER_SRGB;
+ const bool isDestLinear = destTransfer == HAL_DATASPACE_TRANSFER_LINEAR;
+ const bool isDestSRGB = destTransfer == HAL_DATASPACE_TRANSFER_SRGB;
+
+ return !(isSourceLinear && isDestSRGB) && !(isSourceSRGB && isDestLinear) &&
+ sourceTransfer != destTransfer;
+}
+
void SkiaGLRenderEngine::unbindExternalTextureBuffer(uint64_t bufferId) {
std::lock_guard<std::mutex> lock(mRenderingMutex);
mImageCache.erase(bufferId);
@@ -516,14 +546,20 @@
if (iter != mImageCache.end()) {
image = iter->second;
} else {
- image = SkImage::MakeFromAHardwareBuffer(item.buffer->toAHardwareBuffer(),
- item.usePremultipliedAlpha
- ? kPremul_SkAlphaType
- : kUnpremul_SkAlphaType,
- mUseColorManagement
- ? toColorSpace(
- layer->sourceDataspace)
- : SkColorSpace::MakeSRGB());
+ image = SkImage::MakeFromAHardwareBuffer(
+ item.buffer->toAHardwareBuffer(),
+ item.isOpaque ? kOpaque_SkAlphaType
+ : (item.usePremultipliedAlpha ? kPremul_SkAlphaType
+ : kUnpremul_SkAlphaType),
+ mUseColorManagement
+ ? (needsToneMapping(layer->sourceDataspace, display.outputDataspace)
+ // If we need to map to linear space, then
+ // mark the source image with the same
+ // colorspace as the destination surface so
+ // that Skia's color management is a no-op.
+ ? toColorSpace(display.outputDataspace)
+ : toColorSpace(layer->sourceDataspace))
+ : SkColorSpace::MakeSRGB());
mImageCache.insert({item.buffer->getId(), image});
}
@@ -567,7 +603,22 @@
matrix.postConcat(texMatrix);
matrix.postScale(rotatedBufferWidth, rotatedBufferHeight);
- paint.setShader(image->makeShader(matrix));
+ sk_sp<SkShader> shader = image->makeShader(matrix);
+
+ if (mUseColorManagement &&
+ needsToneMapping(layer->sourceDataspace, display.outputDataspace)) {
+ LinearEffect effect = LinearEffect{.inputDataspace = layer->sourceDataspace,
+ .outputDataspace = display.outputDataspace,
+ .undoPremultipliedAlpha = !item.isOpaque &&
+ item.usePremultipliedAlpha};
+ sk_sp<SkRuntimeEffect> runtimeEffect = buildRuntimeEffect(effect);
+ paint.setShader(createLinearEffectShader(shader, effect, runtimeEffect,
+ display.maxLuminance,
+ layer->source.buffer.maxMasteringLuminance,
+ layer->source.buffer.maxContentLuminance));
+ } else {
+ paint.setShader(shader);
+ }
} else {
ATRACE_NAME("DrawColor");
const auto color = layer->source.solidColor;
diff --git a/libs/renderengine/skia/filters/LinearEffect.cpp b/libs/renderengine/skia/filters/LinearEffect.cpp
new file mode 100644
index 0000000..376abdf
--- /dev/null
+++ b/libs/renderengine/skia/filters/LinearEffect.cpp
@@ -0,0 +1,308 @@
+/*
+ * Copyright 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "LinearEffect.h"
+
+#include <SkString.h>
+
+#include <optional>
+
+#include "log/log.h"
+#include "math/mat4.h"
+#include "ui/ColorSpace.h"
+
+namespace android {
+namespace renderengine {
+namespace skia {
+
+static void generateEOTF(ui::Dataspace dataspace, SkString& shader) {
+ switch (dataspace & HAL_DATASPACE_TRANSFER_MASK) {
+ case HAL_DATASPACE_TRANSFER_ST2084:
+ shader.append(R"(
+
+ float3 EOTF(float3 color) {
+ float m1 = (2610.0 / 4096.0) / 4.0;
+ float m2 = (2523.0 / 4096.0) * 128.0;
+ float c1 = (3424.0 / 4096.0);
+ float c2 = (2413.0 / 4096.0) * 32.0;
+ float c3 = (2392.0 / 4096.0) * 32.0;
+
+ float3 tmp = pow(clamp(color, 0.0, 1.0), 1.0 / float3(m2));
+ tmp = max(tmp - c1, 0.0) / (c2 - c3 * tmp);
+ return pow(tmp, 1.0 / float3(m1));
+ }
+ )");
+ break;
+ default:
+ shader.append(R"(
+
+ float EOTF_sRGB(float srgb) {
+ return srgb <= 0.04045 ? srgb / 12.92 : pow((srgb + 0.055) / 1.055, 2.4);
+ }
+
+ float3 EOTF_sRGB(float3 srgb) {
+ return float3(EOTF_sRGB(srgb.r), EOTF_sRGB(srgb.g), EOTF_sRGB(srgb.b));
+ }
+
+ float3 EOTF(float3 srgb) {
+ return sign(srgb.rgb) * EOTF_sRGB(abs(srgb.rgb));
+ }
+ )");
+ break;
+ }
+}
+
+static void generateXYZTransforms(SkString& shader) {
+ shader.append(R"(
+ uniform float4x4 in_rgbToXyz;
+ uniform float4x4 in_xyzToRgb;
+ float3 ToXYZ(float3 rgb) {
+ return clamp((in_rgbToXyz * float4(rgb, 1.0)).rgb, 0.0, 1.0);
+ }
+
+ float3 ToRGB(float3 xyz) {
+ return clamp((in_xyzToRgb * float4(xyz, 1.0)).rgb, 0.0, 1.0);
+ }
+ )");
+}
+
+static void generateOOTF(ui::Dataspace inputDataspace, ui::Dataspace outputDataspace,
+ SkString& shader) {
+ shader.append(R"(
+ uniform float in_displayMaxLuminance;
+ uniform float in_inputMaxLuminance;
+ uniform float in_maxContentLuminance;
+ )");
+ switch (inputDataspace & HAL_DATASPACE_TRANSFER_MASK) {
+ case HAL_DATASPACE_TRANSFER_ST2084:
+ shader.append(R"(
+ float3 ScaleLuminance(float3 xyz) {
+ return xyz * 10000.0;
+ }
+ )");
+
+ switch (outputDataspace & HAL_DATASPACE_TRANSFER_MASK) {
+ default:
+ shader.append(R"(
+ float3 ToneMap(float3 xyz) {
+ float maxInLumi = in_inputMaxLuminance;
+ float maxOutLumi = in_displayMaxLuminance;
+
+ float nits = xyz.y;
+
+ // clamp to max input luminance
+ nits = clamp(nits, 0.0, maxInLumi);
+
+ // scale [0.0, maxInLumi] to [0.0, maxOutLumi]
+ if (maxInLumi <= maxOutLumi) {
+ return xyz * (maxOutLumi / maxInLumi);
+ } else {
+ // three control points
+ const float x0 = 10.0;
+ const float y0 = 17.0;
+ float x1 = maxOutLumi * 0.75;
+ float y1 = x1;
+ float x2 = x1 + (maxInLumi - x1) / 2.0;
+ float y2 = y1 + (maxOutLumi - y1) * 0.75;
+
+ // horizontal distances between the last three control points
+ float h12 = x2 - x1;
+ float h23 = maxInLumi - x2;
+ // tangents at the last three control points
+ float m1 = (y2 - y1) / h12;
+ float m3 = (maxOutLumi - y2) / h23;
+ float m2 = (m1 + m3) / 2.0;
+
+ if (nits < x0) {
+ // scale [0.0, x0] to [0.0, y0] linearly
+ float slope = y0 / x0;
+ return xyz * slope;
+ } else if (nits < x1) {
+ // scale [x0, x1] to [y0, y1] linearly
+ float slope = (y1 - y0) / (x1 - x0);
+ nits = y0 + (nits - x0) * slope;
+ } else if (nits < x2) {
+ // scale [x1, x2] to [y1, y2] using Hermite interp
+ float t = (nits - x1) / h12;
+ nits = (y1 * (1.0 + 2.0 * t) + h12 * m1 * t) * (1.0 - t) * (1.0 - t) +
+ (y2 * (3.0 - 2.0 * t) + h12 * m2 * (t - 1.0)) * t * t;
+ } else {
+ // scale [x2, maxInLumi] to [y2, maxOutLumi] using Hermite interp
+ float t = (nits - x2) / h23;
+ nits = (y2 * (1.0 + 2.0 * t) + h23 * m2 * t) * (1.0 - t) * (1.0 - t) +
+ (maxOutLumi * (3.0 - 2.0 * t) + h23 * m3 * (t - 1.0)) * t * t;
+ }
+ }
+
+ // color.y is greater than x0 and is thus non-zero
+ return xyz * (nits / xyz.y);
+ }
+ )");
+ break;
+ }
+ break;
+ default:
+ shader.append(R"(
+ float3 ScaleLuminance(float3 xyz) {
+ return xyz * in_displayMaxLuminance;
+ }
+
+ float3 ToneMap(float3 xyz) {
+ return xyz;
+ }
+ )");
+ break;
+ }
+
+ switch (outputDataspace & HAL_DATASPACE_TRANSFER_MASK) {
+ case HAL_DATASPACE_TRANSFER_ST2084:
+ shader.append(R"(
+ float3 NormalizeLuminance(float3 xyz) {
+ return xyz / 10000.0;
+ }
+ )");
+ break;
+ default:
+ shader.append(R"(
+ float3 NormalizeLuminance(float3 xyz) {
+ return xyz / in_displayMaxLuminance;
+ }
+ )");
+ break;
+ }
+
+ shader.append(R"(
+ float3 OOTF(float3 xyz) {
+ return NormalizeLuminance(ToneMap(ScaleLuminance(xyz)));
+ }
+ )");
+}
+
+static void generateOETF(ui::Dataspace dataspace, SkString& shader) {
+ switch (dataspace & HAL_DATASPACE_TRANSFER_MASK) {
+ case HAL_DATASPACE_TRANSFER_ST2084:
+ shader.append(R"(
+
+ float3 OETF(float3 xyz) {
+ float m1 = (2610.0 / 4096.0) / 4.0;
+ float m2 = (2523.0 / 4096.0) * 128.0;
+ float c1 = (3424.0 / 4096.0);
+ float c2 = (2413.0 / 4096.0) * 32.0;
+ float c3 = (2392.0 / 4096.0) * 32.0;
+
+ float3 tmp = pow(xyz, float3(m1));
+ tmp = (c1 + c2 * tmp) / (1.0 + c3 * tmp);
+ return pow(tmp, float3(m2));
+ }
+ )");
+ break;
+ default:
+ shader.append(R"(
+ float OETF_sRGB(float linear) {
+ return linear <= 0.0031308 ?
+ linear * 12.92 : (pow(linear, 1.0 / 2.4) * 1.055) - 0.055;
+ }
+
+ float3 OETF_sRGB(float3 linear) {
+ return float3(OETF_sRGB(linear.r), OETF_sRGB(linear.g), OETF_sRGB(linear.b));
+ }
+
+ float3 OETF(float3 linear) {
+ return sign(linear.rgb) * OETF_sRGB(abs(linear.rgb));
+ }
+ )");
+ break;
+ }
+}
+
+static void generateEffectiveOOTF(bool undoPremultipliedAlpha, SkString& shader) {
+ shader.append(R"(
+ in shader input;
+ half4 main(float2 xy) {
+ float4 c = float4(sample(input, xy));
+ )");
+ if (undoPremultipliedAlpha) {
+ shader.append(R"(
+ c.rgb = c.rgb / (c.a + 0.0019);
+ )");
+ }
+ shader.append(R"(
+ c.rgb = OETF(ToRGB(OOTF(ToXYZ(EOTF(c.rgb)))));
+ )");
+ if (undoPremultipliedAlpha) {
+ shader.append(R"(
+ c.rgb = c.rgb * (c.a + 0.0019);
+ )");
+ }
+ shader.append(R"(
+ return c;
+ }
+ )");
+}
+static ColorSpace toColorSpace(ui::Dataspace dataspace) {
+ switch (dataspace & HAL_DATASPACE_STANDARD_MASK) {
+ case HAL_DATASPACE_STANDARD_BT709:
+ return ColorSpace::sRGB();
+ break;
+ case HAL_DATASPACE_STANDARD_DCI_P3:
+ return ColorSpace::DisplayP3();
+ break;
+ case HAL_DATASPACE_STANDARD_BT2020:
+ return ColorSpace::BT2020();
+ break;
+ default:
+ return ColorSpace::sRGB();
+ break;
+ }
+}
+
+sk_sp<SkRuntimeEffect> buildRuntimeEffect(const LinearEffect& linearEffect) {
+ SkString shaderString;
+ generateEOTF(linearEffect.inputDataspace, shaderString);
+ generateXYZTransforms(shaderString);
+ generateOOTF(linearEffect.inputDataspace, linearEffect.outputDataspace, shaderString);
+ generateOETF(linearEffect.outputDataspace, shaderString);
+ generateEffectiveOOTF(linearEffect.undoPremultipliedAlpha, shaderString);
+
+ auto [shader, error] = SkRuntimeEffect::Make(shaderString);
+ if (!shader) {
+ LOG_ALWAYS_FATAL("LinearColorFilter construction error: %s", error.c_str());
+ }
+ return shader;
+}
+
+sk_sp<SkShader> createLinearEffectShader(sk_sp<SkShader> shader, const LinearEffect& linearEffect,
+ sk_sp<SkRuntimeEffect> runtimeEffect,
+ float maxDisplayLuminance, float maxMasteringLuminance,
+ float maxContentLuminance) {
+ SkRuntimeShaderBuilder effectBuilder(runtimeEffect);
+
+ effectBuilder.child("input") = shader;
+
+ ColorSpace inputColorSpace = toColorSpace(linearEffect.inputDataspace);
+ ColorSpace outputColorSpace = toColorSpace(linearEffect.outputDataspace);
+
+ effectBuilder.uniform("in_rgbToXyz") = mat4(inputColorSpace.getRGBtoXYZ());
+ effectBuilder.uniform("in_xyzToRgb") = mat4(outputColorSpace.getXYZtoRGB());
+ effectBuilder.uniform("in_displayMaxLuminance") = maxDisplayLuminance;
+ effectBuilder.uniform("in_inputMaxLuminance") =
+ std::min(maxMasteringLuminance, maxContentLuminance);
+ return effectBuilder.makeShader(nullptr, false);
+}
+
+} // namespace skia
+} // namespace renderengine
+} // namespace android
\ No newline at end of file
diff --git a/libs/renderengine/skia/filters/LinearEffect.h b/libs/renderengine/skia/filters/LinearEffect.h
new file mode 100644
index 0000000..2615669
--- /dev/null
+++ b/libs/renderengine/skia/filters/LinearEffect.h
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <optional>
+
+#include "SkColorMatrix.h"
+#include "SkRuntimeEffect.h"
+#include "SkShader.h"
+#include "ui/GraphicTypes.h"
+
+namespace android {
+namespace renderengine {
+namespace skia {
+
+/**
+ * Arguments for creating an effect that applies color transformations in linear XYZ space.
+ * A linear effect is decomposed into the following steps when operating on an image:
+ * 1. Electrical-Optical Transfer Function (EOTF) maps the input RGB signal into the intended
+ * relative display brightness of the scene in nits for each RGB channel
+ * 2. Transformation matrix from linear RGB brightness to linear XYZ, to operate on display
+ * luminance.
+ * 3. Opto-Optical Transfer Function (OOTF) applies a "rendering intent". This can include tone
+ * mapping to display SDR content alongside HDR content, or any number of subjective transformations
+ * 4. Transformation matrix from linear XYZ back to linear RGB brightness.
+ * 5. Opto-Electronic Transfer Function (OETF) maps the display brightness of the scene back to
+ * output RGB colors.
+ *
+ * For further reading, consult the recommendation in ITU-R BT.2390-4:
+ * https://www.itu.int/dms_pub/itu-r/opb/rep/R-REP-BT.2390-4-2018-PDF-E.pdf
+ *
+ * Skia normally attempts to do its own simple tone mapping, i.e., the working color space is
+ * intended to be the output surface. However, Skia does not support complex tone mapping such as
+ * polynomial interpolation. As such, this filter assumes that tone mapping has not yet been applied
+ * to the source colors. so that the tone mapping process is only applied once by this effect. Tone
+ * mapping is applied when presenting HDR content (content with HLG or PQ transfer functions)
+ * alongside other content, whereby maximum input luminance is mapped to maximum output luminance
+ * and intermediate values are interpolated.
+ */
+struct LinearEffect {
+ // Input dataspace of the source colors.
+ const ui::Dataspace inputDataspace = ui::Dataspace::SRGB;
+
+ // Working dataspace for the output surface, for conversion from linear space.
+ const ui::Dataspace outputDataspace = ui::Dataspace::SRGB;
+
+ // Sets whether alpha premultiplication must be undone.
+ // This is required if the source colors use premultiplied alpha and is not opaque.
+ const bool undoPremultipliedAlpha = false;
+};
+
+sk_sp<SkRuntimeEffect> buildRuntimeEffect(const LinearEffect& linearEffect);
+
+// Generates a shader resulting from applying the a linear effect created from
+// LinearEffectARgs::buildEffect to an inputShader. We also provide additional HDR metadata upon
+// creating the shader:
+// * The max display luminance is the max luminance of the physical display in nits
+// * The max mastering luminance is provided as the max luminance from the SMPTE 2086
+// standard.
+// * The max content luminance is provided as the max light level from the CTA 861.3
+// standard.
+sk_sp<SkShader> createLinearEffectShader(sk_sp<SkShader> inputShader,
+ const LinearEffect& linearEffect,
+ sk_sp<SkRuntimeEffect> runtimeEffect,
+ float maxDisplayLuminance, float maxMasteringLuminance,
+ float maxContentLuminance);
+} // namespace skia
+} // namespace renderengine
+} // namespace android