End-to-end plumbing for dimming SDR layers
Model here is:
* HDR luminance is set to the current display brightness
* SDR luminance is set to the current SDR white point reported by
DisplayManager
Ideally we use scene-referred white points instead, so:
* PQ is always 10k nits
* HLG is always 1k nits
* Everything else is 150-200 nits
So relative dimming thresholds are fixed. But right now this is visually
less jarring (otherwise youtube UI will suddenly dim when autoplaying
HDR video).
Bug: 200310158
Test: Verified that plumbing sdr white point is sent to renderengine
Test: librenderengine_test
Test: libcompositionengine_test
Test: DataspaceUtils_test
Change-Id: I5bcea7941935c43e57cd5434e1ec69b41d31f2b4
diff --git a/libs/renderengine/include/renderengine/DisplaySettings.h b/libs/renderengine/include/renderengine/DisplaySettings.h
index d395d06..b4cab39 100644
--- a/libs/renderengine/include/renderengine/DisplaySettings.h
+++ b/libs/renderengine/include/renderengine/DisplaySettings.h
@@ -57,8 +57,10 @@
// orientation.
uint32_t orientation = ui::Transform::ROT_0;
- // SDR white point, -1f if unknown
- float sdrWhitePointNits = -1.f;
+ // Target luminance of the display. -1f if unknown.
+ // All layers will be dimmed by (max(layer white points) / targetLuminanceNits).
+ // If the target luminance is unknown, then no display-level dimming occurs.
+ float targetLuminanceNits = -1.f;
};
static inline bool operator==(const DisplaySettings& lhs, const DisplaySettings& rhs) {
diff --git a/libs/renderengine/include/renderengine/LayerSettings.h b/libs/renderengine/include/renderengine/LayerSettings.h
index 715f3f8..702e8b0 100644
--- a/libs/renderengine/include/renderengine/LayerSettings.h
+++ b/libs/renderengine/include/renderengine/LayerSettings.h
@@ -171,6 +171,12 @@
// Name associated with the layer for debugging purposes.
std::string name;
+
+ // Luminance of the white point for this layer. Used for linear dimming.
+ // Individual layers will be dimmed by (whitePointNits / maxWhitePoint).
+ // If white point nits are unknown, then this layer is assumed to have the
+ // same luminance as the brightest layer in the scene.
+ float whitePointNits = -1.f;
};
// Keep in sync with custom comparison function in
diff --git a/libs/renderengine/skia/SkiaGLRenderEngine.cpp b/libs/renderengine/skia/SkiaGLRenderEngine.cpp
index 376e279..cc90946 100644
--- a/libs/renderengine/skia/SkiaGLRenderEngine.cpp
+++ b/libs/renderengine/skia/SkiaGLRenderEngine.cpp
@@ -46,6 +46,7 @@
#include <cmath>
#include <cstdint>
#include <memory>
+#include <numeric>
#include "../gl/GLExtensions.h"
#include "Cache.h"
@@ -612,33 +613,33 @@
AutoBackendTexture::CleanupManager& mMgr;
};
-sk_sp<SkShader> SkiaGLRenderEngine::createRuntimeEffectShader(sk_sp<SkShader> shader,
- const LayerSettings& layer,
- const DisplaySettings& display,
- bool undoPremultipliedAlpha,
- bool requiresLinearEffect) {
- const auto stretchEffect = layer.stretchEffect;
+sk_sp<SkShader> SkiaGLRenderEngine::createRuntimeEffectShader(
+ const RuntimeEffectShaderParameters& parameters) {
// The given surface will be stretched by HWUI via matrix transformation
// which gets similar results for most surfaces
// Determine later on if we need to leverage the stertch shader within
// surface flinger
+ const auto& stretchEffect = parameters.layer.stretchEffect;
+ auto shader = parameters.shader;
if (stretchEffect.hasEffect()) {
- const auto targetBuffer = layer.source.buffer.buffer;
+ const auto targetBuffer = parameters.layer.source.buffer.buffer;
const auto graphicBuffer = targetBuffer ? targetBuffer->getBuffer() : nullptr;
- if (graphicBuffer && shader) {
+ if (graphicBuffer && parameters.shader) {
shader = mStretchShaderFactory.createSkShader(shader, stretchEffect);
}
}
- if (requiresLinearEffect) {
- const ui::Dataspace inputDataspace =
- mUseColorManagement ? layer.sourceDataspace : ui::Dataspace::V0_SRGB_LINEAR;
- const ui::Dataspace outputDataspace =
- mUseColorManagement ? display.outputDataspace : ui::Dataspace::V0_SRGB_LINEAR;
+ if (parameters.requiresLinearEffect) {
+ const ui::Dataspace inputDataspace = mUseColorManagement ? parameters.layer.sourceDataspace
+ : ui::Dataspace::V0_SRGB_LINEAR;
+ const ui::Dataspace outputDataspace = mUseColorManagement
+ ? parameters.display.outputDataspace
+ : ui::Dataspace::V0_SRGB_LINEAR;
- auto effect = shaders::LinearEffect{.inputDataspace = inputDataspace,
- .outputDataspace = outputDataspace,
- .undoPremultipliedAlpha = undoPremultipliedAlpha};
+ auto effect =
+ shaders::LinearEffect{.inputDataspace = inputDataspace,
+ .outputDataspace = outputDataspace,
+ .undoPremultipliedAlpha = parameters.undoPremultipliedAlpha};
auto effectIter = mRuntimeEffects.find(effect);
sk_sp<SkRuntimeEffect> runtimeEffect = nullptr;
@@ -648,16 +649,16 @@
} else {
runtimeEffect = effectIter->second;
}
- float maxLuminance = layer.source.buffer.maxLuminanceNits;
- // If the buffer doesn't have a max luminance, treat it as SDR & use the display's SDR
- // white point
- if (maxLuminance <= 0.f) {
- maxLuminance = display.sdrWhitePointNits;
- }
- return createLinearEffectShader(shader, effect, runtimeEffect, layer.colorTransform,
- display.maxLuminance, maxLuminance);
+ mat4 colorTransform = parameters.layer.colorTransform;
+
+ colorTransform *=
+ mat4::scale(vec4(parameters.layerDimmingRatio, parameters.layerDimmingRatio,
+ parameters.layerDimmingRatio, 1.f));
+ return createLinearEffectShader(parameters.shader, effect, runtimeEffect, colorTransform,
+ parameters.display.maxLuminance,
+ parameters.layer.source.buffer.maxLuminanceNits);
}
- return shader;
+ return parameters.shader;
}
void SkiaGLRenderEngine::initCanvas(SkCanvas* canvas, const DisplaySettings& display) {
@@ -730,6 +731,11 @@
return roundedRect;
}
+static bool equalsWithinMargin(float expected, float value, float margin) {
+ LOG_ALWAYS_FATAL_IF(margin < 0.f, "Margin is negative!");
+ return std::abs(expected - value) < margin;
+}
+
void SkiaGLRenderEngine::drawLayersInternal(
const std::shared_ptr<std::promise<RenderEngineResult>>&& resultPromise,
const DisplaySettings& display, const std::vector<LayerSettings>& layers,
@@ -791,6 +797,18 @@
const bool ctModifiesAlpha =
displayColorTransform && !displayColorTransform->isAlphaUnchanged();
+ // Find the max layer white point to determine the max luminance of the scene...
+ const float maxLayerWhitePoint = std::transform_reduce(
+ layers.cbegin(), layers.cend(), 0.f,
+ [](float left, float right) { return std::max(left, right); },
+ [&](const auto& l) { return l.whitePointNits; });
+
+ // ...and compute the dimming ratio if dimming is requested
+ const float displayDimmingRatio = display.targetLuminanceNits > 0.f &&
+ maxLayerWhitePoint > 0.f && display.targetLuminanceNits > maxLayerWhitePoint
+ ? maxLayerWhitePoint / display.targetLuminanceNits
+ : 1.f;
+
// Find if any layers have requested blur, we'll use that info to decide when to render to an
// offscreen buffer and when to render to the native buffer.
sk_sp<SkSurface> activeSurface(dstSurface);
@@ -964,11 +982,14 @@
drawShadow(canvas, rrect, layer.shadow);
}
+ const float layerDimmingRatio = layer.whitePointNits <= 0.f
+ ? displayDimmingRatio
+ : (layer.whitePointNits / maxLayerWhitePoint) * displayDimmingRatio;
+
const bool requiresLinearEffect = layer.colorTransform != mat4() ||
(mUseColorManagement &&
needsToneMapping(layer.sourceDataspace, display.outputDataspace)) ||
- (display.sdrWhitePointNits > 0.f &&
- display.sdrWhitePointNits != display.maxLuminance);
+ !equalsWithinMargin(1.f, layerDimmingRatio, 0.001f);
// quick abort from drawing the remaining portion of the layer
if (layer.skipContentDraw ||
@@ -1067,9 +1088,20 @@
toSkColorSpace(layerDataspace)));
}
- paint.setShader(createRuntimeEffectShader(shader, layer, display,
- !item.isOpaque && item.usePremultipliedAlpha,
- requiresLinearEffect));
+ paint.setShader(createRuntimeEffectShader(
+ RuntimeEffectShaderParameters{.shader = shader,
+ .layer = layer,
+ .display = display,
+ .undoPremultipliedAlpha = !item.isOpaque &&
+ item.usePremultipliedAlpha,
+ .requiresLinearEffect = requiresLinearEffect,
+ .layerDimmingRatio = layerDimmingRatio}));
+
+ // Turn on dithering when dimming beyond this threshold.
+ static constexpr float kDimmingThreshold = 0.2f;
+ if (layerDimmingRatio <= kDimmingThreshold) {
+ paint.setDither(true);
+ }
paint.setAlphaf(layer.alpha);
} else {
ATRACE_NAME("DrawColor");
@@ -1079,9 +1111,13 @@
.fB = color.b,
.fA = layer.alpha},
toSkColorSpace(layerDataspace));
- paint.setShader(createRuntimeEffectShader(shader, layer, display,
- /* undoPremultipliedAlpha */ false,
- requiresLinearEffect));
+ paint.setShader(createRuntimeEffectShader(
+ RuntimeEffectShaderParameters{.shader = shader,
+ .layer = layer,
+ .display = display,
+ .undoPremultipliedAlpha = false,
+ .requiresLinearEffect = requiresLinearEffect,
+ .layerDimmingRatio = layerDimmingRatio}));
}
if (layer.disableBlending) {
diff --git a/libs/renderengine/skia/SkiaGLRenderEngine.h b/libs/renderengine/skia/SkiaGLRenderEngine.h
index 53792f9..a650313 100644
--- a/libs/renderengine/skia/SkiaGLRenderEngine.h
+++ b/libs/renderengine/skia/SkiaGLRenderEngine.h
@@ -106,12 +106,18 @@
void initCanvas(SkCanvas* canvas, const DisplaySettings& display);
void drawShadow(SkCanvas* canvas, const SkRRect& casterRRect,
const ShadowSettings& shadowSettings);
+
// If requiresLinearEffect is true or the layer has a stretchEffect a new shader is returned.
// Otherwise it returns the input shader.
- sk_sp<SkShader> createRuntimeEffectShader(sk_sp<SkShader> shader, const LayerSettings& layer,
- const DisplaySettings& display,
- bool undoPremultipliedAlpha,
- bool requiresLinearEffect);
+ struct RuntimeEffectShaderParameters {
+ sk_sp<SkShader> shader;
+ const LayerSettings& layer;
+ const DisplaySettings& display;
+ bool undoPremultipliedAlpha;
+ bool requiresLinearEffect;
+ float layerDimmingRatio;
+ };
+ sk_sp<SkShader> createRuntimeEffectShader(const RuntimeEffectShaderParameters&);
EGLDisplay mEGLDisplay;
EGLContext mEGLContext;
diff --git a/libs/renderengine/tests/RenderEngineTest.cpp b/libs/renderengine/tests/RenderEngineTest.cpp
index 8259063..eb2b2dc 100644
--- a/libs/renderengine/tests/RenderEngineTest.cpp
+++ b/libs/renderengine/tests/RenderEngineTest.cpp
@@ -208,6 +208,21 @@
renderengine::ExternalTexture::Usage::WRITEABLE);
}
+ std::shared_ptr<renderengine::ExternalTexture> allocateAndFillSourceBuffer(uint32_t width,
+ uint32_t height,
+ ubyte4 color) {
+ const auto buffer = allocateSourceBuffer(width, height);
+ uint8_t* pixels;
+ buffer->getBuffer()->lock(GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN,
+ reinterpret_cast<void**>(&pixels));
+ pixels[0] = color.r;
+ pixels[1] = color.g;
+ pixels[2] = color.b;
+ pixels[3] = color.a;
+ buffer->getBuffer()->unlock();
+ return buffer;
+ }
+
RenderEngineTest() {
const ::testing::TestInfo* const test_info =
::testing::UnitTest::GetInstance()->current_test_info();
@@ -2195,6 +2210,133 @@
expectBufferColor(rect, 0, 128, 0, 128);
}
+TEST_P(RenderEngineTest, testDimming) {
+ if (GetParam()->type() == renderengine::RenderEngine::RenderEngineType::GLES) {
+ return;
+ }
+ initializeRenderEngine();
+
+ const auto displayRect = Rect(3, 1);
+ const renderengine::DisplaySettings display{
+ .physicalDisplay = displayRect,
+ .clip = displayRect,
+ .outputDataspace = ui::Dataspace::V0_SRGB_LINEAR,
+ .targetLuminanceNits = 1000.f,
+ };
+
+ const auto greenBuffer = allocateAndFillSourceBuffer(1, 1, ubyte4(0, 255, 0, 255));
+ const auto blueBuffer = allocateAndFillSourceBuffer(1, 1, ubyte4(0, 0, 255, 255));
+ const auto redBuffer = allocateAndFillSourceBuffer(1, 1, ubyte4(255, 0, 0, 255));
+
+ const renderengine::LayerSettings greenLayer{
+ .geometry.boundaries = FloatRect(0.f, 0.f, 1.f, 1.f),
+ .source =
+ renderengine::PixelSource{
+ .buffer =
+ renderengine::Buffer{
+ .buffer = greenBuffer,
+ .usePremultipliedAlpha = true,
+ },
+ },
+ .alpha = 1.0f,
+ .sourceDataspace = ui::Dataspace::V0_SRGB_LINEAR,
+ .whitePointNits = 200.f,
+ };
+
+ const renderengine::LayerSettings blueLayer{
+ .geometry.boundaries = FloatRect(1.f, 0.f, 2.f, 1.f),
+ .source =
+ renderengine::PixelSource{
+ .buffer =
+ renderengine::Buffer{
+ .buffer = blueBuffer,
+ .usePremultipliedAlpha = true,
+ },
+ },
+ .alpha = 1.0f,
+ .sourceDataspace = ui::Dataspace::V0_SRGB_LINEAR,
+ .whitePointNits = 1000.f / 51.f,
+ };
+
+ const renderengine::LayerSettings redLayer{
+ .geometry.boundaries = FloatRect(2.f, 0.f, 3.f, 1.f),
+ .source =
+ renderengine::PixelSource{
+ .buffer =
+ renderengine::Buffer{
+ .buffer = redBuffer,
+ .usePremultipliedAlpha = true,
+ },
+ },
+ .alpha = 1.0f,
+ .sourceDataspace = ui::Dataspace::V0_SRGB_LINEAR,
+ // When the white point is not set for a layer, just ignore it and treat it as the same
+ // as the max layer
+ .whitePointNits = -1.f,
+ };
+
+ std::vector<renderengine::LayerSettings> layers{greenLayer, blueLayer, redLayer};
+ invokeDraw(display, layers);
+
+ expectBufferColor(Rect(1, 1), 0, 51, 0, 255, 1);
+ expectBufferColor(Rect(1, 0, 2, 1), 0, 0, 5, 255, 1);
+ expectBufferColor(Rect(2, 0, 3, 1), 51, 0, 0, 255, 1);
+}
+
+TEST_P(RenderEngineTest, testDimming_withoutTargetLuminance) {
+ initializeRenderEngine();
+ if (GetParam()->type() == renderengine::RenderEngine::RenderEngineType::GLES) {
+ return;
+ }
+
+ const auto displayRect = Rect(2, 1);
+ const renderengine::DisplaySettings display{
+ .physicalDisplay = displayRect,
+ .clip = displayRect,
+ .outputDataspace = ui::Dataspace::V0_SRGB_LINEAR,
+ .targetLuminanceNits = -1.f,
+ };
+
+ const auto greenBuffer = allocateAndFillSourceBuffer(1, 1, ubyte4(0, 255, 0, 255));
+ const auto blueBuffer = allocateAndFillSourceBuffer(1, 1, ubyte4(0, 0, 255, 255));
+
+ const renderengine::LayerSettings greenLayer{
+ .geometry.boundaries = FloatRect(0.f, 0.f, 1.f, 1.f),
+ .source =
+ renderengine::PixelSource{
+ .buffer =
+ renderengine::Buffer{
+ .buffer = greenBuffer,
+ .usePremultipliedAlpha = true,
+ },
+ },
+ .alpha = 1.0f,
+ .sourceDataspace = ui::Dataspace::V0_SRGB_LINEAR,
+ .whitePointNits = 200.f,
+ };
+
+ const renderengine::LayerSettings blueLayer{
+ .geometry.boundaries = FloatRect(1.f, 0.f, 2.f, 1.f),
+ .source =
+ renderengine::PixelSource{
+ .buffer =
+ renderengine::Buffer{
+ .buffer = blueBuffer,
+ .usePremultipliedAlpha = true,
+ },
+ },
+ .alpha = 1.0f,
+ .sourceDataspace = ui::Dataspace::V0_SRGB_LINEAR,
+ .whitePointNits = 1000.f,
+ };
+
+ std::vector<renderengine::LayerSettings> layers{greenLayer, blueLayer};
+ invokeDraw(display, layers);
+
+ expectBufferColor(Rect(1, 1), 0, 51, 0, 255, 1);
+ expectBufferColor(Rect(1, 0, 2, 1), 0, 0, 255, 255);
+}
+
TEST_P(RenderEngineTest, test_isOpaque) {
initializeRenderEngine();
diff --git a/libs/ui/include_types/ui/DataspaceUtils.h b/libs/ui/include_types/ui/DataspaceUtils.h
new file mode 100644
index 0000000..a461cb4
--- /dev/null
+++ b/libs/ui/include_types/ui/DataspaceUtils.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <ui/GraphicTypes.h>
+
+namespace android {
+
+inline bool isHdrDataspace(ui::Dataspace dataspace) {
+ const auto transfer = dataspace & HAL_DATASPACE_TRANSFER_MASK;
+
+ return transfer == HAL_DATASPACE_TRANSFER_ST2084 || transfer == HAL_DATASPACE_TRANSFER_HLG;
+}
+
+} // namespace android
\ No newline at end of file
diff --git a/libs/ui/tests/Android.bp b/libs/ui/tests/Android.bp
index 0ee15f2..22fbf45 100644
--- a/libs/ui/tests/Android.bp
+++ b/libs/ui/tests/Android.bp
@@ -163,3 +163,13 @@
"-Werror",
],
}
+
+cc_test {
+ name: "DataspaceUtils_test",
+ shared_libs: ["libui"],
+ srcs: ["DataspaceUtils_test.cpp"],
+ cflags: [
+ "-Wall",
+ "-Werror",
+ ],
+}
diff --git a/libs/ui/tests/DataspaceUtils_test.cpp b/libs/ui/tests/DataspaceUtils_test.cpp
new file mode 100644
index 0000000..3e09671
--- /dev/null
+++ b/libs/ui/tests/DataspaceUtils_test.cpp
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#undef LOG_TAG
+#define LOG_TAG "DataspaceUtilsTest"
+
+#include <gtest/gtest.h>
+#include <ui/DataspaceUtils.h>
+
+namespace android {
+
+class DataspaceUtilsTest : public testing::Test {};
+
+TEST_F(DataspaceUtilsTest, isHdrDataspace) {
+ EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_ITU_HLG));
+ EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_ITU_PQ));
+ EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_PQ));
+ EXPECT_TRUE(isHdrDataspace(ui::Dataspace::BT2020_HLG));
+
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SRGB_LINEAR));
+ // scRGB defines a very wide gamut but not an expanded luminance range
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SCRGB_LINEAR));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SRGB));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_SCRGB));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_JFIF));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_BT601_625));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_BT601_525));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::V0_BT709));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DCI_P3_LINEAR));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DCI_P3));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DISPLAY_P3_LINEAR));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DISPLAY_P3));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::ADOBE_RGB));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::BT2020_LINEAR));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::BT2020));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::BT2020_ITU));
+ EXPECT_FALSE(isHdrDataspace(ui::Dataspace::DISPLAY_BT2020));
+}
+
+} // namespace android