Refactor `isHdrDataspace` function.

- Rename it to `getHdrRenderType` and return a ternary enum.
- return the hdr type that we want to treat based on the dataspace,
  format and hdr/sdr ratio.
- pixelformat is optional, in case no source buffer but there is a
  source color.
- hdr/sdr ratio is 1.0f by default, render rengine doesn't take care
  this param.
- The ternary enum has 3 types: just SDR; generic hdr, namely those we
  need to tonemap; display hdr, namely those self-promoting to HDR by
  using extended brightness API.

Bug: 261485283
Test: HdrRenderTypeUtils_test, TextureViewTest#testSDRFromSurfaceViewAndTextureView, OutputLayerUpdateCompositionStateTest

Change-Id: I281687a010bbf5bff555f6fa893002c2a9b324d1
diff --git a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
index b492b6a..4fe6927 100644
--- a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
@@ -13,7 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 #include <DisplayHardware/Hal.h>
 #include <android-base/stringprintf.h>
 #include <compositionengine/DisplayColorProfile.h>
@@ -26,7 +25,7 @@
 #include <cstdint>
 #include "system/graphics-base-v1.0.h"
 
-#include <ui/DataspaceUtils.h>
+#include <ui/HdrRenderTypeUtils.h>
 
 // TODO(b/129481165): remove the #pragma below and fix conversion issues
 #pragma clang diagnostic push
@@ -312,15 +311,26 @@
         }
     }
 
+    auto pixelFormat = layerFEState->buffer ? std::make_optional(static_cast<ui::PixelFormat>(
+                                                      layerFEState->buffer->getPixelFormat()))
+                                            : std::nullopt;
+
+    auto hdrRenderType =
+            getHdrRenderType(outputState.dataspace, pixelFormat, layerFEState->desiredHdrSdrRatio);
+
     // Determine the output dependent dataspace for this layer. If it is
     // colorspace agnostic, it just uses the dataspace chosen for the output to
     // avoid the need for color conversion.
     // For now, also respect the colorspace agnostic flag if we're drawing to HDR, to avoid drastic
     // luminance shift. TODO(b/292162273): we should check if that's true though.
-    state.dataspace = layerFEState->isColorspaceAgnostic && !isHdrDataspace(outputState.dataspace)
+    state.dataspace = layerFEState->isColorspaceAgnostic && hdrRenderType == HdrRenderType::SDR
             ? outputState.dataspace
             : layerFEState->dataspace;
 
+    // re-get HdrRenderType after the dataspace gets changed.
+    hdrRenderType =
+            getHdrRenderType(state.dataspace, pixelFormat, layerFEState->desiredHdrSdrRatio);
+
     // Override the dataspace transfer from 170M to sRGB if the device configuration requests this.
     // We do this here instead of in buffer info so that dumpsys can still report layers that are
     // using the 170M transfer. Also we only do this if the colorspace is not agnostic for the
@@ -335,7 +345,7 @@
     // For hdr content, treat the white point as the display brightness - HDR content should not be
     // boosted or dimmed.
     // If the layer explicitly requests to disable dimming, then don't dim either.
-    if (isHdrDataspace(state.dataspace) ||
+    if (hdrRenderType == HdrRenderType::GENERIC_HDR ||
         getOutput().getState().displayBrightnessNits == getOutput().getState().sdrWhitePointNits ||
         getOutput().getState().displayBrightnessNits == 0.f || !layerFEState->dimmingEnabled) {
         state.dimmingRatio = 1.f;
@@ -344,8 +354,7 @@
         float layerBrightnessNits = getOutput().getState().sdrWhitePointNits;
         // RANGE_EXTENDED can "self-promote" to HDR, but is still rendered for a particular
         // range that we may need to re-adjust to the current display conditions
-        if ((state.dataspace & HAL_DATASPACE_RANGE_MASK) == HAL_DATASPACE_RANGE_EXTENDED &&
-            layerFEState->currentHdrSdrRatio > 1.01f) {
+        if (hdrRenderType == HdrRenderType::DISPLAY_HDR) {
             layerBrightnessNits *= layerFEState->currentHdrSdrRatio;
         }
         state.dimmingRatio =
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 1a0517a..38a36fc 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -50,10 +50,10 @@
 #include <stdlib.h>
 #include <sys/types.h>
 #include <system/graphics-base-v1.0.h>
-#include <ui/DataspaceUtils.h>
 #include <ui/DebugUtils.h>
 #include <ui/FloatRect.h>
 #include <ui/GraphicBuffer.h>
+#include <ui/HdrRenderTypeUtils.h>
 #include <ui/PixelFormat.h>
 #include <ui/Rect.h>
 #include <ui/Transform.h>
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index ef483f6..13a457e 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -81,7 +81,6 @@
 #include <scheduler/FrameTargeter.h>
 #include <sys/types.h>
 #include <ui/ColorSpace.h>
-#include <ui/DataspaceUtils.h>
 #include <ui/DebugUtils.h>
 #include <ui/DisplayId.h>
 #include <ui/DisplayMode.h>
@@ -89,6 +88,7 @@
 #include <ui/DisplayState.h>
 #include <ui/DynamicDisplayInfo.h>
 #include <ui/GraphicBufferAllocator.h>
+#include <ui/HdrRenderTypeUtils.h>
 #include <ui/LayerStack.h>
 #include <ui/PixelFormat.h>
 #include <ui/StaticDisplayInfo.h>
@@ -2796,7 +2796,14 @@
             return false;
         }
     }
-    if (isHdrDataspace(snapshot.dataspace)) {
+    // RANGE_EXTENDED layer may identify themselves as being "HDR"
+    // via a desired hdr/sdr ratio
+    auto pixelFormat = snapshot.buffer
+            ? std::make_optional(static_cast<ui::PixelFormat>(snapshot.buffer->getPixelFormat()))
+            : std::nullopt;
+
+    if (getHdrRenderType(snapshot.dataspace, pixelFormat, snapshot.desiredHdrSdrRatio) !=
+        HdrRenderType::SDR) {
         return true;
     }
     // If the layer is not allowed to be dimmed, treat it as HDR. WindowManager may disable
@@ -2806,12 +2813,6 @@
     if (!snapshot.dimmingEnabled) {
         return true;
     }
-    // RANGE_EXTENDED layers may identify themselves as being "HDR" via a desired sdr/hdr ratio
-    if ((snapshot.dataspace & (int32_t)Dataspace::RANGE_MASK) ==
-                (int32_t)Dataspace::RANGE_EXTENDED &&
-        snapshot.desiredHdrSdrRatio > 1.01f) {
-        return true;
-    }
     return false;
 }