Refactor `isHdrDataspace` function.

- Rename it to `getHdrRenderType` and return a ternary enum.
- return the hdr type that we want to treat based on the dataspace,
  format and hdr/sdr ratio.
- pixelformat is optional, in case no source buffer but there is a
  source color.
- hdr/sdr ratio is 1.0f by default, render rengine doesn't take care
  this param.
- The ternary enum has 3 types: just SDR; generic hdr, namely those we
  need to tonemap; display hdr, namely those self-promoting to HDR by
  using extended brightness API.

Bug: 261485283
Test: HdrRenderTypeUtils_test, TextureViewTest#testSDRFromSurfaceViewAndTextureView, OutputLayerUpdateCompositionStateTest

Change-Id: I281687a010bbf5bff555f6fa893002c2a9b324d1
diff --git a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
index b492b6a..4fe6927 100644
--- a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
@@ -13,7 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 #include <DisplayHardware/Hal.h>
 #include <android-base/stringprintf.h>
 #include <compositionengine/DisplayColorProfile.h>
@@ -26,7 +25,7 @@
 #include <cstdint>
 #include "system/graphics-base-v1.0.h"
 
-#include <ui/DataspaceUtils.h>
+#include <ui/HdrRenderTypeUtils.h>
 
 // TODO(b/129481165): remove the #pragma below and fix conversion issues
 #pragma clang diagnostic push
@@ -312,15 +311,26 @@
         }
     }
 
+    auto pixelFormat = layerFEState->buffer ? std::make_optional(static_cast<ui::PixelFormat>(
+                                                      layerFEState->buffer->getPixelFormat()))
+                                            : std::nullopt;
+
+    auto hdrRenderType =
+            getHdrRenderType(outputState.dataspace, pixelFormat, layerFEState->desiredHdrSdrRatio);
+
     // Determine the output dependent dataspace for this layer. If it is
     // colorspace agnostic, it just uses the dataspace chosen for the output to
     // avoid the need for color conversion.
     // For now, also respect the colorspace agnostic flag if we're drawing to HDR, to avoid drastic
     // luminance shift. TODO(b/292162273): we should check if that's true though.
-    state.dataspace = layerFEState->isColorspaceAgnostic && !isHdrDataspace(outputState.dataspace)
+    state.dataspace = layerFEState->isColorspaceAgnostic && hdrRenderType == HdrRenderType::SDR
             ? outputState.dataspace
             : layerFEState->dataspace;
 
+    // re-get HdrRenderType after the dataspace gets changed.
+    hdrRenderType =
+            getHdrRenderType(state.dataspace, pixelFormat, layerFEState->desiredHdrSdrRatio);
+
     // Override the dataspace transfer from 170M to sRGB if the device configuration requests this.
     // We do this here instead of in buffer info so that dumpsys can still report layers that are
     // using the 170M transfer. Also we only do this if the colorspace is not agnostic for the
@@ -335,7 +345,7 @@
     // For hdr content, treat the white point as the display brightness - HDR content should not be
     // boosted or dimmed.
     // If the layer explicitly requests to disable dimming, then don't dim either.
-    if (isHdrDataspace(state.dataspace) ||
+    if (hdrRenderType == HdrRenderType::GENERIC_HDR ||
         getOutput().getState().displayBrightnessNits == getOutput().getState().sdrWhitePointNits ||
         getOutput().getState().displayBrightnessNits == 0.f || !layerFEState->dimmingEnabled) {
         state.dimmingRatio = 1.f;
@@ -344,8 +354,7 @@
         float layerBrightnessNits = getOutput().getState().sdrWhitePointNits;
         // RANGE_EXTENDED can "self-promote" to HDR, but is still rendered for a particular
         // range that we may need to re-adjust to the current display conditions
-        if ((state.dataspace & HAL_DATASPACE_RANGE_MASK) == HAL_DATASPACE_RANGE_EXTENDED &&
-            layerFEState->currentHdrSdrRatio > 1.01f) {
+        if (hdrRenderType == HdrRenderType::DISPLAY_HDR) {
             layerBrightnessNits *= layerFEState->currentHdrSdrRatio;
         }
         state.dimmingRatio =