Support floating point values for layer crop

Flag: EXEMPT bug fix
Fixes: 310950423
Test: presubmit
Change-Id: I05feb4881a95bc8caad90a3d632b3c7881909bf3
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFECompositionState.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFECompositionState.h
index d1429a2..14a8fd6 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFECompositionState.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFECompositionState.h
@@ -155,7 +155,7 @@
     uint32_t geomBufferTransform{0};
     Rect geomBufferSize;
     Rect geomContentCrop;
-    Rect geomCrop;
+    FloatRect geomCrop;
 
     GenericLayerMetadataMap metadata;
 
diff --git a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
index 091c207..8e5c991 100644
--- a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
@@ -24,6 +24,7 @@
 #include <compositionengine/impl/OutputLayerCompositionState.h>
 #include <cstdint>
 #include "system/graphics-base-v1.0.h"
+#include "ui/FloatRect.h"
 
 #include <ui/HdrRenderTypeUtils.h>
 
@@ -185,35 +186,35 @@
     const auto& layerState = *getLayerFE().getCompositionState();
     const auto& outputState = getOutput().getState();
 
+    // Convert from layer space to layerStackSpace
     // apply the layer's transform, followed by the display's global transform
     // here we're guaranteed that the layer's transform preserves rects
-    Region activeTransparentRegion = layerState.transparentRegionHint;
     const ui::Transform& layerTransform = layerState.geomLayerTransform;
-    const ui::Transform& inverseLayerTransform = layerState.geomInverseLayerTransform;
-    const Rect& bufferSize = layerState.geomBufferSize;
-    Rect activeCrop = layerState.geomCrop;
-    if (!activeCrop.isEmpty() && bufferSize.isValid()) {
-        activeCrop = layerTransform.transform(activeCrop);
-        if (!activeCrop.intersect(outputState.layerStackSpace.getContent(), &activeCrop)) {
-            activeCrop.clear();
-        }
-        activeCrop = inverseLayerTransform.transform(activeCrop, true);
-        // This needs to be here as transform.transform(Rect) computes the
-        // transformed rect and then takes the bounding box of the result before
-        // returning. This means
-        // transform.inverse().transform(transform.transform(Rect)) != Rect
-        // in which case we need to make sure the final rect is clipped to the
-        // display bounds.
-        if (!activeCrop.intersect(bufferSize, &activeCrop)) {
-            activeCrop.clear();
-        }
+    Region activeTransparentRegion = layerTransform.transform(layerState.transparentRegionHint);
+    if (!layerState.geomCrop.isEmpty() && layerState.geomBufferSize.isValid()) {
+        FloatRect activeCrop = layerTransform.transform(layerState.geomCrop);
+        activeCrop = activeCrop.intersect(outputState.layerStackSpace.getContent().toFloatRect());
+        const FloatRect& bufferSize =
+                layerTransform.transform(layerState.geomBufferSize.toFloatRect());
+        activeCrop = activeCrop.intersect(bufferSize);
+
         // mark regions outside the crop as transparent
-        activeTransparentRegion.orSelf(Rect(0, 0, bufferSize.getWidth(), activeCrop.top));
-        activeTransparentRegion.orSelf(
-                Rect(0, activeCrop.bottom, bufferSize.getWidth(), bufferSize.getHeight()));
-        activeTransparentRegion.orSelf(Rect(0, activeCrop.top, activeCrop.left, activeCrop.bottom));
-        activeTransparentRegion.orSelf(
-                Rect(activeCrop.right, activeCrop.top, bufferSize.getWidth(), activeCrop.bottom));
+        Rect topRegion = Rect(layerTransform.transform(
+                FloatRect(0, 0, layerState.geomBufferSize.getWidth(), layerState.geomCrop.top)));
+        Rect bottomRegion = Rect(layerTransform.transform(
+                FloatRect(0, layerState.geomCrop.bottom, layerState.geomBufferSize.getWidth(),
+                          layerState.geomBufferSize.getHeight())));
+        Rect leftRegion = Rect(layerTransform.transform(FloatRect(0, layerState.geomCrop.top,
+                                                                 layerState.geomCrop.left,
+                                                                 layerState.geomCrop.bottom)));
+        Rect rightRegion = Rect(layerTransform.transform(
+                FloatRect(layerState.geomCrop.right, layerState.geomCrop.top,
+                          layerState.geomBufferSize.getWidth(), layerState.geomCrop.bottom)));
+
+        activeTransparentRegion.orSelf(topRegion);
+        activeTransparentRegion.orSelf(bottomRegion);
+        activeTransparentRegion.orSelf(leftRegion);
+        activeTransparentRegion.orSelf(rightRegion);
     }
 
     // reduce uses a FloatRect to provide more accuracy during the
@@ -229,13 +230,14 @@
         geomLayerBounds.right += outset;
         geomLayerBounds.bottom += outset;
     }
-    Rect frame{layerTransform.transform(reduce(geomLayerBounds, activeTransparentRegion))};
-    if (!frame.intersect(outputState.layerStackSpace.getContent(), &frame)) {
-        frame.clear();
-    }
-    const ui::Transform displayTransform{outputState.transform};
 
-    return displayTransform.transform(frame);
+    geomLayerBounds = layerTransform.transform(geomLayerBounds);
+    FloatRect frame = reduce(geomLayerBounds, activeTransparentRegion);
+    frame = frame.intersect(outputState.layerStackSpace.getContent().toFloatRect());
+
+    // convert from layerStackSpace to displaySpace
+    const ui::Transform displayTransform{outputState.transform};
+    return Rect(displayTransform.transform(frame));
 }
 
 uint32_t OutputLayer::calculateOutputRelativeBufferTransform(
diff --git a/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp b/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp
index 1c54469..95cdcd5 100644
--- a/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp
+++ b/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp
@@ -30,6 +30,7 @@
 #include "MockHWC2.h"
 #include "MockHWComposer.h"
 #include "RegionMatcher.h"
+#include "ui/FloatRect.h"
 
 #include <aidl/android/hardware/graphics/composer3/Composition.h>
 
@@ -270,7 +271,7 @@
         mLayerFEState.geomLayerTransform = ui::Transform{TR_IDENT};
         mLayerFEState.geomBufferSize = Rect{0, 0, 1920, 1080};
         mLayerFEState.geomBufferUsesDisplayInverseTransform = false;
-        mLayerFEState.geomCrop = Rect{0, 0, 1920, 1080};
+        mLayerFEState.geomCrop = FloatRect{0, 0, 1920, 1080};
         mLayerFEState.geomLayerBounds = FloatRect{0.f, 0.f, 1920.f, 1080.f};
 
         mOutputState.layerStackSpace.setContent(Rect{0, 0, 1920, 1080});
@@ -296,20 +297,20 @@
 }
 
 TEST_F(OutputLayerDisplayFrameTest, cropAffectsDisplayFrame) {
-    mLayerFEState.geomCrop = Rect{100, 200, 300, 500};
+    mLayerFEState.geomCrop = FloatRect{100, 200, 300, 500};
     const Rect expected{100, 200, 300, 500};
     EXPECT_THAT(calculateOutputDisplayFrame(), expected);
 }
 
 TEST_F(OutputLayerDisplayFrameTest, cropAffectsDisplayFrameRotated) {
-    mLayerFEState.geomCrop = Rect{100, 200, 300, 500};
+    mLayerFEState.geomCrop = FloatRect{100, 200, 300, 500};
     mLayerFEState.geomLayerTransform.set(HAL_TRANSFORM_ROT_90, 1920, 1080);
     const Rect expected{1420, 100, 1720, 300};
     EXPECT_THAT(calculateOutputDisplayFrame(), expected);
 }
 
 TEST_F(OutputLayerDisplayFrameTest, emptyGeomCropIsNotUsedToComputeFrame) {
-    mLayerFEState.geomCrop = Rect{};
+    mLayerFEState.geomCrop = FloatRect{};
     const Rect expected{0, 0, 1920, 1080};
     EXPECT_THAT(calculateOutputDisplayFrame(), expected);
 }