SF: Support xy scaling for rounded corners

Currently only a single corner radius can be applied on a
layer. If the layer's x and y scale do not match, the average
is used to scale the corner radius. This can cause visual
defects in the rendered rounded corners.

This defect can be seen when moving the Camera app
to recents. The camera buffer is submitted with a rotation
and the layer's x and y scales are used to stretch the buffer
to the desired size in portrait mode.

Bug: 145094543, 250491108
Test: atest librenderengine_test
Test: adb shell wm size 1000x1900 & animate camera app to recents
Change-Id: Ie76581eb200a57b847f619f4da0e61758f70dce7
(cherry picked from commit 50c0afe2457ec631fc3c9a7731aa4b9c8e18fd76)
Merged-In: Ie76581eb200a57b847f619f4da0e61758f70dce7
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index bcc94d3..aff94d1 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -406,7 +406,7 @@
     const auto& drawingState{getDrawingState()};
     const auto alpha = static_cast<float>(getAlpha());
     const bool opaque = isOpaque(drawingState);
-    const bool usesRoundedCorners = getRoundedCornerState().radius != 0.f;
+    const bool usesRoundedCorners = hasRoundedCorners();
 
     auto blendMode = Hwc2::IComposerClient::BlendMode::NONE;
     if (!opaque || alpha != 1.0f) {
@@ -482,7 +482,7 @@
     compositionState->hasProtectedContent = isProtected();
     compositionState->dimmingEnabled = isDimmingEnabled();
 
-    const bool usesRoundedCorners = getRoundedCornerState().radius != 0.f;
+    const bool usesRoundedCorners = hasRoundedCorners();
 
     compositionState->isOpaque =
             isOpaque(drawingState) && !usesRoundedCorners && getAlpha() == 1.0_hf;
@@ -1915,27 +1915,22 @@
     const auto& parent = mDrawingParent.promote();
     if (parent != nullptr) {
         parentSettings = parent->getRoundedCornerState();
-        if (parentSettings.radius > 0) {
+        if (parentSettings.hasRoundedCorners()) {
             ui::Transform t = getActiveTransform(getDrawingState());
             t = t.inverse();
             parentSettings.cropRect = t.transform(parentSettings.cropRect);
-            // The rounded corners shader only accepts 1 corner radius for performance reasons,
-            // but a transform matrix can define horizontal and vertical scales.
-            // Let's take the average between both of them and pass into the shader, practically we
-            // never do this type of transformation on windows anyway.
-            auto scaleX = sqrtf(t[0][0] * t[0][0] + t[0][1] * t[0][1]);
-            auto scaleY = sqrtf(t[1][0] * t[1][0] + t[1][1] * t[1][1]);
-            parentSettings.radius *= (scaleX + scaleY) / 2.0f;
+            parentSettings.radius.x *= t.getScaleX();
+            parentSettings.radius.y *= t.getScaleY();
         }
     }
 
     // Get layer settings
     Rect layerCropRect = getCroppedBufferSize(getDrawingState());
-    const float radius = getDrawingState().cornerRadius;
+    const vec2 radius(getDrawingState().cornerRadius, getDrawingState().cornerRadius);
     RoundedCornerState layerSettings(layerCropRect.toFloatRect(), radius);
-    const bool layerSettingsValid = layerSettings.radius > 0 && layerCropRect.isValid();
+    const bool layerSettingsValid = layerSettings.hasRoundedCorners() && layerCropRect.isValid();
 
-    if (layerSettingsValid && parentSettings.radius > 0) {
+    if (layerSettingsValid && parentSettings.hasRoundedCorners()) {
         // If the parent and the layer have rounded corner settings, use the parent settings if the
         // parent crop is entirely inside the layer crop.
         // This has limitations and cause rendering artifacts. See b/200300845 for correct fix.
@@ -1949,7 +1944,7 @@
         }
     } else if (layerSettingsValid) {
         return layerSettings;
-    } else if (parentSettings.radius > 0) {
+    } else if (parentSettings.hasRoundedCorners()) {
         return parentSettings;
     }
     return {};
@@ -2070,7 +2065,8 @@
     layerInfo->set_effective_scaling_mode(getEffectiveScalingMode());
 
     layerInfo->set_requested_corner_radius(getDrawingState().cornerRadius);
-    layerInfo->set_corner_radius(getRoundedCornerState().radius);
+    layerInfo->set_corner_radius(
+            (getRoundedCornerState().radius.x + getRoundedCornerState().radius.y) / 2.0);
     layerInfo->set_background_blur_radius(getBackgroundBlurRadius());
     layerInfo->set_is_trusted_overlay(isTrustedOverlay());
     LayerProtoHelper::writeToProtoDeprecated(transform, layerInfo->mutable_transform());
diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index c547da0..200baf0 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -134,13 +134,14 @@
 
     struct RoundedCornerState {
         RoundedCornerState() = default;
-        RoundedCornerState(FloatRect cropRect, float radius)
+        RoundedCornerState(const FloatRect& cropRect, const vec2& radius)
               : cropRect(cropRect), radius(radius) {}
 
         // Rounded rectangle in local layer coordinate space.
         FloatRect cropRect = FloatRect();
         // Radius of the rounded rectangle.
-        float radius = 0.0f;
+        vec2 radius;
+        bool hasRoundedCorners() const { return radius.x > 0.0f && radius.y > 0.0f; }
     };
 
     using FrameRate = scheduler::LayerInfo::FrameRate;
@@ -596,7 +597,7 @@
     // corner crop does not intersect with its own rounded corner crop.
     virtual RoundedCornerState getRoundedCornerState() const;
 
-    bool hasRoundedCorners() const override { return getRoundedCornerState().radius > .0f; }
+    bool hasRoundedCorners() const override { return getRoundedCornerState().hasRoundedCorners(); }
 
     virtual PixelFormat getPixelFormat() const { return PIXEL_FORMAT_NONE; }
     /**
diff --git a/services/surfaceflinger/tests/unittests/CompositionTest.cpp b/services/surfaceflinger/tests/unittests/CompositionTest.cpp
index bbfedc7..fbc532e 100644
--- a/services/surfaceflinger/tests/unittests/CompositionTest.cpp
+++ b/services/surfaceflinger/tests/unittests/CompositionTest.cpp
@@ -664,7 +664,8 @@
                     EXPECT_EQ(false, layer.source.buffer.isY410BT2020);
                     EXPECT_EQ(true, layer.source.buffer.usePremultipliedAlpha);
                     EXPECT_EQ(false, layer.source.buffer.isOpaque);
-                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.x);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.y);
                     EXPECT_EQ(ui::Dataspace::UNKNOWN, layer.sourceDataspace);
                     EXPECT_EQ(LayerProperties::COLOR[3], layer.alpha);
                     return resultFuture;
@@ -714,7 +715,8 @@
                     EXPECT_EQ(half3(LayerProperties::COLOR[0], LayerProperties::COLOR[1],
                                     LayerProperties::COLOR[2]),
                               layer.source.solidColor);
-                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.x);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.y);
                     EXPECT_EQ(ui::Dataspace::UNKNOWN, layer.sourceDataspace);
                     EXPECT_EQ(LayerProperties::COLOR[3], layer.alpha);
                     return resultFuture;
@@ -792,7 +794,8 @@
                     const renderengine::LayerSettings layer = layerSettings.back();
                     EXPECT_THAT(layer.source.buffer.buffer, IsNull());
                     EXPECT_EQ(half3(0.0f, 0.0f, 0.0f), layer.source.solidColor);
-                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.x);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.y);
                     EXPECT_EQ(ui::Dataspace::UNKNOWN, layer.sourceDataspace);
                     EXPECT_EQ(1.0f, layer.alpha);
                     return resultFuture;