SF: Support xy scaling for rounded corners

Currently only a single corner radius can be applied on a
layer. If the layer's x and y scale do not match, the average
is used to scale the corner radius. This can cause visual
defects in the rendered rounded corners.

This defect can be seen when moving the Camera app
to recents. The camera buffer is submitted with a rotation
and the layer's x and y scales are used to stretch the buffer
to the desired size in portrait mode.

Bug: 145094543
Test: atest librenderengine_test
Test: adb shell wm size 1000x1900 & animate camera app to recents
Change-Id: Ie76581eb200a57b847f619f4da0e61758f70dce7
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 80df399..cd77465 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -409,7 +409,7 @@
     const auto& drawingState{getDrawingState()};
     const auto alpha = static_cast<float>(getAlpha());
     const bool opaque = isOpaque(drawingState);
-    const bool usesRoundedCorners = getRoundedCornerState().radius != 0.f;
+    const bool usesRoundedCorners = hasRoundedCorners();
 
     auto blendMode = Hwc2::IComposerClient::BlendMode::NONE;
     if (!opaque || alpha != 1.0f) {
@@ -485,7 +485,7 @@
     compositionState->hasProtectedContent = isProtected();
     compositionState->dimmingEnabled = isDimmingEnabled();
 
-    const bool usesRoundedCorners = getRoundedCornerState().radius != 0.f;
+    const bool usesRoundedCorners = hasRoundedCorners();
 
     compositionState->isOpaque =
             isOpaque(drawingState) && !usesRoundedCorners && getAlpha() == 1.0_hf;
@@ -1954,27 +1954,22 @@
     const auto& parent = mDrawingParent.promote();
     if (parent != nullptr) {
         parentSettings = parent->getRoundedCornerState();
-        if (parentSettings.radius > 0) {
+        if (parentSettings.hasRoundedCorners()) {
             ui::Transform t = getActiveTransform(getDrawingState());
             t = t.inverse();
             parentSettings.cropRect = t.transform(parentSettings.cropRect);
-            // The rounded corners shader only accepts 1 corner radius for performance reasons,
-            // but a transform matrix can define horizontal and vertical scales.
-            // Let's take the average between both of them and pass into the shader, practically we
-            // never do this type of transformation on windows anyway.
-            auto scaleX = sqrtf(t[0][0] * t[0][0] + t[0][1] * t[0][1]);
-            auto scaleY = sqrtf(t[1][0] * t[1][0] + t[1][1] * t[1][1]);
-            parentSettings.radius *= (scaleX + scaleY) / 2.0f;
+            parentSettings.radius.x *= t.getScaleX();
+            parentSettings.radius.y *= t.getScaleY();
         }
     }
 
     // Get layer settings
     Rect layerCropRect = getCroppedBufferSize(getDrawingState());
-    const float radius = getDrawingState().cornerRadius;
+    const vec2 radius(getDrawingState().cornerRadius, getDrawingState().cornerRadius);
     RoundedCornerState layerSettings(layerCropRect.toFloatRect(), radius);
-    const bool layerSettingsValid = layerSettings.radius > 0 && layerCropRect.isValid();
+    const bool layerSettingsValid = layerSettings.hasRoundedCorners() && layerCropRect.isValid();
 
-    if (layerSettingsValid && parentSettings.radius > 0) {
+    if (layerSettingsValid && parentSettings.hasRoundedCorners()) {
         // If the parent and the layer have rounded corner settings, use the parent settings if the
         // parent crop is entirely inside the layer crop.
         // This has limitations and cause rendering artifacts. See b/200300845 for correct fix.
@@ -1988,7 +1983,7 @@
         }
     } else if (layerSettingsValid) {
         return layerSettings;
-    } else if (parentSettings.radius > 0) {
+    } else if (parentSettings.hasRoundedCorners()) {
         return parentSettings;
     }
     return {};
@@ -2109,7 +2104,8 @@
     layerInfo->set_effective_scaling_mode(getEffectiveScalingMode());
 
     layerInfo->set_requested_corner_radius(getDrawingState().cornerRadius);
-    layerInfo->set_corner_radius(getRoundedCornerState().radius);
+    layerInfo->set_corner_radius(
+            (getRoundedCornerState().radius.x + getRoundedCornerState().radius.y) / 2.0);
     layerInfo->set_background_blur_radius(getBackgroundBlurRadius());
     layerInfo->set_is_trusted_overlay(isTrustedOverlay());
     LayerProtoHelper::writeToProtoDeprecated(transform, layerInfo->mutable_transform());
diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index 911ab78..f1921df 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -137,13 +137,14 @@
 
     struct RoundedCornerState {
         RoundedCornerState() = default;
-        RoundedCornerState(FloatRect cropRect, float radius)
+        RoundedCornerState(const FloatRect& cropRect, const vec2& radius)
               : cropRect(cropRect), radius(radius) {}
 
         // Rounded rectangle in local layer coordinate space.
         FloatRect cropRect = FloatRect();
         // Radius of the rounded rectangle.
-        float radius = 0.0f;
+        vec2 radius;
+        bool hasRoundedCorners() const { return radius.x > 0.0f && radius.y > 0.0f; }
     };
 
     using FrameRate = scheduler::LayerInfo::FrameRate;
@@ -599,7 +600,7 @@
     // corner crop does not intersect with its own rounded corner crop.
     virtual RoundedCornerState getRoundedCornerState() const;
 
-    bool hasRoundedCorners() const override { return getRoundedCornerState().radius > .0f; }
+    bool hasRoundedCorners() const override { return getRoundedCornerState().hasRoundedCorners(); }
 
     virtual PixelFormat getPixelFormat() const { return PIXEL_FORMAT_NONE; }
     /**
diff --git a/services/surfaceflinger/tests/unittests/CompositionTest.cpp b/services/surfaceflinger/tests/unittests/CompositionTest.cpp
index 19eaa19..f6ebfe5 100644
--- a/services/surfaceflinger/tests/unittests/CompositionTest.cpp
+++ b/services/surfaceflinger/tests/unittests/CompositionTest.cpp
@@ -629,7 +629,8 @@
                     EXPECT_EQ(false, layer.source.buffer.isY410BT2020);
                     EXPECT_EQ(true, layer.source.buffer.usePremultipliedAlpha);
                     EXPECT_EQ(false, layer.source.buffer.isOpaque);
-                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.x);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.y);
                     EXPECT_EQ(ui::Dataspace::UNKNOWN, layer.sourceDataspace);
                     EXPECT_EQ(LayerProperties::COLOR[3], layer.alpha);
                     return resultFuture;
@@ -679,7 +680,8 @@
                     EXPECT_EQ(half3(LayerProperties::COLOR[0], LayerProperties::COLOR[1],
                                     LayerProperties::COLOR[2]),
                               layer.source.solidColor);
-                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.x);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.y);
                     EXPECT_EQ(ui::Dataspace::UNKNOWN, layer.sourceDataspace);
                     EXPECT_EQ(LayerProperties::COLOR[3], layer.alpha);
                     return resultFuture;
@@ -757,7 +759,8 @@
                     const renderengine::LayerSettings layer = layerSettings.back();
                     EXPECT_THAT(layer.source.buffer.buffer, IsNull());
                     EXPECT_EQ(half3(0.0f, 0.0f, 0.0f), layer.source.solidColor);
-                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.x);
+                    EXPECT_EQ(0.0, layer.geometry.roundedCornersRadius.y);
                     EXPECT_EQ(ui::Dataspace::UNKNOWN, layer.sourceDataspace);
                     EXPECT_EQ(1.0f, layer.alpha);
                     return resultFuture;