commit 2c1782c6f986debe5ec89d5cdd3a3f08b08d5683
author Garfield Tan <xutan@google.com> Wed Feb 16 15:25:05 2022 -0800
committer Garfield Tan <xutan@google.com> Wed Mar 02 10:28:06 2022 -0800
tree ddcafca867d51d41dabdad9722ab56742a3b5611
parent 8798a4849f761b74221725c408a54f2f55307459

Hide layers that have irreversible transforms

Surface API doesn't prohibit putting interesting values in transforms.
It's especially possible to scale something to 0 at the begining or the
end of an animation, but we should guard SF against ill-intentioned
apps.

Bug: 210837722
Test: KeyguardTests#testDismissKeyguardActivity doesn't crash SF on CF
PC and the warning log is in logcat.

Change-Id: Idaa1cdb0676102a580c4478c860327796c8213f3

libs/ui/Transform.cpp

diff --git a/libs/ui/Transform.cpp b/libs/ui/Transform.cpp
index b34d906..42dd85e 100644
--- a/libs/ui/Transform.cpp
+++ b/libs/ui/Transform.cpp
@@ -134,6 +134,10 @@
     return mMatrix[1][1];
 }
 
+float Transform::det() const {
+    return mMatrix[0][0] * mMatrix[1][1] - mMatrix[0][1] * mMatrix[1][0];
+}
+
 float Transform::getScaleX() const {
     return sqrt((dsdx() * dsdx()) + (dtdx() * dtdx()));
 }
@@ -390,7 +394,7 @@
         const float x = M[2][0];
         const float y = M[2][1];
 
-        const float idet = 1.0f / (a*d - b*c);
+        const float idet = 1.0f / det();
         result.mMatrix[0][0] =  d*idet;
         result.mMatrix[0][1] = -c*idet;
         result.mMatrix[1][0] = -b*idet;

libs/ui/include/ui/Transform.h

diff --git a/libs/ui/include/ui/Transform.h b/libs/ui/include/ui/Transform.h
index 33fbe05..f1178ca 100644
--- a/libs/ui/include/ui/Transform.h
+++ b/libs/ui/include/ui/Transform.h
@@ -77,6 +77,7 @@
     float dtdx() const;
     float dtdy() const;
     float dsdy() const;
+    float det() const;
 
     float getScaleX() const;
     float getScaleY() const;

services/surfaceflinger/Layer.cpp

diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 533acfd..4de8dc2 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -338,6 +338,12 @@
     // Calculate effective layer transform
     mEffectiveTransform = parentTransform * getActiveTransform(s);
 
+    if (CC_UNLIKELY(!isTransformValid())) {
+        ALOGW("Stop computing bounds for %s because it has invalid transformation.",
+              getDebugName());
+        return;
+    }
+
     // Transform parent bounds to layer space
     parentBounds = getActiveTransform(s).inverse().transform(parentBounds);
 
@@ -1326,6 +1332,10 @@
             }
         }
     }
+    if (CC_UNLIKELY(!isTransformValid())) {
+        ALOGW("Hide layer %s because it has invalid transformation.", getDebugName());
+        return true;
+    }
     return s.flags & layer_state_t::eLayerHidden;
 }
 
@@ -1858,6 +1868,11 @@
     return mEffectiveTransform;
 }
 
+bool Layer::isTransformValid() const {
+    float transformDet = getTransform().det();
+    return transformDet != 0 && !isinf(transformDet) && !isnan(transformDet);
+}
+
 half Layer::getAlpha() const {
     const auto& p = mDrawingParent.promote();
 

services/surfaceflinger/Layer.h

diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index 21dd5f4..2b5e337 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -752,6 +752,7 @@
                                   FrameEventHistoryDelta* outDelta);
 
     ui::Transform getTransform() const;
+    bool isTransformValid() const;
 
     // Returns the Alpha of the Surface, accounting for the Alpha
     // of parent Surfaces in the hierarchy (alpha's will be multiplied