commit 2c1782c6f986debe5ec89d5cdd3a3f08b08d5683
author Garfield Tan <xutan@google.com> Wed Feb 16 15:25:05 2022 -0800
committer Garfield Tan <xutan@google.com> Wed Mar 02 10:28:06 2022 -0800
tree ddcafca867d51d41dabdad9722ab56742a3b5611
parent 8798a4849f761b74221725c408a54f2f55307459

Hide layers that have irreversible transforms

Surface API doesn't prohibit putting interesting values in transforms.
It's especially possible to scale something to 0 at the begining or the
end of an animation, but we should guard SF against ill-intentioned
apps.

Bug: 210837722
Test: KeyguardTests#testDismissKeyguardActivity doesn't crash SF on CF
PC and the warning log is in logcat.

Change-Id: Idaa1cdb0676102a580c4478c860327796c8213f3

services/surfaceflinger/Layer.cpp

diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 533acfd..4de8dc2 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -338,6 +338,12 @@
     // Calculate effective layer transform
     mEffectiveTransform = parentTransform * getActiveTransform(s);
 
+    if (CC_UNLIKELY(!isTransformValid())) {
+        ALOGW("Stop computing bounds for %s because it has invalid transformation.",
+              getDebugName());
+        return;
+    }
+
     // Transform parent bounds to layer space
     parentBounds = getActiveTransform(s).inverse().transform(parentBounds);
 
@@ -1326,6 +1332,10 @@
             }
         }
     }
+    if (CC_UNLIKELY(!isTransformValid())) {
+        ALOGW("Hide layer %s because it has invalid transformation.", getDebugName());
+        return true;
+    }
     return s.flags & layer_state_t::eLayerHidden;
 }
 
@@ -1858,6 +1868,11 @@
     return mEffectiveTransform;
 }
 
+bool Layer::isTransformValid() const {
+    float transformDet = getTransform().det();
+    return transformDet != 0 && !isinf(transformDet) && !isnan(transformDet);
+}
+
 half Layer::getAlpha() const {
     const auto& p = mDrawingParent.promote();
 

services/surfaceflinger/Layer.h

diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index 21dd5f4..2b5e337 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -752,6 +752,7 @@
                                   FrameEventHistoryDelta* outDelta);
 
     ui::Transform getTransform() const;
+    bool isTransformValid() const;
 
     // Returns the Alpha of the Surface, accounting for the Alpha
     // of parent Surfaces in the hierarchy (alpha's will be multiplied