SurfaceFlinger: debug layer bounds inset

This is a debug change to make sure layer bounds will not overflow after
an inset operation.

Bug: 137560795
Test: manual tests
Change-Id: Ifd9a8d84877e7f4c1f62c0419b0f86294ab576af
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 1318bc0..d6e86eb 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -2037,6 +2037,13 @@
     return mRemovedFromCurrentState;
 }
 
+// Debug helper for b/137560795
+#define INT32_MIGHT_OVERFLOW(n) (((n) >= INT32_MAX / 2) || ((n) <= INT32_MIN / 2))
+
+#define RECT_BOUNDS_INVALID(rect)                                               \
+    (INT32_MIGHT_OVERFLOW((rect).left) || INT32_MIGHT_OVERFLOW((rect).right) || \
+     INT32_MIGHT_OVERFLOW((rect).bottom) || INT32_MIGHT_OVERFLOW((rect).top))
+
 InputWindowInfo Layer::fillInputInfo() {
     InputWindowInfo info = mDrawingState.inputInfo;
 
@@ -2066,6 +2073,26 @@
         layerBounds = getCroppedBufferSize(getDrawingState());
     }
     layerBounds = t.transform(layerBounds);
+
+    // debug check for b/137560795
+    {
+        if (RECT_BOUNDS_INVALID(layerBounds)) {
+            ALOGE("layer %s bounds are invalid (%" PRIi32 ", %" PRIi32 ", %" PRIi32 ", %" PRIi32
+                  ")",
+                  mName.c_str(), layerBounds.left, layerBounds.top, layerBounds.right,
+                  layerBounds.bottom);
+            std::string out;
+            getTransform().dump(out, "Transform");
+            ALOGE("%s", out.c_str());
+            layerBounds.left = layerBounds.top = layerBounds.right = layerBounds.bottom = 0;
+        }
+
+        if (INT32_MIGHT_OVERFLOW(xSurfaceInset) || INT32_MIGHT_OVERFLOW(ySurfaceInset)) {
+            ALOGE("layer %s surface inset are invalid (%" PRIi32 ", %" PRIi32 ")", mName.c_str(),
+                  int32_t(xSurfaceInset), int32_t(ySurfaceInset));
+            xSurfaceInset = ySurfaceInset = 0;
+        }
+    }
     layerBounds.inset(xSurfaceInset, ySurfaceInset, xSurfaceInset, ySurfaceInset);
 
     // Input coordinate should match the layer bounds.