Provide a fixed transform hint if the layer is in a fixed orientation 1/2

The transform hint is used to prevent allocating a buffer of a
different size when a layer is rotated. The producer can choose to
consume the hint and allocate the buffer with the same size.

Provide the graphic producer a transform hint if the layer and its
children are in an orientation different from the display's
orientation. The caller is responsible for clearing this transform
hint if the layer is no longer in a fixed orientation.

Bug: 152919661
Test: atest VulkanPreTransformTest
Test: confirm with winscope trace, buffers are allocated taking into
account the transform hint in fixed orientation scenarios
Test: go/wm-smoke

Change-Id: Iea9dcf909921802a5be5c44dd61be3274f36bbd8
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 752407a..25929ed1 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -117,6 +117,7 @@
     mCurrentState.metadata = args.metadata;
     mCurrentState.shadowRadius = 0.f;
     mCurrentState.treeHasFrameRateVote = false;
+    mCurrentState.fixedTransformHint = ui::Transform::ROT_INVALID;
 
     // drawing state & current state are identical
     mDrawingState = mCurrentState;
@@ -1333,6 +1334,18 @@
     return true;
 }
 
+bool Layer::setFixedTransformHint(ui::Transform::RotationFlags fixedTransformHint) {
+    if (mCurrentState.fixedTransformHint == fixedTransformHint) {
+        return false;
+    }
+
+    mCurrentState.sequence++;
+    mCurrentState.fixedTransformHint = fixedTransformHint;
+    mCurrentState.modified = true;
+    setTransactionFlags(eTransactionNeeded);
+    return true;
+}
+
 void Layer::updateTreeHasFrameRateVote() {
     const auto traverseTree = [&](const LayerVector::Visitor& visitor) {
         auto parent = getParent();
@@ -1460,19 +1473,19 @@
 }
 
 void Layer::updateTransformHint(const sp<const DisplayDevice>& display) const {
-    uint32_t orientation = 0;
+    ui::Transform::RotationFlags transformHint = ui::Transform::ROT_0;
     // Disable setting transform hint if the debug flag is set.
     if (!mFlinger->mDebugDisableTransformHint) {
         // The transform hint is used to improve performance, but we can
         // only have a single transform hint, it cannot
         // apply to all displays.
         const ui::Transform& planeTransform = display->getTransform();
-        orientation = planeTransform.getOrientation();
-        if (orientation & ui::Transform::ROT_INVALID) {
-            orientation = 0;
+        transformHint = static_cast<ui::Transform::RotationFlags>(planeTransform.getOrientation());
+        if (transformHint & ui::Transform::ROT_INVALID) {
+            transformHint = ui::Transform::ROT_0;
         }
     }
-    setTransformHint(orientation);
+    setTransformHint(transformHint);
 }
 
 // ----------------------------------------------------------------------------
@@ -2076,6 +2089,16 @@
     return parentAlpha * getDrawingState().color.a;
 }
 
+ui::Transform::RotationFlags Layer::getFixedTransformHint() const {
+    ui::Transform::RotationFlags fixedTransformHint = mCurrentState.fixedTransformHint;
+    if (fixedTransformHint != ui::Transform::ROT_INVALID) {
+        return fixedTransformHint;
+    }
+    const auto& p = mCurrentParent.promote();
+    if (!p) return fixedTransformHint;
+    return p->getFixedTransformHint();
+}
+
 half4 Layer::getColor() const {
     const half4 color(getDrawingState().color);
     return half4(color.r, color.g, color.b, getAlpha());