Provide a fixed transform hint if the layer is in a fixed orientation 1/2

The transform hint is used to prevent allocating a buffer of a
different size when a layer is rotated. The producer can choose to
consume the hint and allocate the buffer with the same size.

Provide the graphic producer a transform hint if the layer and its
children are in an orientation different from the display's
orientation. The caller is responsible for clearing this transform
hint if the layer is no longer in a fixed orientation.

Bug: 152919661
Test: atest VulkanPreTransformTest
Test: confirm with winscope trace, buffers are allocated taking into
account the transform hint in fixed orientation scenarios
Test: go/wm-smoke

Change-Id: Iea9dcf909921802a5be5c44dd61be3274f36bbd8
diff --git a/libs/gui/LayerState.cpp b/libs/gui/LayerState.cpp
index f7158d0..e43446a 100644
--- a/libs/gui/LayerState.cpp
+++ b/libs/gui/LayerState.cpp
@@ -116,6 +116,7 @@
     output.writeInt32(frameRateSelectionPriority);
     output.writeFloat(frameRate);
     output.writeByte(frameRateCompatibility);
+    output.writeUint32(fixedTransformHint);
     return NO_ERROR;
 }
 
@@ -198,6 +199,7 @@
     frameRateSelectionPriority = input.readInt32();
     frameRate = input.readFloat();
     frameRateCompatibility = input.readByte();
+    fixedTransformHint = static_cast<ui::Transform::RotationFlags>(input.readUint32());
     return NO_ERROR;
 }
 
@@ -433,6 +435,10 @@
         frameRate = other.frameRate;
         frameRateCompatibility = other.frameRateCompatibility;
     }
+    if (other.what & eFixedTransformHintChanged) {
+        what |= eFixedTransformHintChanged;
+        fixedTransformHint = other.fixedTransformHint;
+    }
     if ((other.what & what) != other.what) {
         ALOGE("Unmerged SurfaceComposer Transaction properties. LayerState::merge needs updating? "
               "other.what=0x%" PRIu64 " what=0x%" PRIu64,
diff --git a/libs/gui/SurfaceComposerClient.cpp b/libs/gui/SurfaceComposerClient.cpp
index a52f298..5922f3a 100644
--- a/libs/gui/SurfaceComposerClient.cpp
+++ b/libs/gui/SurfaceComposerClient.cpp
@@ -1437,6 +1437,22 @@
     return *this;
 }
 
+SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setFixedTransformHint(
+        const sp<SurfaceControl>& sc, int32_t fixedTransformHint) {
+    layer_state_t* s = getLayerState(sc);
+    if (!s) {
+        mStatus = BAD_INDEX;
+        return *this;
+    }
+
+    const ui::Transform::RotationFlags transform = fixedTransformHint == -1
+            ? ui::Transform::ROT_INVALID
+            : ui::Transform::toRotationFlags(static_cast<ui::Rotation>(fixedTransformHint));
+    s->what |= layer_state_t::eFixedTransformHintChanged;
+    s->fixedTransformHint = transform;
+    return *this;
+}
+
 // ---------------------------------------------------------------------------
 
 DisplayState& SurfaceComposerClient::Transaction::getDisplayState(const sp<IBinder>& token) {
diff --git a/libs/gui/include/gui/LayerState.h b/libs/gui/include/gui/LayerState.h
index 2b2f773..e60f677 100644
--- a/libs/gui/include/gui/LayerState.h
+++ b/libs/gui/include/gui/LayerState.h
@@ -35,6 +35,7 @@
 #include <ui/Rect.h>
 #include <ui/Region.h>
 #include <ui/Rotation.h>
+#include <ui/Transform.h>
 #include <utils/Errors.h>
 
 namespace android {
@@ -103,6 +104,7 @@
         eFrameRateChanged = 0x40'00000000,
         eBackgroundBlurRadiusChanged = 0x80'00000000,
         eProducerDisconnect = 0x100'00000000,
+        eFixedTransformHintChanged = 0x200'00000000,
     };
 
     layer_state_t()
@@ -136,7 +138,8 @@
             shadowRadius(0.0f),
             frameRateSelectionPriority(-1),
             frameRate(0.0f),
-            frameRateCompatibility(ANATIVEWINDOW_FRAME_RATE_COMPATIBILITY_DEFAULT) {
+            frameRateCompatibility(ANATIVEWINDOW_FRAME_RATE_COMPATIBILITY_DEFAULT),
+            fixedTransformHint(ui::Transform::ROT_INVALID) {
         matrix.dsdx = matrix.dtdy = 1.0f;
         matrix.dsdy = matrix.dtdx = 0.0f;
         hdrMetadata.validTypes = 0;
@@ -225,6 +228,15 @@
     // Layer frame rate and compatibility. See ANativeWindow_setFrameRate().
     float frameRate;
     int8_t frameRateCompatibility;
+
+    // Set by window manager indicating the layer and all its children are
+    // in a different orientation than the display. The hint suggests that
+    // the graphic producers should receive a transform hint as if the
+    // display was in this orientation. When the display changes to match
+    // the layer orientation, the graphic producer may not need to allocate
+    // a buffer of a different size. -1 means the transform hint is not set,
+    // otherwise the value will be a valid ui::Rotation.
+    ui::Transform::RotationFlags fixedTransformHint;
 };
 
 struct ComposerState {
diff --git a/libs/gui/include/gui/SurfaceComposerClient.h b/libs/gui/include/gui/SurfaceComposerClient.h
index 531aed7..e981a39 100644
--- a/libs/gui/include/gui/SurfaceComposerClient.h
+++ b/libs/gui/include/gui/SurfaceComposerClient.h
@@ -519,6 +519,14 @@
         Transaction& setFrameRate(const sp<SurfaceControl>& sc, float frameRate,
                                   int8_t compatibility);
 
+        // Set by window manager indicating the layer and all its children are
+        // in a different orientation than the display. The hint suggests that
+        // the graphic producers should receive a transform hint as if the
+        // display was in this orientation. When the display changes to match
+        // the layer orientation, the graphic producer may not need to allocate
+        // a buffer of a different size.
+        Transaction& setFixedTransformHint(const sp<SurfaceControl>& sc, int32_t transformHint);
+
         status_t setDisplaySurface(const sp<IBinder>& token,
                 const sp<IGraphicBufferProducer>& bufferProducer);