Provide a fixed transform hint if the layer is in a fixed orientation 1/2

The transform hint is used to prevent allocating a buffer of a
different size when a layer is rotated. The producer can choose to
consume the hint and allocate the buffer with the same size.

Provide the graphic producer a transform hint if the layer and its
children are in an orientation different from the display's
orientation. The caller is responsible for clearing this transform
hint if the layer is no longer in a fixed orientation.

Bug: 152919661
Test: atest VulkanPreTransformTest
Test: confirm with winscope trace, buffers are allocated taking into
account the transform hint in fixed orientation scenarios
Test: go/wm-smoke

Change-Id: Iea9dcf909921802a5be5c44dd61be3274f36bbd8
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index 4ca2074..a2e9e2f 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -2764,7 +2764,7 @@
         processDisplayHotplugEventsLocked();
     }
 
-    if (transactionFlags & (eDisplayLayerStackChanged|eDisplayTransactionNeeded)) {
+    if (transactionFlags & (eTransformHintUpdateNeeded | eDisplayTransactionNeeded)) {
         // The transform hint might have changed for some layers
         // (either because a display has changed, or because a layer
         // as changed).
@@ -2832,7 +2832,6 @@
         });
     }
 
-
     /*
      * Perform our own transaction if needed
      */
@@ -3137,7 +3136,8 @@
 status_t SurfaceFlinger::addClientLayer(const sp<Client>& client, const sp<IBinder>& handle,
                                         const sp<IGraphicBufferProducer>& gbc, const sp<Layer>& lbc,
                                         const sp<IBinder>& parentHandle,
-                                        const sp<Layer>& parentLayer, bool addToCurrentState) {
+                                        const sp<Layer>& parentLayer, bool addToCurrentState,
+                                        uint32_t* outTransformHint) {
     // add this layer to the current state list
     {
         Mutex::Autolock _l(mStateLock);
@@ -3178,6 +3178,14 @@
                                 mGraphicBufferProducerList.size(),
                                 mMaxGraphicBufferProducerListSize, mNumLayers.load());
         }
+
+        if (const auto display = getDefaultDisplayDeviceLocked()) {
+            lbc->updateTransformHint(display);
+        }
+        if (outTransformHint) {
+            *outTransformHint = lbc->getTransformHint();
+        }
+
         mLayersAdded = true;
     }
 
@@ -3679,7 +3687,7 @@
             mCurrentState.layersSortedByZ.add(layer);
             // we need traversal (state changed)
             // AND transaction (list changed)
-            flags |= eTransactionNeeded|eTraversalNeeded|eDisplayLayerStackChanged;
+            flags |= eTransactionNeeded | eTraversalNeeded | eTransformHintUpdateNeeded;
         }
     }
     if (what & layer_state_t::eDeferTransaction_legacy) {
@@ -3776,6 +3784,11 @@
             flags |= eTraversalNeeded;
         }
     }
+    if (what & layer_state_t::eFixedTransformHintChanged) {
+        if (layer->setFixedTransformHint(s.fixedTransformHint)) {
+            flags |= eTraversalNeeded | eTransformHintUpdateNeeded;
+        }
+    }
     // This has to happen after we reparent children because when we reparent to null we remove
     // child layers from current state and remove its relative z. If the children are reparented in
     // the same transaction, then we have to make sure we reparent the children first so we do not
@@ -3863,7 +3876,8 @@
         mirrorLayer->mClonedChild = mirrorFrom->createClone();
     }
 
-    return addClientLayer(client, *outHandle, nullptr, mirrorLayer, nullptr, nullptr, false);
+    return addClientLayer(client, *outHandle, nullptr, mirrorLayer, nullptr, nullptr, false,
+                          nullptr /* outTransformHint */);
 }
 
 status_t SurfaceFlinger::createLayer(const String8& name, const sp<Client>& client, uint32_t w,
@@ -3908,7 +3922,7 @@
             break;
         case ISurfaceComposerClient::eFXSurfaceBufferState:
             result = createBufferStateLayer(client, std::move(uniqueName), w, h, flags,
-                                            std::move(metadata), handle, outTransformHint, &layer);
+                                            std::move(metadata), handle, &layer);
             break;
         case ISurfaceComposerClient::eFXSurfaceEffect:
             // check if buffer size is set for color layer.
@@ -3946,7 +3960,7 @@
 
     bool addToCurrentState = callingThreadHasUnscopedSurfaceFlingerAccess();
     result = addClientLayer(client, *handle, *gbp, layer, parentHandle, parentLayer,
-                            addToCurrentState);
+                            addToCurrentState, outTransformHint);
     if (result != NO_ERROR) {
         return result;
     }
@@ -4023,14 +4037,11 @@
 status_t SurfaceFlinger::createBufferStateLayer(const sp<Client>& client, std::string name,
                                                 uint32_t w, uint32_t h, uint32_t flags,
                                                 LayerMetadata metadata, sp<IBinder>* handle,
-                                                uint32_t* outTransformHint, sp<Layer>* outLayer) {
+                                                sp<Layer>* outLayer) {
     LayerCreationArgs args(this, client, std::move(name), w, h, flags, std::move(metadata));
     args.displayDevice = getDefaultDisplayDevice();
     args.textureName = getNewTexture();
     sp<BufferStateLayer> layer = getFactory().createBufferStateLayer(args);
-    if (outTransformHint) {
-        *outTransformHint = layer->getTransformHint();
-    }
     *handle = layer->getHandle();
     *outLayer = layer;