[sf] Add support for transform hint in new front end

Bug: 238781169
Test: atest VulkanPreTransformTest --rerun-until-failure
Change-Id: Iececcce4101f1a8875ba20e2bfffad1775964ed1
diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshot.h b/services/surfaceflinger/FrontEnd/LayerSnapshot.h
index e22c279..5491d9a 100644
--- a/services/surfaceflinger/FrontEnd/LayerSnapshot.h
+++ b/services/surfaceflinger/FrontEnd/LayerSnapshot.h
@@ -89,6 +89,7 @@
     gui::GameMode gameMode;
     scheduler::LayerInfo::FrameRate frameRate;
     ui::Transform::RotationFlags fixedTransformHint;
+    std::optional<ui::Transform::RotationFlags> transformHint;
     bool handleSkipScreenshotFlag = false;
     int32_t frameRateSelectionPriority;
     LayerHierarchy::TraversalPath mirrorRootPath;
diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
index 2fc9682..bdd4427 100644
--- a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
+++ b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
@@ -22,10 +22,15 @@
 #include "LayerSnapshotBuilder.h"
 #include <gui/TraceUtils.h>
 #include <ui/FloatRect.h>
+
 #include <numeric>
+#include <optional>
+
+#include <gui/TraceUtils.h>
 #include "DisplayHardware/HWC2.h"
 #include "DisplayHardware/Hal.h"
 #include "LayerLog.h"
+#include "LayerSnapshotBuilder.h"
 #include "TimeStats/TimeStats.h"
 #include "ftl/small_map.h"
 
@@ -649,12 +654,14 @@
     snapshot.relativeLayerMetadata.mMap.clear();
 }
 
-uint32_t getDisplayRotationFlags(
-        const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displays,
-        const ui::LayerStack& layerStack) {
-    static frontend::DisplayInfo sDefaultDisplayInfo = {.isPrimary = false};
-    auto display = displays.get(layerStack).value_or(sDefaultDisplayInfo).get();
-    return display.isPrimary ? display.rotationFlags : 0;
+uint32_t getPrimaryDisplayRotationFlags(
+        const display::DisplayMap<ui::LayerStack, frontend::DisplayInfo>& displays) {
+    for (auto& [_, display] : displays) {
+        if (display.isPrimary) {
+            return display.rotationFlags;
+        }
+    }
+    return 0;
 }
 
 void LayerSnapshotBuilder::updateSnapshot(LayerSnapshot& snapshot, const Args& args,
@@ -681,8 +688,7 @@
             ? requested.layerStack
             : parentSnapshot.outputFilter.layerStack;
 
-    uint32_t displayRotationFlags =
-            getDisplayRotationFlags(args.displays, snapshot.outputFilter.layerStack);
+    uint32_t primaryDisplayRotationFlags = getPrimaryDisplayRotationFlags(args.displays);
     const bool forceUpdate = args.forceUpdate == ForceUpdateFlags::ALL ||
             snapshot.changes.any(RequestedLayerState::Changes::Visibility |
                                  RequestedLayerState::Changes::Created);
@@ -696,7 +702,7 @@
                 : Fence::NO_FENCE;
         snapshot.buffer =
                 requested.externalTexture ? requested.externalTexture->getBuffer() : nullptr;
-        snapshot.bufferSize = requested.getBufferSize(displayRotationFlags);
+        snapshot.bufferSize = requested.getBufferSize(primaryDisplayRotationFlags);
         snapshot.geomBufferSize = snapshot.bufferSize;
         snapshot.croppedBufferSize = requested.getCroppedBufferSize(snapshot.bufferSize);
         snapshot.dataspace = requested.dataspace;
@@ -751,15 +757,28 @@
         snapshot.gameMode = requested.metadata.has(gui::METADATA_GAME_MODE)
                 ? requested.gameMode
                 : parentSnapshot.gameMode;
-        snapshot.fixedTransformHint = requested.fixedTransformHint != ui::Transform::ROT_INVALID
-                ? requested.fixedTransformHint
-                : parentSnapshot.fixedTransformHint;
         // Display mirrors are always placed in a VirtualDisplay so we never want to capture layers
         // marked as skip capture
         snapshot.handleSkipScreenshotFlag = parentSnapshot.handleSkipScreenshotFlag ||
                 (requested.layerStackToMirror != ui::INVALID_LAYER_STACK);
     }
 
+    if (forceUpdate || snapshot.changes.any(RequestedLayerState::Changes::AffectsChildren) ||
+        args.displayChanges) {
+        snapshot.fixedTransformHint = requested.fixedTransformHint != ui::Transform::ROT_INVALID
+                ? requested.fixedTransformHint
+                : parentSnapshot.fixedTransformHint;
+
+        if (snapshot.fixedTransformHint != ui::Transform::ROT_INVALID) {
+            snapshot.transformHint = snapshot.fixedTransformHint;
+        } else {
+            const auto display = args.displays.get(snapshot.outputFilter.layerStack);
+            snapshot.transformHint = display.has_value()
+                    ? std::make_optional<>(display->get().transformHint)
+                    : std::nullopt;
+        }
+    }
+
     if (forceUpdate ||
         snapshot.changes.any(RequestedLayerState::Changes::FrameRate |
                              RequestedLayerState::Changes::Hierarchy)) {
@@ -799,7 +818,7 @@
     if (forceUpdate ||
         snapshot.changes.any(RequestedLayerState::Changes::Hierarchy |
                              RequestedLayerState::Changes::Geometry)) {
-        updateLayerBounds(snapshot, requested, parentSnapshot, displayRotationFlags);
+        updateLayerBounds(snapshot, requested, parentSnapshot, primaryDisplayRotationFlags);
         updateRoundedCorner(snapshot, requested, parentSnapshot);
     }
 
@@ -870,10 +889,10 @@
 void LayerSnapshotBuilder::updateLayerBounds(LayerSnapshot& snapshot,
                                              const RequestedLayerState& requested,
                                              const LayerSnapshot& parentSnapshot,
-                                             uint32_t displayRotationFlags) {
+                                             uint32_t primaryDisplayRotationFlags) {
     snapshot.croppedBufferSize = requested.getCroppedBufferSize(snapshot.bufferSize);
     snapshot.geomCrop = requested.crop;
-    snapshot.localTransform = requested.getTransform(displayRotationFlags);
+    snapshot.localTransform = requested.getTransform(primaryDisplayRotationFlags);
     snapshot.localTransformInverse = snapshot.localTransform.inverse();
     snapshot.geomLayerTransform = parentSnapshot.geomLayerTransform * snapshot.localTransform;
     const bool transformWasInvalid = snapshot.invalidTransform;
@@ -887,14 +906,14 @@
                                    requestedT.dsdy(), requestedT.dtdx(), requestedT.dtdy());
         std::string bufferDebug;
         if (requested.externalTexture) {
-            auto unRotBuffer = requested.getUnrotatedBufferSize(displayRotationFlags);
+            auto unRotBuffer = requested.getUnrotatedBufferSize(primaryDisplayRotationFlags);
             auto& destFrame = requested.destinationFrame;
             bufferDebug = base::StringPrintf(" buffer={%d,%d}  displayRot=%d"
                                              " destFrame={%d,%d,%d,%d} unRotBuffer={%d,%d}",
                                              requested.externalTexture->getWidth(),
                                              requested.externalTexture->getHeight(),
-                                             displayRotationFlags, destFrame.left, destFrame.top,
-                                             destFrame.right, destFrame.bottom,
+                                             primaryDisplayRotationFlags, destFrame.left,
+                                             destFrame.top, destFrame.right, destFrame.bottom,
                                              unRotBuffer.getHeight(), unRotBuffer.getWidth());
         }
         ALOGW("Resetting transform for %s because it is invalid.%s%s",
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 64f5c28..c1920ba 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -1505,7 +1505,7 @@
         transformHint = ui::Transform::ROT_0;
     }
 
-    setTransformHint(transformHint);
+    setTransformHintLegacy(transformHint);
 }
 
 // ----------------------------------------------------------------------------
@@ -2865,9 +2865,13 @@
 
 void Layer::releasePendingBuffer(nsecs_t dequeueReadyTime) {
     for (const auto& handle : mDrawingState.callbackHandles) {
-        handle->transformHint = mSkipReportingTransformHint
-                ? std::nullopt
-                : std::make_optional<uint32_t>(mTransformHint);
+        if (mFlinger->mLayerLifecycleManagerEnabled) {
+            handle->transformHint = mTransformHint;
+        } else {
+            handle->transformHint = mSkipReportingTransformHint
+                    ? std::nullopt
+                    : std::make_optional<uint32_t>(mTransformHintLegacy);
+        }
         handle->dequeueReadyTime = dequeueReadyTime;
         handle->currentMaxAcquiredBufferCount =
                 mFlinger->getMaxAcquiredBufferCountForCurrentRefreshRate(mOwnerUid);
@@ -4027,10 +4031,10 @@
     return mBufferInfo.mBuffer ? mBufferInfo.mBuffer->getBuffer() : nullptr;
 }
 
-void Layer::setTransformHint(ui::Transform::RotationFlags displayTransformHint) {
-    mTransformHint = getFixedTransformHint();
-    if (mTransformHint == ui::Transform::ROT_INVALID) {
-        mTransformHint = displayTransformHint;
+void Layer::setTransformHintLegacy(ui::Transform::RotationFlags displayTransformHint) {
+    mTransformHintLegacy = getFixedTransformHint();
+    if (mTransformHintLegacy == ui::Transform::ROT_INVALID) {
+        mTransformHintLegacy = displayTransformHint;
     }
     mSkipReportingTransformHint = false;
 }
diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index c91da3d..552ea37 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -455,8 +455,6 @@
     sp<GraphicBuffer> getBuffer() const;
     const std::shared_ptr<renderengine::ExternalTexture>& getExternalTexture() const;
 
-    ui::Transform::RotationFlags getTransformHint() const { return mTransformHint; }
-
     /*
      * Returns if a frame is ready
      */
@@ -878,6 +876,9 @@
         };
     };
     bool hasBuffer() const { return mBufferInfo.mBuffer != nullptr; }
+    void setTransformHint(std::optional<ui::Transform::RotationFlags> transformHint) {
+        mTransformHint = transformHint;
+    }
 
 protected:
     // For unit tests
@@ -1154,14 +1155,15 @@
     float mBorderWidth;
     half4 mBorderColor;
 
-    void setTransformHint(ui::Transform::RotationFlags);
+    void setTransformHintLegacy(ui::Transform::RotationFlags);
 
     const uint32_t mTextureName;
 
     // Transform hint provided to the producer. This must be accessed holding
     // the mStateLock.
-    ui::Transform::RotationFlags mTransformHint = ui::Transform::ROT_0;
+    ui::Transform::RotationFlags mTransformHintLegacy = ui::Transform::ROT_0;
     bool mSkipReportingTransformHint = true;
+    std::optional<ui::Transform::RotationFlags> mTransformHint = std::nullopt;
 
     ReleaseCallbackId mPreviousReleaseCallbackId = ReleaseCallbackId::INVALID_ID;
     uint64_t mPreviousReleasedFrameNumber = 0;
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index 5664d84..076c683 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -5137,6 +5137,12 @@
         if (layer->setSidebandStream(s.sidebandStream)) flags |= eTraversalNeeded;
     }
     if (what & layer_state_t::eBufferChanged) {
+        std::optional<ui::Transform::RotationFlags> transformHint = std::nullopt;
+        frontend::LayerSnapshot* snapshot = mLayerSnapshotBuilder.getSnapshot(layer->sequence);
+        if (snapshot) {
+            transformHint = snapshot->transformHint;
+        }
+        layer->setTransformHint(transformHint);
         if (layer->setBuffer(composerState.externalTexture, *s.bufferData, postTime,
                              desiredPresentTime, isAutoTimestamp, dequeueBufferTimestamp,
                              frameTimelineInfo)) {