Merge "Provide a fixed transform hint if the layer is in a fixed orientation 1/2" into rvc-dev
diff --git a/libs/gui/LayerState.cpp b/libs/gui/LayerState.cpp
index f7158d0..e43446a 100644
--- a/libs/gui/LayerState.cpp
+++ b/libs/gui/LayerState.cpp
@@ -116,6 +116,7 @@
     output.writeInt32(frameRateSelectionPriority);
     output.writeFloat(frameRate);
     output.writeByte(frameRateCompatibility);
+    output.writeUint32(fixedTransformHint);
     return NO_ERROR;
 }
 
@@ -198,6 +199,7 @@
     frameRateSelectionPriority = input.readInt32();
     frameRate = input.readFloat();
     frameRateCompatibility = input.readByte();
+    fixedTransformHint = static_cast<ui::Transform::RotationFlags>(input.readUint32());
     return NO_ERROR;
 }
 
@@ -433,6 +435,10 @@
         frameRate = other.frameRate;
         frameRateCompatibility = other.frameRateCompatibility;
     }
+    if (other.what & eFixedTransformHintChanged) {
+        what |= eFixedTransformHintChanged;
+        fixedTransformHint = other.fixedTransformHint;
+    }
     if ((other.what & what) != other.what) {
         ALOGE("Unmerged SurfaceComposer Transaction properties. LayerState::merge needs updating? "
               "other.what=0x%" PRIu64 " what=0x%" PRIu64,
diff --git a/libs/gui/SurfaceComposerClient.cpp b/libs/gui/SurfaceComposerClient.cpp
index a52f298..5922f3a 100644
--- a/libs/gui/SurfaceComposerClient.cpp
+++ b/libs/gui/SurfaceComposerClient.cpp
@@ -1437,6 +1437,22 @@
     return *this;
 }
 
+SurfaceComposerClient::Transaction& SurfaceComposerClient::Transaction::setFixedTransformHint(
+        const sp<SurfaceControl>& sc, int32_t fixedTransformHint) {
+    layer_state_t* s = getLayerState(sc);
+    if (!s) {
+        mStatus = BAD_INDEX;
+        return *this;
+    }
+
+    const ui::Transform::RotationFlags transform = fixedTransformHint == -1
+            ? ui::Transform::ROT_INVALID
+            : ui::Transform::toRotationFlags(static_cast<ui::Rotation>(fixedTransformHint));
+    s->what |= layer_state_t::eFixedTransformHintChanged;
+    s->fixedTransformHint = transform;
+    return *this;
+}
+
 // ---------------------------------------------------------------------------
 
 DisplayState& SurfaceComposerClient::Transaction::getDisplayState(const sp<IBinder>& token) {
diff --git a/libs/gui/include/gui/LayerState.h b/libs/gui/include/gui/LayerState.h
index 2b2f773..e60f677 100644
--- a/libs/gui/include/gui/LayerState.h
+++ b/libs/gui/include/gui/LayerState.h
@@ -35,6 +35,7 @@
 #include <ui/Rect.h>
 #include <ui/Region.h>
 #include <ui/Rotation.h>
+#include <ui/Transform.h>
 #include <utils/Errors.h>
 
 namespace android {
@@ -103,6 +104,7 @@
         eFrameRateChanged = 0x40'00000000,
         eBackgroundBlurRadiusChanged = 0x80'00000000,
         eProducerDisconnect = 0x100'00000000,
+        eFixedTransformHintChanged = 0x200'00000000,
     };
 
     layer_state_t()
@@ -136,7 +138,8 @@
             shadowRadius(0.0f),
             frameRateSelectionPriority(-1),
             frameRate(0.0f),
-            frameRateCompatibility(ANATIVEWINDOW_FRAME_RATE_COMPATIBILITY_DEFAULT) {
+            frameRateCompatibility(ANATIVEWINDOW_FRAME_RATE_COMPATIBILITY_DEFAULT),
+            fixedTransformHint(ui::Transform::ROT_INVALID) {
         matrix.dsdx = matrix.dtdy = 1.0f;
         matrix.dsdy = matrix.dtdx = 0.0f;
         hdrMetadata.validTypes = 0;
@@ -225,6 +228,15 @@
     // Layer frame rate and compatibility. See ANativeWindow_setFrameRate().
     float frameRate;
     int8_t frameRateCompatibility;
+
+    // Set by window manager indicating the layer and all its children are
+    // in a different orientation than the display. The hint suggests that
+    // the graphic producers should receive a transform hint as if the
+    // display was in this orientation. When the display changes to match
+    // the layer orientation, the graphic producer may not need to allocate
+    // a buffer of a different size. -1 means the transform hint is not set,
+    // otherwise the value will be a valid ui::Rotation.
+    ui::Transform::RotationFlags fixedTransformHint;
 };
 
 struct ComposerState {
diff --git a/libs/gui/include/gui/SurfaceComposerClient.h b/libs/gui/include/gui/SurfaceComposerClient.h
index 531aed7..e981a39 100644
--- a/libs/gui/include/gui/SurfaceComposerClient.h
+++ b/libs/gui/include/gui/SurfaceComposerClient.h
@@ -519,6 +519,14 @@
         Transaction& setFrameRate(const sp<SurfaceControl>& sc, float frameRate,
                                   int8_t compatibility);
 
+        // Set by window manager indicating the layer and all its children are
+        // in a different orientation than the display. The hint suggests that
+        // the graphic producers should receive a transform hint as if the
+        // display was in this orientation. When the display changes to match
+        // the layer orientation, the graphic producer may not need to allocate
+        // a buffer of a different size.
+        Transaction& setFixedTransformHint(const sp<SurfaceControl>& sc, int32_t transformHint);
+
         status_t setDisplaySurface(const sp<IBinder>& token,
                 const sp<IGraphicBufferProducer>& bufferProducer);
 
diff --git a/services/surfaceflinger/BufferLayer.cpp b/services/surfaceflinger/BufferLayer.cpp
index 5b28384..8ecdd95 100644
--- a/services/surfaceflinger/BufferLayer.cpp
+++ b/services/surfaceflinger/BufferLayer.cpp
@@ -843,6 +843,13 @@
     mDrawingState.inputInfo = tmpInputInfo;
 }
 
+void BufferLayer::setTransformHint(ui::Transform::RotationFlags displayTransformHint) const {
+    mTransformHint = getFixedTransformHint();
+    if (mTransformHint == ui::Transform::ROT_INVALID) {
+        mTransformHint = displayTransformHint;
+    }
+}
+
 } // namespace android
 
 #if defined(__gl_h_)
diff --git a/services/surfaceflinger/BufferLayer.h b/services/surfaceflinger/BufferLayer.h
index 56bab1b..cfccc8a 100644
--- a/services/surfaceflinger/BufferLayer.h
+++ b/services/surfaceflinger/BufferLayer.h
@@ -117,6 +117,10 @@
 
     sp<GraphicBuffer> getBuffer() const override;
 
+    ui::Transform::RotationFlags getTransformHint() const override { return mTransformHint; }
+
+    void setTransformHint(ui::Transform::RotationFlags displayTransformHint) const override;
+
     // -----------------------------------------------------------------------
 
     // -----------------------------------------------------------------------
@@ -205,6 +209,10 @@
 
     virtual uint64_t getHeadFrameNumber(nsecs_t expectedPresentTime) const;
 
+    // Transform hint provided to the producer. This must be accessed holding
+    /// the mStateLock.
+    mutable ui::Transform::RotationFlags mTransformHint = ui::Transform::ROT_0;
+
 private:
     // Returns true if this layer requires filtering
     bool needsFiltering(const sp<const DisplayDevice>& displayDevice) const override;
diff --git a/services/surfaceflinger/BufferQueueLayer.cpp b/services/surfaceflinger/BufferQueueLayer.cpp
index f4e630e..5d80768 100644
--- a/services/surfaceflinger/BufferQueueLayer.cpp
+++ b/services/surfaceflinger/BufferQueueLayer.cpp
@@ -58,8 +58,9 @@
     }
 }
 
-void BufferQueueLayer::setTransformHint(uint32_t orientation) const {
-    mConsumer->setTransformHint(orientation);
+void BufferQueueLayer::setTransformHint(ui::Transform::RotationFlags displayTransformHint) const {
+    BufferLayer::setTransformHint(displayTransformHint);
+    mConsumer->setTransformHint(mTransformHint);
 }
 
 std::vector<OccupancyTracker::Segment> BufferQueueLayer::getOccupancyHistory(bool forceFlush) {
@@ -493,10 +494,6 @@
     if (!mFlinger->isLayerTripleBufferingDisabled()) {
         mProducer->setMaxDequeuedBufferCount(2);
     }
-
-    if (const auto display = mFlinger->getDefaultDisplayDeviceLocked()) {
-        updateTransformHint(display);
-    }
 }
 
 status_t BufferQueueLayer::setDefaultBufferProperties(uint32_t w, uint32_t h, PixelFormat format) {
diff --git a/services/surfaceflinger/BufferQueueLayer.h b/services/surfaceflinger/BufferQueueLayer.h
index 16b4b6e..9bcb63a 100644
--- a/services/surfaceflinger/BufferQueueLayer.h
+++ b/services/surfaceflinger/BufferQueueLayer.h
@@ -43,7 +43,7 @@
 
     void onLayerDisplayed(const sp<Fence>& releaseFence) override;
 
-    void setTransformHint(uint32_t orientation) const override;
+    void setTransformHint(ui::Transform::RotationFlags displayTransformHint) const override;
 
     std::vector<OccupancyTracker::Segment> getOccupancyHistory(bool forceFlush) override;
 
diff --git a/services/surfaceflinger/BufferStateLayer.cpp b/services/surfaceflinger/BufferStateLayer.cpp
index a121ce0..a1ed6d7 100644
--- a/services/surfaceflinger/BufferStateLayer.cpp
+++ b/services/surfaceflinger/BufferStateLayer.cpp
@@ -51,9 +51,6 @@
       : BufferLayer(args), mHwcSlotGenerator(new HwcSlotGenerator()) {
     mOverrideScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
     mCurrentState.dataspace = ui::Dataspace::V0_SRGB;
-    if (const auto display = args.displayDevice) {
-        updateTransformHint(display);
-    }
 }
 
 BufferStateLayer::~BufferStateLayer() {
@@ -108,10 +105,6 @@
     }
 }
 
-void BufferStateLayer::setTransformHint(uint32_t orientation) const {
-    mTransformHint = orientation;
-}
-
 void BufferStateLayer::releasePendingBuffer(nsecs_t dequeueReadyTime) {
     for (const auto& handle : mDrawingState.callbackHandles) {
         handle->transformHint = mTransformHint;
diff --git a/services/surfaceflinger/BufferStateLayer.h b/services/surfaceflinger/BufferStateLayer.h
index 5873a73..00fa7f7 100644
--- a/services/surfaceflinger/BufferStateLayer.h
+++ b/services/surfaceflinger/BufferStateLayer.h
@@ -42,7 +42,6 @@
     const char* getType() const override { return "BufferStateLayer"; }
 
     void onLayerDisplayed(const sp<Fence>& releaseFence) override;
-    void setTransformHint(uint32_t orientation) const override;
     void releasePendingBuffer(nsecs_t dequeueReadyTime) override;
 
     void finalizeFrameEventHistory(const std::shared_ptr<FenceTime>& glDoneFence,
@@ -68,7 +67,6 @@
     }
     Rect getCrop(const Layer::State& s) const;
 
-    uint32_t getTransformHint() const { return mTransformHint; }
     bool setTransform(uint32_t transform) override;
     bool setTransformToDisplayInverse(bool transformToDisplayInverse) override;
     bool setCrop(const Rect& crop) override;
@@ -164,8 +162,6 @@
     bool mReleasePreviousBuffer = false;
     nsecs_t mCallbackHandleAcquireTime = -1;
 
-    mutable uint32_t mTransformHint = 0;
-
     // TODO(marissaw): support sticky transform for LEGACY camera mode
 
     class HwcSlotGenerator : public ClientCache::ErasedRecipient {
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 752407a..25929ed1 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -117,6 +117,7 @@
     mCurrentState.metadata = args.metadata;
     mCurrentState.shadowRadius = 0.f;
     mCurrentState.treeHasFrameRateVote = false;
+    mCurrentState.fixedTransformHint = ui::Transform::ROT_INVALID;
 
     // drawing state & current state are identical
     mDrawingState = mCurrentState;
@@ -1333,6 +1334,18 @@
     return true;
 }
 
+bool Layer::setFixedTransformHint(ui::Transform::RotationFlags fixedTransformHint) {
+    if (mCurrentState.fixedTransformHint == fixedTransformHint) {
+        return false;
+    }
+
+    mCurrentState.sequence++;
+    mCurrentState.fixedTransformHint = fixedTransformHint;
+    mCurrentState.modified = true;
+    setTransactionFlags(eTransactionNeeded);
+    return true;
+}
+
 void Layer::updateTreeHasFrameRateVote() {
     const auto traverseTree = [&](const LayerVector::Visitor& visitor) {
         auto parent = getParent();
@@ -1460,19 +1473,19 @@
 }
 
 void Layer::updateTransformHint(const sp<const DisplayDevice>& display) const {
-    uint32_t orientation = 0;
+    ui::Transform::RotationFlags transformHint = ui::Transform::ROT_0;
     // Disable setting transform hint if the debug flag is set.
     if (!mFlinger->mDebugDisableTransformHint) {
         // The transform hint is used to improve performance, but we can
         // only have a single transform hint, it cannot
         // apply to all displays.
         const ui::Transform& planeTransform = display->getTransform();
-        orientation = planeTransform.getOrientation();
-        if (orientation & ui::Transform::ROT_INVALID) {
-            orientation = 0;
+        transformHint = static_cast<ui::Transform::RotationFlags>(planeTransform.getOrientation());
+        if (transformHint & ui::Transform::ROT_INVALID) {
+            transformHint = ui::Transform::ROT_0;
         }
     }
-    setTransformHint(orientation);
+    setTransformHint(transformHint);
 }
 
 // ----------------------------------------------------------------------------
@@ -2076,6 +2089,16 @@
     return parentAlpha * getDrawingState().color.a;
 }
 
+ui::Transform::RotationFlags Layer::getFixedTransformHint() const {
+    ui::Transform::RotationFlags fixedTransformHint = mCurrentState.fixedTransformHint;
+    if (fixedTransformHint != ui::Transform::ROT_INVALID) {
+        return fixedTransformHint;
+    }
+    const auto& p = mCurrentParent.promote();
+    if (!p) return fixedTransformHint;
+    return p->getFixedTransformHint();
+}
+
 half4 Layer::getColor() const {
     const half4 color(getDrawingState().color);
     return half4(color.r, color.g, color.b, getAlpha());
diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index b630333..224ea19 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -262,6 +262,15 @@
 
         // Indicates whether parents / children of this layer had set FrameRate
         bool treeHasFrameRateVote;
+
+        // Set by window manager indicating the layer and all its children are
+        // in a different orientation than the display. The hint suggests that
+        // the graphic producers should receive a transform hint as if the
+        // display was in this orientation. When the display changes to match
+        // the layer orientation, the graphic producer may not need to allocate
+        // a buffer of a different size. ui::Transform::ROT_INVALID means the
+        // a fixed transform hint is not set.
+        ui::Transform::RotationFlags fixedTransformHint;
     };
 
     explicit Layer(const LayerCreationArgs& args);
@@ -388,6 +397,7 @@
     virtual bool setColorSpaceAgnostic(const bool agnostic);
     bool setShadowRadius(float shadowRadius);
     virtual bool setFrameRateSelectionPriority(int32_t priority);
+    virtual bool setFixedTransformHint(ui::Transform::RotationFlags fixedTransformHint);
     //  If the variable is not set on the layer, it traverses up the tree to inherit the frame
     //  rate priority from its parent.
     virtual int32_t getFrameRateSelectionPriority() const;
@@ -611,7 +621,7 @@
     bool getClearClientTarget(const sp<const DisplayDevice>& display) const;
 
     virtual bool shouldPresentNow(nsecs_t /*expectedPresentTime*/) const { return false; }
-    virtual void setTransformHint(uint32_t /*orientation*/) const { }
+    virtual void setTransformHint(ui::Transform::RotationFlags /*transformHint*/) const {}
 
     /*
      * called after composition.
@@ -689,6 +699,8 @@
 
     virtual sp<GraphicBuffer> getBuffer() const { return nullptr; }
 
+    virtual ui::Transform::RotationFlags getTransformHint() const { return ui::Transform::ROT_0; }
+
     /*
      * Returns if a frame is ready
      */
@@ -733,6 +745,12 @@
     int32_t getBackgroundBlurRadius() const;
     bool drawShadows() const { return mEffectiveShadowRadius > 0.f; };
 
+    // Returns the transform hint set by Window Manager on the layer or one of its parents.
+    // This traverses the current state because the data is needed when creating
+    // the layer(off drawing thread) and the hint should be available before the producer
+    // is ready to acquire a buffer.
+    ui::Transform::RotationFlags getFixedTransformHint() const;
+
     // Returns how rounded corners should be drawn for this layer.
     // This will traverse the hierarchy until it reaches its root, finding topmost rounded
     // corner definition and converting it into current layer's coordinates.
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index 0709a13..3b66fa9 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -2765,7 +2765,7 @@
         processDisplayHotplugEventsLocked();
     }
 
-    if (transactionFlags & (eDisplayLayerStackChanged|eDisplayTransactionNeeded)) {
+    if (transactionFlags & (eTransformHintUpdateNeeded | eDisplayTransactionNeeded)) {
         // The transform hint might have changed for some layers
         // (either because a display has changed, or because a layer
         // as changed).
@@ -2833,7 +2833,6 @@
         });
     }
 
-
     /*
      * Perform our own transaction if needed
      */
@@ -3138,7 +3137,8 @@
 status_t SurfaceFlinger::addClientLayer(const sp<Client>& client, const sp<IBinder>& handle,
                                         const sp<IGraphicBufferProducer>& gbc, const sp<Layer>& lbc,
                                         const sp<IBinder>& parentHandle,
-                                        const sp<Layer>& parentLayer, bool addToCurrentState) {
+                                        const sp<Layer>& parentLayer, bool addToCurrentState,
+                                        uint32_t* outTransformHint) {
     // add this layer to the current state list
     {
         Mutex::Autolock _l(mStateLock);
@@ -3179,6 +3179,14 @@
                                 mGraphicBufferProducerList.size(),
                                 mMaxGraphicBufferProducerListSize, mNumLayers.load());
         }
+
+        if (const auto display = getDefaultDisplayDeviceLocked()) {
+            lbc->updateTransformHint(display);
+        }
+        if (outTransformHint) {
+            *outTransformHint = lbc->getTransformHint();
+        }
+
         mLayersAdded = true;
     }
 
@@ -3680,7 +3688,7 @@
             mCurrentState.layersSortedByZ.add(layer);
             // we need traversal (state changed)
             // AND transaction (list changed)
-            flags |= eTransactionNeeded|eTraversalNeeded|eDisplayLayerStackChanged;
+            flags |= eTransactionNeeded | eTraversalNeeded | eTransformHintUpdateNeeded;
         }
     }
     if (what & layer_state_t::eDeferTransaction_legacy) {
@@ -3777,6 +3785,11 @@
             flags |= eTraversalNeeded;
         }
     }
+    if (what & layer_state_t::eFixedTransformHintChanged) {
+        if (layer->setFixedTransformHint(s.fixedTransformHint)) {
+            flags |= eTraversalNeeded | eTransformHintUpdateNeeded;
+        }
+    }
     // This has to happen after we reparent children because when we reparent to null we remove
     // child layers from current state and remove its relative z. If the children are reparented in
     // the same transaction, then we have to make sure we reparent the children first so we do not
@@ -3864,7 +3877,8 @@
         mirrorLayer->mClonedChild = mirrorFrom->createClone();
     }
 
-    return addClientLayer(client, *outHandle, nullptr, mirrorLayer, nullptr, nullptr, false);
+    return addClientLayer(client, *outHandle, nullptr, mirrorLayer, nullptr, nullptr, false,
+                          nullptr /* outTransformHint */);
 }
 
 status_t SurfaceFlinger::createLayer(const String8& name, const sp<Client>& client, uint32_t w,
@@ -3909,7 +3923,7 @@
             break;
         case ISurfaceComposerClient::eFXSurfaceBufferState:
             result = createBufferStateLayer(client, std::move(uniqueName), w, h, flags,
-                                            std::move(metadata), handle, outTransformHint, &layer);
+                                            std::move(metadata), handle, &layer);
             break;
         case ISurfaceComposerClient::eFXSurfaceEffect:
             // check if buffer size is set for color layer.
@@ -3947,7 +3961,7 @@
 
     bool addToCurrentState = callingThreadHasUnscopedSurfaceFlingerAccess();
     result = addClientLayer(client, *handle, *gbp, layer, parentHandle, parentLayer,
-                            addToCurrentState);
+                            addToCurrentState, outTransformHint);
     if (result != NO_ERROR) {
         return result;
     }
@@ -4024,14 +4038,11 @@
 status_t SurfaceFlinger::createBufferStateLayer(const sp<Client>& client, std::string name,
                                                 uint32_t w, uint32_t h, uint32_t flags,
                                                 LayerMetadata metadata, sp<IBinder>* handle,
-                                                uint32_t* outTransformHint, sp<Layer>* outLayer) {
+                                                sp<Layer>* outLayer) {
     LayerCreationArgs args(this, client, std::move(name), w, h, flags, std::move(metadata));
     args.displayDevice = getDefaultDisplayDevice();
     args.textureName = getNewTexture();
     sp<BufferStateLayer> layer = getFactory().createBufferStateLayer(args);
-    if (outTransformHint) {
-        *outTransformHint = layer->getTransformHint();
-    }
     *handle = layer->getHandle();
     *outLayer = layer;
 
diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h
index 715d5f7..509de0f 100644
--- a/services/surfaceflinger/SurfaceFlinger.h
+++ b/services/surfaceflinger/SurfaceFlinger.h
@@ -117,7 +117,7 @@
     eTransactionNeeded = 0x01,
     eTraversalNeeded = 0x02,
     eDisplayTransactionNeeded = 0x04,
-    eDisplayLayerStackChanged = 0x08,
+    eTransformHintUpdateNeeded = 0x08,
     eTransactionFlushNeeded = 0x10,
     eTransactionMask = 0x1f,
 };
@@ -675,8 +675,7 @@
 
     status_t createBufferStateLayer(const sp<Client>& client, std::string name, uint32_t w,
                                     uint32_t h, uint32_t flags, LayerMetadata metadata,
-                                    sp<IBinder>* outHandle, uint32_t* outTransformHint,
-                                    sp<Layer>* outLayer);
+                                    sp<IBinder>* outHandle, sp<Layer>* outLayer);
 
     status_t createEffectLayer(const sp<Client>& client, std::string name, uint32_t w, uint32_t h,
                                uint32_t flags, LayerMetadata metadata, sp<IBinder>* outHandle,
@@ -701,7 +700,7 @@
     status_t addClientLayer(const sp<Client>& client, const sp<IBinder>& handle,
                             const sp<IGraphicBufferProducer>& gbc, const sp<Layer>& lbc,
                             const sp<IBinder>& parentHandle, const sp<Layer>& parentLayer,
-                            bool addToCurrentState);
+                            bool addToCurrentState, uint32_t* outTransformHint);
 
     // Traverse through all the layers and compute and cache its bounds.
     void computeLayerBounds();