SF: Move/Refactor Layer::setGeometry() to compositionengine::OutputLayer

The front-end Layer now implements only a small bit of functionality
to compute the geometry state of the layer, traversing the parent
hierarchy as needed. The geometry is written to data added to
LayerFECompositionState.

compositionengine::OutputLayer then computes the actual state for the output
the layer is on, storing the result in OutputLayerCompositionState. A
second new function then takes the output-specific layer state (and
output-independent layer state) and sends it to the HWC layer.

Implements partial tests for the new functions.

Test: atest libsurfaceflinger_unittest libcompositionengine_test
Bug: 121291683
Change-Id: Idc770e6c84d046555cc11d75d53fa22c4be4ae58
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h
index 6cc87ba..9f635b9 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h
@@ -24,12 +24,21 @@
 
 namespace compositionengine {
 
+struct LayerFECompositionState;
+
 // Defines the interface used by the CompositionEngine to make requests
 // of the front-end layer
 class LayerFE : public virtual RefBase {
 public:
+    // Latches the output-independent state. If includeGeometry is false, the
+    // geometry state can be skipped.
+    virtual void latchCompositionState(LayerFECompositionState&, bool includeGeometry) const = 0;
+
     // Called after the layer is displayed to update the presentation fence
     virtual void onLayerDisplayed(const sp<Fence>&) = 0;
+
+    // Gets some kind of identifier for the layer for debug purposes.
+    virtual const char* getDebugName() const = 0;
 };
 
 } // namespace compositionengine
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFECompositionState.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFECompositionState.h
index 785a6d7..e6ee078 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFECompositionState.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFECompositionState.h
@@ -41,6 +41,22 @@
     Region geomVisibleRegion;
 
     /*
+     * Geometry state
+     */
+
+    bool isSecure{false};
+    bool geomUsesSourceCrop{false};
+    bool geomBufferUsesDisplayInverseTransform{false};
+    uint32_t geomBufferTransform{0};
+    ui::Transform geomLayerTransform;
+    ui::Transform geomInverseLayerTransform;
+    Rect geomBufferSize;
+    Rect geomContentCrop;
+    Rect geomCrop;
+    Region geomActiveTransparentRegion;
+    FloatRect geomLayerBounds;
+
+    /*
      * Presentation
      */
 
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/OutputLayer.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/OutputLayer.h
index e7a17c4..cd63b57 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/OutputLayer.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/OutputLayer.h
@@ -64,6 +64,15 @@
     // TODO(lpique): Make this protected once it is only internally called.
     virtual CompositionState& editState() = 0;
 
+    // Recalculates the state of the output layer from the output-independent
+    // layer. If includeGeometry is false, the geometry state can be skipped.
+    virtual void updateCompositionState(bool includeGeometry) = 0;
+
+    // Writes the geometry state to the HWC, or does nothing if this layer does
+    // not use the HWC. If includeGeometry is false, the geometry state can be
+    // skipped.
+    virtual void writeStateToHWC(bool includeGeometry) const = 0;
+
     // Debugging
     virtual void dump(std::string& result) const = 0;
 };
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/OutputLayer.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/OutputLayer.h
index d8f0cdd..6a4818f 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/OutputLayer.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/OutputLayer.h
@@ -21,6 +21,8 @@
 
 #include <compositionengine/OutputLayer.h>
 #include <compositionengine/impl/OutputLayerCompositionState.h>
+#include <ui/FloatRect.h>
+#include <ui/Rect.h>
 
 #include "DisplayHardware/DisplayIdentification.h"
 
@@ -41,9 +43,18 @@
     const OutputLayerCompositionState& getState() const override;
     OutputLayerCompositionState& editState() override;
 
+    void updateCompositionState(bool) override;
+    void writeStateToHWC(bool) const override;
+
     void dump(std::string& result) const override;
 
+    virtual FloatRect calculateOutputSourceCrop() const;
+    virtual Rect calculateOutputDisplayFrame() const;
+    virtual uint32_t calculateOutputRelativeBufferTransform() const;
+
 private:
+    Rect calculateInitialCrop() const;
+
     const compositionengine::Output& mOutput;
     std::shared_ptr<compositionengine::Layer> mLayer;
     sp<compositionengine::LayerFE> mLayerFE;
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h
index a0c2a63..aab18db 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h
@@ -17,6 +17,7 @@
 #pragma once
 
 #include <compositionengine/LayerFE.h>
+#include <compositionengine/LayerFECompositionState.h>
 #include <gmock/gmock.h>
 #include <ui/Fence.h>
 
@@ -29,7 +30,10 @@
     LayerFE();
     virtual ~LayerFE();
 
+    MOCK_CONST_METHOD2(latchCompositionState, void(LayerFECompositionState&, bool));
     MOCK_METHOD1(onLayerDisplayed, void(const sp<Fence>&));
+
+    MOCK_CONST_METHOD0(getDebugName, const char*());
 };
 
 } // namespace android::compositionengine::mock
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/OutputLayer.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/OutputLayer.h
index 54c7407..29cd08a 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/OutputLayer.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/OutputLayer.h
@@ -38,6 +38,9 @@
     MOCK_CONST_METHOD0(getState, const impl::OutputLayerCompositionState&());
     MOCK_METHOD0(editState, impl::OutputLayerCompositionState&());
 
+    MOCK_METHOD1(updateCompositionState, void(bool));
+    MOCK_CONST_METHOD1(writeStateToHWC, void(bool));
+
     MOCK_CONST_METHOD1(dump, void(std::string&));
 };
 
diff --git a/services/surfaceflinger/CompositionEngine/src/LayerCompositionState.cpp b/services/surfaceflinger/CompositionEngine/src/LayerCompositionState.cpp
index 517b641..40c4da9 100644
--- a/services/surfaceflinger/CompositionEngine/src/LayerCompositionState.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/LayerCompositionState.cpp
@@ -31,6 +31,25 @@
 
 void dumpFrontEnd(std::string& out, const LayerFECompositionState& state) {
     out.append("      ");
+    dumpVal(out, "isSecure", state.isSecure);
+    dumpVal(out, "geomUsesSourceCrop", state.geomUsesSourceCrop);
+    dumpVal(out, "geomBufferUsesDisplayInverseTransform",
+            state.geomBufferUsesDisplayInverseTransform);
+    dumpVal(out, "geomLayerTransform", state.geomLayerTransform);
+
+    out.append("\n      ");
+    dumpVal(out, "geomBufferSize", state.geomBufferSize);
+    dumpVal(out, "geomContentCrop", state.geomContentCrop);
+    dumpVal(out, "geomCrop", state.geomCrop);
+    dumpVal(out, "geomBufferTransform", state.geomBufferTransform);
+
+    out.append("\n      ");
+    dumpVal(out, "geomActiveTransparentRegion", state.geomActiveTransparentRegion);
+
+    out.append("      ");
+    dumpVal(out, "geomLayerBounds", state.geomLayerBounds);
+
+    out.append("\n      ");
     dumpVal(out, "blend", toString(state.blendMode), state.blendMode);
     dumpVal(out, "alpha", state.alpha);
 
@@ -61,7 +80,7 @@
 } // namespace
 
 void LayerCompositionState::dump(std::string& out) const {
-    out.append("      frontend:\n");
+    out.append("    frontend:\n");
     dumpFrontEnd(out, frontEnd);
 }
 
diff --git a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
index 10da49d..13485b4 100644
--- a/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/OutputLayer.cpp
@@ -19,7 +19,10 @@
 #include <compositionengine/Layer.h>
 #include <compositionengine/LayerFE.h>
 #include <compositionengine/Output.h>
+#include <compositionengine/impl/LayerCompositionState.h>
+#include <compositionengine/impl/OutputCompositionState.h>
 #include <compositionengine/impl/OutputLayer.h>
+#include <compositionengine/impl/OutputLayerCompositionState.h>
 
 #include "DisplayHardware/HWComposer.h"
 
@@ -29,6 +32,18 @@
 
 namespace impl {
 
+namespace {
+
+FloatRect reduce(const FloatRect& win, const Region& exclude) {
+    if (CC_LIKELY(exclude.isEmpty())) {
+        return win;
+    }
+    // Convert through Rect (by rounding) for lack of FloatRegion
+    return Region(Rect{win}).subtract(exclude).getBounds().toFloatRect();
+}
+
+} // namespace
+
 std::unique_ptr<compositionengine::OutputLayer> createOutputLayer(
         const CompositionEngine& compositionEngine, std::optional<DisplayId> displayId,
         const compositionengine::Output& output, std::shared_ptr<compositionengine::Layer> layer,
@@ -77,6 +92,290 @@
     return mState;
 }
 
+Rect OutputLayer::calculateInitialCrop() const {
+    const auto& layerState = mLayer->getState().frontEnd;
+
+    // apply the projection's clipping to the window crop in
+    // layerstack space, and convert-back to layer space.
+    // if there are no window scaling involved, this operation will map to full
+    // pixels in the buffer.
+
+    FloatRect activeCropFloat =
+            reduce(layerState.geomLayerBounds, layerState.geomActiveTransparentRegion);
+
+    const Rect& viewport = mOutput.getState().viewport;
+    const ui::Transform& layerTransform = layerState.geomLayerTransform;
+    const ui::Transform& inverseLayerTransform = layerState.geomInverseLayerTransform;
+    // Transform to screen space.
+    activeCropFloat = layerTransform.transform(activeCropFloat);
+    activeCropFloat = activeCropFloat.intersect(viewport.toFloatRect());
+    // Back to layer space to work with the content crop.
+    activeCropFloat = inverseLayerTransform.transform(activeCropFloat);
+
+    // This needs to be here as transform.transform(Rect) computes the
+    // transformed rect and then takes the bounding box of the result before
+    // returning. This means
+    // transform.inverse().transform(transform.transform(Rect)) != Rect
+    // in which case we need to make sure the final rect is clipped to the
+    // display bounds.
+    Rect activeCrop{activeCropFloat};
+    if (!activeCrop.intersect(layerState.geomBufferSize, &activeCrop)) {
+        activeCrop.clear();
+    }
+    return activeCrop;
+}
+
+FloatRect OutputLayer::calculateOutputSourceCrop() const {
+    const auto& layerState = mLayer->getState().frontEnd;
+    const auto& outputState = mOutput.getState();
+
+    if (!layerState.geomUsesSourceCrop) {
+        return {};
+    }
+
+    // the content crop is the area of the content that gets scaled to the
+    // layer's size. This is in buffer space.
+    FloatRect crop = layerState.geomContentCrop.toFloatRect();
+
+    // In addition there is a WM-specified crop we pull from our drawing state.
+    Rect activeCrop = calculateInitialCrop();
+    const Rect& bufferSize = layerState.geomBufferSize;
+
+    int winWidth = bufferSize.getWidth();
+    int winHeight = bufferSize.getHeight();
+
+    // The bufferSize for buffer state layers can be unbounded ([0, 0, -1, -1])
+    // if display frame hasn't been set and the parent is an unbounded layer.
+    if (winWidth < 0 && winHeight < 0) {
+        return crop;
+    }
+
+    // Transform the window crop to match the buffer coordinate system,
+    // which means using the inverse of the current transform set on the
+    // SurfaceFlingerConsumer.
+    uint32_t invTransform = layerState.geomBufferTransform;
+    if (layerState.geomBufferUsesDisplayInverseTransform) {
+        /*
+         * the code below applies the primary display's inverse transform to the
+         * buffer
+         */
+        uint32_t invTransformOrient = outputState.orientation;
+        // calculate the inverse transform
+        if (invTransformOrient & HAL_TRANSFORM_ROT_90) {
+            invTransformOrient ^= HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_FLIP_H;
+        }
+        // and apply to the current transform
+        invTransform =
+                (ui::Transform(invTransformOrient) * ui::Transform(invTransform)).getOrientation();
+    }
+
+    if (invTransform & HAL_TRANSFORM_ROT_90) {
+        // If the activeCrop has been rotate the ends are rotated but not
+        // the space itself so when transforming ends back we can't rely on
+        // a modification of the axes of rotation. To account for this we
+        // need to reorient the inverse rotation in terms of the current
+        // axes of rotation.
+        bool is_h_flipped = (invTransform & HAL_TRANSFORM_FLIP_H) != 0;
+        bool is_v_flipped = (invTransform & HAL_TRANSFORM_FLIP_V) != 0;
+        if (is_h_flipped == is_v_flipped) {
+            invTransform ^= HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_FLIP_H;
+        }
+        std::swap(winWidth, winHeight);
+    }
+    const Rect winCrop =
+            activeCrop.transform(invTransform, bufferSize.getWidth(), bufferSize.getHeight());
+
+    // below, crop is intersected with winCrop expressed in crop's coordinate space
+    float xScale = crop.getWidth() / float(winWidth);
+    float yScale = crop.getHeight() / float(winHeight);
+
+    float insetL = winCrop.left * xScale;
+    float insetT = winCrop.top * yScale;
+    float insetR = (winWidth - winCrop.right) * xScale;
+    float insetB = (winHeight - winCrop.bottom) * yScale;
+
+    crop.left += insetL;
+    crop.top += insetT;
+    crop.right -= insetR;
+    crop.bottom -= insetB;
+
+    return crop;
+}
+
+Rect OutputLayer::calculateOutputDisplayFrame() const {
+    const auto& layerState = mLayer->getState().frontEnd;
+    const auto& outputState = mOutput.getState();
+
+    // apply the layer's transform, followed by the display's global transform
+    // here we're guaranteed that the layer's transform preserves rects
+    Region activeTransparentRegion = layerState.geomActiveTransparentRegion;
+    const ui::Transform& layerTransform = layerState.geomLayerTransform;
+    const ui::Transform& inverseLayerTransform = layerState.geomInverseLayerTransform;
+    const Rect& bufferSize = layerState.geomBufferSize;
+    Rect activeCrop = layerState.geomCrop;
+    if (!activeCrop.isEmpty() && bufferSize.isValid()) {
+        activeCrop = layerTransform.transform(activeCrop);
+        if (!activeCrop.intersect(outputState.viewport, &activeCrop)) {
+            activeCrop.clear();
+        }
+        activeCrop = inverseLayerTransform.transform(activeCrop, true);
+        // This needs to be here as transform.transform(Rect) computes the
+        // transformed rect and then takes the bounding box of the result before
+        // returning. This means
+        // transform.inverse().transform(transform.transform(Rect)) != Rect
+        // in which case we need to make sure the final rect is clipped to the
+        // display bounds.
+        if (!activeCrop.intersect(bufferSize, &activeCrop)) {
+            activeCrop.clear();
+        }
+        // mark regions outside the crop as transparent
+        activeTransparentRegion.orSelf(Rect(0, 0, bufferSize.getWidth(), activeCrop.top));
+        activeTransparentRegion.orSelf(
+                Rect(0, activeCrop.bottom, bufferSize.getWidth(), bufferSize.getHeight()));
+        activeTransparentRegion.orSelf(Rect(0, activeCrop.top, activeCrop.left, activeCrop.bottom));
+        activeTransparentRegion.orSelf(
+                Rect(activeCrop.right, activeCrop.top, bufferSize.getWidth(), activeCrop.bottom));
+    }
+
+    // reduce uses a FloatRect to provide more accuracy during the
+    // transformation. We then round upon constructing 'frame'.
+    Rect frame{
+            layerTransform.transform(reduce(layerState.geomLayerBounds, activeTransparentRegion))};
+    if (!frame.intersect(outputState.viewport, &frame)) {
+        frame.clear();
+    }
+    const ui::Transform displayTransform{outputState.transform};
+
+    return displayTransform.transform(frame);
+}
+
+uint32_t OutputLayer::calculateOutputRelativeBufferTransform() const {
+    const auto& layerState = mLayer->getState().frontEnd;
+    const auto& outputState = mOutput.getState();
+
+    /*
+     * Transformations are applied in this order:
+     * 1) buffer orientation/flip/mirror
+     * 2) state transformation (window manager)
+     * 3) layer orientation (screen orientation)
+     * (NOTE: the matrices are multiplied in reverse order)
+     */
+    const ui::Transform& layerTransform = layerState.geomLayerTransform;
+    const ui::Transform displayTransform{outputState.orientation};
+    const ui::Transform bufferTransform{layerState.geomBufferTransform};
+    ui::Transform transform(displayTransform * layerTransform * bufferTransform);
+
+    if (layerState.geomBufferUsesDisplayInverseTransform) {
+        /*
+         * the code below applies the primary display's inverse transform to the
+         * buffer
+         */
+        uint32_t invTransform = outputState.orientation;
+        // calculate the inverse transform
+        if (invTransform & HAL_TRANSFORM_ROT_90) {
+            invTransform ^= HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_FLIP_H;
+        }
+
+        /*
+         * Here we cancel out the orientation component of the WM transform.
+         * The scaling and translate components are already included in our bounds
+         * computation so it's enough to just omit it in the composition.
+         * See comment in BufferLayer::prepareClientLayer with ref to b/36727915 for why.
+         */
+        transform = ui::Transform(invTransform) * displayTransform * bufferTransform;
+    }
+
+    // this gives us only the "orientation" component of the transform
+    return transform.getOrientation();
+} // namespace impl
+
+void OutputLayer::updateCompositionState(bool includeGeometry) {
+    if (includeGeometry) {
+        mState.displayFrame = calculateOutputDisplayFrame();
+        mState.sourceCrop = calculateOutputSourceCrop();
+        mState.bufferTransform =
+                static_cast<Hwc2::Transform>(calculateOutputRelativeBufferTransform());
+
+        if ((mLayer->getState().frontEnd.isSecure && !mOutput.getState().isSecure) ||
+            (mState.bufferTransform & ui::Transform::ROT_INVALID)) {
+            mState.forceClientComposition = true;
+        }
+    }
+}
+
+void OutputLayer::writeStateToHWC(bool includeGeometry) const {
+    // Skip doing this if there is no HWC interface
+    if (!mState.hwc) {
+        return;
+    }
+
+    auto& hwcLayer = (*mState.hwc).hwcLayer;
+    if (!hwcLayer) {
+        ALOGE("[%s] failed to write composition state to HWC -- no hwcLayer for output %s",
+              mLayerFE->getDebugName(), mOutput.getName().c_str());
+        return;
+    }
+
+    if (includeGeometry) {
+        // Output dependent state
+
+        if (auto error = hwcLayer->setDisplayFrame(mState.displayFrame);
+            error != HWC2::Error::None) {
+            ALOGE("[%s] Failed to set display frame [%d, %d, %d, %d]: %s (%d)",
+                  mLayerFE->getDebugName(), mState.displayFrame.left, mState.displayFrame.top,
+                  mState.displayFrame.right, mState.displayFrame.bottom, to_string(error).c_str(),
+                  static_cast<int32_t>(error));
+        }
+
+        if (auto error = hwcLayer->setSourceCrop(mState.sourceCrop); error != HWC2::Error::None) {
+            ALOGE("[%s] Failed to set source crop [%.3f, %.3f, %.3f, %.3f]: "
+                  "%s (%d)",
+                  mLayerFE->getDebugName(), mState.sourceCrop.left, mState.sourceCrop.top,
+                  mState.sourceCrop.right, mState.sourceCrop.bottom, to_string(error).c_str(),
+                  static_cast<int32_t>(error));
+        }
+
+        if (auto error = hwcLayer->setZOrder(mState.z); error != HWC2::Error::None) {
+            ALOGE("[%s] Failed to set Z %u: %s (%d)", mLayerFE->getDebugName(), mState.z,
+                  to_string(error).c_str(), static_cast<int32_t>(error));
+        }
+
+        if (auto error =
+                    hwcLayer->setTransform(static_cast<HWC2::Transform>(mState.bufferTransform));
+            error != HWC2::Error::None) {
+            ALOGE("[%s] Failed to set transform %s: %s (%d)", mLayerFE->getDebugName(),
+                  toString(mState.bufferTransform).c_str(), to_string(error).c_str(),
+                  static_cast<int32_t>(error));
+        }
+
+        // Output independent state
+
+        const auto& outputIndependentState = mLayer->getState().frontEnd;
+
+        if (auto error = hwcLayer->setBlendMode(
+                    static_cast<HWC2::BlendMode>(outputIndependentState.blendMode));
+            error != HWC2::Error::None) {
+            ALOGE("[%s] Failed to set blend mode %s: %s (%d)", mLayerFE->getDebugName(),
+                  toString(outputIndependentState.blendMode).c_str(), to_string(error).c_str(),
+                  static_cast<int32_t>(error));
+        }
+
+        if (auto error = hwcLayer->setPlaneAlpha(outputIndependentState.alpha);
+            error != HWC2::Error::None) {
+            ALOGE("[%s] Failed to set plane alpha %.3f: %s (%d)", mLayerFE->getDebugName(),
+                  outputIndependentState.alpha, to_string(error).c_str(),
+                  static_cast<int32_t>(error));
+        }
+
+        if (auto error =
+                    hwcLayer->setInfo(outputIndependentState.type, outputIndependentState.appId);
+            error != HWC2::Error::None) {
+            ALOGE("[%s] Failed to set info %s (%d)", mLayerFE->getDebugName(),
+                  to_string(error).c_str(), static_cast<int32_t>(error));
+        }
+    }
+}
+
 void OutputLayer::dump(std::string& out) const {
     using android::base::StringAppendF;
 
diff --git a/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp b/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp
index f7dcf6f..2060c5a 100644
--- a/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp
+++ b/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp
@@ -23,16 +23,37 @@
 
 #include "MockHWC2.h"
 #include "MockHWComposer.h"
+#include "RectMatcher.h"
 
 namespace android::compositionengine {
 namespace {
 
+using testing::_;
+using testing::Return;
+using testing::ReturnRef;
 using testing::StrictMock;
 
 constexpr DisplayId DEFAULT_DISPLAY_ID = DisplayId{42};
 
+constexpr auto TR_IDENT = 0u;
+constexpr auto TR_FLP_H = HAL_TRANSFORM_FLIP_H;
+constexpr auto TR_FLP_V = HAL_TRANSFORM_FLIP_V;
+constexpr auto TR_ROT_90 = HAL_TRANSFORM_ROT_90;
+constexpr auto TR_ROT_180 = TR_FLP_H | TR_FLP_V;
+constexpr auto TR_ROT_270 = TR_ROT_90 | TR_ROT_180;
+
+const std::string kOutputName{"Test Output"};
+
 class OutputLayerTest : public testing::Test {
 public:
+    OutputLayerTest() {
+        EXPECT_CALL(*mLayerFE, getDebugName()).WillRepeatedly(Return("Test LayerFE"));
+        EXPECT_CALL(mOutput, getName()).WillRepeatedly(ReturnRef(kOutputName));
+
+        EXPECT_CALL(*mLayer, getState()).WillRepeatedly(ReturnRef(mLayerState));
+        EXPECT_CALL(mOutput, getState()).WillRepeatedly(ReturnRef(mOutputState));
+    }
+
     ~OutputLayerTest() override = default;
 
     compositionengine::mock::Output mOutput;
@@ -41,15 +62,18 @@
     sp<compositionengine::mock::LayerFE> mLayerFE{
             new StrictMock<compositionengine::mock::LayerFE>()};
     impl::OutputLayer mOutputLayer{mOutput, mLayer, mLayerFE};
+
+    impl::LayerCompositionState mLayerState;
+    impl::OutputCompositionState mOutputState;
 };
 
-/* ------------------------------------------------------------------------
+/*
  * Basic construction
  */
 
 TEST_F(OutputLayerTest, canInstantiateOutputLayer) {}
 
-/* ------------------------------------------------------------------------
+/*
  * OutputLayer::initialize()
  */
 
@@ -71,15 +95,220 @@
 
     mOutputLayer.initialize(compositionEngine, DEFAULT_DISPLAY_ID);
 
-    const auto& state = mOutputLayer.getState();
-    ASSERT_TRUE(state.hwc);
+    const auto& outputLayerState = mOutputLayer.getState();
+    ASSERT_TRUE(outputLayerState.hwc);
 
-    const auto& hwcState = *state.hwc;
+    const auto& hwcState = *outputLayerState.hwc;
     EXPECT_EQ(&hwcLayer, hwcState.hwcLayer.get());
 
     EXPECT_CALL(hwc, destroyLayer(DEFAULT_DISPLAY_ID, &hwcLayer));
     mOutputLayer.editState().hwc.reset();
 }
 
+/*
+ * OutputLayer::calculateOutputDisplayFrame()
+ */
+
+struct OutputLayerDisplayFrameTest : public OutputLayerTest {
+    OutputLayerDisplayFrameTest() {
+        // Set reasonable default values for a simple case. Each test will
+        // set one specific value to something different.
+
+        mLayerState.frontEnd.geomActiveTransparentRegion = Region{};
+        mLayerState.frontEnd.geomLayerTransform = ui::Transform{TR_IDENT};
+        mLayerState.frontEnd.geomBufferSize = Rect{0, 0, 1920, 1080};
+        mLayerState.frontEnd.geomBufferUsesDisplayInverseTransform = false;
+        mLayerState.frontEnd.geomCrop = Rect{0, 0, 1920, 1080};
+        mLayerState.frontEnd.geomLayerBounds = FloatRect{0.f, 0.f, 1920.f, 1080.f};
+
+        mOutputState.viewport = Rect{0, 0, 1920, 1080};
+        mOutputState.transform = ui::Transform{TR_IDENT};
+    }
+
+    Rect calculateOutputDisplayFrame() {
+        mLayerState.frontEnd.geomInverseLayerTransform =
+                mLayerState.frontEnd.geomLayerTransform.inverse();
+
+        return mOutputLayer.calculateOutputDisplayFrame();
+    }
+};
+
+TEST_F(OutputLayerDisplayFrameTest, correctForSimpleDefaultCase) {
+    const Rect expected{0, 0, 1920, 1080};
+    EXPECT_THAT(calculateOutputDisplayFrame(), RectEq(expected));
+}
+
+TEST_F(OutputLayerDisplayFrameTest, fullActiveTransparentRegionReturnsEmptyFrame) {
+    mLayerState.frontEnd.geomActiveTransparentRegion = Region{Rect{0, 0, 1920, 1080}};
+    const Rect expected{0, 0, 0, 0};
+    EXPECT_THAT(calculateOutputDisplayFrame(), RectEq(expected));
+}
+
+TEST_F(OutputLayerDisplayFrameTest, cropAffectsDisplayFrame) {
+    mLayerState.frontEnd.geomCrop = Rect{100, 200, 300, 500};
+    const Rect expected{100, 200, 300, 500};
+    EXPECT_THAT(calculateOutputDisplayFrame(), RectEq(expected));
+}
+
+TEST_F(OutputLayerDisplayFrameTest, cropAffectsDisplayFrameRotated) {
+    mLayerState.frontEnd.geomCrop = Rect{100, 200, 300, 500};
+    mLayerState.frontEnd.geomLayerTransform.set(HAL_TRANSFORM_ROT_90, 1920, 1080);
+    const Rect expected{1420, 100, 1720, 300};
+    EXPECT_THAT(calculateOutputDisplayFrame(), RectEq(expected));
+}
+
+TEST_F(OutputLayerDisplayFrameTest, emptyGeomCropIsNotUsedToComputeFrame) {
+    mLayerState.frontEnd.geomCrop = Rect{};
+    const Rect expected{0, 0, 1920, 1080};
+    EXPECT_THAT(calculateOutputDisplayFrame(), RectEq(expected));
+}
+
+TEST_F(OutputLayerDisplayFrameTest, geomLayerSnapToBoundsAffectsFrame) {
+    mLayerState.frontEnd.geomLayerBounds = FloatRect{0.f, 0.f, 960.f, 540.f};
+    const Rect expected{0, 0, 960, 540};
+    EXPECT_THAT(calculateOutputDisplayFrame(), RectEq(expected));
+}
+
+TEST_F(OutputLayerDisplayFrameTest, viewportAffectsFrame) {
+    mOutputState.viewport = Rect{0, 0, 960, 540};
+    const Rect expected{0, 0, 960, 540};
+    EXPECT_THAT(calculateOutputDisplayFrame(), RectEq(expected));
+}
+
+TEST_F(OutputLayerDisplayFrameTest, outputTransformAffectsDisplayFrame) {
+    mOutputState.transform = ui::Transform{HAL_TRANSFORM_ROT_90};
+    const Rect expected{-1080, 0, 0, 1920};
+    EXPECT_THAT(calculateOutputDisplayFrame(), RectEq(expected));
+}
+
+/*
+ * OutputLayer::calculateOutputRelativeBufferTransform()
+ */
+
+TEST_F(OutputLayerTest, calculateOutputRelativeBufferTransformTestsNeeded) {
+    mLayerState.frontEnd.geomBufferUsesDisplayInverseTransform = false;
+
+    struct Entry {
+        uint32_t layer;
+        uint32_t buffer;
+        uint32_t display;
+        uint32_t expected;
+    };
+    // Not an exhaustive list of cases, but hopefully enough.
+    const std::array<Entry, 24> testData = {
+            // clang-format off
+            //             layer       buffer      display     expected
+            /*  0 */ Entry{TR_IDENT,   TR_IDENT,   TR_IDENT,   TR_IDENT},
+            /*  1 */ Entry{TR_IDENT,   TR_IDENT,   TR_ROT_90,  TR_ROT_90},
+            /*  2 */ Entry{TR_IDENT,   TR_IDENT,   TR_ROT_180, TR_ROT_180},
+            /*  3 */ Entry{TR_IDENT,   TR_IDENT,   TR_ROT_270, TR_ROT_270},
+
+            /*  4 */ Entry{TR_IDENT,   TR_FLP_H,   TR_IDENT,   TR_FLP_H ^ TR_IDENT},
+            /*  5 */ Entry{TR_IDENT,   TR_FLP_H,   TR_ROT_90,  TR_FLP_H ^ TR_ROT_90},
+            /*  6 */ Entry{TR_IDENT,   TR_FLP_H,   TR_ROT_180, TR_FLP_H ^ TR_ROT_180},
+            /*  7 */ Entry{TR_IDENT,   TR_FLP_H,   TR_ROT_270, TR_FLP_H ^ TR_ROT_270},
+
+            /*  8 */ Entry{TR_IDENT,   TR_FLP_V,   TR_IDENT,   TR_FLP_V},
+            /*  9 */ Entry{TR_IDENT,   TR_ROT_90,  TR_ROT_90,  TR_ROT_180},
+            /* 10 */ Entry{TR_IDENT,   TR_ROT_180, TR_ROT_180, TR_IDENT},
+            /* 11 */ Entry{TR_IDENT,   TR_ROT_270, TR_ROT_270, TR_ROT_180},
+
+            /* 12 */ Entry{TR_ROT_90,  TR_IDENT,   TR_IDENT,   TR_IDENT ^ TR_ROT_90},
+            /* 13 */ Entry{TR_ROT_90,  TR_FLP_H,   TR_ROT_90,  TR_FLP_H ^ TR_ROT_180},
+            /* 14 */ Entry{TR_ROT_90,  TR_IDENT,   TR_ROT_180, TR_IDENT ^ TR_ROT_270},
+            /* 15 */ Entry{TR_ROT_90,  TR_FLP_H,   TR_ROT_270, TR_FLP_H ^ TR_IDENT},
+
+            /* 16 */ Entry{TR_ROT_180, TR_FLP_H,   TR_IDENT,   TR_FLP_H ^ TR_ROT_180},
+            /* 17 */ Entry{TR_ROT_180, TR_IDENT,   TR_ROT_90,  TR_IDENT ^ TR_ROT_270},
+            /* 18 */ Entry{TR_ROT_180, TR_FLP_H,   TR_ROT_180, TR_FLP_H ^ TR_IDENT},
+            /* 19 */ Entry{TR_ROT_180, TR_IDENT,   TR_ROT_270, TR_IDENT ^ TR_ROT_90},
+
+            /* 20 */ Entry{TR_ROT_270, TR_IDENT,   TR_IDENT,   TR_IDENT ^ TR_ROT_270},
+            /* 21 */ Entry{TR_ROT_270, TR_FLP_H,   TR_ROT_90,  TR_FLP_H ^ TR_IDENT},
+            /* 22 */ Entry{TR_ROT_270, TR_FLP_H,   TR_ROT_180, TR_FLP_H ^ TR_ROT_90},
+            /* 23 */ Entry{TR_ROT_270, TR_IDENT,   TR_ROT_270, TR_IDENT ^ TR_ROT_180},
+            // clang-format on
+    };
+
+    for (size_t i = 0; i < testData.size(); i++) {
+        const auto& entry = testData[i];
+
+        mLayerState.frontEnd.geomLayerTransform.set(entry.layer, 1920, 1080);
+        mLayerState.frontEnd.geomBufferTransform = entry.buffer;
+        mOutputState.orientation = entry.display;
+
+        auto actual = mOutputLayer.calculateOutputRelativeBufferTransform();
+        EXPECT_EQ(entry.expected, actual) << "entry " << i;
+    }
+}
+
+/*
+ * OutputLayer::writeStateToHWC()
+ */
+
+struct OutputLayerWriteStateToHWCTest : public OutputLayerTest {
+    static constexpr HWC2::Error kError = HWC2::Error::Unsupported;
+    static constexpr FloatRect kSourceCrop{11.f, 12.f, 13.f, 14.f};
+    static constexpr uint32_t kZOrder = 21u;
+    static constexpr Hwc2::Transform kBufferTransform = static_cast<Hwc2::Transform>(31);
+    static constexpr Hwc2::IComposerClient::BlendMode kBlendMode =
+            static_cast<Hwc2::IComposerClient::BlendMode>(41);
+    static constexpr float kAlpha = 51.f;
+    static constexpr uint32_t kType = 61u;
+    static constexpr uint32_t kAppId = 62u;
+
+    static const Rect kDisplayFrame;
+
+    OutputLayerWriteStateToHWCTest() {
+        auto& outputLayerState = mOutputLayer.editState();
+        outputLayerState.hwc = impl::OutputLayerCompositionState::Hwc(mHwcLayer);
+
+        outputLayerState.displayFrame = kDisplayFrame;
+        outputLayerState.sourceCrop = kSourceCrop;
+        outputLayerState.z = kZOrder;
+        outputLayerState.bufferTransform = static_cast<Hwc2::Transform>(kBufferTransform);
+
+        mLayerState.frontEnd.blendMode = kBlendMode;
+        mLayerState.frontEnd.alpha = kAlpha;
+        mLayerState.frontEnd.type = kType;
+        mLayerState.frontEnd.appId = kAppId;
+    }
+
+    void expectGeometryCommonCalls() {
+        EXPECT_CALL(*mHwcLayer, setDisplayFrame(kDisplayFrame)).WillOnce(Return(kError));
+        EXPECT_CALL(*mHwcLayer, setSourceCrop(kSourceCrop)).WillOnce(Return(kError));
+        EXPECT_CALL(*mHwcLayer, setZOrder(kZOrder)).WillOnce(Return(kError));
+        EXPECT_CALL(*mHwcLayer, setTransform(static_cast<HWC2::Transform>(kBufferTransform)))
+                .WillOnce(Return(kError));
+
+        EXPECT_CALL(*mHwcLayer, setBlendMode(static_cast<HWC2::BlendMode>(kBlendMode)))
+                .WillOnce(Return(kError));
+        EXPECT_CALL(*mHwcLayer, setPlaneAlpha(kAlpha)).WillOnce(Return(kError));
+        EXPECT_CALL(*mHwcLayer, setInfo(kType, kAppId)).WillOnce(Return(kError));
+    }
+
+    std::shared_ptr<HWC2::mock::Layer> mHwcLayer{std::make_shared<StrictMock<HWC2::mock::Layer>>()};
+};
+
+const Rect OutputLayerWriteStateToHWCTest::kDisplayFrame{1001, 1002, 1003, 10044};
+
+TEST_F(OutputLayerWriteStateToHWCTest, doesNothingIfNoHWCState) {
+    mOutputLayer.editState().hwc.reset();
+
+    mOutputLayer.writeStateToHWC(true);
+}
+
+TEST_F(OutputLayerWriteStateToHWCTest, doesNothingIfNoHWCLayer) {
+    mOutputLayer.editState().hwc = impl::OutputLayerCompositionState::Hwc(nullptr);
+
+    mOutputLayer.writeStateToHWC(true);
+}
+
+TEST_F(OutputLayerWriteStateToHWCTest, canSetsAllState) {
+    expectGeometryCommonCalls();
+
+    mOutputLayer.writeStateToHWC(true);
+}
+
 } // namespace
 } // namespace android::compositionengine
diff --git a/services/surfaceflinger/CompositionEngine/tests/RectMatcher.h b/services/surfaceflinger/CompositionEngine/tests/RectMatcher.h
new file mode 100644
index 0000000..d4c76bc
--- /dev/null
+++ b/services/surfaceflinger/CompositionEngine/tests/RectMatcher.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+#include <android-base/stringprintf.h>
+#include <gmock/gmock.h>
+
+namespace {
+
+using android::base::StringAppendF;
+using Rect = android::Rect;
+
+void dumpRect(const Rect& rect, std::string& result, const char* name) {
+    StringAppendF(&result, "%s (%d %d %d %d) ", name, rect.left, rect.top, rect.right, rect.bottom);
+}
+
+// Checks for a region match
+MATCHER_P(RectEq, expected, "") {
+    std::string buf;
+    buf.append("Rects are not equal\n");
+    dumpRect(expected, buf, "expected rect");
+    dumpRect(arg, buf, "actual rect");
+    *result_listener << buf;
+
+    return (expected.left == arg.left) && (expected.top == arg.top) &&
+            (expected.right == arg.right) && (expected.bottom == arg.bottom);
+}
+
+} // namespace