CompositionEngine: base inverse transform on primary display

Layers with geomBufferUsesDisplayInverseTransform set (usually layers
showing a camera preview) were not being rotated correctly on external
displays with hardware composition. Forcing GPU composition avoided
the issue, since BufferLayer.prepareClientLayer() uses the *primary*
display's orientation to create the inverse transformation.

But, the HWC path used each screen's orientation, resulting in a bad
layer rotation when the primary and external displays' orientation
differed. So let's do what the GPU path does.

This, of course, only makes a difference when the primary and secondary
display rotations differ. In our case, we noticed the problem when
running the landscape-oriented CinemaPro app, while our default camera,
which runs in portrait, did not have a problem.

Co-authored-by: Mikael Magnusson <mikael.magnusson@sony.com>

Note: This is an updated version of the change with the same Change-Id
as uploaded to AOSP, as refactoring has continued, and more tests were
added since.

Bug: 155329360
Test: the new unit test fails without the patch
Test: the new unit test passes with the patch
Test: record using Cinema Pro app with an external screen
Reference: I0da22423490a93fe943fd59e6c122aa6aaf30b11
Change-Id: I8b4975a14a0ae42d10e4eeaa66385e72ff00c23d
diff --git a/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp b/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp
index bdacb22..020f93a 100644
--- a/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp
+++ b/services/surfaceflinger/CompositionEngine/tests/OutputLayerTest.cpp
@@ -361,7 +361,7 @@
         mOutputState.orientation = entry.display;
         mOutputState.transform = ui::Transform{entry.display};
 
-        auto actual = mOutputLayer.calculateOutputRelativeBufferTransform();
+        const auto actual = mOutputLayer.calculateOutputRelativeBufferTransform(entry.display);
         EXPECT_EQ(entry.expected, actual) << "entry " << i;
     }
 }
@@ -371,56 +371,109 @@
     mLayerFEState.geomBufferUsesDisplayInverseTransform = true;
 
     struct Entry {
-        uint32_t layer;
+        uint32_t layer; /* shouldn't affect the result, so we just use arbitrary values */
         uint32_t buffer;
         uint32_t display;
+        uint32_t internal;
         uint32_t expected;
     };
-    // Not an exhaustive list of cases, but hopefully enough.
-    const std::array<Entry, 24> testData = {
+    const std::array<Entry, 64> testData = {
             // clang-format off
-            //             layer       buffer      display     expected
-            /*  0 */ Entry{TR_IDENT,   TR_IDENT,   TR_IDENT,   TR_IDENT},
-            /*  1 */ Entry{TR_IDENT,   TR_IDENT,   TR_ROT_90,  TR_IDENT},
-            /*  2 */ Entry{TR_IDENT,   TR_IDENT,   TR_ROT_180, TR_IDENT},
-            /*  3 */ Entry{TR_IDENT,   TR_IDENT,   TR_ROT_270, TR_IDENT},
+            //    layer       buffer      display     internal    expected
+            Entry{TR_IDENT,   TR_IDENT,   TR_IDENT,   TR_IDENT,   TR_IDENT},
+            Entry{TR_IDENT,   TR_IDENT,   TR_IDENT,   TR_ROT_90,  TR_ROT_270},
+            Entry{TR_IDENT,   TR_IDENT,   TR_IDENT,   TR_ROT_180, TR_ROT_180},
+            Entry{TR_IDENT,   TR_IDENT,   TR_IDENT,   TR_ROT_270, TR_ROT_90},
 
-            /*  4 */ Entry{TR_IDENT,   TR_FLP_H,   TR_IDENT,   TR_FLP_H},
-            /*  5 */ Entry{TR_IDENT,   TR_FLP_H,   TR_ROT_90,  TR_FLP_H},
-            /*  6 */ Entry{TR_IDENT,   TR_FLP_H,   TR_ROT_180, TR_FLP_H},
-            /*  7 */ Entry{TR_IDENT,   TR_FLP_H,   TR_ROT_270, TR_FLP_H},
+            Entry{TR_IDENT,   TR_IDENT,   TR_ROT_90,  TR_IDENT,   TR_ROT_90},
+            Entry{TR_ROT_90,  TR_IDENT,   TR_ROT_90,  TR_ROT_90,  TR_IDENT},
+            Entry{TR_ROT_180, TR_IDENT,   TR_ROT_90,  TR_ROT_180, TR_ROT_270},
+            Entry{TR_ROT_90,  TR_IDENT,   TR_ROT_90,  TR_ROT_270, TR_ROT_180},
 
-            /*  8 */ Entry{TR_IDENT,   TR_FLP_V,   TR_IDENT,   TR_FLP_V},
-            /*  9 */ Entry{TR_IDENT,   TR_ROT_90,  TR_ROT_90,  TR_ROT_90},
-            /* 10 */ Entry{TR_IDENT,   TR_ROT_180, TR_ROT_180, TR_ROT_180},
-            /* 11 */ Entry{TR_IDENT,   TR_ROT_270, TR_ROT_270, TR_ROT_270},
+            Entry{TR_ROT_180, TR_IDENT,   TR_ROT_180, TR_IDENT,   TR_ROT_180},
+            Entry{TR_ROT_90,  TR_IDENT,   TR_ROT_180, TR_ROT_90,  TR_ROT_90},
+            Entry{TR_ROT_180, TR_IDENT,   TR_ROT_180, TR_ROT_180, TR_IDENT},
+            Entry{TR_ROT_270, TR_IDENT,   TR_ROT_180, TR_ROT_270, TR_ROT_270},
 
-            /* 12 */ Entry{TR_ROT_90,  TR_IDENT,   TR_IDENT,   TR_IDENT},
-            /* 13 */ Entry{TR_ROT_90,  TR_FLP_H,   TR_ROT_90,  TR_FLP_H},
-            /* 14 */ Entry{TR_ROT_90,  TR_IDENT,   TR_ROT_180, TR_IDENT},
-            /* 15 */ Entry{TR_ROT_90,  TR_FLP_H,   TR_ROT_270, TR_FLP_H},
+            Entry{TR_ROT_270, TR_IDENT,   TR_ROT_270, TR_IDENT,   TR_ROT_270},
+            Entry{TR_ROT_270, TR_IDENT,   TR_ROT_270, TR_ROT_90,  TR_ROT_180},
+            Entry{TR_ROT_180, TR_IDENT,   TR_ROT_270, TR_ROT_180, TR_ROT_90},
+            Entry{TR_IDENT,   TR_IDENT,   TR_ROT_270, TR_ROT_270, TR_IDENT},
 
-            /* 16 */ Entry{TR_ROT_180, TR_FLP_H,   TR_IDENT,   TR_FLP_H},
-            /* 17 */ Entry{TR_ROT_180, TR_IDENT,   TR_ROT_90,  TR_IDENT},
-            /* 18 */ Entry{TR_ROT_180, TR_FLP_H,   TR_ROT_180, TR_FLP_H},
-            /* 19 */ Entry{TR_ROT_180, TR_IDENT,   TR_ROT_270, TR_IDENT},
+            //    layer       buffer      display     internal    expected
+            Entry{TR_IDENT,   TR_ROT_90,  TR_IDENT,   TR_IDENT,   TR_ROT_90},
+            Entry{TR_ROT_90,  TR_ROT_90,  TR_IDENT,   TR_ROT_90,  TR_IDENT},
+            Entry{TR_ROT_180, TR_ROT_90,  TR_IDENT,   TR_ROT_180, TR_ROT_270},
+            Entry{TR_ROT_270, TR_ROT_90,  TR_IDENT,   TR_ROT_270, TR_ROT_180},
 
-            /* 20 */ Entry{TR_ROT_270, TR_IDENT,   TR_IDENT,   TR_IDENT},
-            /* 21 */ Entry{TR_ROT_270, TR_FLP_H,   TR_ROT_90,  TR_FLP_H},
-            /* 22 */ Entry{TR_ROT_270, TR_FLP_H,   TR_ROT_180, TR_FLP_H},
-            /* 23 */ Entry{TR_ROT_270, TR_IDENT,   TR_ROT_270, TR_IDENT},
+            Entry{TR_ROT_90,  TR_ROT_90,  TR_ROT_90,  TR_IDENT,   TR_ROT_180},
+            Entry{TR_ROT_90,  TR_ROT_90,  TR_ROT_90,  TR_ROT_90,  TR_ROT_90},
+            Entry{TR_ROT_90,  TR_ROT_90,  TR_ROT_90,  TR_ROT_180, TR_IDENT},
+            Entry{TR_ROT_270, TR_ROT_90,  TR_ROT_90,  TR_ROT_270, TR_ROT_270},
+
+            Entry{TR_IDENT,   TR_ROT_90,  TR_ROT_180, TR_IDENT,   TR_ROT_270},
+            Entry{TR_ROT_90,  TR_ROT_90,  TR_ROT_180, TR_ROT_90,  TR_ROT_180},
+            Entry{TR_ROT_180, TR_ROT_90,  TR_ROT_180, TR_ROT_180, TR_ROT_90},
+            Entry{TR_ROT_90,  TR_ROT_90,  TR_ROT_180, TR_ROT_270, TR_IDENT},
+
+            Entry{TR_IDENT,   TR_ROT_90,  TR_ROT_270, TR_IDENT,   TR_IDENT},
+            Entry{TR_ROT_270, TR_ROT_90,  TR_ROT_270, TR_ROT_90,  TR_ROT_270},
+            Entry{TR_ROT_180, TR_ROT_90,  TR_ROT_270, TR_ROT_180, TR_ROT_180},
+            Entry{TR_ROT_270, TR_ROT_90,  TR_ROT_270, TR_ROT_270, TR_ROT_90},
+
+            //    layer       buffer      display     internal    expected
+            Entry{TR_IDENT,   TR_ROT_180, TR_IDENT,   TR_IDENT,   TR_ROT_180},
+            Entry{TR_IDENT,   TR_ROT_180, TR_IDENT,   TR_ROT_90,  TR_ROT_90},
+            Entry{TR_ROT_180, TR_ROT_180, TR_IDENT,   TR_ROT_180, TR_IDENT},
+            Entry{TR_ROT_270, TR_ROT_180, TR_IDENT,   TR_ROT_270, TR_ROT_270},
+
+            Entry{TR_IDENT,   TR_ROT_180, TR_ROT_90,  TR_IDENT,   TR_ROT_270},
+            Entry{TR_ROT_90,  TR_ROT_180, TR_ROT_90,  TR_ROT_90,  TR_ROT_180},
+            Entry{TR_ROT_180, TR_ROT_180, TR_ROT_90,  TR_ROT_180, TR_ROT_90},
+            Entry{TR_ROT_180, TR_ROT_180, TR_ROT_90,  TR_ROT_270, TR_IDENT},
+
+            Entry{TR_IDENT,   TR_ROT_180, TR_ROT_180, TR_IDENT,   TR_IDENT},
+            Entry{TR_ROT_180, TR_ROT_180, TR_ROT_180, TR_ROT_90,  TR_ROT_270},
+            Entry{TR_ROT_180, TR_ROT_180, TR_ROT_180, TR_ROT_180, TR_ROT_180},
+            Entry{TR_ROT_270, TR_ROT_180, TR_ROT_180, TR_ROT_270, TR_ROT_90},
+
+            Entry{TR_ROT_270, TR_ROT_180, TR_ROT_270, TR_IDENT,   TR_ROT_90},
+            Entry{TR_ROT_180, TR_ROT_180, TR_ROT_270, TR_ROT_90,  TR_IDENT},
+            Entry{TR_ROT_180, TR_ROT_180, TR_ROT_270, TR_ROT_180, TR_ROT_270},
+            Entry{TR_ROT_270, TR_ROT_180, TR_ROT_270, TR_ROT_270, TR_ROT_180},
+
+            //    layer       buffer      display     internal    expected
+            Entry{TR_IDENT,   TR_ROT_270, TR_IDENT,   TR_IDENT,   TR_ROT_270},
+            Entry{TR_ROT_90,  TR_ROT_270, TR_IDENT,   TR_ROT_90,  TR_ROT_180},
+            Entry{TR_ROT_270, TR_ROT_270, TR_IDENT,   TR_ROT_180, TR_ROT_90},
+            Entry{TR_IDENT,   TR_ROT_270, TR_IDENT,   TR_ROT_270, TR_IDENT},
+
+            Entry{TR_ROT_270, TR_ROT_270, TR_ROT_90,  TR_IDENT,   TR_IDENT},
+            Entry{TR_ROT_90,  TR_ROT_270, TR_ROT_90,  TR_ROT_90,  TR_ROT_270},
+            Entry{TR_ROT_180, TR_ROT_270, TR_ROT_90,  TR_ROT_180, TR_ROT_180},
+            Entry{TR_ROT_90,  TR_ROT_270, TR_ROT_90,  TR_ROT_270, TR_ROT_90},
+
+            Entry{TR_IDENT,   TR_ROT_270, TR_ROT_180, TR_IDENT,   TR_ROT_90},
+            Entry{TR_ROT_270, TR_ROT_270, TR_ROT_180, TR_ROT_90,  TR_IDENT},
+            Entry{TR_ROT_180, TR_ROT_270, TR_ROT_180, TR_ROT_180, TR_ROT_270},
+            Entry{TR_ROT_270, TR_ROT_270, TR_ROT_180, TR_ROT_270, TR_ROT_180},
+
+            Entry{TR_IDENT,   TR_ROT_270, TR_ROT_270, TR_IDENT,   TR_ROT_180},
+            Entry{TR_ROT_90,  TR_ROT_270, TR_ROT_270, TR_ROT_90,  TR_ROT_90},
+            Entry{TR_ROT_270, TR_ROT_270, TR_ROT_270, TR_ROT_180, TR_IDENT},
+            Entry{TR_ROT_270, TR_ROT_270, TR_ROT_270, TR_ROT_270, TR_ROT_270},
             // clang-format on
     };
 
     for (size_t i = 0; i < testData.size(); i++) {
         const auto& entry = testData[i];
 
-        mLayerFEState.geomLayerTransform = ui::Transform{entry.layer};
+        mLayerFEState.geomLayerTransform.set(entry.layer, 1920, 1080);
         mLayerFEState.geomBufferTransform = entry.buffer;
         mOutputState.orientation = entry.display;
         mOutputState.transform = ui::Transform{entry.display};
 
-        auto actual = mOutputLayer.calculateOutputRelativeBufferTransform();
+        const auto actual = mOutputLayer.calculateOutputRelativeBufferTransform(entry.internal);
         EXPECT_EQ(entry.expected, actual) << "entry " << i;
     }
 }
@@ -436,7 +489,7 @@
     // Mock everything called by updateCompositionState to simplify testing it.
     MOCK_CONST_METHOD0(calculateOutputSourceCrop, FloatRect());
     MOCK_CONST_METHOD0(calculateOutputDisplayFrame, Rect());
-    MOCK_CONST_METHOD0(calculateOutputRelativeBufferTransform, uint32_t());
+    MOCK_CONST_METHOD1(calculateOutputRelativeBufferTransform, uint32_t(uint32_t));
 
     // compositionengine::OutputLayer overrides
     const compositionengine::Output& getOutput() const override { return mOutput; }
@@ -463,10 +516,11 @@
 
     ~OutputLayerUpdateCompositionStateTest() = default;
 
-    void setupGeometryChildCallValues() {
+    void setupGeometryChildCallValues(ui::Transform::RotationFlags internalDisplayRotationFlags) {
         EXPECT_CALL(mOutputLayer, calculateOutputSourceCrop()).WillOnce(Return(kSourceCrop));
         EXPECT_CALL(mOutputLayer, calculateOutputDisplayFrame()).WillOnce(Return(kDisplayFrame));
-        EXPECT_CALL(mOutputLayer, calculateOutputRelativeBufferTransform())
+        EXPECT_CALL(mOutputLayer,
+                    calculateOutputRelativeBufferTransform(internalDisplayRotationFlags))
                 .WillOnce(Return(mBufferTransform));
     }
 
@@ -489,7 +543,7 @@
 TEST_F(OutputLayerUpdateCompositionStateTest, doesNothingIfNoFECompositionState) {
     EXPECT_CALL(*mLayerFE, getCompositionState()).WillOnce(Return(nullptr));
 
-    mOutputLayer.updateCompositionState(true, false);
+    mOutputLayer.updateCompositionState(true, false, ui::Transform::RotationFlags::ROT_90);
 }
 
 TEST_F(OutputLayerUpdateCompositionStateTest, setsStateNormally) {
@@ -497,9 +551,9 @@
     mOutputState.isSecure = true;
     mOutputLayer.editState().forceClientComposition = true;
 
-    setupGeometryChildCallValues();
+    setupGeometryChildCallValues(ui::Transform::RotationFlags::ROT_90);
 
-    mOutputLayer.updateCompositionState(true, false);
+    mOutputLayer.updateCompositionState(true, false, ui::Transform::RotationFlags::ROT_90);
 
     validateComputedGeometryState();
 
@@ -511,9 +565,9 @@
     mLayerFEState.isSecure = true;
     mOutputState.isSecure = false;
 
-    setupGeometryChildCallValues();
+    setupGeometryChildCallValues(ui::Transform::RotationFlags::ROT_0);
 
-    mOutputLayer.updateCompositionState(true, false);
+    mOutputLayer.updateCompositionState(true, false, ui::Transform::RotationFlags::ROT_0);
 
     validateComputedGeometryState();
 
@@ -527,9 +581,9 @@
 
     mBufferTransform = ui::Transform::ROT_INVALID;
 
-    setupGeometryChildCallValues();
+    setupGeometryChildCallValues(ui::Transform::RotationFlags::ROT_0);
 
-    mOutputLayer.updateCompositionState(true, false);
+    mOutputLayer.updateCompositionState(true, false, ui::Transform::RotationFlags::ROT_0);
 
     validateComputedGeometryState();
 
@@ -544,7 +598,7 @@
     // should use the layers requested colorspace.
     mLayerFEState.isColorspaceAgnostic = false;
 
-    mOutputLayer.updateCompositionState(false, false);
+    mOutputLayer.updateCompositionState(false, false, ui::Transform::RotationFlags::ROT_0);
 
     EXPECT_EQ(ui::Dataspace::DISPLAY_P3, mOutputLayer.getState().dataspace);
 
@@ -552,7 +606,7 @@
     // should use the colorspace chosen for the whole output.
     mLayerFEState.isColorspaceAgnostic = true;
 
-    mOutputLayer.updateCompositionState(false, false);
+    mOutputLayer.updateCompositionState(false, false, ui::Transform::RotationFlags::ROT_0);
 
     EXPECT_EQ(ui::Dataspace::V0_SCRGB, mOutputLayer.getState().dataspace);
 }
@@ -560,7 +614,7 @@
 TEST_F(OutputLayerUpdateCompositionStateTest, doesNotRecomputeGeometryIfNotRequested) {
     mOutputLayer.editState().forceClientComposition = false;
 
-    mOutputLayer.updateCompositionState(false, false);
+    mOutputLayer.updateCompositionState(false, false, ui::Transform::RotationFlags::ROT_0);
 
     EXPECT_EQ(false, mOutputLayer.getState().forceClientComposition);
 }
@@ -569,7 +623,7 @@
        doesNotClearForceClientCompositionIfNotDoingGeometry) {
     mOutputLayer.editState().forceClientComposition = true;
 
-    mOutputLayer.updateCompositionState(false, false);
+    mOutputLayer.updateCompositionState(false, false, ui::Transform::RotationFlags::ROT_0);
 
     EXPECT_EQ(true, mOutputLayer.getState().forceClientComposition);
 }
@@ -578,7 +632,7 @@
     mLayerFEState.forceClientComposition = true;
     mOutputLayer.editState().forceClientComposition = false;
 
-    mOutputLayer.updateCompositionState(false, false);
+    mOutputLayer.updateCompositionState(false, false, ui::Transform::RotationFlags::ROT_0);
 
     EXPECT_EQ(true, mOutputLayer.getState().forceClientComposition);
 }
@@ -588,7 +642,7 @@
     mOutputLayer.editState().forceClientComposition = false;
     EXPECT_CALL(mDisplayColorProfile, isDataspaceSupported(_)).WillRepeatedly(Return(false));
 
-    mOutputLayer.updateCompositionState(false, false);
+    mOutputLayer.updateCompositionState(false, false, ui::Transform::RotationFlags::ROT_0);
 
     EXPECT_EQ(true, mOutputLayer.getState().forceClientComposition);
 }
@@ -597,15 +651,15 @@
     mLayerFEState.forceClientComposition = false;
     mOutputLayer.editState().forceClientComposition = false;
 
-    mOutputLayer.updateCompositionState(false, true);
+    mOutputLayer.updateCompositionState(false, true, ui::Transform::RotationFlags::ROT_0);
 
     EXPECT_EQ(true, mOutputLayer.getState().forceClientComposition);
 
     mOutputLayer.editState().forceClientComposition = false;
 
-    setupGeometryChildCallValues();
+    setupGeometryChildCallValues(ui::Transform::RotationFlags::ROT_0);
 
-    mOutputLayer.updateCompositionState(true, true);
+    mOutputLayer.updateCompositionState(true, true, ui::Transform::RotationFlags::ROT_0);
 
     EXPECT_EQ(true, mOutputLayer.getState().forceClientComposition);
 }
@@ -810,7 +864,7 @@
     // geomBufferTransform is set to the inverse transform.
     mLayerFEState.geomBufferTransform = TR_ROT_270;
 
-    EXPECT_EQ(TR_IDENT, mOutputLayer.calculateOutputRelativeBufferTransform());
+    EXPECT_EQ(TR_IDENT, mOutputLayer.calculateOutputRelativeBufferTransform(ui::Transform::ROT_90));
 }
 
 TEST_F(OutputLayerWriteStateToHWCTest, canSetPerFrameStateForSolidColor) {