commit 35f0dda298dbce279fd08103142b2f4b34633d65
author Robert Carr <racarr@google.com> Thu May 03 15:47:23 2018 -0700
committer Robert Carr <racarr@google.com> Fri May 11 15:05:26 2018 -0700
tree a8c582ec7c46c10bbcc8788990cfaa227f261262
parent 2da6d8498a8af32e93c24adf480e60427d0f9171

SurfaceFlinger: Factor transformToDisplayInverse in-to buffer rejection

With transform to display inverse, like other transforms, we have a buffer
of one size rotated to fit geometry of another size. We haven't needed this
in the past because the only user of TransformToDisplayInverse was also using
NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW.

Bug: 74400997
Test: Manual
Change-Id: I118e802ff6ce6aca152620bde5d6d7ba890ecaa5

services/surfaceflinger/LayerRejecter.cpp

diff --git a/services/surfaceflinger/LayerRejecter.cpp b/services/surfaceflinger/LayerRejecter.cpp
index a5f0b98..381ea4a 100644
--- a/services/surfaceflinger/LayerRejecter.cpp
+++ b/services/surfaceflinger/LayerRejecter.cpp
@@ -31,6 +31,7 @@
                              bool stickySet,
                              const char* name,
                              int32_t overrideScalingMode,
+                             bool transformToDisplayInverse,
                              bool& freezePositionUpdates)
   : mFront(front),
     mCurrent(current),
@@ -38,6 +39,7 @@
     mStickyTransformSet(stickySet),
     mName(name),
     mOverrideScalingMode(overrideScalingMode),
+    mTransformToDisplayInverse(transformToDisplayInverse),
     mFreezeGeometryUpdates(freezePositionUpdates) {}
 
 bool LayerRejecter::reject(const sp<GraphicBuffer>& buf, const BufferItem& item) {
@@ -54,6 +56,13 @@
         swap(bufWidth, bufHeight);
     }
 
+    if (mTransformToDisplayInverse) {
+        uint32_t invTransform = DisplayDevice::getPrimaryDisplayOrientationTransform();
+        if (invTransform & Transform::ROT_90) {
+            swap(bufWidth, bufHeight);
+        }
+    }
+
     int actualScalingMode = mOverrideScalingMode >= 0 ? mOverrideScalingMode : item.mScalingMode;
     bool isFixedSize = actualScalingMode != NATIVE_WINDOW_SCALING_MODE_FREEZE;
     if (mFront.active != mFront.requested) {