surfaceflinger: use Vulkan coordinate system

In Vulkan coordinate system, (0, 0) always maps to memory address 0.
In GL coordinate system, (0, 0) maps to the bottom-left corner.  The
two agree when doing offscreen rendering.  But when rendering to a
window, GL maps (0, 0) to memory address (winHeight-1)*rowStride,
because that is where the bottom-left pixel is.

Let's pick the simpler one for RenderEngine in preparation for
multiple backends.  As a result, we don't need yswap when rendering
to offscreen buffers (it is hidden behind the RenderEngine API).  It
also makes no sense to flip Y coordinate for both vertex postions
and the viewing volume.

Test: chrome, youtube, camera, screen rotation, screenshots, recents
Change-Id: I6d6317bac3de6f208700a67e518011362c600ced
diff --git a/libs/gui/GLConsumer.cpp b/libs/gui/GLConsumer.cpp
index 5532db4..faf02f3 100644
--- a/libs/gui/GLConsumer.cpp
+++ b/libs/gui/GLConsumer.cpp
@@ -834,10 +834,10 @@
         xform = crop * xform;
     }
 
-    // SurfaceFlinger expects the top of its window textures to be at a Y
-    // coordinate of 0, so GLConsumer must behave the same way.  We don't
-    // want to expose this to applications, however, so we must add an
-    // additional vertical flip to the transform after all the other transforms.
+    // GLConsumer uses the GL convention where (0, 0) is the bottom-left
+    // corner and (1, 1) is the top-right corner.  Add an additional vertical
+    // flip after all other transforms to map from GL convention to buffer
+    // queue memory layout, where (0, 0) is the top-left corner.
     xform = mtxFlipV * xform;
 
     memcpy(outTransform, xform.asArray(), sizeof(xform));
diff --git a/services/surfaceflinger/BufferLayer.cpp b/services/surfaceflinger/BufferLayer.cpp
index 6cfee3f..4da71cc 100644
--- a/services/surfaceflinger/BufferLayer.cpp
+++ b/services/surfaceflinger/BufferLayer.cpp
@@ -609,6 +609,7 @@
     // TODO: we probably want to generate the texture coords with the mesh
     // here we assume that we only have 4 vertices
     renderengine::Mesh::VertexArray<vec2> texCoords(getBE().mMesh.getTexCoordArray<vec2>());
+    // flip texcoords vertically because BufferLayerConsumer expects them to be in GL convention
     texCoords[0] = vec2(left, 1.0f - top);
     texCoords[1] = vec2(left, 1.0f - bottom);
     texCoords[2] = vec2(right, 1.0f - bottom);
diff --git a/services/surfaceflinger/DisplayDevice.cpp b/services/surfaceflinger/DisplayDevice.cpp
index 7fe18d8..3eb0a92 100644
--- a/services/surfaceflinger/DisplayDevice.cpp
+++ b/services/surfaceflinger/DisplayDevice.cpp
@@ -386,8 +386,7 @@
     size_t w = mDisplayWidth;
     size_t h = mDisplayHeight;
     Rect sourceCrop(0, 0, w, h);
-    mFlinger->getRenderEngine().setViewportAndProjection(w, h, sourceCrop, h,
-        false, ui::Transform::ROT_0);
+    mFlinger->getRenderEngine().setViewportAndProjection(w, h, sourceCrop, ui::Transform::ROT_0);
 }
 
 const sp<Fence>& DisplayDevice::getClientTargetAcquireFence() const {
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index e4f8e7d..ecab4e4 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -791,7 +791,6 @@
                             bool useIdentityTransform) const {
     const Layer::State& s(getDrawingState());
     const ui::Transform renderAreaTransform(renderArea.getTransform());
-    const uint32_t height = renderArea.getHeight();
     FloatRect win = computeBounds();
 
     vec2 lt = vec2(win.left, win.top);
@@ -820,9 +819,6 @@
     position[1] = renderAreaTransform.transform(lb);
     position[2] = renderAreaTransform.transform(rb);
     position[3] = renderAreaTransform.transform(rt);
-    for (size_t i = 0; i < 4; i++) {
-        position[i].y = height - position[i].y;
-    }
 }
 
 bool Layer::isSecure() const {
diff --git a/services/surfaceflinger/RenderEngine/gl/GLES20RenderEngine.cpp b/services/surfaceflinger/RenderEngine/gl/GLES20RenderEngine.cpp
index e0f1850..42d4887 100644
--- a/services/surfaceflinger/RenderEngine/gl/GLES20RenderEngine.cpp
+++ b/services/surfaceflinger/RenderEngine/gl/GLES20RenderEngine.cpp
@@ -160,21 +160,15 @@
 }
 
 void GLES20RenderEngine::setViewportAndProjection(size_t vpw, size_t vph, Rect sourceCrop,
-                                                  size_t hwh, bool yswap,
                                                   ui::Transform::orientation_flags rotation) {
     int32_t l = sourceCrop.left;
     int32_t r = sourceCrop.right;
-
-    // In GL, (0, 0) is the bottom-left corner, so flip y coordinates
-    int32_t t = hwh - sourceCrop.top;
-    int32_t b = hwh - sourceCrop.bottom;
-
-    mat4 m;
-    if (yswap) {
-        m = mat4::ortho(l, r, t, b, 0, 1);
-    } else {
-        m = mat4::ortho(l, r, b, t, 0, 1);
+    int32_t b = sourceCrop.bottom;
+    int32_t t = sourceCrop.top;
+    if (mRenderToFbo) {
+        std::swap(t, b);
     }
+    mat4 m = mat4::ortho(l, r, b, t, 0, 1);
 
     // Apply custom rotation to the projection.
     float rot90InRadians = 2.0f * static_cast<float>(M_PI) / 4.0f;
@@ -284,9 +278,13 @@
     *status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
     *texName = tname;
     *fbName = name;
+
+    mRenderToFbo = true;
 }
 
 void GLES20RenderEngine::unbindFramebuffer(uint32_t texName, uint32_t fbName) {
+    mRenderToFbo = false;
+
     glBindFramebuffer(GL_FRAMEBUFFER, 0);
     glDeleteFramebuffers(1, &fbName);
     glDeleteTextures(1, &texName);
diff --git a/services/surfaceflinger/RenderEngine/gl/GLES20RenderEngine.h b/services/surfaceflinger/RenderEngine/gl/GLES20RenderEngine.h
index c830184..e6a7a00 100644
--- a/services/surfaceflinger/RenderEngine/gl/GLES20RenderEngine.h
+++ b/services/surfaceflinger/RenderEngine/gl/GLES20RenderEngine.h
@@ -62,8 +62,8 @@
 
 protected:
     virtual void dump(String8& result);
-    virtual void setViewportAndProjection(size_t vpw, size_t vph, Rect sourceCrop, size_t hwh,
-                                          bool yswap, ui::Transform::orientation_flags rotation);
+    virtual void setViewportAndProjection(size_t vpw, size_t vph, Rect sourceCrop,
+                                          ui::Transform::orientation_flags rotation);
     virtual void setupLayerBlending(bool premultipliedAlpha, bool opaque, bool disableTexture,
                                     const half4& color) override;
 
@@ -108,6 +108,8 @@
     // with PQ or HLG transfer function.
     bool isHdrDataSpace(const ui::Dataspace dataSpace) const;
     bool needsXYZTransformMatrix() const;
+
+    bool mRenderToFbo = false;
 };
 
 }  // namespace gl
diff --git a/services/surfaceflinger/RenderEngine/include/renderengine/RenderEngine.h b/services/surfaceflinger/RenderEngine/include/renderengine/RenderEngine.h
index c532adc..a363cd4 100644
--- a/services/surfaceflinger/RenderEngine/include/renderengine/RenderEngine.h
+++ b/services/surfaceflinger/RenderEngine/include/renderengine/RenderEngine.h
@@ -96,14 +96,15 @@
     virtual void deleteTextures(size_t count, uint32_t const* names) = 0;
     virtual void bindExternalTextureImage(uint32_t texName, const renderengine::Image& image) = 0;
     virtual void readPixels(size_t l, size_t b, size_t w, size_t h, uint32_t* pixels) = 0;
+    // When binding a native buffer, it must be done before setViewportAndProjection
     virtual void bindNativeBufferAsFrameBuffer(ANativeWindowBuffer* buffer,
                                                BindNativeBufferAsFramebuffer* bindHelper) = 0;
     virtual void unbindNativeBufferAsFrameBuffer(BindNativeBufferAsFramebuffer* bindHelper) = 0;
 
     // set-up
     virtual void checkErrors() const;
-    virtual void setViewportAndProjection(size_t vpw, size_t vph, Rect sourceCrop, size_t hwh,
-                                          bool yswap, ui::Transform::orientation_flags rotation) = 0;
+    virtual void setViewportAndProjection(size_t vpw, size_t vph, Rect sourceCrop,
+                                          ui::Transform::orientation_flags rotation) = 0;
     virtual void setupLayerBlending(bool premultipliedAlpha, bool opaque, bool disableTexture,
                                     const half4& color) = 0;
     virtual void setupLayerTexturing(const Texture& texture) = 0;
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index 3529741..633510f 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -5326,15 +5326,12 @@
 }
 
 void SurfaceFlinger::renderScreenImplLocked(const RenderArea& renderArea,
-                                            TraverseLayersFunction traverseLayers, bool yswap,
+                                            TraverseLayersFunction traverseLayers,
                                             bool useIdentityTransform) {
     ATRACE_CALL();
 
     auto& engine(getRenderEngine());
 
-    // get screen geometry
-    const auto raHeight = renderArea.getHeight();
-
     const auto reqWidth = renderArea.getReqWidth();
     const auto reqHeight = renderArea.getReqHeight();
     const auto sourceCrop = renderArea.getSourceCrop();
@@ -5348,8 +5345,7 @@
     engine.checkErrors();
 
     // set-up our viewport
-    engine.setViewportAndProjection(reqWidth, reqHeight, sourceCrop, raHeight, yswap,
-                                    rotation);
+    engine.setViewportAndProjection(reqWidth, reqHeight, sourceCrop, rotation);
     engine.disableTexturing();
 
     const float alpha = RenderArea::getCaptureFillValue(renderArea.getCaptureFill());
@@ -5395,7 +5391,7 @@
     // via an FBO, which means we didn't have to create
     // an EGLSurface and therefore we're not
     // dependent on the context's EGLConfig.
-    renderScreenImplLocked(renderArea, traverseLayers, true, useIdentityTransform);
+    renderScreenImplLocked(renderArea, traverseLayers, useIdentityTransform);
 
     if (DEBUG_SCREENSHOTS) {
         getRenderEngine().finish();
diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h
index 863ab10..898bec7 100644
--- a/services/surfaceflinger/SurfaceFlinger.h
+++ b/services/surfaceflinger/SurfaceFlinger.h
@@ -581,7 +581,7 @@
     void startBootAnim();
 
     void renderScreenImplLocked(const RenderArea& renderArea, TraverseLayersFunction traverseLayers,
-                                bool yswap, bool useIdentityTransform);
+                                bool useIdentityTransform);
     status_t captureScreenCommon(RenderArea& renderArea, TraverseLayersFunction traverseLayers,
                                  sp<GraphicBuffer>* outBuffer,
                                  bool useIdentityTransform);
diff --git a/services/surfaceflinger/tests/unittests/mock/RenderEngine/MockRenderEngine.h b/services/surfaceflinger/tests/unittests/mock/RenderEngine/MockRenderEngine.h
index bf3c09a..7d504c5 100644
--- a/services/surfaceflinger/tests/unittests/mock/RenderEngine/MockRenderEngine.h
+++ b/services/surfaceflinger/tests/unittests/mock/RenderEngine/MockRenderEngine.h
@@ -55,8 +55,8 @@
     MOCK_METHOD2(bindExternalTextureImage, void(uint32_t, const renderengine::Image&));
     MOCK_METHOD5(readPixels, void(size_t, size_t, size_t, size_t, uint32_t*));
     MOCK_CONST_METHOD0(checkErrors, void());
-    MOCK_METHOD6(setViewportAndProjection,
-                 void(size_t, size_t, Rect, size_t, bool, ui::Transform::orientation_flags));
+    MOCK_METHOD4(setViewportAndProjection,
+                 void(size_t, size_t, Rect, ui::Transform::orientation_flags));
     MOCK_METHOD4(setupLayerBlending, void(bool, bool, bool, const half4&));
     MOCK_METHOD1(setupLayerTexturing, void(const Texture&));
     MOCK_METHOD0(setupLayerBlackedOut, void());