Support rounded corners in RenderEngine::drawLayers

Bug: 118461793
Change-Id: I73e4f815f4423b1a80136babba2111f94e38e846
Test: librenderengine_test
diff --git a/libs/renderengine/gl/GLESRenderEngine.cpp b/libs/renderengine/gl/GLESRenderEngine.cpp
index c5c4cc4..94a6650 100644
--- a/libs/renderengine/gl/GLESRenderEngine.cpp
+++ b/libs/renderengine/gl/GLESRenderEngine.cpp
@@ -626,6 +626,25 @@
     return NO_ERROR;
 }
 
+FloatRect GLESRenderEngine::setupLayerCropping(const LayerSettings& layer, Mesh& mesh) {
+    // Translate win by the rounded corners rect coordinates, to have all values in
+    // layer coordinate space.
+    FloatRect cropWin = layer.geometry.boundaries;
+    const FloatRect& roundedCornersCrop = layer.geometry.roundedCornersCrop;
+    cropWin.left -= roundedCornersCrop.left;
+    cropWin.right -= roundedCornersCrop.left;
+    cropWin.top -= roundedCornersCrop.top;
+    cropWin.bottom -= roundedCornersCrop.top;
+    Mesh::VertexArray<vec2> cropCoords(mesh.getCropCoordArray<vec2>());
+    cropCoords[0] = vec2(cropWin.left, cropWin.top);
+    cropCoords[1] = vec2(cropWin.left, cropWin.top + cropWin.getHeight());
+    cropCoords[2] = vec2(cropWin.right, cropWin.top + cropWin.getHeight());
+    cropCoords[3] = vec2(cropWin.right, cropWin.top);
+
+    setupCornerRadiusCropSize(roundedCornersCrop.getWidth(), roundedCornersCrop.getHeight());
+    return cropWin;
+}
+
 status_t GLESRenderEngine::bindFrameBuffer(Framebuffer* framebuffer) {
     GLFramebuffer* glFramebuffer = static_cast<GLFramebuffer*>(framebuffer);
     EGLImageKHR eglImage = glFramebuffer->getEGLImage();
@@ -730,6 +749,7 @@
         position[2] = vec2(bounds.right, bounds.bottom);
         position[3] = vec2(bounds.right, bounds.top);
 
+        setupLayerCropping(layer, mesh);
         setColorTransform(display.colorTransform * layer.colorTransform);
 
         bool usePremultipliedAlpha = true;
@@ -764,7 +784,7 @@
         // Buffer sources will have a black solid color ignored in the shader,
         // so in that scenario the solid color passed here is arbitrary.
         setupLayerBlending(usePremultipliedAlpha, layer.source.buffer.isOpaque, disableTexture,
-                           color, /*cornerRadius=*/0.0);
+                           color, layer.geometry.roundedCornersRadius);
         setSourceDataSpace(layer.sourceDataspace);
 
         drawMesh(mesh);
diff --git a/libs/renderengine/gl/GLESRenderEngine.h b/libs/renderengine/gl/GLESRenderEngine.h
index 2af2aed..511a784 100644
--- a/libs/renderengine/gl/GLESRenderEngine.h
+++ b/libs/renderengine/gl/GLESRenderEngine.h
@@ -128,6 +128,9 @@
     // defined by the clip.
     void setViewportAndProjection(Rect viewport, Rect clip);
     status_t bindExternalTextureBuffer(uint32_t texName, sp<GraphicBuffer> buffer, sp<Fence> fence);
+    // Computes the cropping window for the layer and sets up cropping
+    // coordinates for the mesh.
+    FloatRect setupLayerCropping(const LayerSettings& layer, Mesh& mesh);
 
     EGLDisplay mEGLDisplay;
     EGLConfig mEGLConfig;
diff --git a/libs/renderengine/include/renderengine/LayerSettings.h b/libs/renderengine/include/renderengine/LayerSettings.h
index a1ee37d..4d53205 100644
--- a/libs/renderengine/include/renderengine/LayerSettings.h
+++ b/libs/renderengine/include/renderengine/LayerSettings.h
@@ -69,6 +69,20 @@
 
     // Transform matrix to apply to mesh coordinates.
     mat4 positionTransform = mat4();
+
+    // Radius of rounded corners, if greater than 0. Otherwise, this layer's
+    // corners are not rounded.
+    // Having corner radius will force GPU composition on the layer and its children, drawing it
+    // with a special shader. The shader will receive the radius and the crop rectangle as input,
+    // modifying the opacity of the destination texture, multiplying it by a number between 0 and 1.
+    // We query Layer#getRoundedCornerState() to retrieve the radius as well as the rounded crop
+    // rectangle to figure out how to apply the radius for this layer. The crop rectangle will be
+    // in local layer coordinate space, so we have to take the layer transform into account when
+    // walking up the tree.
+    float roundedCornersRadius = 0.0;
+
+    // Rectangle within which corners will be rounded.
+    FloatRect roundedCornersCrop = FloatRect();
 };
 
 // Descriptor of the source pixels for this layer.
diff --git a/libs/renderengine/tests/RenderEngineTest.cpp b/libs/renderengine/tests/RenderEngineTest.cpp
index b6e4edb..bef25a8 100644
--- a/libs/renderengine/tests/RenderEngineTest.cpp
+++ b/libs/renderengine/tests/RenderEngineTest.cpp
@@ -172,6 +172,12 @@
     template <typename SourceVariant>
     void fillBufferColorTransform();
 
+    template <typename SourceVariant>
+    void fillRedBufferWithRoundedCorners();
+
+    template <typename SourceVariant>
+    void fillBufferWithRoundedCorners();
+
     void fillRedBufferTextureTransform();
 
     void fillBufferTextureTransform();
@@ -501,6 +507,42 @@
     expectBufferColor(fullscreenRect(), 191, 0, 0, 255);
 }
 
+template <typename SourceVariant>
+void RenderEngineTest::fillRedBufferWithRoundedCorners() {
+    renderengine::DisplaySettings settings;
+    settings.physicalDisplay = fullscreenRect();
+    settings.clip = fullscreenRect();
+
+    std::vector<renderengine::LayerSettings> layers;
+
+    renderengine::LayerSettings layer;
+    layer.geometry.boundaries = fullscreenRect().toFloatRect();
+    layer.geometry.roundedCornersRadius = 5.0f;
+    layer.geometry.roundedCornersCrop = fullscreenRect().toFloatRect();
+    SourceVariant::fillColor(layer, 1.0f, 0.0f, 0.0f, this);
+    layer.alpha = 1.0f;
+
+    layers.push_back(layer);
+
+    invokeDraw(settings, layers, mBuffer);
+}
+
+template <typename SourceVariant>
+void RenderEngineTest::fillBufferWithRoundedCorners() {
+    fillRedBufferWithRoundedCorners<SourceVariant>();
+    // Corners should be ignored...
+    expectBufferColor(Rect(0, 0, 1, 1), 0, 0, 0, 0);
+    expectBufferColor(Rect(DEFAULT_DISPLAY_WIDTH - 1, 0, DEFAULT_DISPLAY_WIDTH, 1), 0, 0, 0, 0);
+    expectBufferColor(Rect(0, DEFAULT_DISPLAY_HEIGHT - 1, 1, DEFAULT_DISPLAY_HEIGHT), 0, 0, 0, 0);
+    expectBufferColor(Rect(DEFAULT_DISPLAY_WIDTH - 1, DEFAULT_DISPLAY_HEIGHT - 1,
+                           DEFAULT_DISPLAY_WIDTH, DEFAULT_DISPLAY_HEIGHT),
+                      0, 0, 0, 0);
+    // ...And the non-rounded portion should be red.
+    // Other pixels may be anti-aliased, so let's not check those.
+    expectBufferColor(Rect(5, 5, DEFAULT_DISPLAY_WIDTH - 5, DEFAULT_DISPLAY_HEIGHT - 5), 255, 0, 0,
+                      255);
+}
+
 void RenderEngineTest::fillRedBufferTextureTransform() {
     renderengine::DisplaySettings settings;
     settings.physicalDisplay = fullscreenRect();
@@ -701,6 +743,10 @@
     fillBufferLayerTransform<ColorSourceVariant>();
 }
 
+TEST_F(RenderEngineTest, drawLayers_fillBufferRoundedCorners_colorSource) {
+    fillBufferWithRoundedCorners<ColorSourceVariant>();
+}
+
 TEST_F(RenderEngineTest, drawLayers_fillRedBuffer_opaqueBufferSource) {
     fillRedBuffer<BufferSourceVariant<ForceOpaqueBufferVariant>>();
 }
@@ -745,6 +791,10 @@
     fillBufferLayerTransform<BufferSourceVariant<ForceOpaqueBufferVariant>>();
 }
 
+TEST_F(RenderEngineTest, drawLayers_fillBufferRoundedCorners_opaqueBufferSource) {
+    fillBufferWithRoundedCorners<BufferSourceVariant<ForceOpaqueBufferVariant>>();
+}
+
 TEST_F(RenderEngineTest, drawLayers_fillRedBuffer_bufferSource) {
     fillRedBuffer<BufferSourceVariant<RelaxOpaqueBufferVariant>>();
 }
@@ -789,6 +839,10 @@
     fillBufferLayerTransform<BufferSourceVariant<RelaxOpaqueBufferVariant>>();
 }
 
+TEST_F(RenderEngineTest, drawLayers_fillBufferRoundedCorners_bufferSource) {
+    fillBufferWithRoundedCorners<BufferSourceVariant<RelaxOpaqueBufferVariant>>();
+}
+
 TEST_F(RenderEngineTest, drawLayers_fillBufferTextureTransform) {
     fillBufferTextureTransform();
 }