Fix the order of applying layer alpha and color matrices in GLESRenderEngine

If the system want to do a color inversion,
there is an unexpected result due to applying the layer alpha first.
Solution: RenderEngine should apply
the layer's alpha after multiplying by the color matrix.

For example, given the following color inversion matrix:
| r.r  r.g  r.b |   | -1.0   0.0   0.0 |
| g.r  g.g  g.b | = |  0.0  -1.0   0.0 |
| b.r  b.g  b.b |   |  0.0   0.0  -1.0 |
| Tr   Tg   Tb  |   |  1.0   1.0   1.0 |

We want to get a black layer when applying the color transform to a white layer;
However, the white layer applies layer alpha before color matrix. We can get
the below calculation: When layer alpha is 0, R_out value was always Tr.
Then we do not get a black layer when the system applies color inversion to a
white layer.
R_out = (R_in * layer_alpha) * r.r + Tr

Therefore we apply the color matrix first, so we can get the below calculation.
When the layer alpha is 0, we can get a black layer.
R_out = (R_in * r.r + Tr) * layer_alpha
      = (R_in * r.r * layer_alpha) + (Tr * layer_alpha)

adds RenderEngineTest::fillBufferColorTransformZeroLayerAlpha

Bug: 157204341
Test: color inversion when switching pages in the Setting app
Test: RenderEngineTest
Change-Id: I206184aae84042290b00e411709e04e54579780a
diff --git a/libs/renderengine/tests/RenderEngineTest.cpp b/libs/renderengine/tests/RenderEngineTest.cpp
index 712e5e2..d9dfd8c 100644
--- a/libs/renderengine/tests/RenderEngineTest.cpp
+++ b/libs/renderengine/tests/RenderEngineTest.cpp
@@ -406,6 +406,12 @@
     void fillBufferColorTransform();
 
     template <typename SourceVariant>
+    void fillBufferWithColorTransformZeroLayerAlpha();
+
+    template <typename SourceVariant>
+    void fillBufferColorTransformZeroLayerAlpha();
+
+    template <typename SourceVariant>
     void fillRedBufferWithRoundedCorners();
 
     template <typename SourceVariant>
@@ -765,6 +771,36 @@
 }
 
 template <typename SourceVariant>
+void RenderEngineTest::fillBufferWithColorTransformZeroLayerAlpha() {
+    renderengine::DisplaySettings settings;
+    settings.physicalDisplay = fullscreenRect();
+    settings.clip = Rect(1, 1);
+
+    std::vector<const renderengine::LayerSettings*> layers;
+
+    renderengine::LayerSettings layer;
+    layer.geometry.boundaries = Rect(1, 1).toFloatRect();
+    SourceVariant::fillColor(layer, 0.5f, 0.25f, 0.125f, this);
+    layer.alpha = 0;
+
+    // construct a fake color matrix
+    // simple inverse color
+    settings.colorTransform = mat4(-1, 0, 0, 0, 0, -1, 0, 0, 0, 0, -1, 0, 1, 1, 1, 1);
+
+    layer.geometry.boundaries = Rect(1, 1).toFloatRect();
+
+    layers.push_back(&layer);
+
+    invokeDraw(settings, layers, mBuffer);
+}
+
+template <typename SourceVariant>
+void RenderEngineTest::fillBufferColorTransformZeroLayerAlpha() {
+    fillBufferWithColorTransformZeroLayerAlpha<SourceVariant>();
+    expectBufferColor(fullscreenRect(), 0, 0, 0, 0);
+}
+
+template <typename SourceVariant>
 void RenderEngineTest::fillRedBufferWithRoundedCorners() {
     renderengine::DisplaySettings settings;
     settings.physicalDisplay = fullscreenRect();
@@ -1240,6 +1276,13 @@
     fillBufferWithRoundedCorners<ColorSourceVariant>();
 }
 
+TEST_P(RenderEngineTest, drawLayers_fillBufferColorTransformZeroLayerAlpha_colorSource) {
+    const auto& renderEngineFactory = GetParam();
+    mRE = renderEngineFactory->createRenderEngine();
+
+    fillBufferColorTransformZeroLayerAlpha<ColorSourceVariant>();
+}
+
 TEST_P(RenderEngineTest, drawLayers_fillBufferAndBlurBackground_colorSource) {
     const auto& renderEngineFactory = GetParam();
     mRE = renderEngineFactory->createRenderEngine();
@@ -1337,6 +1380,12 @@
 
     fillBufferWithRoundedCorners<BufferSourceVariant<ForceOpaqueBufferVariant>>();
 }
+TEST_P(RenderEngineTest, drawLayers_fillBufferColorTransformZeroLayerAlpha_opaqueBufferSource) {
+    const auto& renderEngineFactory = GetParam();
+    mRE = renderEngineFactory->createRenderEngine();
+
+    fillBufferColorTransformZeroLayerAlpha<BufferSourceVariant<ForceOpaqueBufferVariant>>();
+}
 
 TEST_P(RenderEngineTest, drawLayers_fillBufferAndBlurBackground_opaqueBufferSource) {
     const auto& renderEngineFactory = GetParam();
@@ -1436,6 +1485,13 @@
     fillBufferWithRoundedCorners<BufferSourceVariant<RelaxOpaqueBufferVariant>>();
 }
 
+TEST_P(RenderEngineTest, drawLayers_fillBufferColorTransformZeroLayerAlpha_bufferSource) {
+    const auto& renderEngineFactory = GetParam();
+    mRE = renderEngineFactory->createRenderEngine();
+
+    fillBufferColorTransformZeroLayerAlpha<BufferSourceVariant<RelaxOpaqueBufferVariant>>();
+}
+
 TEST_P(RenderEngineTest, drawLayers_fillBufferAndBlurBackground_bufferSource) {
     const auto& renderEngineFactory = GetParam();
     mRE = renderEngineFactory->createRenderEngine();