Fix the order of applying layer alpha and color matrices in GLESRenderEngine

If the system want to do a color inversion,
there is an unexpected result due to applying the layer alpha first.
Solution: RenderEngine should apply
the layer's alpha after multiplying by the color matrix.

For example, given the following color inversion matrix:
| r.r  r.g  r.b |   | -1.0   0.0   0.0 |
| g.r  g.g  g.b | = |  0.0  -1.0   0.0 |
| b.r  b.g  b.b |   |  0.0   0.0  -1.0 |
| Tr   Tg   Tb  |   |  1.0   1.0   1.0 |

We want to get a black layer when applying the color transform to a white layer;
However, the white layer applies layer alpha before color matrix. We can get
the below calculation: When layer alpha is 0, R_out value was always Tr.
Then we do not get a black layer when the system applies color inversion to a
white layer.
R_out = (R_in * layer_alpha) * r.r + Tr

Therefore we apply the color matrix first, so we can get the below calculation.
When the layer alpha is 0, we can get a black layer.
R_out = (R_in * r.r + Tr) * layer_alpha
      = (R_in * r.r * layer_alpha) + (Tr * layer_alpha)

adds RenderEngineTest::fillBufferColorTransformZeroLayerAlpha

Bug: 157204341
Test: color inversion when switching pages in the Setting app
Test: RenderEngineTest
Change-Id: I206184aae84042290b00e411709e04e54579780a
Merged-In: I206184aae84042290b00e411709e04e54579780a
diff --git a/libs/renderengine/tests/RenderEngineTest.cpp b/libs/renderengine/tests/RenderEngineTest.cpp
index 3b0d4f7..1ec9412 100644
--- a/libs/renderengine/tests/RenderEngineTest.cpp
+++ b/libs/renderengine/tests/RenderEngineTest.cpp
@@ -345,6 +345,12 @@
     void fillBufferColorTransformCM();
 
     template <typename SourceVariant>
+    void fillBufferWithColorTransformZeroLayerAlpha();
+
+    template <typename SourceVariant>
+    void fillBufferColorTransformZeroLayerAlpha();
+
+    template <typename SourceVariant>
     void fillRedBufferWithRoundedCorners();
 
     template <typename SourceVariant>
@@ -710,6 +716,39 @@
 }
 
 template <typename SourceVariant>
+void RenderEngineTest::fillBufferWithColorTransformZeroLayerAlpha() {
+    renderengine::DisplaySettings settings;
+    settings.physicalDisplay = fullscreenRect();
+    settings.clip = Rect(1, 1);
+
+    std::vector<const renderengine::LayerSettings*> layers;
+
+    renderengine::LayerSettings layer;
+    layer.geometry.boundaries = Rect(1, 1).toFloatRect();
+    SourceVariant::fillColor(layer, 0.5f, 0.25f, 0.125f, this);
+    layer.alpha = 0;
+
+    // construct a fake color matrix
+    // simple inverse color
+    settings.colorTransform = mat4(-1, 0, 0, 0,
+                                   0, -1, 0, 0,
+                                   0, 0, -1, 0,
+                                   1, 1, 1, 1);
+
+    layer.geometry.boundaries = Rect(1, 1).toFloatRect();
+
+    layers.push_back(&layer);
+
+    invokeDraw(settings, layers, mBuffer);
+}
+
+template <typename SourceVariant>
+void RenderEngineTest::fillBufferColorTransformZeroLayerAlpha() {
+    fillBufferWithColorTransformZeroLayerAlpha<SourceVariant>();
+    expectBufferColor(fullscreenRect(), 0, 0, 0, 0);
+}
+
+template <typename SourceVariant>
 void RenderEngineTest::fillRedBufferWithRoundedCorners() {
     renderengine::DisplaySettings settings;
     settings.physicalDisplay = fullscreenRect();
@@ -1116,6 +1155,10 @@
     fillBufferColorTransformCM<ColorSourceVariant>();
 }
 
+TEST_F(RenderEngineTest, drawLayers_fillBufferColorTransformZeroLayerAlpha_colorSource) {
+    fillBufferColorTransformZeroLayerAlpha<ColorSourceVariant>();
+}
+
 TEST_F(RenderEngineTest, drawLayers_fillBufferRoundedCorners_colorSource) {
     fillBufferWithRoundedCorners<ColorSourceVariant>();
 }
@@ -1176,6 +1219,10 @@
     fillBufferColorTransformCM<BufferSourceVariant<ForceOpaqueBufferVariant>>();
 }
 
+TEST_F(RenderEngineTest, drawLayers_fillBufferColorTransformZeroLayerAlpha_opaqueBufferSource) {
+    fillBufferColorTransformZeroLayerAlpha<BufferSourceVariant<ForceOpaqueBufferVariant>>();
+}
+
 TEST_F(RenderEngineTest, drawLayers_fillBufferRoundedCorners_opaqueBufferSource) {
     fillBufferWithRoundedCorners<BufferSourceVariant<ForceOpaqueBufferVariant>>();
 }
@@ -1236,6 +1283,10 @@
     fillBufferColorTransformCM<BufferSourceVariant<RelaxOpaqueBufferVariant>>();
 }
 
+TEST_F(RenderEngineTest, drawLayers_fillBufferColorTransformZeroLayerAlpha_bufferSource) {
+    fillBufferColorTransformZeroLayerAlpha<BufferSourceVariant<RelaxOpaqueBufferVariant>>();
+}
+
 TEST_F(RenderEngineTest, drawLayers_fillBufferRoundedCorners_bufferSource) {
     fillBufferWithRoundedCorners<BufferSourceVariant<RelaxOpaqueBufferVariant>>();
 }