Draw shadows when the casting layer is occluded or transparent

Ignore transparent region when generating the composition layer for shadows. Client may
provide transparent region hints but the shadow should be drawn over the entire layer.
Solves an issue with YouTube in PiP which sets a transparent region where the
SurfaceView is shown causing a shadow of incorrect size to be drawn.

Draw shadows even if the layer is completely occluded by another layer. The layer will
not be composed in this case causing the shadow to not be drawn either. So extend its
visible region by the shadow length so we can check if the shadows will be occluded as
well.

Bug: 136561771
Test: atest libcompositionengine_test
Test: manual tests with pip overriding shadows to be drawn by sf
Change-Id: I4c6cae1716caebe46119ebd1643d8b5e3eda56c3
diff --git a/services/surfaceflinger/CompositionEngine/src/Output.cpp b/services/surfaceflinger/CompositionEngine/src/Output.cpp
index 7e5a720..28d2653 100644
--- a/services/surfaceflinger/CompositionEngine/src/Output.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/Output.cpp
@@ -408,6 +408,11 @@
      */
     Region transparentRegion;
 
+    /*
+     * shadowRegion: Region cast by the layer's shadow.
+     */
+    Region shadowRegion;
+
     // handle hidden surfaces by setting the visible region to empty
     if (CC_UNLIKELY(!layerFEState.isVisible)) {
         return;
@@ -418,7 +423,18 @@
     // Get the visible region
     // TODO(b/121291683): Is it worth creating helper methods on LayerFEState
     // for computations like this?
-    visibleRegion.set(Rect(tr.transform(layerFEState.geomLayerBounds)));
+    const Rect visibleRect(tr.transform(layerFEState.geomLayerBounds));
+    visibleRegion.set(visibleRect);
+
+    if (layerFEState.shadowRadius > 0.0f) {
+        // if the layer casts a shadow, offset the layers visible region and
+        // calculate the shadow region.
+        const int32_t inset = layerFEState.shadowRadius * -1.0f;
+        Rect visibleRectWithShadows(visibleRect);
+        visibleRectWithShadows.inset(inset, inset, inset, inset);
+        visibleRegion.set(visibleRectWithShadows);
+        shadowRegion = visibleRegion.subtract(visibleRect);
+    }
 
     if (visibleRegion.isEmpty()) {
         return;
@@ -444,7 +460,7 @@
         // Otherwise we don't try and compute the opaque region since there may
         // be errors at the edges, and we treat the entire layer as
         // translucent.
-        opaqueRegion = visibleRegion;
+        opaqueRegion.set(visibleRect);
     }
 
     // Clip the covered region to the visible region
@@ -510,7 +526,7 @@
     // Compute the visible non-transparent region
     Region visibleNonTransparentRegion = visibleRegion.subtract(transparentRegion);
 
-    // Peform the final check to see if this layer is visible on this output
+    // Perform the final check to see if this layer is visible on this output
     // TODO(b/121291683): Why does this not use visibleRegion? (see outputSpaceVisibleRegion below)
     const auto& outputState = getState();
     Region drawRegion(outputState.transform.transform(visibleNonTransparentRegion));
@@ -519,6 +535,8 @@
         return;
     }
 
+    Region visibleNonShadowRegion = visibleRegion.subtract(shadowRegion);
+
     // The layer is visible. Either reuse the existing outputLayer if we have
     // one, or create a new one if we do not.
     auto result = ensureOutputLayer(prevOutputLayerIndex, layer, layerFE);
@@ -529,8 +547,9 @@
     outputLayerState.visibleRegion = visibleRegion;
     outputLayerState.visibleNonTransparentRegion = visibleNonTransparentRegion;
     outputLayerState.coveredRegion = coveredRegion;
-    outputLayerState.outputSpaceVisibleRegion = outputState.transform.transform(
-            outputLayerState.visibleRegion.intersect(outputState.viewport));
+    outputLayerState.outputSpaceVisibleRegion =
+            outputState.transform.transform(visibleNonShadowRegion.intersect(outputState.viewport));
+    outputLayerState.shadowRegion = shadowRegion;
 }
 
 void Output::setReleasedLayers(const compositionengine::CompositionRefreshArgs&) {
@@ -884,7 +903,7 @@
             continue;
         }
 
-        bool clientComposition = layer->requiresClientComposition();
+        const bool clientComposition = layer->requiresClientComposition();
 
         // We clear the client target for non-client composed layers if
         // requested by the HWC. We skip this if the layer is not an opaque
@@ -921,8 +940,15 @@
                     }
                 }
 
-                layer->editState().clientCompositionTimestamp = systemTime();
-                clientCompositionLayers.push_back(*result);
+                // If the layer casts a shadow but the content casting the shadow is occluded, skip
+                // composing the non-shadow content and only draw the shadows.
+                const bool skipNonShadowContentComposition = clientComposition &&
+                        layerState.visibleRegion.subtract(layerState.shadowRegion).isEmpty();
+
+                if (!skipNonShadowContentComposition) {
+                    layer->editState().clientCompositionTimestamp = systemTime();
+                    clientCompositionLayers.push_back(*result);
+                }
             }
         }