Merge "[Shadows] Create a new composition layer for shadows [6/n]"
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h
index e585769..0a70165 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/LayerFE.h
@@ -93,6 +93,13 @@
     virtual std::optional<renderengine::LayerSettings> prepareClientComposition(
             ClientCompositionTargetSettings&) = 0;
 
+    // Returns the LayerSettings used to draw shadows around a layer. It is passed
+    // to RenderEngine::drawLayers. Returns nullopt_t if the layer does not render
+    // shadows.
+    virtual std::optional<renderengine::LayerSettings> prepareShadowClientComposition(
+            const renderengine::LayerSettings& layerSettings, const Rect& displayViewport,
+            ui::Dataspace outputDataspace) = 0;
+
     // Called after the layer is displayed to update the presentation fence
     virtual void onLayerDisplayed(const sp<Fence>&) = 0;
 
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/Output.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/Output.h
index 5f42ea1..076fdad 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/Output.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/Output.h
@@ -272,7 +272,7 @@
     virtual bool getSkipColorTransform() const = 0;
     virtual FrameFences presentAndGetFrameFences() = 0;
     virtual std::vector<renderengine::LayerSettings> generateClientCompositionRequests(
-            bool supportsProtectedContent, Region& clearRegion) = 0;
+            bool supportsProtectedContent, Region& clearRegion, ui::Dataspace outputDataspace) = 0;
     virtual void appendRegionFlashRequests(
             const Region& flashRegion,
             std::vector<renderengine::LayerSettings>& clientCompositionLayers) = 0;
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/Output.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/Output.h
index a2342ae..159e928 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/Output.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/impl/Output.h
@@ -97,7 +97,8 @@
     bool getSkipColorTransform() const override;
     compositionengine::Output::FrameFences presentAndGetFrameFences() override;
     std::vector<renderengine::LayerSettings> generateClientCompositionRequests(
-            bool supportsProtectedContent, Region& clearRegion) override;
+            bool supportsProtectedContent, Region& clearRegion,
+            ui::Dataspace outputDataspace) override;
     void appendRegionFlashRequests(const Region&,
                                    std::vector<renderengine::LayerSettings>&) override;
     void setExpensiveRenderingExpected(bool enabled) override;
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h
index 3eada3c..739490f 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/LayerFE.h
@@ -38,6 +38,10 @@
     MOCK_METHOD1(prepareClientComposition,
                  std::optional<renderengine::LayerSettings>(
                          compositionengine::LayerFE::ClientCompositionTargetSettings&));
+    MOCK_METHOD3(prepareShadowClientComposition,
+                 std::optional<renderengine::LayerSettings>(const renderengine::LayerSettings&,
+                                                            const Rect&, ui::Dataspace));
+
     MOCK_METHOD1(onLayerDisplayed, void(const sp<Fence>&));
 
     MOCK_CONST_METHOD0(getDebugName, const char*());
diff --git a/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/Output.h b/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/Output.h
index 02e68fc..7f5bd06 100644
--- a/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/Output.h
+++ b/services/surfaceflinger/CompositionEngine/include/compositionengine/mock/Output.h
@@ -106,8 +106,8 @@
     MOCK_METHOD0(postFramebuffer, void());
     MOCK_METHOD0(presentAndGetFrameFences, compositionengine::Output::FrameFences());
 
-    MOCK_METHOD2(generateClientCompositionRequests,
-                 std::vector<renderengine::LayerSettings>(bool, Region&));
+    MOCK_METHOD3(generateClientCompositionRequests,
+                 std::vector<renderengine::LayerSettings>(bool, Region&, ui::Dataspace));
     MOCK_METHOD2(appendRegionFlashRequests,
                  void(const Region&, std::vector<renderengine::LayerSettings>&));
     MOCK_METHOD1(setExpensiveRenderingExpected, void(bool));
diff --git a/services/surfaceflinger/CompositionEngine/src/Output.cpp b/services/surfaceflinger/CompositionEngine/src/Output.cpp
index 6877f8b..9cc0540 100644
--- a/services/surfaceflinger/CompositionEngine/src/Output.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/Output.cpp
@@ -799,7 +799,8 @@
     // Generate the client composition requests for the layers on this output.
     std::vector<renderengine::LayerSettings> clientCompositionLayers =
             generateClientCompositionRequests(supportsProtectedContent,
-                                              clientCompositionDisplay.clearRegion);
+                                              clientCompositionDisplay.clearRegion,
+                                              clientCompositionDisplay.outputDataspace);
     appendRegionFlashRequests(debugRegion, clientCompositionLayers);
 
     // If we the display is secure, protected content support is enabled, and at
@@ -850,7 +851,7 @@
 }
 
 std::vector<renderengine::LayerSettings> Output::generateClientCompositionRequests(
-        bool supportsProtectedContent, Region& clearRegion) {
+        bool supportsProtectedContent, Region& clearRegion, ui::Dataspace outputDataspace) {
     std::vector<renderengine::LayerSettings> clientCompositionLayers;
     ALOGV("Rendering client layers");
 
@@ -902,6 +903,13 @@
                     layerSettings.source.solidColor = half3(0.0, 0.0, 0.0);
                     layerSettings.alpha = half(0.0);
                     layerSettings.disableBlending = true;
+                } else {
+                    std::optional<renderengine::LayerSettings> shadowLayer =
+                            layerFE.prepareShadowClientComposition(*result, outputState.viewport,
+                                                                   outputDataspace);
+                    if (shadowLayer) {
+                        clientCompositionLayers.push_back(*shadowLayer);
+                    }
                 }
 
                 layer->editState().clientCompositionTimestamp = systemTime();
diff --git a/services/surfaceflinger/CompositionEngine/tests/OutputTest.cpp b/services/surfaceflinger/CompositionEngine/tests/OutputTest.cpp
index 110760c..9e79062 100644
--- a/services/surfaceflinger/CompositionEngine/tests/OutputTest.cpp
+++ b/services/surfaceflinger/CompositionEngine/tests/OutputTest.cpp
@@ -2279,8 +2279,8 @@
         // Sets up the helper functions called by the function under test to use
         // mock implementations.
         MOCK_CONST_METHOD0(getSkipColorTransform, bool());
-        MOCK_METHOD2(generateClientCompositionRequests,
-                     std::vector<renderengine::LayerSettings>(bool, Region&));
+        MOCK_METHOD3(generateClientCompositionRequests,
+                     std::vector<renderengine::LayerSettings>(bool, Region&, ui::Dataspace));
         MOCK_METHOD2(appendRegionFlashRequests,
                      void(const Region&, std::vector<renderengine::LayerSettings>&));
         MOCK_METHOD1(setExpensiveRenderingExpected, void(bool));
@@ -2358,7 +2358,7 @@
     EXPECT_CALL(*mRenderSurface, dequeueBuffer(_)).WillOnce(Return(mOutputBuffer));
 
     EXPECT_CALL(mOutput, getSkipColorTransform()).WillOnce(Return(false));
-    EXPECT_CALL(mOutput, generateClientCompositionRequests(false, _)).Times(1);
+    EXPECT_CALL(mOutput, generateClientCompositionRequests(false, _, _)).Times(1);
     EXPECT_CALL(mOutput, appendRegionFlashRequests(RegionEq(kDebugRegion), _)).Times(1);
     EXPECT_CALL(mOutput, setExpensiveRenderingExpected(true)).Times(1);
     EXPECT_CALL(mOutput, setExpensiveRenderingExpected(false)).Times(1);
@@ -2375,9 +2375,10 @@
     struct OutputPartialMock : public OutputPartialMockBase {
         // compositionengine::Output overrides
         std::vector<renderengine::LayerSettings> generateClientCompositionRequests(
-                bool supportsProtectedContent, Region& clearRegion) override {
+                bool supportsProtectedContent, Region& clearRegion,
+                ui::Dataspace dataspace) override {
             return impl::Output::generateClientCompositionRequests(supportsProtectedContent,
-                                                                   clearRegion);
+                                                                   clearRegion, dataspace);
         }
     };
 
@@ -2436,6 +2437,8 @@
     EXPECT_CALL(leftOutputLayer, needsFiltering()).WillRepeatedly(Return(false));
     EXPECT_CALL(leftLayer, getFEState()).WillRepeatedly(ReturnRef(leftLayerFEState));
     EXPECT_CALL(leftLayerFE, prepareClientComposition(_)).WillOnce(Return(leftLayerRESettings));
+    EXPECT_CALL(leftLayerFE, prepareShadowClientComposition(_, _, _))
+            .WillOnce(Return(std::optional<renderengine::LayerSettings>()));
     EXPECT_CALL(leftOutputLayer, editState()).WillRepeatedly(ReturnRef(leftOutputLayerState));
 
     EXPECT_CALL(rightOutputLayer, getState()).WillRepeatedly(ReturnRef(rightOutputLayerState));
@@ -2445,6 +2448,8 @@
     EXPECT_CALL(rightOutputLayer, needsFiltering()).WillRepeatedly(Return(false));
     EXPECT_CALL(rightLayer, getFEState()).WillRepeatedly(ReturnRef(rightLayerFEState));
     EXPECT_CALL(rightLayerFE, prepareClientComposition(_)).WillOnce(Return(rightLayerRESettings));
+    EXPECT_CALL(rightLayerFE, prepareShadowClientComposition(_, _, _))
+            .WillOnce(Return(std::optional<renderengine::LayerSettings>()));
     EXPECT_CALL(rightOutputLayer, editState()).WillRepeatedly(ReturnRef(rightOutputLayerState));
 
     EXPECT_CALL(mOutput, getOutputLayerCount()).WillRepeatedly(Return(2u));
@@ -2468,8 +2473,8 @@
 
     constexpr bool supportsProtectedContent = false;
     Region clearRegion;
-    auto requests =
-            mOutput.generateClientCompositionRequests(supportsProtectedContent, clearRegion);
+    auto requests = mOutput.generateClientCompositionRequests(supportsProtectedContent, clearRegion,
+                                                              mOutput.getState().targetDataspace);
 
     ASSERT_EQ(2u, requests.size());
     EXPECT_EQ(leftLayerColor, requests[0].source.solidColor);
@@ -2518,8 +2523,8 @@
 
     constexpr bool supportsProtectedContent = false;
     Region clearRegion;
-    auto requests =
-            mOutput.generateClientCompositionRequests(supportsProtectedContent, clearRegion);
+    auto requests = mOutput.generateClientCompositionRequests(supportsProtectedContent, clearRegion,
+                                                              mOutput.getState().targetDataspace);
 
     EXPECT_EQ(0u, requests.size());
 }
@@ -2596,8 +2601,8 @@
 
     constexpr bool supportsProtectedContent = false;
     Region clearRegion;
-    auto requests =
-            mOutput.generateClientCompositionRequests(supportsProtectedContent, clearRegion);
+    auto requests = mOutput.generateClientCompositionRequests(supportsProtectedContent, clearRegion,
+                                                              mOutput.getState().targetDataspace);
 
     const half3 clearColor{0.f, 0.f, 0.f};
 
diff --git a/services/surfaceflinger/DisplayDevice.h b/services/surfaceflinger/DisplayDevice.h
index ce4e1e6..74f47ff 100644
--- a/services/surfaceflinger/DisplayDevice.h
+++ b/services/surfaceflinger/DisplayDevice.h
@@ -246,6 +246,7 @@
                       uint32_t reqHeight, ui::Dataspace reqDataSpace,
                       ui::Transform::orientation_flags rotation, bool allowSecureLayers = true)
           : RenderArea(reqWidth, reqHeight, CaptureFill::OPAQUE, reqDataSpace,
+                       device->getViewport(),
                        getDisplayRotation(rotation, device->getInstallOrientation())),
             mDevice(device),
             mSourceCrop(sourceCrop),
diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 3d00352..35fc4be 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -571,6 +571,53 @@
     return layerSettings;
 }
 
+std::optional<renderengine::LayerSettings> Layer::prepareShadowClientComposition(
+        const renderengine::LayerSettings& casterLayerSettings, const Rect& displayViewport,
+        ui::Dataspace outputDataspace) {
+    renderengine::ShadowSettings shadow = getShadowSettings(displayViewport);
+    if (shadow.length <= 0.f) {
+        return {};
+    }
+
+    const float casterAlpha = casterLayerSettings.alpha;
+    const bool casterIsOpaque = ((casterLayerSettings.source.buffer.buffer != nullptr) &&
+                                 casterLayerSettings.source.buffer.isOpaque);
+
+    renderengine::LayerSettings shadowLayer = casterLayerSettings;
+    shadowLayer.shadow = shadow;
+
+    // If the casting layer is translucent, we need to fill in the shadow underneath the layer.
+    // Otherwise the generated shadow will only be shown around the casting layer.
+    shadowLayer.shadow.casterIsTranslucent = !casterIsOpaque || (casterAlpha < 1.0f);
+    shadowLayer.shadow.ambientColor *= casterAlpha;
+    shadowLayer.shadow.spotColor *= casterAlpha;
+    shadowLayer.sourceDataspace = outputDataspace;
+    shadowLayer.source.buffer.buffer = nullptr;
+
+    if (shadowLayer.shadow.ambientColor.a <= 0.f && shadowLayer.shadow.spotColor.a <= 0.f) {
+        return {};
+    }
+
+    float casterCornerRadius = shadowLayer.geometry.roundedCornersRadius;
+    const FloatRect& cornerRadiusCropRect = casterLayerSettings.geometry.roundedCornersCrop;
+    const FloatRect& casterRect = shadowLayer.geometry.boundaries;
+
+    // crop used to set the corner radius may be larger than the content rect. Adjust the corner
+    // radius accordingly.
+    if (casterCornerRadius > 0.f) {
+        float cropRectOffset = std::max(std::abs(cornerRadiusCropRect.top - casterRect.top),
+                                        std::abs(cornerRadiusCropRect.left - casterRect.left));
+        if (cropRectOffset > casterCornerRadius) {
+            casterCornerRadius = 0;
+        } else {
+            casterCornerRadius -= cropRectOffset;
+        }
+        shadowLayer.geometry.roundedCornersRadius = casterCornerRadius;
+    }
+
+    return shadowLayer;
+}
+
 Hwc2::IComposerClient::Composition Layer::getCompositionType(
         const sp<const DisplayDevice>& display) const {
     const auto outputLayer = findOutputLayerForDisplay(display);
diff --git a/services/surfaceflinger/Layer.h b/services/surfaceflinger/Layer.h
index 82e558f..843d3ae 100644
--- a/services/surfaceflinger/Layer.h
+++ b/services/surfaceflinger/Layer.h
@@ -510,6 +510,9 @@
     void latchCursorCompositionState(compositionengine::LayerFECompositionState&) const override;
     std::optional<renderengine::LayerSettings> prepareClientComposition(
             compositionengine::LayerFE::ClientCompositionTargetSettings&) override;
+    std::optional<renderengine::LayerSettings> prepareShadowClientComposition(
+            const renderengine::LayerSettings& layerSettings, const Rect& displayViewport,
+            ui::Dataspace outputDataspace) override;
     void onLayerDisplayed(const sp<Fence>& releaseFence) override;
     const char* getDebugName() const override;
 
diff --git a/services/surfaceflinger/RenderArea.h b/services/surfaceflinger/RenderArea.h
index edc6442..532572f 100644
--- a/services/surfaceflinger/RenderArea.h
+++ b/services/surfaceflinger/RenderArea.h
@@ -22,13 +22,14 @@
     static float getCaptureFillValue(CaptureFill captureFill);
 
     RenderArea(uint32_t reqWidth, uint32_t reqHeight, CaptureFill captureFill,
-               ui::Dataspace reqDataSpace,
+               ui::Dataspace reqDataSpace, const Rect& displayViewport,
                ui::Transform::orientation_flags rotation = ui::Transform::ROT_0)
           : mReqWidth(reqWidth),
             mReqHeight(reqHeight),
             mReqDataSpace(reqDataSpace),
             mCaptureFill(captureFill),
-            mRotationFlags(rotation) {}
+            mRotationFlags(rotation),
+            mDisplayViewport(displayViewport) {}
 
     virtual ~RenderArea() = default;
 
@@ -80,12 +81,16 @@
 
     virtual const sp<const DisplayDevice> getDisplayDevice() const = 0;
 
+    // Returns the source display viewport.
+    const Rect& getDisplayViewport() const { return mDisplayViewport; }
+
 private:
     const uint32_t mReqWidth;
     const uint32_t mReqHeight;
     const ui::Dataspace mReqDataSpace;
     const CaptureFill mCaptureFill;
     const ui::Transform::orientation_flags mRotationFlags;
+    const Rect mDisplayViewport;
 };
 
 } // namespace android
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index f3d3224..12206e5 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -4930,8 +4930,12 @@
     }
     // Couldn't find display by displayId. Try to get display by layerStack since virtual displays
     // may not have a displayId.
+    return getDisplayByLayerStack(displayOrLayerStack);
+}
+
+const sp<DisplayDevice> SurfaceFlinger::getDisplayByLayerStack(uint64_t layerStack) {
     for (const auto& [token, display] : mDisplays) {
-        if (display->getLayerStack() == displayOrLayerStack) {
+        if (display->getLayerStack() == layerStack) {
             return display;
         }
     }
@@ -4987,8 +4991,8 @@
     public:
         LayerRenderArea(SurfaceFlinger* flinger, const sp<Layer>& layer, const Rect crop,
                         int32_t reqWidth, int32_t reqHeight, Dataspace reqDataSpace,
-                        bool childrenOnly)
-              : RenderArea(reqWidth, reqHeight, CaptureFill::CLEAR, reqDataSpace),
+                        bool childrenOnly, const Rect& displayViewport)
+              : RenderArea(reqWidth, reqHeight, CaptureFill::CLEAR, reqDataSpace, displayViewport),
                 mLayer(layer),
                 mCrop(crop),
                 mNeedsFiltering(false),
@@ -5072,7 +5076,7 @@
     sp<Layer> parent;
     Rect crop(sourceCrop);
     std::unordered_set<sp<Layer>, ISurfaceComposer::SpHash<Layer>> excludeLayers;
-
+    Rect displayViewport;
     {
         Mutex::Autolock _l(mStateLock);
 
@@ -5117,6 +5121,13 @@
                 return NAME_NOT_FOUND;
             }
         }
+
+        auto display = getDisplayByLayerStack(parent->getLayerStack());
+        if (!display) {
+            return BAD_VALUE;
+        }
+
+        displayViewport = display->getViewport();
     } // mStateLock
 
     // really small crop or frameScale
@@ -5127,7 +5138,8 @@
         reqHeight = 1;
     }
 
-    LayerRenderArea renderArea(this, parent, crop, reqWidth, reqHeight, reqDataspace, childrenOnly);
+    LayerRenderArea renderArea(this, parent, crop, reqWidth, reqHeight, reqDataspace, childrenOnly,
+                               displayViewport);
     auto traverseLayers = [parent, childrenOnly,
                            &excludeLayers](const LayerVector::Visitor& visitor) {
         parent->traverseChildrenInZOrder(LayerVector::StateSet::Drawing, [&](Layer* layer) {
@@ -5253,6 +5265,7 @@
     const auto rotation = renderArea.getRotationFlags();
     const auto transform = renderArea.getTransform();
     const auto sourceCrop = renderArea.getSourceCrop();
+    const auto& displayViewport = renderArea.getDisplayViewport();
 
     renderengine::DisplaySettings clientCompositionDisplay;
     std::vector<renderengine::LayerSettings> clientCompositionLayers;
@@ -5337,6 +5350,12 @@
         };
         auto result = layer->prepareClientComposition(targetSettings);
         if (result) {
+            std::optional<renderengine::LayerSettings> shadowLayer =
+                    layer->prepareShadowClientComposition(*result, displayViewport,
+                                                          clientCompositionDisplay.outputDataspace);
+            if (shadowLayer) {
+                clientCompositionLayers.push_back(*shadowLayer);
+            }
             clientCompositionLayers.push_back(*result);
         }
     });
diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h
index 5f1580b..7144608 100644
--- a/services/surfaceflinger/SurfaceFlinger.h
+++ b/services/surfaceflinger/SurfaceFlinger.h
@@ -688,6 +688,7 @@
                                  const sp<GraphicBuffer>& buffer, bool useIdentityTransform,
                                  bool& outCapturedSecureLayers);
     const sp<DisplayDevice> getDisplayByIdOrLayerStack(uint64_t displayOrLayerStack);
+    const sp<DisplayDevice> getDisplayByLayerStack(uint64_t layerStack);
     status_t captureScreenImplLocked(const RenderArea& renderArea,
                                      TraverseLayersFunction traverseLayers,
                                      ANativeWindowBuffer* buffer, bool useIdentityTransform,