SF: Consolidate layer-to-output filtering

Add ui::LayerFilter for less repetitive CE plumbing. Make ui::LayerStack
a type, and (unlike the alias) use it everywhere. Remove redundant state
in CE's DisplayCreationArgs.

Bug: 182939859
Test: Display cutout is excluded in screenshots.
Test: libcompositionengine_test
Test: libsurfaceflinger_unittest
Test: SurfaceFlinger_test
Test: libgui_test
Change-Id: Ib854d354af7aef7168001c34297e875b71d53622
diff --git a/services/surfaceflinger/CompositionEngine/src/Display.cpp b/services/surfaceflinger/CompositionEngine/src/Display.cpp
index 2f2c686..2cc5fae 100644
--- a/services/surfaceflinger/CompositionEngine/src/Display.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/Display.cpp
@@ -51,12 +51,9 @@
 
 void Display::setConfiguration(const compositionengine::DisplayCreationArgs& args) {
     mId = args.id;
-    mIsVirtual = !args.connectionType;
     mPowerAdvisor = args.powerAdvisor;
     editState().isSecure = args.isSecure;
     editState().displaySpace.bounds = Rect(args.pixels);
-    setLayerStackFilter(args.layerStackId,
-                        args.connectionType == ui::DisplayConnectionType::Internal);
     setName(args.name);
 }
 
@@ -73,7 +70,7 @@
 }
 
 bool Display::isVirtual() const {
-    return mIsVirtual;
+    return VirtualDisplayId::tryCast(mId).has_value();
 }
 
 std::optional<DisplayId> Display::getDisplayId() const {
@@ -117,8 +114,8 @@
         return;
     }
 
-    if (mIsVirtual) {
-        ALOGW("%s: Invalid operation on virtual display", __FUNCTION__);
+    if (isVirtual()) {
+        ALOGW("%s: Invalid operation on virtual display", __func__);
         return;
     }
 
@@ -136,7 +133,7 @@
     StringAppendF(&out, "   Composition Display State: [\"%s\"]", getName().c_str());
 
     out.append("\n   ");
-    dumpVal(out, "isVirtual", mIsVirtual);
+    dumpVal(out, "isVirtual", isVirtual());
     dumpVal(out, "DisplayId", to_string(mId));
     out.append("\n");
 
diff --git a/services/surfaceflinger/CompositionEngine/src/DumpHelpers.cpp b/services/surfaceflinger/CompositionEngine/src/DumpHelpers.cpp
index 5565396..01c368d 100644
--- a/services/surfaceflinger/CompositionEngine/src/DumpHelpers.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/DumpHelpers.cpp
@@ -63,6 +63,18 @@
     dumpVal(out, name, valueName.c_str(), value);
 }
 
+void dumpVal(std::string& out, const char* name, ui::LayerFilter filter) {
+    out.append(name);
+    out.append("={");
+    dumpVal(out, "layerStack", filter.layerStack.id);
+    dumpVal(out, "toInternalDisplay", filter.toInternalDisplay);
+    out.push_back('}');
+}
+
+void dumpVal(std::string& out, const char* name, ui::Size size) {
+    StringAppendF(&out, "%s=[%d %d] ", name, size.width, size.height);
+}
+
 void dumpVal(std::string& out, const char* name, const FloatRect& rect) {
     StringAppendF(&out, "%s=[%f %f %f %f] ", name, rect.left, rect.top, rect.right, rect.bottom);
 }
@@ -80,10 +92,6 @@
     out.append(" ");
 }
 
-void dumpVal(std::string& out, const char* name, const ui::Size& size) {
-    StringAppendF(&out, "%s=[%d %d] ", name, size.width, size.height);
-}
-
 void dumpVal(std::string& out, const char* name, const mat4& tr) {
     StringAppendF(&out,
                   "%s=["
diff --git a/services/surfaceflinger/CompositionEngine/src/Output.cpp b/services/surfaceflinger/CompositionEngine/src/Output.cpp
index 6ac488b..5e40802 100644
--- a/services/surfaceflinger/CompositionEngine/src/Output.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/Output.cpp
@@ -227,11 +227,8 @@
     return static_cast<ui::Transform::RotationFlags>(getState().transform.getOrientation());
 }
 
-void Output::setLayerStackFilter(uint32_t layerStackId, bool isInternal) {
-    auto& outputState = editState();
-    outputState.layerStackId = layerStackId;
-    outputState.layerStackInternal = isInternal;
-
+void Output::setLayerFilter(ui::LayerFilter filter) {
+    editState().layerFilter = filter;
     dirtyEntireOutput();
 }
 
@@ -380,17 +377,13 @@
     return dirty;
 }
 
-bool Output::belongsInOutput(std::optional<uint32_t> layerStackId, bool internalOnly) const {
-    // The layerStackId's must match, and also the layer must not be internal
-    // only when not on an internal output.
-    const auto& outputState = getState();
-    return layerStackId && (*layerStackId == outputState.layerStackId) &&
-            (!internalOnly || outputState.layerStackInternal);
+bool Output::includesLayer(ui::LayerFilter filter) const {
+    return getState().layerFilter.includes(filter);
 }
 
-bool Output::belongsInOutput(const sp<compositionengine::LayerFE>& layerFE) const {
+bool Output::includesLayer(const sp<LayerFE>& layerFE) const {
     const auto* layerFEState = layerFE->getCompositionState();
-    return layerFEState && belongsInOutput(layerFEState->layerStackId, layerFEState->internalOnly);
+    return layerFEState && includesLayer(layerFEState->outputFilter);
 }
 
 std::unique_ptr<compositionengine::OutputLayer> Output::createOutputLayer(
@@ -496,8 +489,8 @@
         layerFE->prepareCompositionState(compositionengine::LayerFE::StateSubset::BasicGeometry);
     }
 
-    // Only consider the layers on the given layer stack
-    if (!belongsInOutput(layerFE)) {
+    // Only consider the layers on this output
+    if (!includesLayer(layerFE)) {
         return;
     }
 
@@ -1122,7 +1115,7 @@
     // bounds its framebuffer cache but Skia RenderEngine has no current policy. The best fix is
     // probably to encapsulate the output buffer into a structure that dispatches resource cleanup
     // over to RenderEngine, in which case this flag can be removed from the drawLayers interface.
-    const bool useFramebufferCache = outputState.layerStackInternal;
+    const bool useFramebufferCache = outputState.layerFilter.toInternalDisplay;
     status_t status =
             renderEngine.drawLayers(clientCompositionDisplay, clientCompositionLayerPointers, tex,
                                     useFramebufferCache, std::move(fd), &readyFence);
diff --git a/services/surfaceflinger/CompositionEngine/src/OutputCompositionState.cpp b/services/surfaceflinger/CompositionEngine/src/OutputCompositionState.cpp
index ee30ad8..acc9216 100644
--- a/services/surfaceflinger/CompositionEngine/src/OutputCompositionState.cpp
+++ b/services/surfaceflinger/CompositionEngine/src/OutputCompositionState.cpp
@@ -28,9 +28,7 @@
     dumpVal(out, "usesDeviceComposition", usesDeviceComposition);
     dumpVal(out, "flipClientTarget", flipClientTarget);
     dumpVal(out, "reusedClientComposition", reusedClientComposition);
-
-    dumpVal(out, "layerStack", layerStackId);
-    dumpVal(out, "layerStackInternal", layerStackInternal);
+    dumpVal(out, "layerFilter", layerFilter);
 
     out.append("\n   ");