[sf] fix overflow in snapshot generator

Fixes a couple of issues that caused the snapshot
builder to crash from a stack overflow:
- mirror layers were temporarily rooted to the display
with the default layer stack
- there was no check to prevent a layer from mirroring
itself (causing a cycle in the hierarchy)

Also modifies the layer trace generator to easily generate
test data for the transaction trace test suite.

Test: atest transactiontrace_testsuite
Fixes: 282110579
Change-Id: I2954f838e333b857b4b755e11666c2580d97c2b6
diff --git a/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp b/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp
index cd9515c..c9eb9c4 100644
--- a/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp
+++ b/services/surfaceflinger/FrontEnd/LayerLifecycleManager.cpp
@@ -28,6 +28,14 @@
 
 using namespace ftl::flag_operators;
 
+namespace {
+// Returns true if the layer is root of a display and can be mirrored by mirroringLayer
+bool canMirrorRootLayer(RequestedLayerState& mirroringLayer, RequestedLayerState& rootLayer) {
+    return rootLayer.isRoot() && rootLayer.layerStack == mirroringLayer.layerStackToMirror &&
+            rootLayer.id != mirroringLayer.id;
+}
+} // namespace
+
 void LayerLifecycleManager::addLayers(std::vector<std::unique_ptr<RequestedLayerState>> newLayers) {
     if (newLayers.empty()) {
         return;
@@ -46,11 +54,15 @@
         layer.parentId = linkLayer(layer.parentId, layer.id);
         layer.relativeParentId = linkLayer(layer.relativeParentId, layer.id);
         if (layer.layerStackToMirror != ui::INVALID_LAYER_STACK) {
+            // Set mirror layer's default layer stack to -1 so it doesn't end up rendered on a
+            // display accidentally.
+            layer.layerStack = ui::INVALID_LAYER_STACK;
+
             // if this layer is mirroring a display, then walk though all the existing root layers
             // for the layer stack and add them as children to be mirrored.
             mDisplayMirroringLayers.emplace_back(layer.id);
             for (auto& rootLayer : mLayers) {
-                if (rootLayer->isRoot() && rootLayer->layerStack == layer.layerStackToMirror) {
+                if (canMirrorRootLayer(layer, *rootLayer)) {
                     layer.mirrorIds.emplace_back(rootLayer->id);
                     linkLayer(rootLayer->id, layer.id);
                 }
@@ -383,10 +395,9 @@
 // and updates its list of layers that its mirroring. This function should be called when a new
 // root layer is added, removed or moved to another display.
 void LayerLifecycleManager::updateDisplayMirrorLayers(RequestedLayerState& rootLayer) {
-    for (uint32_t mirrorLayerId : mDisplayMirroringLayers) {
-        RequestedLayerState* mirrorLayer = getLayerFromId(mirrorLayerId);
-        bool canBeMirrored =
-                rootLayer.isRoot() && rootLayer.layerStack == mirrorLayer->layerStackToMirror;
+    for (uint32_t mirroringLayerId : mDisplayMirroringLayers) {
+        RequestedLayerState* mirrorLayer = getLayerFromId(mirroringLayerId);
+        bool canBeMirrored = canMirrorRootLayer(*mirrorLayer, rootLayer);
         bool currentlyMirrored =
                 std::find(mirrorLayer->mirrorIds.begin(), mirrorLayer->mirrorIds.end(),
                           rootLayer.id) != mirrorLayer->mirrorIds.end();
diff --git a/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.cpp b/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.cpp
index 72a11c6..519ef44 100644
--- a/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.cpp
+++ b/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.cpp
@@ -41,7 +41,7 @@
 using namespace ftl::flag_operators;
 
 bool LayerTraceGenerator::generate(const proto::TransactionTraceFile& traceFile,
-                                   const char* outputLayersTracePath) {
+                                   const char* outputLayersTracePath, bool onlyLastEntry) {
     if (traceFile.entry_size() == 0) {
         ALOGD("Trace file is empty");
         return false;
@@ -158,9 +158,11 @@
                                                                   layerTracing.getFlags())
                                           .generate(hierarchyBuilder.getHierarchy());
         auto displayProtos = LayerProtoHelper::writeDisplayInfoToProto(displayInfos);
-        layerTracing.notify(visibleRegionsDirty, entry.elapsed_realtime_nanos(), entry.vsync_id(),
-                            &layersProto, {}, &displayProtos);
-        layerTracing.appendToStream(out);
+        if (!onlyLastEntry || (i == traceFile.entry_size() - 1)) {
+            layerTracing.notify(visibleRegionsDirty, entry.elapsed_realtime_nanos(),
+                                entry.vsync_id(), &layersProto, {}, &displayProtos);
+            layerTracing.appendToStream(out);
+        }
     }
     layerTracing.disable("", /*writeToFile=*/false);
     out.close();
diff --git a/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.h b/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.h
index ee1ea6c..e41d1e6 100644
--- a/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.h
+++ b/services/surfaceflinger/Tracing/tools/LayerTraceGenerator.h
@@ -21,6 +21,7 @@
 namespace android {
 class LayerTraceGenerator {
 public:
-    bool generate(const proto::TransactionTraceFile&, const char* outputLayersTracePath);
+    bool generate(const proto::TransactionTraceFile&, const char* outputLayersTracePath,
+                  bool onlyLastEntry);
 };
 } // namespace android
\ No newline at end of file
diff --git a/services/surfaceflinger/Tracing/tools/main.cpp b/services/surfaceflinger/Tracing/tools/main.cpp
index c440c19..5ca87e4 100644
--- a/services/surfaceflinger/Tracing/tools/main.cpp
+++ b/services/surfaceflinger/Tracing/tools/main.cpp
@@ -26,9 +26,9 @@
 using namespace android;
 
 int main(int argc, char** argv) {
-    if (argc > 3) {
+    if (argc > 4) {
         std::cout << "Usage: " << argv[0]
-                  << " [transaction-trace-path] [output-layers-trace-path]\n";
+                  << " [transaction-trace-path] [output-layers-trace-path] [--last-entry-only]\n";
         return -1;
     }
 
@@ -48,12 +48,16 @@
     }
 
     const char* outputLayersTracePath =
-            (argc == 3) ? argv[2] : "/data/misc/wmtrace/layers_trace.winscope";
-    ;
+            (argc >= 3) ? argv[2] : "/data/misc/wmtrace/layers_trace.winscope";
+
+    const bool generateLastEntryOnly =
+            argc >= 4 && std::string_view(argv[3]) == "--last-entry-only";
+
     ALOGD("Generating %s...", outputLayersTracePath);
     std::cout << "Generating " << outputLayersTracePath << "\n";
 
-    if (!LayerTraceGenerator().generate(transactionTraceFile, outputLayersTracePath)) {
+    if (!LayerTraceGenerator().generate(transactionTraceFile, outputLayersTracePath,
+                                        generateLastEntryOnly)) {
         std::cout << "Error: Failed to generate layers trace " << outputLayersTracePath;
         return -1;
     }
diff --git a/services/surfaceflinger/tests/tracing/TransactionTraceTestSuite.cpp b/services/surfaceflinger/tests/tracing/TransactionTraceTestSuite.cpp
index 2b29530..b8a5e79 100644
--- a/services/surfaceflinger/tests/tracing/TransactionTraceTestSuite.cpp
+++ b/services/surfaceflinger/tests/tracing/TransactionTraceTestSuite.cpp
@@ -59,8 +59,8 @@
         std::string actualLayersTracePath =
                 std::string(temp_dir.path) + "/" + expectedLayersFilename + "_actual";
 
-        EXPECT_TRUE(
-                LayerTraceGenerator().generate(mTransactionTrace, actualLayersTracePath.c_str()))
+        EXPECT_TRUE(LayerTraceGenerator().generate(mTransactionTrace, actualLayersTracePath.c_str(),
+                                                   /*onlyLastEntry=*/true))
                 << "Failed to generate layers trace from " << transactionTracePath;
         EXPECT_TRUE(std::filesystem::exists(std::filesystem::path(actualLayersTracePath)));
         parseLayersTraceFromFile(actualLayersTracePath.c_str(), mActualLayersTraceProto);
@@ -86,9 +86,9 @@
 std::vector<std::filesystem::path> TransactionTraceTestSuite::sTransactionTraces{};
 
 struct LayerInfo {
-    int32_t id;
+    uint64_t id;
     std::string name;
-    int32_t parent;
+    uint64_t parent;
     int z;
     uint64_t curr_frame;
     float x;
@@ -119,8 +119,8 @@
 }
 
 struct find_id : std::unary_function<LayerInfo, bool> {
-    int id;
-    find_id(int id) : id(id) {}
+    uint64_t id;
+    find_id(uint64_t id) : id(id) {}
     bool operator()(LayerInfo const& m) const { return m.id == id; }
 };
 
@@ -136,9 +136,9 @@
         touchableRegionBounds = touchableRegion.bounds();
     }
 
-    return {proto.id(),
+    return {static_cast<uint64_t>(proto.id()),
             proto.name(),
-            proto.parent(),
+            static_cast<uint64_t>(proto.parent()),
             proto.z(),
             proto.curr_frame(),
             proto.has_position() ? proto.position().x() : -1,
@@ -150,7 +150,7 @@
 
 static std::vector<LayerInfo> getLayerInfosFromProto(
         android::surfaceflinger::LayersTraceProto& entry) {
-    std::unordered_map<int32_t /* snapshotId*/, int32_t /*layerId*/> snapshotIdToLayerId;
+    std::unordered_map<uint64_t /* snapshotId*/, uint64_t /*layerId*/> snapshotIdToLayerId;
     std::vector<LayerInfo> layers;
     layers.reserve(static_cast<size_t>(entry.layers().layers_size()));
     bool mapSnapshotIdToLayerId = false;
@@ -158,7 +158,12 @@
         auto layer = entry.layers().layers(i);
         LayerInfo layerInfo = getLayerInfoFromProto(layer);
 
-        snapshotIdToLayerId[layerInfo.id] = static_cast<int32_t>(layer.original_id());
+        uint64_t layerId = layerInfo.name.find("(Mirror)") == std::string::npos
+                ? static_cast<uint64_t>(layer.original_id())
+                : static_cast<uint64_t>(layer.original_id()) | 1ull << 63;
+
+        snapshotIdToLayerId[layerInfo.id] = layerId;
+
         if (layer.original_id() != 0) {
             mapSnapshotIdToLayerId = true;
         }
@@ -172,7 +177,7 @@
     for (auto& layer : layers) {
         layer.id = snapshotIdToLayerId[layer.id];
         auto it = snapshotIdToLayerId.find(layer.parent);
-        layer.parent = it == snapshotIdToLayerId.end() ? -1 : it->second;
+        layer.parent = it == snapshotIdToLayerId.end() ? static_cast<uint64_t>(-1) : it->second;
     }
     return layers;
 }
@@ -189,7 +194,6 @@
 
     std::vector<LayerInfo> expectedLayers = getLayerInfosFromProto(expectedLastEntry);
     std::vector<LayerInfo> actualLayers = getLayerInfosFromProto(actualLastEntry);
-    ;
 
     size_t i = 0;
     for (; i < actualLayers.size() && i < expectedLayers.size(); i++) {
@@ -197,9 +201,9 @@
                                find_id(expectedLayers[i].id));
         EXPECT_NE(it, actualLayers.end());
         EXPECT_EQ(expectedLayers[i], *it);
-        ALOGV("Validating %s[%d] parent=%d z=%d frame=%" PRIu64, expectedLayers[i].name.c_str(),
-              expectedLayers[i].id, expectedLayers[i].parent, expectedLayers[i].z,
-              expectedLayers[i].curr_frame);
+        ALOGV("Validating %s[%" PRIu64 "] parent=%" PRIu64 " z=%d frame=%" PRIu64,
+              expectedLayers[i].name.c_str(), expectedLayers[i].id, expectedLayers[i].parent,
+              expectedLayers[i].z, expectedLayers[i].curr_frame);
     }
 
     EXPECT_EQ(expectedLayers.size(), actualLayers.size());
@@ -208,9 +212,9 @@
         for (size_t j = 0; j < actualLayers.size(); j++) {
             if (std::find_if(expectedLayers.begin(), expectedLayers.end(),
                              find_id(actualLayers[j].id)) == expectedLayers.end()) {
-                ALOGD("actualLayers [%d]:%s parent=%d z=%d frame=%" PRIu64, actualLayers[j].id,
-                      actualLayers[j].name.c_str(), actualLayers[j].parent, actualLayers[j].z,
-                      actualLayers[j].curr_frame);
+                ALOGD("actualLayers [%" PRIu64 "]:%s parent=%" PRIu64 " z=%d frame=%" PRIu64,
+                      actualLayers[j].id, actualLayers[j].name.c_str(), actualLayers[j].parent,
+                      actualLayers[j].z, actualLayers[j].curr_frame);
             }
         }
         FAIL();
@@ -220,9 +224,9 @@
         for (size_t j = 0; j < expectedLayers.size(); j++) {
             if (std::find_if(actualLayers.begin(), actualLayers.end(),
                              find_id(expectedLayers[j].id)) == actualLayers.end()) {
-                ALOGD("expectedLayers [%d]:%s parent=%d z=%d frame=%" PRIu64, expectedLayers[j].id,
-                      expectedLayers[j].name.c_str(), expectedLayers[j].parent, expectedLayers[j].z,
-                      expectedLayers[j].curr_frame);
+                ALOGD("expectedLayers [%" PRIu64 "]:%s parent=%" PRIu64 " z=%d frame=%" PRIu64,
+                      expectedLayers[j].id, expectedLayers[j].name.c_str(),
+                      expectedLayers[j].parent, expectedLayers[j].z, expectedLayers[j].curr_frame);
             }
         }
         FAIL();
diff --git a/services/surfaceflinger/tests/tracing/readme.md b/services/surfaceflinger/tests/tracing/readme.md
index 3e80a74..f545a3c 100644
--- a/services/surfaceflinger/tests/tracing/readme.md
+++ b/services/surfaceflinger/tests/tracing/readme.md
@@ -14,7 +14,9 @@
 #### Workflow ####
 Add transaction traces that resulted in front end bugs along
 with the layer trace after fixing the issue. The layer trace
-can be generated by using the layertracegenerator tool. The
+can be generated by using the layertracegenerator tool. Use the
+--last-entry-only flag to generate only the last entry in the
+trace. This will keep the test data to a manageable size. The
 main goal of this test suite is to add regression tests with
 minimal effort.
 
diff --git a/services/surfaceflinger/tests/tracing/testdata/layers_trace_b282110579.winscope b/services/surfaceflinger/tests/tracing/testdata/layers_trace_b282110579.winscope
new file mode 100644
index 0000000..3246453
--- /dev/null
+++ b/services/surfaceflinger/tests/tracing/testdata/layers_trace_b282110579.winscope
Binary files differ
diff --git a/services/surfaceflinger/tests/tracing/testdata/transactions_trace_b282110579.winscope b/services/surfaceflinger/tests/tracing/testdata/transactions_trace_b282110579.winscope
new file mode 100644
index 0000000..ecb9431
--- /dev/null
+++ b/services/surfaceflinger/tests/tracing/testdata/transactions_trace_b282110579.winscope
Binary files differ