libsnapshot: Add a way to detach snapuserd.

Currently, when init unlinks /dev/socket/snapuserd, the daemon currently
polling the socket will fail in accept4(). This works because the
process won't exit until all threads have terminated, but it's
confusing, and it creates log spam.

Instead, add an explicit "detach" message so that snapuserd will stop
listening for new connections and will immediately join on remaining
threads. This way snapuserd will gracefully exit when all threads are
complete, and we're guaranteed no new threads will be created.

This will only be used for first-stage instances of snapuserd.

Bug: 173476209
Test: full OTA with VABC, no accept4() errors
Change-Id: Ibeffa4a35043ed43a70166eeee86204e2b3d03aa
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h
index 2f671c2..aa9ba6e 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h
@@ -70,6 +70,11 @@
     // Wait for snapuserd to disassociate with a dm-user control device. This
     // must ONLY be called if the control device has already been deleted.
     bool WaitForDeviceDelete(const std::string& control_device);
+
+    // Detach snapuserd. This shuts down the listener socket, and will cause
+    // snapuserd to gracefully exit once all handler threads have terminated.
+    // This should only be used on first-stage instances of snapuserd.
+    bool DetachSnapuserd();
 };
 
 }  // namespace snapshot
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h
index cadfd71..1491aac 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h
@@ -41,6 +41,7 @@
     QUERY,
     STOP,
     DELETE,
+    DETACH,
     INVALID,
 };
 
@@ -106,6 +107,7 @@
     bool IsTerminating() { return terminating_; }
 
     void RunThread(std::shared_ptr<DmUserHandler> handler);
+    void JoinAllThreads();
 
     // Find a DmUserHandler within a lock.
     HandlerList::iterator FindHandler(std::lock_guard<std::mutex>* proof_of_lock,
diff --git a/fs_mgr/libsnapshot/snapuserd_client.cpp b/fs_mgr/libsnapshot/snapuserd_client.cpp
index 9ee13db..7282bff 100644
--- a/fs_mgr/libsnapshot/snapuserd_client.cpp
+++ b/fs_mgr/libsnapshot/snapuserd_client.cpp
@@ -210,5 +210,13 @@
     return num_sectors;
 }
 
+bool SnapuserdClient::DetachSnapuserd() {
+    if (!Sendmsg("detach")) {
+        LOG(ERROR) << "Failed to detach snapuserd.";
+        return false;
+    }
+    return true;
+}
+
 }  // namespace snapshot
 }  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd_server.cpp b/fs_mgr/libsnapshot/snapuserd_server.cpp
index 9d57ab0..7a5cead 100644
--- a/fs_mgr/libsnapshot/snapuserd_server.cpp
+++ b/fs_mgr/libsnapshot/snapuserd_server.cpp
@@ -38,6 +38,7 @@
     if (input == "stop") return DaemonOperations::STOP;
     if (input == "query") return DaemonOperations::QUERY;
     if (input == "delete") return DaemonOperations::DELETE;
+    if (input == "detach") return DaemonOperations::DETACH;
 
     return DaemonOperations::INVALID;
 }
@@ -72,19 +73,7 @@
 
 void SnapuserdServer::ShutdownThreads() {
     StopThreads();
-
-    // Acquire the thread list within the lock.
-    std::vector<std::shared_ptr<DmUserHandler>> dm_users;
-    {
-        std::lock_guard<std::mutex> guard(lock_);
-        dm_users = std::move(dm_users_);
-    }
-
-    for (auto& client : dm_users) {
-        auto& th = client->thread();
-
-        if (th.joinable()) th.join();
-    }
+    JoinAllThreads();
 }
 
 const std::string& DmUserHandler::GetMiscName() const {
@@ -214,6 +203,10 @@
             }
             return Sendmsg(fd, "success");
         }
+        case DaemonOperations::DETACH: {
+            terminating_ = true;
+            return Sendmsg(fd, "success");
+        }
         default: {
             LOG(ERROR) << "Received unknown message type from client";
             Sendmsg(fd, "fail");
@@ -234,7 +227,7 @@
 
     LOG(INFO) << "Exiting thread for handler: " << handler->GetMiscName();
 
-    // If the main thread called /emoveHandler, the handler was already removed
+    // If the main thread called RemoveHandler, the handler was already removed
     // from within the lock, and calling RemoveHandler again has no effect.
     RemoveHandler(handler->GetMiscName(), false);
 }
@@ -286,9 +279,26 @@
             }
         }
     }
+
+    JoinAllThreads();
     return true;
 }
 
+void SnapuserdServer::JoinAllThreads() {
+    // Acquire the thread list within the lock.
+    std::vector<std::shared_ptr<DmUserHandler>> dm_users;
+    {
+        std::lock_guard<std::mutex> guard(lock_);
+        dm_users = std::move(dm_users_);
+    }
+
+    for (auto& client : dm_users) {
+        auto& th = client->thread();
+
+        if (th.joinable()) th.join();
+    }
+}
+
 void SnapuserdServer::AddWatchedFd(android::base::borrowed_fd fd) {
     struct pollfd p = {};
     p.fd = fd.get();