snapuserd: Add an API call to wait for device deletion.
This adds a new message to the daemon protocol, which waits for a device
to be deleted. The caller must ensure that the corresponding control
device is actually going away (eg, the device containing the dm-user
table entry has been deleted). Otherwise, this will hang.
This will allow libsnapshot to safely delete the cow since any
outstanding references will be closed.
This also refactors DmUserHandler so that it's freed (and removed from
the handler list) if its corresponding thread exits of its own accord.
Bug: 168554689
Test: vts_libsnapshot_test
Change-Id: I8e97c543eec84874c88795a493470e992dc476fc
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd.h b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd.h
index 2f727d6..80f87d9 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd.h
@@ -77,7 +77,7 @@
int ReadDiskExceptions(chunk_t chunk, size_t size);
int ReadData(chunk_t chunk, size_t size);
- std::string GetControlDevicePath() { return control_device_; }
+ const std::string& GetControlDevicePath() { return control_device_; }
private:
int ProcessReplaceOp(const CowOperation* cow_op);
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h
index dffd481..0bbdaa5 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h
@@ -53,6 +53,10 @@
int RestartSnapuserd(std::vector<std::vector<std::string>>& vec);
bool InitializeSnapuserd(const std::string& cow_device, const std::string& backing_device,
const std::string& control_device);
+
+ // Wait for snapuserd to disassociate with a dm-user control device. This
+ // must ONLY be called if the control device has already been deleted.
+ bool WaitForDeviceDelete(const std::string& control_device);
};
} // namespace snapshot
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h
index 357acac..6bcd666 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h
@@ -21,12 +21,14 @@
#include <functional>
#include <future>
#include <iostream>
+#include <mutex>
#include <sstream>
#include <string>
#include <thread>
#include <vector>
#include <android-base/unique_fd.h>
+#include <libsnapshot/snapuserd.h>
namespace android {
namespace snapshot {
@@ -37,19 +39,23 @@
START,
QUERY,
STOP,
+ DELETE,
INVALID,
};
class DmUserHandler {
private:
- std::unique_ptr<std::thread> threadHandler_;
+ std::thread thread_;
+ std::unique_ptr<Snapuserd> snapuserd_;
public:
- void SetThreadHandler(std::function<void(void)> func) {
- threadHandler_ = std::make_unique<std::thread>(func);
- }
+ explicit DmUserHandler(std::unique_ptr<Snapuserd>&& snapuserd)
+ : snapuserd_(std::move(snapuserd)) {}
- std::unique_ptr<std::thread>& GetThreadHandler() { return threadHandler_; }
+ const std::unique_ptr<Snapuserd>& snapuserd() const { return snapuserd_; }
+ std::thread& thread() { return thread_; }
+
+ const std::string& GetControlDevice() const;
};
class Stoppable {
@@ -61,9 +67,6 @@
virtual ~Stoppable() {}
- virtual void ThreadStart(std::string cow_device, std::string backing_device,
- std::string control_device) = 0;
-
bool StopRequested() {
// checks if value in future object is available
if (futureObj_.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout)
@@ -78,9 +81,11 @@
private:
android::base::unique_fd sockfd_;
bool terminating_;
- std::vector<std::unique_ptr<DmUserHandler>> dm_users_;
std::vector<struct pollfd> watched_fds_;
+ std::mutex lock_;
+ std::vector<std::unique_ptr<DmUserHandler>> dm_users_;
+
void AddWatchedFd(android::base::borrowed_fd fd);
void AcceptClient();
bool HandleClient(android::base::borrowed_fd fd, int revents);
@@ -88,17 +93,21 @@
bool Sendmsg(android::base::borrowed_fd fd, const std::string& msg);
bool Receivemsg(android::base::borrowed_fd fd, const std::string& msg);
- void ThreadStart(std::string cow_device, std::string backing_device,
- std::string control_device) override;
void ShutdownThreads();
+ bool WaitForDelete(const std::string& control_device);
DaemonOperations Resolveop(std::string& input);
std::string GetDaemonStatus();
void Parsemsg(std::string const& msg, const char delim, std::vector<std::string>& out);
void SetTerminating() { terminating_ = true; }
-
bool IsTerminating() { return terminating_; }
+ void RunThread(DmUserHandler* handler);
+
+ // Remove a DmUserHandler from dm_users_, searching by its control device.
+ // If none is found, return nullptr.
+ std::unique_ptr<DmUserHandler> RemoveHandler(const std::string& control_device);
+
public:
SnapuserdServer() { terminating_ = false; }
~SnapuserdServer();
diff --git a/fs_mgr/libsnapshot/snapuserd_client.cpp b/fs_mgr/libsnapshot/snapuserd_client.cpp
index 532e585..fcc41a7 100644
--- a/fs_mgr/libsnapshot/snapuserd_client.cpp
+++ b/fs_mgr/libsnapshot/snapuserd_client.cpp
@@ -123,6 +123,20 @@
return true;
}
+bool SnapuserdClient::WaitForDeviceDelete(const std::string& control_device) {
+ std::string msg = "delete," + control_device;
+ if (!Sendmsg(msg)) {
+ LOG(ERROR) << "Failed to send message " << msg << " to snapuserd";
+ return false;
+ }
+ std::string response = Receivemsg();
+ if (response != "success") {
+ LOG(ERROR) << "Failed waiting to delete device " << control_device;
+ return false;
+ }
+ return true;
+}
+
std::string SnapuserdClient::Receivemsg() {
int ret;
struct timeval tv;
diff --git a/fs_mgr/libsnapshot/snapuserd_server.cpp b/fs_mgr/libsnapshot/snapuserd_server.cpp
index 48a3b2a..6b8cdd9 100644
--- a/fs_mgr/libsnapshot/snapuserd_server.cpp
+++ b/fs_mgr/libsnapshot/snapuserd_server.cpp
@@ -36,6 +36,7 @@
if (input == "start") return DaemonOperations::START;
if (input == "stop") return DaemonOperations::STOP;
if (input == "query") return DaemonOperations::QUERY;
+ if (input == "delete") return DaemonOperations::DELETE;
return DaemonOperations::INVALID;
}
@@ -68,33 +69,25 @@
}
}
-// new thread
-void SnapuserdServer::ThreadStart(std::string cow_device, std::string backing_device,
- std::string control_device) {
- Snapuserd snapd(cow_device, backing_device, control_device);
- if (!snapd.Init()) {
- LOG(ERROR) << "Snapuserd: Init failed";
- return;
- }
-
- while (StopRequested() == false) {
- int ret = snapd.Run();
-
- if (ret < 0) {
- LOG(ERROR) << "Snapuserd: Thread terminating as control device is de-registered";
- break;
- }
- }
-}
-
void SnapuserdServer::ShutdownThreads() {
StopThreads();
- for (auto& client : dm_users_) {
- auto& th = client->GetThreadHandler();
-
- if (th->joinable()) th->join();
+ // Acquire the thread list within the lock.
+ std::vector<std::unique_ptr<DmUserHandler>> dm_users;
+ {
+ std::lock_guard<std::mutex> guard(lock_);
+ dm_users = std::move(dm_users_);
}
+
+ for (auto& client : dm_users) {
+ auto& th = client->thread();
+
+ if (th.joinable()) th.join();
+ }
+}
+
+const std::string& DmUserHandler::GetControlDevice() const {
+ return snapuserd_->GetControlDevicePath();
}
bool SnapuserdServer::Sendmsg(android::base::borrowed_fd fd, const std::string& msg) {
@@ -135,10 +128,25 @@
// start,<cow_device_path>,<source_device_path>,<control_device>
//
// Start the new thread which binds to dm-user misc device
- auto handler = std::make_unique<DmUserHandler>();
- handler->SetThreadHandler(
- std::bind(&SnapuserdServer::ThreadStart, this, out[1], out[2], out[3]));
- dm_users_.push_back(std::move(handler));
+ if (out.size() != 4) {
+ LOG(ERROR) << "Malformed start message, " << out.size() << " parts";
+ return Sendmsg(fd, "fail");
+ }
+
+ auto snapuserd = std::make_unique<Snapuserd>(out[1], out[2], out[3]);
+ if (!snapuserd->Init()) {
+ LOG(ERROR) << "Failed to initialize Snapuserd";
+ return Sendmsg(fd, "fail");
+ }
+
+ auto handler = std::make_unique<DmUserHandler>(std::move(snapuserd));
+ {
+ std::lock_guard<std::mutex> lock(lock_);
+
+ handler->thread() =
+ std::thread(std::bind(&SnapuserdServer::RunThread, this, handler.get()));
+ dm_users_.push_back(std::move(handler));
+ }
return Sendmsg(fd, "success");
}
case DaemonOperations::STOP: {
@@ -162,6 +170,18 @@
// be ready to receive control message.
return Sendmsg(fd, GetDaemonStatus());
}
+ case DaemonOperations::DELETE: {
+ // Message format:
+ // delete,<cow_device_path>
+ if (out.size() != 2) {
+ LOG(ERROR) << "Malformed delete message, " << out.size() << " parts";
+ return Sendmsg(fd, "fail");
+ }
+ if (!WaitForDelete(out[1])) {
+ return Sendmsg(fd, "fail");
+ }
+ return Sendmsg(fd, "success");
+ }
default: {
LOG(ERROR) << "Received unknown message type from client";
Sendmsg(fd, "fail");
@@ -170,6 +190,23 @@
}
}
+void SnapuserdServer::RunThread(DmUserHandler* handler) {
+ while (!StopRequested()) {
+ if (handler->snapuserd()->Run() < 0) {
+ LOG(INFO) << "Snapuserd: Thread terminating as control device is de-registered";
+ break;
+ }
+ }
+
+ if (auto client = RemoveHandler(handler->GetControlDevice())) {
+ // The main thread did not receive a WaitForDelete request for this
+ // control device. Since we transferred ownership within the lock,
+ // we know join() was never called, and will never be called. We can
+ // safely detach here.
+ client->thread().detach();
+ }
+}
+
bool SnapuserdServer::Start(const std::string& socketname) {
sockfd_.reset(android_get_control_socket(socketname.c_str()));
if (sockfd_ >= 0) {
@@ -260,5 +297,37 @@
SetTerminating();
}
+std::unique_ptr<DmUserHandler> SnapuserdServer::RemoveHandler(const std::string& control_device) {
+ std::unique_ptr<DmUserHandler> client;
+ {
+ std::lock_guard<std::mutex> lock(lock_);
+ auto iter = dm_users_.begin();
+ while (iter != dm_users_.end()) {
+ if ((*iter)->GetControlDevice() == control_device) {
+ client = std::move(*iter);
+ iter = dm_users_.erase(iter);
+ break;
+ }
+ iter++;
+ }
+ }
+ return client;
+}
+
+bool SnapuserdServer::WaitForDelete(const std::string& control_device) {
+ auto client = RemoveHandler(control_device);
+
+ // Client already deleted.
+ if (!client) {
+ return true;
+ }
+
+ auto& th = client->thread();
+ if (th.joinable()) {
+ th.join();
+ }
+ return true;
+}
+
} // namespace snapshot
} // namespace android