diff --git a/debuggerd/crasher/arm/crashglue.S b/debuggerd/crasher/arm/crashglue.S
index e4adf40..0def8ae 100644
--- a/debuggerd/crasher/arm/crashglue.S
+++ b/debuggerd/crasher/arm/crashglue.S
@@ -23,10 +23,11 @@
 	ldr lr, [lr]
 	b .
 	.cfi_endproc
+	.size crash1, .-crash1
 
-.globl crashnostack
-.type crashnostack, %function
-crashnostack:
+.globl crash_no_stack
+.type crash_no_stack, %function
+crash_no_stack:
 	.cfi_startproc
 	mov r1, sp
 	.cfi_def_cfa_register r1
@@ -35,3 +36,4 @@
 	ldr r0, [r0]
 	b .
 	.cfi_endproc
+	.size crash_no_stack, .-crash_no_stack
diff --git a/debuggerd/crasher/arm64/crashglue.S b/debuggerd/crasher/arm64/crashglue.S
index 97c824e..c56e19a 100644
--- a/debuggerd/crasher/arm64/crashglue.S
+++ b/debuggerd/crasher/arm64/crashglue.S
@@ -41,11 +41,12 @@
 	ldr x30, [x30]
 	b .
 	.cfi_endproc
+	.size crash1, .-crash1
 
 
-.globl crashnostack
-.type crashnostack, %function
-crashnostack:
+.globl crash_no_stack
+.type crash_no_stack, %function
+crash_no_stack:
 	.cfi_startproc
 	mov x1, sp
 	.cfi_def_cfa_register x1
@@ -54,3 +55,41 @@
 	ldr x0, [x0]
 	b .
 	.cfi_endproc
+	.size crash_no_stack, .-crash_no_stack
+
+
+.globl crash_bti
+.type crash_bti, %function
+crash_bti:
+	.cfi_startproc
+	adr x16, 1f
+	br x16
+1:	// Deliberatly not a bti instruction so we crash here.
+	b .
+	.cfi_endproc
+	.size crash_bti, .-crash_bti
+
+
+.globl crash_pac
+.type crash_pac, %function
+crash_pac:
+	.cfi_startproc
+	paciasp
+	// Since sp is a pac input, this ensures a mismatch.
+	sub sp, sp, #16
+	autiasp
+	b .
+	.cfi_endproc
+	.size crash_pac, .-crash_pac
+
+// Set the PAC and BTI bits for this object file.
+.section .note.gnu.property, "a"
+.balign 8
+.long 4
+.long 0x10
+.long 0x5
+.asciz "GNU"
+.long 0xc0000000
+.long 4
+.long 0x3
+.long 0
diff --git a/debuggerd/crasher/crasher.cpp b/debuggerd/crasher/crasher.cpp
index 12ba502..3b52776 100644
--- a/debuggerd/crasher/crasher.cpp
+++ b/debuggerd/crasher/crasher.cpp
@@ -19,6 +19,7 @@
 #include <assert.h>
 #include <dirent.h>
 #include <errno.h>
+#include <error.h>
 #include <fcntl.h>
 #include <pthread.h>
 #include <signal.h>
@@ -29,6 +30,9 @@
 #include <sys/prctl.h>
 #include <unistd.h>
 
+#include <android-base/file.h>
+#include <android-base/strings.h>
+
 // We test both kinds of logging.
 #include <android-base/logging.h>
 #include <log/log.h>
@@ -59,8 +63,10 @@
 // Avoid name mangling so that stacks are more readable.
 extern "C" {
 
-void crash1(void);
-void crashnostack(void);
+void crash1();
+void crash_no_stack();
+void crash_bti();
+void crash_pac();
 
 int do_action(const char* arg);
 
@@ -196,13 +202,6 @@
     fprintf(stderr, "  fdsan_file            close a file descriptor that's owned by a FILE*\n");
     fprintf(stderr, "  fdsan_dir             close a file descriptor that's owned by a DIR*\n");
     fprintf(stderr, "  seccomp               fail a seccomp check\n");
-#if defined(__arm__)
-    fprintf(stderr, "  kuser_helper_version  call kuser_helper_version\n");
-    fprintf(stderr, "  kuser_get_tls         call kuser_get_tls\n");
-    fprintf(stderr, "  kuser_cmpxchg         call kuser_cmpxchg\n");
-    fprintf(stderr, "  kuser_memory_barrier  call kuser_memory_barrier\n");
-    fprintf(stderr, "  kuser_cmpxchg64       call kuser_cmpxchg64\n");
-#endif
     fprintf(stderr, "  xom                   read execute-only memory\n");
     fprintf(stderr, "\n");
     fprintf(stderr, "  LOG_ALWAYS_FATAL      call liblog LOG_ALWAYS_FATAL\n");
@@ -223,6 +222,20 @@
     fprintf(stderr, "\n");
     fprintf(stderr, "  no_new_privs          set PR_SET_NO_NEW_PRIVS and then abort\n");
     fprintf(stderr, "\n");
+#if defined(__arm__)
+    fprintf(stderr, "Also, since this is an arm32 binary:\n");
+    fprintf(stderr, "  kuser_helper_version  call kuser_helper_version\n");
+    fprintf(stderr, "  kuser_get_tls         call kuser_get_tls\n");
+    fprintf(stderr, "  kuser_cmpxchg         call kuser_cmpxchg\n");
+    fprintf(stderr, "  kuser_memory_barrier  call kuser_memory_barrier\n");
+    fprintf(stderr, "  kuser_cmpxchg64       call kuser_cmpxchg64\n");
+#endif
+#if defined(__aarch64__)
+    fprintf(stderr, "Also, since this is an arm64 binary:\n");
+    fprintf(stderr, "  bti                   fail a branch target identification (BTI) check\n");
+    fprintf(stderr, "  pac                   fail a pointer authentication (PAC) check\n");
+#endif
+    fprintf(stderr, "\n");
     fprintf(stderr, "prefix any of the above with 'thread-' to run on a new thread\n");
     fprintf(stderr, "prefix any of the above with 'exhaustfd-' to exhaust\n");
     fprintf(stderr, "all available file descriptors before crashing.\n");
@@ -231,6 +244,21 @@
     return EXIT_FAILURE;
 }
 
+[[maybe_unused]] static void CheckCpuFeature(const std::string& name) {
+    std::string cpuinfo;
+    if (!android::base::ReadFileToString("/proc/cpuinfo", &cpuinfo)) {
+        error(1, errno, "couldn't read /proc/cpuinfo");
+    }
+    std::vector<std::string> lines = android::base::Split(cpuinfo, "\n");
+    for (std::string_view line : lines) {
+        if (!android::base::ConsumePrefix(&line, "Features\t:")) continue;
+        std::vector<std::string> features = android::base::Split(std::string(line), " ");
+        if (std::find(features.begin(), features.end(), name) == features.end()) {
+          error(1, 0, "/proc/cpuinfo does not report feature '%s'", name.c_str());
+        }
+    }
+}
+
 noinline int do_action(const char* arg) {
     // Prefixes.
     if (!strncmp(arg, "wait-", strlen("wait-"))) {
@@ -256,7 +284,7 @@
     } else if (!strcasecmp(arg, "stack-overflow")) {
       overflow_stack(nullptr);
     } else if (!strcasecmp(arg, "nostack")) {
-      crashnostack();
+      crash_no_stack();
     } else if (!strcasecmp(arg, "exit")) {
       exit(1);
     } else if (!strcasecmp(arg, "call-null")) {
@@ -350,6 +378,14 @@
     } else if (!strcasecmp(arg, "kuser_cmpxchg64")) {
         return __kuser_cmpxchg64(0, 0, 0);
 #endif
+#if defined(__aarch64__)
+    } else if (!strcasecmp(arg, "bti")) {
+        CheckCpuFeature("bti");
+        crash_bti();
+    } else if (!strcasecmp(arg, "pac")) {
+        CheckCpuFeature("paca");
+        crash_pac();
+#endif
     } else if (!strcasecmp(arg, "no_new_privs")) {
         if (prctl(PR_SET_NO_NEW_PRIVS, 1) != 0) {
           fprintf(stderr, "prctl(PR_SET_NO_NEW_PRIVS, 1) failed: %s\n", strerror(errno));
diff --git a/debuggerd/crasher/riscv64/crashglue.S b/debuggerd/crasher/riscv64/crashglue.S
index 42f59b3..f179e33 100644
--- a/debuggerd/crasher/riscv64/crashglue.S
+++ b/debuggerd/crasher/riscv64/crashglue.S
@@ -43,10 +43,11 @@
 	ld t2, 0(zero)
 	j .
 	.cfi_endproc
+	.size crash1, .-crash1
 
 
-.globl crashnostack
-crashnostack:
+.globl crash_no_stack
+crash_no_stack:
 	.cfi_startproc
 	mv t1, sp
 	.cfi_def_cfa_register t1
@@ -54,3 +55,4 @@
 	ld t2, 0(zero)
 	j .
 	.cfi_endproc
+	.size crash_no_stack, .-crash_no_stack
diff --git a/debuggerd/crasher/x86/crashglue.S b/debuggerd/crasher/x86/crashglue.S
index e8eb3a7..453035b 100644
--- a/debuggerd/crasher/x86/crashglue.S
+++ b/debuggerd/crasher/x86/crashglue.S
@@ -6,13 +6,15 @@
 
 	movl $0, %edx
 	jmp *%edx
+	.size crash1, .-crash1
 
 
-.globl crashnostack
-crashnostack:
+.globl crash_no_stack
+crash_no_stack:
 	.cfi_startproc
 	movl %esp, %eax
 	.cfi_def_cfa_register %eax
 	movl $0, %esp
 	movl (%esp), %ebx
 	.cfi_endproc
+	.size crash_no_stack, .-crash_no_stack
diff --git a/debuggerd/crasher/x86_64/crashglue.S b/debuggerd/crasher/x86_64/crashglue.S
index 8f67214..c3d39c4 100644
--- a/debuggerd/crasher/x86_64/crashglue.S
+++ b/debuggerd/crasher/x86_64/crashglue.S
@@ -6,13 +6,15 @@
 
 	movl $0, %edx
 	jmp *%rdx
+	.size crash1, .-crash1
 
 
-.globl crashnostack
-crashnostack:
+.globl crash_no_stack
+crash_no_stack:
 	.cfi_startproc
 	movq %rsp, %rax
 	.cfi_def_cfa_register %rax
 	movq $0, %rsp
 	movq (%rsp), %rbx
 	.cfi_endproc
+	.size crash_no_stack, .-crash_no_stack
diff --git a/debuggerd/handler/debuggerd_handler.cpp b/debuggerd/handler/debuggerd_handler.cpp
index 1e5365d..01365f2 100644
--- a/debuggerd/handler/debuggerd_handler.cpp
+++ b/debuggerd/handler/debuggerd_handler.cpp
@@ -552,8 +552,14 @@
   }
 
   debugger_process_info process_info = {};
+  if (g_callbacks.get_process_info) {
+    process_info = g_callbacks.get_process_info();
+  }
   uintptr_t si_val = reinterpret_cast<uintptr_t>(info->si_ptr);
   if (signal_number == BIONIC_SIGNAL_DEBUGGER) {
+    // Applications can set abort messages via android_set_abort_message without
+    // actually aborting; ignore those messages in non-fatal dumps.
+    process_info.abort_msg = nullptr;
     if (info->si_code == SI_QUEUE && info->si_pid == __getpid()) {
       // Allow for the abort message to be explicitly specified via the sigqueue value.
       // Keep the bottom bit intact for representing whether we want a backtrace or a tombstone.
@@ -562,8 +568,6 @@
         info->si_ptr = reinterpret_cast<void*>(si_val & 1);
       }
     }
-  } else if (g_callbacks.get_process_info) {
-    process_info = g_callbacks.get_process_info();
   }
 
   gwp_asan_callbacks_t gwp_asan_callbacks = {};
diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp
index 0bd07ed..71a228e 100644
--- a/fastboot/fastboot.cpp
+++ b/fastboot/fastboot.cpp
@@ -1801,6 +1801,7 @@
     CancelSnapshotIfNeeded();
 
     tasks_ = CollectTasks();
+
     for (auto& task : tasks_) {
         task->Run();
     }
@@ -1815,7 +1816,18 @@
     } else {
         tasks = CollectTasksFromImageList();
     }
-
+    if (fp_->exclude_dynamic_partitions) {
+        auto is_non_static_flash_task = [](const auto& task) -> bool {
+            if (auto flash_task = task->AsFlashTask()) {
+                if (!should_flash_in_userspace(flash_task->GetPartitionAndSlot())) {
+                    return false;
+                }
+            }
+            return true;
+        };
+        tasks.erase(std::remove_if(tasks.begin(), tasks.end(), is_non_static_flash_task),
+                    tasks.end());
+    }
     return tasks;
 }
 
@@ -1944,7 +1956,6 @@
     }
     ZipImageSource zp = ZipImageSource(zip);
     fp->source = &zp;
-    fp->wants_wipe = false;
     FlashAllTool tool(fp);
     tool.Flash();
 
@@ -2220,6 +2231,7 @@
                                       {"disable-verification", no_argument, 0, 0},
                                       {"disable-verity", no_argument, 0, 0},
                                       {"disable-super-optimization", no_argument, 0, 0},
+                                      {"exclude-dynamic-partitions", no_argument, 0, 0},
                                       {"disable-fastboot-info", no_argument, 0, 0},
                                       {"force", no_argument, 0, 0},
                                       {"fs-options", required_argument, 0, 0},
@@ -2261,6 +2273,9 @@
                 g_disable_verity = true;
             } else if (name == "disable-super-optimization") {
                 fp->should_optimize_flash_super = false;
+            } else if (name == "exclude-dynamic-partitions") {
+                fp->exclude_dynamic_partitions = true;
+                fp->should_optimize_flash_super = false;
             } else if (name == "disable-fastboot-info") {
                 fp->should_use_fastboot_info = false;
             } else if (name == "force") {
diff --git a/fastboot/fastboot.h b/fastboot/fastboot.h
index dc57149..75b8d29 100644
--- a/fastboot/fastboot.h
+++ b/fastboot/fastboot.h
@@ -99,6 +99,7 @@
     bool force_flash = false;
     bool should_optimize_flash_super = true;
     bool should_use_fastboot_info = true;
+    bool exclude_dynamic_partitions = false;
     uint64_t sparse_limit = 0;
 
     std::string slot_override;
diff --git a/fastboot/task.cpp b/fastboot/task.cpp
index bf64f0e..146064c 100644
--- a/fastboot/task.cpp
+++ b/fastboot/task.cpp
@@ -32,7 +32,7 @@
 
 void FlashTask::Run() {
     auto flash = [&](const std::string& partition) {
-        if (should_flash_in_userspace(partition) && !is_userspace_fastboot()) {
+        if (should_flash_in_userspace(partition) && !is_userspace_fastboot() && !fp_->force_flash) {
             die("The partition you are trying to flash is dynamic, and "
                 "should be flashed via fastbootd. Please run:\n"
                 "\n"
diff --git a/fs_mgr/Android.bp b/fs_mgr/Android.bp
index 4e4d20e..87db98b 100644
--- a/fs_mgr/Android.bp
+++ b/fs_mgr/Android.bp
@@ -171,6 +171,25 @@
     ],
 }
 
+cc_library_static {
+    name: "libfs_mgr_file_wait",
+    defaults: ["fs_mgr_defaults"],
+    export_include_dirs: ["include"],
+    cflags: [
+        "-D_FILE_OFFSET_BITS=64",
+    ],
+    srcs: [
+        "file_wait.cpp",
+    ],
+    shared_libs: [
+        "libbase",
+    ],
+    host_supported: true,
+    ramdisk_available: true,
+    vendor_ramdisk_available: true,
+    recovery_available: true,
+}
+
 cc_binary {
     name: "remount",
     defaults: ["fs_mgr_defaults"],
diff --git a/fs_mgr/fs_mgr_remount.cpp b/fs_mgr/fs_mgr_remount.cpp
index 4c45828..4b3a5d3 100644
--- a/fs_mgr/fs_mgr_remount.cpp
+++ b/fs_mgr/fs_mgr_remount.cpp
@@ -128,12 +128,11 @@
 }
 
 static android::sp<android::os::IVold> GetVold() {
+    auto sm = android::defaultServiceManager();
     while (true) {
-        if (auto sm = android::defaultServiceManager()) {
-            if (auto binder = sm->getService(android::String16("vold"))) {
-                if (auto vold = android::interface_cast<android::os::IVold>(binder)) {
-                    return vold;
-                }
+        if (auto binder = sm->checkService(android::String16("vold"))) {
+            if (auto vold = android::interface_cast<android::os::IVold>(binder)) {
+                return vold;
             }
         }
         std::this_thread::sleep_for(2s);
diff --git a/fs_mgr/libfiemap/Android.bp b/fs_mgr/libfiemap/Android.bp
index ddda648..c8d5756 100644
--- a/fs_mgr/libfiemap/Android.bp
+++ b/fs_mgr/libfiemap/Android.bp
@@ -24,6 +24,7 @@
     vendor_ramdisk_available: true,
     recovery_available: true,
     export_include_dirs: ["include"],
+    host_supported: true,
 }
 
 filegroup {
diff --git a/fs_mgr/libfstab/boot_config.cpp b/fs_mgr/libfstab/boot_config.cpp
index 53c843e..b21495e 100644
--- a/fs_mgr/libfstab/boot_config.cpp
+++ b/fs_mgr/libfstab/boot_config.cpp
@@ -17,11 +17,9 @@
 #include <algorithm>
 #include <iterator>
 #include <string>
-#include <vector>
 
 #include <android-base/file.h>
 #include <android-base/properties.h>
-#include <android-base/stringprintf.h>
 #include <android-base/strings.h>
 
 #include "fstab_priv.h"
@@ -35,7 +33,7 @@
     static const std::string kAndroidDtDir = [] {
         std::string android_dt_dir;
         if ((GetBootconfig("androidboot.android_dt_dir", &android_dt_dir) ||
-             fs_mgr_get_boot_config_from_kernel_cmdline("android_dt_dir", &android_dt_dir)) &&
+             GetKernelCmdline("androidboot.android_dt_dir", &android_dt_dir)) &&
             !android_dt_dir.empty()) {
             // Ensure the returned path ends with a /
             if (android_dt_dir.back() != '/') {
@@ -110,13 +108,10 @@
     return GetBootconfigFromString(bootconfig, key, out);
 }
 
-}  // namespace fs_mgr
-}  // namespace android
-
-std::vector<std::pair<std::string, std::string>> fs_mgr_parse_cmdline(const std::string& cmdline) {
+void ImportKernelCmdlineFromString(const std::string& cmdline,
+                                   const std::function<void(std::string, std::string)>& fn) {
     static constexpr char quote = '"';
 
-    std::vector<std::pair<std::string, std::string>> result;
     size_t base = 0;
     while (true) {
         // skip quoted spans
@@ -127,56 +122,54 @@
             if ((found = cmdline.find(quote, found + 1)) == cmdline.npos) break;
             ++found;
         }
-        std::string piece;
-        auto source = cmdline.substr(base, found - base);
-        std::remove_copy(source.begin(), source.end(),
-                         std::back_insert_iterator<std::string>(piece), quote);
+        std::string piece = cmdline.substr(base, found - base);
+        piece.erase(std::remove(piece.begin(), piece.end(), quote), piece.end());
         auto equal_sign = piece.find('=');
         if (equal_sign == piece.npos) {
             if (!piece.empty()) {
                 // no difference between <key> and <key>=
-                result.emplace_back(std::move(piece), "");
+                fn(std::move(piece), "");
             }
         } else {
-            result.emplace_back(piece.substr(0, equal_sign), piece.substr(equal_sign + 1));
+            std::string value = piece.substr(equal_sign + 1);
+            piece.resize(equal_sign);
+            fn(std::move(piece), std::move(value));
         }
         if (found == cmdline.npos) break;
         base = found + 1;
     }
-
-    return result;
 }
 
-bool fs_mgr_get_boot_config_from_kernel(const std::string& cmdline, const std::string& android_key,
-                                        std::string* out_val) {
-    FSTAB_CHECK(out_val != nullptr);
-
-    const std::string cmdline_key("androidboot." + android_key);
-    for (const auto& [key, value] : fs_mgr_parse_cmdline(cmdline)) {
-        if (key == cmdline_key) {
-            *out_val = value;
-            return true;
+bool GetKernelCmdlineFromString(const std::string& cmdline, const std::string& key,
+                                std::string* out) {
+    bool found = false;
+    ImportKernelCmdlineFromString(cmdline, [&](std::string config_key, std::string value) {
+        if (!found && config_key == key) {
+            *out = std::move(value);
+            found = true;
         }
-    }
-
-    *out_val = "";
-    return false;
+    });
+    return found;
 }
 
-// Tries to get the given boot config value from kernel cmdline.
-// Returns true if successfully found, false otherwise.
-bool fs_mgr_get_boot_config_from_kernel_cmdline(const std::string& key, std::string* out_val) {
+void ImportKernelCmdline(const std::function<void(std::string, std::string)>& fn) {
     std::string cmdline;
-    if (!android::base::ReadFileToString("/proc/cmdline", &cmdline)) return false;
-    if (!cmdline.empty() && cmdline.back() == '\n') {
-        cmdline.pop_back();
-    }
-    return fs_mgr_get_boot_config_from_kernel(cmdline, key, out_val);
+    android::base::ReadFileToString("/proc/cmdline", &cmdline);
+    ImportKernelCmdlineFromString(android::base::Trim(cmdline), fn);
 }
 
-// Tries to get the boot config value in device tree, properties and
-// kernel cmdline (in that order).  Returns 'true' if successfully
-// found, 'false' otherwise.
+bool GetKernelCmdline(const std::string& key, std::string* out) {
+    std::string cmdline;
+    android::base::ReadFileToString("/proc/cmdline", &cmdline);
+    return GetKernelCmdlineFromString(android::base::Trim(cmdline), key, out);
+}
+
+}  // namespace fs_mgr
+}  // namespace android
+
+// Tries to get the boot config value in device tree, properties, kernel bootconfig and kernel
+// cmdline (in that order).
+// Returns 'true' if successfully found, 'false' otherwise.
 bool fs_mgr_get_boot_config(const std::string& key, std::string* out_val) {
     FSTAB_CHECK(out_val != nullptr);
 
@@ -204,7 +197,7 @@
     }
 
     // finally, fallback to kernel cmdline, properties may not be ready yet
-    if (fs_mgr_get_boot_config_from_kernel_cmdline(key, out_val)) {
+    if (android::fs_mgr::GetKernelCmdline(config_key, out_val)) {
         return true;
     }
 
diff --git a/fs_mgr/libfstab/fstab.cpp b/fs_mgr/libfstab/fstab.cpp
index 3cbd8c8..32460b1 100644
--- a/fs_mgr/libfstab/fstab.cpp
+++ b/fs_mgr/libfstab/fstab.cpp
@@ -831,9 +831,8 @@
     Fstab default_fstab;
     const std::string default_fstab_path = GetFstabPath();
     if (!default_fstab_path.empty() && ReadFstabFromFile(default_fstab_path, &default_fstab)) {
-        for (auto&& entry : default_fstab) {
-            fstab->emplace_back(std::move(entry));
-        }
+        fstab->insert(fstab->end(), std::make_move_iterator(default_fstab.begin()),
+                      std::make_move_iterator(default_fstab.end()));
     } else {
         LINFO << __FUNCTION__ << "(): failed to find device default fstab";
     }
@@ -863,11 +862,11 @@
 }
 
 std::set<std::string> GetBootDevices() {
+    std::set<std::string> boot_devices;
     // First check bootconfig, then kernel commandline, then the device tree
     std::string value;
     if (GetBootconfig("androidboot.boot_devices", &value) ||
         GetBootconfig("androidboot.boot_device", &value)) {
-        std::set<std::string> boot_devices;
         // split by spaces and trim the trailing comma.
         for (std::string_view device : android::base::Split(value, " ")) {
             base::ConsumeSuffix(&device, ",");
@@ -876,25 +875,20 @@
         return boot_devices;
     }
 
-    std::string dt_file_name = GetAndroidDtDir() + "boot_devices";
-    if (fs_mgr_get_boot_config_from_kernel_cmdline("boot_devices", &value) ||
-        ReadDtFile(dt_file_name, &value)) {
-        auto boot_devices = Split(value, ",");
-        return std::set<std::string>(boot_devices.begin(), boot_devices.end());
+    const std::string dt_file_name = GetAndroidDtDir() + "boot_devices";
+    if (GetKernelCmdline("androidboot.boot_devices", &value) || ReadDtFile(dt_file_name, &value)) {
+        auto boot_devices_list = Split(value, ",");
+        return {std::make_move_iterator(boot_devices_list.begin()),
+                std::make_move_iterator(boot_devices_list.end())};
     }
 
-    std::string cmdline;
-    if (android::base::ReadFileToString("/proc/cmdline", &cmdline)) {
-        std::set<std::string> boot_devices;
-        const std::string cmdline_key = "androidboot.boot_device";
-        for (const auto& [key, value] : fs_mgr_parse_cmdline(cmdline)) {
-            if (key == cmdline_key) {
-                boot_devices.emplace(value);
-            }
+    ImportKernelCmdline([&](std::string key, std::string value) {
+        if (key == "androidboot.boot_device") {
+            boot_devices.emplace(std::move(value));
         }
-        if (!boot_devices.empty()) {
-            return boot_devices;
-        }
+    });
+    if (!boot_devices.empty()) {
+        return boot_devices;
     }
 
     // Fallback to extract boot devices from fstab.
diff --git a/fs_mgr/libfstab/fstab_priv.h b/fs_mgr/libfstab/fstab_priv.h
index cc56825..5105da0 100644
--- a/fs_mgr/libfstab/fstab_priv.h
+++ b/fs_mgr/libfstab/fstab_priv.h
@@ -18,17 +18,11 @@
 
 #include <functional>
 #include <string>
-#include <utility>
-#include <vector>
 
 #include <fstab/fstab.h>
 
 // Do not include logging_macros.h here as this header is used by fs_mgr, too.
 
-std::vector<std::pair<std::string, std::string>> fs_mgr_parse_cmdline(const std::string& cmdline);
-bool fs_mgr_get_boot_config_from_kernel(const std::string& cmdline, const std::string& key,
-                                        std::string* out_val);
-bool fs_mgr_get_boot_config_from_kernel_cmdline(const std::string& key, std::string* out_val);
 bool fs_mgr_get_boot_config(const std::string& key, std::string* out_val);
 
 bool fs_mgr_update_for_slotselect(android::fs_mgr::Fstab* fstab);
@@ -48,5 +42,11 @@
 bool GetBootconfigFromString(const std::string& bootconfig, const std::string& key,
                              std::string* out);
 
+void ImportKernelCmdlineFromString(const std::string& cmdline,
+                                   const std::function<void(std::string, std::string)>& fn);
+
+bool GetKernelCmdlineFromString(const std::string& cmdline, const std::string& key,
+                                std::string* out);
+
 }  // namespace fs_mgr
 }  // namespace android
diff --git a/fs_mgr/libfstab/include/fstab/fstab.h b/fs_mgr/libfstab/include/fstab/fstab.h
index 79a07ee..150a47d 100644
--- a/fs_mgr/libfstab/include/fstab/fstab.h
+++ b/fs_mgr/libfstab/include/fstab/fstab.h
@@ -137,5 +137,13 @@
 // Otherwise returns false and |*out| is not modified.
 bool GetBootconfig(const std::string& key, std::string* out);
 
+// Import the kernel cmdline by calling the callback |fn| with each key-value pair.
+void ImportKernelCmdline(const std::function<void(std::string, std::string)>& fn);
+
+// Get the kernel cmdline value for |key|.
+// Returns true if |key| is found in the kernel cmdline.
+// Otherwise returns false and |*out| is not modified.
+bool GetKernelCmdline(const std::string& key, std::string* out);
+
 }  // namespace fs_mgr
 }  // namespace android
diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp
index bd017ff..04b5a02 100644
--- a/fs_mgr/libsnapshot/Android.bp
+++ b/fs_mgr/libsnapshot/Android.bp
@@ -44,7 +44,7 @@
         "libext2_uuid",
         "libext4_utils",
         "libfstab",
-        "libsnapshot_snapuserd",
+        "libsnapuserd_client",
         "libz",
     ],
     header_libs: [
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
index 3a81f63..c9a4dee 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
@@ -161,6 +161,10 @@
     kCowCompressLz4 = 3,
     kCowCompressZstd = 4,
 };
+struct CowCompression {
+    CowCompressionAlgorithm algorithm = kCowCompressNone;
+    uint32_t compression_level = 0;
+};
 
 static constexpr uint8_t kCowReadAheadNotStarted = 0;
 static constexpr uint8_t kCowReadAheadInProgress = 1;
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
index d6194eb..74b8bb8 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
@@ -18,14 +18,11 @@
 
 #include <condition_variable>
 #include <cstdint>
-#include <future>
 #include <memory>
 #include <mutex>
 #include <optional>
 #include <queue>
 #include <string>
-#include <thread>
-#include <utility>
 #include <vector>
 
 #include <android-base/unique_fd.h>
@@ -110,16 +107,17 @@
 
 class CompressWorker {
   public:
-    CompressWorker(CowCompressionAlgorithm compression, uint32_t block_size);
+    CompressWorker(CowCompression compression, uint32_t block_size);
     bool RunThread();
     void EnqueueCompressBlocks(const void* buffer, size_t num_blocks);
     bool GetCompressedBuffers(std::vector<std::basic_string<uint8_t>>* compressed_buf);
     void Finalize();
-    static std::basic_string<uint8_t> Compress(CowCompressionAlgorithm compression,
-                                               const void* data, size_t length);
+    static uint32_t GetDefaultCompressionLevel(CowCompressionAlgorithm compression);
+    static std::basic_string<uint8_t> Compress(CowCompression compression, const void* data,
+                                               size_t length);
 
-    static bool CompressBlocks(CowCompressionAlgorithm compression, size_t block_size,
-                               const void* buffer, size_t num_blocks,
+    static bool CompressBlocks(CowCompression compression, size_t block_size, const void* buffer,
+                               size_t num_blocks,
                                std::vector<std::basic_string<uint8_t>>* compressed_data);
 
   private:
@@ -130,7 +128,7 @@
         std::vector<std::basic_string<uint8_t>> compressed_data;
     };
 
-    CowCompressionAlgorithm compression_;
+    CowCompression compression_;
     uint32_t block_size_;
 
     std::queue<CompressWork> work_queue_;
@@ -139,7 +137,6 @@
     std::condition_variable cv_;
     bool stopped_ = false;
 
-    std::basic_string<uint8_t> Compress(const void* data, size_t length);
     bool CompressBlocks(const void* buffer, size_t num_blocks,
                         std::vector<std::basic_string<uint8_t>>* compressed_data);
 };
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp
index a4a0ad6..96d6016 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp
@@ -46,24 +46,47 @@
     } else if (name == "none" || name.empty()) {
         return {kCowCompressNone};
     } else {
+        LOG(ERROR) << "unable to determine default compression algorithm for: " << name;
         return {};
     }
 }
 
-std::basic_string<uint8_t> CompressWorker::Compress(const void* data, size_t length) {
-    return Compress(compression_, data, length);
+// 1. Default compression level is determined by compression algorithm
+// 2. There might be compatibility issues if a value is changed here, as  some older versions of
+// Android will assume a different compression level, causing cow_size estimation differences that
+// will lead to OTA failure. Ensure that the device and OTA package use the same compression level
+// for OTA to succeed.
+uint32_t CompressWorker::GetDefaultCompressionLevel(CowCompressionAlgorithm compression) {
+    switch (compression) {
+        case kCowCompressGz: {
+            return Z_BEST_COMPRESSION;
+        }
+        case kCowCompressBrotli: {
+            return BROTLI_DEFAULT_QUALITY;
+        }
+        case kCowCompressLz4: {
+            break;
+        }
+        case kCowCompressZstd: {
+            return ZSTD_defaultCLevel();
+        }
+        case kCowCompressNone: {
+            break;
+        }
+    }
+    return 0;
 }
 
-std::basic_string<uint8_t> CompressWorker::Compress(CowCompressionAlgorithm compression,
-                                                    const void* data, size_t length) {
-    switch (compression) {
+std::basic_string<uint8_t> CompressWorker::Compress(CowCompression compression, const void* data,
+                                                    size_t length) {
+    switch (compression.algorithm) {
         case kCowCompressGz: {
             const auto bound = compressBound(length);
             std::basic_string<uint8_t> buffer(bound, '\0');
 
             uLongf dest_len = bound;
             auto rv = compress2(buffer.data(), &dest_len, reinterpret_cast<const Bytef*>(data),
-                                length, Z_BEST_COMPRESSION);
+                                length, compression.compression_level);
             if (rv != Z_OK) {
                 LOG(ERROR) << "compress2 returned: " << rv;
                 return {};
@@ -81,8 +104,8 @@
 
             size_t encoded_size = bound;
             auto rv = BrotliEncoderCompress(
-                    BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, length,
-                    reinterpret_cast<const uint8_t*>(data), &encoded_size, buffer.data());
+                    compression.compression_level, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE,
+                    length, reinterpret_cast<const uint8_t*>(data), &encoded_size, buffer.data());
             if (!rv) {
                 LOG(ERROR) << "BrotliEncoderCompress failed";
                 return {};
@@ -117,8 +140,8 @@
         }
         case kCowCompressZstd: {
             std::basic_string<uint8_t> buffer(ZSTD_compressBound(length), '\0');
-            const auto compressed_size =
-                    ZSTD_compress(buffer.data(), buffer.size(), data, length, 0);
+            const auto compressed_size = ZSTD_compress(buffer.data(), buffer.size(), data, length,
+                                                       compression.compression_level);
             if (compressed_size <= 0) {
                 LOG(ERROR) << "ZSTD compression failed " << compressed_size;
                 return {};
@@ -133,7 +156,7 @@
             return buffer;
         }
         default:
-            LOG(ERROR) << "unhandled compression type: " << compression;
+            LOG(ERROR) << "unhandled compression type: " << compression.algorithm;
             break;
     }
     return {};
@@ -143,7 +166,7 @@
     return CompressBlocks(compression_, block_size_, buffer, num_blocks, compressed_data);
 }
 
-bool CompressWorker::CompressBlocks(CowCompressionAlgorithm compression, size_t block_size,
+bool CompressWorker::CompressBlocks(CowCompression compression, size_t block_size,
                                     const void* buffer, size_t num_blocks,
                                     std::vector<std::basic_string<uint8_t>>* compressed_data) {
     const uint8_t* iter = reinterpret_cast<const uint8_t*>(buffer);
@@ -255,7 +278,7 @@
     cv_.notify_all();
 }
 
-CompressWorker::CompressWorker(CowCompressionAlgorithm compression, uint32_t block_size)
+CompressWorker::CompressWorker(CowCompression compression, uint32_t block_size)
     : compression_(compression), block_size_(block_size) {}
 
 }  // namespace snapshot
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_decompress.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/cow_decompress.cpp
index da90cc0..3692c1a 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_decompress.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/cow_decompress.cpp
@@ -18,6 +18,7 @@
 
 #include <array>
 #include <cstring>
+#include <memory>
 #include <utility>
 #include <vector>
 
@@ -62,6 +63,8 @@
         return IDecompressor::Brotli();
     } else if (compressor == "gz") {
         return IDecompressor::Gz();
+    } else if (compressor == "zstd") {
+        return IDecompressor::Zstd();
     } else {
         return nullptr;
     }
@@ -211,10 +214,6 @@
     return true;
 }
 
-std::unique_ptr<IDecompressor> IDecompressor::Gz() {
-    return std::unique_ptr<IDecompressor>(new GzDecompressor());
-}
-
 class BrotliDecompressor final : public StreamDecompressor {
   public:
     ~BrotliDecompressor();
@@ -275,10 +274,6 @@
     return true;
 }
 
-std::unique_ptr<IDecompressor> IDecompressor::Brotli() {
-    return std::unique_ptr<IDecompressor>(new BrotliDecompressor());
-}
-
 class Lz4Decompressor final : public IDecompressor {
   public:
     ~Lz4Decompressor() override = default;
@@ -382,6 +377,14 @@
     }
 };
 
+std::unique_ptr<IDecompressor> IDecompressor::Brotli() {
+    return std::make_unique<BrotliDecompressor>();
+}
+
+std::unique_ptr<IDecompressor> IDecompressor::Gz() {
+    return std::make_unique<GzDecompressor>();
+}
+
 std::unique_ptr<IDecompressor> IDecompressor::Lz4() {
     return std::make_unique<Lz4Decompressor>();
 }
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp
index 31b9a58..2258d9f 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp
@@ -472,14 +472,15 @@
     if (strcmp(GetParam(), "none") == 0) {
         GTEST_SKIP();
     }
-
+    CowCompression compression;
     auto algorithm = CompressionAlgorithmFromString(GetParam());
     ASSERT_TRUE(algorithm.has_value());
+    compression.algorithm = algorithm.value();
 
     std::string expected = "The quick brown fox jumps over the lazy dog.";
     expected.resize(4096, '\0');
 
-    auto result = CompressWorker::Compress(*algorithm, expected.data(), expected.size());
+    auto result = CompressWorker::Compress(compression, expected.data(), expected.size());
     ASSERT_FALSE(result.empty());
 
     HorribleStream<uint8_t> stream(result);
@@ -1408,6 +1409,18 @@
     ASSERT_TRUE(iter->AtEnd());
 }
 
+TEST_F(CowTest, ParseOptionsTest) {
+    CowOptions options;
+    std::vector<std::pair<std::string, bool>> testcases = {
+            {"gz,4", true},   {"gz,4,4", false}, {"lz4,4", true}, {"brotli,4", true},
+            {"zstd,4", true}, {"zstd,x", false}, {"zs,4", false}, {"zstd.4", false}};
+    for (size_t i = 0; i < testcases.size(); i++) {
+        options.compression = testcases[i].first;
+        CowWriterV2 writer(options, GetCowFd());
+        ASSERT_EQ(writer.Initialize(), testcases[i].second);
+    }
+}
+
 TEST_F(CowTest, LegacyRevMergeOpItrTest) {
     CowOptions options;
     options.cluster_ops = 5;
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp
index c549969..6d04c6a 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp
@@ -20,8 +20,8 @@
 #include <sys/uio.h>
 #include <unistd.h>
 
+#include <future>
 #include <limits>
-#include <queue>
 
 #include <android-base/file.h>
 #include <android-base/logging.h>
@@ -39,6 +39,8 @@
 #include <sys/ioctl.h>
 #include <unistd.h>
 
+#include "android-base/parseint.h"
+#include "android-base/strings.h"
 #include "parser_v2.h"
 
 // The info messages here are spammy, but as useful for update_engine. Disable
@@ -119,12 +121,29 @@
 }
 
 bool CowWriterV2::ParseOptions() {
-    auto algorithm = CompressionAlgorithmFromString(options_.compression);
+    auto parts = android::base::Split(options_.compression, ",");
+
+    if (parts.size() > 2) {
+        LOG(ERROR) << "failed to parse compression parameters: invalid argument count: "
+                   << parts.size() << " " << options_.compression;
+        return false;
+    }
+    auto algorithm = CompressionAlgorithmFromString(parts[0]);
     if (!algorithm) {
         LOG(ERROR) << "unrecognized compression: " << options_.compression;
         return false;
     }
-    compression_ = *algorithm;
+    if (parts.size() > 1) {
+        if (!android::base::ParseUint(parts[1], &compression_.compression_level)) {
+            LOG(ERROR) << "failed to parse compression level invalid type: " << parts[1];
+            return false;
+        }
+    } else {
+        compression_.compression_level =
+                CompressWorker::GetDefaultCompressionLevel(algorithm.value());
+    }
+
+    compression_.algorithm = *algorithm;
 
     if (options_.cluster_ops == 1) {
         LOG(ERROR) << "Clusters must contain at least two operations to function.";
@@ -366,7 +385,7 @@
     while (num_blocks) {
         size_t pending_blocks = (std::min(kProcessingBlocks, num_blocks));
 
-        if (compression_ && num_compress_threads_ > 1) {
+        if (compression_.algorithm && num_compress_threads_ > 1) {
             if (!CompressBlocks(pending_blocks, iter)) {
                 return false;
             }
@@ -386,7 +405,7 @@
                 op.source = next_data_pos_;
             }
 
-            if (compression_) {
+            if (compression_.algorithm) {
                 auto data = [&, this]() {
                     if (num_compress_threads_ > 1) {
                         auto data = std::move(*buf_iter_);
@@ -398,7 +417,7 @@
                         return data;
                     }
                 }();
-                op.compression = compression_;
+                op.compression = compression_.algorithm;
                 op.data_length = static_cast<uint16_t>(data.size());
 
                 if (!WriteOperation(op, data.data(), data.size())) {
@@ -507,8 +526,8 @@
         }
     }
 
-    // Footer should be at the end of a file, so if there is data after the current block, end it
-    // and start a new cluster.
+    // Footer should be at the end of a file, so if there is data after the current block, end
+    // it and start a new cluster.
     if (cluster_size_ && current_data_size_ > 0) {
         EmitCluster();
         extra_cluster = true;
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h
index 809ae57..3f357e0 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h
@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include <future>
 #include "writer_base.h"
 
 namespace android {
@@ -63,7 +64,7 @@
 
   private:
     CowFooter footer_{};
-    CowCompressionAlgorithm compression_ = kCowCompressNone;
+    CowCompression compression_;
     uint64_t current_op_pos_ = 0;
     uint64_t next_op_pos_ = 0;
     uint64_t next_data_pos_ = 0;
diff --git a/fs_mgr/libsnapshot/snapuserd/Android.bp b/fs_mgr/libsnapshot/snapuserd/Android.bp
index a9b96e2..6548cc8 100644
--- a/fs_mgr/libsnapshot/snapuserd/Android.bp
+++ b/fs_mgr/libsnapshot/snapuserd/Android.bp
@@ -19,7 +19,7 @@
 }
 
 cc_defaults {
-    name: "libsnapshot_snapuserd_defaults",
+    name: "libsnapuserd_client_defaults",
     defaults: [
         "fs_mgr_defaults",
     ],
@@ -33,15 +33,15 @@
 }
 
 cc_library_static {
-    name: "libsnapshot_snapuserd",
+    name: "libsnapuserd_client",
     defaults: [
         "fs_mgr_defaults",
-        "libsnapshot_snapuserd_defaults",
+        "libsnapuserd_client_defaults",
     ],
     recovery_available: true,
     static_libs: [
         "libcutils_sockets",
-        "libfs_mgr",
+        "libfs_mgr_file_wait",
     ],
     shared_libs: [
         "libbase",
@@ -60,32 +60,37 @@
     local_include_dirs: ["include/"],
     srcs: [
         "dm-snapshot-merge/snapuserd.cpp",
-        "dm-snapshot-merge/snapuserd_worker.cpp",
         "dm-snapshot-merge/snapuserd_readahead.cpp",
+        "dm-snapshot-merge/snapuserd_worker.cpp",
+        "dm_user_block_server.cpp",
         "snapuserd_buffer.cpp",
         "user-space-merge/handler_manager.cpp",
+        "user-space-merge/merge_worker.cpp",
         "user-space-merge/read_worker.cpp",
         "user-space-merge/snapuserd_core.cpp",
-        "user-space-merge/snapuserd_merge.cpp",
         "user-space-merge/snapuserd_readahead.cpp",
         "user-space-merge/snapuserd_transitions.cpp",
         "user-space-merge/snapuserd_verify.cpp",
         "user-space-merge/worker.cpp",
+        "utility.cpp",
     ],
     static_libs: [
         "libbase",
         "libdm",
+        "libext2_uuid",
         "libext4_utils",
         "libsnapshot_cow",
         "liburing",
     ],
     include_dirs: ["bionic/libc/kernel"],
+    export_include_dirs: ["include"],
     header_libs: [
         "libstorage_literals_headers",
     ],
     ramdisk_available: true,
     vendor_ramdisk_available: true,
     recovery_available: true,
+    host_supported: true,
 }
 
 cc_defaults {
@@ -104,12 +109,13 @@
         "libbrotli",
         "libcutils_sockets",
         "libdm",
-        "libfs_mgr",
+        "libext2_uuid",
+        "libfs_mgr_file_wait",
         "libgflags",
         "liblog",
         "libsnapshot_cow",
-        "libsnapshot_snapuserd",
         "libsnapuserd",
+        "libsnapuserd_client",
         "libz",
         "liblz4",
         "libext4_utils",
@@ -144,6 +150,9 @@
     init_rc: [
         "snapuserd.rc",
     ],
+    static_libs: [
+        "libsnapuserd_client",
+    ],
     ramdisk_available: false,
     vendor_ramdisk_available: true,
 }
@@ -186,12 +195,13 @@
         "libbrotli",
         "libgtest",
         "libsnapshot_cow",
-        "libsnapshot_snapuserd",
+        "libsnapuserd_client",
         "libcutils_sockets",
         "libz",
-        "libfs_mgr",
         "libdm",
+        "libext2_uuid",
         "libext4_utils",
+        "libfs_mgr_file_wait",
     ],
     header_libs: [
         "libstorage_literals_headers",
@@ -211,6 +221,9 @@
         "libsnapshot_cow_defaults",
     ],
     srcs: [
+        "testing/dm_user_harness.cpp",
+        "testing/harness.cpp",
+        "testing/host_harness.cpp",
         "user-space-merge/snapuserd_test.cpp",
     ],
     shared_libs: [
@@ -221,17 +234,20 @@
         "libbrotli",
         "libcutils_sockets",
         "libdm",
+        "libext2_uuid",
         "libext4_utils",
-        "libfs_mgr",
+        "libfs_mgr_file_wait",
         "libgflags",
         "libgtest",
         "libsnapshot_cow",
-        "libsnapshot_snapuserd",
         "libsnapuserd",
         "liburing",
         "libz",
     ],
-    include_dirs: ["bionic/libc/kernel"],
+    include_dirs: [
+        "bionic/libc/kernel",
+        ".",
+    ],
     header_libs: [
         "libstorage_literals_headers",
         "libfiemap_headers",
diff --git a/fs_mgr/libsnapshot/snapuserd/dm_user_block_server.cpp b/fs_mgr/libsnapshot/snapuserd/dm_user_block_server.cpp
new file mode 100644
index 0000000..e988335
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/dm_user_block_server.cpp
@@ -0,0 +1,152 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <snapuserd/dm_user_block_server.h>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <snapuserd/snapuserd_kernel.h>
+#include "snapuserd_logging.h"
+
+namespace android {
+namespace snapshot {
+
+using android::base::unique_fd;
+
+DmUserBlockServer::DmUserBlockServer(const std::string& misc_name, unique_fd&& ctrl_fd,
+                                     Delegate* delegate, size_t buffer_size)
+    : misc_name_(misc_name), ctrl_fd_(std::move(ctrl_fd)), delegate_(delegate) {
+    buffer_.Initialize(buffer_size);
+}
+
+bool DmUserBlockServer::ProcessRequests() {
+    struct dm_user_header* header = buffer_.GetHeaderPtr();
+    if (!android::base::ReadFully(ctrl_fd_, header, sizeof(*header))) {
+        if (errno != ENOTBLK) {
+            SNAP_PLOG(ERROR) << "Control-read failed";
+        }
+
+        SNAP_PLOG(DEBUG) << "ReadDmUserHeader failed....";
+        return false;
+    }
+
+    SNAP_LOG(DEBUG) << "Daemon: msg->seq: " << std::dec << header->seq;
+    SNAP_LOG(DEBUG) << "Daemon: msg->len: " << std::dec << header->len;
+    SNAP_LOG(DEBUG) << "Daemon: msg->sector: " << std::dec << header->sector;
+    SNAP_LOG(DEBUG) << "Daemon: msg->type: " << std::dec << header->type;
+    SNAP_LOG(DEBUG) << "Daemon: msg->flags: " << std::dec << header->flags;
+
+    if (!ProcessRequest(header)) {
+        if (header->type != DM_USER_RESP_ERROR) {
+            SendError();
+        }
+        return false;
+    }
+    return true;
+}
+
+bool DmUserBlockServer::ProcessRequest(dm_user_header* header) {
+    // Use the same header buffer as the response header.
+    int request_type = header->type;
+    header->type = DM_USER_RESP_SUCCESS;
+    header_response_ = true;
+
+    // Reset the output buffer.
+    buffer_.ResetBufferOffset();
+
+    switch (request_type) {
+        case DM_USER_REQ_MAP_READ:
+            return delegate_->RequestSectors(header->sector, header->len);
+
+        case DM_USER_REQ_MAP_WRITE:
+            // We should not get any write request to dm-user as we mount all
+            // partitions as read-only.
+            SNAP_LOG(ERROR) << "Unexpected write request from dm-user";
+            return false;
+
+        default:
+            SNAP_LOG(ERROR) << "Unexpected request from dm-user: " << request_type;
+            return false;
+    }
+}
+
+void* DmUserBlockServer::GetResponseBuffer(size_t size, size_t to_write) {
+    return buffer_.AcquireBuffer(size, to_write);
+}
+
+bool DmUserBlockServer::SendBufferedIo() {
+    return WriteDmUserPayload(buffer_.GetPayloadBytesWritten());
+}
+
+void DmUserBlockServer::SendError() {
+    struct dm_user_header* header = buffer_.GetHeaderPtr();
+    header->type = DM_USER_RESP_ERROR;
+    // This is an issue with the dm-user interface. There
+    // is no way to propagate the I/O error back to dm-user
+    // if we have already communicated the header back. Header
+    // is responded once at the beginning; however I/O can
+    // be processed in chunks. If we encounter an I/O error
+    // somewhere in the middle of the processing, we can't communicate
+    // this back to dm-user.
+    //
+    // TODO: Fix the interface
+    CHECK(header_response_);
+
+    WriteDmUserPayload(0);
+}
+
+bool DmUserBlockServer::WriteDmUserPayload(size_t size) {
+    size_t payload_size = size;
+    void* buf = buffer_.GetPayloadBufPtr();
+    if (header_response_) {
+        payload_size += sizeof(struct dm_user_header);
+        buf = buffer_.GetBufPtr();
+    }
+
+    if (!android::base::WriteFully(ctrl_fd_, buf, payload_size)) {
+        SNAP_PLOG(ERROR) << "Write to dm-user failed size: " << payload_size;
+        return false;
+    }
+
+    // After the first header is sent in response to a request, we cannot
+    // send any additional headers.
+    header_response_ = false;
+
+    // Reset the buffer for use by the next request.
+    buffer_.ResetBufferOffset();
+    return true;
+}
+
+DmUserBlockServerOpener::DmUserBlockServerOpener(const std::string& misc_name,
+                                                 const std::string& dm_user_path)
+    : misc_name_(misc_name), dm_user_path_(dm_user_path) {}
+
+std::unique_ptr<IBlockServer> DmUserBlockServerOpener::Open(IBlockServer::Delegate* delegate,
+                                                            size_t buffer_size) {
+    unique_fd fd(open(dm_user_path_.c_str(), O_RDWR | O_CLOEXEC));
+    if (fd < 0) {
+        SNAP_PLOG(ERROR) << "Could not open dm-user path: " << dm_user_path_;
+        return nullptr;
+    }
+    return std::make_unique<DmUserBlockServer>(misc_name_, std::move(fd), delegate, buffer_size);
+}
+
+std::shared_ptr<IBlockServerOpener> DmUserBlockServerFactory::CreateOpener(
+        const std::string& misc_name) {
+    auto dm_path = "/dev/dm-user/" + misc_name;
+    return std::make_shared<DmUserBlockServerOpener>(misc_name, dm_path);
+}
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/include/snapuserd/block_server.h b/fs_mgr/libsnapshot/snapuserd/include/snapuserd/block_server.h
new file mode 100644
index 0000000..406bf11
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/include/snapuserd/block_server.h
@@ -0,0 +1,95 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <memory>
+
+namespace android {
+namespace snapshot {
+
+// These interfaces model the block device driver component of snapuserd (eg,
+// dm-user).
+
+// An open connection to a userspace block device control
+class IBlockServer {
+  public:
+    class Delegate {
+      public:
+        virtual ~Delegate() {}
+
+        // Respond to a request for reading a contiguous run of sectors. This
+        // call should be followed by calls to GetResponseBuffer/CommitBuffer
+        // until the |size| is fulfilled.
+        //
+        // If false is returned, an error will be automatically reported unless
+        // SendError was called.
+        virtual bool RequestSectors(uint64_t sector, uint64_t size) = 0;
+    };
+
+    virtual ~IBlockServer() {}
+
+    // Process I/O requests. This can block the worker thread until either a
+    // request is available or the underlying connection has been destroyed.
+    //
+    // True indicates that one or more requests was processed. False indicates
+    // an unrecoverable condition and processing should stop.
+    virtual bool ProcessRequests() = 0;
+
+    // Return a buffer for fulfilling a RequestSectors request. This buffer
+    // is valid until calling SendBufferedIo. This cannot be called outside
+    // of RequestSectors().
+    //
+    // "to_write" must be <= "size". If it is < size, the excess bytes are
+    // available for writing, but will not be send via SendBufferedIo, and
+    // may be reallocated in the next call to GetResponseBuffer.
+    //
+    // All buffers returned are invalidated after SendBufferedIo or returning
+    // control from RequestSectors.
+    virtual void* GetResponseBuffer(size_t size, size_t to_write) = 0;
+
+    // Send all outstanding buffers to the driver, in order. This should
+    // be called at least once in response to RequestSectors. This returns
+    // ownership of any buffers returned by GetResponseBuffer.
+    //
+    // If false is returned, an error is automatically reported to the driver.
+    virtual bool SendBufferedIo() = 0;
+
+    void* GetResponseBuffer(size_t size) { return GetResponseBuffer(size, size); }
+};
+
+class IBlockServerOpener {
+  public:
+    virtual ~IBlockServerOpener() = default;
+
+    // Open a connection to the service. This is called on the daemon thread.
+    //
+    // buffer_size is the maximum amount of buffered I/O to use.
+    virtual std::unique_ptr<IBlockServer> Open(IBlockServer::Delegate* delegate,
+                                               size_t buffer_size) = 0;
+};
+
+class IBlockServerFactory {
+  public:
+    virtual ~IBlockServerFactory() {}
+
+    // Return a new IBlockServerOpener given a unique device name.
+    virtual std::shared_ptr<IBlockServerOpener> CreateOpener(const std::string& misc_name) = 0;
+};
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/include/snapuserd/dm_user_block_server.h b/fs_mgr/libsnapshot/snapuserd/include/snapuserd/dm_user_block_server.h
new file mode 100644
index 0000000..f1f8da1
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/include/snapuserd/dm_user_block_server.h
@@ -0,0 +1,68 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <android-base/unique_fd.h>
+
+#include <string>
+
+#include <snapuserd/block_server.h>
+#include <snapuserd/snapuserd_buffer.h>
+
+namespace android {
+namespace snapshot {
+
+class DmUserBlockServer : public IBlockServer {
+  public:
+    DmUserBlockServer(const std::string& misc_name, android::base::unique_fd&& ctrl_fd,
+                      Delegate* delegate, size_t buffer_size);
+
+    bool ProcessRequests() override;
+    void* GetResponseBuffer(size_t size, size_t to_write) override;
+    bool SendBufferedIo() override;
+    void SendError();
+
+  private:
+    bool ProcessRequest(dm_user_header* header);
+    bool WriteDmUserPayload(size_t size);
+
+    std::string misc_name_;
+    android::base::unique_fd ctrl_fd_;
+    Delegate* delegate_;
+
+    // Per-request state.
+    BufferSink buffer_;
+    bool header_response_ = false;
+};
+
+class DmUserBlockServerOpener : public IBlockServerOpener {
+  public:
+    DmUserBlockServerOpener(const std::string& misc_name, const std::string& dm_user_path);
+
+    std::unique_ptr<IBlockServer> Open(IBlockServer::Delegate* delegate,
+                                       size_t buffer_size) override;
+
+  private:
+    std::string misc_name_;
+    std::string dm_user_path_;
+};
+
+class DmUserBlockServerFactory : public IBlockServerFactory {
+  public:
+    std::shared_ptr<IBlockServerOpener> CreateOpener(const std::string& misc_name) override;
+};
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/include/snapuserd/snapuserd_kernel.h b/fs_mgr/libsnapshot/snapuserd/include/snapuserd/snapuserd_kernel.h
index 46952c4..7ab75dc 100644
--- a/fs_mgr/libsnapshot/snapuserd/include/snapuserd/snapuserd_kernel.h
+++ b/fs_mgr/libsnapshot/snapuserd/include/snapuserd/snapuserd_kernel.h
@@ -14,6 +14,8 @@
 
 #pragma once
 
+#include <linux/types.h>
+
 namespace android {
 namespace snapshot {
 
@@ -40,6 +42,7 @@
  * multiple of 512 bytes. Hence these two constants.
  */
 static constexpr uint32_t SECTOR_SHIFT = 9;
+static constexpr uint64_t SECTOR_SIZE = (1ULL << SECTOR_SHIFT);
 
 static constexpr size_t BLOCK_SZ = 4096;
 static constexpr size_t BLOCK_SHIFT = (__builtin_ffs(BLOCK_SZ) - 1);
@@ -69,7 +72,7 @@
 
     /* In sectors */
     uint32_t chunk_size;
-} __packed;
+} __attribute__((packed));
 
 // A disk exception is a mapping of old_chunk to new_chunk
 // old_chunk is the chunk ID of a dm-snapshot device.
@@ -77,7 +80,7 @@
 struct disk_exception {
     uint64_t old_chunk;
     uint64_t new_chunk;
-} __packed;
+} __attribute__((packed));
 
 // Control structures to communicate with dm-user
 // It comprises of header and a payload
diff --git a/fs_mgr/libsnapshot/snapuserd/snapuserd_buffer.cpp b/fs_mgr/libsnapshot/snapuserd/snapuserd_buffer.cpp
index 35065e6..490c0e6 100644
--- a/fs_mgr/libsnapshot/snapuserd/snapuserd_buffer.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/snapuserd_buffer.cpp
@@ -23,9 +23,9 @@
 namespace snapshot {
 
 void BufferSink::Initialize(size_t size) {
-    buffer_size_ = size;
+    buffer_size_ = size + sizeof(struct dm_user_header);
     buffer_offset_ = 0;
-    buffer_ = std::make_unique<uint8_t[]>(size);
+    buffer_ = std::make_unique<uint8_t[]>(buffer_size_);
 }
 
 void* BufferSink::AcquireBuffer(size_t size, size_t to_write) {
diff --git a/fs_mgr/libsnapshot/snapuserd/snapuserd_logging.h b/fs_mgr/libsnapshot/snapuserd/snapuserd_logging.h
new file mode 100644
index 0000000..bc470ce
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/snapuserd_logging.h
@@ -0,0 +1,20 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <android-base/logging.h>
+
+#define SNAP_LOG(level) LOG(level) << misc_name_ << ": "
+#define SNAP_PLOG(level) PLOG(level) << misc_name_ << ": "
diff --git a/fs_mgr/libsnapshot/snapuserd/testing/dm_user_harness.cpp b/fs_mgr/libsnapshot/snapuserd/testing/dm_user_harness.cpp
new file mode 100644
index 0000000..7cadf25
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/testing/dm_user_harness.cpp
@@ -0,0 +1,67 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "dm_user_harness.h"
+
+#include <fcntl.h>
+
+#include <android-base/file.h>
+#include <fs_mgr/file_wait.h>
+#include <libdm/dm.h>
+#include <snapuserd/dm_user_block_server.h>
+
+namespace android {
+namespace snapshot {
+
+using namespace std::chrono_literals;
+using android::base::unique_fd;
+
+DmUserDevice::DmUserDevice(std::unique_ptr<Tempdevice>&& dev) : dev_(std::move(dev)) {}
+
+const std::string& DmUserDevice::GetPath() {
+    return dev_->path();
+}
+
+bool DmUserDevice::Destroy() {
+    return dev_->Destroy();
+}
+
+DmUserTestHarness::DmUserTestHarness() {
+    block_server_factory_ = std::make_unique<DmUserBlockServerFactory>();
+}
+
+std::unique_ptr<IUserDevice> DmUserTestHarness::CreateUserDevice(const std::string& dev_name,
+                                                                 const std::string& misc_name,
+                                                                 uint64_t num_sectors) {
+    android::dm::DmTable dmuser_table;
+    dmuser_table.Emplace<android::dm::DmTargetUser>(0, num_sectors, misc_name);
+    auto dev = std::make_unique<Tempdevice>(dev_name, dmuser_table);
+    if (!dev->valid()) {
+        return nullptr;
+    }
+
+    auto misc_device = "/dev/dm-user/" + misc_name;
+    if (!android::fs_mgr::WaitForFile(misc_device, 10s)) {
+        return nullptr;
+    }
+
+    return std::make_unique<DmUserDevice>(std::move(dev));
+}
+
+IBlockServerFactory* DmUserTestHarness::GetBlockServerFactory() {
+    return block_server_factory_.get();
+}
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/testing/dm_user_harness.h b/fs_mgr/libsnapshot/snapuserd/testing/dm_user_harness.h
new file mode 100644
index 0000000..cf26bed
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/testing/dm_user_harness.h
@@ -0,0 +1,54 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <android-base/unique_fd.h>
+
+#include "harness.h"
+#include "temp_device.h"
+
+namespace android {
+namespace snapshot {
+
+using android::base::unique_fd;
+
+class DmUserBlockServerFactory;
+
+class DmUserDevice final : public IUserDevice {
+  public:
+    explicit DmUserDevice(std::unique_ptr<Tempdevice>&& dev);
+    const std::string& GetPath() override;
+    bool Destroy() override;
+
+  private:
+    std::unique_ptr<Tempdevice> dev_;
+};
+
+class DmUserTestHarness final : public ITestHarness {
+  public:
+    DmUserTestHarness();
+
+    std::unique_ptr<IUserDevice> CreateUserDevice(const std::string& dev_name,
+                                                  const std::string& misc_name,
+                                                  uint64_t num_sectors) override;
+    IBlockServerFactory* GetBlockServerFactory() override;
+    bool HasUserDevice() override { return true; }
+
+  private:
+    std::unique_ptr<DmUserBlockServerFactory> block_server_factory_;
+};
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/testing/harness.cpp b/fs_mgr/libsnapshot/snapuserd/testing/harness.cpp
new file mode 100644
index 0000000..02ae549
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/testing/harness.cpp
@@ -0,0 +1,116 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "harness.h"
+
+#ifdef __ANDROID__
+#include <linux/memfd.h>
+#endif
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include <android-base/file.h>
+#include <ext4_utils/ext4_utils.h>
+#include <libdm/loop_control.h>
+#include "snapuserd_logging.h"
+
+namespace android {
+namespace snapshot {
+
+using namespace std::chrono_literals;
+using android::base::unique_fd;
+using android::dm::LoopDevice;
+
+#ifdef __ANDROID__
+// Prefer this on device since it is a real block device, which is more similar
+// to how we use snapuserd.
+class MemoryBackedDevice final : public IBackingDevice {
+  public:
+    bool Init(uint64_t size) {
+        memfd_.reset(memfd_create("snapuserd_test", MFD_ALLOW_SEALING));
+        if (memfd_ < 0) {
+            PLOG(ERROR) << "memfd_create failed";
+            return false;
+        }
+        if (ftruncate(memfd_.get(), size) < 0) {
+            PLOG(ERROR) << "ftruncate failed";
+            return false;
+        }
+        if (fcntl(memfd_.get(), F_ADD_SEALS, F_SEAL_GROW | F_SEAL_SHRINK) < 0) {
+            PLOG(ERROR) << "fcntl seal failed";
+            return false;
+        }
+        dev_ = std::make_unique<LoopDevice>(memfd_, 10s);
+        return dev_->valid();
+    }
+    const std::string& GetPath() override { return dev_->device(); }
+    uint64_t GetSize() override {
+        unique_fd fd(open(GetPath().c_str(), O_RDONLY | O_CLOEXEC));
+        if (fd < 0) {
+            PLOG(ERROR) << "open failed: " << GetPath();
+            return 0;
+        }
+        return get_block_device_size(fd.get());
+    }
+
+  private:
+    unique_fd memfd_;
+    std::unique_ptr<LoopDevice> dev_;
+};
+#endif
+
+class FileBackedDevice final : public IBackingDevice {
+  public:
+    bool Init(uint64_t size) {
+        if (temp_.fd < 0) {
+            return false;
+        }
+        if (ftruncate(temp_.fd, size) < 0) {
+            PLOG(ERROR) << "ftruncate failed: " << temp_.path;
+            return false;
+        }
+        path_ = temp_.path;
+        return true;
+    }
+
+    const std::string& GetPath() override { return path_; }
+    uint64_t GetSize() override {
+        off_t off = lseek(temp_.fd, 0, SEEK_END);
+        if (off < 0) {
+            PLOG(ERROR) << "lseek failed: " << temp_.path;
+            return 0;
+        }
+        return off;
+    }
+
+  private:
+    TemporaryFile temp_;
+    std::string path_;
+};
+
+std::unique_ptr<IBackingDevice> ITestHarness::CreateBackingDevice(uint64_t size) {
+#ifdef __ANDROID__
+    auto dev = std::make_unique<MemoryBackedDevice>();
+#else
+    auto dev = std::make_unique<FileBackedDevice>();
+#endif
+    if (!dev->Init(size)) {
+        return nullptr;
+    }
+    return dev;
+}
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/testing/harness.h b/fs_mgr/libsnapshot/snapuserd/testing/harness.h
new file mode 100644
index 0000000..ffe9f7b
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/testing/harness.h
@@ -0,0 +1,56 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <stddef.h>
+#include <sys/types.h>
+
+#include <memory>
+
+#include <android-base/unique_fd.h>
+#include <snapuserd/block_server.h>
+
+namespace android {
+namespace snapshot {
+
+// Interface for a "block driver in userspace" device.
+class IUserDevice {
+  public:
+    virtual ~IUserDevice() {}
+    virtual const std::string& GetPath() = 0;
+    virtual bool Destroy() = 0;
+};
+
+// Interface for an fd/temp file that is a block device when possible.
+class IBackingDevice {
+  public:
+    virtual ~IBackingDevice() {}
+    virtual const std::string& GetPath() = 0;
+    virtual uint64_t GetSize() = 0;
+};
+
+class ITestHarness {
+  public:
+    virtual ~ITestHarness() {}
+    virtual std::unique_ptr<IUserDevice> CreateUserDevice(const std::string& dev_name,
+                                                          const std::string& misc_name,
+                                                          uint64_t num_sectors) = 0;
+    virtual IBlockServerFactory* GetBlockServerFactory() = 0;
+    virtual bool HasUserDevice() = 0;
+    virtual std::unique_ptr<IBackingDevice> CreateBackingDevice(uint64_t size);
+};
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/testing/host_harness.cpp b/fs_mgr/libsnapshot/snapuserd/testing/host_harness.cpp
new file mode 100644
index 0000000..0d230ad
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/testing/host_harness.cpp
@@ -0,0 +1,112 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "host_harness.h"
+
+#include "snapuserd_logging.h"
+
+namespace android {
+namespace snapshot {
+
+void TestBlockServerQueue::WaitForShutdown() {
+    std::unique_lock lock(m_);
+    if (shutdown_) {
+        return;
+    }
+    cv_.wait(lock, [this]() -> bool { return shutdown_; });
+}
+
+void TestBlockServerQueue::Shutdown() {
+    std::unique_lock lock(m_);
+    shutdown_ = true;
+    cv_.notify_all();
+}
+
+TestBlockServer::TestBlockServer(std::shared_ptr<TestBlockServerQueue> queue,
+                                 const std::string& misc_name)
+    : queue_(queue), misc_name_(misc_name) {}
+
+bool TestBlockServer::ProcessRequests() {
+    queue_->WaitForShutdown();
+    return false;
+}
+
+void* TestBlockServer::GetResponseBuffer(size_t size, size_t to_write) {
+    std::string buffer(size, '\0');
+    buffered_.emplace_back(std::move(buffer), to_write);
+    return buffered_.back().first.data();
+}
+
+bool TestBlockServer::SendBufferedIo() {
+    for (const auto& [data, to_write] : buffered_) {
+        sent_io_ += data.substr(0, to_write);
+    }
+    buffered_.clear();
+    return true;
+}
+
+TestBlockServerOpener::TestBlockServerOpener(std::shared_ptr<TestBlockServerQueue> queue,
+                                             const std::string& misc_name)
+    : queue_(queue), misc_name_(misc_name) {}
+
+std::unique_ptr<IBlockServer> TestBlockServerOpener::Open(IBlockServer::Delegate*, size_t) {
+    return std::make_unique<TestBlockServer>(queue_, misc_name_);
+}
+
+std::shared_ptr<TestBlockServerOpener> TestBlockServerFactory::CreateTestOpener(
+        const std::string& misc_name) {
+    if (queues_.count(misc_name)) {
+        LOG(ERROR) << "Cannot create opener for " << misc_name << ", already exists";
+        return nullptr;
+    }
+    auto queue = std::make_shared<TestBlockServerQueue>();
+    queues_.emplace(misc_name, queue);
+    return std::make_shared<TestBlockServerOpener>(queue, misc_name);
+}
+
+std::shared_ptr<IBlockServerOpener> TestBlockServerFactory::CreateOpener(
+        const std::string& misc_name) {
+    return CreateTestOpener(misc_name);
+}
+
+bool TestBlockServerFactory::DeleteQueue(const std::string& misc_name) {
+    auto iter = queues_.find(misc_name);
+    if (iter == queues_.end()) {
+        LOG(ERROR) << "Cannot delete queue " << misc_name << ", not found";
+        return false;
+    }
+    iter->second->Shutdown();
+    queues_.erase(iter);
+    return true;
+}
+
+HostUserDevice::HostUserDevice(TestBlockServerFactory* factory, const std::string& misc_name)
+    : factory_(factory), misc_name_(misc_name) {}
+
+bool HostUserDevice::Destroy() {
+    return factory_->DeleteQueue(misc_name_);
+}
+
+std::unique_ptr<IUserDevice> HostTestHarness::CreateUserDevice(const std::string&,
+                                                               const std::string& misc_name,
+                                                               uint64_t) {
+    return std::make_unique<HostUserDevice>(&factory_, misc_name);
+}
+
+IBlockServerFactory* HostTestHarness::GetBlockServerFactory() {
+    return &factory_;
+}
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/testing/host_harness.h b/fs_mgr/libsnapshot/snapuserd/testing/host_harness.h
new file mode 100644
index 0000000..ec0bd29
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/testing/host_harness.h
@@ -0,0 +1,105 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <condition_variable>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "harness.h"
+
+namespace android {
+namespace snapshot {
+
+class TestBlockServerQueue final {
+  public:
+    void WaitForShutdown();
+    void Shutdown();
+
+  private:
+    std::mutex m_;
+    std::condition_variable cv_;
+    bool shutdown_ = false;
+};
+
+class TestBlockServer final : public IBlockServer {
+  public:
+    TestBlockServer(std::shared_ptr<TestBlockServerQueue> queue, const std::string& misc_name);
+    bool ProcessRequests() override;
+    void* GetResponseBuffer(size_t size, size_t to_write) override;
+    bool SendBufferedIo() override;
+
+    std::string&& sent_io() { return std::move(sent_io_); }
+
+  private:
+    std::shared_ptr<TestBlockServerQueue> queue_;
+    std::string misc_name_;
+    std::string sent_io_;
+    std::vector<std::pair<std::string, size_t>> buffered_;
+};
+
+class TestBlockServerOpener final : public IBlockServerOpener {
+  public:
+    TestBlockServerOpener(std::shared_ptr<TestBlockServerQueue> queue,
+                          const std::string& misc_name);
+    std::unique_ptr<IBlockServer> Open(IBlockServer::Delegate* delegate,
+                                       size_t buffer_size) override;
+
+  private:
+    std::shared_ptr<TestBlockServerQueue> queue_;
+    std::string misc_name_;
+};
+
+class TestBlockServerFactory final : public IBlockServerFactory {
+  public:
+    std::shared_ptr<IBlockServerOpener> CreateOpener(const std::string& misc_name) override;
+    std::shared_ptr<TestBlockServerOpener> CreateTestOpener(const std::string& misc_name);
+    bool DeleteQueue(const std::string& misc_name);
+
+  private:
+    std::unordered_map<std::string, std::shared_ptr<TestBlockServerQueue>> queues_;
+};
+
+class TestBlockServerFactory;
+
+class HostUserDevice final : public IUserDevice {
+  public:
+    HostUserDevice(TestBlockServerFactory* factory, const std::string& misc_name);
+    const std::string& GetPath() override { return empty_path_; }
+    bool Destroy();
+
+  private:
+    TestBlockServerFactory* factory_;
+    std::string misc_name_;
+    std::string empty_path_;
+};
+
+class HostTestHarness final : public ITestHarness {
+  public:
+    std::unique_ptr<IUserDevice> CreateUserDevice(const std::string& dev_name,
+                                                  const std::string& misc_name,
+                                                  uint64_t num_sectors) override;
+    IBlockServerFactory* GetBlockServerFactory() override;
+    bool HasUserDevice() override { return false; }
+
+  private:
+    TestBlockServerFactory factory_;
+};
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/testing/temp_device.h b/fs_mgr/libsnapshot/snapuserd/testing/temp_device.h
new file mode 100644
index 0000000..2a6d3ea
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/testing/temp_device.h
@@ -0,0 +1,72 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <chrono>
+#include <string>
+
+#include <libdm/dm.h>
+
+namespace android {
+namespace snapshot {
+
+using android::dm::DeviceMapper;
+using android::dm::DmTable;
+
+class Tempdevice {
+  public:
+    Tempdevice(const std::string& name, const DmTable& table)
+        : dm_(DeviceMapper::Instance()), name_(name), valid_(false) {
+        valid_ = dm_.CreateDevice(name, table, &path_, std::chrono::seconds(5));
+    }
+    Tempdevice(Tempdevice&& other) noexcept
+        : dm_(other.dm_), name_(other.name_), path_(other.path_), valid_(other.valid_) {
+        other.valid_ = false;
+    }
+    ~Tempdevice() {
+        if (valid_) {
+            dm_.DeleteDeviceIfExists(name_);
+        }
+    }
+    bool Destroy() {
+        if (!valid_) {
+            return true;
+        }
+        valid_ = false;
+        return dm_.DeleteDeviceIfExists(name_);
+    }
+    const std::string& path() const { return path_; }
+    const std::string& name() const { return name_; }
+    bool valid() const { return valid_; }
+
+    Tempdevice(const Tempdevice&) = delete;
+    Tempdevice& operator=(const Tempdevice&) = delete;
+
+    Tempdevice& operator=(Tempdevice&& other) noexcept {
+        name_ = other.name_;
+        valid_ = other.valid_;
+        other.valid_ = false;
+        return *this;
+    }
+
+  private:
+    DeviceMapper& dm_;
+    std::string name_;
+    std::string path_;
+    bool valid_;
+};
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp
index 4105b4b..d979e20 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp
@@ -14,13 +14,14 @@
 
 #include "handler_manager.h"
 
+#include <pthread.h>
 #include <sys/eventfd.h>
 
 #include <android-base/logging.h>
 
+#include "merge_worker.h"
 #include "read_worker.h"
 #include "snapuserd_core.h"
-#include "snapuserd_merge.h"
 
 namespace android {
 namespace snapshot {
@@ -50,9 +51,10 @@
 std::shared_ptr<HandlerThread> SnapshotHandlerManager::AddHandler(
         const std::string& misc_name, const std::string& cow_device_path,
         const std::string& backing_device, const std::string& base_path_merge,
-        int num_worker_threads, bool use_iouring, bool perform_verification) {
+        std::shared_ptr<IBlockServerOpener> opener, int num_worker_threads, bool use_iouring,
+        bool perform_verification) {
     auto snapuserd = std::make_shared<SnapshotHandler>(misc_name, cow_device_path, backing_device,
-                                                       base_path_merge, num_worker_threads,
+                                                       base_path_merge, opener, num_worker_threads,
                                                        use_iouring, perform_verification);
     if (!snapuserd->InitCowDevice()) {
         LOG(ERROR) << "Failed to initialize Snapuserd";
@@ -131,6 +133,8 @@
 void SnapshotHandlerManager::RunThread(std::shared_ptr<HandlerThread> handler) {
     LOG(INFO) << "Entering thread for handler: " << handler->misc_name();
 
+    pthread_setname_np(pthread_self(), "Handler");
+
     if (!handler->snapuserd()->Start()) {
         LOG(ERROR) << " Failed to launch all worker threads";
     }
@@ -200,9 +204,8 @@
 
     handler->snapuserd()->MonitorMerge();
 
-    if (!is_merge_monitor_started_) {
-        std::thread(&SnapshotHandlerManager::MonitorMerge, this).detach();
-        is_merge_monitor_started_ = true;
+    if (!merge_monitor_.joinable()) {
+        merge_monitor_ = std::thread(&SnapshotHandlerManager::MonitorMerge, this);
     }
 
     merge_handlers_.push(handler);
@@ -219,6 +222,7 @@
 }
 
 void SnapshotHandlerManager::MonitorMerge() {
+    pthread_setname_np(pthread_self(), "Merge Monitor");
     while (!stop_monitor_merge_thread_) {
         uint64_t testVal;
         ssize_t ret =
@@ -356,8 +360,12 @@
         if (th.joinable()) th.join();
     }
 
-    stop_monitor_merge_thread_ = true;
-    WakeupMonitorMergeThread();
+    if (merge_monitor_.joinable()) {
+        stop_monitor_merge_thread_ = true;
+        WakeupMonitorMergeThread();
+
+        merge_monitor_.join();
+    }
 }
 
 auto SnapshotHandlerManager::FindHandler(std::lock_guard<std::mutex>* proof_of_lock,
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.h
index b7ddac1..b1605f0 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.h
@@ -21,6 +21,7 @@
 #include <vector>
 
 #include <android-base/unique_fd.h>
+#include <snapuserd/block_server.h>
 
 namespace android {
 namespace snapshot {
@@ -55,6 +56,7 @@
                                                       const std::string& cow_device_path,
                                                       const std::string& backing_device,
                                                       const std::string& base_path_merge,
+                                                      std::shared_ptr<IBlockServerOpener> opener,
                                                       int num_worker_threads, bool use_iouring,
                                                       bool perform_verification) = 0;
 
@@ -91,6 +93,7 @@
                                               const std::string& cow_device_path,
                                               const std::string& backing_device,
                                               const std::string& base_path_merge,
+                                              std::shared_ptr<IBlockServerOpener> opener,
                                               int num_worker_threads, bool use_iouring,
                                               bool perform_verification) override;
     bool StartHandler(const std::string& misc_name) override;
@@ -119,9 +122,9 @@
     std::mutex lock_;
     HandlerList dm_users_;
 
-    bool is_merge_monitor_started_ = false;
     bool stop_monitor_merge_thread_ = false;
     int active_merge_threads_ = 0;
+    std::thread merge_monitor_;
     int num_partitions_merge_complete_ = 0;
     std::queue<std::shared_ptr<HandlerThread>> merge_handlers_;
     android::base::unique_fd monitor_merge_event_fd_;
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/merge_worker.cpp
similarity index 97%
rename from fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp
rename to fs_mgr/libsnapshot/snapuserd/user-space-merge/merge_worker.cpp
index 517148d..11b8d7c 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/merge_worker.cpp
@@ -13,9 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "snapuserd_merge.h"
+#include "merge_worker.h"
+
+#include <pthread.h>
 
 #include "snapuserd_core.h"
+#include "utility.h"
 
 namespace android {
 namespace snapshot {
@@ -197,6 +200,7 @@
         // Wait for RA thread to notify that the merge window
         // is ready for merging.
         if (!snapuserd_->WaitForMergeBegin()) {
+            SNAP_LOG(ERROR) << "Failed waiting for merge to begin";
             return false;
         }
 
@@ -302,7 +306,7 @@
                     // will fallback to synchronous I/O.
                     ret = io_uring_wait_cqe(ring_.get(), &cqe);
                     if (ret) {
-                        SNAP_LOG(ERROR) << "Merge: io_uring_wait_cqe failed: " << ret;
+                        SNAP_LOG(ERROR) << "Merge: io_uring_wait_cqe failed: " << strerror(-ret);
                         status = false;
                         break;
                     }
@@ -545,17 +549,22 @@
 
 bool MergeWorker::Run() {
     SNAP_LOG(DEBUG) << "Waiting for merge begin...";
+
+    pthread_setname_np(pthread_self(), "MergeWorker");
+
     if (!snapuserd_->WaitForMergeBegin()) {
         SNAP_LOG(ERROR) << "Merge terminated early...";
         return true;
     }
 
-    if (setpriority(PRIO_PROCESS, gettid(), kNiceValueForMergeThreads)) {
-        SNAP_PLOG(ERROR) << "Failed to set priority for TID: " << gettid();
+    if (!SetThreadPriority(kNiceValueForMergeThreads)) {
+        SNAP_PLOG(ERROR) << "Failed to set thread priority";
     }
 
     SNAP_LOG(INFO) << "Merge starting..";
 
+    bufsink_.Initialize(PAYLOAD_BUFFER_SZ);
+
     if (!Init()) {
         SNAP_LOG(ERROR) << "Merge thread initialization failed...";
         snapuserd_->MergeFailed();
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/merge_worker.h
similarity index 98%
rename from fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.h
rename to fs_mgr/libsnapshot/snapuserd/user-space-merge/merge_worker.h
index f35147f..478d4c8 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/merge_worker.h
@@ -39,6 +39,7 @@
     void FinalizeIouring();
 
   private:
+    BufferSink bufsink_;
     std::unique_ptr<ICowOpIter> cowop_iter_;
     std::unique_ptr<struct io_uring> ring_;
     size_t ra_block_index_ = 0;
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
index 7d2e3a6..b9ecfa5 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
@@ -16,7 +16,10 @@
 
 #include "read_worker.h"
 
+#include <pthread.h>
+
 #include "snapuserd_core.h"
+#include "utility.h"
 
 namespace android {
 namespace snapshot {
@@ -26,18 +29,18 @@
 using android::base::unique_fd;
 
 void ReadWorker::CloseFds() {
-    ctrl_fd_ = {};
+    block_server_ = {};
     backing_store_fd_ = {};
     Worker::CloseFds();
 }
 
 ReadWorker::ReadWorker(const std::string& cow_device, const std::string& backing_device,
-                       const std::string& control_device, const std::string& misc_name,
-                       const std::string& base_path_merge,
-                       std::shared_ptr<SnapshotHandler> snapuserd)
+                       const std::string& misc_name, const std::string& base_path_merge,
+                       std::shared_ptr<SnapshotHandler> snapuserd,
+                       std::shared_ptr<IBlockServerOpener> opener)
     : Worker(cow_device, misc_name, base_path_merge, snapuserd),
       backing_store_device_(backing_device),
-      control_device_(control_device) {}
+      block_server_opener_(opener) {}
 
 // Start the replace operation. This will read the
 // internal COW format and if the block is compressed,
@@ -197,9 +200,9 @@
         return false;
     }
 
-    ctrl_fd_.reset(open(control_device_.c_str(), O_RDWR));
-    if (ctrl_fd_ < 0) {
-        SNAP_PLOG(ERROR) << "Unable to open " << control_device_;
+    block_server_ = block_server_opener_->Open(this, PAYLOAD_BUFFER_SZ);
+    if (!block_server_) {
+        SNAP_PLOG(ERROR) << "Unable to open block server";
         return false;
     }
     return true;
@@ -208,13 +211,15 @@
 bool ReadWorker::Run() {
     SNAP_LOG(INFO) << "Processing snapshot I/O requests....";
 
-    if (setpriority(PRIO_PROCESS, gettid(), kNiceValueForMergeThreads)) {
-        SNAP_PLOG(ERROR) << "Failed to set priority for TID: " << gettid();
+    pthread_setname_np(pthread_self(), "ReadWorker");
+
+    if (!SetThreadPriority(kNiceValueForMergeThreads)) {
+        SNAP_PLOG(ERROR) << "Failed to set thread priority";
     }
 
     // Start serving IO
     while (true) {
-        if (!ProcessIORequest()) {
+        if (!block_server_->ProcessRequests()) {
             break;
         }
     }
@@ -225,29 +230,6 @@
     return true;
 }
 
-// Send the payload/data back to dm-user misc device.
-bool ReadWorker::WriteDmUserPayload(size_t size) {
-    size_t payload_size = size;
-    void* buf = bufsink_.GetPayloadBufPtr();
-    if (header_response_) {
-        payload_size += sizeof(struct dm_user_header);
-        buf = bufsink_.GetBufPtr();
-    }
-
-    if (!android::base::WriteFully(ctrl_fd_, buf, payload_size)) {
-        SNAP_PLOG(ERROR) << "Write to dm-user failed size: " << payload_size;
-        return false;
-    }
-
-    // After the first header is sent in response to a request, we cannot
-    // send any additional headers.
-    header_response_ = false;
-
-    // Reset the buffer for use by the next request.
-    bufsink_.ResetBufferOffset();
-    return true;
-}
-
 bool ReadWorker::ReadDataFromBaseDevice(sector_t sector, void* buffer, size_t read_size) {
     CHECK(read_size <= BLOCK_SZ);
 
@@ -281,7 +263,7 @@
                                        std::make_pair(sector, nullptr), SnapshotHandler::compare);
             bool not_found = (it == chunk_vec.end() || it->first != sector);
 
-            void* buffer = bufsink_.AcquireBuffer(BLOCK_SZ, size);
+            void* buffer = block_server_->GetResponseBuffer(BLOCK_SZ, size);
             if (!buffer) {
                 SNAP_LOG(ERROR) << "AcquireBuffer failed in ReadAlignedSector";
                 return false;
@@ -334,7 +316,8 @@
     int num_sectors_skip = sector - it->first;
     size_t skip_size = num_sectors_skip << SECTOR_SHIFT;
     size_t write_size = std::min(size, BLOCK_SZ - skip_size);
-    auto buffer = reinterpret_cast<uint8_t*>(bufsink_.AcquireBuffer(BLOCK_SZ, write_size));
+    auto buffer =
+            reinterpret_cast<uint8_t*>(block_server_->GetResponseBuffer(BLOCK_SZ, write_size));
     if (!buffer) {
         SNAP_LOG(ERROR) << "ProcessCowOp failed to allocate buffer";
         return -1;
@@ -462,7 +445,7 @@
         CHECK(diff_size <= BLOCK_SZ);
 
         size_t read_size = std::min(remaining_size, diff_size);
-        void* buffer = bufsink_.AcquireBuffer(BLOCK_SZ, read_size);
+        void* buffer = block_server_->GetResponseBuffer(BLOCK_SZ, read_size);
         if (!buffer) {
             SNAP_LOG(ERROR) << "AcquireBuffer failed in ReadUnalignedSector";
             return false;
@@ -488,88 +471,17 @@
     return true;
 }
 
-void ReadWorker::RespondIOError() {
-    struct dm_user_header* header = bufsink_.GetHeaderPtr();
-    header->type = DM_USER_RESP_ERROR;
-    // This is an issue with the dm-user interface. There
-    // is no way to propagate the I/O error back to dm-user
-    // if we have already communicated the header back. Header
-    // is responded once at the beginning; however I/O can
-    // be processed in chunks. If we encounter an I/O error
-    // somewhere in the middle of the processing, we can't communicate
-    // this back to dm-user.
-    //
-    // TODO: Fix the interface
-    CHECK(header_response_);
-
-    WriteDmUserPayload(0);
-}
-
-bool ReadWorker::DmuserReadRequest() {
-    struct dm_user_header* header = bufsink_.GetHeaderPtr();
-
+bool ReadWorker::RequestSectors(uint64_t sector, uint64_t len) {
     // Unaligned I/O request
-    if (!IsBlockAligned(header->sector << SECTOR_SHIFT)) {
-        return ReadUnalignedSector(header->sector, header->len);
+    if (!IsBlockAligned(sector << SECTOR_SHIFT)) {
+        return ReadUnalignedSector(sector, len);
     }
 
-    return ReadAlignedSector(header->sector, header->len);
+    return ReadAlignedSector(sector, len);
 }
 
 bool ReadWorker::SendBufferedIo() {
-    return WriteDmUserPayload(bufsink_.GetPayloadBytesWritten());
-}
-
-bool ReadWorker::ProcessIORequest() {
-    // Read Header from dm-user misc device. This gives
-    // us the sector number for which IO is issued by dm-snapshot device
-    struct dm_user_header* header = bufsink_.GetHeaderPtr();
-    if (!android::base::ReadFully(ctrl_fd_, header, sizeof(*header))) {
-        if (errno != ENOTBLK) {
-            SNAP_PLOG(ERROR) << "Control-read failed";
-        }
-
-        SNAP_PLOG(DEBUG) << "ReadDmUserHeader failed....";
-        return false;
-    }
-
-    SNAP_LOG(DEBUG) << "Daemon: msg->seq: " << std::dec << header->seq;
-    SNAP_LOG(DEBUG) << "Daemon: msg->len: " << std::dec << header->len;
-    SNAP_LOG(DEBUG) << "Daemon: msg->sector: " << std::dec << header->sector;
-    SNAP_LOG(DEBUG) << "Daemon: msg->type: " << std::dec << header->type;
-    SNAP_LOG(DEBUG) << "Daemon: msg->flags: " << std::dec << header->flags;
-
-    // Use the same header buffer as the response header.
-    int request_type = header->type;
-    header->type = DM_USER_RESP_SUCCESS;
-    header_response_ = true;
-
-    // Reset the output buffer.
-    bufsink_.ResetBufferOffset();
-
-    bool ok;
-    switch (request_type) {
-        case DM_USER_REQ_MAP_READ:
-            ok = DmuserReadRequest();
-            break;
-
-        case DM_USER_REQ_MAP_WRITE:
-            // TODO: We should not get any write request
-            // to dm-user as we mount all partitions
-            // as read-only. Need to verify how are TRIM commands
-            // handled during mount.
-            ok = false;
-            break;
-
-        default:
-            ok = false;
-            break;
-    }
-
-    if (!ok && header->type != DM_USER_RESP_ERROR) {
-        RespondIOError();
-    }
-    return ok;
+    return block_server_->SendBufferedIo();
 }
 
 }  // namespace snapshot
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.h
index 8d6f663..6dbae81 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.h
@@ -17,28 +17,28 @@
 #include <utility>
 #include <vector>
 
+#include <snapuserd/block_server.h>
 #include "worker.h"
 
 namespace android {
 namespace snapshot {
 
-class ReadWorker : public Worker {
+class ReadWorker : public Worker, public IBlockServer::Delegate {
   public:
     ReadWorker(const std::string& cow_device, const std::string& backing_device,
-               const std::string& control_device, const std::string& misc_name,
-               const std::string& base_path_merge, std::shared_ptr<SnapshotHandler> snapuserd);
+               const std::string& misc_name, const std::string& base_path_merge,
+               std::shared_ptr<SnapshotHandler> snapuserd,
+               std::shared_ptr<IBlockServerOpener> opener);
 
     bool Run();
     bool Init() override;
     void CloseFds() override;
+    bool RequestSectors(uint64_t sector, uint64_t size) override;
+
+    IBlockServer* block_server() const { return block_server_.get(); }
 
   private:
-    // Functions interacting with dm-user
-    bool ProcessIORequest();
-    bool WriteDmUserPayload(size_t size);
-    bool DmuserReadRequest();
     bool SendBufferedIo();
-    void RespondIOError();
 
     bool ProcessCowOp(const CowOperation* cow_op, void* buffer);
     bool ProcessXorOp(const CowOperation* cow_op, void* buffer);
@@ -60,10 +60,9 @@
     std::string backing_store_device_;
     unique_fd backing_store_fd_;
 
-    std::string control_device_;
-    unique_fd ctrl_fd_;
+    std::shared_ptr<IBlockServerOpener> block_server_opener_;
+    std::unique_ptr<IBlockServer> block_server_;
 
-    bool header_response_ = false;
     std::basic_string<uint8_t> xor_buffer_;
 };
 
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
index 2dd2ec0..c295851 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
@@ -22,9 +22,10 @@
 #include <android-base/properties.h>
 #include <android-base/scopeguard.h>
 #include <android-base/strings.h>
+#include <snapuserd/dm_user_block_server.h>
 
+#include "merge_worker.h"
 #include "read_worker.h"
-#include "snapuserd_merge.h"
 
 namespace android {
 namespace snapshot {
@@ -35,12 +36,12 @@
 
 SnapshotHandler::SnapshotHandler(std::string misc_name, std::string cow_device,
                                  std::string backing_device, std::string base_path_merge,
-                                 int num_worker_threads, bool use_iouring,
-                                 bool perform_verification) {
+                                 std::shared_ptr<IBlockServerOpener> opener, int num_worker_threads,
+                                 bool use_iouring, bool perform_verification) {
     misc_name_ = std::move(misc_name);
     cow_device_ = std::move(cow_device);
     backing_store_device_ = std::move(backing_device);
-    control_device_ = "/dev/dm-user/" + misc_name_;
+    block_server_opener_ = std::move(opener);
     base_path_merge_ = std::move(base_path_merge);
     num_worker_threads_ = num_worker_threads;
     is_io_uring_enabled_ = use_iouring;
@@ -49,8 +50,9 @@
 
 bool SnapshotHandler::InitializeWorkers() {
     for (int i = 0; i < num_worker_threads_; i++) {
-        auto wt = std::make_unique<ReadWorker>(cow_device_, backing_store_device_, control_device_,
-                                               misc_name_, base_path_merge_, GetSharedPtr());
+        auto wt = std::make_unique<ReadWorker>(cow_device_, backing_store_device_, misc_name_,
+                                               base_path_merge_, GetSharedPtr(),
+                                               block_server_opener_);
         if (!wt->Init()) {
             SNAP_LOG(ERROR) << "Thread initialization failed";
             return false;
@@ -280,20 +282,6 @@
         return false;
     }
 
-    unique_fd fd(TEMP_FAILURE_RETRY(open(base_path_merge_.c_str(), O_RDONLY | O_CLOEXEC)));
-    if (fd < 0) {
-        SNAP_LOG(ERROR) << "Cannot open block device";
-        return false;
-    }
-
-    uint64_t dev_sz = get_block_device_size(fd.get());
-    if (!dev_sz) {
-        SNAP_LOG(ERROR) << "Failed to find block device size: " << base_path_merge_;
-        return false;
-    }
-
-    num_sectors_ = dev_sz >> SECTOR_SHIFT;
-
     return ReadMetadata();
 }
 
@@ -307,8 +295,6 @@
     if (ra_thread_) {
         ra_thread_status =
                 std::async(std::launch::async, &ReadAhead::RunThread, read_ahead_thread_.get());
-
-        SNAP_LOG(INFO) << "Read-ahead thread started";
     }
 
     // Launch worker threads
@@ -460,5 +446,21 @@
     merge_thread_ = nullptr;
 }
 
+uint64_t SnapshotHandler::GetNumSectors() const {
+    unique_fd fd(TEMP_FAILURE_RETRY(open(base_path_merge_.c_str(), O_RDONLY | O_CLOEXEC)));
+    if (fd < 0) {
+        SNAP_LOG(ERROR) << "Cannot open base path: " << base_path_merge_;
+        return false;
+    }
+
+    uint64_t dev_sz = get_block_device_size(fd.get());
+    if (!dev_sz) {
+        SNAP_LOG(ERROR) << "Failed to find block device size: " << base_path_merge_;
+        return false;
+    }
+
+    return dev_sz / SECTOR_SIZE;
+}
+
 }  // namespace snapshot
 }  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
index cdc38c0..622fc50 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
@@ -28,6 +28,7 @@
 #include <iostream>
 #include <limits>
 #include <mutex>
+#include <ostream>
 #include <string>
 #include <thread>
 #include <unordered_map>
@@ -43,6 +44,7 @@
 #include <libsnapshot/cow_reader.h>
 #include <libsnapshot/cow_writer.h>
 #include <liburing.h>
+#include <snapuserd/block_server.h>
 #include <snapuserd/snapuserd_buffer.h>
 #include <snapuserd/snapuserd_kernel.h>
 #include <storage_literals/storage_literals.h>
@@ -67,12 +69,13 @@
 #define SNAP_PLOG(level) PLOG(level) << misc_name_ << ": "
 
 enum class MERGE_IO_TRANSITION {
+    INVALID,
     MERGE_READY,
     MERGE_BEGIN,
     MERGE_FAILED,
     MERGE_COMPLETE,
     IO_TERMINATED,
-    READ_AHEAD_FAILURE,
+    READ_AHEAD_FAILURE
 };
 
 class MergeWorker;
@@ -102,14 +105,14 @@
 class SnapshotHandler : public std::enable_shared_from_this<SnapshotHandler> {
   public:
     SnapshotHandler(std::string misc_name, std::string cow_device, std::string backing_device,
-                    std::string base_path_merge, int num_workers, bool use_iouring,
-                    bool perform_verification);
+                    std::string base_path_merge, std::shared_ptr<IBlockServerOpener> opener,
+                    int num_workers, bool use_iouring, bool perform_verification);
     bool InitCowDevice();
     bool Start();
 
     const std::string& GetControlDevicePath() { return control_device_; }
     const std::string& GetMiscName() { return misc_name_; }
-    const uint64_t& GetNumSectors() { return num_sectors_; }
+    uint64_t GetNumSectors() const;
     const bool& IsAttached() const { return attached_; }
     void AttachControlDevice() { attached_ = true; }
 
@@ -202,8 +205,6 @@
 
     unique_fd cow_fd_;
 
-    uint64_t num_sectors_;
-
     std::unique_ptr<CowReader> reader_;
 
     // chunk_vec stores the pseudo mapping of sector
@@ -221,7 +222,7 @@
     bool populate_data_from_cow_ = false;
     bool ra_thread_ = false;
     int total_ra_blocks_merged_ = 0;
-    MERGE_IO_TRANSITION io_state_;
+    MERGE_IO_TRANSITION io_state_ = MERGE_IO_TRANSITION::INVALID;
     std::unique_ptr<ReadAhead> read_ahead_thread_;
     std::unordered_map<uint64_t, void*> read_ahead_buffer_map_;
 
@@ -244,7 +245,10 @@
 
     std::unique_ptr<struct io_uring> ring_;
     std::unique_ptr<UpdateVerify> update_verify_;
+    std::shared_ptr<IBlockServerOpener> block_server_opener_;
 };
 
+std::ostream& operator<<(std::ostream& os, MERGE_IO_TRANSITION value);
+
 }  // namespace snapshot
 }  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
index 8755820..d2128c5 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
@@ -16,7 +16,10 @@
 
 #include "snapuserd_readahead.h"
 
+#include <pthread.h>
+
 #include "snapuserd_core.h"
+#include "utility.h"
 
 namespace android {
 namespace snapshot {
@@ -427,7 +430,7 @@
         // will fallback to synchronous I/O.
         int ret = io_uring_wait_cqe(ring_.get(), &cqe);
         if (ret) {
-            SNAP_LOG(ERROR) << "Read-ahead - io_uring_wait_cqe failed: " << ret;
+            SNAP_LOG(ERROR) << "Read-ahead - io_uring_wait_cqe failed: " << strerror(-ret);
             status = false;
             break;
         }
@@ -690,6 +693,7 @@
     // window. If there is a crash during this time frame, merge should resume
     // based on the contents of the scratch space.
     if (!snapuserd_->WaitForMergeReady()) {
+        SNAP_LOG(ERROR) << "ReadAhead failed to wait for merge ready";
         return false;
     }
 
@@ -751,6 +755,10 @@
 }
 
 bool ReadAhead::RunThread() {
+    SNAP_LOG(INFO) << "ReadAhead thread started.";
+
+    pthread_setname_np(pthread_self(), "ReadAhead");
+
     if (!InitializeFds()) {
         return false;
     }
@@ -765,10 +773,11 @@
 
     InitializeIouring();
 
-    if (setpriority(PRIO_PROCESS, gettid(), kNiceValueForMergeThreads)) {
-        SNAP_PLOG(ERROR) << "Failed to set priority for TID: " << gettid();
+    if (!SetThreadPriority(kNiceValueForMergeThreads)) {
+        SNAP_PLOG(ERROR) << "Failed to set thread priority";
     }
 
+    SNAP_LOG(INFO) << "ReadAhead processing.";
     while (!RAIterDone()) {
         if (!ReadAheadIOStart()) {
             break;
@@ -779,7 +788,7 @@
     CloseFds();
     reader_->CloseCowFd();
 
-    SNAP_LOG(INFO) << " ReadAhead thread terminating....";
+    SNAP_LOG(INFO) << " ReadAhead thread terminating.";
     return true;
 }
 
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_server.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_server.cpp
index c953f1a..13b9a00 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_server.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_server.cpp
@@ -31,6 +31,7 @@
 #include <android-base/scopeguard.h>
 #include <android-base/strings.h>
 #include <fs_mgr/file_wait.h>
+#include <snapuserd/dm_user_block_server.h>
 #include <snapuserd/snapuserd_client.h>
 #include "snapuserd_server.h"
 
@@ -48,6 +49,7 @@
 UserSnapshotServer::UserSnapshotServer() {
     terminating_ = false;
     handlers_ = std::make_unique<SnapshotHandlerManager>();
+    block_server_factory_ = std::make_unique<DmUserBlockServerFactory>();
 }
 
 UserSnapshotServer::~UserSnapshotServer() {
@@ -130,7 +132,12 @@
             return Sendmsg(fd, "fail");
         }
 
-        auto retval = "success," + std::to_string(handler->snapuserd()->GetNumSectors());
+        auto num_sectors = handler->snapuserd()->GetNumSectors();
+        if (!num_sectors) {
+            return Sendmsg(fd, "fail");
+        }
+
+        auto retval = "success," + std::to_string(num_sectors);
         return Sendmsg(fd, retval);
     } else if (cmd == "start") {
         // Message format:
@@ -358,8 +365,11 @@
         perform_verification = false;
     }
 
+    auto opener = block_server_factory_->CreateOpener(misc_name);
+
     return handlers_->AddHandler(misc_name, cow_device_path, backing_device, base_path_merge,
-                                 num_worker_threads, io_uring_enabled_, perform_verification);
+                                 opener, num_worker_threads, io_uring_enabled_,
+                                 perform_verification);
 }
 
 bool UserSnapshotServer::WaitForSocket() {
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_server.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_server.h
index 988c01a..be28541 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_server.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_server.h
@@ -31,6 +31,7 @@
 #include <vector>
 
 #include <android-base/unique_fd.h>
+#include <snapuserd/block_server.h>
 #include "handler_manager.h"
 #include "snapuserd_core.h"
 
@@ -50,6 +51,7 @@
     bool is_server_running_ = false;
     bool io_uring_enabled_ = false;
     std::unique_ptr<ISnapshotHandlerManager> handlers_;
+    std::unique_ptr<IBlockServerFactory> block_server_factory_;
 
     std::mutex lock_;
 
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
index efe0c14..01fe06f 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
@@ -17,7 +17,6 @@
 
 #include <fcntl.h>
 #include <linux/fs.h>
-#include <linux/memfd.h>
 #include <sys/ioctl.h>
 #include <sys/stat.h>
 #include <sys/syscall.h>
@@ -37,10 +36,16 @@
 #include <libdm/dm.h>
 #include <libdm/loop_control.h>
 #include <libsnapshot/cow_writer.h>
+#include <snapuserd/dm_user_block_server.h>
 #include <storage_literals/storage_literals.h>
-
 #include "handler_manager.h"
+#include "merge_worker.h"
+#include "read_worker.h"
 #include "snapuserd_core.h"
+#include "testing/dm_user_harness.h"
+#include "testing/host_harness.h"
+#include "testing/temp_device.h"
+#include "utility.h"
 
 DEFINE_string(force_config, "", "Force testing mode with iouring disabled");
 
@@ -53,186 +58,54 @@
 using namespace std::chrono_literals;
 using namespace android::dm;
 using namespace std;
+using testing::AssertionFailure;
+using testing::AssertionResult;
+using testing::AssertionSuccess;
 
-class Tempdevice {
-  public:
-    Tempdevice(const std::string& name, const DmTable& table)
-        : dm_(DeviceMapper::Instance()), name_(name), valid_(false) {
-        valid_ = dm_.CreateDevice(name, table, &path_, std::chrono::seconds(5));
-    }
-    Tempdevice(Tempdevice&& other) noexcept
-        : dm_(other.dm_), name_(other.name_), path_(other.path_), valid_(other.valid_) {
-        other.valid_ = false;
-    }
-    ~Tempdevice() {
-        if (valid_) {
-            dm_.DeleteDeviceIfExists(name_);
-        }
-    }
-    bool Destroy() {
-        if (!valid_) {
-            return true;
-        }
-        valid_ = false;
-        return dm_.DeleteDeviceIfExists(name_);
-    }
-    const std::string& path() const { return path_; }
-    const std::string& name() const { return name_; }
-    bool valid() const { return valid_; }
-
-    Tempdevice(const Tempdevice&) = delete;
-    Tempdevice& operator=(const Tempdevice&) = delete;
-
-    Tempdevice& operator=(Tempdevice&& other) noexcept {
-        name_ = other.name_;
-        valid_ = other.valid_;
-        other.valid_ = false;
-        return *this;
-    }
-
-  private:
-    DeviceMapper& dm_;
-    std::string name_;
-    std::string path_;
-    bool valid_;
-};
-
-class SnapuserdTest : public ::testing::Test {
-  public:
-    bool SetupDefault();
-    bool SetupOrderedOps();
-    bool SetupOrderedOpsInverted();
-    bool SetupCopyOverlap_1();
-    bool SetupCopyOverlap_2();
-    bool Merge();
-    void ValidateMerge();
-    void ReadSnapshotDeviceAndValidate();
-    void Shutdown();
-    void MergeInterrupt();
-    void MergeInterruptFixed(int duration);
-    void MergeInterruptRandomly(int max_duration);
-    void StartMerge();
-    void CheckMergeCompletion();
-
-    static const uint64_t kSectorSize = 512;
-
+class SnapuserdTestBase : public ::testing::Test {
   protected:
-    void SetUp() override {}
-    void TearDown() override { Shutdown(); }
-
-  private:
-    void SetupImpl();
-
-    void SimulateDaemonRestart();
-
-    std::unique_ptr<ICowWriter> CreateCowDeviceInternal();
-    void CreateCowDevice();
-    void CreateCowDeviceOrderedOps();
-    void CreateCowDeviceOrderedOpsInverted();
-    void CreateCowDeviceWithCopyOverlap_1();
-    void CreateCowDeviceWithCopyOverlap_2();
-    bool SetupDaemon();
+    void SetUp() override;
+    void TearDown() override;
     void CreateBaseDevice();
-    void InitCowDevice();
+    void CreateCowDevice();
     void SetDeviceControlName();
-    void InitDaemon();
-    void CreateDmUserDevice();
+    std::unique_ptr<ICowWriter> CreateCowDeviceInternal();
 
-    unique_ptr<LoopDevice> base_loop_;
-    unique_ptr<Tempdevice> dmuser_dev_;
-
+    std::unique_ptr<ITestHarness> harness_;
+    size_t size_ = 10_MiB;
+    int total_base_size_ = 0;
     std::string system_device_ctrl_name_;
     std::string system_device_name_;
 
+    unique_ptr<IBackingDevice> base_dev_;
     unique_fd base_fd_;
+
     std::unique_ptr<TemporaryFile> cow_system_;
+
     std::unique_ptr<uint8_t[]> orig_buffer_;
-    std::unique_ptr<uint8_t[]> merged_buffer_;
-    SnapshotHandlerManager handlers_;
-    bool setup_ok_ = false;
-    bool merge_ok_ = false;
-    size_t size_ = 100_MiB;
-    int cow_num_sectors_;
-    int total_base_size_;
 };
 
-static unique_fd CreateTempFile(const std::string& name, size_t size) {
-    unique_fd fd(syscall(__NR_memfd_create, name.c_str(), MFD_ALLOW_SEALING));
-    if (fd < 0) {
-        return {};
-    }
-    if (size) {
-        if (ftruncate(fd, size) < 0) {
-            perror("ftruncate");
-            return {};
-        }
-        if (fcntl(fd, F_ADD_SEALS, F_SEAL_GROW | F_SEAL_SHRINK) < 0) {
-            perror("fcntl");
-            return {};
-        }
-    }
-    return fd;
+void SnapuserdTestBase::SetUp() {
+#if __ANDROID__
+    harness_ = std::make_unique<DmUserTestHarness>();
+#else
+    harness_ = std::make_unique<HostTestHarness>();
+#endif
 }
 
-void SnapuserdTest::Shutdown() {
-    ASSERT_TRUE(dmuser_dev_->Destroy());
+void SnapuserdTestBase::TearDown() {}
 
-    auto misc_device = "/dev/dm-user/" + system_device_ctrl_name_;
-    ASSERT_TRUE(handlers_.DeleteHandler(system_device_ctrl_name_));
-    ASSERT_TRUE(android::fs_mgr::WaitForFileDeleted(misc_device, 10s));
-    handlers_.TerminateMergeThreads();
-}
-
-bool SnapuserdTest::SetupDefault() {
-    SetupImpl();
-    return setup_ok_;
-}
-
-bool SnapuserdTest::SetupOrderedOps() {
-    CreateBaseDevice();
-    CreateCowDeviceOrderedOps();
-    return SetupDaemon();
-}
-
-bool SnapuserdTest::SetupOrderedOpsInverted() {
-    CreateBaseDevice();
-    CreateCowDeviceOrderedOpsInverted();
-    return SetupDaemon();
-}
-
-bool SnapuserdTest::SetupCopyOverlap_1() {
-    CreateBaseDevice();
-    CreateCowDeviceWithCopyOverlap_1();
-    return SetupDaemon();
-}
-
-bool SnapuserdTest::SetupCopyOverlap_2() {
-    CreateBaseDevice();
-    CreateCowDeviceWithCopyOverlap_2();
-    return SetupDaemon();
-}
-
-bool SnapuserdTest::SetupDaemon() {
-    SetDeviceControlName();
-
-    CreateDmUserDevice();
-    InitCowDevice();
-    InitDaemon();
-
-    setup_ok_ = true;
-
-    return setup_ok_;
-}
-
-void SnapuserdTest::CreateBaseDevice() {
-    unique_fd rnd_fd;
-
+void SnapuserdTestBase::CreateBaseDevice() {
     total_base_size_ = (size_ * 5);
-    base_fd_ = CreateTempFile("base_device", total_base_size_);
+
+    base_dev_ = harness_->CreateBackingDevice(total_base_size_);
+    ASSERT_NE(base_dev_, nullptr);
+
+    base_fd_.reset(open(base_dev_->GetPath().c_str(), O_RDWR | O_CLOEXEC));
     ASSERT_GE(base_fd_, 0);
 
-    rnd_fd.reset(open("/dev/random", O_RDONLY));
-    ASSERT_TRUE(rnd_fd > 0);
+    unique_fd rnd_fd(open("/dev/random", O_RDONLY));
+    ASSERT_GE(rnd_fd, 0);
 
     std::unique_ptr<uint8_t[]> random_buffer = std::make_unique<uint8_t[]>(1_MiB);
 
@@ -242,13 +115,220 @@
     }
 
     ASSERT_EQ(lseek(base_fd_, 0, SEEK_SET), 0);
+}
 
-    base_loop_ = std::make_unique<LoopDevice>(base_fd_, 10s);
-    ASSERT_TRUE(base_loop_->valid());
+std::unique_ptr<ICowWriter> SnapuserdTestBase::CreateCowDeviceInternal() {
+    cow_system_ = std::make_unique<TemporaryFile>();
+
+    CowOptions options;
+    options.compression = "gz";
+
+    unique_fd fd(cow_system_->fd);
+    cow_system_->fd = -1;
+
+    return CreateCowWriter(kDefaultCowVersion, options, std::move(fd));
+}
+
+void SnapuserdTestBase::CreateCowDevice() {
+    unique_fd rnd_fd;
+    loff_t offset = 0;
+
+    auto writer = CreateCowDeviceInternal();
+    ASSERT_NE(writer, nullptr);
+
+    rnd_fd.reset(open("/dev/random", O_RDONLY));
+    ASSERT_TRUE(rnd_fd > 0);
+
+    std::unique_ptr<uint8_t[]> random_buffer_1_ = std::make_unique<uint8_t[]>(size_);
+
+    // Fill random data
+    for (size_t j = 0; j < (size_ / 1_MiB); j++) {
+        ASSERT_EQ(ReadFullyAtOffset(rnd_fd, (char*)random_buffer_1_.get() + offset, 1_MiB, 0),
+                  true);
+
+        offset += 1_MiB;
+    }
+
+    size_t num_blocks = size_ / writer->GetBlockSize();
+    size_t blk_end_copy = num_blocks * 2;
+    size_t source_blk = num_blocks - 1;
+    size_t blk_src_copy = blk_end_copy - 1;
+
+    uint32_t sequence[num_blocks * 2];
+    // Sequence for Copy ops
+    for (int i = 0; i < num_blocks; i++) {
+        sequence[i] = num_blocks - 1 - i;
+    }
+    // Sequence for Xor ops
+    for (int i = 0; i < num_blocks; i++) {
+        sequence[num_blocks + i] = 5 * num_blocks - 1 - i;
+    }
+    ASSERT_TRUE(writer->AddSequenceData(2 * num_blocks, sequence));
+
+    size_t x = num_blocks;
+    while (1) {
+        ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy));
+        x -= 1;
+        if (x == 0) {
+            break;
+        }
+        source_blk -= 1;
+        blk_src_copy -= 1;
+    }
+
+    source_blk = num_blocks;
+    blk_src_copy = blk_end_copy;
+
+    ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_));
+
+    size_t blk_zero_copy_start = source_blk + num_blocks;
+    size_t blk_zero_copy_end = blk_zero_copy_start + num_blocks;
+
+    ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks));
+
+    size_t blk_random2_replace_start = blk_zero_copy_end;
+
+    ASSERT_TRUE(writer->AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_));
+
+    size_t blk_xor_start = blk_random2_replace_start + num_blocks;
+    size_t xor_offset = BLOCK_SZ / 2;
+    ASSERT_TRUE(writer->AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks,
+                                     xor_offset));
+
+    // Flush operations
+    ASSERT_TRUE(writer->Finalize());
+    // Construct the buffer required for validation
+    orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
+    std::string zero_buffer(size_, 0);
+    ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), size_, size_), true);
+    memcpy((char*)orig_buffer_.get() + size_, random_buffer_1_.get(), size_);
+    memcpy((char*)orig_buffer_.get() + (size_ * 2), (void*)zero_buffer.c_str(), size_);
+    memcpy((char*)orig_buffer_.get() + (size_ * 3), random_buffer_1_.get(), size_);
+    ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, &orig_buffer_.get()[size_ * 4], size_,
+                                               size_ + xor_offset),
+              true);
+    for (int i = 0; i < size_; i++) {
+        orig_buffer_.get()[(size_ * 4) + i] =
+                (uint8_t)(orig_buffer_.get()[(size_ * 4) + i] ^ random_buffer_1_.get()[i]);
+    }
+}
+
+void SnapuserdTestBase::SetDeviceControlName() {
+    system_device_name_.clear();
+    system_device_ctrl_name_.clear();
+
+    std::string str(cow_system_->path);
+    std::size_t found = str.find_last_of("/\\");
+    ASSERT_NE(found, std::string::npos);
+    system_device_name_ = str.substr(found + 1);
+
+    system_device_ctrl_name_ = system_device_name_ + "-ctrl";
+}
+
+class SnapuserdTest : public SnapuserdTestBase {
+  public:
+    void SetupDefault();
+    void SetupOrderedOps();
+    void SetupOrderedOpsInverted();
+    void SetupCopyOverlap_1();
+    void SetupCopyOverlap_2();
+    bool Merge();
+    void ValidateMerge();
+    void ReadSnapshotDeviceAndValidate();
+    void Shutdown();
+    void MergeInterrupt();
+    void MergeInterruptFixed(int duration);
+    void MergeInterruptRandomly(int max_duration);
+    bool StartMerge();
+    void CheckMergeCompletion();
+
+    static const uint64_t kSectorSize = 512;
+
+  protected:
+    void SetUp() override;
+    void TearDown() override;
+
+    void SetupImpl();
+
+    void SimulateDaemonRestart();
+
+    void CreateCowDeviceOrderedOps();
+    void CreateCowDeviceOrderedOpsInverted();
+    void CreateCowDeviceWithCopyOverlap_1();
+    void CreateCowDeviceWithCopyOverlap_2();
+    void SetupDaemon();
+    void InitCowDevice();
+    void InitDaemon();
+    void CreateUserDevice();
+
+    unique_ptr<IUserDevice> dmuser_dev_;
+
+    std::unique_ptr<uint8_t[]> merged_buffer_;
+    std::unique_ptr<SnapshotHandlerManager> handlers_;
+    int cow_num_sectors_;
+};
+
+void SnapuserdTest::SetUp() {
+    ASSERT_NO_FATAL_FAILURE(SnapuserdTestBase::SetUp());
+    handlers_ = std::make_unique<SnapshotHandlerManager>();
+}
+
+void SnapuserdTest::TearDown() {
+    SnapuserdTestBase::TearDown();
+    Shutdown();
+}
+
+void SnapuserdTest::Shutdown() {
+    if (dmuser_dev_) {
+        ASSERT_TRUE(dmuser_dev_->Destroy());
+    }
+
+    auto misc_device = "/dev/dm-user/" + system_device_ctrl_name_;
+    ASSERT_TRUE(handlers_->DeleteHandler(system_device_ctrl_name_));
+    ASSERT_TRUE(android::fs_mgr::WaitForFileDeleted(misc_device, 10s));
+    handlers_->TerminateMergeThreads();
+    handlers_->JoinAllThreads();
+    handlers_ = std::make_unique<SnapshotHandlerManager>();
+}
+
+void SnapuserdTest::SetupDefault() {
+    ASSERT_NO_FATAL_FAILURE(SetupImpl());
+}
+
+void SnapuserdTest::SetupOrderedOps() {
+    ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+    ASSERT_NO_FATAL_FAILURE(CreateCowDeviceOrderedOps());
+    ASSERT_NO_FATAL_FAILURE(SetupDaemon());
+}
+
+void SnapuserdTest::SetupOrderedOpsInverted() {
+    ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+    ASSERT_NO_FATAL_FAILURE(CreateCowDeviceOrderedOpsInverted());
+    ASSERT_NO_FATAL_FAILURE(SetupDaemon());
+}
+
+void SnapuserdTest::SetupCopyOverlap_1() {
+    ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+    ASSERT_NO_FATAL_FAILURE(CreateCowDeviceWithCopyOverlap_1());
+    ASSERT_NO_FATAL_FAILURE(SetupDaemon());
+}
+
+void SnapuserdTest::SetupCopyOverlap_2() {
+    ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+    ASSERT_NO_FATAL_FAILURE(CreateCowDeviceWithCopyOverlap_2());
+    ASSERT_NO_FATAL_FAILURE(SetupDaemon());
+}
+
+void SnapuserdTest::SetupDaemon() {
+    SetDeviceControlName();
+
+    ASSERT_NO_FATAL_FAILURE(CreateUserDevice());
+    ASSERT_NO_FATAL_FAILURE(InitCowDevice());
+    ASSERT_NO_FATAL_FAILURE(InitDaemon());
 }
 
 void SnapuserdTest::ReadSnapshotDeviceAndValidate() {
-    unique_fd fd(open(dmuser_dev_->path().c_str(), O_RDONLY));
+    unique_fd fd(open(dmuser_dev_->GetPath().c_str(), O_RDONLY));
     ASSERT_GE(fd, 0);
     std::unique_ptr<uint8_t[]> snapuserd_buffer = std::make_unique<uint8_t[]>(size_);
 
@@ -278,19 +358,6 @@
     ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 4), size_), 0);
 }
 
-std::unique_ptr<ICowWriter> SnapuserdTest::CreateCowDeviceInternal() {
-    std::string path = android::base::GetExecutableDirectory();
-    cow_system_ = std::make_unique<TemporaryFile>(path);
-
-    CowOptions options;
-    options.compression = "gz";
-
-    unique_fd fd(cow_system_->fd);
-    cow_system_->fd = -1;
-
-    return CreateCowWriter(kDefaultCowVersion, options, std::move(fd));
-}
-
 void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() {
     auto writer = CreateCowDeviceInternal();
     ASSERT_NE(writer, nullptr);
@@ -487,145 +554,42 @@
     }
 }
 
-void SnapuserdTest::CreateCowDevice() {
-    unique_fd rnd_fd;
-    loff_t offset = 0;
-
-    auto writer = CreateCowDeviceInternal();
-    ASSERT_NE(writer, nullptr);
-
-    rnd_fd.reset(open("/dev/random", O_RDONLY));
-    ASSERT_TRUE(rnd_fd > 0);
-
-    std::unique_ptr<uint8_t[]> random_buffer_1_ = std::make_unique<uint8_t[]>(size_);
-
-    // Fill random data
-    for (size_t j = 0; j < (size_ / 1_MiB); j++) {
-        ASSERT_EQ(ReadFullyAtOffset(rnd_fd, (char*)random_buffer_1_.get() + offset, 1_MiB, 0),
-                  true);
-
-        offset += 1_MiB;
-    }
-
-    size_t num_blocks = size_ / writer->GetBlockSize();
-    size_t blk_end_copy = num_blocks * 2;
-    size_t source_blk = num_blocks - 1;
-    size_t blk_src_copy = blk_end_copy - 1;
-
-    uint32_t sequence[num_blocks * 2];
-    // Sequence for Copy ops
-    for (int i = 0; i < num_blocks; i++) {
-        sequence[i] = num_blocks - 1 - i;
-    }
-    // Sequence for Xor ops
-    for (int i = 0; i < num_blocks; i++) {
-        sequence[num_blocks + i] = 5 * num_blocks - 1 - i;
-    }
-    ASSERT_TRUE(writer->AddSequenceData(2 * num_blocks, sequence));
-
-    size_t x = num_blocks;
-    while (1) {
-        ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy));
-        x -= 1;
-        if (x == 0) {
-            break;
-        }
-        source_blk -= 1;
-        blk_src_copy -= 1;
-    }
-
-    source_blk = num_blocks;
-    blk_src_copy = blk_end_copy;
-
-    ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_));
-
-    size_t blk_zero_copy_start = source_blk + num_blocks;
-    size_t blk_zero_copy_end = blk_zero_copy_start + num_blocks;
-
-    ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks));
-
-    size_t blk_random2_replace_start = blk_zero_copy_end;
-
-    ASSERT_TRUE(writer->AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_));
-
-    size_t blk_xor_start = blk_random2_replace_start + num_blocks;
-    size_t xor_offset = BLOCK_SZ / 2;
-    ASSERT_TRUE(writer->AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks,
-                                     xor_offset));
-
-    // Flush operations
-    ASSERT_TRUE(writer->Finalize());
-    // Construct the buffer required for validation
-    orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
-    std::string zero_buffer(size_, 0);
-    ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), size_, size_), true);
-    memcpy((char*)orig_buffer_.get() + size_, random_buffer_1_.get(), size_);
-    memcpy((char*)orig_buffer_.get() + (size_ * 2), (void*)zero_buffer.c_str(), size_);
-    memcpy((char*)orig_buffer_.get() + (size_ * 3), random_buffer_1_.get(), size_);
-    ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, &orig_buffer_.get()[size_ * 4], size_,
-                                               size_ + xor_offset),
-              true);
-    for (int i = 0; i < size_; i++) {
-        orig_buffer_.get()[(size_ * 4) + i] =
-                (uint8_t)(orig_buffer_.get()[(size_ * 4) + i] ^ random_buffer_1_.get()[i]);
-    }
-}
-
 void SnapuserdTest::InitCowDevice() {
     bool use_iouring = true;
     if (FLAGS_force_config == "iouring_disabled") {
         use_iouring = false;
     }
 
+    auto factory = harness_->GetBlockServerFactory();
+    auto opener = factory->CreateOpener(system_device_ctrl_name_);
     auto handler =
-            handlers_.AddHandler(system_device_ctrl_name_, cow_system_->path, base_loop_->device(),
-                                 base_loop_->device(), 1, use_iouring, false);
+            handlers_->AddHandler(system_device_ctrl_name_, cow_system_->path, base_dev_->GetPath(),
+                                  base_dev_->GetPath(), opener, 1, use_iouring, false);
     ASSERT_NE(handler, nullptr);
     ASSERT_NE(handler->snapuserd(), nullptr);
+#ifdef __ANDROID__
     ASSERT_NE(handler->snapuserd()->GetNumSectors(), 0);
+#endif
 }
 
-void SnapuserdTest::SetDeviceControlName() {
-    system_device_name_.clear();
-    system_device_ctrl_name_.clear();
-
-    std::string str(cow_system_->path);
-    std::size_t found = str.find_last_of("/\\");
-    ASSERT_NE(found, std::string::npos);
-    system_device_name_ = str.substr(found + 1);
-
-    system_device_ctrl_name_ = system_device_name_ + "-ctrl";
-}
-
-void SnapuserdTest::CreateDmUserDevice() {
-    unique_fd fd(TEMP_FAILURE_RETRY(open(base_loop_->device().c_str(), O_RDONLY | O_CLOEXEC)));
-    ASSERT_TRUE(fd > 0);
-
-    uint64_t dev_sz = get_block_device_size(fd.get());
-    ASSERT_TRUE(dev_sz > 0);
+void SnapuserdTest::CreateUserDevice() {
+    auto dev_sz = base_dev_->GetSize();
+    ASSERT_NE(dev_sz, 0);
 
     cow_num_sectors_ = dev_sz >> 9;
 
-    DmTable dmuser_table;
-    ASSERT_TRUE(dmuser_table.AddTarget(
-            std::make_unique<DmTargetUser>(0, cow_num_sectors_, system_device_ctrl_name_)));
-    ASSERT_TRUE(dmuser_table.valid());
-
-    dmuser_dev_ = std::make_unique<Tempdevice>(system_device_name_, dmuser_table);
-    ASSERT_TRUE(dmuser_dev_->valid());
-    ASSERT_FALSE(dmuser_dev_->path().empty());
-
-    auto misc_device = "/dev/dm-user/" + system_device_ctrl_name_;
-    ASSERT_TRUE(android::fs_mgr::WaitForFile(misc_device, 10s));
+    dmuser_dev_ = harness_->CreateUserDevice(system_device_name_, system_device_ctrl_name_,
+                                             cow_num_sectors_);
+    ASSERT_NE(dmuser_dev_, nullptr);
 }
 
 void SnapuserdTest::InitDaemon() {
-    ASSERT_TRUE(handlers_.StartHandler(system_device_ctrl_name_));
+    ASSERT_TRUE(handlers_->StartHandler(system_device_ctrl_name_));
 }
 
 void SnapuserdTest::CheckMergeCompletion() {
     while (true) {
-        double percentage = handlers_.GetMergePercentage();
+        double percentage = handlers_->GetMergePercentage();
         if ((int)percentage == 100) {
             break;
         }
@@ -635,27 +599,26 @@
 }
 
 void SnapuserdTest::SetupImpl() {
-    CreateBaseDevice();
-    CreateCowDevice();
+    ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+    ASSERT_NO_FATAL_FAILURE(CreateCowDevice());
 
     SetDeviceControlName();
 
-    CreateDmUserDevice();
-    InitCowDevice();
-    InitDaemon();
-
-    setup_ok_ = true;
+    ASSERT_NO_FATAL_FAILURE(CreateUserDevice());
+    ASSERT_NO_FATAL_FAILURE(InitCowDevice());
+    ASSERT_NO_FATAL_FAILURE(InitDaemon());
 }
 
 bool SnapuserdTest::Merge() {
-    StartMerge();
+    if (!StartMerge()) {
+        return false;
+    }
     CheckMergeCompletion();
-    merge_ok_ = true;
-    return merge_ok_;
+    return true;
 }
 
-void SnapuserdTest::StartMerge() {
-    ASSERT_TRUE(handlers_.InitiateMerge(system_device_ctrl_name_));
+bool SnapuserdTest::StartMerge() {
+    return handlers_->InitiateMerge(system_device_ctrl_name_);
 }
 
 void SnapuserdTest::ValidateMerge() {
@@ -666,158 +629,260 @@
 }
 
 void SnapuserdTest::SimulateDaemonRestart() {
-    Shutdown();
+    ASSERT_NO_FATAL_FAILURE(Shutdown());
     std::this_thread::sleep_for(500ms);
     SetDeviceControlName();
-    CreateDmUserDevice();
-    InitCowDevice();
-    InitDaemon();
+    ASSERT_NO_FATAL_FAILURE(CreateUserDevice());
+    ASSERT_NO_FATAL_FAILURE(InitCowDevice());
+    ASSERT_NO_FATAL_FAILURE(InitDaemon());
 }
 
 void SnapuserdTest::MergeInterruptRandomly(int max_duration) {
     std::srand(std::time(nullptr));
-    StartMerge();
+    ASSERT_TRUE(StartMerge());
 
     for (int i = 0; i < 20; i++) {
         int duration = std::rand() % max_duration;
         std::this_thread::sleep_for(std::chrono::milliseconds(duration));
-        SimulateDaemonRestart();
-        StartMerge();
+        ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
+        ASSERT_TRUE(StartMerge());
     }
 
-    SimulateDaemonRestart();
+    ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
     ASSERT_TRUE(Merge());
 }
 
 void SnapuserdTest::MergeInterruptFixed(int duration) {
-    StartMerge();
+    ASSERT_TRUE(StartMerge());
 
     for (int i = 0; i < 25; i++) {
         std::this_thread::sleep_for(std::chrono::milliseconds(duration));
-        SimulateDaemonRestart();
-        StartMerge();
+        ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
+        ASSERT_TRUE(StartMerge());
     }
 
-    SimulateDaemonRestart();
+    ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
     ASSERT_TRUE(Merge());
 }
 
 void SnapuserdTest::MergeInterrupt() {
     // Interrupt merge at various intervals
-    StartMerge();
+    ASSERT_TRUE(StartMerge());
     std::this_thread::sleep_for(250ms);
-    SimulateDaemonRestart();
+    ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
 
-    StartMerge();
+    ASSERT_TRUE(StartMerge());
     std::this_thread::sleep_for(250ms);
-    SimulateDaemonRestart();
+    ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
 
-    StartMerge();
+    ASSERT_TRUE(StartMerge());
     std::this_thread::sleep_for(150ms);
-    SimulateDaemonRestart();
+    ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
 
-    StartMerge();
+    ASSERT_TRUE(StartMerge());
     std::this_thread::sleep_for(100ms);
-    SimulateDaemonRestart();
+    ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
 
-    StartMerge();
+    ASSERT_TRUE(StartMerge());
     std::this_thread::sleep_for(800ms);
-    SimulateDaemonRestart();
+    ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
 
-    StartMerge();
+    ASSERT_TRUE(StartMerge());
     std::this_thread::sleep_for(600ms);
-    SimulateDaemonRestart();
+    ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
 
     ASSERT_TRUE(Merge());
 }
 
 TEST_F(SnapuserdTest, Snapshot_IO_TEST) {
-    ASSERT_TRUE(SetupDefault());
+    if (!harness_->HasUserDevice()) {
+        GTEST_SKIP() << "Skipping snapshot read; not supported";
+    }
+    ASSERT_NO_FATAL_FAILURE(SetupDefault());
     // I/O before merge
-    ReadSnapshotDeviceAndValidate();
+    ASSERT_NO_FATAL_FAILURE(ReadSnapshotDeviceAndValidate());
     ASSERT_TRUE(Merge());
     ValidateMerge();
     // I/O after merge - daemon should read directly
     // from base device
-    ReadSnapshotDeviceAndValidate();
-    Shutdown();
+    ASSERT_NO_FATAL_FAILURE(ReadSnapshotDeviceAndValidate());
 }
 
 TEST_F(SnapuserdTest, Snapshot_MERGE_IO_TEST) {
-    ASSERT_TRUE(SetupDefault());
+    if (!harness_->HasUserDevice()) {
+        GTEST_SKIP() << "Skipping snapshot read; not supported";
+    }
+    ASSERT_NO_FATAL_FAILURE(SetupDefault());
     // Issue I/O before merge begins
     std::async(std::launch::async, &SnapuserdTest::ReadSnapshotDeviceAndValidate, this);
     // Start the merge
     ASSERT_TRUE(Merge());
     ValidateMerge();
-    Shutdown();
 }
 
 TEST_F(SnapuserdTest, Snapshot_MERGE_IO_TEST_1) {
-    ASSERT_TRUE(SetupDefault());
+    if (!harness_->HasUserDevice()) {
+        GTEST_SKIP() << "Skipping snapshot read; not supported";
+    }
+    ASSERT_NO_FATAL_FAILURE(SetupDefault());
     // Start the merge
-    StartMerge();
+    ASSERT_TRUE(StartMerge());
     // Issue I/O in parallel when merge is in-progress
     std::async(std::launch::async, &SnapuserdTest::ReadSnapshotDeviceAndValidate, this);
     CheckMergeCompletion();
     ValidateMerge();
-    Shutdown();
 }
 
 TEST_F(SnapuserdTest, Snapshot_Merge_Resume) {
-    ASSERT_TRUE(SetupDefault());
-    MergeInterrupt();
+    ASSERT_NO_FATAL_FAILURE(SetupDefault());
+    ASSERT_NO_FATAL_FAILURE(MergeInterrupt());
     ValidateMerge();
-    Shutdown();
 }
 
 TEST_F(SnapuserdTest, Snapshot_COPY_Overlap_TEST_1) {
-    ASSERT_TRUE(SetupCopyOverlap_1());
+    ASSERT_NO_FATAL_FAILURE(SetupCopyOverlap_1());
     ASSERT_TRUE(Merge());
     ValidateMerge();
-    Shutdown();
 }
 
 TEST_F(SnapuserdTest, Snapshot_COPY_Overlap_TEST_2) {
-    ASSERT_TRUE(SetupCopyOverlap_2());
+    ASSERT_NO_FATAL_FAILURE(SetupCopyOverlap_2());
     ASSERT_TRUE(Merge());
     ValidateMerge();
-    Shutdown();
 }
 
 TEST_F(SnapuserdTest, Snapshot_COPY_Overlap_Merge_Resume_TEST) {
-    ASSERT_TRUE(SetupCopyOverlap_1());
-    MergeInterrupt();
+    ASSERT_NO_FATAL_FAILURE(SetupCopyOverlap_1());
+    ASSERT_NO_FATAL_FAILURE(MergeInterrupt());
     ValidateMerge();
-    Shutdown();
 }
 
 TEST_F(SnapuserdTest, Snapshot_Merge_Crash_Fixed_Ordered) {
-    ASSERT_TRUE(SetupOrderedOps());
-    MergeInterruptFixed(300);
+    ASSERT_NO_FATAL_FAILURE(SetupOrderedOps());
+    ASSERT_NO_FATAL_FAILURE(MergeInterruptFixed(300));
     ValidateMerge();
-    Shutdown();
 }
 
 TEST_F(SnapuserdTest, Snapshot_Merge_Crash_Random_Ordered) {
-    ASSERT_TRUE(SetupOrderedOps());
-    MergeInterruptRandomly(500);
+    ASSERT_NO_FATAL_FAILURE(SetupOrderedOps());
+    ASSERT_NO_FATAL_FAILURE(MergeInterruptRandomly(500));
     ValidateMerge();
-    Shutdown();
 }
 
 TEST_F(SnapuserdTest, Snapshot_Merge_Crash_Fixed_Inverted) {
-    ASSERT_TRUE(SetupOrderedOpsInverted());
-    MergeInterruptFixed(50);
+    ASSERT_NO_FATAL_FAILURE(SetupOrderedOpsInverted());
+    ASSERT_NO_FATAL_FAILURE(MergeInterruptFixed(50));
     ValidateMerge();
-    Shutdown();
 }
 
 TEST_F(SnapuserdTest, Snapshot_Merge_Crash_Random_Inverted) {
-    ASSERT_TRUE(SetupOrderedOpsInverted());
-    MergeInterruptRandomly(50);
+    ASSERT_NO_FATAL_FAILURE(SetupOrderedOpsInverted());
+    ASSERT_NO_FATAL_FAILURE(MergeInterruptRandomly(50));
     ValidateMerge();
-    Shutdown();
+}
+
+class HandlerTest : public SnapuserdTestBase {
+  protected:
+    void SetUp() override;
+    void TearDown() override;
+
+    AssertionResult ReadSectors(sector_t sector, uint64_t size, void* buffer);
+
+    TestBlockServerFactory factory_;
+    std::shared_ptr<TestBlockServerOpener> opener_;
+    std::shared_ptr<SnapshotHandler> handler_;
+    std::unique_ptr<ReadWorker> read_worker_;
+    TestBlockServer* block_server_;
+    std::future<bool> handler_thread_;
+};
+
+void HandlerTest::SetUp() {
+    ASSERT_NO_FATAL_FAILURE(SnapuserdTestBase::SetUp());
+    ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+    ASSERT_NO_FATAL_FAILURE(CreateCowDevice());
+    ASSERT_NO_FATAL_FAILURE(SetDeviceControlName());
+
+    opener_ = factory_.CreateTestOpener(system_device_ctrl_name_);
+    ASSERT_NE(opener_, nullptr);
+
+    handler_ = std::make_shared<SnapshotHandler>(system_device_ctrl_name_, cow_system_->path,
+                                                 base_dev_->GetPath(), base_dev_->GetPath(),
+                                                 opener_, 1, false, false);
+    ASSERT_TRUE(handler_->InitCowDevice());
+    ASSERT_TRUE(handler_->InitializeWorkers());
+
+    read_worker_ = std::make_unique<ReadWorker>(cow_system_->path, base_dev_->GetPath(),
+                                                system_device_ctrl_name_, base_dev_->GetPath(),
+                                                handler_->GetSharedPtr(), opener_);
+    ASSERT_TRUE(read_worker_->Init());
+    block_server_ = static_cast<TestBlockServer*>(read_worker_->block_server());
+
+    handler_thread_ = std::async(std::launch::async, &SnapshotHandler::Start, handler_.get());
+}
+
+void HandlerTest::TearDown() {
+    ASSERT_TRUE(factory_.DeleteQueue(system_device_ctrl_name_));
+    ASSERT_TRUE(handler_thread_.get());
+    SnapuserdTestBase::TearDown();
+}
+
+AssertionResult HandlerTest::ReadSectors(sector_t sector, uint64_t size, void* buffer) {
+    if (!read_worker_->RequestSectors(sector, size)) {
+        return AssertionFailure() << "request sectors failed";
+    }
+
+    std::string result = std::move(block_server_->sent_io());
+    if (result.size() != size) {
+        return AssertionFailure() << "size mismatch in result, got " << result.size()
+                                  << ", expected " << size;
+    }
+
+    memcpy(buffer, result.data(), size);
+    return AssertionSuccess();
+}
+
+// This test mirrors ReadSnapshotDeviceAndValidate.
+TEST_F(HandlerTest, Read) {
+    std::unique_ptr<uint8_t[]> snapuserd_buffer = std::make_unique<uint8_t[]>(size_);
+
+    // COPY
+    loff_t offset = 0;
+    ASSERT_TRUE(ReadSectors(offset / SECTOR_SIZE, size_, snapuserd_buffer.get()));
+    ASSERT_EQ(memcmp(snapuserd_buffer.get(), orig_buffer_.get(), size_), 0);
+
+    // REPLACE
+    offset += size_;
+    ASSERT_TRUE(ReadSectors(offset / SECTOR_SIZE, size_, snapuserd_buffer.get()));
+    ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + size_, size_), 0);
+
+    // ZERO
+    offset += size_;
+    ASSERT_TRUE(ReadSectors(offset / SECTOR_SIZE, size_, snapuserd_buffer.get()));
+    ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 2), size_), 0);
+
+    // REPLACE
+    offset += size_;
+    ASSERT_TRUE(ReadSectors(offset / SECTOR_SIZE, size_, snapuserd_buffer.get()));
+    ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 3), size_), 0);
+
+    // XOR
+    offset += size_;
+    ASSERT_TRUE(ReadSectors(offset / SECTOR_SIZE, size_, snapuserd_buffer.get()));
+    ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 4), size_), 0);
+}
+
+TEST_F(HandlerTest, ReadUnalignedSector) {
+    std::unique_ptr<uint8_t[]> snapuserd_buffer = std::make_unique<uint8_t[]>(BLOCK_SZ);
+
+    ASSERT_TRUE(ReadSectors(1, BLOCK_SZ, snapuserd_buffer.get()));
+    ASSERT_EQ(memcmp(snapuserd_buffer.get(), orig_buffer_.get() + SECTOR_SIZE, BLOCK_SZ), 0);
+}
+
+TEST_F(HandlerTest, ReadUnalignedSize) {
+    std::unique_ptr<uint8_t[]> snapuserd_buffer = std::make_unique<uint8_t[]>(SECTOR_SIZE);
+
+    ASSERT_TRUE(ReadSectors(0, SECTOR_SIZE, snapuserd_buffer.get()));
+    ASSERT_EQ(memcmp(snapuserd_buffer.get(), orig_buffer_.get(), SECTOR_SIZE), 0);
 }
 
 }  // namespace snapshot
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp
index 28c9f68..52e4f89 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp
@@ -199,6 +199,7 @@
 
             if (io_state_ == MERGE_IO_TRANSITION::READ_AHEAD_FAILURE ||
                 io_state_ == MERGE_IO_TRANSITION::IO_TERMINATED) {
+                SNAP_LOG(ERROR) << "WaitForMergeBegin failed with state: " << io_state_;
                 return false;
             }
         }
@@ -211,6 +212,7 @@
 
         if (io_state_ == MERGE_IO_TRANSITION::READ_AHEAD_FAILURE ||
             io_state_ == MERGE_IO_TRANSITION::IO_TERMINATED) {
+            SNAP_LOG(ERROR) << "WaitForMergeBegin failed with state: " << io_state_;
             return false;
         }
 
@@ -277,6 +279,7 @@
         if (io_state_ == MERGE_IO_TRANSITION::MERGE_FAILED ||
             io_state_ == MERGE_IO_TRANSITION::MERGE_COMPLETE ||
             io_state_ == MERGE_IO_TRANSITION::IO_TERMINATED) {
+            SNAP_LOG(ERROR) << "Wait for merge ready failed: " << io_state_;
             return false;
         }
         return true;
@@ -668,5 +671,26 @@
     }
 }
 
+std::ostream& operator<<(std::ostream& os, MERGE_IO_TRANSITION value) {
+    switch (value) {
+        case MERGE_IO_TRANSITION::INVALID:
+            return os << "INVALID";
+        case MERGE_IO_TRANSITION::MERGE_READY:
+            return os << "MERGE_READY";
+        case MERGE_IO_TRANSITION::MERGE_BEGIN:
+            return os << "MERGE_BEGIN";
+        case MERGE_IO_TRANSITION::MERGE_FAILED:
+            return os << "MERGE_FAILED";
+        case MERGE_IO_TRANSITION::MERGE_COMPLETE:
+            return os << "MERGE_COMPLETE";
+        case MERGE_IO_TRANSITION::IO_TERMINATED:
+            return os << "IO_TERMINATED";
+        case MERGE_IO_TRANSITION::READ_AHEAD_FAILURE:
+            return os << "READ_AHEAD_FAILURE";
+        default:
+            return os << "unknown";
+    }
+}
+
 }  // namespace snapshot
 }  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.cpp
index aa15630..65208fb 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.cpp
@@ -27,18 +27,7 @@
     snapuserd_ = snapuserd;
 }
 
-void Worker::InitializeBufsink() {
-    // Allocate the buffer which is used to communicate between
-    // daemon and dm-user. The buffer comprises of header and a fixed payload.
-    // If the dm-user requests a big IO, the IO will be broken into chunks
-    // of PAYLOAD_BUFFER_SZ.
-    size_t buf_size = sizeof(struct dm_user_header) + PAYLOAD_BUFFER_SZ;
-    bufsink_.Initialize(buf_size);
-}
-
 bool Worker::Init() {
-    InitializeBufsink();
-
     if (!InitializeFds()) {
         return false;
     }
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.h
index 813b159..c89d1b4 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.h
@@ -40,14 +40,11 @@
     virtual bool Init();
 
   protected:
-    // Initialization
-    void InitializeBufsink();
     bool InitializeFds();
     bool InitReader();
     virtual void CloseFds() { base_path_merge_fd_ = {}; }
 
     std::unique_ptr<CowReader> reader_;
-    BufferSink bufsink_;
 
     std::string misc_name_;  // Needed for SNAP_LOG.
 
diff --git a/fs_mgr/libsnapshot/snapuserd/utility.cpp b/fs_mgr/libsnapshot/snapuserd/utility.cpp
new file mode 100644
index 0000000..a84a7c1
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/utility.cpp
@@ -0,0 +1,36 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "utility.h"
+
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include <android-base/file.h>
+
+namespace android {
+namespace snapshot {
+
+using android::base::unique_fd;
+
+bool SetThreadPriority([[maybe_unused]] int priority) {
+#ifdef __ANDROID__
+    return setpriority(PRIO_PROCESS, gettid(), priority) != -1;
+#else
+    return true;
+#endif
+}
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/utility.h b/fs_mgr/libsnapshot/snapuserd/utility.h
new file mode 100644
index 0000000..58ec3e6
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/utility.h
@@ -0,0 +1,23 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+namespace android {
+namespace snapshot {
+
+bool SetThreadPriority(int priority);
+
+}  // namespace snapshot
+}  // namespace android
diff --git a/fs_mgr/tests/fs_mgr_test.cpp b/fs_mgr/tests/fs_mgr_test.cpp
index fbed371..322bf1b 100644
--- a/fs_mgr/tests/fs_mgr_test.cpp
+++ b/fs_mgr/tests/fs_mgr_test.cpp
@@ -224,23 +224,32 @@
 
 }  // namespace
 
-TEST(fs_mgr, fs_mgr_parse_cmdline) {
-    EXPECT_EQ(result_space, fs_mgr_parse_cmdline(cmdline));
+TEST(fs_mgr, ImportKernelCmdline) {
+    std::vector<std::pair<std::string, std::string>> result;
+    ImportKernelCmdlineFromString(
+            cmdline, [&](std::string key, std::string value) { result.emplace_back(key, value); });
+    EXPECT_THAT(result, ContainerEq(result_space));
 }
 
-TEST(fs_mgr, fs_mgr_get_boot_config_from_kernel_cmdline) {
+TEST(fs_mgr, GetKernelCmdline) {
     std::string content;
-    for (const auto& entry : result_space) {
-        static constexpr char androidboot[] = "androidboot.";
-        if (!android::base::StartsWith(entry.first, androidboot)) continue;
-        auto key = entry.first.substr(strlen(androidboot));
-        EXPECT_TRUE(fs_mgr_get_boot_config_from_kernel(cmdline, key, &content)) << " for " << key;
-        EXPECT_EQ(entry.second, content);
+    for (const auto& [key, value] : result_space) {
+        EXPECT_TRUE(GetKernelCmdlineFromString(cmdline, key, &content)) << " for " << key;
+        EXPECT_EQ(content, value);
     }
-    EXPECT_FALSE(fs_mgr_get_boot_config_from_kernel(cmdline, "vbmeta.avb_versio", &content));
-    EXPECT_TRUE(content.empty()) << content;
-    EXPECT_FALSE(fs_mgr_get_boot_config_from_kernel(cmdline, "nospace", &content));
-    EXPECT_TRUE(content.empty()) << content;
+
+    const std::string kUnmodifiedToken = "<UNMODIFIED>";
+    content = kUnmodifiedToken;
+    EXPECT_FALSE(GetKernelCmdlineFromString(cmdline, "", &content));
+    EXPECT_EQ(content, kUnmodifiedToken) << "output parameter shouldn't be overridden";
+
+    content = kUnmodifiedToken;
+    EXPECT_FALSE(GetKernelCmdlineFromString(cmdline, "androidboot.vbmeta.avb_versio", &content));
+    EXPECT_EQ(content, kUnmodifiedToken) << "output parameter shouldn't be overridden";
+
+    content = kUnmodifiedToken;
+    EXPECT_FALSE(GetKernelCmdlineFromString(bootconfig, "androidboot.nospace", &content));
+    EXPECT_EQ(content, kUnmodifiedToken) << "output parameter shouldn't be overridden";
 }
 
 TEST(fs_mgr, ImportBootconfig) {
@@ -260,17 +269,15 @@
 
     const std::string kUnmodifiedToken = "<UNMODIFIED>";
     content = kUnmodifiedToken;
-    EXPECT_FALSE(android::fs_mgr::GetBootconfigFromString(bootconfig, "", &content));
+    EXPECT_FALSE(GetBootconfigFromString(bootconfig, "", &content));
     EXPECT_EQ(content, kUnmodifiedToken) << "output parameter shouldn't be overridden";
 
     content = kUnmodifiedToken;
-    EXPECT_FALSE(android::fs_mgr::GetBootconfigFromString(
-            bootconfig, "androidboot.vbmeta.avb_versio", &content));
+    EXPECT_FALSE(GetBootconfigFromString(bootconfig, "androidboot.vbmeta.avb_versio", &content));
     EXPECT_EQ(content, kUnmodifiedToken) << "output parameter shouldn't be overridden";
 
     content = kUnmodifiedToken;
-    EXPECT_FALSE(
-            android::fs_mgr::GetBootconfigFromString(bootconfig, "androidboot.nospace", &content));
+    EXPECT_FALSE(GetBootconfigFromString(bootconfig, "androidboot.nospace", &content));
     EXPECT_EQ(content, kUnmodifiedToken) << "output parameter shouldn't be overridden";
 }
 
diff --git a/init/devices.cpp b/init/devices.cpp
index 6b8474e..e76786a 100644
--- a/init/devices.cpp
+++ b/init/devices.cpp
@@ -204,7 +204,7 @@
                 partition_map.emplace_back(map_pieces[0], map_pieces[1]);
             }
         };
-        ImportKernelCmdline(parser);
+        android::fs_mgr::ImportKernelCmdline(parser);
         android::fs_mgr::ImportBootconfig(parser);
         return partition_map;
     }();
diff --git a/init/init.cpp b/init/init.cpp
index da63fdc..4bb8eec 100644
--- a/init/init.cpp
+++ b/init/init.cpp
@@ -832,6 +832,12 @@
     CHECKCALL(mount("tmpfs", "/apex", "tmpfs", MS_NOEXEC | MS_NOSUID | MS_NODEV,
                     "mode=0755,uid=0,gid=0"));
 
+    if (NeedsTwoMountNamespaces()) {
+        // /bootstrap-apex is used to mount "bootstrap" APEXes.
+        CHECKCALL(mount("tmpfs", "/bootstrap-apex", "tmpfs", MS_NOEXEC | MS_NOSUID | MS_NODEV,
+                        "mode=0755,uid=0,gid=0"));
+    }
+
     // /linkerconfig is used to keep generated linker configuration
     CHECKCALL(mount("tmpfs", "/linkerconfig", "tmpfs", MS_NOEXEC | MS_NOSUID | MS_NODEV,
                     "mode=0755,uid=0,gid=0"));
diff --git a/init/mount_namespace.cpp b/init/mount_namespace.cpp
index 5b53d50..7918f23 100644
--- a/init/mount_namespace.cpp
+++ b/init/mount_namespace.cpp
@@ -66,15 +66,6 @@
     return ret;
 }
 
-// In case we have two sets of APEXes (non-updatable, updatable), we need two separate mount
-// namespaces.
-static bool NeedsTwoMountNamespaces() {
-    if (IsRecoveryMode()) return false;
-    // In microdroid, there's only one set of APEXes in built-in directories include block devices.
-    if (IsMicrodroid()) return false;
-    return true;
-}
-
 static android::base::unique_fd bootstrap_ns_fd;
 static android::base::unique_fd default_ns_fd;
 
@@ -83,6 +74,15 @@
 
 }  // namespace
 
+// In case we have two sets of APEXes (non-updatable, updatable), we need two separate mount
+// namespaces.
+bool NeedsTwoMountNamespaces() {
+    if (IsRecoveryMode()) return false;
+    // In microdroid, there's only one set of APEXes in built-in directories include block devices.
+    if (IsMicrodroid()) return false;
+    return true;
+}
+
 bool SetupMountNamespaces() {
     // Set the propagation type of / as shared so that any mounting event (e.g.
     // /data) is by default visible to all processes. When private mounting is
@@ -163,6 +163,23 @@
             PLOG(ERROR) << "Cannot switch back to bootstrap mount namespace";
             return false;
         }
+
+        // Some components (e.g. servicemanager) need to access bootstrap
+        // APEXes from the default mount namespace. To achieve that, we bind-mount
+        // /apex to /bootstrap-apex in the bootstrap mount namespace. Since /bootstrap-apex
+        // is "shared", the mounts are visible in the default mount namespace as well.
+        //
+        // The end result will look like:
+        //   in the bootstrap mount namespace:
+        //     /apex  (== /bootstrap-apex)
+        //       {bootstrap APEXes from the read-only partition}
+        //
+        //   in the default mount namespace:
+        //     /bootstrap-apex
+        //       {bootstrap APEXes from the read-only partition}
+        //     /apex
+        //       {APEXes, can be from /data partition}
+        if (!(BindMount("/bootstrap-apex", "/apex"))) return false;
     } else {
         // Otherwise, default == bootstrap
         default_ns_fd.reset(OpenMountNamespace());
diff --git a/init/mount_namespace.h b/init/mount_namespace.h
index 5e3dab2..43c5476 100644
--- a/init/mount_namespace.h
+++ b/init/mount_namespace.h
@@ -24,9 +24,12 @@
 enum MountNamespace { NS_BOOTSTRAP, NS_DEFAULT };
 
 bool SetupMountNamespaces();
+
 base::Result<void> SwitchToMountNamespaceIfNeeded(MountNamespace target_mount_namespace);
 
 base::Result<MountNamespace> GetCurrentMountNamespace();
 
+bool NeedsTwoMountNamespaces();
+
 }  // namespace init
 }  // namespace android
diff --git a/init/property_service.cpp b/init/property_service.cpp
index 3557787..f5de17d 100644
--- a/init/property_service.cpp
+++ b/init/property_service.cpp
@@ -1342,7 +1342,7 @@
 constexpr auto ANDROIDBOOT_PREFIX = "androidboot."sv;
 
 static void ProcessKernelCmdline() {
-    ImportKernelCmdline([&](const std::string& key, const std::string& value) {
+    android::fs_mgr::ImportKernelCmdline([&](const std::string& key, const std::string& value) {
         if (StartsWith(key, ANDROIDBOOT_PREFIX)) {
             InitPropertySet("ro.boot." + key.substr(ANDROIDBOOT_PREFIX.size()), value);
         }
diff --git a/init/selinux.cpp b/init/selinux.cpp
index 6fb028b..ebdcaa6 100644
--- a/init/selinux.cpp
+++ b/init/selinux.cpp
@@ -101,23 +101,14 @@
 enum EnforcingStatus { SELINUX_PERMISSIVE, SELINUX_ENFORCING };
 
 EnforcingStatus StatusFromProperty() {
-    EnforcingStatus status = SELINUX_ENFORCING;
-
-    ImportKernelCmdline([&](const std::string& key, const std::string& value) {
-        if (key == "androidboot.selinux" && value == "permissive") {
-            status = SELINUX_PERMISSIVE;
-        }
-    });
-
-    if (status == SELINUX_ENFORCING) {
-        std::string value;
-        if (android::fs_mgr::GetBootconfig("androidboot.selinux", &value) &&
-            value == "permissive") {
-            status = SELINUX_PERMISSIVE;
-        }
+    std::string value;
+    if (android::fs_mgr::GetKernelCmdline("androidboot.selinux", &value) && value == "permissive") {
+        return SELINUX_PERMISSIVE;
     }
-
-    return status;
+    if (android::fs_mgr::GetBootconfig("androidboot.selinux", &value) && value == "permissive") {
+        return SELINUX_PERMISSIVE;
+    }
+    return SELINUX_ENFORCING;
 }
 
 bool IsEnforcing() {
@@ -766,7 +757,7 @@
     selinux_android_restorecon("/dev/device-mapper", 0);
 
     selinux_android_restorecon("/apex", 0);
-
+    selinux_android_restorecon("/bootstrap-apex", 0);
     selinux_android_restorecon("/linkerconfig", 0);
 
     // adb remount, snapshot-based updates, and DSUs all create files during
diff --git a/init/util.cpp b/init/util.cpp
index b34d45d..e760a59 100644
--- a/init/util.cpp
+++ b/init/util.cpp
@@ -242,18 +242,6 @@
     return -1;
 }
 
-void ImportKernelCmdline(const std::function<void(const std::string&, const std::string&)>& fn) {
-    std::string cmdline;
-    android::base::ReadFileToString("/proc/cmdline", &cmdline);
-
-    for (const auto& entry : android::base::Split(android::base::Trim(cmdline), " ")) {
-        std::vector<std::string> pieces = android::base::Split(entry, "=");
-        if (pieces.size() == 2) {
-            fn(pieces[0], pieces[1]);
-        }
-    }
-}
-
 bool make_dir(const std::string& path, mode_t mode) {
     std::string secontext;
     if (SelabelLookupFileContext(path, mode, &secontext) && !secontext.empty()) {
diff --git a/init/util.h b/init/util.h
index 0b0ef63..2d02182 100644
--- a/init/util.h
+++ b/init/util.h
@@ -53,8 +53,7 @@
 Result<uid_t> DecodeUid(const std::string& name);
 
 bool mkdir_recursive(const std::string& pathname, mode_t mode);
-int wait_for_file(const char *filename, std::chrono::nanoseconds timeout);
-void ImportKernelCmdline(const std::function<void(const std::string&, const std::string&)>&);
+int wait_for_file(const char* filename, std::chrono::nanoseconds timeout);
 bool make_dir(const std::string& path, mode_t mode);
 bool is_dir(const char* pathname);
 Result<std::string> ExpandProps(const std::string& src);
diff --git a/libcutils/fs_config.cpp b/libcutils/fs_config.cpp
index f90a1bc..26ac576 100644
--- a/libcutils/fs_config.cpp
+++ b/libcutils/fs_config.cpp
@@ -214,6 +214,7 @@
 #endif
     { 00755, AID_ROOT,      AID_ROOT,      0, "first_stage_ramdisk/system/bin/resize2fs" },
     { 00755, AID_ROOT,      AID_ROOT,      0, "first_stage_ramdisk/system/bin/snapuserd" },
+    { 00755, AID_ROOT,      AID_ROOT,      0, "first_stage_ramdisk/system/bin/snapuserd_ramdisk" },
     { 00755, AID_ROOT,      AID_ROOT,      0, "first_stage_ramdisk/system/bin/tune2fs" },
     { 00755, AID_ROOT,      AID_ROOT,      0, "first_stage_ramdisk/system/bin/fsck.f2fs" },
     // generic defaults
diff --git a/rootdir/Android.mk b/rootdir/Android.mk
index 3362872..5218753 100644
--- a/rootdir/Android.mk
+++ b/rootdir/Android.mk
@@ -91,7 +91,7 @@
 #
 # create some directories (some are mount points) and symlinks
 LOCAL_POST_INSTALL_CMD := mkdir -p $(addprefix $(TARGET_ROOT_OUT)/, \
-    dev proc sys system data data_mirror odm oem acct config storage mnt apex debug_ramdisk \
+    dev proc sys system data data_mirror odm oem acct config storage mnt apex bootstrap-apex debug_ramdisk \
     linkerconfig second_stage_resources postinstall $(BOARD_ROOT_EXTRA_FOLDERS)); \
     ln -sf /system/bin $(TARGET_ROOT_OUT)/bin; \
     ln -sf /system/etc $(TARGET_ROOT_OUT)/etc; \
diff --git a/trusty/line-coverage/Android.bp b/trusty/line-coverage/Android.bp
new file mode 100644
index 0000000..36a73aa
--- /dev/null
+++ b/trusty/line-coverage/Android.bp
@@ -0,0 +1,36 @@
+// Copyright (C) 2023 The Android Open-Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+cc_library {
+    name: "libtrusty_line_coverage",
+    vendor_available: true,
+    srcs: [
+        "coverage.cpp",
+    ],
+    export_include_dirs: [
+        "include",
+    ],
+    shared_libs: [
+        "libbase",
+        "libext2_uuid",
+        "liblog",
+        "libdmabufheap",
+        "libtrusty",
+    ],
+}
+
diff --git a/trusty/line-coverage/coverage.cpp b/trusty/line-coverage/coverage.cpp
new file mode 100644
index 0000000..57b7025
--- /dev/null
+++ b/trusty/line-coverage/coverage.cpp
@@ -0,0 +1,191 @@
+/*
+ * Copyright (C) 2023 The Android Open Sourete Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "line-coverage"
+
+#include <BufferAllocator/BufferAllocator.h>
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/unique_fd.h>
+#include <assert.h>
+#include <log/log.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/uio.h>
+#include <trusty/line-coverage/coverage.h>
+#include <trusty/line-coverage/tipc.h>
+#include <trusty/tipc.h>
+#include <iostream>
+
+#define LINE_COVERAGE_CLIENT_PORT "com.android.trusty.linecoverage.client"
+
+struct control {
+    /* Written by controller, read by instrumented TA */
+    uint64_t        cntrl_flags;
+
+    /* Written by instrumented TA, read by controller */
+    uint64_t        oper_flags;
+    uint64_t        write_buffer_start_count;
+    uint64_t        write_buffer_complete_count;
+};
+
+namespace android {
+namespace trusty {
+namespace line_coverage {
+
+using ::android::base::ErrnoError;
+using ::android::base::Error;
+using ::std::string;
+
+static inline uintptr_t RoundPageUp(uintptr_t val) {
+    return (val + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1);
+}
+
+CoverageRecord::CoverageRecord(string tipc_dev, struct uuid* uuid)
+    : tipc_dev_(std::move(tipc_dev)),
+      coverage_srv_fd_(-1),
+      uuid_(*uuid),
+      record_len_(0),
+      shm_(NULL),
+      shm_len_(0) {}
+
+CoverageRecord::~CoverageRecord() {
+    if (shm_) {
+        munmap((void*)shm_, shm_len_);
+    }
+}
+
+volatile void *CoverageRecord::getShm() {
+    if(!IsOpen()) {
+        fprintf(stderr, "Warning! SHM is NULL!\n");
+    }
+    return shm_;
+}
+
+Result<void> CoverageRecord::Rpc(struct line_coverage_client_req* req, \
+                                  int req_fd, \
+                                  struct line_coverage_client_resp* resp) {
+    int rc;
+
+    if (req_fd < 0) {
+        rc = write(coverage_srv_fd_, req, sizeof(*req));
+    } else {
+        iovec iov = {
+                .iov_base = req,
+                .iov_len = sizeof(*req),
+        };
+
+        trusty_shm shm = {
+                .fd = req_fd,
+                .transfer = TRUSTY_SHARE,
+        };
+
+        rc = tipc_send(coverage_srv_fd_, &iov, 1, &shm, 1);
+    }
+
+    if (rc != (int)sizeof(*req)) {
+        return ErrnoError() << "failed to send request to coverage server: ";
+    }
+
+    rc = read(coverage_srv_fd_, resp, sizeof(*resp));
+    if (rc != (int)sizeof(*resp)) {
+        return ErrnoError() << "failed to read reply from coverage server: ";
+    }
+
+    if (resp->hdr.cmd != (req->hdr.cmd | LINE_COVERAGE_CLIENT_CMD_RESP_BIT)) {
+        return ErrnoError() << "unknown response cmd: " << resp->hdr.cmd;
+    }
+
+    return {};
+}
+
+Result<void> CoverageRecord::Open(int fd) {
+    struct line_coverage_client_req req;
+    struct line_coverage_client_resp resp;
+
+    if (shm_) {
+        return {}; /* already initialized */
+    }
+
+    coverage_srv_fd_= fd;
+
+    req.hdr.cmd = LINE_COVERAGE_CLIENT_CMD_OPEN;
+    req.open_args.uuid = uuid_;
+    auto ret = Rpc(&req, -1, &resp);
+    if (!ret.ok()) {
+        return Error() << "failed to open coverage client: " << ret.error();
+    }
+    record_len_ = resp.open_args.record_len;
+    shm_len_ = RoundPageUp(record_len_);
+
+    BufferAllocator allocator;
+
+    fd = allocator.Alloc("system", shm_len_);
+    if (fd < 0) {
+        return ErrnoError() << "failed to create dmabuf of size " << shm_len_
+                            << " err code: " << fd;
+    }
+    unique_fd dma_buf(fd);
+
+    void* shm = mmap(0, shm_len_, PROT_READ | PROT_WRITE, MAP_SHARED, dma_buf, 0);
+    if (shm == MAP_FAILED) {
+        return ErrnoError() << "failed to map memfd: ";
+    }
+
+    req.hdr.cmd = LINE_COVERAGE_CLIENT_CMD_SHARE_RECORD;
+    req.share_record_args.shm_len = shm_len_;
+    ret = Rpc(&req, dma_buf, &resp);
+    if (!ret.ok()) {
+        return Error() << "failed to send shared memory: " << ret.error();
+    }
+
+    shm_ = shm;
+
+    req.hdr.cmd = LINE_COVERAGE_CLIENT_CMD_OPEN;
+    req.open_args.uuid = uuid_;
+    ret = Rpc(&req, -1, &resp);
+    if (!ret.ok()) {
+        return Error() << "failed to open coverage client: " << ret.error();
+    }
+
+    return {};
+}
+
+bool CoverageRecord::IsOpen() {
+    return shm_;
+}
+
+Result<void> CoverageRecord::SaveFile(const std::string& filename) {
+    if(!IsOpen()) {
+        return ErrnoError() << "Warning! SHM is NULL!";
+    }
+    android::base::unique_fd output_fd(TEMP_FAILURE_RETRY(creat(filename.c_str(), 00644)));
+    if (!output_fd.ok()) {
+        return ErrnoError() << "Could not open output file";
+    }
+
+    uintptr_t* begin = (uintptr_t*)((char *)shm_ + sizeof(struct control));
+    bool ret = WriteFully(output_fd, begin, record_len_);
+    if(!ret) {
+        fprintf(stderr, "Coverage write to file failed\n");
+    }
+
+    return {};
+}
+
+}  // namespace line_coverage
+}  // namespace trusty
+}  // namespace android
diff --git a/trusty/line-coverage/include/trusty/line-coverage/coverage.h b/trusty/line-coverage/include/trusty/line-coverage/coverage.h
new file mode 100644
index 0000000..53901be
--- /dev/null
+++ b/trusty/line-coverage/include/trusty/line-coverage/coverage.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <optional>
+#include <string>
+
+#include <android-base/result.h>
+#include <android-base/unique_fd.h>
+#include <stdint.h>
+#include <trusty/line-coverage/tipc.h>
+
+namespace android {
+namespace trusty {
+namespace line_coverage {
+
+using android::base::Result;
+using android::base::unique_fd;
+
+class CoverageRecord {
+  public:
+    CoverageRecord(std::string tipc_dev, struct uuid* uuid);
+
+    ~CoverageRecord();
+    Result<void> Open(int fd);
+    bool IsOpen();
+    Result<void> SaveFile(const std::string& filename);
+    volatile void* getShm();
+
+  private:
+    Result<void> Rpc(struct line_coverage_client_req* req, \
+                      int req_fd, \
+                      struct line_coverage_client_resp* resp);
+
+    Result<std::pair<size_t, size_t>> GetRegionBounds(uint32_t region_type);
+
+    std::string tipc_dev_;
+    int coverage_srv_fd_;
+    struct uuid uuid_;
+    size_t record_len_;
+    volatile void* shm_;
+    size_t shm_len_;
+};
+
+}  // namespace line_coverage
+}  // namespace trusty
+}  // namespace android
diff --git a/trusty/line-coverage/include/trusty/line-coverage/tipc.h b/trusty/line-coverage/include/trusty/line-coverage/tipc.h
new file mode 100644
index 0000000..20e855c
--- /dev/null
+++ b/trusty/line-coverage/include/trusty/line-coverage/tipc.h
@@ -0,0 +1,140 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* This file needs to be kept in-sync with its counterpart on Trusty side */
+
+#pragma once
+
+#include <stdint.h>
+#include <trusty/line-coverage/uuid.h>
+
+
+#define LINE_COVERAGE_CLIENT_PORT "com.android.trusty.linecoverage.client"
+
+/**
+ * enum line_coverage_client_cmd - command identifiers for coverage client interface
+ * @LINE_COVERAGE_CLIENT_CMD_RESP_BIT:     response bit set as part of response
+ * @LINE_COVERAGE_CLIENT_CMD_SHIFT:        number of bits used by response bit
+ * @LINE_COVERAGE_CLIENT_CMD_OPEN:         command to open coverage record
+ * @LINE_COVERAGE_CLIENT_CMD_SHARE_RECORD: command to register a shared memory region
+ *                                    where coverage record will be written to
+ */
+enum line_coverage_client_cmd {
+    LINE_COVERAGE_CLIENT_CMD_RESP_BIT = 1U,
+    LINE_COVERAGE_CLIENT_CMD_SHIFT = 1U,
+    LINE_COVERAGE_CLIENT_CMD_OPEN = (1U << LINE_COVERAGE_CLIENT_CMD_SHIFT),
+    LINE_COVERAGE_CLIENT_CMD_SHARE_RECORD = (2U << LINE_COVERAGE_CLIENT_CMD_SHIFT),
+    LINE_COVERAGE_CLIENT_CMD_SEND_LIST = (3U << LINE_COVERAGE_CLIENT_CMD_SHIFT),
+};
+
+/**
+ * struct line_coverage_client_hdr - header for coverage client messages
+ * @cmd: command identifier
+ *
+ * Note that no messages return a status code. Any error on the server side
+ * results in the connection being closed. So, operations can be assumed to be
+ * successful if they return a response.
+ */
+struct line_coverage_client_hdr {
+    uint32_t cmd;
+};
+
+/**
+ * struct line_coverage_client_open_req - arguments for request to open coverage
+ *                                   record
+ * @uuid: UUID of target TA
+ *
+ * There is one coverage record per TA. @uuid is used to identify both the TA
+ * and corresponding coverage record.
+ */
+struct line_coverage_client_open_req {
+    struct uuid uuid;
+};
+
+/**
+ * struct line_coverage_client_open_resp - arguments for response to open coverage
+ *                                    record
+ * @record_len: length of coverage record that will be emitted by target TA
+ *
+ * Shared memory allocated for this coverage record must larger than
+ * @record_len.
+ */
+struct line_coverage_client_open_resp {
+    uint32_t record_len;
+};
+
+/**
+ * struct line_coverage_client_send_list_resp - arguments for response to send list
+ *                                   record
+ * @uuid: UUID of TA that connected to aggregator
+ */
+struct line_coverage_client_send_list_resp {
+    struct uuid uuid;
+};
+
+/**
+ * struct line_coverage_client_send_list_resp - arguments for response to send list
+ *                                   record
+ * @index: index of the list being requested
+ */
+struct line_coverage_client_send_list_req {
+    uint32_t index;
+};
+
+/**
+ * struct line_coverage_client_share_record_req - arguments for request to share
+ *                                           memory for coverage record
+ * @shm_len: length of memory region being shared
+ *
+ * A handle to a memory region must be sent along with this message. This memory
+ * is used to store coverage record.
+ *
+ * Upon success, this memory region can be assumed to be shared between the
+ * client and target TA.
+ */
+struct line_coverage_client_share_record_req {
+    uint32_t shm_len;
+};
+
+/**
+ * struct line_coverage_client_req - structure for a coverage client request
+ * @hdr:               message header
+ * @open_args:         arguments for %COVERAGE_CLIENT_CMD_OPEN request
+ * @share_record_args: arguments for %COVERAGE_CLIENT_CMD_SHARE_RECORD request
+ * @index: arguments for %COVERAGE_CLIENT_CMD_SHARE_RECORD request
+ */
+struct line_coverage_client_req {
+    struct line_coverage_client_hdr hdr;
+    union {
+        struct line_coverage_client_open_req open_args;
+        struct line_coverage_client_share_record_req share_record_args;
+        struct line_coverage_client_send_list_req send_list_args;
+    };
+};
+
+/**
+ * struct line_coverage_client_resp - structure for a coverage client response
+ * @hdr:       message header
+ * @open_args: arguments for %COVERAGE_CLIENT_CMD_OPEN response
+ * @send_list_args: arguments for %COVERAGE_CLIENT_CMD_SHARE_RECORD response
+ */
+struct line_coverage_client_resp {
+    struct line_coverage_client_hdr hdr;
+    union {
+        struct line_coverage_client_open_resp open_args;
+        struct line_coverage_client_send_list_resp send_list_args;
+    };
+};
diff --git a/trusty/line-coverage/include/trusty/line-coverage/uuid.h b/trusty/line-coverage/include/trusty/line-coverage/uuid.h
new file mode 100644
index 0000000..1873980
--- /dev/null
+++ b/trusty/line-coverage/include/trusty/line-coverage/uuid.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <string.h>
+
+#define UUCMP(u1, u2) if (u1 != u2) return u1 < u2
+
+struct uuid {
+    uint32_t time_low;
+    uint16_t time_mid;
+    uint16_t time_hi_and_version;
+    uint8_t clock_seq_and_node[8];
+
+    bool operator<(const struct uuid& rhs) const
+    {
+        UUCMP(time_low, rhs.time_low);
+        UUCMP(time_mid, rhs.time_mid);
+        UUCMP(time_hi_and_version, rhs.time_hi_and_version);
+        return memcmp(clock_seq_and_node, rhs.clock_seq_and_node, 8);
+    }
+};
diff --git a/trusty/utils/coverage-controller/Android.bp b/trusty/utils/coverage-controller/Android.bp
new file mode 100644
index 0000000..1aa88cc
--- /dev/null
+++ b/trusty/utils/coverage-controller/Android.bp
@@ -0,0 +1,34 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+cc_binary {
+    name: "coverage-controller",
+    vendor: true,
+
+    srcs: ["controller.cpp"],
+    shared_libs: [
+        "libc",
+        "liblog",
+        "libbase",
+        "libdmabufheap",
+    ],
+    static_libs: [
+        "libtrusty",
+        "libtrusty_line_coverage",
+    ],
+}
diff --git a/trusty/utils/coverage-controller/controller.cpp b/trusty/utils/coverage-controller/controller.cpp
new file mode 100644
index 0000000..730c010
--- /dev/null
+++ b/trusty/utils/coverage-controller/controller.cpp
@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <getopt.h>
+#include <trusty/line-coverage/coverage.h>
+#include <trusty/tipc.h>
+#include <array>
+#include <memory>
+#include <vector>
+
+#include "controller.h"
+
+#define READ_ONCE(x) (*((volatile __typeof__(x) *) &(x)))
+#define WRITE_ONCE(x, val) (*((volatile __typeof__(val) *) &(x)) = (val))
+
+namespace android {
+namespace trusty {
+namespace controller {
+
+using ::android::trusty::line_coverage::CoverageRecord;
+
+void Controller::run(std::string output_dir) {
+    connectCoverageServer();
+    struct control *control;
+    uint64_t complete_cnt = 0, start_cnt = 0, flags;
+
+    while(1) {
+        setUpShm();
+
+        for (int index = 0; index < record_list_.size(); index++) {
+            control = (struct control *)record_list_[index]->getShm();
+            start_cnt = READ_ONCE((control->write_buffer_start_count));
+            complete_cnt = READ_ONCE(control->write_buffer_complete_count);
+            flags = READ_ONCE(control->cntrl_flags);
+
+            if (complete_cnt != counters[index] && start_cnt == complete_cnt) {
+                WRITE_ONCE(control->cntrl_flags, FLAG_NONE);
+                std::string fmt = "/%d.%lu.profraw";
+                int sz = std::snprintf(nullptr, 0, fmt.c_str(), index, counters[index]);
+                std::string filename(sz+1, '.');
+                std::sprintf(filename.data(), fmt.c_str(), index, counters[index]);
+                filename.insert(0, output_dir);
+                android::base::Result<void> res = record_list_[index]->SaveFile(filename);
+                counters[index]++;
+            }
+            if(complete_cnt == counters[index] &&
+                !(flags & FLAG_RUN)) {
+                flags |= FLAG_RUN;
+                WRITE_ONCE(control->cntrl_flags, flags);
+            }
+        }
+    }
+}
+
+void Controller::connectCoverageServer() {
+    coverage_srv_fd = tipc_connect(TIPC_DEV, LINE_COVERAGE_CLIENT_PORT);
+    if (coverage_srv_fd < 0) {
+        fprintf(stderr, \
+                "Error: Failed to connect to Trusty coverage server: %d\n", coverage_srv_fd);
+        return;
+    }
+}
+
+void Controller::setUpShm() {
+    struct line_coverage_client_req req;
+    struct line_coverage_client_resp resp;
+    uint32_t cur_index = record_list_.size();
+    struct uuid zero_uuid = {0, 0, 0, { 0 }};
+    req.hdr.cmd = LINE_COVERAGE_CLIENT_CMD_SEND_LIST;
+    int rc = write(coverage_srv_fd, &req, sizeof(req));
+        if (rc != (int)sizeof(req)) {
+            fprintf(stderr, "failed to send request to coverage server: %d\n", rc);
+            return;
+    }
+
+    while(1) {
+        rc = read(coverage_srv_fd, &resp, sizeof(resp));
+        if (rc != (int)sizeof(resp)) {
+            fprintf(stderr, "failed to read reply from coverage server:: %d\n", rc);
+        }
+
+        if (resp.hdr.cmd == (req.hdr.cmd | LINE_COVERAGE_CLIENT_CMD_RESP_BIT)) {
+            if (!memcmp(&resp.send_list_args.uuid, &zero_uuid, sizeof(struct uuid))) {
+                break;
+            }
+            if(uuid_set_.find(resp.send_list_args.uuid) == uuid_set_.end()) {
+                uuid_set_.insert(resp.send_list_args.uuid);
+                record_list_.push_back(std::make_unique<CoverageRecord>(TIPC_DEV,
+                                                                    &resp.send_list_args.uuid));
+                counters.push_back(0);
+            }
+        }
+        else {
+            fprintf(stderr, "Unknown response header\n");
+        }
+        cur_index++;
+        req.hdr.cmd = LINE_COVERAGE_CLIENT_CMD_SEND_LIST;
+        req.send_list_args.index = cur_index;
+        int rc = write(coverage_srv_fd, &req, sizeof(req));
+        if (rc != (int)sizeof(req)) {
+            fprintf(stderr, "failed to send request to coverage server: %d\n", rc);
+        }
+    }
+
+    for(int ind = 0 ; ind < record_list_.size() ; ind++) {
+        record_list_[ind]->Open(coverage_srv_fd);
+    }
+}
+
+
+}  // namespace controller
+}  // namespace trusty
+}  // namespace android
+
+int main(int argc, char* argv[]) {
+
+    std::string optarg = "";
+    do {
+        int c;
+        c = getopt(argc, argv, "o");
+
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+        case 'o':
+            break;
+        default:
+            fprintf(stderr, "usage: %s -o [output_directory]\n", argv[0]);
+            exit(EXIT_FAILURE);
+        }
+    } while (1);
+
+    if (argc > optind + 1) {
+        fprintf(stderr, "%s: too many arguments\n", argv[0]);
+        exit(EXIT_FAILURE);
+    }
+
+    if (argc > optind) {
+        optarg = argv[optind];
+    }
+    if (optarg.size()==0) {
+        optarg = "data/local/tmp";
+    }
+
+    android::trusty::controller::Controller cur;
+    cur.run(optarg);
+
+    return EXIT_SUCCESS;
+}
\ No newline at end of file
diff --git a/trusty/utils/coverage-controller/controller.h b/trusty/utils/coverage-controller/controller.h
new file mode 100644
index 0000000..b771c16
--- /dev/null
+++ b/trusty/utils/coverage-controller/controller.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <trusty/line-coverage/coverage.h>
+#include <array>
+#include <memory>
+#include <vector>
+#include <set>
+
+#define TIPC_DEV "/dev/trusty-ipc-dev0"
+#define TEST_SRV_PORT "com.android.trusty.sancov.test.srv"
+#define TEST_SRV_MODULE "srv.syms.elf"
+
+#define FLAG_NONE               0x0
+#define FLAG_RUN                0x1
+#define FLAG_TOGGLE_CLEAR       0x2
+
+struct control {
+    /* Written by controller, read by instrumented TA */
+    uint64_t        cntrl_flags;
+
+    /* Written by instrumented TA, read by controller */
+    uint64_t        oper_flags;
+    uint64_t        write_buffer_start_count;
+    uint64_t        write_buffer_complete_count;
+};
+
+namespace android {
+namespace trusty {
+namespace controller {
+
+class Controller {
+  public:
+    public:
+        void run(std::string output_dir);
+
+    private:
+        std::vector<std::unique_ptr<line_coverage::CoverageRecord>>record_list_;
+        std::set<struct uuid>uuid_set_;
+        std::vector<uint64_t> counters;
+        int coverage_srv_fd;
+
+        void connectCoverageServer();
+        void setUpShm();
+};
+
+}  // namespace controller
+}  // namespace trusty
+}  // namespace android