Merge changes Ib33dc593,I6550682f,Ic0681cbf,Iaaf96a37,I89b15492 into main
* changes:
snapuserd: Factor a base class out of tests.
snapuserd: Add error propagation to tests.
snapuserd: Add diagnostics for debugging races.
snapuserd: Fix race condition in HandleManager shutdown.
snapuserd: Remove test dependence on LoopDevice.
diff --git a/fs_mgr/libsnapshot/snapuserd/Android.bp b/fs_mgr/libsnapshot/snapuserd/Android.bp
index 6251e5a..fc260b6 100644
--- a/fs_mgr/libsnapshot/snapuserd/Android.bp
+++ b/fs_mgr/libsnapshot/snapuserd/Android.bp
@@ -221,6 +221,7 @@
],
srcs: [
"testing/dm_user_harness.cpp",
+ "testing/harness.cpp",
"user-space-merge/snapuserd_test.cpp",
],
shared_libs: [
diff --git a/fs_mgr/libsnapshot/snapuserd/testing/harness.cpp b/fs_mgr/libsnapshot/snapuserd/testing/harness.cpp
new file mode 100644
index 0000000..02ae549
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/testing/harness.cpp
@@ -0,0 +1,116 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "harness.h"
+
+#ifdef __ANDROID__
+#include <linux/memfd.h>
+#endif
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include <android-base/file.h>
+#include <ext4_utils/ext4_utils.h>
+#include <libdm/loop_control.h>
+#include "snapuserd_logging.h"
+
+namespace android {
+namespace snapshot {
+
+using namespace std::chrono_literals;
+using android::base::unique_fd;
+using android::dm::LoopDevice;
+
+#ifdef __ANDROID__
+// Prefer this on device since it is a real block device, which is more similar
+// to how we use snapuserd.
+class MemoryBackedDevice final : public IBackingDevice {
+ public:
+ bool Init(uint64_t size) {
+ memfd_.reset(memfd_create("snapuserd_test", MFD_ALLOW_SEALING));
+ if (memfd_ < 0) {
+ PLOG(ERROR) << "memfd_create failed";
+ return false;
+ }
+ if (ftruncate(memfd_.get(), size) < 0) {
+ PLOG(ERROR) << "ftruncate failed";
+ return false;
+ }
+ if (fcntl(memfd_.get(), F_ADD_SEALS, F_SEAL_GROW | F_SEAL_SHRINK) < 0) {
+ PLOG(ERROR) << "fcntl seal failed";
+ return false;
+ }
+ dev_ = std::make_unique<LoopDevice>(memfd_, 10s);
+ return dev_->valid();
+ }
+ const std::string& GetPath() override { return dev_->device(); }
+ uint64_t GetSize() override {
+ unique_fd fd(open(GetPath().c_str(), O_RDONLY | O_CLOEXEC));
+ if (fd < 0) {
+ PLOG(ERROR) << "open failed: " << GetPath();
+ return 0;
+ }
+ return get_block_device_size(fd.get());
+ }
+
+ private:
+ unique_fd memfd_;
+ std::unique_ptr<LoopDevice> dev_;
+};
+#endif
+
+class FileBackedDevice final : public IBackingDevice {
+ public:
+ bool Init(uint64_t size) {
+ if (temp_.fd < 0) {
+ return false;
+ }
+ if (ftruncate(temp_.fd, size) < 0) {
+ PLOG(ERROR) << "ftruncate failed: " << temp_.path;
+ return false;
+ }
+ path_ = temp_.path;
+ return true;
+ }
+
+ const std::string& GetPath() override { return path_; }
+ uint64_t GetSize() override {
+ off_t off = lseek(temp_.fd, 0, SEEK_END);
+ if (off < 0) {
+ PLOG(ERROR) << "lseek failed: " << temp_.path;
+ return 0;
+ }
+ return off;
+ }
+
+ private:
+ TemporaryFile temp_;
+ std::string path_;
+};
+
+std::unique_ptr<IBackingDevice> ITestHarness::CreateBackingDevice(uint64_t size) {
+#ifdef __ANDROID__
+ auto dev = std::make_unique<MemoryBackedDevice>();
+#else
+ auto dev = std::make_unique<FileBackedDevice>();
+#endif
+ if (!dev->Init(size)) {
+ return nullptr;
+ }
+ return dev;
+}
+
+} // namespace snapshot
+} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/testing/harness.h b/fs_mgr/libsnapshot/snapuserd/testing/harness.h
index c0d528f..ffe9f7b 100644
--- a/fs_mgr/libsnapshot/snapuserd/testing/harness.h
+++ b/fs_mgr/libsnapshot/snapuserd/testing/harness.h
@@ -33,6 +33,14 @@
virtual bool Destroy() = 0;
};
+// Interface for an fd/temp file that is a block device when possible.
+class IBackingDevice {
+ public:
+ virtual ~IBackingDevice() {}
+ virtual const std::string& GetPath() = 0;
+ virtual uint64_t GetSize() = 0;
+};
+
class ITestHarness {
public:
virtual ~ITestHarness() {}
@@ -41,6 +49,7 @@
uint64_t num_sectors) = 0;
virtual IBlockServerFactory* GetBlockServerFactory() = 0;
virtual bool HasUserDevice() = 0;
+ virtual std::unique_ptr<IBackingDevice> CreateBackingDevice(uint64_t size);
};
} // namespace snapshot
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp
index fd41cd4..d979e20 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp
@@ -14,6 +14,7 @@
#include "handler_manager.h"
+#include <pthread.h>
#include <sys/eventfd.h>
#include <android-base/logging.h>
@@ -132,6 +133,8 @@
void SnapshotHandlerManager::RunThread(std::shared_ptr<HandlerThread> handler) {
LOG(INFO) << "Entering thread for handler: " << handler->misc_name();
+ pthread_setname_np(pthread_self(), "Handler");
+
if (!handler->snapuserd()->Start()) {
LOG(ERROR) << " Failed to launch all worker threads";
}
@@ -201,9 +204,8 @@
handler->snapuserd()->MonitorMerge();
- if (!is_merge_monitor_started_) {
- std::thread(&SnapshotHandlerManager::MonitorMerge, this).detach();
- is_merge_monitor_started_ = true;
+ if (!merge_monitor_.joinable()) {
+ merge_monitor_ = std::thread(&SnapshotHandlerManager::MonitorMerge, this);
}
merge_handlers_.push(handler);
@@ -220,6 +222,7 @@
}
void SnapshotHandlerManager::MonitorMerge() {
+ pthread_setname_np(pthread_self(), "Merge Monitor");
while (!stop_monitor_merge_thread_) {
uint64_t testVal;
ssize_t ret =
@@ -357,8 +360,12 @@
if (th.joinable()) th.join();
}
- stop_monitor_merge_thread_ = true;
- WakeupMonitorMergeThread();
+ if (merge_monitor_.joinable()) {
+ stop_monitor_merge_thread_ = true;
+ WakeupMonitorMergeThread();
+
+ merge_monitor_.join();
+ }
}
auto SnapshotHandlerManager::FindHandler(std::lock_guard<std::mutex>* proof_of_lock,
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.h
index 2a6da96..b1605f0 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.h
@@ -122,9 +122,9 @@
std::mutex lock_;
HandlerList dm_users_;
- bool is_merge_monitor_started_ = false;
bool stop_monitor_merge_thread_ = false;
int active_merge_threads_ = 0;
+ std::thread merge_monitor_;
int num_partitions_merge_complete_ = 0;
std::queue<std::shared_ptr<HandlerThread>> merge_handlers_;
android::base::unique_fd monitor_merge_event_fd_;
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/merge_worker.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/merge_worker.cpp
index 2d69721..11b8d7c 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/merge_worker.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/merge_worker.cpp
@@ -15,6 +15,8 @@
*/
#include "merge_worker.h"
+#include <pthread.h>
+
#include "snapuserd_core.h"
#include "utility.h"
@@ -198,6 +200,7 @@
// Wait for RA thread to notify that the merge window
// is ready for merging.
if (!snapuserd_->WaitForMergeBegin()) {
+ SNAP_LOG(ERROR) << "Failed waiting for merge to begin";
return false;
}
@@ -303,7 +306,7 @@
// will fallback to synchronous I/O.
ret = io_uring_wait_cqe(ring_.get(), &cqe);
if (ret) {
- SNAP_LOG(ERROR) << "Merge: io_uring_wait_cqe failed: " << ret;
+ SNAP_LOG(ERROR) << "Merge: io_uring_wait_cqe failed: " << strerror(-ret);
status = false;
break;
}
@@ -546,6 +549,9 @@
bool MergeWorker::Run() {
SNAP_LOG(DEBUG) << "Waiting for merge begin...";
+
+ pthread_setname_np(pthread_self(), "MergeWorker");
+
if (!snapuserd_->WaitForMergeBegin()) {
SNAP_LOG(ERROR) << "Merge terminated early...";
return true;
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
index e2c292b..b9ecfa5 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
@@ -16,6 +16,8 @@
#include "read_worker.h"
+#include <pthread.h>
+
#include "snapuserd_core.h"
#include "utility.h"
@@ -209,6 +211,8 @@
bool ReadWorker::Run() {
SNAP_LOG(INFO) << "Processing snapshot I/O requests....";
+ pthread_setname_np(pthread_self(), "ReadWorker");
+
if (!SetThreadPriority(kNiceValueForMergeThreads)) {
SNAP_PLOG(ERROR) << "Failed to set thread priority";
}
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
index 5a82ca1..c295851 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
@@ -295,8 +295,6 @@
if (ra_thread_) {
ra_thread_status =
std::async(std::launch::async, &ReadAhead::RunThread, read_ahead_thread_.get());
-
- SNAP_LOG(INFO) << "Read-ahead thread started";
}
// Launch worker threads
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
index a45e0bc..622fc50 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
@@ -28,6 +28,7 @@
#include <iostream>
#include <limits>
#include <mutex>
+#include <ostream>
#include <string>
#include <thread>
#include <unordered_map>
@@ -68,12 +69,13 @@
#define SNAP_PLOG(level) PLOG(level) << misc_name_ << ": "
enum class MERGE_IO_TRANSITION {
+ INVALID,
MERGE_READY,
MERGE_BEGIN,
MERGE_FAILED,
MERGE_COMPLETE,
IO_TERMINATED,
- READ_AHEAD_FAILURE,
+ READ_AHEAD_FAILURE
};
class MergeWorker;
@@ -220,7 +222,7 @@
bool populate_data_from_cow_ = false;
bool ra_thread_ = false;
int total_ra_blocks_merged_ = 0;
- MERGE_IO_TRANSITION io_state_;
+ MERGE_IO_TRANSITION io_state_ = MERGE_IO_TRANSITION::INVALID;
std::unique_ptr<ReadAhead> read_ahead_thread_;
std::unordered_map<uint64_t, void*> read_ahead_buffer_map_;
@@ -246,5 +248,7 @@
std::shared_ptr<IBlockServerOpener> block_server_opener_;
};
+std::ostream& operator<<(std::ostream& os, MERGE_IO_TRANSITION value);
+
} // namespace snapshot
} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
index 8b799ea..d2128c5 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
@@ -16,6 +16,8 @@
#include "snapuserd_readahead.h"
+#include <pthread.h>
+
#include "snapuserd_core.h"
#include "utility.h"
@@ -428,7 +430,7 @@
// will fallback to synchronous I/O.
int ret = io_uring_wait_cqe(ring_.get(), &cqe);
if (ret) {
- SNAP_LOG(ERROR) << "Read-ahead - io_uring_wait_cqe failed: " << ret;
+ SNAP_LOG(ERROR) << "Read-ahead - io_uring_wait_cqe failed: " << strerror(-ret);
status = false;
break;
}
@@ -691,6 +693,7 @@
// window. If there is a crash during this time frame, merge should resume
// based on the contents of the scratch space.
if (!snapuserd_->WaitForMergeReady()) {
+ SNAP_LOG(ERROR) << "ReadAhead failed to wait for merge ready";
return false;
}
@@ -752,6 +755,10 @@
}
bool ReadAhead::RunThread() {
+ SNAP_LOG(INFO) << "ReadAhead thread started.";
+
+ pthread_setname_np(pthread_self(), "ReadAhead");
+
if (!InitializeFds()) {
return false;
}
@@ -770,6 +777,7 @@
SNAP_PLOG(ERROR) << "Failed to set thread priority";
}
+ SNAP_LOG(INFO) << "ReadAhead processing.";
while (!RAIterDone()) {
if (!ReadAheadIOStart()) {
break;
@@ -780,7 +788,7 @@
CloseFds();
reader_->CloseCowFd();
- SNAP_LOG(INFO) << " ReadAhead thread terminating....";
+ SNAP_LOG(INFO) << " ReadAhead thread terminating.";
return true;
}
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
index c407834..9f7a91d 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
@@ -17,7 +17,6 @@
#include <fcntl.h>
#include <linux/fs.h>
-#include <linux/memfd.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/syscall.h>
@@ -57,146 +56,46 @@
using namespace android::dm;
using namespace std;
-class SnapuserdTest : public ::testing::Test {
- public:
- bool SetupDefault();
- bool SetupOrderedOps();
- bool SetupOrderedOpsInverted();
- bool SetupCopyOverlap_1();
- bool SetupCopyOverlap_2();
- bool Merge();
- void ValidateMerge();
- void ReadSnapshotDeviceAndValidate();
- void Shutdown();
- void MergeInterrupt();
- void MergeInterruptFixed(int duration);
- void MergeInterruptRandomly(int max_duration);
- void StartMerge();
- void CheckMergeCompletion();
-
- static const uint64_t kSectorSize = 512;
-
+class SnapuserdTestBase : public ::testing::Test {
protected:
void SetUp() override;
- void TearDown() override { Shutdown(); }
-
- void SetupImpl();
-
- void SimulateDaemonRestart();
-
- std::unique_ptr<ICowWriter> CreateCowDeviceInternal();
- void CreateCowDevice();
- void CreateCowDeviceOrderedOps();
- void CreateCowDeviceOrderedOpsInverted();
- void CreateCowDeviceWithCopyOverlap_1();
- void CreateCowDeviceWithCopyOverlap_2();
- bool SetupDaemon();
+ void TearDown() override;
void CreateBaseDevice();
- void InitCowDevice();
+ void CreateCowDevice();
void SetDeviceControlName();
- void InitDaemon();
- void CreateUserDevice();
+ std::unique_ptr<ICowWriter> CreateCowDeviceInternal();
- unique_ptr<LoopDevice> base_loop_;
- unique_ptr<IUserDevice> dmuser_dev_;
-
+ std::unique_ptr<ITestHarness> harness_;
+ size_t size_ = 100_MiB;
+ int total_base_size_ = 0;
std::string system_device_ctrl_name_;
std::string system_device_name_;
+ unique_ptr<IBackingDevice> base_dev_;
unique_fd base_fd_;
+
std::unique_ptr<TemporaryFile> cow_system_;
+
std::unique_ptr<uint8_t[]> orig_buffer_;
- std::unique_ptr<uint8_t[]> merged_buffer_;
- SnapshotHandlerManager handlers_;
- bool setup_ok_ = false;
- bool merge_ok_ = false;
- size_t size_ = 100_MiB;
- int cow_num_sectors_;
- int total_base_size_;
- std::unique_ptr<ITestHarness> harness_;
};
-static unique_fd CreateTempFile(const std::string& name, size_t size) {
- unique_fd fd(syscall(__NR_memfd_create, name.c_str(), MFD_ALLOW_SEALING));
- if (fd < 0) {
- return {};
- }
- if (size) {
- if (ftruncate(fd, size) < 0) {
- perror("ftruncate");
- return {};
- }
- if (fcntl(fd, F_ADD_SEALS, F_SEAL_GROW | F_SEAL_SHRINK) < 0) {
- perror("fcntl");
- return {};
- }
- }
- return fd;
-}
-
-void SnapuserdTest::SetUp() {
+void SnapuserdTestBase::SetUp() {
harness_ = std::make_unique<DmUserTestHarness>();
}
-void SnapuserdTest::Shutdown() {
- ASSERT_TRUE(dmuser_dev_->Destroy());
+void SnapuserdTestBase::TearDown() {}
- auto misc_device = "/dev/dm-user/" + system_device_ctrl_name_;
- ASSERT_TRUE(handlers_.DeleteHandler(system_device_ctrl_name_));
- ASSERT_TRUE(android::fs_mgr::WaitForFileDeleted(misc_device, 10s));
- handlers_.TerminateMergeThreads();
-}
-
-bool SnapuserdTest::SetupDefault() {
- SetupImpl();
- return setup_ok_;
-}
-
-bool SnapuserdTest::SetupOrderedOps() {
- CreateBaseDevice();
- CreateCowDeviceOrderedOps();
- return SetupDaemon();
-}
-
-bool SnapuserdTest::SetupOrderedOpsInverted() {
- CreateBaseDevice();
- CreateCowDeviceOrderedOpsInverted();
- return SetupDaemon();
-}
-
-bool SnapuserdTest::SetupCopyOverlap_1() {
- CreateBaseDevice();
- CreateCowDeviceWithCopyOverlap_1();
- return SetupDaemon();
-}
-
-bool SnapuserdTest::SetupCopyOverlap_2() {
- CreateBaseDevice();
- CreateCowDeviceWithCopyOverlap_2();
- return SetupDaemon();
-}
-
-bool SnapuserdTest::SetupDaemon() {
- SetDeviceControlName();
-
- CreateUserDevice();
- InitCowDevice();
- InitDaemon();
-
- setup_ok_ = true;
-
- return setup_ok_;
-}
-
-void SnapuserdTest::CreateBaseDevice() {
- unique_fd rnd_fd;
-
+void SnapuserdTestBase::CreateBaseDevice() {
total_base_size_ = (size_ * 5);
- base_fd_ = CreateTempFile("base_device", total_base_size_);
+
+ base_dev_ = harness_->CreateBackingDevice(total_base_size_);
+ ASSERT_NE(base_dev_, nullptr);
+
+ base_fd_.reset(open(base_dev_->GetPath().c_str(), O_RDWR | O_CLOEXEC));
ASSERT_GE(base_fd_, 0);
- rnd_fd.reset(open("/dev/random", O_RDONLY));
- ASSERT_TRUE(rnd_fd > 0);
+ unique_fd rnd_fd(open("/dev/random", O_RDONLY));
+ ASSERT_GE(rnd_fd, 0);
std::unique_ptr<uint8_t[]> random_buffer = std::make_unique<uint8_t[]>(1_MiB);
@@ -206,9 +105,215 @@
}
ASSERT_EQ(lseek(base_fd_, 0, SEEK_SET), 0);
+}
- base_loop_ = std::make_unique<LoopDevice>(base_fd_, 10s);
- ASSERT_TRUE(base_loop_->valid());
+std::unique_ptr<ICowWriter> SnapuserdTestBase::CreateCowDeviceInternal() {
+ std::string path = android::base::GetExecutableDirectory();
+ cow_system_ = std::make_unique<TemporaryFile>(path);
+
+ CowOptions options;
+ options.compression = "gz";
+
+ unique_fd fd(cow_system_->fd);
+ cow_system_->fd = -1;
+
+ return CreateCowWriter(kDefaultCowVersion, options, std::move(fd));
+}
+
+void SnapuserdTestBase::CreateCowDevice() {
+ unique_fd rnd_fd;
+ loff_t offset = 0;
+
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
+
+ rnd_fd.reset(open("/dev/random", O_RDONLY));
+ ASSERT_TRUE(rnd_fd > 0);
+
+ std::unique_ptr<uint8_t[]> random_buffer_1_ = std::make_unique<uint8_t[]>(size_);
+
+ // Fill random data
+ for (size_t j = 0; j < (size_ / 1_MiB); j++) {
+ ASSERT_EQ(ReadFullyAtOffset(rnd_fd, (char*)random_buffer_1_.get() + offset, 1_MiB, 0),
+ true);
+
+ offset += 1_MiB;
+ }
+
+ size_t num_blocks = size_ / writer->GetBlockSize();
+ size_t blk_end_copy = num_blocks * 2;
+ size_t source_blk = num_blocks - 1;
+ size_t blk_src_copy = blk_end_copy - 1;
+
+ uint32_t sequence[num_blocks * 2];
+ // Sequence for Copy ops
+ for (int i = 0; i < num_blocks; i++) {
+ sequence[i] = num_blocks - 1 - i;
+ }
+ // Sequence for Xor ops
+ for (int i = 0; i < num_blocks; i++) {
+ sequence[num_blocks + i] = 5 * num_blocks - 1 - i;
+ }
+ ASSERT_TRUE(writer->AddSequenceData(2 * num_blocks, sequence));
+
+ size_t x = num_blocks;
+ while (1) {
+ ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy));
+ x -= 1;
+ if (x == 0) {
+ break;
+ }
+ source_blk -= 1;
+ blk_src_copy -= 1;
+ }
+
+ source_blk = num_blocks;
+ blk_src_copy = blk_end_copy;
+
+ ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_));
+
+ size_t blk_zero_copy_start = source_blk + num_blocks;
+ size_t blk_zero_copy_end = blk_zero_copy_start + num_blocks;
+
+ ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks));
+
+ size_t blk_random2_replace_start = blk_zero_copy_end;
+
+ ASSERT_TRUE(writer->AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_));
+
+ size_t blk_xor_start = blk_random2_replace_start + num_blocks;
+ size_t xor_offset = BLOCK_SZ / 2;
+ ASSERT_TRUE(writer->AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks,
+ xor_offset));
+
+ // Flush operations
+ ASSERT_TRUE(writer->Finalize());
+ // Construct the buffer required for validation
+ orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
+ std::string zero_buffer(size_, 0);
+ ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), size_, size_), true);
+ memcpy((char*)orig_buffer_.get() + size_, random_buffer_1_.get(), size_);
+ memcpy((char*)orig_buffer_.get() + (size_ * 2), (void*)zero_buffer.c_str(), size_);
+ memcpy((char*)orig_buffer_.get() + (size_ * 3), random_buffer_1_.get(), size_);
+ ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, &orig_buffer_.get()[size_ * 4], size_,
+ size_ + xor_offset),
+ true);
+ for (int i = 0; i < size_; i++) {
+ orig_buffer_.get()[(size_ * 4) + i] =
+ (uint8_t)(orig_buffer_.get()[(size_ * 4) + i] ^ random_buffer_1_.get()[i]);
+ }
+}
+
+void SnapuserdTestBase::SetDeviceControlName() {
+ system_device_name_.clear();
+ system_device_ctrl_name_.clear();
+
+ std::string str(cow_system_->path);
+ std::size_t found = str.find_last_of("/\\");
+ ASSERT_NE(found, std::string::npos);
+ system_device_name_ = str.substr(found + 1);
+
+ system_device_ctrl_name_ = system_device_name_ + "-ctrl";
+}
+
+class SnapuserdTest : public SnapuserdTestBase {
+ public:
+ void SetupDefault();
+ void SetupOrderedOps();
+ void SetupOrderedOpsInverted();
+ void SetupCopyOverlap_1();
+ void SetupCopyOverlap_2();
+ bool Merge();
+ void ValidateMerge();
+ void ReadSnapshotDeviceAndValidate();
+ void Shutdown();
+ void MergeInterrupt();
+ void MergeInterruptFixed(int duration);
+ void MergeInterruptRandomly(int max_duration);
+ bool StartMerge();
+ void CheckMergeCompletion();
+
+ static const uint64_t kSectorSize = 512;
+
+ protected:
+ void SetUp() override;
+ void TearDown() override;
+
+ void SetupImpl();
+
+ void SimulateDaemonRestart();
+
+ void CreateCowDeviceOrderedOps();
+ void CreateCowDeviceOrderedOpsInverted();
+ void CreateCowDeviceWithCopyOverlap_1();
+ void CreateCowDeviceWithCopyOverlap_2();
+ void SetupDaemon();
+ void InitCowDevice();
+ void InitDaemon();
+ void CreateUserDevice();
+
+ unique_ptr<IUserDevice> dmuser_dev_;
+
+ std::unique_ptr<uint8_t[]> merged_buffer_;
+ std::unique_ptr<SnapshotHandlerManager> handlers_;
+ int cow_num_sectors_;
+};
+
+void SnapuserdTest::SetUp() {
+ ASSERT_NO_FATAL_FAILURE(SnapuserdTestBase::SetUp());
+ handlers_ = std::make_unique<SnapshotHandlerManager>();
+}
+
+void SnapuserdTest::TearDown() {
+ SnapuserdTestBase::TearDown();
+ Shutdown();
+}
+
+void SnapuserdTest::Shutdown() {
+ ASSERT_TRUE(dmuser_dev_->Destroy());
+
+ auto misc_device = "/dev/dm-user/" + system_device_ctrl_name_;
+ ASSERT_TRUE(handlers_->DeleteHandler(system_device_ctrl_name_));
+ ASSERT_TRUE(android::fs_mgr::WaitForFileDeleted(misc_device, 10s));
+ handlers_->TerminateMergeThreads();
+ handlers_->JoinAllThreads();
+ handlers_ = std::make_unique<SnapshotHandlerManager>();
+}
+
+void SnapuserdTest::SetupDefault() {
+ ASSERT_NO_FATAL_FAILURE(SetupImpl());
+}
+
+void SnapuserdTest::SetupOrderedOps() {
+ ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+ ASSERT_NO_FATAL_FAILURE(CreateCowDeviceOrderedOps());
+ ASSERT_NO_FATAL_FAILURE(SetupDaemon());
+}
+
+void SnapuserdTest::SetupOrderedOpsInverted() {
+ ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+ ASSERT_NO_FATAL_FAILURE(CreateCowDeviceOrderedOpsInverted());
+ ASSERT_NO_FATAL_FAILURE(SetupDaemon());
+}
+
+void SnapuserdTest::SetupCopyOverlap_1() {
+ ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+ ASSERT_NO_FATAL_FAILURE(CreateCowDeviceWithCopyOverlap_1());
+ ASSERT_NO_FATAL_FAILURE(SetupDaemon());
+}
+
+void SnapuserdTest::SetupCopyOverlap_2() {
+ ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+ ASSERT_NO_FATAL_FAILURE(CreateCowDeviceWithCopyOverlap_2());
+ ASSERT_NO_FATAL_FAILURE(SetupDaemon());
+}
+
+void SnapuserdTest::SetupDaemon() {
+ SetDeviceControlName();
+
+ ASSERT_NO_FATAL_FAILURE(CreateUserDevice());
+ ASSERT_NO_FATAL_FAILURE(InitCowDevice());
+ ASSERT_NO_FATAL_FAILURE(InitDaemon());
}
void SnapuserdTest::ReadSnapshotDeviceAndValidate() {
@@ -242,19 +347,6 @@
ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 4), size_), 0);
}
-std::unique_ptr<ICowWriter> SnapuserdTest::CreateCowDeviceInternal() {
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
-
- CowOptions options;
- options.compression = "gz";
-
- unique_fd fd(cow_system_->fd);
- cow_system_->fd = -1;
-
- return CreateCowWriter(kDefaultCowVersion, options, std::move(fd));
-}
-
void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() {
auto writer = CreateCowDeviceInternal();
ASSERT_NE(writer, nullptr);
@@ -451,90 +543,6 @@
}
}
-void SnapuserdTest::CreateCowDevice() {
- unique_fd rnd_fd;
- loff_t offset = 0;
-
- auto writer = CreateCowDeviceInternal();
- ASSERT_NE(writer, nullptr);
-
- rnd_fd.reset(open("/dev/random", O_RDONLY));
- ASSERT_TRUE(rnd_fd > 0);
-
- std::unique_ptr<uint8_t[]> random_buffer_1_ = std::make_unique<uint8_t[]>(size_);
-
- // Fill random data
- for (size_t j = 0; j < (size_ / 1_MiB); j++) {
- ASSERT_EQ(ReadFullyAtOffset(rnd_fd, (char*)random_buffer_1_.get() + offset, 1_MiB, 0),
- true);
-
- offset += 1_MiB;
- }
-
- size_t num_blocks = size_ / writer->GetBlockSize();
- size_t blk_end_copy = num_blocks * 2;
- size_t source_blk = num_blocks - 1;
- size_t blk_src_copy = blk_end_copy - 1;
-
- uint32_t sequence[num_blocks * 2];
- // Sequence for Copy ops
- for (int i = 0; i < num_blocks; i++) {
- sequence[i] = num_blocks - 1 - i;
- }
- // Sequence for Xor ops
- for (int i = 0; i < num_blocks; i++) {
- sequence[num_blocks + i] = 5 * num_blocks - 1 - i;
- }
- ASSERT_TRUE(writer->AddSequenceData(2 * num_blocks, sequence));
-
- size_t x = num_blocks;
- while (1) {
- ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy));
- x -= 1;
- if (x == 0) {
- break;
- }
- source_blk -= 1;
- blk_src_copy -= 1;
- }
-
- source_blk = num_blocks;
- blk_src_copy = blk_end_copy;
-
- ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_));
-
- size_t blk_zero_copy_start = source_blk + num_blocks;
- size_t blk_zero_copy_end = blk_zero_copy_start + num_blocks;
-
- ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks));
-
- size_t blk_random2_replace_start = blk_zero_copy_end;
-
- ASSERT_TRUE(writer->AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_));
-
- size_t blk_xor_start = blk_random2_replace_start + num_blocks;
- size_t xor_offset = BLOCK_SZ / 2;
- ASSERT_TRUE(writer->AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks,
- xor_offset));
-
- // Flush operations
- ASSERT_TRUE(writer->Finalize());
- // Construct the buffer required for validation
- orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
- std::string zero_buffer(size_, 0);
- ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), size_, size_), true);
- memcpy((char*)orig_buffer_.get() + size_, random_buffer_1_.get(), size_);
- memcpy((char*)orig_buffer_.get() + (size_ * 2), (void*)zero_buffer.c_str(), size_);
- memcpy((char*)orig_buffer_.get() + (size_ * 3), random_buffer_1_.get(), size_);
- ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, &orig_buffer_.get()[size_ * 4], size_,
- size_ + xor_offset),
- true);
- for (int i = 0; i < size_; i++) {
- orig_buffer_.get()[(size_ * 4) + i] =
- (uint8_t)(orig_buffer_.get()[(size_ * 4) + i] ^ random_buffer_1_.get()[i]);
- }
-}
-
void SnapuserdTest::InitCowDevice() {
bool use_iouring = true;
if (FLAGS_force_config == "iouring_disabled") {
@@ -544,8 +552,8 @@
auto factory = harness_->GetBlockServerFactory();
auto opener = factory->CreateOpener(system_device_ctrl_name_);
auto handler =
- handlers_.AddHandler(system_device_ctrl_name_, cow_system_->path, base_loop_->device(),
- base_loop_->device(), opener, 1, use_iouring, false);
+ handlers_->AddHandler(system_device_ctrl_name_, cow_system_->path, base_dev_->GetPath(),
+ base_dev_->GetPath(), opener, 1, use_iouring, false);
ASSERT_NE(handler, nullptr);
ASSERT_NE(handler->snapuserd(), nullptr);
#ifdef __ANDROID__
@@ -553,24 +561,9 @@
#endif
}
-void SnapuserdTest::SetDeviceControlName() {
- system_device_name_.clear();
- system_device_ctrl_name_.clear();
-
- std::string str(cow_system_->path);
- std::size_t found = str.find_last_of("/\\");
- ASSERT_NE(found, std::string::npos);
- system_device_name_ = str.substr(found + 1);
-
- system_device_ctrl_name_ = system_device_name_ + "-ctrl";
-}
-
void SnapuserdTest::CreateUserDevice() {
- unique_fd fd(TEMP_FAILURE_RETRY(open(base_loop_->device().c_str(), O_RDONLY | O_CLOEXEC)));
- ASSERT_TRUE(fd > 0);
-
- uint64_t dev_sz = get_block_device_size(fd.get());
- ASSERT_TRUE(dev_sz > 0);
+ auto dev_sz = base_dev_->GetSize();
+ ASSERT_NE(dev_sz, 0);
cow_num_sectors_ = dev_sz >> 9;
@@ -580,12 +573,12 @@
}
void SnapuserdTest::InitDaemon() {
- ASSERT_TRUE(handlers_.StartHandler(system_device_ctrl_name_));
+ ASSERT_TRUE(handlers_->StartHandler(system_device_ctrl_name_));
}
void SnapuserdTest::CheckMergeCompletion() {
while (true) {
- double percentage = handlers_.GetMergePercentage();
+ double percentage = handlers_->GetMergePercentage();
if ((int)percentage == 100) {
break;
}
@@ -595,27 +588,26 @@
}
void SnapuserdTest::SetupImpl() {
- CreateBaseDevice();
- CreateCowDevice();
+ ASSERT_NO_FATAL_FAILURE(CreateBaseDevice());
+ ASSERT_NO_FATAL_FAILURE(CreateCowDevice());
SetDeviceControlName();
- CreateUserDevice();
- InitCowDevice();
- InitDaemon();
-
- setup_ok_ = true;
+ ASSERT_NO_FATAL_FAILURE(CreateUserDevice());
+ ASSERT_NO_FATAL_FAILURE(InitCowDevice());
+ ASSERT_NO_FATAL_FAILURE(InitDaemon());
}
bool SnapuserdTest::Merge() {
- StartMerge();
+ if (!StartMerge()) {
+ return false;
+ }
CheckMergeCompletion();
- merge_ok_ = true;
- return merge_ok_;
+ return true;
}
-void SnapuserdTest::StartMerge() {
- ASSERT_TRUE(handlers_.InitiateMerge(system_device_ctrl_name_));
+bool SnapuserdTest::StartMerge() {
+ return handlers_->InitiateMerge(system_device_ctrl_name_);
}
void SnapuserdTest::ValidateMerge() {
@@ -626,67 +618,67 @@
}
void SnapuserdTest::SimulateDaemonRestart() {
- Shutdown();
+ ASSERT_NO_FATAL_FAILURE(Shutdown());
std::this_thread::sleep_for(500ms);
SetDeviceControlName();
- CreateUserDevice();
- InitCowDevice();
- InitDaemon();
+ ASSERT_NO_FATAL_FAILURE(CreateUserDevice());
+ ASSERT_NO_FATAL_FAILURE(InitCowDevice());
+ ASSERT_NO_FATAL_FAILURE(InitDaemon());
}
void SnapuserdTest::MergeInterruptRandomly(int max_duration) {
std::srand(std::time(nullptr));
- StartMerge();
+ ASSERT_TRUE(StartMerge());
for (int i = 0; i < 20; i++) {
int duration = std::rand() % max_duration;
std::this_thread::sleep_for(std::chrono::milliseconds(duration));
- SimulateDaemonRestart();
- StartMerge();
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
+ ASSERT_TRUE(StartMerge());
}
- SimulateDaemonRestart();
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
ASSERT_TRUE(Merge());
}
void SnapuserdTest::MergeInterruptFixed(int duration) {
- StartMerge();
+ ASSERT_TRUE(StartMerge());
for (int i = 0; i < 25; i++) {
std::this_thread::sleep_for(std::chrono::milliseconds(duration));
- SimulateDaemonRestart();
- StartMerge();
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
+ ASSERT_TRUE(StartMerge());
}
- SimulateDaemonRestart();
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
ASSERT_TRUE(Merge());
}
void SnapuserdTest::MergeInterrupt() {
// Interrupt merge at various intervals
- StartMerge();
+ ASSERT_TRUE(StartMerge());
std::this_thread::sleep_for(250ms);
- SimulateDaemonRestart();
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
- StartMerge();
+ ASSERT_TRUE(StartMerge());
std::this_thread::sleep_for(250ms);
- SimulateDaemonRestart();
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
- StartMerge();
+ ASSERT_TRUE(StartMerge());
std::this_thread::sleep_for(150ms);
- SimulateDaemonRestart();
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
- StartMerge();
+ ASSERT_TRUE(StartMerge());
std::this_thread::sleep_for(100ms);
- SimulateDaemonRestart();
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
- StartMerge();
+ ASSERT_TRUE(StartMerge());
std::this_thread::sleep_for(800ms);
- SimulateDaemonRestart();
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
- StartMerge();
+ ASSERT_TRUE(StartMerge());
std::this_thread::sleep_for(600ms);
- SimulateDaemonRestart();
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
ASSERT_TRUE(Merge());
}
@@ -695,98 +687,87 @@
if (!harness_->HasUserDevice()) {
GTEST_SKIP() << "Skipping snapshot read; not supported";
}
- ASSERT_TRUE(SetupDefault());
+ ASSERT_NO_FATAL_FAILURE(SetupDefault());
// I/O before merge
- ReadSnapshotDeviceAndValidate();
+ ASSERT_NO_FATAL_FAILURE(ReadSnapshotDeviceAndValidate());
ASSERT_TRUE(Merge());
ValidateMerge();
// I/O after merge - daemon should read directly
// from base device
- ReadSnapshotDeviceAndValidate();
- Shutdown();
+ ASSERT_NO_FATAL_FAILURE(ReadSnapshotDeviceAndValidate());
}
TEST_F(SnapuserdTest, Snapshot_MERGE_IO_TEST) {
if (!harness_->HasUserDevice()) {
GTEST_SKIP() << "Skipping snapshot read; not supported";
}
- ASSERT_TRUE(SetupDefault());
+ ASSERT_NO_FATAL_FAILURE(SetupDefault());
// Issue I/O before merge begins
std::async(std::launch::async, &SnapuserdTest::ReadSnapshotDeviceAndValidate, this);
// Start the merge
ASSERT_TRUE(Merge());
ValidateMerge();
- Shutdown();
}
TEST_F(SnapuserdTest, Snapshot_MERGE_IO_TEST_1) {
if (!harness_->HasUserDevice()) {
GTEST_SKIP() << "Skipping snapshot read; not supported";
}
- ASSERT_TRUE(SetupDefault());
+ ASSERT_NO_FATAL_FAILURE(SetupDefault());
// Start the merge
- StartMerge();
+ ASSERT_TRUE(StartMerge());
// Issue I/O in parallel when merge is in-progress
std::async(std::launch::async, &SnapuserdTest::ReadSnapshotDeviceAndValidate, this);
CheckMergeCompletion();
ValidateMerge();
- Shutdown();
}
TEST_F(SnapuserdTest, Snapshot_Merge_Resume) {
- ASSERT_TRUE(SetupDefault());
- MergeInterrupt();
+ ASSERT_NO_FATAL_FAILURE(SetupDefault());
+ ASSERT_NO_FATAL_FAILURE(MergeInterrupt());
ValidateMerge();
- Shutdown();
}
TEST_F(SnapuserdTest, Snapshot_COPY_Overlap_TEST_1) {
- ASSERT_TRUE(SetupCopyOverlap_1());
+ ASSERT_NO_FATAL_FAILURE(SetupCopyOverlap_1());
ASSERT_TRUE(Merge());
ValidateMerge();
- Shutdown();
}
TEST_F(SnapuserdTest, Snapshot_COPY_Overlap_TEST_2) {
- ASSERT_TRUE(SetupCopyOverlap_2());
+ ASSERT_NO_FATAL_FAILURE(SetupCopyOverlap_2());
ASSERT_TRUE(Merge());
ValidateMerge();
- Shutdown();
}
TEST_F(SnapuserdTest, Snapshot_COPY_Overlap_Merge_Resume_TEST) {
- ASSERT_TRUE(SetupCopyOverlap_1());
- MergeInterrupt();
+ ASSERT_NO_FATAL_FAILURE(SetupCopyOverlap_1());
+ ASSERT_NO_FATAL_FAILURE(MergeInterrupt());
ValidateMerge();
- Shutdown();
}
TEST_F(SnapuserdTest, Snapshot_Merge_Crash_Fixed_Ordered) {
- ASSERT_TRUE(SetupOrderedOps());
- MergeInterruptFixed(300);
+ ASSERT_NO_FATAL_FAILURE(SetupOrderedOps());
+ ASSERT_NO_FATAL_FAILURE(MergeInterruptFixed(300));
ValidateMerge();
- Shutdown();
}
TEST_F(SnapuserdTest, Snapshot_Merge_Crash_Random_Ordered) {
- ASSERT_TRUE(SetupOrderedOps());
- MergeInterruptRandomly(500);
+ ASSERT_NO_FATAL_FAILURE(SetupOrderedOps());
+ ASSERT_NO_FATAL_FAILURE(MergeInterruptRandomly(500));
ValidateMerge();
- Shutdown();
}
TEST_F(SnapuserdTest, Snapshot_Merge_Crash_Fixed_Inverted) {
- ASSERT_TRUE(SetupOrderedOpsInverted());
- MergeInterruptFixed(50);
+ ASSERT_NO_FATAL_FAILURE(SetupOrderedOpsInverted());
+ ASSERT_NO_FATAL_FAILURE(MergeInterruptFixed(50));
ValidateMerge();
- Shutdown();
}
TEST_F(SnapuserdTest, Snapshot_Merge_Crash_Random_Inverted) {
- ASSERT_TRUE(SetupOrderedOpsInverted());
- MergeInterruptRandomly(50);
+ ASSERT_NO_FATAL_FAILURE(SetupOrderedOpsInverted());
+ ASSERT_NO_FATAL_FAILURE(MergeInterruptRandomly(50));
ValidateMerge();
- Shutdown();
}
} // namespace snapshot
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp
index 28c9f68..52e4f89 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp
@@ -199,6 +199,7 @@
if (io_state_ == MERGE_IO_TRANSITION::READ_AHEAD_FAILURE ||
io_state_ == MERGE_IO_TRANSITION::IO_TERMINATED) {
+ SNAP_LOG(ERROR) << "WaitForMergeBegin failed with state: " << io_state_;
return false;
}
}
@@ -211,6 +212,7 @@
if (io_state_ == MERGE_IO_TRANSITION::READ_AHEAD_FAILURE ||
io_state_ == MERGE_IO_TRANSITION::IO_TERMINATED) {
+ SNAP_LOG(ERROR) << "WaitForMergeBegin failed with state: " << io_state_;
return false;
}
@@ -277,6 +279,7 @@
if (io_state_ == MERGE_IO_TRANSITION::MERGE_FAILED ||
io_state_ == MERGE_IO_TRANSITION::MERGE_COMPLETE ||
io_state_ == MERGE_IO_TRANSITION::IO_TERMINATED) {
+ SNAP_LOG(ERROR) << "Wait for merge ready failed: " << io_state_;
return false;
}
return true;
@@ -668,5 +671,26 @@
}
}
+std::ostream& operator<<(std::ostream& os, MERGE_IO_TRANSITION value) {
+ switch (value) {
+ case MERGE_IO_TRANSITION::INVALID:
+ return os << "INVALID";
+ case MERGE_IO_TRANSITION::MERGE_READY:
+ return os << "MERGE_READY";
+ case MERGE_IO_TRANSITION::MERGE_BEGIN:
+ return os << "MERGE_BEGIN";
+ case MERGE_IO_TRANSITION::MERGE_FAILED:
+ return os << "MERGE_FAILED";
+ case MERGE_IO_TRANSITION::MERGE_COMPLETE:
+ return os << "MERGE_COMPLETE";
+ case MERGE_IO_TRANSITION::IO_TERMINATED:
+ return os << "IO_TERMINATED";
+ case MERGE_IO_TRANSITION::READ_AHEAD_FAILURE:
+ return os << "READ_AHEAD_FAILURE";
+ default:
+ return os << "unknown";
+ }
+}
+
} // namespace snapshot
} // namespace android