Merge "s/master/main/" into main
diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp
index 3ce8141..b12e584 100644
--- a/fastboot/fastboot.cpp
+++ b/fastboot/fastboot.cpp
@@ -1525,7 +1525,7 @@
fb->ResizePartition(pname, std::to_string(buf.image_size));
}
std::string flash_pname = repack_ramdisk(pname, &buf, fp->fb);
- flash_buf(fp->source, flash_pname, &buf, apply_vbmeta);
+ flash_buf(fp->source.get(), flash_pname, &buf, apply_vbmeta);
}
// Sets slot_override as the active slot. If slot_override is blank,
@@ -1679,7 +1679,7 @@
}
for (size_t i = 0; i < tasks->size(); i++) {
if (auto flash_task = tasks->at(i)->AsFlashTask()) {
- if (FlashTask::IsDynamicParitition(fp->source, flash_task)) {
+ if (FlashTask::IsDynamicParitition(fp->source.get(), flash_task)) {
if (!loc) {
loc = i;
}
@@ -1810,7 +1810,8 @@
if (fp_->exclude_dynamic_partitions) {
auto is_non_static_flash_task = [&](const auto& task) -> bool {
if (auto flash_task = task->AsFlashTask()) {
- if (!should_flash_in_userspace(fp_->source, flash_task->GetPartitionAndSlot())) {
+ if (!should_flash_in_userspace(fp_->source.get(),
+ flash_task->GetPartitionAndSlot())) {
return false;
}
}
@@ -1884,9 +1885,10 @@
// On these devices, secondary slots must be flashed as physical
// partitions (otherwise they would not mount on first boot). To enforce
// this, we delete any logical partitions for the "other" slot.
- if (is_retrofit_device(fp_->source)) {
+ if (is_retrofit_device(fp_->source.get())) {
std::string partition_name = image->part_name + "_" + slot;
- if (image->IsSecondary() && should_flash_in_userspace(fp_->source, partition_name)) {
+ if (image->IsSecondary() &&
+ should_flash_in_userspace(fp_->source.get(), partition_name)) {
tasks.emplace_back(std::make_unique<DeleteTask>(fp_, partition_name));
}
}
@@ -1949,8 +1951,7 @@
if (error != 0) {
die("failed to open zip file '%s': %s", filename, ErrorCodeString(error));
}
- ZipImageSource zp = ZipImageSource(zip);
- fp->source = &zp;
+ fp->source.reset(new ZipImageSource(zip));
FlashAllTool tool(fp);
tool.Flash();
@@ -1971,8 +1972,7 @@
}
static void do_flashall(FlashingPlan* fp) {
- LocalImageSource s = LocalImageSource();
- fp->source = &s;
+ fp->source.reset(new LocalImageSource());
FlashAllTool tool(fp);
tool.Flash();
}
@@ -2089,7 +2089,7 @@
die("Cannot read image: %s", strerror(errno));
}
- flash_buf(fp->source, partition, &buf, is_vbmeta_partition(partition));
+ flash_buf(fp->source.get(), partition, &buf, is_vbmeta_partition(partition));
return;
failed:
diff --git a/fastboot/fastboot.h b/fastboot/fastboot.h
index 4859ceb..cd7bc2d 100644
--- a/fastboot/fastboot.h
+++ b/fastboot/fastboot.h
@@ -28,6 +28,7 @@
#pragma once
#include <functional>
+#include <memory>
#include <string>
#include "fastboot_driver_interface.h"
#include "filesystem.h"
@@ -89,7 +90,7 @@
// If the image uses the default slot, or the user specified "all", then
// the paired string will be empty. If the image requests a specific slot
// (for example, system_other) it is specified instead.
- ImageSource* source;
+ std::unique_ptr<ImageSource> source;
bool wants_wipe = false;
bool skip_reboot = false;
bool wants_set_active = false;
diff --git a/fastboot/task.cpp b/fastboot/task.cpp
index 4b2b9e3..25c5a6e 100644
--- a/fastboot/task.cpp
+++ b/fastboot/task.cpp
@@ -41,7 +41,7 @@
void FlashTask::Run() {
auto flash = [&](const std::string& partition) {
- if (should_flash_in_userspace(fp_->source, partition) && !is_userspace_fastboot() &&
+ if (should_flash_in_userspace(fp_->source.get(), partition) && !is_userspace_fastboot() &&
!fp_->force_flash) {
die("The partition you are trying to flash is dynamic, and "
"should be flashed via fastbootd. Please run:\n"
@@ -174,7 +174,7 @@
LOG(VERBOSE) << "Cannot optimize flashing super for all slots";
return nullptr;
}
- if (!CanOptimize(fp->source, tasks)) {
+ if (!CanOptimize(fp->source.get(), tasks)) {
return nullptr;
}
diff --git a/fastboot/task_test.cpp b/fastboot/task_test.cpp
index 1e25b6f..9cde1a8 100644
--- a/fastboot/task_test.cpp
+++ b/fastboot/task_test.cpp
@@ -192,8 +192,7 @@
GTEST_SKIP();
}
- LocalImageSource s;
- fp->source = &s;
+ fp->source.reset(new LocalImageSource);
fp->sparse_limit = std::numeric_limits<int64_t>::max();
fastboot::MockFastbootDriver fb;
@@ -239,8 +238,7 @@
GTEST_SKIP();
}
- LocalImageSource s;
- fp->source = &s;
+ fp->source.reset(new LocalImageSource);
fastboot::MockFastbootDriver fb;
fp->fb = &fb;
@@ -260,7 +258,7 @@
ParseFastbootInfoLine(fp.get(), android::base::Tokenize(test.first, " "));
auto flash_task = task->AsFlashTask();
ASSERT_FALSE(flash_task == nullptr);
- ASSERT_EQ(FlashTask::IsDynamicParitition(fp->source, flash_task), test.second);
+ ASSERT_EQ(FlashTask::IsDynamicParitition(fp->source.get(), flash_task), test.second);
}
}
@@ -269,8 +267,7 @@
GTEST_SKIP();
}
- LocalImageSource s;
- fp->source = &s;
+ fp->source.reset(new LocalImageSource);
fp->sparse_limit = std::numeric_limits<int64_t>::max();
fastboot::MockFastbootDriver fb;
@@ -301,7 +298,7 @@
for (auto& test : patternmatchtest) {
std::vector<std::unique_ptr<Task>> tasks = ParseFastbootInfo(fp.get(), test.first);
tasks.erase(std::remove_if(tasks.begin(), tasks.end(), remove_if_callback), tasks.end());
- ASSERT_EQ(OptimizedFlashSuperTask::CanOptimize(fp->source, tasks), test.second);
+ ASSERT_EQ(OptimizedFlashSuperTask::CanOptimize(fp->source.get(), tasks), test.second);
}
}
@@ -312,8 +309,7 @@
GTEST_SKIP();
}
- LocalImageSource s;
- fp->source = &s;
+ fp->source.reset(new LocalImageSource);
fp->sparse_limit = std::numeric_limits<int64_t>::max();
fastboot::MockFastbootDriver fb;
@@ -362,7 +358,7 @@
contains_optimized_task = true;
}
if (auto flash_task = task->AsFlashTask()) {
- if (FlashTask::IsDynamicParitition(fp->source, flash_task)) {
+ if (FlashTask::IsDynamicParitition(fp->source.get(), flash_task)) {
return false;
}
}
diff --git a/fs_mgr/libfs_avb/tests/avb_util_test.cpp b/fs_mgr/libfs_avb/tests/avb_util_test.cpp
index 2e34920..ee83cda 100644
--- a/fs_mgr/libfs_avb/tests/avb_util_test.cpp
+++ b/fs_mgr/libfs_avb/tests/avb_util_test.cpp
@@ -655,10 +655,12 @@
" Partition Name: boot\n"
" Rollback Index Location: 1\n"
" Public key (sha1): cdbb77177f731920bbe0a0f94f84d9038ae0617d\n"
+ " Flags: 0\n"
" Chain Partition descriptor:\n"
" Partition Name: system\n"
" Rollback Index Location: 2\n"
- " Public key (sha1): 2597c218aae470a130f61162feaae70afd97f011\n",
+ " Public key (sha1): 2597c218aae470a130f61162feaae70afd97f011\n"
+ " Flags: 0\n",
InfoImage("vbmeta.img"));
android::base::unique_fd fd(open(vbmeta_path.value().c_str(), O_RDONLY | O_CLOEXEC));
@@ -876,10 +878,12 @@
" Partition Name: boot\n"
" Rollback Index Location: 1\n"
" Public key (sha1): cdbb77177f731920bbe0a0f94f84d9038ae0617d\n"
+ " Flags: 0\n"
" Chain Partition descriptor:\n"
" Partition Name: vbmeta_system\n"
" Rollback Index Location: 2\n"
- " Public key (sha1): 2597c218aae470a130f61162feaae70afd97f011\n",
+ " Public key (sha1): 2597c218aae470a130f61162feaae70afd97f011\n"
+ " Flags: 0\n",
InfoImage("vbmeta.img"));
bool fatal_error = false;
@@ -909,7 +913,8 @@
" Chain Partition descriptor:\n"
" Partition Name: system\n"
" Rollback Index Location: 3\n"
- " Public key (sha1): 2597c218aae470a130f61162feaae70afd97f011\n",
+ " Public key (sha1): 2597c218aae470a130f61162feaae70afd97f011\n"
+ " Flags: 0\n",
InfoImage("vbmeta_system.img"));
chained_descriptors = GetChainPartitionInfo(LoadVBMetaData("vbmeta_system.img"), &fatal_error);
diff --git a/fs_mgr/libfs_avb/tests/basic_test.cpp b/fs_mgr/libfs_avb/tests/basic_test.cpp
index 1c47c07..d49affb 100644
--- a/fs_mgr/libfs_avb/tests/basic_test.cpp
+++ b/fs_mgr/libfs_avb/tests/basic_test.cpp
@@ -268,10 +268,12 @@
" Partition Name: boot\n"
" Rollback Index Location: 1\n"
" Public key (sha1): cdbb77177f731920bbe0a0f94f84d9038ae0617d\n"
+ " Flags: 0\n"
" Chain Partition descriptor:\n"
" Partition Name: system\n"
" Rollback Index Location: 2\n"
- " Public key (sha1): 2597c218aae470a130f61162feaae70afd97f011\n",
+ " Public key (sha1): 2597c218aae470a130f61162feaae70afd97f011\n"
+ " Flags: 0\n",
InfoImage("vbmeta.img"));
}
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp
index a6dee4f..83b5a12 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp
@@ -22,6 +22,7 @@
#include <string>
#include <vector>
+#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/unique_fd.h>
#include <gflags/gflags.h>
@@ -38,11 +39,13 @@
DEFINE_bool(verify_merge_sequence, false, "Verify merge order sequencing");
DEFINE_bool(show_merge_sequence, false, "Show merge order sequence");
DEFINE_bool(show_raw_ops, false, "Show raw ops directly from the underlying parser");
+DEFINE_string(extract_to, "", "Extract the COW contents to the given file");
namespace android {
namespace snapshot {
using android::base::borrowed_fd;
+using android::base::unique_fd;
void MyLogger(android::base::LogId, android::base::LogSeverity severity, const char*, const char*,
unsigned int, const char* message) {
@@ -53,7 +56,7 @@
}
}
-static void ShowBad(CowReader& reader, const struct CowOperation* op) {
+static void ShowBad(CowReader& reader, const CowOperation* op) {
size_t count;
auto buffer = std::make_unique<uint8_t[]>(op->data_length);
@@ -104,12 +107,21 @@
}
static bool Inspect(const std::string& path) {
- android::base::unique_fd fd(open(path.c_str(), O_RDONLY));
+ unique_fd fd(open(path.c_str(), O_RDONLY));
if (fd < 0) {
PLOG(ERROR) << "open failed: " << path;
return false;
}
+ unique_fd extract_to;
+ if (!FLAGS_extract_to.empty()) {
+ extract_to.reset(open(FLAGS_extract_to.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0664));
+ if (extract_to < 0) {
+ PLOG(ERROR) << "could not open " << FLAGS_extract_to << " for writing";
+ return false;
+ }
+ }
+
CowReader reader;
auto start_time = std::chrono::steady_clock::now();
@@ -186,12 +198,23 @@
if (!FLAGS_silent && FLAGS_show_ops) std::cout << *op << "\n";
- if (FLAGS_decompress && op->type == kCowReplaceOp && op->compression != kCowCompressNone) {
+ if ((FLAGS_decompress || extract_to >= 0) && op->type == kCowReplaceOp) {
if (reader.ReadData(op, buffer.data(), buffer.size()) < 0) {
std::cerr << "Failed to decompress for :" << *op << "\n";
success = false;
if (FLAGS_show_bad_data) ShowBad(reader, op);
}
+ if (extract_to >= 0) {
+ off_t offset = uint64_t(op->new_block) * header.block_size;
+ if (!android::base::WriteFullyAtOffset(extract_to, buffer.data(), buffer.size(),
+ offset)) {
+ PLOG(ERROR) << "failed to write block " << op->new_block;
+ return false;
+ }
+ }
+ } else if (extract_to >= 0 && !IsMetadataOp(*op) && op->type != kCowZeroOp) {
+ PLOG(ERROR) << "Cannot extract op yet: " << *op;
+ return false;
}
if (op->type == kCowSequenceOp && FLAGS_show_merge_sequence) {
diff --git a/fs_mgr/libsnapshot/snapuserd/Android.bp b/fs_mgr/libsnapshot/snapuserd/Android.bp
index 47a8685..1b0c563 100644
--- a/fs_mgr/libsnapshot/snapuserd/Android.bp
+++ b/fs_mgr/libsnapshot/snapuserd/Android.bp
@@ -293,3 +293,48 @@
"vts",
],
}
+
+cc_binary_host {
+ name: "snapuserd_extractor",
+ defaults: [
+ "fs_mgr_defaults",
+ "libsnapshot_cow_defaults",
+ ],
+ srcs: [
+ "testing/dm_user_harness.cpp",
+ "testing/harness.cpp",
+ "testing/host_harness.cpp",
+ "user-space-merge/extractor.cpp",
+ "snapuserd_extractor.cpp",
+ ],
+ cflags: [
+ "-D_FILE_OFFSET_BITS=64",
+ "-Wall",
+ "-Werror",
+ ],
+ shared_libs: [
+ "libbase",
+ "liblog",
+ ],
+ static_libs: [
+ "libbrotli",
+ "libcutils_sockets",
+ "libdm",
+ "libext2_uuid",
+ "libext4_utils",
+ "libfs_mgr_file_wait",
+ "libgflags",
+ "libsnapshot_cow",
+ "libsnapuserd",
+ "liburing",
+ "libz",
+ ],
+ include_dirs: [
+ "bionic/libc/kernel",
+ ".",
+ ],
+ header_libs: [
+ "libstorage_literals_headers",
+ "libfiemap_headers",
+ ],
+}
diff --git a/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/snapuserd_worker.cpp b/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/snapuserd_worker.cpp
index 559dcc7..571b352 100644
--- a/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/snapuserd_worker.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/snapuserd_worker.cpp
@@ -103,7 +103,7 @@
ssize_t rv = reader_->ReadData(cow_op, buffer, BLOCK_SZ);
if (rv != BLOCK_SZ) {
SNAP_LOG(ERROR) << "ProcessReplaceOp failed for block " << cow_op->new_block
- << ", return = " << rv;
+ << ", return = " << rv << ", COW operation = " << *cow_op;
return false;
}
return true;
diff --git a/fs_mgr/libsnapshot/snapuserd/snapuserd_extractor.cpp b/fs_mgr/libsnapshot/snapuserd/snapuserd_extractor.cpp
new file mode 100644
index 0000000..f46cd5b
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/snapuserd_extractor.cpp
@@ -0,0 +1,68 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fcntl.h>
+#include <linux/fs.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <iostream>
+#include <memory>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/unique_fd.h>
+#include <gflags/gflags.h>
+#include "user-space-merge/extractor.h"
+
+using namespace std::string_literals;
+
+DEFINE_string(base, "", "Base device/image");
+DEFINE_string(cow, "", "COW device/image");
+DEFINE_string(out, "", "Output path");
+DEFINE_int32(num_sectors, 0, "Number of sectors to read");
+
+int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) {
+ android::base::InitLogging(argv);
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+ if (FLAGS_out.empty()) {
+ LOG(ERROR) << "Missing -out argument.";
+ return 1;
+ }
+ if (FLAGS_base.empty()) {
+ LOG(ERROR) << "Missing -base argument.";
+ return 1;
+ }
+ if (FLAGS_cow.empty()) {
+ LOG(ERROR) << "missing -out argument.";
+ return 1;
+ }
+ if (!FLAGS_num_sectors) {
+ LOG(ERROR) << "missing -num_sectors argument.";
+ return 1;
+ }
+
+ android::snapshot::Extractor extractor(FLAGS_base, FLAGS_cow);
+ if (!extractor.Init()) {
+ return 1;
+ }
+ if (!extractor.Extract(FLAGS_num_sectors, FLAGS_out)) {
+ return 1;
+ }
+ return 0;
+}
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.cpp
new file mode 100644
index 0000000..c5718d5
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.cpp
@@ -0,0 +1,90 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "extractor.h"
+
+#include <fcntl.h>
+#include <linux/fs.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <iostream>
+#include <memory>
+
+#include <android-base/file.h>
+#include <android-base/properties.h>
+#include <android-base/unique_fd.h>
+
+using android::base::unique_fd;
+using namespace std::string_literals;
+
+namespace android {
+namespace snapshot {
+
+Extractor::Extractor(const std::string& base_path, const std::string& cow_path)
+ : base_path_(base_path), cow_path_(cow_path), control_name_("test") {}
+
+bool Extractor::Init() {
+ auto opener = factory_.CreateTestOpener(control_name_);
+ handler_ = std::make_shared<SnapshotHandler>(control_name_, cow_path_, base_path_, base_path_,
+ opener, 1, false, false);
+ if (!handler_->InitCowDevice()) {
+ return false;
+ }
+ if (!handler_->InitializeWorkers()) {
+ return false;
+ }
+
+ read_worker_ = std::make_unique<ReadWorker>(cow_path_, base_path_, control_name_, base_path_,
+ handler_->GetSharedPtr(), opener);
+ if (!read_worker_->Init()) {
+ return false;
+ }
+ block_server_ = static_cast<TestBlockServer*>(read_worker_->block_server());
+
+ handler_thread_ = std::async(std::launch::async, &SnapshotHandler::Start, handler_.get());
+ return true;
+}
+
+Extractor::~Extractor() {
+ factory_.DeleteQueue(control_name_);
+}
+
+bool Extractor::Extract(off_t num_sectors, const std::string& out_path) {
+ unique_fd out_fd(open(out_path.c_str(), O_RDWR | O_CLOEXEC | O_TRUNC | O_CREAT, 0664));
+ if (out_fd < 0) {
+ PLOG(ERROR) << "Could not open for writing: " << out_path;
+ return false;
+ }
+
+ for (off_t i = 0; i < num_sectors; i++) {
+ if (!read_worker_->RequestSectors(i, 512)) {
+ LOG(ERROR) << "Read sector " << i << " failed.";
+ return false;
+ }
+ std::string result = std::move(block_server_->sent_io());
+ off_t offset = i * 512;
+ if (!android::base::WriteFullyAtOffset(out_fd, result.data(), result.size(), offset)) {
+ PLOG(ERROR) << "write failed";
+ return false;
+ }
+ }
+ return true;
+}
+
+} // namespace snapshot
+} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.h
new file mode 100644
index 0000000..65285b1
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.h
@@ -0,0 +1,51 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <thread>
+
+#include <android-base/unique_fd.h>
+#include "merge_worker.h"
+#include "read_worker.h"
+#include "snapuserd_core.h"
+#include "testing/host_harness.h"
+
+namespace android {
+namespace snapshot {
+
+class Extractor final {
+ public:
+ Extractor(const std::string& base_path, const std::string& cow_path);
+ ~Extractor();
+
+ bool Init();
+ bool Extract(off_t num_sectors, const std::string& out_path);
+
+ private:
+ std::string base_path_;
+ std::string cow_path_;
+
+ TestBlockServerFactory factory_;
+ HostTestHarness harness_;
+ std::string control_name_;
+ std::shared_ptr<SnapshotHandler> handler_;
+ std::unique_ptr<ReadWorker> read_worker_;
+ std::future<bool> handler_thread_;
+ TestBlockServer* block_server_ = nullptr;
+};
+
+} // namespace snapshot
+} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
index b9ecfa5..5cb13e8 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
@@ -283,7 +283,8 @@
// We found the sector in mapping. Check the type of COW OP and
// process it.
if (!ProcessCowOp(it->second, buffer)) {
- SNAP_LOG(ERROR) << "ProcessCowOp failed";
+ SNAP_LOG(ERROR)
+ << "ProcessCowOp failed, sector = " << sector << ", size = " << sz;
return false;
}
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
index c295851..e886ec3 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
@@ -173,6 +173,10 @@
}
SNAP_LOG(INFO) << "Merge-ops: " << header.num_merge_ops;
+ if (header.num_merge_ops) {
+ resume_merge_ = true;
+ SNAP_LOG(INFO) << "Resume Snapshot-merge";
+ }
if (!MmapMetadata()) {
SNAP_LOG(ERROR) << "mmap failed";
@@ -295,6 +299,11 @@
if (ra_thread_) {
ra_thread_status =
std::async(std::launch::async, &ReadAhead::RunThread, read_ahead_thread_.get());
+ // If this is a merge-resume path, wait until RA thread is fully up as
+ // the data has to be re-constructed from the scratch space.
+ if (resume_merge_ && ShouldReconstructDataFromCow()) {
+ WaitForRaThreadToStart();
+ }
}
// Launch worker threads
@@ -307,7 +316,9 @@
std::async(std::launch::async, &MergeWorker::Run, merge_thread_.get());
// Now that the worker threads are up, scan the partitions.
- if (perform_verification_) {
+ // If the snapshot-merge is being resumed, there is no need to scan as the
+ // current slot is already marked as boot complete.
+ if (perform_verification_ && !resume_merge_) {
update_verify_->VerifyUpdatePartition();
}
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
index e401c11..f88406d 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
@@ -147,6 +147,8 @@
void WakeupMonitorMergeThread();
void WaitForMergeComplete();
bool WaitForMergeBegin();
+ void RaThreadStarted();
+ void WaitForRaThreadToStart();
void NotifyRAForMergeReady();
bool WaitForMergeReady();
void MergeFailed();
@@ -221,6 +223,7 @@
// Read-ahead related
bool populate_data_from_cow_ = false;
bool ra_thread_ = false;
+ bool ra_thread_started_ = false;
int total_ra_blocks_merged_ = 0;
MERGE_IO_TRANSITION io_state_ = MERGE_IO_TRANSITION::INVALID;
std::unique_ptr<ReadAhead> read_ahead_thread_;
@@ -242,6 +245,7 @@
bool scratch_space_ = false;
int num_worker_threads_ = kNumWorkerThreads;
bool perform_verification_ = true;
+ bool resume_merge_ = false;
std::unique_ptr<struct io_uring> ring_;
std::unique_ptr<UpdateVerify> update_verify_;
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
index d2128c5..998d233 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
@@ -206,6 +206,7 @@
return false;
}
+ snapuserd_->RaThreadStarted();
SNAP_LOG(INFO) << "ReconstructDataFromCow success";
notify_read_ahead_failed.Cancel();
return true;
@@ -716,9 +717,13 @@
total_ra_blocks_completed_ += total_blocks_merged_;
snapuserd_->SetMergedBlockCountForNextCommit(total_blocks_merged_);
- // Flush the data only if we have a overlapping blocks in the region
+ // Flush the scratch data - Technically, we should flush only for overlapping
+ // blocks; However, since this region is mmap'ed, the dirty pages can still
+ // get flushed to disk at any random point in time. Instead, make sure
+ // the data in scratch is in the correct state before merge thread resumes.
+ //
// Notify the Merge thread to resume merging this window
- if (!snapuserd_->ReadAheadIOCompleted(overlap_)) {
+ if (!snapuserd_->ReadAheadIOCompleted(true)) {
SNAP_LOG(ERROR) << "ReadAheadIOCompleted failed...";
snapuserd_->ReadAheadIOFailed();
return false;
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
index 620ecbd..65f31cf 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
@@ -235,9 +235,11 @@
bool Merge();
void ValidateMerge();
void ReadSnapshotDeviceAndValidate();
+ void ReadSnapshotAndValidateOverlappingBlocks();
void Shutdown();
void MergeInterrupt();
void MergeInterruptFixed(int duration);
+ void MergeInterruptAndValidate(int duration);
void MergeInterruptRandomly(int max_duration);
bool StartMerge();
void CheckMergeCompletion();
@@ -358,6 +360,76 @@
ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 4), size_), 0);
}
+void SnapuserdTest::ReadSnapshotAndValidateOverlappingBlocks() {
+ // Open COW device
+ unique_fd fd(open(cow_system_->path, O_RDONLY));
+ ASSERT_GE(fd, 0);
+
+ CowReader reader;
+ ASSERT_TRUE(reader.Parse(fd));
+
+ const auto& header = reader.GetHeader();
+ size_t total_mapped_addr_length = header.prefix.header_size + BUFFER_REGION_DEFAULT_SIZE;
+
+ ASSERT_GE(header.prefix.major_version, 2);
+
+ void* mapped_addr = mmap(NULL, total_mapped_addr_length, PROT_READ, MAP_SHARED, fd.get(), 0);
+ ASSERT_NE(mapped_addr, MAP_FAILED);
+
+ bool populate_data_from_scratch = false;
+ struct BufferState* ra_state =
+ reinterpret_cast<struct BufferState*>((char*)mapped_addr + header.prefix.header_size);
+ if (ra_state->read_ahead_state == kCowReadAheadDone) {
+ populate_data_from_scratch = true;
+ }
+
+ size_t num_merge_ops = header.num_merge_ops;
+ // We have some partial merge operations completed.
+ // To test the merge-resume path, forcefully corrupt the data of the base
+ // device for the offsets where the merge is still pending.
+ if (num_merge_ops && populate_data_from_scratch) {
+ std::string corrupt_buffer(4096, 0);
+ // Corrupt two blocks from the point where the merge has to be resumed by
+ // writing down zeroe's.
+ //
+ // Now, since this is a merge-resume path, the "correct" data should be
+ // in the scratch space of the COW device. When there is an I/O request
+ // from the snapshot device, the data has to be retrieved from the
+ // scratch space. If not and I/O is routed to the base device, we
+ // may end up with corruption.
+ off_t corrupt_offset = (num_merge_ops + 2) * 4096;
+
+ if (corrupt_offset < size_) {
+ ASSERT_EQ(android::base::WriteFullyAtOffset(base_fd_, (void*)corrupt_buffer.c_str(),
+ 4096, corrupt_offset),
+ true);
+ corrupt_offset -= 4096;
+ ASSERT_EQ(android::base::WriteFullyAtOffset(base_fd_, (void*)corrupt_buffer.c_str(),
+ 4096, corrupt_offset),
+ true);
+ fsync(base_fd_.get());
+ }
+ }
+
+ // Time to read the snapshot device.
+ unique_fd snapshot_fd(open(dmuser_dev_->GetPath().c_str(), O_RDONLY | O_DIRECT | O_SYNC));
+ ASSERT_GE(snapshot_fd, 0);
+
+ void* buff_addr;
+ ASSERT_EQ(posix_memalign(&buff_addr, 4096, size_), 0);
+
+ std::unique_ptr<void, decltype(&::free)> snapshot_buffer(buff_addr, ::free);
+
+ // Scan the entire snapshot device and read the data and verify data
+ // integrity. Since the base device was forcefully corrupted, the data from
+ // this scan should be retrieved from scratch space of the COW partition.
+ //
+ // Furthermore, after the merge is complete, base device data is again
+ // verified as the aforementioned corrupted blocks aren't persisted.
+ ASSERT_EQ(ReadFullyAtOffset(snapshot_fd, snapshot_buffer.get(), size_, 0), true);
+ ASSERT_EQ(memcmp(snapshot_buffer.get(), orig_buffer_.get(), size_), 0);
+}
+
void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() {
auto writer = CreateCowDeviceInternal();
ASSERT_NE(writer, nullptr);
@@ -665,6 +737,20 @@
ASSERT_TRUE(Merge());
}
+void SnapuserdTest::MergeInterruptAndValidate(int duration) {
+ ASSERT_TRUE(StartMerge());
+
+ for (int i = 0; i < 15; i++) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(duration));
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
+ ReadSnapshotAndValidateOverlappingBlocks();
+ ASSERT_TRUE(StartMerge());
+ }
+
+ ASSERT_NO_FATAL_FAILURE(SimulateDaemonRestart());
+ ASSERT_TRUE(Merge());
+}
+
void SnapuserdTest::MergeInterrupt() {
// Interrupt merge at various intervals
ASSERT_TRUE(StartMerge());
@@ -761,6 +847,15 @@
ValidateMerge();
}
+TEST_F(SnapuserdTest, Snapshot_COPY_Overlap_Merge_Resume_IO_Validate_TEST) {
+ if (!harness_->HasUserDevice()) {
+ GTEST_SKIP() << "Skipping snapshot read; not supported";
+ }
+ ASSERT_NO_FATAL_FAILURE(SetupCopyOverlap_2());
+ ASSERT_NO_FATAL_FAILURE(MergeInterruptAndValidate(2));
+ ValidateMerge();
+}
+
TEST_F(SnapuserdTest, Snapshot_Merge_Crash_Fixed_Ordered) {
ASSERT_NO_FATAL_FAILURE(SetupOrderedOps());
ASSERT_NO_FATAL_FAILURE(MergeInterruptFixed(300));
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp
index f3e0019..8d090bf 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_transitions.cpp
@@ -366,6 +366,26 @@
}
}
+void SnapshotHandler::RaThreadStarted() {
+ std::unique_lock<std::mutex> lock(lock_);
+ ra_thread_started_ = true;
+}
+
+void SnapshotHandler::WaitForRaThreadToStart() {
+ auto now = std::chrono::system_clock::now();
+ auto deadline = now + 3s;
+ {
+ std::unique_lock<std::mutex> lock(lock_);
+ while (!ra_thread_started_) {
+ auto status = cv.wait_until(lock, deadline);
+ if (status == std::cv_status::timeout) {
+ SNAP_LOG(ERROR) << "Read-ahead thread did not start";
+ return;
+ }
+ }
+ }
+}
+
std::string SnapshotHandler::GetMergeStatus() {
bool merge_not_initiated = false;
bool merge_monitored = false;
@@ -618,7 +638,6 @@
std::unordered_map<uint64_t, void*>::iterator it = read_ahead_buffer_map_.find(block);
if (it == read_ahead_buffer_map_.end()) {
- SNAP_LOG(ERROR) << "Block: " << block << " not found in RA buffer";
return false;
}
@@ -642,6 +661,13 @@
MERGE_GROUP_STATE state = blk_state->merge_state_;
switch (state) {
case MERGE_GROUP_STATE::GROUP_MERGE_PENDING: {
+ // If this is a merge-resume path, check if the data is
+ // available from scratch space. Data from scratch space takes
+ // higher precedence than from source device for overlapping
+ // blocks.
+ if (resume_merge_ && GetRABuffer(&lock, new_block, buffer)) {
+ return (MERGE_GROUP_STATE::GROUP_MERGE_IN_PROGRESS);
+ }
blk_state->num_ios_in_progress += 1; // ref count
[[fallthrough]];
}
diff --git a/init/capabilities.cpp b/init/capabilities.cpp
index ab6ff03..0e2cd2a 100644
--- a/init/capabilities.cpp
+++ b/init/capabilities.cpp
@@ -66,18 +66,12 @@
CAP_MAP_ENTRY(WAKE_ALARM),
CAP_MAP_ENTRY(BLOCK_SUSPEND),
CAP_MAP_ENTRY(AUDIT_READ),
-#if defined(__BIONIC__)
CAP_MAP_ENTRY(PERFMON),
CAP_MAP_ENTRY(BPF),
CAP_MAP_ENTRY(CHECKPOINT_RESTORE),
-#endif
};
-#if defined(__BIONIC__)
static_assert(CAP_LAST_CAP == CAP_CHECKPOINT_RESTORE, "CAP_LAST_CAP is not CAP_CHECKPOINT_RESTORE");
-#else
-static_assert(CAP_LAST_CAP == CAP_AUDIT_READ, "CAP_LAST_CAP is not CAP_AUDIT_READ");
-#endif
static bool ComputeCapAmbientSupported() {
#if defined(__ANDROID__)
diff --git a/init/capabilities.h b/init/capabilities.h
index 891e0ac..fc80c98 100644
--- a/init/capabilities.h
+++ b/init/capabilities.h
@@ -21,17 +21,6 @@
#include <string>
#include <type_traits>
-#if !defined(__ANDROID__)
-#ifndef CAP_BLOCK_SUSPEND
-#define CAP_BLOCK_SUSPEND 36
-#endif
-#ifndef CAP_AUDIT_READ
-#define CAP_AUDIT_READ 37
-#endif
-#undef CAP_LAST_CAP
-#define CAP_LAST_CAP CAP_AUDIT_READ
-#endif
-
namespace android {
namespace init {
diff --git a/rootdir/etc/linker.config.json b/rootdir/etc/linker.config.json
index 47f77b1..d72ac66 100644
--- a/rootdir/etc/linker.config.json
+++ b/rootdir/etc/linker.config.json
@@ -11,9 +11,6 @@
"libsigchain.so",
// TODO(b/122876336): Remove libpac.so once it's migrated to Webview
"libpac.so",
- // TODO(b/184872979): Remove libbinder_rpc_unstable.so once stablized and
- // added to libbinder_ndk.
- "libbinder_rpc_unstable.so",
// TODO(b/120786417 or b/134659294): libicuuc.so
// and libicui18n.so are kept for app compat.
"libicui18n.so",