Merge "init_first_stage: Disable ThinLTO" into main
diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp
index 61c7204..0bd07ed 100644
--- a/fastboot/fastboot.cpp
+++ b/fastboot/fastboot.cpp
@@ -120,6 +120,9 @@
static std::vector<Image> images = {
// clang-format off
{ "boot", "boot.img", "boot.sig", "boot", false, ImageType::BootCritical },
+ { "bootloader",
+ "bootloader.img", "", "bootloader",
+ true, ImageType::Extra },
{ "init_boot",
"init_boot.img", "init_boot.sig",
"init_boot",
@@ -132,6 +135,7 @@
{ "odm_dlkm", "odm_dlkm.img", "odm_dlkm.sig", "odm_dlkm", true, ImageType::Normal },
{ "product", "product.img", "product.sig", "product", true, ImageType::Normal },
{ "pvmfw", "pvmfw.img", "pvmfw.sig", "pvmfw", true, ImageType::BootCritical },
+ { "radio", "radio.img", "", "radio", true, ImageType::Extra },
{ "recovery", "recovery.img", "recovery.sig", "recovery", true, ImageType::BootCritical },
{ "super", "super.img", "super.sig", "super", true, ImageType::Extra },
{ "system", "system.img", "system.sig", "system", false, ImageType::Normal },
@@ -630,6 +634,9 @@
" --skip-reboot Don't reboot device after flashing.\n"
" --disable-verity Sets disable-verity when flashing vbmeta.\n"
" --disable-verification Sets disable-verification when flashing vbmeta.\n"
+ " --disable-super-optimization\n"
+ " Disables optimizations on flashing super partition.\n"
+ " --disable-fastboot-info Will collects tasks from image list rather than $OUT/fastboot-info.txt.\n"
" --fs-options=OPTION[,OPTION]\n"
" Enable filesystem features. OPTION supports casefold, projid, compress\n"
// TODO: remove --unbuffered?
diff --git a/fs_mgr/Android.bp b/fs_mgr/Android.bp
index 15600fd..0a836e4 100644
--- a/fs_mgr/Android.bp
+++ b/fs_mgr/Android.bp
@@ -49,7 +49,6 @@
sanitize: {
misc_undefined: ["integer"],
},
- local_include_dirs: ["include/"],
cflags: [
"-Wall",
"-Werror",
@@ -60,6 +59,7 @@
name: "libfs_mgr_defaults",
defaults: ["fs_mgr_defaults"],
export_include_dirs: ["include"],
+ local_include_dirs: ["include/"],
include_dirs: ["system/vold"],
cflags: [
"-D_FILE_OFFSET_BITS=64",
@@ -188,6 +188,7 @@
],
host_supported: true,
defaults: ["fs_mgr_defaults"],
+ local_include_dirs: ["include/"],
srcs: [
"fs_mgr_fstab.cpp",
"fs_mgr_boot_config.cpp",
diff --git a/fs_mgr/libsnapshot/snapshot.cpp b/fs_mgr/libsnapshot/snapshot.cpp
index fbea79b..09d35cf 100644
--- a/fs_mgr/libsnapshot/snapshot.cpp
+++ b/fs_mgr/libsnapshot/snapshot.cpp
@@ -2490,9 +2490,6 @@
}
created_devices.EmplaceBack<AutoUnmapDevice>(&dm_, name);
- remaining_time = GetRemainingTime(params.timeout_ms, begin);
- if (remaining_time.count() < 0) return false;
-
cow_device = new_cow_device;
}
@@ -2507,6 +2504,9 @@
// the user-space will not start the merge. We have to explicitly inform the
// daemon to resume the merge. Check ProcessUpdateState() call stack.
if (!UpdateUsesUserSnapshots(lock)) {
+ remaining_time = GetRemainingTime(params.timeout_ms, begin);
+ if (remaining_time.count() < 0) return false;
+
std::string path;
if (!MapSnapshot(lock, params.GetPartitionName(), base_device, cow_device, remaining_time,
&path)) {
diff --git a/fs_mgr/libsnapshot/snapuserd/Android.bp b/fs_mgr/libsnapshot/snapuserd/Android.bp
index 9fe567a..a9b96e2 100644
--- a/fs_mgr/libsnapshot/snapuserd/Android.bp
+++ b/fs_mgr/libsnapshot/snapuserd/Android.bp
@@ -57,18 +57,20 @@
defaults: [
"fs_mgr_defaults",
],
+ local_include_dirs: ["include/"],
srcs: [
"dm-snapshot-merge/snapuserd.cpp",
"dm-snapshot-merge/snapuserd_worker.cpp",
"dm-snapshot-merge/snapuserd_readahead.cpp",
"snapuserd_buffer.cpp",
"user-space-merge/handler_manager.cpp",
+ "user-space-merge/read_worker.cpp",
"user-space-merge/snapuserd_core.cpp",
- "user-space-merge/snapuserd_dm_user.cpp",
"user-space-merge/snapuserd_merge.cpp",
"user-space-merge/snapuserd_readahead.cpp",
"user-space-merge/snapuserd_transitions.cpp",
"user-space-merge/snapuserd_verify.cpp",
+ "user-space-merge/worker.cpp",
],
static_libs: [
"libbase",
diff --git a/fs_mgr/libsnapshot/snapuserd/include/snapuserd/snapuserd_buffer.h b/fs_mgr/libsnapshot/snapuserd/include/snapuserd/snapuserd_buffer.h
index a6b6a7f..c5ca2b1 100644
--- a/fs_mgr/libsnapshot/snapuserd/include/snapuserd/snapuserd_buffer.h
+++ b/fs_mgr/libsnapshot/snapuserd/include/snapuserd/snapuserd_buffer.h
@@ -36,6 +36,21 @@
struct dm_user_header* GetHeaderPtr();
void ResetBufferOffset() { buffer_offset_ = 0; }
void* GetPayloadBufPtr();
+ loff_t GetPayloadBytesWritten() { return buffer_offset_; }
+
+ // Same as calling GetPayloadBuffer and then UpdateBufferOffset.
+ //
+ // This is preferred over GetPayloadBuffer as it does not require a
+ // separate call to UpdateBufferOffset.
+ void* AcquireBuffer(size_t size) { return AcquireBuffer(size, size); }
+
+ // Same as AcquireBuffer, but separates the requested size from the buffer
+ // offset. This is useful for a situation where a full run of data will be
+ // read, but only a partial amount will be returned.
+ //
+ // If size != to_write, the excess bytes may be reallocated by the next
+ // call to AcquireBuffer.
+ void* AcquireBuffer(size_t size, size_t to_write);
private:
std::unique_ptr<uint8_t[]> buffer_;
@@ -43,19 +58,5 @@
size_t buffer_size_;
};
-class XorSink final {
- public:
- void Initialize(BufferSink* sink, size_t size);
- void Reset();
- void* GetBuffer(size_t requested, size_t* actual);
- bool ReturnData(void* buffer, size_t len);
-
- private:
- BufferSink* bufsink_;
- std::unique_ptr<uint8_t[]> buffer_;
- size_t buffer_size_;
- size_t returned_;
-};
-
} // namespace snapshot
} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/snapuserd_buffer.cpp b/fs_mgr/libsnapshot/snapuserd/snapuserd_buffer.cpp
index ab763ab..35065e6 100644
--- a/fs_mgr/libsnapshot/snapuserd/snapuserd_buffer.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/snapuserd_buffer.cpp
@@ -15,6 +15,8 @@
*/
#include <snapuserd/snapuserd_buffer.h>
+
+#include <android-base/logging.h>
#include <snapuserd/snapuserd_kernel.h>
namespace android {
@@ -26,11 +28,23 @@
buffer_ = std::make_unique<uint8_t[]>(size);
}
-void* BufferSink::GetPayloadBuffer(size_t size) {
- if ((buffer_size_ - buffer_offset_) < size) return nullptr;
+void* BufferSink::AcquireBuffer(size_t size, size_t to_write) {
+ CHECK(to_write <= size);
+ void* ptr = GetPayloadBuffer(size);
+ if (!ptr) {
+ return nullptr;
+ }
+ UpdateBufferOffset(to_write);
+ return ptr;
+}
+
+void* BufferSink::GetPayloadBuffer(size_t size) {
char* buffer = reinterpret_cast<char*>(GetBufPtr());
struct dm_user_message* msg = (struct dm_user_message*)(&(buffer[0]));
+ if ((buffer_size_ - buffer_offset_ - sizeof(msg->header)) < size) {
+ return nullptr;
+ }
return (char*)msg->payload.buf + buffer_offset_;
}
@@ -59,38 +73,5 @@
return msg->payload.buf;
}
-void XorSink::Initialize(BufferSink* sink, size_t size) {
- bufsink_ = sink;
- buffer_size_ = size;
- returned_ = 0;
- buffer_ = std::make_unique<uint8_t[]>(size);
-}
-
-void XorSink::Reset() {
- returned_ = 0;
-}
-
-void* XorSink::GetBuffer(size_t requested, size_t* actual) {
- if (requested > buffer_size_) {
- *actual = buffer_size_;
- } else {
- *actual = requested;
- }
- return buffer_.get();
-}
-
-bool XorSink::ReturnData(void* buffer, size_t len) {
- uint8_t* xor_data = reinterpret_cast<uint8_t*>(buffer);
- uint8_t* buff = reinterpret_cast<uint8_t*>(bufsink_->GetPayloadBuffer(len + returned_));
- if (buff == nullptr) {
- return false;
- }
- for (size_t i = 0; i < len; i++) {
- buff[returned_ + i] ^= xor_data[i];
- }
- returned_ += len;
- return true;
-}
-
} // namespace snapshot
} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp
index bdba5c0..4105b4b 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/handler_manager.cpp
@@ -18,7 +18,9 @@
#include <android-base/logging.h>
+#include "read_worker.h"
#include "snapuserd_core.h"
+#include "snapuserd_merge.h"
namespace android {
namespace snapshot {
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_dm_user.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
similarity index 73%
rename from fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_dm_user.cpp
rename to fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
index 2b9d14e..7d2e3a6 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_dm_user.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.cpp
@@ -14,6 +14,8 @@
* limitations under the License.
*/
+#include "read_worker.h"
+
#include "snapuserd_core.h"
namespace android {
@@ -23,64 +25,24 @@
using namespace android::dm;
using android::base::unique_fd;
-Worker::Worker(const std::string& cow_device, const std::string& backing_device,
- const std::string& control_device, const std::string& misc_name,
- const std::string& base_path_merge, std::shared_ptr<SnapshotHandler> snapuserd) {
- cow_device_ = cow_device;
- backing_store_device_ = backing_device;
- control_device_ = control_device;
- misc_name_ = misc_name;
- base_path_merge_ = base_path_merge;
- snapuserd_ = snapuserd;
+void ReadWorker::CloseFds() {
+ ctrl_fd_ = {};
+ backing_store_fd_ = {};
+ Worker::CloseFds();
}
-bool Worker::InitializeFds() {
- backing_store_fd_.reset(open(backing_store_device_.c_str(), O_RDONLY));
- if (backing_store_fd_ < 0) {
- SNAP_PLOG(ERROR) << "Open Failed: " << backing_store_device_;
- return false;
- }
-
- cow_fd_.reset(open(cow_device_.c_str(), O_RDWR));
- if (cow_fd_ < 0) {
- SNAP_PLOG(ERROR) << "Open Failed: " << cow_device_;
- return false;
- }
-
- ctrl_fd_.reset(open(control_device_.c_str(), O_RDWR));
- if (ctrl_fd_ < 0) {
- SNAP_PLOG(ERROR) << "Unable to open " << control_device_;
- return false;
- }
-
- // Base device used by merge thread
- base_path_merge_fd_.reset(open(base_path_merge_.c_str(), O_RDWR));
- if (base_path_merge_fd_ < 0) {
- SNAP_PLOG(ERROR) << "Open Failed: " << base_path_merge_;
- return false;
- }
-
- return true;
-}
-
-bool Worker::InitReader() {
- reader_ = snapuserd_->CloneReaderForWorker();
-
- if (!reader_->InitForMerge(std::move(cow_fd_))) {
- return false;
- }
- return true;
-}
+ReadWorker::ReadWorker(const std::string& cow_device, const std::string& backing_device,
+ const std::string& control_device, const std::string& misc_name,
+ const std::string& base_path_merge,
+ std::shared_ptr<SnapshotHandler> snapuserd)
+ : Worker(cow_device, misc_name, base_path_merge, snapuserd),
+ backing_store_device_(backing_device),
+ control_device_(control_device) {}
// Start the replace operation. This will read the
// internal COW format and if the block is compressed,
// it will be de-compressed.
-bool Worker::ProcessReplaceOp(const CowOperation* cow_op) {
- void* buffer = bufsink_.GetPayloadBuffer(BLOCK_SZ);
- if (!buffer) {
- SNAP_LOG(ERROR) << "ProcessReplaceOp failed to allocate buffer";
- return false;
- }
+bool ReadWorker::ProcessReplaceOp(const CowOperation* cow_op, void* buffer) {
if (!reader_->ReadData(cow_op, buffer, BLOCK_SZ)) {
SNAP_LOG(ERROR) << "ProcessReplaceOp failed for block " << cow_op->new_block;
return false;
@@ -88,12 +50,7 @@
return true;
}
-bool Worker::ReadFromSourceDevice(const CowOperation* cow_op) {
- void* buffer = bufsink_.GetPayloadBuffer(BLOCK_SZ);
- if (buffer == nullptr) {
- SNAP_LOG(ERROR) << "ReadFromBaseDevice: Failed to get payload buffer";
- return false;
- }
+bool ReadWorker::ReadFromSourceDevice(const CowOperation* cow_op, void* buffer) {
uint64_t offset;
if (!reader_->GetSourceOffset(cow_op, &offset)) {
SNAP_LOG(ERROR) << "ReadFromSourceDevice: Failed to get source offset";
@@ -118,60 +75,43 @@
// Start the copy operation. This will read the backing
// block device which is represented by cow_op->source.
-bool Worker::ProcessCopyOp(const CowOperation* cow_op) {
- if (!ReadFromSourceDevice(cow_op)) {
+bool ReadWorker::ProcessCopyOp(const CowOperation* cow_op, void* buffer) {
+ if (!ReadFromSourceDevice(cow_op, buffer)) {
return false;
}
-
return true;
}
-bool Worker::ProcessXorOp(const CowOperation* cow_op) {
- if (!ReadFromSourceDevice(cow_op)) {
+bool ReadWorker::ProcessXorOp(const CowOperation* cow_op, void* buffer) {
+ if (!ReadFromSourceDevice(cow_op, buffer)) {
return false;
}
- xorsink_.Reset();
-
- size_t actual = 0;
- void* buffer = xorsink_.GetBuffer(BLOCK_SZ, &actual);
- if (!buffer || actual < BLOCK_SZ) {
- SNAP_LOG(ERROR) << "ProcessXorOp failed to get buffer of " << BLOCK_SZ << " size, got "
- << actual;
- return false;
+ if (xor_buffer_.empty()) {
+ xor_buffer_.resize(BLOCK_SZ);
}
- ssize_t size = reader_->ReadData(cow_op, buffer, BLOCK_SZ);
+ CHECK(xor_buffer_.size() == BLOCK_SZ);
+
+ ssize_t size = reader_->ReadData(cow_op, xor_buffer_.data(), xor_buffer_.size());
if (size != BLOCK_SZ) {
SNAP_LOG(ERROR) << "ProcessXorOp failed for block " << cow_op->new_block
<< ", return value: " << size;
return false;
}
- if (!xorsink_.ReturnData(buffer, size)) {
- SNAP_LOG(ERROR) << "ProcessXorOp failed to return data";
- return false;
+
+ auto xor_out = reinterpret_cast<uint8_t*>(buffer);
+ for (size_t i = 0; i < BLOCK_SZ; i++) {
+ xor_out[i] ^= xor_buffer_[i];
}
return true;
}
-bool Worker::ProcessZeroOp() {
- // Zero out the entire block
- void* buffer = bufsink_.GetPayloadBuffer(BLOCK_SZ);
- if (buffer == nullptr) {
- SNAP_LOG(ERROR) << "ProcessZeroOp: Failed to get payload buffer";
- return false;
- }
-
+bool ReadWorker::ProcessZeroOp(void* buffer) {
memset(buffer, 0, BLOCK_SZ);
return true;
}
-bool Worker::ProcessOrderedOp(const CowOperation* cow_op) {
- void* buffer = bufsink_.GetPayloadBuffer(BLOCK_SZ);
- if (buffer == nullptr) {
- SNAP_LOG(ERROR) << "ProcessOrderedOp: Failed to get payload buffer";
- return false;
- }
-
+bool ReadWorker::ProcessOrderedOp(const CowOperation* cow_op, void* buffer) {
MERGE_GROUP_STATE state = snapuserd_->ProcessMergingBlock(cow_op->new_block, buffer);
switch (state) {
@@ -181,7 +121,7 @@
SNAP_LOG(DEBUG) << "Merge-completed: Reading from base device sector: "
<< (cow_op->new_block >> SECTOR_SHIFT)
<< " Block-number: " << cow_op->new_block;
- if (!ReadDataFromBaseDevice(ChunkToSector(cow_op->new_block), BLOCK_SZ)) {
+ if (!ReadDataFromBaseDevice(ChunkToSector(cow_op->new_block), buffer, BLOCK_SZ)) {
SNAP_LOG(ERROR) << "ReadDataFromBaseDevice at sector: "
<< (cow_op->new_block >> SECTOR_SHIFT) << " after merge-complete.";
return false;
@@ -191,9 +131,9 @@
case MERGE_GROUP_STATE::GROUP_MERGE_PENDING: {
bool ret;
if (cow_op->type == kCowCopyOp) {
- ret = ProcessCopyOp(cow_op);
+ ret = ProcessCopyOp(cow_op, buffer);
} else {
- ret = ProcessXorOp(cow_op);
+ ret = ProcessXorOp(cow_op, buffer);
}
// I/O is complete - decrement the refcount irrespective of the return
@@ -218,7 +158,7 @@
return false;
}
-bool Worker::ProcessCowOp(const CowOperation* cow_op) {
+bool ReadWorker::ProcessCowOp(const CowOperation* cow_op, void* buffer) {
if (cow_op == nullptr) {
SNAP_LOG(ERROR) << "ProcessCowOp: Invalid cow_op";
return false;
@@ -226,17 +166,17 @@
switch (cow_op->type) {
case kCowReplaceOp: {
- return ProcessReplaceOp(cow_op);
+ return ProcessReplaceOp(cow_op, buffer);
}
case kCowZeroOp: {
- return ProcessZeroOp();
+ return ProcessZeroOp(buffer);
}
case kCowCopyOp:
[[fallthrough]];
case kCowXorOp: {
- return ProcessOrderedOp(cow_op);
+ return ProcessOrderedOp(cow_op, buffer);
}
default: {
@@ -246,31 +186,26 @@
return false;
}
-void Worker::InitializeBufsink() {
- // Allocate the buffer which is used to communicate between
- // daemon and dm-user. The buffer comprises of header and a fixed payload.
- // If the dm-user requests a big IO, the IO will be broken into chunks
- // of PAYLOAD_BUFFER_SZ.
- size_t buf_size = sizeof(struct dm_user_header) + PAYLOAD_BUFFER_SZ;
- bufsink_.Initialize(buf_size);
-}
-
-bool Worker::Init() {
- InitializeBufsink();
- xorsink_.Initialize(&bufsink_, BLOCK_SZ);
-
- if (!InitializeFds()) {
+bool ReadWorker::Init() {
+ if (!Worker::Init()) {
return false;
}
- if (!InitReader()) {
+ backing_store_fd_.reset(open(backing_store_device_.c_str(), O_RDONLY));
+ if (backing_store_fd_ < 0) {
+ SNAP_PLOG(ERROR) << "Open Failed: " << backing_store_device_;
return false;
}
+ ctrl_fd_.reset(open(control_device_.c_str(), O_RDWR));
+ if (ctrl_fd_ < 0) {
+ SNAP_PLOG(ERROR) << "Unable to open " << control_device_;
+ return false;
+ }
return true;
}
-bool Worker::RunThread() {
+bool ReadWorker::Run() {
SNAP_LOG(INFO) << "Processing snapshot I/O requests....";
if (setpriority(PRIO_PROCESS, gettid(), kNiceValueForMergeThreads)) {
@@ -291,7 +226,7 @@
}
// Send the payload/data back to dm-user misc device.
-bool Worker::WriteDmUserPayload(size_t size) {
+bool ReadWorker::WriteDmUserPayload(size_t size) {
size_t payload_size = size;
void* buf = bufsink_.GetPayloadBufPtr();
if (header_response_) {
@@ -307,18 +242,15 @@
// After the first header is sent in response to a request, we cannot
// send any additional headers.
header_response_ = false;
+
+ // Reset the buffer for use by the next request.
+ bufsink_.ResetBufferOffset();
return true;
}
-bool Worker::ReadDataFromBaseDevice(sector_t sector, size_t read_size) {
+bool ReadWorker::ReadDataFromBaseDevice(sector_t sector, void* buffer, size_t read_size) {
CHECK(read_size <= BLOCK_SZ);
- void* buffer = bufsink_.GetPayloadBuffer(BLOCK_SZ);
- if (buffer == nullptr) {
- SNAP_LOG(ERROR) << "ReadFromBaseDevice: Failed to get payload buffer";
- return false;
- }
-
loff_t offset = sector << SECTOR_SHIFT;
if (!android::base::ReadFullyAtOffset(base_path_merge_fd_, buffer, read_size, offset)) {
SNAP_PLOG(ERROR) << "ReadDataFromBaseDevice failed. fd: " << base_path_merge_fd_
@@ -329,7 +261,7 @@
return true;
}
-bool Worker::ReadAlignedSector(sector_t sector, size_t sz) {
+bool ReadWorker::ReadAlignedSector(sector_t sector, size_t sz) {
size_t remaining_size = sz;
std::vector<std::pair<sector_t, const CowOperation*>>& chunk_vec = snapuserd_->GetChunkVec();
int ret = 0;
@@ -339,7 +271,6 @@
size_t read_size = std::min(PAYLOAD_BUFFER_SZ, remaining_size);
size_t total_bytes_read = 0;
- bufsink_.ResetBufferOffset();
while (read_size) {
// We need to check every 4k block to verify if it is
@@ -350,11 +281,17 @@
std::make_pair(sector, nullptr), SnapshotHandler::compare);
bool not_found = (it == chunk_vec.end() || it->first != sector);
+ void* buffer = bufsink_.AcquireBuffer(BLOCK_SZ, size);
+ if (!buffer) {
+ SNAP_LOG(ERROR) << "AcquireBuffer failed in ReadAlignedSector";
+ return false;
+ }
+
if (not_found) {
// Block not found in map - which means this block was not
// changed as per the OTA. Just route the I/O to the base
// device.
- if (!ReadDataFromBaseDevice(sector, size)) {
+ if (!ReadDataFromBaseDevice(sector, buffer, size)) {
SNAP_LOG(ERROR) << "ReadDataFromBaseDevice failed";
return false;
}
@@ -363,25 +300,24 @@
} else {
// We found the sector in mapping. Check the type of COW OP and
// process it.
- if (!ProcessCowOp(it->second)) {
+ if (!ProcessCowOp(it->second, buffer)) {
SNAP_LOG(ERROR) << "ProcessCowOp failed";
return false;
}
- ret = BLOCK_SZ;
+ ret = std::min(BLOCK_SZ, read_size);
}
read_size -= ret;
total_bytes_read += ret;
sector += (ret >> SECTOR_SHIFT);
- bufsink_.UpdateBufferOffset(ret);
}
- if (!WriteDmUserPayload(total_bytes_read)) {
+ if (!SendBufferedIo()) {
return false;
}
- SNAP_LOG(DEBUG) << "WriteDmUserPayload success total_bytes_read: " << total_bytes_read
+ SNAP_LOG(DEBUG) << "SendBufferedIo success total_bytes_read: " << total_bytes_read
<< " remaining_size: " << remaining_size;
remaining_size -= total_bytes_read;
} while (remaining_size > 0);
@@ -389,43 +325,39 @@
return true;
}
-int Worker::ReadUnalignedSector(
+int ReadWorker::ReadUnalignedSector(
sector_t sector, size_t size,
std::vector<std::pair<sector_t, const CowOperation*>>::iterator& it) {
- size_t skip_sector_size = 0;
-
SNAP_LOG(DEBUG) << "ReadUnalignedSector: sector " << sector << " size: " << size
<< " Aligned sector: " << it->first;
- if (!ProcessCowOp(it->second)) {
+ int num_sectors_skip = sector - it->first;
+ size_t skip_size = num_sectors_skip << SECTOR_SHIFT;
+ size_t write_size = std::min(size, BLOCK_SZ - skip_size);
+ auto buffer = reinterpret_cast<uint8_t*>(bufsink_.AcquireBuffer(BLOCK_SZ, write_size));
+ if (!buffer) {
+ SNAP_LOG(ERROR) << "ProcessCowOp failed to allocate buffer";
+ return -1;
+ }
+
+ if (!ProcessCowOp(it->second, buffer)) {
SNAP_LOG(ERROR) << "ReadUnalignedSector: " << sector << " failed of size: " << size
<< " Aligned sector: " << it->first;
return -1;
}
- int num_sectors_skip = sector - it->first;
-
- if (num_sectors_skip > 0) {
- skip_sector_size = num_sectors_skip << SECTOR_SHIFT;
- char* buffer = reinterpret_cast<char*>(bufsink_.GetBufPtr());
- struct dm_user_message* msg = (struct dm_user_message*)(&(buffer[0]));
-
- if (skip_sector_size == BLOCK_SZ) {
+ if (skip_size) {
+ if (skip_size == BLOCK_SZ) {
SNAP_LOG(ERROR) << "Invalid un-aligned IO request at sector: " << sector
<< " Base-sector: " << it->first;
return -1;
}
-
- memmove(msg->payload.buf, (char*)msg->payload.buf + skip_sector_size,
- (BLOCK_SZ - skip_sector_size));
+ memmove(buffer, buffer + skip_size, write_size);
}
-
- bufsink_.ResetBufferOffset();
- return std::min(size, (BLOCK_SZ - skip_sector_size));
+ return write_size;
}
-bool Worker::ReadUnalignedSector(sector_t sector, size_t size) {
- bufsink_.ResetBufferOffset();
+bool ReadWorker::ReadUnalignedSector(sector_t sector, size_t size) {
std::vector<std::pair<sector_t, const CowOperation*>>& chunk_vec = snapuserd_->GetChunkVec();
auto it = std::lower_bound(chunk_vec.begin(), chunk_vec.end(), std::make_pair(sector, nullptr),
@@ -494,7 +426,6 @@
// requested offset in this case is beyond the last mapped COW op size (which
// is block 1 in this case).
- size_t total_bytes_read = 0;
size_t remaining_size = size;
int ret = 0;
if (!merge_complete && (requested_offset >= final_offset) &&
@@ -508,11 +439,10 @@
}
remaining_size -= ret;
- total_bytes_read += ret;
sector += (ret >> SECTOR_SHIFT);
// Send the data back
- if (!WriteDmUserPayload(total_bytes_read)) {
+ if (!SendBufferedIo()) {
return false;
}
@@ -530,26 +460,21 @@
// Find the diff of the aligned offset
size_t diff_size = aligned_offset - requested_offset;
CHECK(diff_size <= BLOCK_SZ);
- if (remaining_size < diff_size) {
- if (!ReadDataFromBaseDevice(sector, remaining_size)) {
- return false;
- }
- total_bytes_read += remaining_size;
- if (!WriteDmUserPayload(total_bytes_read)) {
- return false;
- }
- } else {
- if (!ReadDataFromBaseDevice(sector, diff_size)) {
- return false;
- }
+ size_t read_size = std::min(remaining_size, diff_size);
+ void* buffer = bufsink_.AcquireBuffer(BLOCK_SZ, read_size);
+ if (!buffer) {
+ SNAP_LOG(ERROR) << "AcquireBuffer failed in ReadUnalignedSector";
+ return false;
+ }
+ if (!ReadDataFromBaseDevice(sector, buffer, read_size)) {
+ return false;
+ }
+ if (!SendBufferedIo()) {
+ return false;
+ }
- total_bytes_read += diff_size;
-
- if (!WriteDmUserPayload(total_bytes_read)) {
- return false;
- }
-
+ if (remaining_size >= diff_size) {
remaining_size -= diff_size;
size_t num_sectors_read = (diff_size >> SECTOR_SHIFT);
sector += num_sectors_read;
@@ -563,7 +488,7 @@
return true;
}
-void Worker::RespondIOError() {
+void ReadWorker::RespondIOError() {
struct dm_user_header* header = bufsink_.GetHeaderPtr();
header->type = DM_USER_RESP_ERROR;
// This is an issue with the dm-user interface. There
@@ -580,7 +505,7 @@
WriteDmUserPayload(0);
}
-bool Worker::DmuserReadRequest() {
+bool ReadWorker::DmuserReadRequest() {
struct dm_user_header* header = bufsink_.GetHeaderPtr();
// Unaligned I/O request
@@ -591,7 +516,11 @@
return ReadAlignedSector(header->sector, header->len);
}
-bool Worker::ProcessIORequest() {
+bool ReadWorker::SendBufferedIo() {
+ return WriteDmUserPayload(bufsink_.GetPayloadBytesWritten());
+}
+
+bool ReadWorker::ProcessIORequest() {
// Read Header from dm-user misc device. This gives
// us the sector number for which IO is issued by dm-snapshot device
struct dm_user_header* header = bufsink_.GetHeaderPtr();
@@ -615,6 +544,9 @@
header->type = DM_USER_RESP_SUCCESS;
header_response_ = true;
+ // Reset the output buffer.
+ bufsink_.ResetBufferOffset();
+
bool ok;
switch (request_type) {
case DM_USER_REQ_MAP_READ:
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.h
new file mode 100644
index 0000000..8d6f663
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/read_worker.h
@@ -0,0 +1,71 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <utility>
+#include <vector>
+
+#include "worker.h"
+
+namespace android {
+namespace snapshot {
+
+class ReadWorker : public Worker {
+ public:
+ ReadWorker(const std::string& cow_device, const std::string& backing_device,
+ const std::string& control_device, const std::string& misc_name,
+ const std::string& base_path_merge, std::shared_ptr<SnapshotHandler> snapuserd);
+
+ bool Run();
+ bool Init() override;
+ void CloseFds() override;
+
+ private:
+ // Functions interacting with dm-user
+ bool ProcessIORequest();
+ bool WriteDmUserPayload(size_t size);
+ bool DmuserReadRequest();
+ bool SendBufferedIo();
+ void RespondIOError();
+
+ bool ProcessCowOp(const CowOperation* cow_op, void* buffer);
+ bool ProcessXorOp(const CowOperation* cow_op, void* buffer);
+ bool ProcessOrderedOp(const CowOperation* cow_op, void* buffer);
+ bool ProcessCopyOp(const CowOperation* cow_op, void* buffer);
+ bool ProcessReplaceOp(const CowOperation* cow_op, void* buffer);
+ bool ProcessZeroOp(void* buffer);
+
+ bool ReadAlignedSector(sector_t sector, size_t sz);
+ bool ReadUnalignedSector(sector_t sector, size_t size);
+ int ReadUnalignedSector(sector_t sector, size_t size,
+ std::vector<std::pair<sector_t, const CowOperation*>>::iterator& it);
+ bool ReadFromSourceDevice(const CowOperation* cow_op, void* buffer);
+ bool ReadDataFromBaseDevice(sector_t sector, void* buffer, size_t read_size);
+
+ constexpr bool IsBlockAligned(size_t size) { return ((size & (BLOCK_SZ - 1)) == 0); }
+ constexpr sector_t ChunkToSector(chunk_t chunk) { return chunk << CHUNK_SHIFT; }
+
+ std::string backing_store_device_;
+ unique_fd backing_store_fd_;
+
+ std::string control_device_;
+ unique_fd ctrl_fd_;
+
+ bool header_response_ = false;
+ std::basic_string<uint8_t> xor_buffer_;
+};
+
+} // namespace snapshot
+} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
index 8e1212b..e52d752 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
@@ -23,6 +23,9 @@
#include <android-base/scopeguard.h>
#include <android-base/strings.h>
+#include "read_worker.h"
+#include "snapuserd_merge.h"
+
namespace android {
namespace snapshot {
@@ -46,9 +49,8 @@
bool SnapshotHandler::InitializeWorkers() {
for (int i = 0; i < num_worker_threads_; i++) {
- std::unique_ptr<Worker> wt =
- std::make_unique<Worker>(cow_device_, backing_store_device_, control_device_,
- misc_name_, base_path_merge_, GetSharedPtr());
+ auto wt = std::make_unique<ReadWorker>(cow_device_, backing_store_device_, control_device_,
+ misc_name_, base_path_merge_, GetSharedPtr());
if (!wt->Init()) {
SNAP_LOG(ERROR) << "Thread initialization failed";
return false;
@@ -57,8 +59,8 @@
worker_threads_.push_back(std::move(wt));
}
- merge_thread_ = std::make_unique<Worker>(cow_device_, backing_store_device_, control_device_,
- misc_name_, base_path_merge_, GetSharedPtr());
+ merge_thread_ = std::make_unique<MergeWorker>(cow_device_, misc_name_, base_path_merge_,
+ GetSharedPtr());
read_ahead_thread_ = std::make_unique<ReadAhead>(cow_device_, backing_store_device_, misc_name_,
GetSharedPtr());
@@ -312,11 +314,11 @@
// Launch worker threads
for (int i = 0; i < worker_threads_.size(); i++) {
threads.emplace_back(
- std::async(std::launch::async, &Worker::RunThread, worker_threads_[i].get()));
+ std::async(std::launch::async, &ReadWorker::Run, worker_threads_[i].get()));
}
std::future<bool> merge_thread =
- std::async(std::launch::async, &Worker::RunMergeThread, merge_thread_.get());
+ std::async(std::launch::async, &MergeWorker::Run, merge_thread_.get());
// Now that the worker threads are up, scan the partitions.
if (perform_verification_) {
@@ -452,5 +454,11 @@
return update_verify_->CheckPartitionVerification();
}
+void SnapshotHandler::FreeResources() {
+ worker_threads_.clear();
+ read_ahead_thread_ = nullptr;
+ merge_thread_ = nullptr;
+}
+
} // namespace snapshot
} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
index c984a61..cdc38c0 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
@@ -75,7 +75,8 @@
READ_AHEAD_FAILURE,
};
-class SnapshotHandler;
+class MergeWorker;
+class ReadWorker;
enum class MERGE_GROUP_STATE {
GROUP_MERGE_PENDING,
@@ -98,102 +99,6 @@
: merge_state_(state), num_ios_in_progress(n_ios) {}
};
-class Worker {
- public:
- Worker(const std::string& cow_device, const std::string& backing_device,
- const std::string& control_device, const std::string& misc_name,
- const std::string& base_path_merge, std::shared_ptr<SnapshotHandler> snapuserd);
- bool RunThread();
- bool RunMergeThread();
- bool Init();
-
- private:
- // Initialization
- void InitializeBufsink();
- bool InitializeFds();
- bool InitReader();
- void CloseFds() {
- ctrl_fd_ = {};
- backing_store_fd_ = {};
- base_path_merge_fd_ = {};
- }
-
- // Functions interacting with dm-user
- bool WriteDmUserPayload(size_t size);
- bool DmuserReadRequest();
-
- // IO Path
- bool ProcessIORequest();
- bool IsBlockAligned(size_t size) { return ((size & (BLOCK_SZ - 1)) == 0); }
-
- bool ReadDataFromBaseDevice(sector_t sector, size_t read_size);
- bool ReadFromSourceDevice(const CowOperation* cow_op);
-
- bool ReadAlignedSector(sector_t sector, size_t sz);
- bool ReadUnalignedSector(sector_t sector, size_t size);
- int ReadUnalignedSector(sector_t sector, size_t size,
- std::vector<std::pair<sector_t, const CowOperation*>>::iterator& it);
- void RespondIOError();
-
- // Processing COW operations
- bool ProcessCowOp(const CowOperation* cow_op);
- bool ProcessReplaceOp(const CowOperation* cow_op);
- bool ProcessZeroOp();
-
- // Handles Copy and Xor
- bool ProcessCopyOp(const CowOperation* cow_op);
- bool ProcessXorOp(const CowOperation* cow_op);
- bool ProcessOrderedOp(const CowOperation* cow_op);
-
- // Merge related ops
- bool Merge();
- bool AsyncMerge();
- bool SyncMerge();
- bool MergeOrderedOps();
- bool MergeOrderedOpsAsync();
- bool MergeReplaceZeroOps();
- int PrepareMerge(uint64_t* source_offset, int* pending_ops,
- std::vector<const CowOperation*>* replace_zero_vec = nullptr);
-
- sector_t ChunkToSector(chunk_t chunk) { return chunk << CHUNK_SHIFT; }
- chunk_t SectorToChunk(sector_t sector) { return sector >> CHUNK_SHIFT; }
-
- bool InitializeIouring();
- void FinalizeIouring();
-
- std::unique_ptr<CowReader> reader_;
- BufferSink bufsink_;
- XorSink xorsink_;
-
- std::string cow_device_;
- std::string backing_store_device_;
- std::string control_device_;
- std::string misc_name_;
- std::string base_path_merge_;
-
- unique_fd cow_fd_;
- unique_fd backing_store_fd_;
- unique_fd base_path_merge_fd_;
- unique_fd ctrl_fd_;
- bool header_response_ = false;
-
- std::unique_ptr<ICowOpIter> cowop_iter_;
- size_t ra_block_index_ = 0;
- uint64_t blocks_merged_in_group_ = 0;
- bool merge_async_ = false;
- // Queue depth of 8 seems optimal. We don't want
- // to have a huge depth as it may put more memory pressure
- // on the kernel worker threads given that we use
- // IOSQE_ASYNC flag - ASYNC flags can potentially
- // result in EINTR; Since we don't restart
- // syscalls and fallback to synchronous I/O, we
- // don't want huge queue depth
- int queue_depth_ = 8;
- std::unique_ptr<struct io_uring> ring_;
-
- std::shared_ptr<SnapshotHandler> snapuserd_;
-};
-
class SnapshotHandler : public std::enable_shared_from_this<SnapshotHandler> {
public:
SnapshotHandler(std::string misc_name, std::string cow_device, std::string backing_device,
@@ -212,11 +117,7 @@
bool CommitMerge(int num_merge_ops);
void CloseFds() { cow_fd_ = {}; }
- void FreeResources() {
- worker_threads_.clear();
- read_ahead_thread_ = nullptr;
- merge_thread_ = nullptr;
- }
+ void FreeResources();
bool InitializeWorkers();
std::unique_ptr<CowReader> CloneReaderForWorker();
@@ -315,7 +216,7 @@
void* mapped_addr_;
size_t total_mapped_addr_length_;
- std::vector<std::unique_ptr<Worker>> worker_threads_;
+ std::vector<std::unique_ptr<ReadWorker>> worker_threads_;
// Read-ahead related
bool populate_data_from_cow_ = false;
bool ra_thread_ = false;
@@ -330,7 +231,7 @@
// Merge Block state
std::vector<std::unique_ptr<MergeGroupState>> merge_blk_state_;
- std::unique_ptr<Worker> merge_thread_;
+ std::unique_ptr<MergeWorker> merge_thread_;
double merge_completion_percentage_;
bool merge_initiated_ = false;
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp
index ce95b76..517148d 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+#include "snapuserd_merge.h"
#include "snapuserd_core.h"
@@ -23,8 +24,13 @@
using namespace android::dm;
using android::base::unique_fd;
-int Worker::PrepareMerge(uint64_t* source_offset, int* pending_ops,
- std::vector<const CowOperation*>* replace_zero_vec) {
+MergeWorker::MergeWorker(const std::string& cow_device, const std::string& misc_name,
+ const std::string& base_path_merge,
+ std::shared_ptr<SnapshotHandler> snapuserd)
+ : Worker(cow_device, misc_name, base_path_merge, snapuserd) {}
+
+int MergeWorker::PrepareMerge(uint64_t* source_offset, int* pending_ops,
+ std::vector<const CowOperation*>* replace_zero_vec) {
int num_ops = *pending_ops;
int nr_consecutive = 0;
bool checkOrderedOp = (replace_zero_vec == nullptr);
@@ -70,7 +76,7 @@
return nr_consecutive;
}
-bool Worker::MergeReplaceZeroOps() {
+bool MergeWorker::MergeReplaceZeroOps() {
// Flush after merging 2MB. Since all ops are independent and there is no
// dependency between COW ops, we will flush the data and the number
// of ops merged in COW block device. If there is a crash, we will
@@ -99,20 +105,21 @@
for (size_t i = 0; i < replace_zero_vec.size(); i++) {
const CowOperation* cow_op = replace_zero_vec[i];
+
+ void* buffer = bufsink_.AcquireBuffer(BLOCK_SZ);
+ if (!buffer) {
+ SNAP_LOG(ERROR) << "AcquireBuffer failed in MergeReplaceOps";
+ return false;
+ }
if (cow_op->type == kCowReplaceOp) {
- if (!ProcessReplaceOp(cow_op)) {
- SNAP_LOG(ERROR) << "Merge - ReplaceOp failed for block: " << cow_op->new_block;
+ if (!reader_->ReadData(cow_op, buffer, BLOCK_SZ)) {
+ SNAP_LOG(ERROR) << "Failed to read COW in merge";
return false;
}
} else {
CHECK(cow_op->type == kCowZeroOp);
- if (!ProcessZeroOp()) {
- SNAP_LOG(ERROR) << "Merge ZeroOp failed.";
- return false;
- }
+ memset(buffer, 0, BLOCK_SZ);
}
-
- bufsink_.UpdateBufferOffset(BLOCK_SZ);
}
size_t io_size = linear_blocks * BLOCK_SZ;
@@ -149,7 +156,7 @@
if (snapuserd_->IsIOTerminated()) {
SNAP_LOG(ERROR)
- << "MergeReplaceZeroOps: Worker threads terminated - shutting down merge";
+ << "MergeReplaceZeroOps: MergeWorker threads terminated - shutting down merge";
return false;
}
}
@@ -173,7 +180,7 @@
return true;
}
-bool Worker::MergeOrderedOpsAsync() {
+bool MergeWorker::MergeOrderedOpsAsync() {
void* mapped_addr = snapuserd_->GetMappedAddr();
void* read_ahead_buffer =
static_cast<void*>((char*)mapped_addr + snapuserd_->GetBufferDataOffset());
@@ -354,7 +361,7 @@
return true;
}
-bool Worker::MergeOrderedOps() {
+bool MergeWorker::MergeOrderedOps() {
void* mapped_addr = snapuserd_->GetMappedAddr();
void* read_ahead_buffer =
static_cast<void*>((char*)mapped_addr + snapuserd_->GetBufferDataOffset());
@@ -439,7 +446,7 @@
return true;
}
-bool Worker::AsyncMerge() {
+bool MergeWorker::AsyncMerge() {
if (!MergeOrderedOpsAsync()) {
SNAP_LOG(ERROR) << "MergeOrderedOpsAsync failed - Falling back to synchronous I/O";
// Reset the iter so that we retry the merge
@@ -455,7 +462,7 @@
return true;
}
-bool Worker::SyncMerge() {
+bool MergeWorker::SyncMerge() {
if (!MergeOrderedOps()) {
SNAP_LOG(ERROR) << "Merge failed for ordered ops";
return false;
@@ -465,7 +472,7 @@
return true;
}
-bool Worker::Merge() {
+bool MergeWorker::Merge() {
cowop_iter_ = reader_->GetOpIter(true);
bool retry = false;
@@ -511,7 +518,7 @@
return true;
}
-bool Worker::InitializeIouring() {
+bool MergeWorker::InitializeIouring() {
if (!snapuserd_->IsIouringSupported()) {
return false;
}
@@ -530,13 +537,13 @@
return true;
}
-void Worker::FinalizeIouring() {
+void MergeWorker::FinalizeIouring() {
if (merge_async_) {
io_uring_queue_exit(ring_.get());
}
}
-bool Worker::RunMergeThread() {
+bool MergeWorker::Run() {
SNAP_LOG(DEBUG) << "Waiting for merge begin...";
if (!snapuserd_->WaitForMergeBegin()) {
SNAP_LOG(ERROR) << "Merge terminated early...";
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.h
new file mode 100644
index 0000000..f35147f
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.h
@@ -0,0 +1,58 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "worker.h"
+
+#include <liburing.h>
+
+namespace android {
+namespace snapshot {
+
+class MergeWorker : public Worker {
+ public:
+ MergeWorker(const std::string& cow_device, const std::string& misc_name,
+ const std::string& base_path_merge, std::shared_ptr<SnapshotHandler> snapuserd);
+ bool Run();
+
+ private:
+ int PrepareMerge(uint64_t* source_offset, int* pending_ops,
+ std::vector<const CowOperation*>* replace_zero_vec = nullptr);
+ bool MergeReplaceZeroOps();
+ bool MergeOrderedOps();
+ bool MergeOrderedOpsAsync();
+ bool Merge();
+ bool AsyncMerge();
+ bool SyncMerge();
+ bool InitializeIouring();
+ void FinalizeIouring();
+
+ private:
+ std::unique_ptr<ICowOpIter> cowop_iter_;
+ std::unique_ptr<struct io_uring> ring_;
+ size_t ra_block_index_ = 0;
+ uint64_t blocks_merged_in_group_ = 0;
+ bool merge_async_ = false;
+ // Queue depth of 8 seems optimal. We don't want
+ // to have a huge depth as it may put more memory pressure
+ // on the kernel worker threads given that we use
+ // IOSQE_ASYNC flag - ASYNC flags can potentially
+ // result in EINTR; Since we don't restart
+ // syscalls and fallback to synchronous I/O, we
+ // don't want huge queue depth
+ int queue_depth_ = 8;
+};
+
+} // namespace snapshot
+} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
index 399f7b8..3e9588b 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
@@ -492,7 +492,7 @@
if (xor_op_index < xor_op_vec.size()) {
const CowOperation* xor_op = xor_op_vec[xor_op_index];
if (xor_op->new_block == new_block) {
- void* buffer = bufsink_.GetPayloadBuffer(BLOCK_SZ);
+ void* buffer = bufsink_.AcquireBuffer(BLOCK_SZ);
if (!buffer) {
SNAP_LOG(ERROR) << "ReadAhead - failed to allocate buffer for block: "
<< xor_op->new_block;
@@ -506,7 +506,6 @@
}
xor_op_index += 1;
- bufsink_.UpdateBufferOffset(BLOCK_SZ);
}
}
block_index += 1;
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.cpp
new file mode 100644
index 0000000..aa15630
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.cpp
@@ -0,0 +1,80 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "worker.h"
+
+#include "snapuserd_core.h"
+
+namespace android {
+namespace snapshot {
+
+Worker::Worker(const std::string& cow_device, const std::string& misc_name,
+ const std::string& base_path_merge, std::shared_ptr<SnapshotHandler> snapuserd) {
+ cow_device_ = cow_device;
+ misc_name_ = misc_name;
+ base_path_merge_ = base_path_merge;
+ snapuserd_ = snapuserd;
+}
+
+void Worker::InitializeBufsink() {
+ // Allocate the buffer which is used to communicate between
+ // daemon and dm-user. The buffer comprises of header and a fixed payload.
+ // If the dm-user requests a big IO, the IO will be broken into chunks
+ // of PAYLOAD_BUFFER_SZ.
+ size_t buf_size = sizeof(struct dm_user_header) + PAYLOAD_BUFFER_SZ;
+ bufsink_.Initialize(buf_size);
+}
+
+bool Worker::Init() {
+ InitializeBufsink();
+
+ if (!InitializeFds()) {
+ return false;
+ }
+
+ if (!InitReader()) {
+ return false;
+ }
+
+ return true;
+}
+
+bool Worker::InitReader() {
+ reader_ = snapuserd_->CloneReaderForWorker();
+
+ if (!reader_->InitForMerge(std::move(cow_fd_))) {
+ return false;
+ }
+ return true;
+}
+
+bool Worker::InitializeFds() {
+ cow_fd_.reset(open(cow_device_.c_str(), O_RDWR));
+ if (cow_fd_ < 0) {
+ SNAP_PLOG(ERROR) << "Open Failed: " << cow_device_;
+ return false;
+ }
+
+ // Base device used by merge thread
+ base_path_merge_fd_.reset(open(base_path_merge_.c_str(), O_RDWR));
+ if (base_path_merge_fd_ < 0) {
+ SNAP_PLOG(ERROR) << "Open Failed: " << base_path_merge_;
+ return false;
+ }
+
+ return true;
+}
+
+} // namespace snapshot
+} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.h
new file mode 100644
index 0000000..813b159
--- /dev/null
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/worker.h
@@ -0,0 +1,65 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <stddef.h>
+
+#include <memory>
+#include <string>
+
+#include <android-base/unique_fd.h>
+#include <libsnapshot/cow_reader.h>
+#include <snapuserd/snapuserd_buffer.h>
+#include <snapuserd/snapuserd_kernel.h>
+
+namespace android {
+namespace snapshot {
+
+using android::base::unique_fd;
+
+class SnapshotHandler;
+
+class Worker {
+ public:
+ Worker(const std::string& cow_device, const std::string& misc_name,
+ const std::string& base_path_merge, std::shared_ptr<SnapshotHandler> snapuserd);
+ virtual ~Worker() = default;
+
+ virtual bool Init();
+
+ protected:
+ // Initialization
+ void InitializeBufsink();
+ bool InitializeFds();
+ bool InitReader();
+ virtual void CloseFds() { base_path_merge_fd_ = {}; }
+
+ std::unique_ptr<CowReader> reader_;
+ BufferSink bufsink_;
+
+ std::string misc_name_; // Needed for SNAP_LOG.
+
+ unique_fd base_path_merge_fd_;
+
+ std::shared_ptr<SnapshotHandler> snapuserd_;
+
+ private:
+ std::string cow_device_;
+ std::string base_path_merge_;
+ unique_fd cow_fd_;
+};
+
+} // namespace snapshot
+} // namespace android
diff --git a/init/Android.bp b/init/Android.bp
index 0b8c7dc..ee34215 100644
--- a/init/Android.bp
+++ b/init/Android.bp
@@ -166,11 +166,9 @@
"libbootloader_message",
"libc++fs",
"libcgrouprc_format",
- "libfsverity_init",
"liblmkd_utils",
"liblz4",
"libzstd",
- "libmini_keyctl_static",
"libmodprobe",
"libprocinfo",
"libprotobuf-cpp-lite",
diff --git a/init/devices.cpp b/init/devices.cpp
index d29ffd6..7c23492 100644
--- a/init/devices.cpp
+++ b/init/devices.cpp
@@ -568,6 +568,8 @@
return;
} else if (uevent.subsystem == "misc" && StartsWith(uevent.device_name, "dm-user/")) {
devpath = "/dev/dm-user/" + uevent.device_name.substr(8);
+ } else if (uevent.subsystem == "misc" && uevent.device_name == "vfio/vfio") {
+ devpath = "/dev/" + uevent.device_name;
} else {
devpath = "/dev/" + Basename(uevent.path);
}
diff --git a/init/selinux.cpp b/init/selinux.cpp
index e0ef491..51093d8 100644
--- a/init/selinux.cpp
+++ b/init/selinux.cpp
@@ -74,10 +74,8 @@
#include <android-base/unique_fd.h>
#include <fs_avb/fs_avb.h>
#include <fs_mgr.h>
-#include <fsverity_init.h>
#include <libgsi/libgsi.h>
#include <libsnapshot/snapshot.h>
-#include <mini_keyctl_utils.h>
#include <selinux/android.h>
#include <ziparchive/zip_archive.h>
@@ -510,7 +508,6 @@
constexpr const char* kSigningCertRelease =
"/system/etc/selinux/com.android.sepolicy.cert-release.der";
-constexpr const char* kFsVerityProcPath = "/proc/sys/fs/verity";
const std::string kSepolicyApexMetadataDir = "/metadata/sepolicy/";
const std::string kSepolicyApexSystemDir = "/system/etc/selinux/apex/";
const std::string kSepolicyZip = "SEPolicy.zip";
@@ -614,24 +611,6 @@
return {};
}
-Result<void> LoadSepolicyApexCerts() {
- key_serial_t keyring_id = android::GetKeyringId(".fs-verity");
- if (keyring_id < 0) {
- return Error() << "Failed to find .fs-verity keyring id";
- }
-
- // TODO(b/199914227) the release key should always exist. Once it's checked in, start
- // throwing an error here if it doesn't exist.
- if (access(kSigningCertRelease, F_OK) == 0) {
- LoadKeyFromFile(keyring_id, "fsv_sepolicy_apex_release", kSigningCertRelease);
- }
- return {};
-}
-
-Result<void> SepolicyFsVerityCheck() {
- return Error() << "TODO implement support for fsverity SEPolicy.";
-}
-
Result<void> SepolicyCheckSignature(const std::string& dir) {
std::string signature;
if (!android::base::ReadFileToString(dir + kSepolicySignature, &signature)) {
@@ -654,18 +633,7 @@
return verifySignature(sepolicyStr, signature, *releaseKey);
}
-Result<void> SepolicyVerify(const std::string& dir, bool supportsFsVerity) {
- if (supportsFsVerity) {
- auto fsVerityCheck = SepolicyFsVerityCheck();
- if (fsVerityCheck.ok()) {
- return fsVerityCheck;
- }
- // TODO(b/199914227) If the device supports fsverity, but we fail here, we should fail to
- // boot and not carry on. For now, fallback to a signature checkuntil the fsverity
- // logic is implemented.
- LOG(INFO) << "Falling back to standard signature check. " << fsVerityCheck.error();
- }
-
+Result<void> SepolicyVerify(const std::string& dir) {
auto sepolicySignature = SepolicyCheckSignature(dir);
if (!sepolicySignature.ok()) {
return Error() << "Apex SEPolicy failed signature check";
@@ -698,23 +666,19 @@
// 6. Sets selinux into enforcing mode and continues normal booting.
//
void PrepareApexSepolicy() {
- bool supportsFsVerity = access(kFsVerityProcPath, F_OK) == 0;
- if (supportsFsVerity) {
- auto loadSepolicyApexCerts = LoadSepolicyApexCerts();
- if (!loadSepolicyApexCerts.ok()) {
- // TODO(b/199914227) If the device supports fsverity, but we fail here, we should fail
- // to boot and not carry on. For now, fallback to a signature checkuntil the fsverity
- // logic is implemented.
- LOG(INFO) << loadSepolicyApexCerts.error();
- }
- }
// If apex sepolicy zip exists in /metadata/sepolicy, use that, otherwise use version on
- // /system.
- auto dir = (access((kSepolicyApexMetadataDir + kSepolicyZip).c_str(), F_OK) == 0)
- ? kSepolicyApexMetadataDir
- : kSepolicyApexSystemDir;
+ // /system. If neither exists, do nothing.
+ std::string dir;
+ if (access((kSepolicyApexMetadataDir + kSepolicyZip).c_str(), F_OK) == 0) {
+ dir = kSepolicyApexMetadataDir;
+ } else if (access((kSepolicyApexSystemDir + kSepolicyZip).c_str(), F_OK) == 0) {
+ dir = kSepolicyApexSystemDir;
+ } else {
+ LOG(INFO) << "APEX Sepolicy not found";
+ return;
+ }
- auto sepolicyVerify = SepolicyVerify(dir, supportsFsVerity);
+ auto sepolicyVerify = SepolicyVerify(dir);
if (!sepolicyVerify.ok()) {
LOG(INFO) << "Error: " << sepolicyVerify.error();
// If signature verification fails, fall back to version on /system.
diff --git a/libcutils/include/cutils/threads.h b/libcutils/include/cutils/threads.h
index 0082c6c..1886184 100644
--- a/libcutils/include/cutils/threads.h
+++ b/libcutils/include/cutils/threads.h
@@ -31,7 +31,7 @@
//
// Deprecated: use android::base::GetThreadId instead, which doesn't truncate on Mac/Windows.
//
-#if !defined(__GLIBC__) || __GLIBC__ >= 2 && __GLIBC_MINOR__ < 32
+#if !defined(__GLIBC__) || __GLIBC__ >= 2 && __GLIBC_MINOR__ < 30
extern pid_t gettid();
#endif
diff --git a/libcutils/threads.cpp b/libcutils/threads.cpp
index 6ece7a3..2638720 100644
--- a/libcutils/threads.cpp
+++ b/libcutils/threads.cpp
@@ -25,9 +25,9 @@
#include <windows.h>
#endif
-#if defined(__BIONIC__) || defined(__GLIBC__) && __GLIBC_MINOR__ >= 32
+#if defined(__BIONIC__) || defined(__GLIBC__) && __GLIBC_MINOR__ >= 30
// No definition needed for Android because we'll just pick up bionic's copy.
-// No definition needed for Glibc >= 2.32 because it exposes its own copy.
+// No definition needed for Glibc >= 2.30 because it exposes its own copy.
#else
pid_t gettid() {
#if defined(__APPLE__)
diff --git a/libutils/String8.cpp b/libutils/String8.cpp
index 82f5cb6..8d312b5 100644
--- a/libutils/String8.cpp
+++ b/libutils/String8.cpp
@@ -393,6 +393,11 @@
}
bool String8::removeAll(const char* other) {
+ ALOG_ASSERT(other, "String8::removeAll() requires a non-NULL string");
+
+ if (*other == '\0')
+ return true;
+
ssize_t index = find(other);
if (index < 0) return false;
diff --git a/libutils/String8_fuzz.cpp b/libutils/String8_fuzz.cpp
index faf49b6..e5dcd31 100644
--- a/libutils/String8_fuzz.cpp
+++ b/libutils/String8_fuzz.cpp
@@ -45,6 +45,8 @@
str1->toLower();
},
[](FuzzedDataProvider*, android::String8* str1, android::String8* str2) -> void {
+ if (str2->size() == 0) return;
+
str1->removeAll(str2->c_str());
},
[](FuzzedDataProvider*, android::String8* str1, android::String8* str2) -> void {
diff --git a/libutils/String8_test.cpp b/libutils/String8_test.cpp
index 1356cd0..35fd512 100644
--- a/libutils/String8_test.cpp
+++ b/libutils/String8_test.cpp
@@ -114,3 +114,21 @@
EXPECT_EQ(NO_MEMORY, s.append("baz", SIZE_MAX));
EXPECT_STREQ("foobar", s);
}
+
+TEST_F(String8Test, removeAll) {
+ String8 s("Hello, world!");
+
+ // NULL input should cause an assertion failure and error message in logcat
+ EXPECT_DEATH(s.removeAll(NULL), "");
+
+ // expect to return true and string content should remain unchanged
+ EXPECT_TRUE(s.removeAll(""));
+ EXPECT_STREQ("Hello, world!", s);
+
+ // expect to return false
+ EXPECT_FALSE(s.removeAll("x"));
+ EXPECT_STREQ("Hello, world!", s);
+
+ EXPECT_TRUE(s.removeAll("o"));
+ EXPECT_STREQ("Hell, wrld!", s);
+}
diff --git a/rootdir/init.rc b/rootdir/init.rc
index cdf67b9..d2499ef 100644
--- a/rootdir/init.rc
+++ b/rootdir/init.rc
@@ -604,8 +604,8 @@
chmod 0700 /metadata/vold
mkdir /metadata/password_slots 0771 root system
mkdir /metadata/bootstat 0750 system log
- mkdir /metadata/ota 0700 root system
- mkdir /metadata/ota/snapshots 0700 root system
+ mkdir /metadata/ota 0750 root system
+ mkdir /metadata/ota/snapshots 0750 root system
mkdir /metadata/userspacereboot 0770 root system
mkdir /metadata/watchdog 0770 root system
@@ -1021,13 +1021,9 @@
# Must start after 'derive_classpath' to have *CLASSPATH variables set.
start odsign
- # Before we can lock keys and proceed to the next boot stage, wait for
- # odsign to be done with the key
+ # Wait for odsign to be done with the key.
wait_for_prop odsign.key.done 1
- # Lock the fs-verity keyring, so no more keys can be added
- exec -- /system/bin/fsverity_init --lock
-
# Bump the boot level to 1000000000; this prevents further on-device signing.
# This is a special value that shuts down the thread which listens for
# further updates.
diff --git a/rootdir/ueventd.rc b/rootdir/ueventd.rc
index 0b7ffb8..60dcc2a 100644
--- a/rootdir/ueventd.rc
+++ b/rootdir/ueventd.rc
@@ -23,6 +23,11 @@
subsystem dma_heap
devname uevent_devpath
dirname /dev/dma_heap
+
+subsystem vfio
+ devname uevent_devpath
+ dirname /dev/vfio
+
# ueventd can only set permissions on device nodes and their associated
# sysfs attributes, not on arbitrary paths.
#
@@ -43,6 +48,7 @@
/dev/binder 0666 root root
/dev/hwbinder 0666 root root
/dev/vndbinder 0666 root root
+/dev/vfio/* 0666 root root
/dev/pmsg0 0222 root log
/dev/dma_heap/system 0444 system system
diff --git a/trusty/storage/proxy/Android.bp b/trusty/storage/proxy/Android.bp
index 2e97ee0..e362b8b 100644
--- a/trusty/storage/proxy/Android.bp
+++ b/trusty/storage/proxy/Android.bp
@@ -33,6 +33,7 @@
shared_libs: [
"libbase",
+ "libbinder_ndk",
"libcutils",
"liblog",
"libhardware_legacy",
diff --git a/trusty/storage/proxy/proxy.c b/trusty/storage/proxy/proxy.c
index c89c5b6..3b744ec 100644
--- a/trusty/storage/proxy/proxy.c
+++ b/trusty/storage/proxy/proxy.c
@@ -24,6 +24,7 @@
#include <sys/stat.h>
#include <unistd.h>
+#include <android/binder_process.h>
#include <cutils/android_filesystem_config.h>
#include "checkpoint_handling.h"
@@ -238,6 +239,15 @@
/* parse arguments */
parse_args(argc, argv);
+ /*
+ * Start binder threadpool. At least one extra binder thread is needed to
+ * connect to the wakelock service without relying on polling. If we poll on
+ * the main thread we end up pausing for at least 1s even if the service
+ * starts faster.
+ */
+ ABinderProcess_setThreadPoolMaxThreadCount(1);
+ ABinderProcess_startThreadPool();
+
/* initialize secure storage directory */
rc = storage_init(ss_data_root);
if (rc < 0) return EXIT_FAILURE;
diff --git a/trusty/storage/proxy/rpmb.c b/trusty/storage/proxy/rpmb.c
index 22a85a7..1f5d107 100644
--- a/trusty/storage/proxy/rpmb.c
+++ b/trusty/storage/proxy/rpmb.c
@@ -399,6 +399,14 @@
bool is_request_write = req->reliable_write_size > 0;
+ /*
+ * Internally this call connects to the suspend service, which will cause
+ * this service to start if not already running. If the binder thread pool
+ * has not been started at this point, this call will block and poll for the
+ * service every 1s. We need to make sure the thread pool is started to
+ * receive an async notification that the service is started to avoid
+ * blocking (see main).
+ */
wl_rc = acquire_wake_lock(PARTIAL_WAKE_LOCK, UFS_WAKE_LOCK_NAME);
if (wl_rc < 0) {
ALOGE("%s: failed to acquire wakelock: %d, %s\n", __func__, wl_rc, strerror(errno));