libsnapshot: Add XorOp to v3 Cow
Adding in XOR operation to v3 writer. Parser has to go through ops and
create a mapping of XOR operation to it's data
Test: cow_api_test
Bug: 307452468
Change-Id: I51e13a59ba472b62bdcc05921f2e5d6e2c8ad2af
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
index 755a3af..1ab6ada 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
@@ -183,7 +183,7 @@
uint64_t num_total_data_ops_{};
uint64_t num_ordered_ops_to_merge_{};
bool has_seq_ops_{};
- std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc_;
+ std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> xor_data_loc_;
ReaderFlags reader_flag_;
bool is_merge_{};
};
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp
index 93c31bb..296987a 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp
@@ -82,7 +82,7 @@
cow->merge_op_start_ = merge_op_start_;
cow->num_total_data_ops_ = num_total_data_ops_;
cow->num_ordered_ops_to_merge_ = num_ordered_ops_to_merge_;
- cow->data_loc_ = data_loc_;
+ cow->xor_data_loc_ = xor_data_loc_;
cow->block_pos_index_ = block_pos_index_;
cow->is_merge_ = is_merge_;
return cow;
@@ -139,7 +139,6 @@
LOG(ERROR) << "Unknown version: " << header_.prefix.major_version;
return false;
}
-
if (!parser->Parse(fd, header_, label)) {
return false;
}
@@ -154,7 +153,7 @@
footer_ = parser->footer();
fd_size_ = parser->fd_size();
last_label_ = parser->last_label();
- data_loc_ = parser->data_loc();
+ xor_data_loc_ = parser->xor_data_loc();
// If we're resuming a write, we're not ready to merge
if (label.has_value()) return true;
@@ -682,11 +681,10 @@
uint64_t offset;
if (op->type == kCowXorOp) {
- offset = data_loc_->at(op->new_block);
+ offset = xor_data_loc_->at(op->new_block);
} else {
offset = GetCowOpSourceInfoData(*op);
}
-
if (!decompressor) {
CowDataStream stream(this, offset + ignore_bytes, op->data_length - ignore_bytes);
return stream.ReadFully(buffer, buffer_size);
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp
index 76509de..993630b 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp
@@ -82,7 +82,8 @@
}
for (const auto& op : *parser.get_v2ops()) {
std::cout << op << "\n";
- if (auto iter = parser.data_loc()->find(op.new_block); iter != parser.data_loc()->end()) {
+ if (auto iter = parser.xor_data_loc()->find(op.new_block);
+ iter != parser.xor_data_loc()->end()) {
std::cout << " data loc: " << iter->second << "\n";
}
}
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/parser_base.h b/fs_mgr/libsnapshot/libsnapshot_cow/parser_base.h
index 2783407..837b33e 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/parser_base.h
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/parser_base.h
@@ -38,7 +38,9 @@
std::optional<uint64_t> label = {}) = 0;
virtual bool Translate(TranslatedCowOps* out) = 0;
virtual std::optional<CowFooter> footer() const { return std::nullopt; }
- virtual std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc() const = 0;
+ std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> xor_data_loc() {
+ return xor_data_loc_;
+ };
uint64_t fd_size() const { return fd_size_; }
const std::optional<uint64_t>& last_label() const { return last_label_; }
@@ -47,6 +49,7 @@
CowHeaderV3 header_ = {};
uint64_t fd_size_;
std::optional<uint64_t> last_label_;
+ std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> xor_data_loc_ = {};
};
} // namespace snapshot
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.cpp
index 51489c8..08a43a4 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.cpp
@@ -63,7 +63,7 @@
bool CowParserV2::ParseOps(borrowed_fd fd, std::optional<uint64_t> label) {
uint64_t pos;
- auto data_loc = std::make_shared<std::unordered_map<uint64_t, uint64_t>>();
+ auto xor_data_loc = std::make_shared<std::unordered_map<uint64_t, uint64_t>>();
// Skip the scratch space
if (header_.prefix.major_version >= 2 && (header_.buffer_size > 0)) {
@@ -111,7 +111,7 @@
auto& current_op = ops_buffer->data()[current_op_num];
current_op_num++;
if (current_op.type == kCowXorOp) {
- data_loc->insert({current_op.new_block, data_pos});
+ xor_data_loc->insert({current_op.new_block, data_pos});
}
pos += sizeof(CowOperationV2) + GetNextOpOffset(current_op, header_.cluster_ops);
data_pos += current_op.data_length + GetNextDataOffset(current_op, header_.cluster_ops);
@@ -193,7 +193,7 @@
v2_ops_ = ops_buffer;
v2_ops_->shrink_to_fit();
- data_loc_ = data_loc;
+ xor_data_loc_ = xor_data_loc;
return true;
}
@@ -239,9 +239,5 @@
return true;
}
-std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> CowParserV2::data_loc() const {
- return data_loc_;
-}
-
} // namespace snapshot
} // namespace android
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.h b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.h
index 318a3ac..f9ee2e5 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.h
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.h
@@ -35,10 +35,8 @@
const CowHeader& header() const { return header_; }
std::shared_ptr<std::vector<CowOperationV2>> get_v2ops() { return v2_ops_; }
- std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc() const override;
private:
- std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc_;
bool ParseOps(android::base::borrowed_fd fd, std::optional<uint64_t> label);
std::shared_ptr<std::vector<CowOperationV2>> v2_ops_;
std::optional<CowFooter> footer_;
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.cpp
index 23ddaae..a8a63d8 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.cpp
@@ -58,6 +58,10 @@
return ParseOps(fd, label);
}
+off_t CowParserV3::GetDataOffset() const {
+ return sizeof(CowHeaderV3) + header_.buffer_size + header_.op_count_max * sizeof(CowOperation);
+}
+
bool CowParserV3::ParseOps(borrowed_fd fd, std::optional<uint64_t> label) {
ops_ = std::make_shared<std::vector<CowOperationV3>>();
ops_->resize(header_.op_count);
@@ -69,12 +73,24 @@
return false;
}
+ // fill out mapping of XOR op data location
+ uint64_t data_pos = GetDataOffset();
+
+ xor_data_loc_ = std::make_shared<std::unordered_map<uint64_t, uint64_t>>();
+
+ for (auto op : *ops_) {
+ if (op.type == kCowXorOp) {
+ xor_data_loc_->insert({op.new_block, data_pos});
+ }
+ data_pos += op.data_length;
+ }
// :TODO: sequence buffer & resume buffer follow
// Once we implement labels, we'll have to discard unused ops and adjust
// the header as needed.
CHECK(!label);
ops_->shrink_to_fit();
+
return true;
}
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.h b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.h
index 5760cf2..e2663cc 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.h
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/parser_v3.h
@@ -45,13 +45,10 @@
bool Parse(android::base::borrowed_fd fd, const CowHeaderV3& header,
std::optional<uint64_t> label = {}) override;
bool Translate(TranslatedCowOps* out) override;
- std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc() const override {
- return nullptr;
- };
private:
bool ParseOps(android::base::borrowed_fd fd, std::optional<uint64_t> label);
-
+ off_t GetDataOffset() const;
CowHeaderV3 header_ = {};
std::shared_ptr<std::vector<CowOperationV3>> ops_;
};
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/test_v3.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/test_v3.cpp
index c5d7a02..3277c99 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/test_v3.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/test_v3.cpp
@@ -218,7 +218,7 @@
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
- const auto& header = reader.GetHeader();
+ const auto& header = reader.header_v3();
ASSERT_EQ(header.prefix.magic, kCowMagicNumber);
ASSERT_EQ(header.prefix.major_version, 3);
ASSERT_EQ(header.prefix.minor_version, kCowVersionMinor);
@@ -242,5 +242,84 @@
ASSERT_EQ(i, 100);
}
+TEST_F(CowTestV3, XorOp) {
+ CowOptions options;
+ options.op_count_max = 100;
+ auto writer = CreateCowWriter(3, options, GetCowFd());
+
+ std::string data = "This is test data-1. Testing xor";
+ data.resize(options.block_size, '\0');
+ ASSERT_TRUE(writer->AddXorBlocks(50, data.data(), data.size(), 24, 10));
+ ASSERT_TRUE(writer->Finalize());
+
+ ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
+
+ CowReader reader;
+
+ ASSERT_TRUE(reader.Parse(cow_->fd));
+
+ const auto& header = reader.header_v3();
+ ASSERT_EQ(header.op_count, 1);
+
+ auto iter = reader.GetOpIter();
+ ASSERT_NE(iter, nullptr);
+ ASSERT_FALSE(iter->AtEnd());
+ auto op = iter->Get();
+ std::string sink(data.size(), '\0');
+
+ ASSERT_EQ(op->type, kCowXorOp);
+ ASSERT_EQ(op->data_length, 4096);
+ ASSERT_EQ(op->new_block, 50);
+ ASSERT_EQ(GetCowOpSourceInfoData(*op), 98314); // 4096 * 24 + 10
+ ASSERT_TRUE(ReadData(reader, op, sink.data(), sink.size()));
+ ASSERT_EQ(sink, data);
+}
+
+TEST_F(CowTestV3, ConsecutiveXorOp) {
+ CowOptions options;
+ options.op_count_max = 100;
+ auto writer = CreateCowWriter(3, options, GetCowFd());
+
+ std::string data;
+ data.resize(options.block_size * 5);
+ for (int i = 0; i < data.size(); i++) {
+ data[i] = char(rand() % 256);
+ }
+
+ ASSERT_TRUE(writer->AddXorBlocks(50, data.data(), data.size(), 24, 10));
+ ASSERT_TRUE(writer->Finalize());
+
+ ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
+
+ CowReader reader;
+
+ ASSERT_TRUE(reader.Parse(cow_->fd));
+
+ const auto& header = reader.header_v3();
+ ASSERT_EQ(header.op_count, 5);
+
+ auto iter = reader.GetOpIter();
+ ASSERT_NE(iter, nullptr);
+ ASSERT_FALSE(iter->AtEnd());
+
+ std::string sink(data.size(), '\0');
+ size_t i = 0;
+
+ while (!iter->AtEnd()) {
+ auto op = iter->Get();
+ ASSERT_EQ(op->type, kCowXorOp);
+ ASSERT_EQ(op->data_length, 4096);
+ ASSERT_EQ(op->new_block, 50 + i);
+ ASSERT_EQ(GetCowOpSourceInfoData(*op), 98314 + (i * options.block_size)); // 4096 * 24 + 10
+ ASSERT_TRUE(
+ ReadData(reader, op, sink.data() + (i * options.block_size), options.block_size));
+ iter->Next();
+ i++;
+ }
+ ASSERT_EQ(sink, data);
+
+ ASSERT_EQ(i, 5);
+}
+
} // namespace snapshot
} // namespace android
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.cpp
index aeb088d..13b6157 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v3.cpp
@@ -163,7 +163,6 @@
}
next_data_pos_ =
sizeof(CowHeaderV3) + header_.buffer_size + header_.op_count_max * sizeof(CowOperation);
-
return true;
}
@@ -187,19 +186,12 @@
bool CowWriterV3::EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,
uint32_t old_block, uint16_t offset) {
- LOG(ERROR) << __LINE__ << " " << __FILE__ << " <- function here should never be called";
- if (new_block_start || old_block || offset || data || size) return false;
- return false;
+ return EmitBlocks(new_block_start, data, size, old_block, offset, kCowXorOp);
}
bool CowWriterV3::EmitBlocks(uint64_t new_block_start, const void* data, size_t size,
uint64_t old_block, uint16_t offset, uint8_t type) {
const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
-
- // Placing here until we support XOR ops
- CHECK_EQ(old_block, 0);
- CHECK_EQ(offset, 0);
-
const size_t num_blocks = (size / header_.block_size);
for (size_t i = 0; i < num_blocks; i++) {
@@ -207,12 +199,18 @@
op.new_block = new_block_start + i;
op.type = type;
- op.source_info = next_data_pos_;
op.data_length = static_cast<uint16_t>(header_.block_size);
+
+ if (type == kCowXorOp) {
+ op.source_info = (old_block + i) * header_.block_size + offset;
+ } else {
+ op.source_info = next_data_pos_;
+ }
if (!WriteOperation(op, iter, header_.block_size)) {
LOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
+
iter += header_.block_size;
}