Add XorExtentWriter
This class is built on top of BlockExtentWriter(block aligned writer)
and ExtentMap. It will look at the extent map, convert blocks in the
extent map to XOR type, and convert everythign else to COW_REPLACE.
Test: th
Bug: 177104308
Change-Id: I6f9df83ed09f1c5a6aae9f266412132b002acd29
diff --git a/Android.bp b/Android.bp
index b3fbccf..83be871 100644
--- a/Android.bp
+++ b/Android.bp
@@ -242,6 +242,7 @@
"payload_consumer/partition_writer.cc",
"payload_consumer/partition_writer_factory_android.cc",
"payload_consumer/vabc_partition_writer.cc",
+ "payload_consumer/xor_extent_writer.cc",
"payload_consumer/block_extent_writer.cc",
"payload_consumer/snapshot_extent_writer.cc",
"payload_consumer/postinstall_runner_action.cc",
@@ -806,6 +807,7 @@
"payload_consumer/extent_writer_unittest.cc",
"payload_consumer/extent_map_unittest.cc",
"payload_consumer/snapshot_extent_writer_unittest.cc",
+ "payload_consumer/xor_extent_writer_unittest.cc",
"payload_consumer/fake_file_descriptor.cc",
"payload_consumer/file_descriptor_utils_unittest.cc",
"payload_consumer/file_writer_unittest.cc",
diff --git a/common/utils.cc b/common/utils.cc
index 5dbb445..25df540 100644
--- a/common/utils.cc
+++ b/common/utils.cc
@@ -1040,6 +1040,11 @@
return ErrorCode::kSuccess;
}
+std::unique_ptr<android::base::MappedFile> GetReadonlyZeroBlock(size_t size) {
+ android::base::unique_fd fd{HANDLE_EINTR(open("/dev/zero", O_RDONLY))};
+ return android::base::MappedFile::FromFd(fd, 0, size, PROT_READ);
+}
+
} // namespace utils
} // namespace chromeos_update_engine
diff --git a/common/utils.h b/common/utils.h
index 4e78097..9ed0c7b 100644
--- a/common/utils.h
+++ b/common/utils.h
@@ -36,6 +36,7 @@
#include <brillo/key_value_store.h>
#include <brillo/secure_blob.h>
+#include "android-base/mapped_file.h"
#include "update_engine/common/action.h"
#include "update_engine/common/action_processor.h"
#include "update_engine/common/constants.h"
@@ -357,6 +358,8 @@
ErrorCode IsTimestampNewer(const std::string& old_version,
const std::string& new_version);
+std::unique_ptr<android::base::MappedFile> GetReadonlyZeroBlock(size_t size);
+
} // namespace utils
// Utility class to close a file descriptor
diff --git a/payload_consumer/block_extent_writer.cc b/payload_consumer/block_extent_writer.cc
index a5c990c..e50526c 100644
--- a/payload_consumer/block_extent_writer.cc
+++ b/payload_consumer/block_extent_writer.cc
@@ -45,7 +45,7 @@
if (buffer_.empty() && count >= cur_extent_size) {
if (!WriteExtent(data, cur_extent, block_size_)) {
- LOG(ERROR) << "AddRawBlocks(" << cur_extent.start_block() << ", " << data
+ LOG(ERROR) << "WriteExtent(" << cur_extent.start_block() << ", " << data
<< ", " << cur_extent_size << ") failed.";
// return value is expected to be greater than 0. Return 0 to signal error
// condition
diff --git a/payload_consumer/block_extent_writer.h b/payload_consumer/block_extent_writer.h
index 7c90459..f9c7b15 100644
--- a/payload_consumer/block_extent_writer.h
+++ b/payload_consumer/block_extent_writer.h
@@ -41,6 +41,7 @@
virtual bool WriteExtent(const void* bytes,
const Extent& extent,
size_t block_size) = 0;
+ size_t BlockSize() const { return block_size_; }
private:
bool NextExtent();
diff --git a/payload_consumer/xor_extent_writer.cc b/payload_consumer/xor_extent_writer.cc
new file mode 100644
index 0000000..5c89c93
--- /dev/null
+++ b/payload_consumer/xor_extent_writer.cc
@@ -0,0 +1,115 @@
+//
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <algorithm>
+#include <optional>
+
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_consumer/xor_extent_writer.h"
+#include "update_engine/payload_generator/extent_utils.h"
+
+namespace chromeos_update_engine {
+
+// Returns true on success.
+bool XORExtentWriter::WriteExtent(const void* bytes,
+ const Extent& extent,
+ const size_t size) {
+ brillo::Blob xor_block_data;
+ const auto xor_extents = xor_map_.GetIntersectingExtents(extent);
+ for (const auto& xor_ext : xor_extents) {
+ const auto merge_op_opt = xor_map_.Get(xor_ext);
+ if (!merge_op_opt.has_value()) {
+ LOG(ERROR)
+ << xor_ext
+ << " isn't in XOR map but it's returned by GetIntersectingExtents(), "
+ "this is a bug inside GetIntersectingExtents";
+ return false;
+ }
+
+ const auto merge_op = merge_op_opt.value();
+ TEST_AND_RETURN_FALSE(merge_op->has_src_extent());
+ TEST_AND_RETURN_FALSE(merge_op->has_dst_extent());
+ if (merge_op->dst_extent() != xor_ext) {
+ LOG(ERROR) << "Each xor extent is expected to correspond to a complete "
+ "MergeOp, extent in value: "
+ << merge_op->dst_extent() << " extent in key: " << xor_ext;
+ return false;
+ }
+ if (xor_ext.start_block() + xor_ext.num_blocks() >
+ extent.start_block() + extent.num_blocks()) {
+ LOG(ERROR) << "CowXor merge op extent should be completely inside "
+ "InstallOp's extent. merge op extent: "
+ << xor_ext << " InstallOp extent: " << extent;
+ return false;
+ }
+ const auto src_offset = merge_op->src_offset();
+ const auto src_block = merge_op->src_extent().start_block();
+ xor_block_data.resize(BlockSize() * xor_ext.num_blocks());
+ ssize_t bytes_read = 0;
+ TEST_AND_RETURN_FALSE_ERRNO(
+ utils::PReadAll(source_fd_,
+ xor_block_data.data(),
+ xor_block_data.size(),
+ src_offset + src_block * BlockSize(),
+ &bytes_read));
+ if (bytes_read != static_cast<ssize_t>(xor_block_data.size())) {
+ LOG(ERROR) << "bytes_read: " << bytes_read;
+ return false;
+ }
+
+ const auto i = xor_ext.start_block() - extent.start_block();
+
+ const auto dst_block_data =
+ static_cast<const unsigned char*>(bytes) + i * BlockSize();
+ std::transform(xor_block_data.cbegin(),
+ xor_block_data.cbegin() + xor_block_data.size(),
+ dst_block_data,
+ xor_block_data.begin(),
+ std::bit_xor<unsigned char>{});
+ TEST_AND_RETURN_FALSE(cow_writer_->AddXorBlocks(xor_ext.start_block(),
+ xor_block_data.data(),
+ xor_block_data.size(),
+ src_block,
+ src_offset));
+ }
+ const auto replace_extents = xor_map_.GetNonIntersectingExtents(extent);
+ return WriteReplaceExtents(replace_extents, extent, bytes, size);
+}
+
+bool XORExtentWriter::WriteReplaceExtents(
+ const std::vector<Extent>& replace_extents,
+ const Extent& extent,
+ const void* bytes,
+ size_t size) {
+ const uint64_t new_block_start = extent.start_block();
+ for (const auto& ext : replace_extents) {
+ if (ext.start_block() + ext.num_blocks() >
+ extent.start_block() + extent.num_blocks()) {
+ LOG(ERROR) << "CowReplace merge op extent should be completely inside "
+ "InstallOp's extent. merge op extent: "
+ << ext << " InstallOp extent: " << extent;
+ return false;
+ }
+ const auto i = ext.start_block() - new_block_start;
+ const auto dst_block_data =
+ static_cast<const unsigned char*>(bytes) + i * BlockSize();
+ TEST_AND_RETURN_FALSE(cow_writer_->AddRawBlocks(
+ ext.start_block(), dst_block_data, ext.num_blocks() * BlockSize()));
+ }
+ return true;
+}
+
+} // namespace chromeos_update_engine
diff --git a/payload_consumer/xor_extent_writer.h b/payload_consumer/xor_extent_writer.h
new file mode 100644
index 0000000..94b5a9e
--- /dev/null
+++ b/payload_consumer/xor_extent_writer.h
@@ -0,0 +1,66 @@
+//
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef UPDATE_ENGINE_PAYLOAD_CONSUMER_XOR_EXTENT_WRITER_H_
+#define UPDATE_ENGINE_PAYLOAD_CONSUMER_XOR_EXTENT_WRITER_H_
+
+#include "update_engine/payload_consumer/block_extent_writer.h"
+#include "update_engine/payload_consumer/extent_map.h"
+#include "update_engine/payload_consumer/extent_reader.h"
+#include "update_engine/payload_consumer/extent_writer.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+#include "update_engine/payload_generator/extent_utils.h"
+
+#include <update_engine/update_metadata.pb.h>
+#include <libsnapshot/cow_writer.h>
+
+namespace chromeos_update_engine {
+
+// An extent writer that will selectively convert some of the blocks into an XOR
+// block. All blocks that appear in |xor_map| will be converted,
+class XORExtentWriter : public BlockExtentWriter {
+ public:
+ XORExtentWriter(const InstallOperation& op,
+ FileDescriptorPtr source_fd,
+ android::snapshot::ICowWriter* cow_writer,
+ const ExtentMap<const CowMergeOperation*>& xor_map)
+ : src_extents_(op.src_extents()),
+ source_fd_(source_fd),
+ xor_map_(xor_map),
+ cow_writer_(cow_writer) {
+ CHECK(source_fd->IsOpen());
+ }
+ ~XORExtentWriter() = default;
+
+ // Returns true on success.
+ bool WriteExtent(const void* bytes,
+ const Extent& extent,
+ size_t size) override;
+
+ private:
+ bool WriteReplaceExtents(const std::vector<Extent>& replace_extents,
+ const Extent& extent,
+ const void* bytes,
+ size_t size);
+ const google::protobuf::RepeatedPtrField<Extent>& src_extents_;
+ const FileDescriptorPtr source_fd_;
+ const ExtentMap<const CowMergeOperation*>& xor_map_;
+ android::snapshot::ICowWriter* cow_writer_;
+};
+
+} // namespace chromeos_update_engine
+
+#endif // UPDATE_ENGINE_PAYLOAD_CONSUMER_XOR_EXTENT_WRITER_H_
diff --git a/payload_consumer/xor_extent_writer_unittest.cc b/payload_consumer/xor_extent_writer_unittest.cc
new file mode 100644
index 0000000..7f35bc2
--- /dev/null
+++ b/payload_consumer/xor_extent_writer_unittest.cc
@@ -0,0 +1,134 @@
+//
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <memory>
+
+#include <unistd.h>
+
+#include <android-base/file.h>
+#include <gtest/gtest.h>
+#include <libsnapshot/mock_snapshot_writer.h>
+
+#include "common/utils.h"
+#include "update_engine/payload_consumer/extent_map.h"
+#include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/payload_consumer/xor_extent_writer.h"
+#include "update_engine/payload_generator/delta_diff_generator.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+#include "update_engine/payload_generator/merge_sequence_generator.h"
+#include "update_engine/update_metadata.pb.h"
+
+namespace chromeos_update_engine {
+
+using testing::_;
+using testing::Args;
+using testing::Return;
+
+class XorExtentWriterTest : public ::testing::Test {
+ public:
+ static constexpr size_t NUM_BLOCKS = 50;
+ void SetUp() override {
+ ASSERT_EQ(ftruncate64(source_part_.fd, kBlockSize * NUM_BLOCKS), 0);
+ ASSERT_EQ(ftruncate64(target_part_.fd, kBlockSize * NUM_BLOCKS), 0);
+
+ // Fill source part with 1s, as we are computing XOR between source and
+ // target data later.
+ ASSERT_EQ(lseek(source_part_.fd, 0, SEEK_SET), 0);
+ brillo::Blob buffer(kBlockSize);
+ std::fill(buffer.begin(), buffer.end(), 1);
+ for (size_t i = 0; i < NUM_BLOCKS; i++) {
+ ASSERT_EQ(write(source_part_.fd, buffer.data(), buffer.size()),
+ static_cast<ssize_t>(buffer.size()));
+ }
+ ASSERT_EQ(fsync(source_part_.fd), 0);
+ ASSERT_EQ(fsync(target_part_.fd), 0);
+ ASSERT_TRUE(source_fd_->Open(source_part_.path, O_RDONLY | O_CREAT, 0644));
+ }
+ InstallOperation op_;
+ FileDescriptorPtr source_fd_ = std::make_shared<EintrSafeFileDescriptor>();
+ ExtentMap<const CowMergeOperation*> xor_map_;
+ android::snapshot::MockSnapshotWriter cow_writer_;
+ TemporaryFile source_part_;
+ TemporaryFile target_part_;
+};
+
+MATCHER_P2(BytesEqual,
+ bytes,
+ size,
+ "Check if args match expected value byte for byte") {
+ return std::get<1>(arg) == size && std::get<0>(arg) != nullptr &&
+ memcmp(std::get<0>(arg), bytes, size) == 0;
+}
+
+TEST_F(XorExtentWriterTest, StreamTest) {
+ constexpr auto COW_XOR = CowMergeOperation::COW_XOR;
+ ON_CALL(cow_writer_, EmitXorBlocks(_, _, _, _, _))
+ .WillByDefault(Return(true));
+ const auto op1 = CreateCowMergeOperation(
+ ExtentForRange(5, 2), ExtentForRange(5, 2), COW_XOR);
+ ASSERT_TRUE(xor_map_.AddExtent(op1.dst_extent(), &op1));
+ *op_.add_src_extents() = op1.src_extent();
+ *op_.add_dst_extents() = op1.dst_extent();
+
+ const auto op2 = CreateCowMergeOperation(
+ ExtentForRange(45, 2), ExtentForRange(456, 2), COW_XOR);
+ ASSERT_TRUE(xor_map_.AddExtent(op2.dst_extent(), &op2));
+ *op_.add_src_extents() = ExtentForRange(45, 3);
+ *op_.add_dst_extents() = ExtentForRange(455, 3);
+
+ const auto op3 = CreateCowMergeOperation(
+ ExtentForRange(12, 2), ExtentForRange(321, 2), COW_XOR, 777);
+ ASSERT_TRUE(xor_map_.AddExtent(op3.dst_extent(), &op3));
+ *op_.add_src_extents() = ExtentForRange(12, 4);
+ *op_.add_dst_extents() = ExtentForRange(320, 4);
+ XORExtentWriter writer_{op_, source_fd_, &cow_writer_, xor_map_};
+
+ // OTA op:
+ // [5-6] => [5-6], [45-47] => [455-457], [12-15] => [320-323]
+
+ // merge op:
+ // [5-6] => [5-6], [45-46] => [456-457], [12-13] => [321-322]
+
+ // Expected result:
+ // [5-7], [45-47], [12-14] should be XOR blocks
+ // [320], [323], [455] should be regular replace blocks
+
+ auto zeros = utils::GetReadonlyZeroBlock(kBlockSize * 10);
+ EXPECT_CALL(cow_writer_,
+ EmitRawBlocks(455, zeros->data() + 2 * kBlockSize, kBlockSize))
+ .With(Args<1, 2>(BytesEqual(zeros->data(), kBlockSize)))
+ .WillOnce(Return(true));
+ EXPECT_CALL(cow_writer_,
+ EmitRawBlocks(320, zeros->data() + 5 * kBlockSize, kBlockSize))
+ .With(Args<1, 2>(BytesEqual(zeros->data(), kBlockSize)))
+ .WillOnce(Return(true));
+ EXPECT_CALL(cow_writer_,
+ EmitRawBlocks(323, zeros->data() + 8 * kBlockSize, kBlockSize))
+ .With(Args<1, 2>(BytesEqual(zeros->data(), kBlockSize)))
+ .WillOnce(Return(true));
+
+ EXPECT_CALL(cow_writer_, EmitXorBlocks(5, _, kBlockSize * 2, 5, 0))
+ .WillOnce(Return(true));
+ EXPECT_CALL(cow_writer_, EmitXorBlocks(456, _, kBlockSize * 2, 45, 0))
+ .WillOnce(Return(true));
+ EXPECT_CALL(cow_writer_, EmitXorBlocks(321, _, kBlockSize * 2, 12, 777))
+ .WillOnce(Return(true));
+
+ ASSERT_TRUE(writer_.Init(op_.dst_extents(), kBlockSize));
+ ASSERT_TRUE(writer_.Write(zeros->data(), 9 * kBlockSize));
+}
+
+} // namespace chromeos_update_engine
diff --git a/payload_generator/extent_utils.cc b/payload_generator/extent_utils.cc
index f4a9ff0..612fd67 100644
--- a/payload_generator/extent_utils.cc
+++ b/payload_generator/extent_utils.cc
@@ -26,6 +26,7 @@
#include <base/macros.h>
#include <base/strings/stringprintf.h>
+#include "update_engine/common/utils.h"
#include "update_engine/payload_consumer/payload_constants.h"
#include "update_engine/payload_generator/annotated_operation.h"
#include "update_engine/payload_generator/extent_ranges.h"
@@ -161,14 +162,44 @@
return result;
}
-bool operator==(const Extent& a, const Extent& b) {
+bool operator==(const Extent& a, const Extent& b) noexcept {
return a.start_block() == b.start_block() && a.num_blocks() == b.num_blocks();
}
+bool operator!=(const Extent& a, const Extent& b) noexcept {
+ return !(a == b);
+}
+
std::ostream& operator<<(std::ostream& out, const Extent& extent) {
out << "[" << extent.start_block() << " - "
<< extent.start_block() + extent.num_blocks() - 1 << "]";
return out;
}
+template <typename T>
+std::ostream& PrintExtents(std::ostream& out, const T& extents) {
+ if (extents.begin() == extents.end()) {
+ out << "{}";
+ return out;
+ }
+ out << "{";
+ auto begin = extents.begin();
+ out << *begin;
+ for (const auto& ext : Range{++begin, extents.end()}) {
+ out << ", " << ext;
+ }
+ out << "}";
+ return out;
+}
+
+std::ostream& operator<<(std::ostream& out,
+ const std::vector<Extent>& extents) {
+ return PrintExtents(out, extents);
+}
+std::ostream& operator<<(
+ std::ostream& out,
+ const google::protobuf::RepeatedPtrField<Extent>& extents) {
+ return PrintExtents(out, extents);
+}
+
} // namespace chromeos_update_engine
diff --git a/payload_generator/extent_utils.h b/payload_generator/extent_utils.h
index 2bd6626..f5bb39c 100644
--- a/payload_generator/extent_utils.h
+++ b/payload_generator/extent_utils.h
@@ -86,7 +86,9 @@
uint64_t block_offset,
uint64_t block_count);
-bool operator==(const Extent& a, const Extent& b);
+bool operator==(const Extent& a, const Extent& b) noexcept;
+
+bool operator!=(const Extent& a, const Extent& b) noexcept;
// TODO(zhangkelvin) This is ugly. Rewrite using C++20's coroutine once
// that's available. Unfortunately with C++17 this is the best I could do.
@@ -126,6 +128,10 @@
};
std::ostream& operator<<(std::ostream& out, const Extent& extent);
+std::ostream& operator<<(std::ostream& out, const std::vector<Extent>& extent);
+std::ostream& operator<<(
+ std::ostream& out,
+ const google::protobuf::RepeatedPtrField<Extent>& extent);
template <typename Container>
size_t GetNthBlock(const Container& extents, const size_t n) {