Implement SnapshotExtentWriter
Part of the effort to integrate update_engine with VABC.
SnapshotExtentWriter is the proxy between update_engine and VABC.
Test: treehugger
Bug: 168554689
Change-Id: Ib5afe8dac80658d3155f3fe9bcc42e9c3c78d6cb
diff --git a/Android.bp b/Android.bp
index 178b7da..29e6025 100644
--- a/Android.bp
+++ b/Android.bp
@@ -728,6 +728,7 @@
"payload_consumer/download_action_android_unittest.cc",
"payload_consumer/extent_reader_unittest.cc",
"payload_consumer/extent_writer_unittest.cc",
+ "payload_consumer/snapshot_extent_writer_unittest.cc",
"payload_consumer/fake_file_descriptor.cc",
"payload_consumer/file_descriptor_utils_unittest.cc",
"payload_consumer/file_writer_unittest.cc",
diff --git a/payload_consumer/snapshot_extent_writer.cc b/payload_consumer/snapshot_extent_writer.cc
index 882d1f7..5693c9b 100644
--- a/payload_consumer/snapshot_extent_writer.cc
+++ b/payload_consumer/snapshot_extent_writer.cc
@@ -13,6 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//
+
#include "update_engine/payload_consumer/snapshot_extent_writer.h"
#include <algorithm>
@@ -23,6 +24,7 @@
#include "update_engine/update_metadata.pb.h"
namespace chromeos_update_engine {
+
SnapshotExtentWriter::SnapshotExtentWriter(
android::snapshot::ICowWriter* cow_writer)
: cow_writer_(cow_writer) {
@@ -30,25 +32,82 @@
}
SnapshotExtentWriter::~SnapshotExtentWriter() {
- CHECK(buffer_.empty());
+ CHECK(buffer_.empty()) << buffer_.size();
}
bool SnapshotExtentWriter::Init(
FileDescriptorPtr /*fd*/,
const google::protobuf::RepeatedPtrField<Extent>& extents,
- uint32_t /*block_size*/) {
- // TODO(zhangkelvin) Implement this
+ uint32_t block_size) {
+ extents_ = extents;
+ cur_extent_idx_ = 0;
+ buffer_.clear();
+ buffer_.reserve(block_size);
+ block_size_ = block_size;
return true;
}
+size_t SnapshotExtentWriter::ConsumeWithBuffer(const uint8_t* data,
+ size_t count) {
+ CHECK_LT(cur_extent_idx_, static_cast<size_t>(extents_.size()));
+ const auto& cur_extent = extents_[cur_extent_idx_];
+ const auto cur_extent_size = cur_extent.num_blocks() * block_size_;
+
+ if (buffer_.empty() && count >= cur_extent_size) {
+ TEST_AND_RETURN_FALSE(cow_writer_->AddRawBlocks(
+ cur_extent.start_block(), data, cur_extent_size));
+ if (!next_extent()) {
+ CHECK_EQ(count, cur_extent_size)
+ << "Exhausted all blocks, but still have " << count - cur_extent_size
+ << " bytes left";
+ }
+ return cur_extent_size;
+ }
+ CHECK_LT(buffer_.size(), cur_extent_size)
+ << "Data left in buffer should never be >= cur_extent_size, otherwise "
+ "we should have send that data to CowWriter. Buffer size: "
+ << buffer_.size() << " current extent size: " << cur_extent_size;
+ size_t bytes_to_copy =
+ std::min<size_t>(count, cur_extent_size - buffer_.size());
+ CHECK_GT(bytes_to_copy, 0U);
+
+ buffer_.insert(buffer_.end(), data, data + bytes_to_copy);
+ CHECK_LE(buffer_.size(), cur_extent_size);
+
+ if (buffer_.size() == cur_extent_size) {
+ TEST_AND_RETURN_FALSE(cow_writer_->AddRawBlocks(
+ cur_extent.start_block(), buffer_.data(), buffer_.size()));
+ buffer_.clear();
+ if (!next_extent()) {
+ CHECK_EQ(count, bytes_to_copy) << "Exhausted all blocks, but still have "
+ << count - bytes_to_copy << " bytes left";
+ }
+ }
+ return bytes_to_copy;
+}
+
// Returns true on success.
// This will construct a COW_REPLACE operation and forward it to CowWriter. It
// is important that caller does not perform SOURCE_COPY operation on this
// class, otherwise raw data will be stored. Caller should find ways to use
// COW_COPY whenever possible.
bool SnapshotExtentWriter::Write(const void* bytes, size_t count) {
- // TODO(zhangkelvin) Implement this
+ if (count == 0) {
+ return true;
+ }
+ CHECK_NE(extents_.size(), 0);
+
+ auto data = static_cast<const uint8_t*>(bytes);
+ while (count > 0) {
+ auto bytes_written = ConsumeWithBuffer(data, count);
+ data += bytes_written;
+ count -= bytes_written;
+ }
return true;
}
+bool SnapshotExtentWriter::next_extent() {
+ cur_extent_idx_++;
+ return cur_extent_idx_ < static_cast<size_t>(extents_.size());
+}
} // namespace chromeos_update_engine
diff --git a/payload_consumer/snapshot_extent_writer.h b/payload_consumer/snapshot_extent_writer.h
index 43a8317..fb4b548 100644
--- a/payload_consumer/snapshot_extent_writer.h
+++ b/payload_consumer/snapshot_extent_writer.h
@@ -13,14 +13,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//
+
#include <cstdint>
#include <vector>
#include <libsnapshot/cow_writer.h>
#include "update_engine/payload_consumer/extent_writer.h"
+#include "update_engine/update_metadata.pb.h"
namespace chromeos_update_engine {
+
class SnapshotExtentWriter : public chromeos_update_engine::ExtentWriter {
public:
explicit SnapshotExtentWriter(android::snapshot::ICowWriter* cow_writer);
@@ -37,11 +40,16 @@
bool Write(const void* bytes, size_t count) override;
private:
+ bool next_extent();
+ size_t ConsumeWithBuffer(const uint8_t* bytes, size_t count);
// It's a non-owning pointer, because PartitionWriter owns the CowWruter. This
// allows us to use a single instance of CowWriter for all operations applied
// to the same partition.
- [[maybe_unused]] android::snapshot::ICowWriter* cow_writer_;
- [[maybe_unused]] google::protobuf::RepeatedPtrField<Extent> extents_;
- [[maybe_unused]] std::vector<uint8_t> buffer_;
+ android::snapshot::ICowWriter* cow_writer_;
+ google::protobuf::RepeatedPtrField<Extent> extents_;
+ size_t cur_extent_idx_;
+ std::vector<uint8_t> buffer_;
+ size_t block_size_;
};
+
} // namespace chromeos_update_engine
diff --git a/payload_consumer/snapshot_extent_writer_unittest.cc b/payload_consumer/snapshot_extent_writer_unittest.cc
new file mode 100644
index 0000000..0e22482
--- /dev/null
+++ b/payload_consumer/snapshot_extent_writer_unittest.cc
@@ -0,0 +1,180 @@
+//
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <array>
+#include <cstring>
+#include <map>
+#include <numeric>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include <google/protobuf/message_lite.h>
+#include <libsnapshot/cow_writer.h>
+
+#include "update_engine/payload_consumer/snapshot_extent_writer.h"
+#include "update_engine/payload_generator/delta_diff_generator.h"
+#include "update_engine/update_metadata.pb.h"
+
+namespace chromeos_update_engine {
+
+class FakeCowWriter : public android::snapshot::ICowWriter {
+ public:
+ struct CowOp {
+ enum { COW_COPY, COW_REPLACE, COW_ZERO } type;
+ std::vector<unsigned char> data;
+ union {
+ size_t source_block;
+ size_t num_blocks;
+ };
+ };
+ using ICowWriter::ICowWriter;
+ ~FakeCowWriter() = default;
+
+ bool EmitCopy(uint64_t new_block, uint64_t old_block) override {
+ operations_[new_block] = {.type = CowOp::COW_COPY,
+ .source_block = static_cast<size_t>(old_block)};
+ return true;
+ }
+ bool EmitRawBlocks(uint64_t new_block_start,
+ const void* data,
+ size_t size) override {
+ auto&& op = operations_[new_block_start];
+ const auto uint8_ptr = static_cast<const unsigned char*>(data);
+ op.data.insert(op.data.end(), uint8_ptr, uint8_ptr + size);
+ return true;
+ }
+ bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override {
+ operations_[new_block_start] = {.type = CowOp::COW_ZERO};
+ return true;
+ }
+ bool Finalize() override {
+ finalize_called_ = true;
+ return true;
+ }
+
+ bool EmitLabel(uint64_t label) {
+ label_count_++;
+ return true;
+ }
+
+ // Return number of bytes the cow image occupies on disk.
+ uint64_t GetCowSize() override {
+ return std::accumulate(
+ operations_.begin(), operations_.end(), 0, [](auto&& acc, auto&& op) {
+ return acc + op.second.data.size();
+ });
+ }
+ bool Contains(size_t block) {
+ return operations_.find(block) != operations_.end();
+ }
+ bool finalize_called_ = true;
+ size_t label_count_ = 0;
+ std::map<size_t, CowOp> operations_;
+};
+
+class SnapshotExtentWriterTest : public ::testing::Test {
+ public:
+ void SetUp() override {}
+
+ protected:
+ android::snapshot::CowOptions options_ = {
+ .block_size = static_cast<uint32_t>(kBlockSize)};
+ FakeCowWriter cow_writer_{options_};
+ SnapshotExtentWriter writer_{&cow_writer_};
+};
+
+void AddExtent(google::protobuf::RepeatedPtrField<Extent>* extents,
+ size_t start_block,
+ size_t num_blocks) {
+ auto&& extent = extents->Add();
+ extent->set_start_block(start_block);
+ extent->set_num_blocks(num_blocks);
+}
+
+TEST_F(SnapshotExtentWriterTest, BufferWrites) {
+ google::protobuf::RepeatedPtrField<Extent> extents;
+ AddExtent(&extents, 123, 1);
+ writer_.Init(nullptr, extents, kBlockSize);
+
+ std::vector<uint8_t> buf(kBlockSize, 0);
+ buf[123] = 231;
+ buf[231] = 123;
+ buf[buf.size() - 1] = 255;
+
+ writer_.Write(buf.data(), kBlockSize - 1);
+ ASSERT_TRUE(cow_writer_.operations_.empty())
+ << "Haven't send data of a complete block yet, CowWriter should not be "
+ "invoked.";
+ writer_.Write(buf.data() + kBlockSize - 1, 1);
+ ASSERT_TRUE(cow_writer_.Contains(123))
+ << "Once a block of data is sent to SnapshotExtentWriter, it should "
+ "forward data to cow_writer.";
+ ASSERT_EQ(cow_writer_.operations_.size(), 1U);
+ ASSERT_EQ(buf, cow_writer_.operations_[123].data);
+}
+
+TEST_F(SnapshotExtentWriterTest, NonBufferedWrites) {
+ google::protobuf::RepeatedPtrField<Extent> extents;
+ AddExtent(&extents, 123, 1);
+ AddExtent(&extents, 125, 1);
+ writer_.Init(nullptr, extents, kBlockSize);
+
+ std::vector<uint8_t> buf(kBlockSize * 2, 0);
+ buf[123] = 231;
+ buf[231] = 123;
+ buf[buf.size() - 1] = 255;
+
+ writer_.Write(buf.data(), buf.size());
+ ASSERT_TRUE(cow_writer_.Contains(123));
+ ASSERT_TRUE(cow_writer_.Contains(125));
+
+ ASSERT_EQ(cow_writer_.operations_.size(), 2U);
+ auto actual_data = cow_writer_.operations_[123].data;
+ actual_data.insert(actual_data.end(),
+ cow_writer_.operations_[125].data.begin(),
+ cow_writer_.operations_[125].data.end());
+ ASSERT_EQ(buf, actual_data);
+}
+
+TEST_F(SnapshotExtentWriterTest, WriteAcrossBlockBoundary) {
+ google::protobuf::RepeatedPtrField<Extent> extents;
+ AddExtent(&extents, 123, 1);
+ AddExtent(&extents, 125, 2);
+ writer_.Init(nullptr, extents, kBlockSize);
+
+ std::vector<uint8_t> buf(kBlockSize * 3);
+ std::memset(buf.data(), 0, buf.size());
+ buf[123] = 231;
+ buf[231] = 123;
+ buf[buf.size() - 1] = 255;
+ buf[kBlockSize - 1] = 254;
+
+ writer_.Write(buf.data(), kBlockSize - 1);
+ ASSERT_TRUE(cow_writer_.operations_.empty())
+ << "Haven't send data of a complete block yet, CowWriter should not be "
+ "invoked.";
+ writer_.Write(buf.data() + kBlockSize - 1, 1 + kBlockSize * 2);
+ ASSERT_TRUE(cow_writer_.Contains(123));
+ ASSERT_TRUE(cow_writer_.Contains(125));
+
+ ASSERT_EQ(cow_writer_.operations_.size(), 2U);
+ auto actual_data = cow_writer_.operations_[123].data;
+ actual_data.insert(actual_data.end(),
+ cow_writer_.operations_[125].data.begin(),
+ cow_writer_.operations_[125].data.end());
+ ASSERT_EQ(buf, actual_data);
+}
+} // namespace chromeos_update_engine