update_engine: Add ExtentReader
Currently Each operation in DeltaPerformer has the responsibility of
reading from the source device by itself. Adding an
ExtentReader (similar to ExtentWriter) aggregates all these
responsibilities on DeltaPerformer.source_fd_. This simplifies the
design as each operation does not need to invent is own open/read/close
etc. This specially will be used in conjunction with bspatch and puffin.
BUG=chromium:761138
TEST=FEATURES="test" emerge-amd64-generic update_engine;
Change-Id: I2c7a21a02bd1df06bbaa7ebf374a13f654768721
Reviewed-on: https://chromium-review.googlesource.com/651336
Commit-Ready: Amin Hassani <ahassani@chromium.org>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Andrew de los Reyes <adlr@chromium.org>
Reviewed-by: Sen Jiang <senj@chromium.org>
diff --git a/Android.mk b/Android.mk
index 96b4a90..12b6fc0 100644
--- a/Android.mk
+++ b/Android.mk
@@ -127,6 +127,7 @@
payload_consumer/bzip_extent_writer.cc \
payload_consumer/delta_performer.cc \
payload_consumer/download_action.cc \
+ payload_consumer/extent_reader.cc \
payload_consumer/extent_writer.cc \
payload_consumer/file_descriptor.cc \
payload_consumer/file_descriptor_utils.cc \
@@ -924,6 +925,7 @@
payload_consumer/bzip_extent_writer_unittest.cc \
payload_consumer/delta_performer_integration_test.cc \
payload_consumer/delta_performer_unittest.cc \
+ payload_consumer/extent_reader_unittest.cc \
payload_consumer/extent_writer_unittest.cc \
payload_consumer/fake_file_descriptor.cc \
payload_consumer/file_descriptor_utils_unittest.cc \
diff --git a/payload_consumer/extent_reader.cc b/payload_consumer/extent_reader.cc
new file mode 100644
index 0000000..428ecfe
--- /dev/null
+++ b/payload_consumer/extent_reader.cc
@@ -0,0 +1,97 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_consumer/extent_reader.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_consumer/payload_constants.h"
+
+using google::protobuf::RepeatedPtrField;
+
+namespace chromeos_update_engine {
+
+bool DirectExtentReader::Init(FileDescriptorPtr fd,
+ const RepeatedPtrField<Extent>& extents,
+ uint32_t block_size) {
+ fd_ = fd;
+ extents_ = extents;
+ block_size_ = block_size;
+ cur_extent_ = extents_.begin();
+
+ extents_upper_bounds_.reserve(extents_.size() + 1);
+ // We add this pad as the first element to not bother with boundary checks
+ // later.
+ extents_upper_bounds_.emplace_back(0);
+ for (const auto& extent : extents_) {
+ total_size_ += extent.num_blocks() * block_size_;
+ extents_upper_bounds_.emplace_back(total_size_);
+ }
+ return true;
+}
+
+bool DirectExtentReader::Seek(uint64_t offset) {
+ TEST_AND_RETURN_FALSE(offset <= total_size_);
+ if (offset_ == offset) {
+ return true;
+ }
+ // The first item is zero and upper_bound never returns it because it always
+ // return the item which is greater than the given value.
+ auto extent_idx = std::upper_bound(
+ extents_upper_bounds_.begin(), extents_upper_bounds_.end(), offset) -
+ extents_upper_bounds_.begin() - 1;
+ cur_extent_ = std::next(extents_.begin(), extent_idx);
+ offset_ = offset;
+ cur_extent_bytes_read_ = offset_ - extents_upper_bounds_[extent_idx];
+ return true;
+}
+
+bool DirectExtentReader::Read(void* buffer, size_t count) {
+ auto bytes = reinterpret_cast<uint8_t*>(buffer);
+ uint64_t bytes_read = 0;
+ while (bytes_read < count) {
+ if (cur_extent_ == extents_.end()) {
+ TEST_AND_RETURN_FALSE(bytes_read == count);
+ }
+ uint64_t bytes_to_read = std::min(
+ count - bytes_read,
+ cur_extent_->num_blocks() * block_size_ - cur_extent_bytes_read_);
+
+ ssize_t out_bytes_read;
+ TEST_AND_RETURN_FALSE(utils::PReadAll(
+ fd_,
+ bytes + bytes_read,
+ bytes_to_read,
+ cur_extent_->start_block() * block_size_ + cur_extent_bytes_read_,
+ &out_bytes_read));
+ TEST_AND_RETURN_FALSE(out_bytes_read ==
+ static_cast<ssize_t>(bytes_to_read));
+
+ bytes_read += bytes_to_read;
+ cur_extent_bytes_read_ += bytes_to_read;
+ offset_ += bytes_to_read;
+ if (cur_extent_bytes_read_ == cur_extent_->num_blocks() * block_size_) {
+ // We have to advance the cur_extent_;
+ cur_extent_++;
+ cur_extent_bytes_read_ = 0;
+ }
+ }
+ return true;
+}
+
+} // namespace chromeos_update_engine
diff --git a/payload_consumer/extent_reader.h b/payload_consumer/extent_reader.h
new file mode 100644
index 0000000..3f9e4c8
--- /dev/null
+++ b/payload_consumer/extent_reader.h
@@ -0,0 +1,82 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef UPDATE_ENGINE_PAYLOAD_CONSUMER_EXTENT_READER_H_
+#define UPDATE_ENGINE_PAYLOAD_CONSUMER_EXTENT_READER_H_
+
+#include <vector>
+
+#include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/update_metadata.pb.h"
+
+namespace chromeos_update_engine {
+
+// ExtentReader is an abstract class with reads from a given file descriptor at
+// the extents given.
+class ExtentReader {
+ public:
+ virtual ~ExtentReader() = default;
+
+ // Initializes |ExtentReader|
+ virtual bool Init(FileDescriptorPtr fd,
+ const google::protobuf::RepeatedPtrField<Extent>& extents,
+ uint32_t block_size) = 0;
+
+ // Seeks to the given |offset| assuming all extents are concatenated together.
+ virtual bool Seek(uint64_t offset) = 0;
+
+ // Returns true on success.
+ virtual bool Read(void* buffer, size_t count) = 0;
+};
+
+// DirectExtentReader is probably the simplest ExtentReader implementation.
+// It reads the data directly from the extents.
+class DirectExtentReader : public ExtentReader {
+ public:
+ DirectExtentReader() = default;
+ ~DirectExtentReader() override = default;
+
+ bool Init(FileDescriptorPtr fd,
+ const google::protobuf::RepeatedPtrField<Extent>& extents,
+ uint32_t block_size) override;
+ bool Seek(uint64_t offset) override;
+ bool Read(void* bytes, size_t count) override;
+
+ private:
+ FileDescriptorPtr fd_{nullptr};
+ google::protobuf::RepeatedPtrField<Extent> extents_;
+ size_t block_size_{0};
+
+ // Current extent being read from |fd_|.
+ google::protobuf::RepeatedPtrField<Extent>::iterator cur_extent_;
+
+ // Bytes read from |cur_extent_| thus far.
+ uint64_t cur_extent_bytes_read_{0};
+
+ // Offset assuming all extents are concatenated.
+ uint64_t offset_{0};
+
+ // The accelaring upper bounds for |extents_| if we assume all extents are
+ // concatenated.
+ std::vector<uint64_t> extents_upper_bounds_;
+ uint64_t total_size_{0};
+
+ DISALLOW_COPY_AND_ASSIGN(DirectExtentReader);
+};
+
+} // namespace chromeos_update_engine
+
+#endif // UPDATE_ENGINE_PAYLOAD_CONSUMER_EXTENT_READER_H_
diff --git a/payload_consumer/extent_reader_unittest.cc b/payload_consumer/extent_reader_unittest.cc
new file mode 100644
index 0000000..9821775
--- /dev/null
+++ b/payload_consumer/extent_reader_unittest.cc
@@ -0,0 +1,170 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_consumer/extent_reader.h"
+
+#include <fcntl.h>
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include <brillo/secure_blob.h>
+#include <gtest/gtest.h>
+
+#include "update_engine/common/test_utils.h"
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_consumer/file_descriptor.h"
+#include "update_engine/payload_consumer/payload_constants.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+#include "update_engine/payload_generator/extent_utils.h"
+
+using chromeos_update_engine::test_utils::ExpectVectorsEq;
+using std::min;
+using std::string;
+using std::vector;
+
+namespace chromeos_update_engine {
+
+namespace {
+const size_t kBlockSize = 8;
+const size_t kRandomIterations = 1000;
+} // namespace
+
+class ExtentReaderTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ sample_.resize(4096 * 10);
+ srand(time(nullptr));
+ unsigned int rand_seed;
+ for (size_t i = 0; i < sample_.size(); i++) {
+ sample_[i] = rand_r(&rand_seed) % 256;
+ }
+ ASSERT_TRUE(utils::WriteFile(
+ temp_file_.path().c_str(), sample_.data(), sample_.size()));
+
+ fd_.reset(new EintrSafeFileDescriptor());
+ ASSERT_TRUE(fd_->Open(temp_file_.path().c_str(), O_RDONLY, 0600));
+ }
+ void TearDown() override { fd_->Close(); }
+
+ void ReadExtents(vector<Extent> extents, brillo::Blob* blob) {
+ blob->clear();
+ for (const auto& extent : extents) {
+ blob->insert(
+ blob->end(),
+ &sample_[extent.start_block() * kBlockSize],
+ &sample_[(extent.start_block() + extent.num_blocks()) * kBlockSize]);
+ }
+ }
+
+ FileDescriptorPtr fd_;
+ test_utils::ScopedTempFile temp_file_{"ExtentReaderTest-file.XXXXXX"};
+ brillo::Blob sample_;
+};
+
+TEST_F(ExtentReaderTest, SimpleTest) {
+ vector<Extent> extents = {ExtentForRange(1, 1)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+ EXPECT_TRUE(reader.Seek(0));
+ brillo::Blob blob1(BlocksInExtents(extents) * kBlockSize);
+ EXPECT_TRUE(reader.Read(blob1.data(), blob1.size()));
+ brillo::Blob blob2;
+ ReadExtents(extents, &blob2);
+ ExpectVectorsEq(blob1, blob2);
+}
+
+TEST_F(ExtentReaderTest, ZeroExtentLengthTest) {
+ vector<Extent> extents = {ExtentForRange(1, 0)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+ EXPECT_TRUE(reader.Seek(0));
+ brillo::Blob blob(1);
+ EXPECT_TRUE(reader.Read(blob.data(), 0));
+ EXPECT_FALSE(reader.Read(blob.data(), 1));
+}
+
+TEST_F(ExtentReaderTest, NoExtentTest) {
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {}, kBlockSize));
+ EXPECT_TRUE(reader.Seek(0));
+ brillo::Blob blob(1);
+ EXPECT_TRUE(reader.Read(blob.data(), 0));
+ EXPECT_FALSE(reader.Read(blob.data(), 1));
+}
+
+TEST_F(ExtentReaderTest, OverflowExtentTest) {
+ vector<Extent> extents = {ExtentForRange(1, 1)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+ EXPECT_TRUE(reader.Seek(0));
+ brillo::Blob blob(BlocksInExtents(extents) * kBlockSize + 1);
+ EXPECT_FALSE(reader.Read(blob.data(), blob.size()));
+}
+
+TEST_F(ExtentReaderTest, SeekOverflow1Test) {
+ vector<Extent> extents = {ExtentForRange(1, 0)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+ EXPECT_TRUE(reader.Seek(0));
+ EXPECT_FALSE(reader.Seek(1));
+}
+
+TEST_F(ExtentReaderTest, SeekOverflow2Test) {
+ DirectExtentReader reader;
+ reader.Init(fd_, {}, kBlockSize);
+ EXPECT_TRUE(reader.Seek(0));
+ EXPECT_FALSE(reader.Seek(1));
+}
+
+TEST_F(ExtentReaderTest, SeekOverflow3Test) {
+ vector<Extent> extents = {ExtentForRange(1, 1)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+ // Seek to the end of the extents should be fine as long as nothing is read.
+ EXPECT_TRUE(reader.Seek(kBlockSize));
+ EXPECT_FALSE(reader.Seek(kBlockSize + 1));
+}
+
+TEST_F(ExtentReaderTest, RandomReadTest) {
+ vector<Extent> extents = {ExtentForRange(0, 0),
+ ExtentForRange(1, 1),
+ ExtentForRange(3, 0),
+ ExtentForRange(4, 2),
+ ExtentForRange(7, 1)};
+ DirectExtentReader reader;
+ EXPECT_TRUE(reader.Init(fd_, {extents.begin(), extents.end()}, kBlockSize));
+
+ brillo::Blob result;
+ ReadExtents(extents, &result);
+
+ brillo::Blob blob(BlocksInExtents(extents) * kBlockSize);
+ srand(time(nullptr));
+ uint32_t rand_seed;
+ for (size_t idx = 0; idx < kRandomIterations; idx++) {
+ // zero to full size available.
+ size_t start = rand_r(&rand_seed) % blob.size();
+ size_t size = rand_r(&rand_seed) % (blob.size() - start);
+ EXPECT_TRUE(reader.Seek(start));
+ EXPECT_TRUE(reader.Read(blob.data(), size));
+ for (size_t i = 0; i < size; i++) {
+ ASSERT_EQ(blob[i], result[start + i]);
+ }
+ }
+}
+
+} // namespace chromeos_update_engine
diff --git a/update_engine.gyp b/update_engine.gyp
index 7a1ad95..41ee1e8 100644
--- a/update_engine.gyp
+++ b/update_engine.gyp
@@ -187,6 +187,7 @@
'payload_consumer/bzip_extent_writer.cc',
'payload_consumer/delta_performer.cc',
'payload_consumer/download_action.cc',
+ 'payload_consumer/extent_reader.cc',
'payload_consumer/extent_writer.cc',
'payload_consumer/file_descriptor.cc',
'payload_consumer/file_descriptor_utils.cc',
@@ -532,6 +533,7 @@
'payload_consumer/delta_performer_integration_test.cc',
'payload_consumer/delta_performer_unittest.cc',
'payload_consumer/download_action_unittest.cc',
+ 'payload_consumer/extent_reader_unittest.cc',
'payload_consumer/extent_writer_unittest.cc',
'payload_consumer/fake_file_descriptor.cc',
'payload_consumer/file_descriptor_utils_unittest.cc',