Merge "Removing duplicate CollectImages()"
diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp
index fb0e18b..287285b 100644
--- a/fastboot/fastboot.cpp
+++ b/fastboot/fastboot.cpp
@@ -2230,7 +2230,9 @@
{"version", no_argument, 0, 0},
{0, 0, 0, 0}};
- serial = getenv("ANDROID_SERIAL");
+ serial = getenv("FASTBOOT_DEVICE");
+ if (!serial)
+ serial = getenv("ANDROID_SERIAL");
int c;
while ((c = getopt_long(argc, argv, "a::hls:S:vw", longopts, &longindex)) != -1) {
diff --git a/fastboot/fastboot_driver.h b/fastboot/fastboot_driver.h
index 3d6c7b0..6ac26ce 100644
--- a/fastboot/fastboot_driver.h
+++ b/fastboot/fastboot_driver.h
@@ -28,6 +28,7 @@
#pragma once
#include <cstdlib>
#include <deque>
+#include <functional>
#include <limits>
#include <string>
#include <vector>
diff --git a/fastboot/storage.cpp b/fastboot/storage.cpp
index dc733b9..629ebc8 100644
--- a/fastboot/storage.cpp
+++ b/fastboot/storage.cpp
@@ -18,6 +18,7 @@
#include <android-base/logging.h>
#include <fstream>
+#include <iterator>
#include "storage.h"
#include "util.h"
@@ -62,4 +63,4 @@
LOG(FATAL) << "Cannot create directory: " << home_fastboot_path_;
}
return FileLock(devices_lock_path_);
-}
\ No newline at end of file
+}
diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp
index cba6844..e931bec 100644
--- a/fs_mgr/libsnapshot/Android.bp
+++ b/fs_mgr/libsnapshot/Android.bp
@@ -174,12 +174,13 @@
"libsnapshot_cow_defaults",
],
srcs: [
- "libsnapshot_cow/cow_decompress.cpp",
- "libsnapshot_cow/cow_reader.cpp",
- "libsnapshot_cow/cow_writer.cpp",
- "libsnapshot_cow/cow_format.cpp",
"libsnapshot_cow/cow_compress.cpp",
+ "libsnapshot_cow/cow_decompress.cpp",
+ "libsnapshot_cow/cow_format.cpp",
+ "libsnapshot_cow/cow_reader.cpp",
"libsnapshot_cow/parser_v2.cpp",
+ "libsnapshot_cow/writer_base.cpp",
+ "libsnapshot_cow/writer_v2.cpp",
],
host_supported: true,
recovery_available: true,
@@ -371,7 +372,7 @@
"libsnapshot_cow_defaults",
],
srcs: [
- "libsnapshot_cow/cow_api_test.cpp",
+ "libsnapshot_cow/test_v2.cpp",
],
cflags: [
"-D_FILE_OFFSET_BITS=64",
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
index b228dff..dd626bc 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
@@ -31,6 +31,10 @@
static constexpr uint32_t kMinCowVersion = 1;
static constexpr uint32_t kMaxCowVersion = 2;
+// Normally, this should be kMaxCowVersion. When a new version is under testing
+// it may be the previous value of kMaxCowVersion.
+static constexpr uint32_t kDefaultCowVersion = 2;
+
// This header appears as the first sequence of bytes in the COW. All fields
// in the layout are little-endian encoded. The on-disk layout is:
//
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
index 7881f35..af2d3ef 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
@@ -1,16 +1,16 @@
-// Copyright (C) 2019 The Android Open Source Project
+// copyright (c) 2019 the android open source project
//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
+// licensed under the apache license, version 2.0 (the "license");
+// you may not use this file except in compliance with the license.
+// you may obtain a copy of the license at
//
-// http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/license-2.0
//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+// unless required by applicable law or agreed to in writing, software
+// distributed under the license is distributed on an "as is" basis,
+// without warranties or conditions of any kind, either express or implied.
+// see the license for the specific language governing permissions and
+// limitations under the license.
#pragma once
@@ -61,30 +61,28 @@
// will occur in the sequence they were added to the COW.
class ICowWriter {
public:
- explicit ICowWriter(const CowOptions& options) : options_(options) {}
-
virtual ~ICowWriter() {}
// Encode an operation that copies the contents of |old_block| to the
// location of |new_block|. 'num_blocks' is the number of contiguous
// COPY operations from |old_block| to |new_block|.
- bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1);
+ virtual bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0;
// Encode a sequence of raw blocks. |size| must be a multiple of the block size.
- bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size);
+ virtual bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0;
// Add a sequence of xor'd blocks. |size| must be a multiple of the block size.
- bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block,
- uint16_t offset);
+ virtual bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size,
+ uint32_t old_block, uint16_t offset) = 0;
// Encode a sequence of zeroed blocks. |size| must be a multiple of the block size.
- bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks);
+ virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;
// Add a label to the op sequence.
- bool AddLabel(uint64_t label);
+ virtual bool AddLabel(uint64_t label) = 0;
// Add sequence data for op merging. Data is a list of the destination block numbers.
- bool AddSequenceData(size_t num_ops, const uint32_t* data);
+ virtual bool AddSequenceData(size_t num_ops, const uint32_t* data) = 0;
// Flush all pending writes. This must be called before closing the writer
// to ensure that the correct headers and footers are written.
@@ -93,21 +91,8 @@
// Return number of bytes the cow image occupies on disk.
virtual uint64_t GetCowSize() = 0;
- const CowOptions& options() { return options_; }
-
- protected:
- virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0;
- virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0;
- virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,
- uint32_t old_block, uint16_t offset) = 0;
- virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;
- virtual bool EmitLabel(uint64_t label) = 0;
- virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) = 0;
-
- bool ValidateNewBlock(uint64_t new_block);
-
- protected:
- CowOptions options_;
+ virtual uint32_t GetBlockSize() const = 0;
+ virtual std::optional<uint32_t> GetMaxBlocks() const = 0;
};
class CompressWorker {
@@ -146,96 +131,15 @@
std::vector<std::basic_string<uint8_t>>* compressed_data);
};
-class CowWriter : public ICowWriter {
- public:
- explicit CowWriter(const CowOptions& options);
- ~CowWriter();
+// Create an ICowWriter not backed by any file. This is useful for estimating
+// the final size of a cow file.
+std::unique_ptr<ICowWriter> CreateCowEstimator(uint32_t version, const CowOptions& options);
- // Set up the writer.
- // The file starts from the beginning.
- //
- // If fd is < 0, the CowWriter will be opened against /dev/null. This is for
- // computing COW sizes without using storage space.
- bool Initialize(android::base::unique_fd&& fd);
- bool Initialize(android::base::borrowed_fd fd);
- // Set up a writer, assuming that the given label is the last valid label.
- // This will result in dropping any labels that occur after the given on, and will fail
- // if the given label does not appear.
- bool InitializeAppend(android::base::unique_fd&&, uint64_t label);
- bool InitializeAppend(android::base::borrowed_fd fd, uint64_t label);
-
- bool Finalize() override;
-
- uint64_t GetCowSize() override;
-
- protected:
- virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;
- virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
- virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,
- uint32_t old_block, uint16_t offset) override;
- virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
- virtual bool EmitLabel(uint64_t label) override;
- virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) override;
-
- private:
- bool EmitCluster();
- bool EmitClusterIfNeeded();
- bool EmitBlocks(uint64_t new_block_start, const void* data, size_t size, uint64_t old_block,
- uint16_t offset, uint8_t type);
- void SetupHeaders();
- void SetupWriteOptions();
- bool ParseOptions();
- bool OpenForWrite();
- bool OpenForAppend(uint64_t label);
- bool GetDataPos(uint64_t* pos);
- bool WriteRawData(const void* data, size_t size);
- bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0);
- void AddOperation(const CowOperation& op);
- void InitPos();
- void InitBatchWrites();
- void InitWorkers();
- bool FlushCluster();
-
- bool CompressBlocks(size_t num_blocks, const void* data);
- bool SetFd(android::base::borrowed_fd fd);
- bool Sync();
- bool Truncate(off_t length);
- bool EnsureSpaceAvailable(const uint64_t bytes_needed) const;
-
- private:
- android::base::unique_fd owned_fd_;
- android::base::borrowed_fd fd_;
- CowHeader header_{};
- CowFooter footer_{};
- CowCompressionAlgorithm compression_ = kCowCompressNone;
- uint64_t current_op_pos_ = 0;
- uint64_t next_op_pos_ = 0;
- uint64_t next_data_pos_ = 0;
- uint64_t current_data_pos_ = 0;
- ssize_t total_data_written_ = 0;
- uint32_t cluster_size_ = 0;
- uint32_t current_cluster_size_ = 0;
- uint64_t current_data_size_ = 0;
- bool is_dev_null_ = false;
- bool merge_in_progress_ = false;
- bool is_block_device_ = false;
- uint64_t cow_image_size_ = INT64_MAX;
-
- int num_compress_threads_ = 1;
- std::vector<std::unique_ptr<CompressWorker>> compress_threads_;
- std::vector<std::future<bool>> threads_;
- std::vector<std::basic_string<uint8_t>> compressed_buf_;
- std::vector<std::basic_string<uint8_t>>::iterator buf_iter_;
-
- std::vector<std::unique_ptr<CowOperation>> opbuffer_vec_;
- std::vector<std::unique_ptr<uint8_t[]>> databuffer_vec_;
- std::unique_ptr<struct iovec[]> cowop_vec_;
- int op_vec_index_ = 0;
-
- std::unique_ptr<struct iovec[]> data_vec_;
- int data_vec_index_ = 0;
- bool batch_write_ = false;
-};
+// Create an ICowWriter of the given version and options. If a label is given,
+// the writer is opened in append mode.
+std::unique_ptr<ICowWriter> CreateCowWriter(uint32_t version, const CowOptions& options,
+ android::base::unique_fd&& fd,
+ std::optional<uint64_t> label = {});
} // namespace snapshot
} // namespace android
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h
index d798e25..52e3a9c 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h
@@ -23,30 +23,23 @@
public:
using FileDescriptor = ISnapshotWriter::FileDescriptor;
- explicit MockSnapshotWriter(const CowOptions& options) : ISnapshotWriter(options) {}
- MockSnapshotWriter() : ISnapshotWriter({}) {}
-
MOCK_METHOD(bool, Finalize, (), (override));
// Return number of bytes the cow image occupies on disk.
MOCK_METHOD(uint64_t, GetCowSize, (), (override));
- MOCK_METHOD(bool, EmitCopy, (uint64_t, uint64_t, uint64_t), (override));
- MOCK_METHOD(bool, EmitRawBlocks, (uint64_t, const void*, size_t), (override));
- MOCK_METHOD(bool, EmitXorBlocks, (uint32_t, const void*, size_t, uint32_t, uint16_t),
+ MOCK_METHOD(bool, AddCopy, (uint64_t, uint64_t, uint64_t), (override));
+ MOCK_METHOD(bool, AddRawBlocks, (uint64_t, const void*, size_t), (override));
+ MOCK_METHOD(bool, AddXorBlocks, (uint32_t, const void*, size_t, uint32_t, uint16_t),
(override));
- MOCK_METHOD(bool, EmitZeroBlocks, (uint64_t, uint64_t), (override));
- MOCK_METHOD(bool, EmitLabel, (uint64_t), (override));
- MOCK_METHOD(bool, EmitSequenceData, (size_t, const uint32_t*), (override));
-
- // Open the writer in write mode (no append).
+ MOCK_METHOD(bool, AddZeroBlocks, (uint64_t, uint64_t), (override));
+ MOCK_METHOD(bool, AddLabel, (uint64_t), (override));
+ MOCK_METHOD(bool, AddSequenceData, (size_t, const uint32_t*), (override));
MOCK_METHOD(bool, Initialize, (), (override));
+ MOCK_METHOD(bool, InitializeAppend, (uint64_t), (override));
MOCK_METHOD(bool, VerifyMergeOps, (), (override, const, noexcept));
-
- // Open the writer in append mode, with the last label to resume
- // from. See CowWriter::InitializeAppend.
- MOCK_METHOD(bool, InitializeAppend, (uint64_t label), (override));
-
MOCK_METHOD(std::unique_ptr<FileDescriptor>, OpenReader, (), (override));
+ MOCK_METHOD(uint32_t, GetBlockSize, (), (override, const));
+ MOCK_METHOD(std::optional<uint32_t>, GetMaxBlocks, (), (override, const));
};
} // namespace android::snapshot
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h
index 8f6344c..2653a60 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h
@@ -31,13 +31,7 @@
public:
using FileDescriptor = chromeos_update_engine::FileDescriptor;
- explicit ISnapshotWriter(const CowOptions& options);
-
- // Set the source device. This is used for AddCopy() operations, if the
- // underlying writer needs the original bytes (for example if backed by
- // dm-snapshot or if writing directly to an unsnapshotted region). The
- // device is only opened on the first operation that requires it.
- void SetSourceDevice(const std::string& source_device);
+ virtual ~ISnapshotWriter() {}
// Open the writer in write mode (no append).
virtual bool Initialize() = 0;
@@ -47,15 +41,8 @@
virtual bool InitializeAppend(uint64_t label) = 0;
virtual std::unique_ptr<FileDescriptor> OpenReader() = 0;
+
virtual bool VerifyMergeOps() const noexcept = 0;
-
- protected:
- android::base::borrowed_fd GetSourceFd();
-
- std::optional<std::string> source_device_;
-
- private:
- android::base::unique_fd source_fd_;
};
// Send writes to a COW or a raw device directly, based on a threshold.
@@ -63,6 +50,8 @@
public:
CompressedSnapshotWriter(const CowOptions& options);
+ void SetSourceDevice(const std::string& source_device);
+
// Sets the COW device; this is required.
bool SetCowDevice(android::base::unique_fd&& cow_device);
@@ -70,23 +59,34 @@
bool InitializeAppend(uint64_t label) override;
bool Finalize() override;
uint64_t GetCowSize() override;
+ uint32_t GetBlockSize() const override;
+ std::optional<uint32_t> GetMaxBlocks() const override;
std::unique_ptr<FileDescriptor> OpenReader() override;
bool VerifyMergeOps() const noexcept;
- protected:
- bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;
- bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
- bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block,
- uint16_t offset) override;
- bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
- bool EmitLabel(uint64_t label) override;
- bool EmitSequenceData(size_t num_ops, const uint32_t* data) override;
+ bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;
+ bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
+ bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block,
+ uint16_t offset) override;
+ bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
+ bool AddLabel(uint64_t label) override;
+ bool AddSequenceData(size_t num_ops, const uint32_t* data) override;
private:
std::unique_ptr<CowReader> OpenCowReader() const;
- android::base::unique_fd cow_device_;
+ android::base::borrowed_fd GetSourceFd();
- std::unique_ptr<CowWriter> cow_;
+ CowOptions options_;
+
+ // Set the source device. This is used for AddCopy() operations, if the
+ // underlying writer needs the original bytes (for example if backed by
+ // dm-snapshot or if writing directly to an unsnapshotted region). The
+ // device is only opened on the first operation that requires it.
+ std::optional<std::string> source_device_;
+ android::base::unique_fd source_fd_;
+
+ android::base::unique_fd cow_device_;
+ std::unique_ptr<ICowWriter> cow_;
};
} // namespace snapshot
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp
index 2157d0f..ff3ccec 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp
@@ -19,10 +19,13 @@
#include <unistd.h>
#include <android-base/logging.h>
+#include "writer_v2.h"
namespace android {
namespace snapshot {
+using android::base::unique_fd;
+
std::ostream& operator<<(std::ostream& os, CowOperation const& op) {
os << "CowOperation(type:";
if (op.type == kCowCopyOp)
@@ -103,5 +106,27 @@
}
}
+std::unique_ptr<ICowWriter> CreateCowWriter(uint32_t version, const CowOptions& options,
+ unique_fd&& fd, std::optional<uint64_t> label) {
+ std::unique_ptr<CowWriterBase> base;
+ switch (version) {
+ case 1:
+ case 2:
+ base = std::make_unique<CowWriterV2>(options, std::move(fd));
+ break;
+ default:
+ LOG(ERROR) << "Cannot create unknown cow version: " << version;
+ return nullptr;
+ }
+ if (!base->Initialize(label)) {
+ return nullptr;
+ }
+ return base;
+}
+
+std::unique_ptr<ICowWriter> CreateCowEstimator(uint32_t version, const CowOptions& options) {
+ return CreateCowWriter(version, options, unique_fd{-1}, std::nullopt);
+}
+
} // namespace snapshot
} // namespace android
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp
similarity index 91%
rename from fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp
rename to fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp
index 8f80bb3..120b2c0 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp
@@ -25,7 +25,9 @@
#include <libsnapshot/cow_reader.h>
#include <libsnapshot/cow_writer.h>
#include "cow_decompress.h"
+#include "writer_v2.h"
+using android::base::unique_fd;
using testing::AssertionFailure;
using testing::AssertionResult;
using testing::AssertionSuccess;
@@ -42,6 +44,8 @@
virtual void TearDown() override { cow_ = nullptr; }
+ unique_fd GetCowFd() { return unique_fd{dup(cow_->fd)}; }
+
std::unique_ptr<TemporaryFile> cow_;
};
@@ -53,9 +57,9 @@
TEST_F(CowTest, CopyContiguous) {
CowOptions options;
options.cluster_ops = 0;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
ASSERT_TRUE(writer.AddCopy(10, 1000, 100));
ASSERT_TRUE(writer.Finalize());
@@ -96,9 +100,9 @@
TEST_F(CowTest, ReadWrite) {
CowOptions options;
options.cluster_ops = 0;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
@@ -175,9 +179,9 @@
TEST_F(CowTest, ReadWriteXor) {
CowOptions options;
options.cluster_ops = 0;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
@@ -256,9 +260,9 @@
CowOptions options;
options.cluster_ops = 0;
options.compression = "gz";
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
@@ -296,9 +300,9 @@
options.compression = GetParam();
options.num_compress_threads = 2;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
std::string xor_data = "This is test data-1. Testing xor";
xor_data.resize(options.block_size, '\0');
@@ -374,9 +378,9 @@
options.num_compress_threads = 1;
options.cluster_ops = 0;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
std::string data = "Testing replace ops without batch writes";
data.resize(options.block_size * 1024, '\0');
@@ -497,9 +501,9 @@
CowOptions options;
options.compression = "gz";
options.cluster_ops = 2;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
@@ -562,9 +566,9 @@
CowOptions options;
options.compression = "gz";
options.cluster_ops = 0;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
std::string data = "This is some data, believe it";
data.resize(options.block_size * 2, '\0');
@@ -595,12 +599,12 @@
TEST_F(CowTest, GetSize) {
CowOptions options;
options.cluster_ops = 0;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
if (ftruncate(cow_->fd, 0) < 0) {
perror("Fails to set temp file size");
FAIL();
}
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
@@ -621,8 +625,8 @@
TEST_F(CowTest, AppendLabelSmall) {
CowOptions options;
options.cluster_ops = 0;
- auto writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
@@ -632,8 +636,8 @@
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
- writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 3));
+ writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize({3}));
std::string data2 = "More data!";
data2.resize(options.block_size, '\0');
@@ -688,8 +692,8 @@
TEST_F(CowTest, AppendLabelMissing) {
CowOptions options;
options.cluster_ops = 0;
- auto writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
ASSERT_TRUE(writer->AddLabel(0));
std::string data = "This is some data, believe it";
@@ -701,9 +705,9 @@
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
- writer = std::make_unique<CowWriter>(options);
- ASSERT_FALSE(writer->InitializeAppend(cow_->fd, 1));
- ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 0));
+ writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_FALSE(writer->Initialize({1}));
+ ASSERT_TRUE(writer->Initialize({0}));
ASSERT_TRUE(writer->AddZeroBlocks(51, 1));
ASSERT_TRUE(writer->Finalize());
@@ -740,8 +744,8 @@
TEST_F(CowTest, AppendExtendedCorrupted) {
CowOptions options;
options.cluster_ops = 0;
- auto writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
ASSERT_TRUE(writer->AddLabel(5));
@@ -763,8 +767,8 @@
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
- writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 5));
+ writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize({5}));
ASSERT_TRUE(writer->Finalize());
@@ -791,8 +795,8 @@
TEST_F(CowTest, AppendbyLabel) {
CowOptions options;
options.cluster_ops = 0;
- auto writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
std::string data = "This is some data, believe it";
data.resize(options.block_size * 2, '\0');
@@ -810,9 +814,9 @@
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
- writer = std::make_unique<CowWriter>(options);
- ASSERT_FALSE(writer->InitializeAppend(cow_->fd, 12));
- ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 5));
+ writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_FALSE(writer->Initialize({12}));
+ ASSERT_TRUE(writer->Initialize({5}));
// This should drop label 6
ASSERT_TRUE(writer->Finalize());
@@ -879,8 +883,8 @@
TEST_F(CowTest, ClusterTest) {
CowOptions options;
options.cluster_ops = 4;
- auto writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
@@ -976,16 +980,16 @@
TEST_F(CowTest, ClusterAppendTest) {
CowOptions options;
options.cluster_ops = 3;
- auto writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
ASSERT_TRUE(writer->AddLabel(50));
ASSERT_TRUE(writer->Finalize()); // Adds a cluster op, should be dropped on append
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
- writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 50));
+ writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize({50}));
std::string data2 = "More data!";
data2.resize(options.block_size, '\0');
@@ -1037,8 +1041,8 @@
TEST_F(CowTest, AppendAfterFinalize) {
CowOptions options;
options.cluster_ops = 0;
- auto writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
@@ -1058,8 +1062,8 @@
ASSERT_TRUE(reader.Parse(cow_->fd));
}
-AssertionResult WriteDataBlock(CowWriter* writer, uint64_t new_block, std::string data) {
- data.resize(writer->options().block_size, '\0');
+AssertionResult WriteDataBlock(ICowWriter* writer, uint64_t new_block, std::string data) {
+ data.resize(writer->GetBlockSize(), '\0');
if (!writer->AddRawBlocks(new_block, data.data(), data.size())) {
return AssertionFailure() << "Failed to add raw block";
}
@@ -1088,8 +1092,8 @@
TEST_F(CowTest, ResumeMidCluster) {
CowOptions options;
options.cluster_ops = 7;
- auto writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
ASSERT_TRUE(WriteDataBlock(writer.get(), 1, "Block 1"));
ASSERT_TRUE(WriteDataBlock(writer.get(), 2, "Block 2"));
@@ -1099,8 +1103,8 @@
ASSERT_TRUE(WriteDataBlock(writer.get(), 4, "Block 4"));
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
- writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1));
+ writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize({1}));
ASSERT_TRUE(WriteDataBlock(writer.get(), 4, "Block 4"));
ASSERT_TRUE(WriteDataBlock(writer.get(), 5, "Block 5"));
ASSERT_TRUE(WriteDataBlock(writer.get(), 6, "Block 6"));
@@ -1145,8 +1149,8 @@
CowOptions options;
int cluster_ops = 5;
options.cluster_ops = cluster_ops;
- auto writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
ASSERT_TRUE(WriteDataBlock(writer.get(), 1, "Block 1"));
ASSERT_TRUE(WriteDataBlock(writer.get(), 2, "Block 2"));
@@ -1160,8 +1164,8 @@
ASSERT_TRUE(WriteDataBlock(writer.get(), 8, "Block 8"));
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
- writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1));
+ writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize({1}));
ASSERT_TRUE(WriteDataBlock(writer.get(), 4, "Block 4"));
ASSERT_TRUE(WriteDataBlock(writer.get(), 5, "Block 5"));
ASSERT_TRUE(WriteDataBlock(writer.get(), 6, "Block 6"));
@@ -1205,8 +1209,8 @@
TEST_F(CowTest, DeleteMidCluster) {
CowOptions options;
options.cluster_ops = 7;
- auto writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
ASSERT_TRUE(WriteDataBlock(writer.get(), 1, "Block 1"));
ASSERT_TRUE(WriteDataBlock(writer.get(), 2, "Block 2"));
@@ -1218,8 +1222,8 @@
ASSERT_TRUE(WriteDataBlock(writer.get(), 6, "Block 6"));
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
- writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1));
+ writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize({1}));
ASSERT_TRUE(writer->Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@@ -1255,14 +1259,14 @@
TEST_F(CowTest, BigSeqOp) {
CowOptions options;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
const int seq_len = std::numeric_limits<uint16_t>::max() / sizeof(uint32_t) + 1;
uint32_t sequence[seq_len];
for (int i = 0; i < seq_len; i++) {
sequence[i] = i + 1;
}
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
ASSERT_TRUE(writer.AddSequenceData(seq_len, sequence));
ASSERT_TRUE(writer.AddZeroBlocks(1, seq_len));
@@ -1287,14 +1291,14 @@
TEST_F(CowTest, MissingSeqOp) {
CowOptions options;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
const int seq_len = 10;
uint32_t sequence[seq_len];
for (int i = 0; i < seq_len; i++) {
sequence[i] = i + 1;
}
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
ASSERT_TRUE(writer.AddSequenceData(seq_len, sequence));
ASSERT_TRUE(writer.AddZeroBlocks(1, seq_len - 1));
@@ -1308,14 +1312,14 @@
TEST_F(CowTest, ResumeSeqOp) {
CowOptions options;
- auto writer = std::make_unique<CowWriter>(options);
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
const int seq_len = 10;
uint32_t sequence[seq_len];
for (int i = 0; i < seq_len; i++) {
sequence[i] = i + 1;
}
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ ASSERT_TRUE(writer->Initialize());
ASSERT_TRUE(writer->AddSequenceData(seq_len, sequence));
ASSERT_TRUE(writer->AddZeroBlocks(1, seq_len / 2));
@@ -1328,8 +1332,8 @@
auto itr = reader->GetRevMergeOpIter();
ASSERT_TRUE(itr->AtEnd());
- writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1));
+ writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize({1}));
ASSERT_TRUE(writer->AddZeroBlocks(1 + seq_len / 2, seq_len / 2));
ASSERT_TRUE(writer->Finalize());
@@ -1358,10 +1362,10 @@
CowOptions options;
options.cluster_ops = 5;
options.num_merge_ops = 1;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
uint32_t sequence[] = {2, 10, 6, 7, 3, 5};
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
ASSERT_TRUE(writer.AddSequenceData(6, sequence));
ASSERT_TRUE(writer.AddCopy(6, 13));
@@ -1408,9 +1412,9 @@
CowOptions options;
options.cluster_ops = 5;
options.num_merge_ops = 1;
- CowWriter writer(options);
+ CowWriterV2 writer(options, GetCowFd());
- ASSERT_TRUE(writer.Initialize(cow_->fd));
+ ASSERT_TRUE(writer.Initialize());
ASSERT_TRUE(writer.AddCopy(2, 11));
ASSERT_TRUE(writer.AddCopy(10, 12));
@@ -1459,10 +1463,10 @@
options.num_merge_ops = 1;
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
- auto writer = std::make_unique<CowWriter>(options);
+ auto writer = std::make_unique<CowWriterV2>(options, GetCowFd());
CowReader reader;
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ ASSERT_TRUE(writer->Initialize());
ASSERT_TRUE(writer->AddCopy(3, 2));
ASSERT_TRUE(writer->AddCopy(2, 1));
@@ -1471,14 +1475,14 @@
ASSERT_TRUE(reader.Parse(cow_->fd));
ASSERT_TRUE(reader.VerifyMergeOps());
- ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1));
+ ASSERT_TRUE(writer->Initialize({1}));
ASSERT_TRUE(writer->AddCopy(4, 2));
ASSERT_TRUE(writer->Finalize());
ASSERT_TRUE(reader.Parse(cow_->fd));
ASSERT_FALSE(reader.VerifyMergeOps());
- writer = std::make_unique<CowWriter>(options);
- ASSERT_TRUE(writer->Initialize(cow_->fd));
+ writer = std::make_unique<CowWriterV2>(options, GetCowFd());
+ ASSERT_TRUE(writer->Initialize());
ASSERT_TRUE(writer->AddCopy(2, 1));
ASSERT_TRUE(writer->AddXorBlocks(3, &data, data.size(), 1, 1));
ASSERT_TRUE(writer->Finalize());
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.cpp
new file mode 100644
index 0000000..22e63d0
--- /dev/null
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.cpp
@@ -0,0 +1,163 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "writer_base.h"
+
+#include <fcntl.h>
+#include <linux/fs.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <android-base/logging.h>
+
+// The info messages here are spammy, but as useful for update_engine. Disable
+// them when running on the host.
+#ifdef __ANDROID__
+#define LOG_INFO LOG(INFO)
+#else
+#define LOG_INFO LOG(VERBOSE)
+#endif
+
+namespace android {
+namespace snapshot {
+
+using android::base::borrowed_fd;
+using android::base::unique_fd;
+
+namespace {
+std::string GetFdPath(borrowed_fd fd) {
+ const auto fd_path = "/proc/self/fd/" + std::to_string(fd.get());
+ std::string file_path(512, '\0');
+ const auto err = readlink(fd_path.c_str(), file_path.data(), file_path.size());
+ if (err <= 0) {
+ PLOG(ERROR) << "Failed to determine path for fd " << fd.get();
+ file_path.clear();
+ } else {
+ file_path.resize(err);
+ }
+ return file_path;
+}
+} // namespace
+
+CowWriterBase::CowWriterBase(const CowOptions& options, unique_fd&& fd)
+ : options_(options), fd_(std::move(fd)) {}
+
+bool CowWriterBase::InitFd() {
+ if (fd_.get() < 0) {
+ fd_.reset(open("/dev/null", O_RDWR | O_CLOEXEC));
+ if (fd_ < 0) {
+ PLOG(ERROR) << "open /dev/null failed";
+ return false;
+ }
+ is_dev_null_ = true;
+ return true;
+ }
+
+ struct stat stat {};
+ if (fstat(fd_.get(), &stat) < 0) {
+ PLOG(ERROR) << "fstat failed";
+ return false;
+ }
+ const auto file_path = GetFdPath(fd_);
+ is_block_device_ = S_ISBLK(stat.st_mode);
+ if (is_block_device_) {
+ uint64_t size_in_bytes = 0;
+ if (ioctl(fd_.get(), BLKGETSIZE64, &size_in_bytes)) {
+ PLOG(ERROR) << "Failed to get total size for: " << fd_.get();
+ return false;
+ }
+ cow_image_size_ = size_in_bytes;
+ LOG_INFO << "COW image " << file_path << " has size " << size_in_bytes;
+ } else {
+ LOG_INFO << "COW image " << file_path
+ << " is not a block device, assuming unlimited space.";
+ }
+ return true;
+}
+
+bool CowWriterBase::AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) {
+ CHECK(num_blocks != 0);
+
+ for (size_t i = 0; i < num_blocks; i++) {
+ if (!ValidateNewBlock(new_block + i)) {
+ return false;
+ }
+ }
+
+ return EmitCopy(new_block, old_block, num_blocks);
+}
+
+bool CowWriterBase::AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) {
+ if (size % options_.block_size != 0) {
+ LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of "
+ << options_.block_size;
+ return false;
+ }
+
+ uint64_t num_blocks = size / options_.block_size;
+ uint64_t last_block = new_block_start + num_blocks - 1;
+ if (!ValidateNewBlock(last_block)) {
+ return false;
+ }
+ return EmitRawBlocks(new_block_start, data, size);
+}
+
+bool CowWriterBase::AddXorBlocks(uint32_t new_block_start, const void* data, size_t size,
+ uint32_t old_block, uint16_t offset) {
+ if (size % options_.block_size != 0) {
+ LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of "
+ << options_.block_size;
+ return false;
+ }
+
+ uint64_t num_blocks = size / options_.block_size;
+ uint64_t last_block = new_block_start + num_blocks - 1;
+ if (!ValidateNewBlock(last_block)) {
+ return false;
+ }
+ if (offset >= options_.block_size) {
+ LOG(ERROR) << "AddXorBlocks: offset " << offset << " is not less than "
+ << options_.block_size;
+ }
+ return EmitXorBlocks(new_block_start, data, size, old_block, offset);
+}
+
+bool CowWriterBase::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
+ uint64_t last_block = new_block_start + num_blocks - 1;
+ if (!ValidateNewBlock(last_block)) {
+ return false;
+ }
+ return EmitZeroBlocks(new_block_start, num_blocks);
+}
+
+bool CowWriterBase::AddLabel(uint64_t label) {
+ return EmitLabel(label);
+}
+
+bool CowWriterBase::AddSequenceData(size_t num_ops, const uint32_t* data) {
+ return EmitSequenceData(num_ops, data);
+}
+
+bool CowWriterBase::ValidateNewBlock(uint64_t new_block) {
+ if (options_.max_blocks && new_block >= options_.max_blocks.value()) {
+ LOG(ERROR) << "New block " << new_block << " exceeds maximum block count "
+ << options_.max_blocks.value();
+ return false;
+ }
+ return true;
+}
+
+} // namespace snapshot
+} // namespace android
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.h b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.h
new file mode 100644
index 0000000..8fa9065
--- /dev/null
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.h
@@ -0,0 +1,71 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <libsnapshot/cow_writer.h>
+
+namespace android {
+namespace snapshot {
+
+class CowWriterBase : public ICowWriter {
+ public:
+ CowWriterBase(const CowOptions& options, android::base::unique_fd&& fd);
+ virtual ~CowWriterBase() {}
+
+ // Set up the writer.
+ // The file starts from the beginning.
+ //
+ // If fd is < 0, the CowWriter will be opened against /dev/null. This is for
+ // computing COW sizes without using storage space.
+ //
+ // If a label is given, any operations after the given label will be dropped.
+ // If the given label is not found, Initialize will fail.
+ virtual bool Initialize(std::optional<uint64_t> label = {}) = 0;
+
+ bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;
+ bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
+ bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block,
+ uint16_t offset) override;
+ bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
+ bool AddLabel(uint64_t label) override;
+ bool AddSequenceData(size_t num_ops, const uint32_t* data) override;
+ uint32_t GetBlockSize() const override { return options_.block_size; }
+ std::optional<uint32_t> GetMaxBlocks() const override { return options_.max_blocks; }
+
+ const CowOptions& options() const { return options_; }
+
+ protected:
+ virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0;
+ virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0;
+ virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,
+ uint32_t old_block, uint16_t offset) = 0;
+ virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;
+ virtual bool EmitLabel(uint64_t label) = 0;
+ virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) = 0;
+
+ bool InitFd();
+ bool ValidateNewBlock(uint64_t new_block);
+
+ CowOptions options_;
+ CowHeader header_{};
+
+ android::base::unique_fd fd_;
+ bool is_dev_null_ = false;
+ bool is_block_device_ = false;
+ uint64_t cow_image_size_ = INT64_MAX;
+};
+
+} // namespace snapshot
+} // namespace android
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_writer.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp
similarity index 75%
rename from fs_mgr/libsnapshot/libsnapshot_cow/cow_writer.cpp
rename to fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp
index 1eaa038..b6603da 100644
--- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_writer.cpp
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp
@@ -37,6 +37,8 @@
#include <sys/ioctl.h>
#include <unistd.h>
+#include "writer_v2.h"
+
// The info messages here are spammy, but as useful for update_engine. Disable
// them when running on the host.
#ifdef __ANDROID__
@@ -48,104 +50,17 @@
namespace android {
namespace snapshot {
-namespace {
-std::string GetFdPath(int fd) {
- const auto fd_path = "/proc/self/fd/" + std::to_string(fd);
- std::string file_path(512, '\0');
- const auto err = readlink(fd_path.c_str(), file_path.data(), file_path.size());
- if (err <= 0) {
- PLOG(ERROR) << "Failed to determine path for fd " << fd;
- file_path.clear();
- } else {
- file_path.resize(err);
- }
- return file_path;
-}
-} // namespace
-
static_assert(sizeof(off_t) == sizeof(uint64_t));
-using android::base::borrowed_fd;
using android::base::unique_fd;
-bool ICowWriter::AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) {
- CHECK(num_blocks != 0);
-
- for (size_t i = 0; i < num_blocks; i++) {
- if (!ValidateNewBlock(new_block + i)) {
- return false;
- }
- }
-
- return EmitCopy(new_block, old_block, num_blocks);
-}
-
-bool ICowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) {
- if (size % options_.block_size != 0) {
- LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of "
- << options_.block_size;
- return false;
- }
-
- uint64_t num_blocks = size / options_.block_size;
- uint64_t last_block = new_block_start + num_blocks - 1;
- if (!ValidateNewBlock(last_block)) {
- return false;
- }
- return EmitRawBlocks(new_block_start, data, size);
-}
-
-bool ICowWriter::AddXorBlocks(uint32_t new_block_start, const void* data, size_t size,
- uint32_t old_block, uint16_t offset) {
- if (size % options_.block_size != 0) {
- LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of "
- << options_.block_size;
- return false;
- }
-
- uint64_t num_blocks = size / options_.block_size;
- uint64_t last_block = new_block_start + num_blocks - 1;
- if (!ValidateNewBlock(last_block)) {
- return false;
- }
- if (offset >= options_.block_size) {
- LOG(ERROR) << "AddXorBlocks: offset " << offset << " is not less than "
- << options_.block_size;
- }
- return EmitXorBlocks(new_block_start, data, size, old_block, offset);
-}
-
-bool ICowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
- uint64_t last_block = new_block_start + num_blocks - 1;
- if (!ValidateNewBlock(last_block)) {
- return false;
- }
- return EmitZeroBlocks(new_block_start, num_blocks);
-}
-
-bool ICowWriter::AddLabel(uint64_t label) {
- return EmitLabel(label);
-}
-
-bool ICowWriter::AddSequenceData(size_t num_ops, const uint32_t* data) {
- return EmitSequenceData(num_ops, data);
-}
-
-bool ICowWriter::ValidateNewBlock(uint64_t new_block) {
- if (options_.max_blocks && new_block >= options_.max_blocks.value()) {
- LOG(ERROR) << "New block " << new_block << " exceeds maximum block count "
- << options_.max_blocks.value();
- return false;
- }
- return true;
-}
-
-CowWriter::CowWriter(const CowOptions& options) : ICowWriter(options), fd_(-1) {
+CowWriterV2::CowWriterV2(const CowOptions& options, unique_fd&& fd)
+ : CowWriterBase(options, std::move(fd)) {
SetupHeaders();
SetupWriteOptions();
}
-CowWriter::~CowWriter() {
+CowWriterV2::~CowWriterV2() {
for (size_t i = 0; i < compress_threads_.size(); i++) {
CompressWorker* worker = compress_threads_[i].get();
if (worker) {
@@ -164,7 +79,7 @@
compress_threads_.clear();
}
-void CowWriter::SetupWriteOptions() {
+void CowWriterV2::SetupWriteOptions() {
num_compress_threads_ = options_.num_compress_threads;
if (!num_compress_threads_) {
@@ -184,7 +99,7 @@
}
}
-void CowWriter::SetupHeaders() {
+void CowWriterV2::SetupHeaders() {
header_ = {};
header_.prefix.magic = kCowMagicNumber;
header_.prefix.major_version = kCowVersionMajor;
@@ -201,7 +116,7 @@
footer_.op.type = kCowFooterOp;
}
-bool CowWriter::ParseOptions() {
+bool CowWriterV2::ParseOptions() {
auto algorithm = CompressionAlgorithmFromString(options_.compression);
if (!algorithm) {
LOG(ERROR) << "unrecognized compression: " << options_.compression;
@@ -216,42 +131,7 @@
return true;
}
-bool CowWriter::SetFd(android::base::borrowed_fd fd) {
- if (fd.get() < 0) {
- owned_fd_.reset(open("/dev/null", O_RDWR | O_CLOEXEC));
- if (owned_fd_ < 0) {
- PLOG(ERROR) << "open /dev/null failed";
- return false;
- }
- fd_ = owned_fd_;
- is_dev_null_ = true;
- } else {
- fd_ = fd;
-
- struct stat stat {};
- if (fstat(fd.get(), &stat) < 0) {
- PLOG(ERROR) << "fstat failed";
- return false;
- }
- const auto file_path = GetFdPath(fd.get());
- is_block_device_ = S_ISBLK(stat.st_mode);
- if (is_block_device_) {
- uint64_t size_in_bytes = 0;
- if (ioctl(fd.get(), BLKGETSIZE64, &size_in_bytes)) {
- PLOG(ERROR) << "Failed to get total size for: " << fd.get();
- return false;
- }
- cow_image_size_ = size_in_bytes;
- LOG_INFO << "COW image " << file_path << " has size " << size_in_bytes;
- } else {
- LOG_INFO << "COW image " << file_path
- << " is not a block device, assuming unlimited space.";
- }
- }
- return true;
-}
-
-void CowWriter::InitBatchWrites() {
+void CowWriterV2::InitBatchWrites() {
if (batch_write_) {
cowop_vec_ = std::make_unique<struct iovec[]>(header_.cluster_ops);
data_vec_ = std::make_unique<struct iovec[]>(header_.cluster_ops);
@@ -277,7 +157,7 @@
LOG_INFO << "Batch writes: " << batch_write;
}
-void CowWriter::InitWorkers() {
+void CowWriterV2::InitWorkers() {
if (num_compress_threads_ <= 1) {
LOG_INFO << "Not creating new threads for compression.";
return;
@@ -291,44 +171,27 @@
LOG_INFO << num_compress_threads_ << " thread used for compression";
}
-bool CowWriter::Initialize(unique_fd&& fd) {
- owned_fd_ = std::move(fd);
- return Initialize(borrowed_fd{owned_fd_});
-}
-
-bool CowWriter::Initialize(borrowed_fd fd) {
- if (!SetFd(fd) || !ParseOptions()) {
+bool CowWriterV2::Initialize(std::optional<uint64_t> label) {
+ if (!InitFd() || !ParseOptions()) {
return false;
}
-
- if (!OpenForWrite()) {
- return false;
+ if (!label) {
+ if (!OpenForWrite()) {
+ return false;
+ }
+ } else {
+ if (!OpenForAppend(*label)) {
+ return false;
+ }
}
- InitWorkers();
+ if (!compress_threads_.size()) {
+ InitWorkers();
+ }
return true;
}
-bool CowWriter::InitializeAppend(android::base::unique_fd&& fd, uint64_t label) {
- owned_fd_ = std::move(fd);
- return InitializeAppend(android::base::borrowed_fd{owned_fd_}, label);
-}
-
-bool CowWriter::InitializeAppend(android::base::borrowed_fd fd, uint64_t label) {
- if (!SetFd(fd) || !ParseOptions()) {
- return false;
- }
-
- bool ret = OpenForAppend(label);
-
- if (ret && !compress_threads_.size()) {
- InitWorkers();
- }
-
- return ret;
-}
-
-void CowWriter::InitPos() {
+void CowWriterV2::InitPos() {
next_op_pos_ = sizeof(header_) + header_.buffer_size;
cluster_size_ = header_.cluster_ops * sizeof(CowOperation);
if (header_.cluster_ops) {
@@ -340,7 +203,7 @@
current_data_size_ = 0;
}
-bool CowWriter::OpenForWrite() {
+bool CowWriterV2::OpenForWrite() {
// This limitation is tied to the data field size in CowOperation.
if (header_.block_size > std::numeric_limits<uint16_t>::max()) {
LOG(ERROR) << "Block size is too large";
@@ -388,7 +251,7 @@
return true;
}
-bool CowWriter::OpenForAppend(uint64_t label) {
+bool CowWriterV2::OpenForAppend(uint64_t label) {
auto reader = std::make_unique<CowReader>();
std::queue<CowOperation> toAdd;
@@ -424,7 +287,7 @@
return EmitClusterIfNeeded();
}
-bool CowWriter::EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) {
+bool CowWriterV2::EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) {
CHECK(!merge_in_progress_);
for (size_t i = 0; i < num_blocks; i++) {
@@ -440,16 +303,16 @@
return true;
}
-bool CowWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) {
+bool CowWriterV2::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) {
return EmitBlocks(new_block_start, data, size, 0, 0, kCowReplaceOp);
}
-bool CowWriter::EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,
- uint32_t old_block, uint16_t offset) {
+bool CowWriterV2::EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,
+ uint32_t old_block, uint16_t offset) {
return EmitBlocks(new_block_start, data, size, old_block, offset, kCowXorOp);
}
-bool CowWriter::CompressBlocks(size_t num_blocks, const void* data) {
+bool CowWriterV2::CompressBlocks(size_t num_blocks, const void* data) {
size_t num_threads = (num_blocks == 1) ? 1 : num_compress_threads_;
size_t num_blocks_per_thread = num_blocks / num_threads;
const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
@@ -483,8 +346,8 @@
return true;
}
-bool CowWriter::EmitBlocks(uint64_t new_block_start, const void* data, size_t size,
- uint64_t old_block, uint16_t offset, uint8_t type) {
+bool CowWriterV2::EmitBlocks(uint64_t new_block_start, const void* data, size_t size,
+ uint64_t old_block, uint16_t offset, uint8_t type) {
CHECK(!merge_in_progress_);
const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
@@ -558,7 +421,7 @@
return true;
}
-bool CowWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
+bool CowWriterV2::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
CHECK(!merge_in_progress_);
for (uint64_t i = 0; i < num_blocks; i++) {
CowOperation op = {};
@@ -570,7 +433,7 @@
return true;
}
-bool CowWriter::EmitLabel(uint64_t label) {
+bool CowWriterV2::EmitLabel(uint64_t label) {
CHECK(!merge_in_progress_);
CowOperation op = {};
op.type = kCowLabelOp;
@@ -578,7 +441,7 @@
return WriteOperation(op) && Sync();
}
-bool CowWriter::EmitSequenceData(size_t num_ops, const uint32_t* data) {
+bool CowWriterV2::EmitSequenceData(size_t num_ops, const uint32_t* data) {
CHECK(!merge_in_progress_);
size_t to_add = 0;
size_t max_ops = (header_.block_size * 2) / sizeof(uint32_t);
@@ -598,7 +461,7 @@
return true;
}
-bool CowWriter::EmitCluster() {
+bool CowWriterV2::EmitCluster() {
CowOperation op = {};
op.type = kCowClusterOp;
// Next cluster starts after remainder of current cluster and the next data block.
@@ -606,7 +469,7 @@
return WriteOperation(op);
}
-bool CowWriter::EmitClusterIfNeeded() {
+bool CowWriterV2::EmitClusterIfNeeded() {
// If there isn't room for another op and the cluster end op, end the current cluster
if (cluster_size_ && cluster_size_ < current_cluster_size_ + 2 * sizeof(CowOperation)) {
if (!EmitCluster()) return false;
@@ -614,7 +477,7 @@
return true;
}
-bool CowWriter::Finalize() {
+bool CowWriterV2::Finalize() {
if (!FlushCluster()) {
LOG(ERROR) << "Finalize: FlushCluster() failed";
return false;
@@ -688,7 +551,7 @@
return Sync();
}
-uint64_t CowWriter::GetCowSize() {
+uint64_t CowWriterV2::GetCowSize() {
if (current_data_size_ > 0) {
return next_data_pos_ + sizeof(footer_);
} else {
@@ -696,7 +559,7 @@
}
}
-bool CowWriter::GetDataPos(uint64_t* pos) {
+bool CowWriterV2::GetDataPos(uint64_t* pos) {
off_t offs = lseek(fd_.get(), 0, SEEK_CUR);
if (offs < 0) {
PLOG(ERROR) << "lseek failed";
@@ -706,7 +569,7 @@
return true;
}
-bool CowWriter::EnsureSpaceAvailable(const uint64_t bytes_needed) const {
+bool CowWriterV2::EnsureSpaceAvailable(const uint64_t bytes_needed) const {
if (bytes_needed > cow_image_size_) {
LOG(ERROR) << "No space left on COW device. Required: " << bytes_needed
<< ", available: " << cow_image_size_;
@@ -716,7 +579,7 @@
return true;
}
-bool CowWriter::FlushCluster() {
+bool CowWriterV2::FlushCluster() {
ssize_t ret;
if (op_vec_index_) {
@@ -745,7 +608,7 @@
return true;
}
-bool CowWriter::WriteOperation(const CowOperation& op, const void* data, size_t size) {
+bool CowWriterV2::WriteOperation(const CowOperation& op, const void* data, size_t size) {
if (!EnsureSpaceAvailable(next_op_pos_ + sizeof(op))) {
return false;
}
@@ -793,7 +656,7 @@
return EmitClusterIfNeeded();
}
-void CowWriter::AddOperation(const CowOperation& op) {
+void CowWriterV2::AddOperation(const CowOperation& op) {
footer_.op.num_ops++;
if (op.type == kCowClusterOp) {
@@ -808,14 +671,14 @@
next_op_pos_ += sizeof(CowOperation) + GetNextOpOffset(op, header_.cluster_ops);
}
-bool CowWriter::WriteRawData(const void* data, const size_t size) {
+bool CowWriterV2::WriteRawData(const void* data, const size_t size) {
if (!android::base::WriteFullyAtOffset(fd_, data, size, next_data_pos_)) {
return false;
}
return true;
}
-bool CowWriter::Sync() {
+bool CowWriterV2::Sync() {
if (is_dev_null_) {
return true;
}
@@ -826,7 +689,7 @@
return true;
}
-bool CowWriter::Truncate(off_t length) {
+bool CowWriterV2::Truncate(off_t length) {
if (is_dev_null_ || is_block_device_) {
return true;
}
diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h
new file mode 100644
index 0000000..809ae57
--- /dev/null
+++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h
@@ -0,0 +1,94 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "writer_base.h"
+
+namespace android {
+namespace snapshot {
+
+class CowWriterV2 : public CowWriterBase {
+ public:
+ explicit CowWriterV2(const CowOptions& options, android::base::unique_fd&& fd);
+ ~CowWriterV2() override;
+
+ bool Initialize(std::optional<uint64_t> label = {}) override;
+ bool Finalize() override;
+ uint64_t GetCowSize() override;
+
+ protected:
+ virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;
+ virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
+ virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,
+ uint32_t old_block, uint16_t offset) override;
+ virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
+ virtual bool EmitLabel(uint64_t label) override;
+ virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) override;
+
+ private:
+ bool EmitCluster();
+ bool EmitClusterIfNeeded();
+ bool EmitBlocks(uint64_t new_block_start, const void* data, size_t size, uint64_t old_block,
+ uint16_t offset, uint8_t type);
+ void SetupHeaders();
+ void SetupWriteOptions();
+ bool ParseOptions();
+ bool OpenForWrite();
+ bool OpenForAppend(uint64_t label);
+ bool GetDataPos(uint64_t* pos);
+ bool WriteRawData(const void* data, size_t size);
+ bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0);
+ void AddOperation(const CowOperation& op);
+ void InitPos();
+ void InitBatchWrites();
+ void InitWorkers();
+ bool FlushCluster();
+
+ bool CompressBlocks(size_t num_blocks, const void* data);
+ bool Sync();
+ bool Truncate(off_t length);
+ bool EnsureSpaceAvailable(const uint64_t bytes_needed) const;
+
+ private:
+ CowFooter footer_{};
+ CowCompressionAlgorithm compression_ = kCowCompressNone;
+ uint64_t current_op_pos_ = 0;
+ uint64_t next_op_pos_ = 0;
+ uint64_t next_data_pos_ = 0;
+ uint64_t current_data_pos_ = 0;
+ ssize_t total_data_written_ = 0;
+ uint32_t cluster_size_ = 0;
+ uint32_t current_cluster_size_ = 0;
+ uint64_t current_data_size_ = 0;
+ bool merge_in_progress_ = false;
+
+ int num_compress_threads_ = 1;
+ std::vector<std::unique_ptr<CompressWorker>> compress_threads_;
+ std::vector<std::future<bool>> threads_;
+ std::vector<std::basic_string<uint8_t>> compressed_buf_;
+ std::vector<std::basic_string<uint8_t>>::iterator buf_iter_;
+
+ std::vector<std::unique_ptr<CowOperation>> opbuffer_vec_;
+ std::vector<std::unique_ptr<uint8_t[]>> databuffer_vec_;
+ std::unique_ptr<struct iovec[]> cowop_vec_;
+ int op_vec_index_ = 0;
+
+ std::unique_ptr<struct iovec[]> data_vec_;
+ int data_vec_index_ = 0;
+ bool batch_write_ = false;
+};
+
+} // namespace snapshot
+} // namespace android
diff --git a/fs_mgr/libsnapshot/snapshot.cpp b/fs_mgr/libsnapshot/snapshot.cpp
index e114d25..5920bc2 100644
--- a/fs_mgr/libsnapshot/snapshot.cpp
+++ b/fs_mgr/libsnapshot/snapshot.cpp
@@ -45,6 +45,7 @@
#include <android/snapshot/snapshot.pb.h>
#include <libsnapshot/snapshot_stats.h>
#include "device_info.h"
+#include "libsnapshot_cow/writer_v2.h"
#include "partition_cow_creator.h"
#include "snapshot_metadata_updater.h"
#include "snapshot_reader.h"
@@ -3557,8 +3558,8 @@
}
options.compression = it->second.compression_algorithm();
- CowWriter writer(options);
- if (!writer.Initialize(fd) || !writer.Finalize()) {
+ CowWriterV2 writer(options, std::move(fd));
+ if (!writer.Initialize(std::nullopt) || !writer.Finalize()) {
LOG(ERROR) << "Could not initialize COW device for " << target_partition->name();
return Return::Error();
}
diff --git a/fs_mgr/libsnapshot/snapshot_test.cpp b/fs_mgr/libsnapshot/snapshot_test.cpp
index 757f6f1..dac1b77 100644
--- a/fs_mgr/libsnapshot/snapshot_test.cpp
+++ b/fs_mgr/libsnapshot/snapshot_test.cpp
@@ -1220,13 +1220,13 @@
SHA256_CTX ctx;
SHA256_Init(&ctx);
- if (!writer->options().max_blocks) {
+ if (!writer->GetMaxBlocks()) {
LOG(ERROR) << "CowWriter must specify maximum number of blocks";
return false;
}
- const auto num_blocks = writer->options().max_blocks.value();
+ const auto num_blocks = writer->GetMaxBlocks().value();
- const auto block_size = writer->options().block_size;
+ const auto block_size = writer->GetBlockSize();
std::string block(block_size, '\0');
for (uint64_t i = 0; i < num_blocks; i++) {
if (!ReadFully(rand, block.data(), block.size())) {
@@ -1254,13 +1254,13 @@
if (auto res = MapUpdateSnapshot(name, &writer); !res) {
return res;
}
- if (!writer->options().max_blocks || !*writer->options().max_blocks) {
+ if (!writer->GetMaxBlocks() || !*writer->GetMaxBlocks()) {
return AssertionFailure() << "No max blocks set for " << name << " writer";
}
- uint64_t src_block = (old_size / writer->options().block_size) - 1;
+ uint64_t src_block = (old_size / writer->GetBlockSize()) - 1;
uint64_t dst_block = 0;
- uint64_t max_blocks = *writer->options().max_blocks;
+ uint64_t max_blocks = *writer->GetMaxBlocks();
while (dst_block < max_blocks && dst_block < src_block) {
if (!writer->AddCopy(dst_block, src_block)) {
return AssertionFailure() << "Unable to add copy for " << name << " for blocks "
diff --git a/fs_mgr/libsnapshot/snapshot_writer.cpp b/fs_mgr/libsnapshot/snapshot_writer.cpp
index 6a3906e..0ea424c 100644
--- a/fs_mgr/libsnapshot/snapshot_writer.cpp
+++ b/fs_mgr/libsnapshot/snapshot_writer.cpp
@@ -19,6 +19,7 @@
#include <android-base/file.h>
#include <android-base/logging.h>
#include <payload_consumer/file_descriptor.h>
+#include "libsnapshot_cow/writer_v2.h"
#include "snapshot_reader.h"
namespace android {
@@ -28,13 +29,11 @@
using android::base::unique_fd;
using chromeos_update_engine::FileDescriptor;
-ISnapshotWriter::ISnapshotWriter(const CowOptions& options) : ICowWriter(options) {}
-
-void ISnapshotWriter::SetSourceDevice(const std::string& source_device) {
+void CompressedSnapshotWriter::SetSourceDevice(const std::string& source_device) {
source_device_ = {source_device};
}
-borrowed_fd ISnapshotWriter::GetSourceFd() {
+borrowed_fd CompressedSnapshotWriter::GetSourceFd() {
if (!source_device_) {
LOG(ERROR) << "Attempted to read from source device but none was set";
return borrowed_fd{-1};
@@ -50,12 +49,10 @@
return source_fd_;
}
-CompressedSnapshotWriter::CompressedSnapshotWriter(const CowOptions& options)
- : ISnapshotWriter(options) {}
+CompressedSnapshotWriter::CompressedSnapshotWriter(const CowOptions& options) : options_(options) {}
bool CompressedSnapshotWriter::SetCowDevice(android::base::unique_fd&& cow_device) {
cow_device_ = std::move(cow_device);
- cow_ = std::make_unique<CowWriter>(options_);
return true;
}
@@ -106,47 +103,76 @@
reader->SetSourceDevice(*source_device_);
}
- const auto& cow_options = options();
- if (cow_options.max_blocks) {
- reader->SetBlockDeviceSize(*cow_options.max_blocks * cow_options.block_size);
+ if (options_.max_blocks) {
+ reader->SetBlockDeviceSize(*options_.max_blocks * options_.block_size);
}
return reader;
}
-bool CompressedSnapshotWriter::EmitCopy(uint64_t new_block, uint64_t old_block,
- uint64_t num_blocks) {
+bool CompressedSnapshotWriter::AddCopy(uint64_t new_block, uint64_t old_block,
+ uint64_t num_blocks) {
return cow_->AddCopy(new_block, old_block, num_blocks);
}
-bool CompressedSnapshotWriter::EmitRawBlocks(uint64_t new_block_start, const void* data,
- size_t size) {
+bool CompressedSnapshotWriter::AddRawBlocks(uint64_t new_block_start, const void* data,
+ size_t size) {
return cow_->AddRawBlocks(new_block_start, data, size);
}
-bool CompressedSnapshotWriter::EmitXorBlocks(uint32_t new_block_start, const void* data,
- size_t size, uint32_t old_block, uint16_t offset) {
+bool CompressedSnapshotWriter::AddXorBlocks(uint32_t new_block_start, const void* data, size_t size,
+ uint32_t old_block, uint16_t offset) {
return cow_->AddXorBlocks(new_block_start, data, size, old_block, offset);
}
-bool CompressedSnapshotWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
+bool CompressedSnapshotWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
return cow_->AddZeroBlocks(new_block_start, num_blocks);
}
-bool CompressedSnapshotWriter::EmitLabel(uint64_t label) {
+bool CompressedSnapshotWriter::AddLabel(uint64_t label) {
return cow_->AddLabel(label);
}
-bool CompressedSnapshotWriter::EmitSequenceData(size_t num_ops, const uint32_t* data) {
+bool CompressedSnapshotWriter::AddSequenceData(size_t num_ops, const uint32_t* data) {
return cow_->AddSequenceData(num_ops, data);
}
bool CompressedSnapshotWriter::Initialize() {
- return cow_->Initialize(cow_device_);
+ unique_fd cow_fd(dup(cow_device_.get()));
+ if (cow_fd < 0) {
+ PLOG(ERROR) << "dup COW device";
+ return false;
+ }
+
+ auto cow = std::make_unique<CowWriterV2>(options_, std::move(cow_fd));
+ if (!cow->Initialize(std::nullopt)) {
+ return false;
+ }
+ cow_ = std::move(cow);
+ return true;
}
bool CompressedSnapshotWriter::InitializeAppend(uint64_t label) {
- return cow_->InitializeAppend(cow_device_, label);
+ unique_fd cow_fd(dup(cow_device_.get()));
+ if (cow_fd < 0) {
+ PLOG(ERROR) << "dup COW device";
+ return false;
+ }
+
+ auto cow = std::make_unique<CowWriterV2>(options_, std::move(cow_fd));
+ if (!cow->Initialize(label)) {
+ return false;
+ }
+ cow_ = std::move(cow);
+ return true;
+}
+
+uint32_t CompressedSnapshotWriter::GetBlockSize() const {
+ return cow_->GetBlockSize();
+}
+
+std::optional<uint32_t> CompressedSnapshotWriter::GetMaxBlocks() const {
+ return cow_->GetMaxBlocks();
}
} // namespace snapshot
diff --git a/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp b/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp
index 3c4ab2e..737c480 100644
--- a/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp
@@ -122,6 +122,7 @@
void SimulateDaemonRestart();
void StartMerge();
+ std::unique_ptr<ICowWriter> CreateCowDeviceInternal();
void CreateCowDevice();
void CreateCowDeviceOrderedOps();
void CreateCowDeviceOrderedOpsInverted();
@@ -164,6 +165,7 @@
private:
void InitMetadata();
+ std::unique_ptr<ICowWriter> CreateCowDeviceInternal();
void CreateCowDevice();
void CreateCowPartialFilledArea();
@@ -258,6 +260,19 @@
}
}
+std::unique_ptr<ICowWriter> CowSnapuserdTest::CreateCowDeviceInternal() {
+ std::string path = android::base::GetExecutableDirectory();
+ cow_system_ = std::make_unique<TemporaryFile>(path);
+
+ CowOptions options;
+ options.compression = "gz";
+
+ unique_fd fd(cow_system_->fd);
+ cow_system_->fd = -1;
+
+ return CreateCowWriter(kDefaultCowVersion, options, std::move(fd));
+}
+
void CowSnapuserdTest::ReadLastBlock() {
unique_fd rnd_fd;
total_base_size_ = BLOCK_SZ * 2;
@@ -280,9 +295,6 @@
base_loop_ = std::make_unique<LoopDevice>(base_fd_, 10s);
ASSERT_TRUE(base_loop_->valid());
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
-
std::unique_ptr<uint8_t[]> random_buffer_1_ = std::make_unique<uint8_t[]>(total_base_size_);
loff_t offset = 0;
@@ -294,16 +306,13 @@
offset += BLOCK_SZ;
}
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
+ ASSERT_TRUE(writer->AddRawBlocks(0, random_buffer_1_.get(), BLOCK_SZ));
+ ASSERT_TRUE(writer->AddRawBlocks(1, (char*)random_buffer_1_.get() + BLOCK_SZ, BLOCK_SZ));
- ASSERT_TRUE(writer.AddRawBlocks(0, random_buffer_1_.get(), BLOCK_SZ));
- ASSERT_TRUE(writer.AddRawBlocks(1, (char*)random_buffer_1_.get() + BLOCK_SZ, BLOCK_SZ));
-
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
SetDeviceControlName();
@@ -381,22 +390,16 @@
}
void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_2() {
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
-
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
-
- size_t num_blocks = size_ / options.block_size;
+ size_t num_blocks = size_ / writer->GetBlockSize();
size_t x = num_blocks;
size_t blk_src_copy = 0;
// Create overlapping copy operations
while (1) {
- ASSERT_TRUE(writer.AddCopy(blk_src_copy, blk_src_copy + 1));
+ ASSERT_TRUE(writer->AddCopy(blk_src_copy, blk_src_copy + 1));
x -= 1;
if (x == 1) {
break;
@@ -405,7 +408,7 @@
}
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
// Construct the buffer required for validation
orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
@@ -433,22 +436,16 @@
}
void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_1() {
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
-
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
-
- size_t num_blocks = size_ / options.block_size;
+ size_t num_blocks = size_ / writer->GetBlockSize();
size_t x = num_blocks;
size_t blk_src_copy = num_blocks - 1;
// Create overlapping copy operations
while (1) {
- ASSERT_TRUE(writer.AddCopy(blk_src_copy + 1, blk_src_copy));
+ ASSERT_TRUE(writer->AddCopy(blk_src_copy + 1, blk_src_copy));
x -= 1;
if (x == 0) {
ASSERT_EQ(blk_src_copy, 0);
@@ -458,7 +455,7 @@
}
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
// Construct the buffer required for validation
orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
@@ -468,10 +465,11 @@
true);
// Merged operations
- ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), options.block_size, 0),
+ ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), writer->GetBlockSize(),
+ 0),
true);
ASSERT_EQ(android::base::ReadFullyAtOffset(
- base_fd_, (char*)orig_buffer_.get() + options.block_size, size_, 0),
+ base_fd_, (char*)orig_buffer_.get() + writer->GetBlockSize(), size_, 0),
true);
}
@@ -479,8 +477,8 @@
unique_fd rnd_fd;
loff_t offset = 0;
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
rnd_fd.reset(open("/dev/random", O_RDONLY));
ASSERT_TRUE(rnd_fd > 0);
@@ -495,13 +493,7 @@
offset += 1_MiB;
}
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
-
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
-
- size_t num_blocks = size_ / options.block_size;
+ size_t num_blocks = size_ / writer->GetBlockSize();
size_t blk_end_copy = num_blocks * 3;
size_t source_blk = num_blocks - 1;
size_t blk_src_copy = blk_end_copy - 1;
@@ -509,7 +501,7 @@
size_t x = num_blocks;
while (1) {
- ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy));
+ ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy));
x -= 1;
if (x == 0) {
break;
@@ -519,12 +511,12 @@
}
for (size_t i = num_blocks; i > 0; i--) {
- ASSERT_TRUE(writer.AddXorBlocks(num_blocks + i - 1,
- &random_buffer_1_.get()[options.block_size * (i - 1)],
- options.block_size, 2 * num_blocks + i - 1, xor_offset));
+ ASSERT_TRUE(writer->AddXorBlocks(
+ num_blocks + i - 1, &random_buffer_1_.get()[writer->GetBlockSize() * (i - 1)],
+ writer->GetBlockSize(), 2 * num_blocks + i - 1, xor_offset));
}
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
// Construct the buffer required for validation
orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
// Read the entire base device
@@ -542,8 +534,8 @@
unique_fd rnd_fd;
loff_t offset = 0;
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
rnd_fd.reset(open("/dev/random", O_RDONLY));
ASSERT_TRUE(rnd_fd > 0);
@@ -559,20 +551,14 @@
}
memset(random_buffer_1_.get(), 0, size_);
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
-
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
-
- size_t num_blocks = size_ / options.block_size;
+ size_t num_blocks = size_ / writer->GetBlockSize();
size_t x = num_blocks;
size_t source_blk = 0;
size_t blk_src_copy = 2 * num_blocks;
uint16_t xor_offset = 5;
while (1) {
- ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy));
+ ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy));
x -= 1;
if (x == 0) {
@@ -582,10 +568,10 @@
blk_src_copy += 1;
}
- ASSERT_TRUE(writer.AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks,
- xor_offset));
+ ASSERT_TRUE(writer->AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks,
+ xor_offset));
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
// Construct the buffer required for validation
orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
// Read the entire base device
@@ -603,8 +589,8 @@
unique_fd rnd_fd;
loff_t offset = 0;
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
rnd_fd.reset(open("/dev/random", O_RDONLY));
ASSERT_TRUE(rnd_fd > 0);
@@ -619,13 +605,7 @@
offset += 1_MiB;
}
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
-
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
-
- size_t num_blocks = size_ / options.block_size;
+ size_t num_blocks = size_ / writer->GetBlockSize();
size_t blk_end_copy = num_blocks * 2;
size_t source_blk = num_blocks - 1;
size_t blk_src_copy = blk_end_copy - 1;
@@ -639,11 +619,11 @@
for (int i = 0; i < num_blocks; i++) {
sequence[num_blocks + i] = 5 * num_blocks - 1 - i;
}
- ASSERT_TRUE(writer.AddSequenceData(2 * num_blocks, sequence));
+ ASSERT_TRUE(writer->AddSequenceData(2 * num_blocks, sequence));
size_t x = num_blocks;
while (1) {
- ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy));
+ ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy));
x -= 1;
if (x == 0) {
break;
@@ -655,24 +635,24 @@
source_blk = num_blocks;
blk_src_copy = blk_end_copy;
- ASSERT_TRUE(writer.AddRawBlocks(source_blk, random_buffer_1_.get(), size_));
+ ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_));
size_t blk_zero_copy_start = source_blk + num_blocks;
size_t blk_zero_copy_end = blk_zero_copy_start + num_blocks;
- ASSERT_TRUE(writer.AddZeroBlocks(blk_zero_copy_start, num_blocks));
+ ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks));
size_t blk_random2_replace_start = blk_zero_copy_end;
- ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_));
+ ASSERT_TRUE(writer->AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_));
size_t blk_xor_start = blk_random2_replace_start + num_blocks;
size_t xor_offset = BLOCK_SZ / 2;
- ASSERT_TRUE(writer.AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks,
- xor_offset));
+ ASSERT_TRUE(writer->AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks,
+ xor_offset));
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
// Construct the buffer required for validation
orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
std::string zero_buffer(size_, 0);
@@ -902,29 +882,36 @@
ASSERT_TRUE(Merge());
}
-void CowSnapuserdMetadataTest::CreateCowPartialFilledArea() {
+std::unique_ptr<ICowWriter> CowSnapuserdMetadataTest::CreateCowDeviceInternal() {
std::string path = android::base::GetExecutableDirectory();
cow_system_ = std::make_unique<TemporaryFile>(path);
CowOptions options;
options.compression = "gz";
- CowWriter writer(options);
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
+ unique_fd fd(cow_system_->fd);
+ cow_system_->fd = -1;
+
+ return CreateCowWriter(kDefaultCowVersion, options, std::move(fd));
+}
+
+void CowSnapuserdMetadataTest::CreateCowPartialFilledArea() {
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
// Area 0 is completely filled with 256 exceptions
for (int i = 0; i < 256; i++) {
- ASSERT_TRUE(writer.AddCopy(i, 256 + i));
+ ASSERT_TRUE(writer->AddCopy(i, 256 + i));
}
// Area 1 is partially filled with 2 copy ops and 10 zero ops
- ASSERT_TRUE(writer.AddCopy(500, 1000));
- ASSERT_TRUE(writer.AddCopy(501, 1001));
+ ASSERT_TRUE(writer->AddCopy(500, 1000));
+ ASSERT_TRUE(writer->AddCopy(501, 1001));
- ASSERT_TRUE(writer.AddZeroBlocks(300, 10));
+ ASSERT_TRUE(writer->AddZeroBlocks(300, 10));
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
}
void CowSnapuserdMetadataTest::ValidatePartialFilledArea() {
@@ -956,8 +943,8 @@
unique_fd rnd_fd;
loff_t offset = 0;
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
rnd_fd.reset(open("/dev/random", O_RDONLY));
ASSERT_TRUE(rnd_fd > 0);
@@ -972,50 +959,44 @@
offset += 1_MiB;
}
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
-
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
-
- size_t num_blocks = size_ / options.block_size;
+ size_t num_blocks = size_ / writer->GetBlockSize();
// Overlapping region. This has to be split
// into two batch operations
- ASSERT_TRUE(writer.AddCopy(23, 20));
- ASSERT_TRUE(writer.AddCopy(22, 19));
- ASSERT_TRUE(writer.AddCopy(21, 18));
- ASSERT_TRUE(writer.AddCopy(20, 17));
- ASSERT_TRUE(writer.AddCopy(19, 16));
- ASSERT_TRUE(writer.AddCopy(18, 15));
+ ASSERT_TRUE(writer->AddCopy(23, 20));
+ ASSERT_TRUE(writer->AddCopy(22, 19));
+ ASSERT_TRUE(writer->AddCopy(21, 18));
+ ASSERT_TRUE(writer->AddCopy(20, 17));
+ ASSERT_TRUE(writer->AddCopy(19, 16));
+ ASSERT_TRUE(writer->AddCopy(18, 15));
// Contiguous region but blocks in ascending order
// Daemon has to ensure that these blocks are merged
// in a batch
- ASSERT_TRUE(writer.AddCopy(50, 75));
- ASSERT_TRUE(writer.AddCopy(51, 76));
- ASSERT_TRUE(writer.AddCopy(52, 77));
- ASSERT_TRUE(writer.AddCopy(53, 78));
+ ASSERT_TRUE(writer->AddCopy(50, 75));
+ ASSERT_TRUE(writer->AddCopy(51, 76));
+ ASSERT_TRUE(writer->AddCopy(52, 77));
+ ASSERT_TRUE(writer->AddCopy(53, 78));
// Dis-contiguous region
- ASSERT_TRUE(writer.AddCopy(110, 130));
- ASSERT_TRUE(writer.AddCopy(105, 125));
- ASSERT_TRUE(writer.AddCopy(100, 120));
+ ASSERT_TRUE(writer->AddCopy(110, 130));
+ ASSERT_TRUE(writer->AddCopy(105, 125));
+ ASSERT_TRUE(writer->AddCopy(100, 120));
// Overlap
- ASSERT_TRUE(writer.AddCopy(25, 30));
- ASSERT_TRUE(writer.AddCopy(30, 31));
+ ASSERT_TRUE(writer->AddCopy(25, 30));
+ ASSERT_TRUE(writer->AddCopy(30, 31));
size_t source_blk = num_blocks;
- ASSERT_TRUE(writer.AddRawBlocks(source_blk, random_buffer_1_.get(), size_));
+ ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_));
size_t blk_zero_copy_start = source_blk + num_blocks;
- ASSERT_TRUE(writer.AddZeroBlocks(blk_zero_copy_start, num_blocks));
+ ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks));
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
}
void CowSnapuserdMetadataTest::InitMetadata() {
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
index 57f9e7a..efe0c14 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
@@ -125,6 +125,7 @@
void SimulateDaemonRestart();
+ std::unique_ptr<ICowWriter> CreateCowDeviceInternal();
void CreateCowDevice();
void CreateCowDeviceOrderedOps();
void CreateCowDeviceOrderedOpsInverted();
@@ -277,23 +278,30 @@
ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 4), size_), 0);
}
-void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() {
+std::unique_ptr<ICowWriter> SnapuserdTest::CreateCowDeviceInternal() {
std::string path = android::base::GetExecutableDirectory();
cow_system_ = std::make_unique<TemporaryFile>(path);
CowOptions options;
options.compression = "gz";
- CowWriter writer(options);
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
+ unique_fd fd(cow_system_->fd);
+ cow_system_->fd = -1;
- size_t num_blocks = size_ / options.block_size;
+ return CreateCowWriter(kDefaultCowVersion, options, std::move(fd));
+}
+
+void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() {
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
+
+ size_t num_blocks = size_ / writer->GetBlockSize();
size_t x = num_blocks;
size_t blk_src_copy = 0;
// Create overlapping copy operations
while (1) {
- ASSERT_TRUE(writer.AddCopy(blk_src_copy, blk_src_copy + 1));
+ ASSERT_TRUE(writer->AddCopy(blk_src_copy, blk_src_copy + 1));
x -= 1;
if (x == 1) {
break;
@@ -302,7 +310,7 @@
}
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
// Construct the buffer required for validation
orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
@@ -330,22 +338,16 @@
}
void SnapuserdTest::CreateCowDeviceWithCopyOverlap_1() {
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
-
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
-
- size_t num_blocks = size_ / options.block_size;
+ size_t num_blocks = size_ / writer->GetBlockSize();
size_t x = num_blocks;
size_t blk_src_copy = num_blocks - 1;
// Create overlapping copy operations
while (1) {
- ASSERT_TRUE(writer.AddCopy(blk_src_copy + 1, blk_src_copy));
+ ASSERT_TRUE(writer->AddCopy(blk_src_copy + 1, blk_src_copy));
x -= 1;
if (x == 0) {
ASSERT_EQ(blk_src_copy, 0);
@@ -355,7 +357,7 @@
}
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
// Construct the buffer required for validation
orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
@@ -365,10 +367,11 @@
true);
// Merged operations
- ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), options.block_size, 0),
+ ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), writer->GetBlockSize(),
+ 0),
true);
ASSERT_EQ(android::base::ReadFullyAtOffset(
- base_fd_, (char*)orig_buffer_.get() + options.block_size, size_, 0),
+ base_fd_, (char*)orig_buffer_.get() + writer->GetBlockSize(), size_, 0),
true);
}
@@ -376,8 +379,8 @@
unique_fd rnd_fd;
loff_t offset = 0;
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
rnd_fd.reset(open("/dev/random", O_RDONLY));
ASSERT_TRUE(rnd_fd > 0);
@@ -392,13 +395,7 @@
offset += 1_MiB;
}
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
-
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
-
- size_t num_blocks = size_ / options.block_size;
+ size_t num_blocks = size_ / writer->GetBlockSize();
size_t blk_end_copy = num_blocks * 3;
size_t source_blk = num_blocks - 1;
size_t blk_src_copy = blk_end_copy - 1;
@@ -406,7 +403,7 @@
size_t x = num_blocks;
while (1) {
- ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy));
+ ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy));
x -= 1;
if (x == 0) {
break;
@@ -416,12 +413,12 @@
}
for (size_t i = num_blocks; i > 0; i--) {
- ASSERT_TRUE(writer.AddXorBlocks(num_blocks + i - 1,
- &random_buffer_1_.get()[options.block_size * (i - 1)],
- options.block_size, 2 * num_blocks + i - 1, xor_offset));
+ ASSERT_TRUE(writer->AddXorBlocks(
+ num_blocks + i - 1, &random_buffer_1_.get()[writer->GetBlockSize() * (i - 1)],
+ writer->GetBlockSize(), 2 * num_blocks + i - 1, xor_offset));
}
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
// Construct the buffer required for validation
orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
// Read the entire base device
@@ -439,8 +436,8 @@
unique_fd rnd_fd;
loff_t offset = 0;
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
rnd_fd.reset(open("/dev/random", O_RDONLY));
ASSERT_TRUE(rnd_fd > 0);
@@ -456,20 +453,14 @@
}
memset(random_buffer_1_.get(), 0, size_);
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
-
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
-
- size_t num_blocks = size_ / options.block_size;
+ size_t num_blocks = size_ / writer->GetBlockSize();
size_t x = num_blocks;
size_t source_blk = 0;
size_t blk_src_copy = 2 * num_blocks;
uint16_t xor_offset = 5;
while (1) {
- ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy));
+ ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy));
x -= 1;
if (x == 0) {
@@ -479,10 +470,10 @@
blk_src_copy += 1;
}
- ASSERT_TRUE(writer.AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks,
- xor_offset));
+ ASSERT_TRUE(writer->AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks,
+ xor_offset));
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
// Construct the buffer required for validation
orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
// Read the entire base device
@@ -500,8 +491,8 @@
unique_fd rnd_fd;
loff_t offset = 0;
- std::string path = android::base::GetExecutableDirectory();
- cow_system_ = std::make_unique<TemporaryFile>(path);
+ auto writer = CreateCowDeviceInternal();
+ ASSERT_NE(writer, nullptr);
rnd_fd.reset(open("/dev/random", O_RDONLY));
ASSERT_TRUE(rnd_fd > 0);
@@ -516,13 +507,7 @@
offset += 1_MiB;
}
- CowOptions options;
- options.compression = "gz";
- CowWriter writer(options);
-
- ASSERT_TRUE(writer.Initialize(cow_system_->fd));
-
- size_t num_blocks = size_ / options.block_size;
+ size_t num_blocks = size_ / writer->GetBlockSize();
size_t blk_end_copy = num_blocks * 2;
size_t source_blk = num_blocks - 1;
size_t blk_src_copy = blk_end_copy - 1;
@@ -536,11 +521,11 @@
for (int i = 0; i < num_blocks; i++) {
sequence[num_blocks + i] = 5 * num_blocks - 1 - i;
}
- ASSERT_TRUE(writer.AddSequenceData(2 * num_blocks, sequence));
+ ASSERT_TRUE(writer->AddSequenceData(2 * num_blocks, sequence));
size_t x = num_blocks;
while (1) {
- ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy));
+ ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy));
x -= 1;
if (x == 0) {
break;
@@ -552,24 +537,24 @@
source_blk = num_blocks;
blk_src_copy = blk_end_copy;
- ASSERT_TRUE(writer.AddRawBlocks(source_blk, random_buffer_1_.get(), size_));
+ ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_));
size_t blk_zero_copy_start = source_blk + num_blocks;
size_t blk_zero_copy_end = blk_zero_copy_start + num_blocks;
- ASSERT_TRUE(writer.AddZeroBlocks(blk_zero_copy_start, num_blocks));
+ ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks));
size_t blk_random2_replace_start = blk_zero_copy_end;
- ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_));
+ ASSERT_TRUE(writer->AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_));
size_t blk_xor_start = blk_random2_replace_start + num_blocks;
size_t xor_offset = BLOCK_SZ / 2;
- ASSERT_TRUE(writer.AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks,
- xor_offset));
+ ASSERT_TRUE(writer->AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks,
+ xor_offset));
// Flush operations
- ASSERT_TRUE(writer.Finalize());
+ ASSERT_TRUE(writer->Finalize());
// Construct the buffer required for validation
orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);
std::string zero_buffer(size_, 0);
diff --git a/init/README.md b/init/README.md
index 6bdff4a..5fced19 100644
--- a/init/README.md
+++ b/init/README.md
@@ -344,11 +344,14 @@
intended to be used with the `exec_start` builtin for any must-have checks during boot.
`restart_period <seconds>`
-> If a non-oneshot service exits, it will be restarted at its start time plus
- this period. It defaults to 5s to rate limit crashing services.
- This can be increased for services that are meant to run periodically. For
- example, it may be set to 3600 to indicate that the service should run every hour
- or 86400 to indicate that the service should run every day.
+> If a non-oneshot service exits, it will be restarted at its previous start time plus this period.
+ The default value is 5s. This can be used to implement periodic services together with the
+ `timeout_period` command below. For example, it may be set to 3600 to indicate that the service
+ should run every hour or 86400 to indicate that the service should run every day. This can be set
+ to a value shorter than 5s for example 0, but the minimum 5s delay is enforced if the restart was
+ due to a crash. This is to rate limit persistentally crashing services. In other words,
+ `<seconds>` smaller than 5 is respected only when the service exits deliverately and successfully
+ (i.e. by calling exit(0)).
`rlimit <resource> <cur> <max>`
> This applies the given rlimit to the service. rlimits are inherited by child
diff --git a/init/reboot.cpp b/init/reboot.cpp
index 27a7876..3351c4c 100644
--- a/init/reboot.cpp
+++ b/init/reboot.cpp
@@ -680,8 +680,8 @@
<< "': " << result.error();
}
s->SetShutdownCritical();
- } else if (do_shutdown_animation) {
- continue;
+ } else if (do_shutdown_animation && s->classnames().count("animation") > 0) {
+ // Need these for shutdown animations.
} else if (s->IsShutdownCritical()) {
// Start shutdown critical service if not started.
if (auto result = s->Start(); !result.ok()) {
diff --git a/init/service.cpp b/init/service.cpp
index c152081..2945708 100644
--- a/init/service.cpp
+++ b/init/service.cpp
@@ -308,6 +308,7 @@
pid_ = 0;
flags_ &= (~SVC_RUNNING);
start_order_ = 0;
+ was_last_exit_ok_ = siginfo.si_code == CLD_EXITED && siginfo.si_status == 0;
// Oneshot processes go into the disabled state on exit,
// except when manually restarted.
@@ -361,7 +362,8 @@
// If we crash > 4 times in 'fatal_crash_window_' minutes or before boot_completed,
// reboot into bootloader or set crashing property
boot_clock::time_point now = boot_clock::now();
- if (((flags_ & SVC_CRITICAL) || is_process_updatable) && !(flags_ & SVC_RESTART)) {
+ if (((flags_ & SVC_CRITICAL) || is_process_updatable) && !(flags_ & SVC_RESTART) &&
+ !was_last_exit_ok_) {
bool boot_completed = GetBoolProperty("sys.boot_completed", false);
if (now < time_crashed_ + fatal_crash_window_ || !boot_completed) {
if (++crash_count_ > 4) {
diff --git a/init/service.h b/init/service.h
index ce7c0da..b858eef 100644
--- a/init/service.h
+++ b/init/service.h
@@ -19,6 +19,7 @@
#include <signal.h>
#include <sys/types.h>
+#include <algorithm>
#include <chrono>
#include <memory>
#include <optional>
@@ -115,6 +116,7 @@
pid_t pid() const { return pid_; }
android::base::boot_clock::time_point time_started() const { return time_started_; }
int crash_count() const { return crash_count_; }
+ int was_last_exit_ok() const { return was_last_exit_ok_; }
uid_t uid() const { return proc_attr_.uid(); }
gid_t gid() const { return proc_attr_.gid; }
int namespace_flags() const { return namespaces_.flags; }
@@ -130,7 +132,15 @@
bool process_cgroup_empty() const { return process_cgroup_empty_; }
unsigned long start_order() const { return start_order_; }
void set_sigstop(bool value) { sigstop_ = value; }
- std::chrono::seconds restart_period() const { return restart_period_; }
+ std::chrono::seconds restart_period() const {
+ // If the service exited abnormally or due to timeout, late limit the restart even if
+ // restart_period is set to a very short value.
+ // If not, i.e. restart after a deliberate and successful exit, respect the period.
+ if (!was_last_exit_ok_) {
+ return std::max(restart_period_, default_restart_period_);
+ }
+ return restart_period_;
+ }
std::optional<std::chrono::seconds> timeout_period() const { return timeout_period_; }
const std::vector<std::string>& args() const { return args_; }
bool is_updatable() const { return updatable_; }
@@ -172,6 +182,8 @@
bool upgraded_mte_ = false; // whether we upgraded async MTE -> sync MTE before
std::chrono::minutes fatal_crash_window_ = 4min; // fatal() when more than 4 crashes in it
std::optional<std::string> fatal_reboot_target_; // reboot target of fatal handler
+ bool was_last_exit_ok_ =
+ true; // true if the service never exited, or exited with status code 0
std::optional<CapSet> capabilities_;
ProcessAttributes proc_attr_;
@@ -214,7 +226,8 @@
bool sigstop_ = false;
- std::chrono::seconds restart_period_ = 5s;
+ const std::chrono::seconds default_restart_period_ = 5s;
+ std::chrono::seconds restart_period_ = default_restart_period_;
std::optional<std::chrono::seconds> timeout_period_;
bool updatable_ = false;
diff --git a/init/service_parser.cpp b/init/service_parser.cpp
index d46e1f7..a1b2cc5 100644
--- a/init/service_parser.cpp
+++ b/init/service_parser.cpp
@@ -370,8 +370,8 @@
Result<void> ServiceParser::ParseRestartPeriod(std::vector<std::string>&& args) {
int period;
- if (!ParseInt(args[1], &period, 5)) {
- return Error() << "restart_period value must be an integer >= 5";
+ if (!ParseInt(args[1], &period, 0)) {
+ return Error() << "restart_period value must be an integer >= 0";
}
service_->restart_period_ = std::chrono::seconds(period);
return {};
diff --git a/libmodprobe/libmodprobe.cpp b/libmodprobe/libmodprobe.cpp
index 1971f01..858b955 100644
--- a/libmodprobe/libmodprobe.cpp
+++ b/libmodprobe/libmodprobe.cpp
@@ -439,54 +439,58 @@
return module_blocklist_.count(canonical_name) > 0;
}
-// Another option to load kernel modules. load in independent modules in parallel
-// and then update dependency list of other remaining modules, repeat these steps
-// until all modules are loaded.
+// Another option to load kernel modules. load independent modules dependencies
+// in parallel and then update dependency list of other remaining modules,
+// repeat these steps until all modules are loaded.
+// Discard all blocklist.
+// Softdeps are taken care in InsmodWithDeps().
bool Modprobe::LoadModulesParallel(int num_threads) {
bool ret = true;
- int count = -1;
- std::map<std::string, std::set<std::string>> mod_with_deps;
+ std::unordered_map<std::string, std::vector<std::string>> mod_with_deps;
// Get dependencies
for (const auto& module : module_load_) {
+ // Skip blocklist modules
+ if (IsBlocklisted(module)) {
+ LOG(VERBOSE) << "LMP: Blocklist: Module " << module << " skipping...";
+ continue;
+ }
auto dependencies = GetDependencies(MakeCanonical(module));
-
- for (auto dep = dependencies.rbegin(); dep != dependencies.rend(); dep++) {
- mod_with_deps[module].emplace(*dep);
+ if (dependencies.empty()) {
+ LOG(ERROR) << "LMP: Hard-dep: Module " << module
+ << " not in .dep file";
+ return false;
}
+ mod_with_deps[MakeCanonical(module)] = dependencies;
}
- // Get soft dependencies
- for (const auto& [it_mod, it_softdep] : module_pre_softdep_) {
- if (mod_with_deps.find(MakeCanonical(it_softdep)) != mod_with_deps.end()) {
- mod_with_deps[MakeCanonical(it_mod)].emplace(
- GetDependencies(MakeCanonical(it_softdep))[0]);
- }
- }
-
- // Get soft post dependencies
- for (const auto& [it_mod, it_softdep] : module_post_softdep_) {
- if (mod_with_deps.find(MakeCanonical(it_softdep)) != mod_with_deps.end()) {
- mod_with_deps[MakeCanonical(it_softdep)].emplace(
- GetDependencies(MakeCanonical(it_mod))[0]);
- }
- }
-
- while (!mod_with_deps.empty() && count != module_loaded_.size()) {
+ while (!mod_with_deps.empty()) {
std::vector<std::thread> threads;
std::vector<std::string> mods_path_to_load;
std::mutex vector_lock;
- count = module_loaded_.size();
// Find independent modules
for (const auto& [it_mod, it_dep] : mod_with_deps) {
- if (it_dep.size() == 1) {
- if (module_options_[it_mod].find("load_sequential=1") != std::string::npos) {
- if (!LoadWithAliases(it_mod, true) && !IsBlocklisted(it_mod)) {
- return false;
- }
- } else {
- mods_path_to_load.emplace_back(it_mod);
+ auto itd_last = it_dep.rbegin();
+ if (itd_last == it_dep.rend())
+ continue;
+
+ auto cnd_last = MakeCanonical(*itd_last);
+ // Hard-dependencies cannot be blocklisted
+ if (IsBlocklisted(cnd_last)) {
+ LOG(ERROR) << "LMP: Blocklist: Module-dep " << cnd_last
+ << " : failed to load module " << it_mod;
+ return false;
+ }
+
+ if (module_options_[cnd_last].find("load_sequential=1") != std::string::npos) {
+ if (!LoadWithAliases(cnd_last, true)) {
+ return false;
+ }
+ } else {
+ if (std::find(mods_path_to_load.begin(), mods_path_to_load.end(),
+ cnd_last) == mods_path_to_load.end()) {
+ mods_path_to_load.emplace_back(cnd_last);
}
}
}
@@ -502,7 +506,7 @@
lk.unlock();
ret_load &= LoadWithAliases(mod_to_load, true);
lk.lock();
- if (!ret_load && !IsBlocklisted(mod_to_load)) {
+ if (!ret_load) {
ret &= ret_load;
}
}
@@ -521,13 +525,18 @@
std::lock_guard guard(module_loaded_lock_);
// Remove loaded module form mod_with_deps and soft dependencies of other modules
for (const auto& module_loaded : module_loaded_) {
- mod_with_deps.erase(module_loaded);
+ if (mod_with_deps.find(module_loaded) != mod_with_deps.end()) {
+ mod_with_deps.erase(module_loaded);
+ }
}
// Remove loaded module form dependencies of other modules which are not loaded yet
for (const auto& module_loaded_path : module_loaded_paths_) {
for (auto& [mod, deps] : mod_with_deps) {
- deps.erase(module_loaded_path);
+ auto it = std::find(deps.begin(), deps.end(), module_loaded_path);
+ if (it != deps.end()) {
+ deps.erase(it);
+ }
}
}
}