Merge changes Ie79ab525,I3befc71f
* changes:
libsnapshot: Refactor COW reading/writing of footers.
libsnapshot: Don't try to truncate block devices
diff --git a/fs_mgr/libsnapshot/cow_api_test.cpp b/fs_mgr/libsnapshot/cow_api_test.cpp
index 9a44020..4db6584 100644
--- a/fs_mgr/libsnapshot/cow_api_test.cpp
+++ b/fs_mgr/libsnapshot/cow_api_test.cpp
@@ -264,10 +264,7 @@
ASSERT_EQ(size_before, size_after);
struct stat buf;
- if (fstat(cow_->fd, &buf) < 0) {
- perror("Fails to determine size of cow image written");
- FAIL();
- }
+ ASSERT_GE(fstat(cow_->fd, &buf), 0) << strerror(errno);
ASSERT_EQ(buf.st_size, writer.GetCowSize());
}
@@ -408,7 +405,7 @@
// Get the last known good label
CowReader label_reader;
uint64_t label;
- ASSERT_TRUE(label_reader.Parse(cow_->fd));
+ ASSERT_TRUE(label_reader.Parse(cow_->fd, {5}));
ASSERT_TRUE(label_reader.GetLastLabel(&label));
ASSERT_EQ(label, 5);
diff --git a/fs_mgr/libsnapshot/cow_reader.cpp b/fs_mgr/libsnapshot/cow_reader.cpp
index 81d1eee..6b7ada5 100644
--- a/fs_mgr/libsnapshot/cow_reader.cpp
+++ b/fs_mgr/libsnapshot/cow_reader.cpp
@@ -31,14 +31,7 @@
namespace android {
namespace snapshot {
-CowReader::CowReader()
- : fd_(-1),
- header_(),
- footer_(),
- fd_size_(0),
- has_footer_(false),
- last_label_(0),
- has_last_label_(false) {}
+CowReader::CowReader() : fd_(-1), header_(), fd_size_(0) {}
static void SHA256(const void*, size_t, uint8_t[]) {
#if 0
@@ -49,12 +42,12 @@
#endif
}
-bool CowReader::Parse(android::base::unique_fd&& fd) {
+bool CowReader::Parse(android::base::unique_fd&& fd, std::optional<uint64_t> label) {
owned_fd_ = std::move(fd);
- return Parse(android::base::borrowed_fd{owned_fd_});
+ return Parse(android::base::borrowed_fd{owned_fd_}, label);
}
-bool CowReader::Parse(android::base::borrowed_fd fd) {
+bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label) {
fd_ = fd;
auto pos = lseek(fd_.get(), 0, SEEK_END);
@@ -99,105 +92,107 @@
return false;
}
- auto footer_pos = lseek(fd_.get(), -header_.footer_size, SEEK_END);
- if (footer_pos != fd_size_ - header_.footer_size) {
- LOG(ERROR) << "Failed to read full footer!";
- return false;
- }
- if (!android::base::ReadFully(fd_, &footer_, sizeof(footer_))) {
- PLOG(ERROR) << "read footer failed";
- return false;
- }
- has_footer_ = (footer_.op.type == kCowFooterOp);
- return ParseOps();
+ return ParseOps(label);
}
-bool CowReader::ParseOps() {
+bool CowReader::ParseOps(std::optional<uint64_t> label) {
uint64_t pos = lseek(fd_.get(), sizeof(header_), SEEK_SET);
if (pos != sizeof(header_)) {
PLOG(ERROR) << "lseek ops failed";
return false;
}
- std::optional<uint64_t> next_last_label;
+
auto ops_buffer = std::make_shared<std::vector<CowOperation>>();
- if (has_footer_) ops_buffer->reserve(footer_.op.num_ops);
- uint64_t current_op_num = 0;
- // Look until we reach the last possible non-footer position.
- uint64_t last_pos = fd_size_ - (has_footer_ ? sizeof(footer_) : sizeof(CowOperation));
// Alternating op and data
- while (pos < last_pos) {
- ops_buffer->resize(current_op_num + 1);
- if (!android::base::ReadFully(fd_, ops_buffer->data() + current_op_num,
- sizeof(CowOperation))) {
+ while (true) {
+ ops_buffer->emplace_back();
+ if (!android::base::ReadFully(fd_, &ops_buffer->back(), sizeof(CowOperation))) {
PLOG(ERROR) << "read op failed";
return false;
}
- auto& current_op = ops_buffer->data()[current_op_num];
- pos = lseek(fd_.get(), GetNextOpOffset(current_op), SEEK_CUR);
- if (pos == uint64_t(-1)) {
+
+ auto& current_op = ops_buffer->back();
+ off_t offs = lseek(fd_.get(), GetNextOpOffset(current_op), SEEK_CUR);
+ if (offs < 0) {
PLOG(ERROR) << "lseek next op failed";
return false;
}
- current_op_num++;
- if (next_last_label) {
- last_label_ = next_last_label.value();
- has_last_label_ = true;
- }
+ pos = static_cast<uint64_t>(offs);
+
if (current_op.type == kCowLabelOp) {
- // If we don't have a footer, the last label may be incomplete.
- // If we see any operation after it, we can infer the flush finished.
- if (has_footer_) {
- has_last_label_ = true;
- last_label_ = current_op.source;
- } else {
- next_last_label = {current_op.source};
+ last_label_ = {current_op.source};
+
+ // If we reach the requested label, stop reading.
+ if (label && label.value() == current_op.source) {
+ break;
}
} else if (current_op.type == kCowFooterOp) {
- memcpy(&footer_.op, ¤t_op, sizeof(footer_.op));
- // we don't consider this an operation for the checksum
- current_op_num--;
- if (android::base::ReadFully(fd_, &footer_.data, sizeof(footer_.data))) {
- has_footer_ = true;
- if (next_last_label) {
- last_label_ = next_last_label.value();
- has_last_label_ = true;
- }
+ footer_.emplace();
+
+ CowFooter* footer = &footer_.value();
+ memcpy(&footer_->op, ¤t_op, sizeof(footer->op));
+
+ if (!android::base::ReadFully(fd_, &footer->data, sizeof(footer->data))) {
+ LOG(ERROR) << "Could not read COW footer";
+ return false;
}
+
+ // Drop the footer from the op stream.
+ ops_buffer->pop_back();
break;
}
}
+ // To successfully parse a COW file, we need either:
+ // (1) a label to read up to, and for that label to be found, or
+ // (2) a valid footer.
+ if (label) {
+ if (!last_label_) {
+ LOG(ERROR) << "Did not find label " << label.value()
+ << " while reading COW (no labels found)";
+ return false;
+ }
+ if (last_label_.value() != label.value()) {
+ LOG(ERROR) << "Did not find label " << label.value()
+ << ", last label=" << last_label_.value();
+ return false;
+ }
+ } else if (!footer_) {
+ LOG(ERROR) << "No COW footer found";
+ return false;
+ }
+
uint8_t csum[32];
memset(csum, 0, sizeof(uint8_t) * 32);
- if (has_footer_) {
- if (ops_buffer->size() != footer_.op.num_ops) {
+ if (footer_) {
+ if (ops_buffer->size() != footer_->op.num_ops) {
LOG(ERROR) << "num ops does not match";
return false;
}
- if (ops_buffer->size() * sizeof(CowOperation) != footer_.op.ops_size) {
+ if (ops_buffer->size() * sizeof(CowOperation) != footer_->op.ops_size) {
LOG(ERROR) << "ops size does not match ";
return false;
}
- SHA256(&footer_.op, sizeof(footer_.op), footer_.data.footer_checksum);
- if (memcmp(csum, footer_.data.ops_checksum, sizeof(csum)) != 0) {
+ SHA256(&footer_->op, sizeof(footer_->op), footer_->data.footer_checksum);
+ if (memcmp(csum, footer_->data.ops_checksum, sizeof(csum)) != 0) {
LOG(ERROR) << "ops checksum does not match";
return false;
}
- SHA256(ops_buffer.get()->data(), footer_.op.ops_size, csum);
- if (memcmp(csum, footer_.data.ops_checksum, sizeof(csum)) != 0) {
+ SHA256(ops_buffer.get()->data(), footer_->op.ops_size, csum);
+ if (memcmp(csum, footer_->data.ops_checksum, sizeof(csum)) != 0) {
LOG(ERROR) << "ops checksum does not match";
return false;
}
} else {
- LOG(INFO) << "No Footer, recovered data";
+ LOG(INFO) << "No COW Footer, recovered data";
}
if (header_.num_merge_ops > 0) {
uint64_t merge_ops = header_.num_merge_ops;
uint64_t metadata_ops = 0;
- current_op_num = 0;
+ uint64_t current_op_num = 0;
CHECK(ops_buffer->size() >= merge_ops);
while (merge_ops) {
@@ -223,14 +218,14 @@
}
bool CowReader::GetFooter(CowFooter* footer) {
- if (!has_footer_) return false;
- *footer = footer_;
+ if (!footer_) return false;
+ *footer = footer_.value();
return true;
}
bool CowReader::GetLastLabel(uint64_t* label) {
- if (!has_last_label_) return false;
- *label = last_label_;
+ if (!last_label_) return false;
+ *label = last_label_.value();
return true;
}
@@ -308,8 +303,8 @@
bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {
// Validate the offset, taking care to acknowledge possible overflow of offset+len.
- if (offset < sizeof(header_) || offset >= fd_size_ - sizeof(footer_) || len >= fd_size_ ||
- offset + len > fd_size_ - sizeof(footer_)) {
+ if (offset < sizeof(header_) || offset >= fd_size_ - sizeof(CowFooter) || len >= fd_size_ ||
+ offset + len > fd_size_ - sizeof(CowFooter)) {
LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes";
return false;
}
diff --git a/fs_mgr/libsnapshot/cow_writer.cpp b/fs_mgr/libsnapshot/cow_writer.cpp
index dcf259c..957ba35 100644
--- a/fs_mgr/libsnapshot/cow_writer.cpp
+++ b/fs_mgr/libsnapshot/cow_writer.cpp
@@ -122,6 +122,13 @@
is_dev_null_ = true;
} else {
fd_ = fd;
+
+ struct stat stat;
+ if (fstat(fd.get(), &stat) < 0) {
+ PLOG(ERROR) << "fstat failed";
+ return false;
+ }
+ is_block_device_ = S_ISBLK(stat.st_mode);
}
return true;
}
@@ -184,12 +191,10 @@
bool CowWriter::OpenForAppend(uint64_t label) {
auto reader = std::make_unique<CowReader>();
std::queue<CowOperation> toAdd;
- bool found_label = false;
- if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) {
+ if (!reader->Parse(fd_, {label}) || !reader->GetHeader(&header_)) {
return false;
}
- reader->GetFooter(&footer_);
options_.block_size = header_.block_size;
@@ -199,30 +204,19 @@
ops_.resize(0);
auto iter = reader->GetOpIter();
- while (!iter->Done() && !found_label) {
- const CowOperation& op = iter->Get();
-
- if (op.type == kCowFooterOp) break;
- if (op.type == kCowLabelOp && op.source == label) found_label = true;
- AddOperation(op);
-
+ while (!iter->Done()) {
+ AddOperation(iter->Get());
iter->Next();
}
- if (!found_label) {
- LOG(ERROR) << "Failed to find last label";
- return false;
- }
-
// Free reader so we own the descriptor position again.
reader = nullptr;
- // Position for new writing
- if (ftruncate(fd_.get(), next_op_pos_) != 0) {
- PLOG(ERROR) << "Failed to trim file";
+ // Remove excess data
+ if (!Truncate(next_op_pos_)) {
return false;
}
- if (lseek(fd_.get(), 0, SEEK_END) < 0) {
+ if (lseek(fd_.get(), next_op_pos_, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
@@ -445,5 +439,16 @@
return Sync();
}
+bool CowWriter::Truncate(off_t length) {
+ if (is_dev_null_ || is_block_device_) {
+ return true;
+ }
+ if (ftruncate(fd_.get(), length) < 0) {
+ PLOG(ERROR) << "Failed to truncate.";
+ return false;
+ }
+ return true;
+}
+
} // namespace snapshot
} // namespace android
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
index ad6b008..be69225 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
@@ -18,6 +18,7 @@
#include <functional>
#include <memory>
+#include <optional>
#include <android-base/unique_fd.h>
#include <libsnapshot/cow_format.h>
@@ -116,8 +117,10 @@
public:
CowReader();
- bool Parse(android::base::unique_fd&& fd);
- bool Parse(android::base::borrowed_fd fd);
+ // Parse the COW, optionally, up to the given label. If no label is
+ // specified, the COW must have an intact footer.
+ bool Parse(android::base::unique_fd&& fd, std::optional<uint64_t> label = {});
+ bool Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label = {});
bool GetHeader(CowHeader* header) override;
bool GetFooter(CowFooter* footer) override;
@@ -138,16 +141,14 @@
void UpdateMergeProgress(uint64_t merge_ops) { header_.num_merge_ops += merge_ops; }
private:
- bool ParseOps();
+ bool ParseOps(std::optional<uint64_t> label);
android::base::unique_fd owned_fd_;
android::base::borrowed_fd fd_;
CowHeader header_;
- CowFooter footer_;
+ std::optional<CowFooter> footer_;
uint64_t fd_size_;
- bool has_footer_;
- uint64_t last_label_;
- bool has_last_label_;
+ std::optional<uint64_t> last_label_;
std::shared_ptr<std::vector<CowOperation>> ops_;
};
diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
index d0a861b..e9320b0 100644
--- a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
+++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
@@ -123,6 +123,7 @@
bool SetFd(android::base::borrowed_fd fd);
bool Sync();
+ bool Truncate(off_t length);
private:
android::base::unique_fd owned_fd_;
@@ -133,6 +134,7 @@
uint64_t next_op_pos_ = 0;
bool is_dev_null_ = false;
bool merge_in_progress_ = false;
+ bool is_block_device_ = false;
// :TODO: this is not efficient, but stringstream ubsan aborts because some
// bytes overflow a signed char.