update_engine: Implement SOURCE_COPY/SOURCE_BSDIFF operations.
Implement SOURCE_COPY and SOURCE_BSDIFF in DeltaPerformer. These new
operations are the same as MOVE and BSDIFF, respectively, except
that they read from the source partition rather than the destination
partition.
Also fills in the source paths in omaha_response_handler_action, which
were previously not being set.
BUG=chromium:461632
TEST=`FEATURES=test emerge-link update_engine` and manual testing.
Change-Id: I81eba5780c73ed920a5db72702d9d7c67dbaa673
Reviewed-on: https://chromium-review.googlesource.com/263747
Reviewed-by: Allie Wood <alliewood@chromium.org>
Commit-Queue: Allie Wood <alliewood@chromium.org>
Trybot-Ready: Allie Wood <alliewood@chromium.org>
Tested-by: Allie Wood <alliewood@chromium.org>
diff --git a/delta_performer.cc b/delta_performer.cc
index 0544e12..a307c63 100644
--- a/delta_performer.cc
+++ b/delta_performer.cc
@@ -621,6 +621,16 @@
else if (op.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF)
op_result = HandleOpResult(
PerformBsdiffOperation(op, is_kernel_partition), "bsdiff", error);
+ else if (op.type() ==
+ DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY)
+ op_result =
+ HandleOpResult(PerformSourceCopyOperation(op, is_kernel_partition),
+ "source_copy", error);
+ else if (op.type() ==
+ DeltaArchiveManifest_InstallOperation_Type_SOURCE_BSDIFF)
+ op_result =
+ HandleOpResult(PerformSourceBsdiffOperation(op, is_kernel_partition),
+ "source_bsdiff", error);
else
op_result = HandleOpResult(false, "unknown", error);
@@ -641,9 +651,11 @@
bool DeltaPerformer::CanPerformInstallOperation(
const chromeos_update_engine::DeltaArchiveManifest_InstallOperation&
operation) {
- // Move operations don't require any data blob, so they can always
- // be performed.
- if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE)
+ // Move and source_copy operations don't require any data blob, so they can
+ // always be performed.
+ if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE ||
+ operation.type() ==
+ DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY)
return true;
// See if we have the entire data blob in the buffer
@@ -769,6 +781,84 @@
return true;
}
+namespace {
+
+// Takes |extents| and fills an empty vector |blocks| with a block index for
+// each block in |extents|. For example, [(3, 2), (8, 1)] would give [3, 4, 8].
+void ExtentsToBlocks(const RepeatedPtrField<Extent>& extents,
+ vector<uint64_t>* blocks) {
+ for (Extent ext : extents) {
+ for (uint64_t j = 0; j < ext.num_blocks(); j++)
+ blocks->push_back(ext.start_block() + j);
+ }
+}
+
+// Takes |extents| and returns the number of blocks in those extents.
+uint64_t GetBlockCount(const RepeatedPtrField<Extent>& extents) {
+ uint64_t sum = 0;
+ for (Extent ext : extents) {
+ sum += ext.num_blocks();
+ }
+ return sum;
+}
+
+} // namespace
+
+bool DeltaPerformer::PerformSourceCopyOperation(
+ const DeltaArchiveManifest_InstallOperation& operation,
+ bool is_kernel_partition) {
+ if (operation.has_src_length())
+ TEST_AND_RETURN_FALSE(operation.src_length() % block_size_ == 0);
+ if (operation.has_dst_length())
+ TEST_AND_RETURN_FALSE(operation.dst_length() % block_size_ == 0);
+
+ uint64_t blocks_to_read = GetBlockCount(operation.src_extents());
+ uint64_t blocks_to_write = GetBlockCount(operation.dst_extents());
+ TEST_AND_RETURN_FALSE(blocks_to_write == blocks_to_read);
+
+ // Create vectors of all the individual src/dst blocks.
+ vector<uint64_t> src_blocks;
+ vector<uint64_t> dst_blocks;
+ ExtentsToBlocks(operation.src_extents(), &src_blocks);
+ ExtentsToBlocks(operation.dst_extents(), &dst_blocks);
+ DCHECK_EQ(src_blocks.size(), blocks_to_read);
+ DCHECK_EQ(src_blocks.size(), dst_blocks.size());
+
+ FileDescriptorPtr src_fd =
+ is_kernel_partition ? source_kernel_fd_ : source_fd_;
+ FileDescriptorPtr dst_fd = is_kernel_partition? kernel_fd_ : fd_;
+
+ chromeos::Blob buf(block_size_);
+ ssize_t bytes_read = 0;
+ // Read/write one block at a time.
+ for (uint64_t i = 0; i < blocks_to_read; i++) {
+ ssize_t bytes_read_this_iteration = 0;
+ uint64_t src_block = src_blocks[i];
+ uint64_t dst_block = dst_blocks[i];
+
+ // Read in bytes.
+ TEST_AND_RETURN_FALSE(
+ utils::PReadAll(src_fd,
+ buf.data(),
+ block_size_,
+ src_block * block_size_,
+ &bytes_read_this_iteration));
+
+ // Write bytes out.
+ TEST_AND_RETURN_FALSE(
+ utils::PWriteAll(dst_fd,
+ buf.data(),
+ block_size_,
+ dst_block * block_size_));
+
+ bytes_read += bytes_read_this_iteration;
+ TEST_AND_RETURN_FALSE(bytes_read_this_iteration ==
+ static_cast<ssize_t>(block_size_));
+ }
+ DCHECK_EQ(bytes_read, static_cast<ssize_t>(blocks_to_read * block_size_));
+ return true;
+}
+
bool DeltaPerformer::ExtentsToBsdiffPositionsString(
const RepeatedPtrField<Extent>& extents,
uint64_t block_size,
@@ -834,14 +924,10 @@
ResetUpdateProgress(prefs_, true);
}
- vector<string> cmd;
const string& path = is_kernel_partition ? kernel_path_ : path_;
- cmd.push_back(kBspatchPath);
- cmd.push_back(path);
- cmd.push_back(path);
- cmd.push_back(temp_filename);
- cmd.push_back(input_positions);
- cmd.push_back(output_positions);
+ vector<string> cmd{kBspatchPath, path, path, temp_filename,
+ input_positions, output_positions};
+
int return_code = 0;
TEST_AND_RETURN_FALSE(
Subprocess::SynchronousExecFlags(cmd,
@@ -867,6 +953,62 @@
return true;
}
+bool DeltaPerformer::PerformSourceBsdiffOperation(
+ const DeltaArchiveManifest_InstallOperation& operation,
+ bool is_kernel_partition) {
+ // Since we delete data off the beginning of the buffer as we use it,
+ // the data we need should be exactly at the beginning of the buffer.
+ TEST_AND_RETURN_FALSE(buffer_offset_ == operation.data_offset());
+ TEST_AND_RETURN_FALSE(buffer_.size() >= operation.data_length());
+ if (operation.has_src_length())
+ TEST_AND_RETURN_FALSE(operation.src_length() % block_size_ == 0);
+ if (operation.has_dst_length())
+ TEST_AND_RETURN_FALSE(operation.dst_length() % block_size_ == 0);
+
+ string input_positions;
+ TEST_AND_RETURN_FALSE(ExtentsToBsdiffPositionsString(operation.src_extents(),
+ block_size_,
+ operation.src_length(),
+ &input_positions));
+ string output_positions;
+ TEST_AND_RETURN_FALSE(ExtentsToBsdiffPositionsString(operation.dst_extents(),
+ block_size_,
+ operation.dst_length(),
+ &output_positions));
+
+ string temp_filename;
+ TEST_AND_RETURN_FALSE(utils::MakeTempFile("/tmp/au_patch.XXXXXX",
+ &temp_filename,
+ nullptr));
+ ScopedPathUnlinker path_unlinker(temp_filename);
+ {
+ int fd = open(temp_filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
+ ScopedFdCloser fd_closer(&fd);
+ TEST_AND_RETURN_FALSE(
+ utils::WriteAll(fd, buffer_.data(), operation.data_length()));
+ }
+
+ // Update the buffer to release the patch data memory as soon as the patch
+ // file is written out.
+ DiscardBuffer(true);
+
+ const string& src_path = is_kernel_partition ?
+ install_plan_->kernel_source_path :
+ install_plan_->source_path;
+ const string& dst_path = is_kernel_partition ? kernel_path_ : path_;
+ vector<string> cmd{kBspatchPath, src_path, dst_path, temp_filename,
+ input_positions, output_positions};
+
+ int return_code = 0;
+ TEST_AND_RETURN_FALSE(
+ Subprocess::SynchronousExecFlags(cmd,
+ G_SPAWN_LEAVE_DESCRIPTORS_OPEN,
+ &return_code,
+ nullptr));
+ TEST_AND_RETURN_FALSE(return_code == 0);
+ return true;
+}
+
bool DeltaPerformer::ExtractSignatureMessage(
const DeltaArchiveManifest_InstallOperation& operation) {
if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_REPLACE ||
diff --git a/delta_performer.h b/delta_performer.h
index fd234a2..a57788d 100644
--- a/delta_performer.h
+++ b/delta_performer.h
@@ -130,7 +130,7 @@
// unavailable; it returns ErrorCode::kSignedDeltaPayloadExpectedError if the
// public key is available but the delta payload doesn't include a signature.
ErrorCode VerifyPayload(const std::string& update_check_response_hash,
- const uint64_t update_check_response_size);
+ const uint64_t update_check_response_size);
// Reads from the update manifest the expected sizes and hashes of the target
// kernel and rootfs partitions. These values can be used for applied update
@@ -276,6 +276,12 @@
bool PerformBsdiffOperation(
const DeltaArchiveManifest_InstallOperation& operation,
bool is_kernel_partition);
+ bool PerformSourceCopyOperation(
+ const DeltaArchiveManifest_InstallOperation& operation,
+ bool is_kernel_partition);
+ bool PerformSourceBsdiffOperation(
+ const DeltaArchiveManifest_InstallOperation& operation,
+ bool is_kernel_partition);
// Returns true if the payload signature message has been extracted from
// |operation|, false otherwise.
diff --git a/delta_performer_unittest.cc b/delta_performer_unittest.cc
index 2b0bc45..e969c57 100644
--- a/delta_performer_unittest.cc
+++ b/delta_performer_unittest.cc
@@ -62,6 +62,7 @@
struct DeltaState {
string a_img;
string b_img;
+ string result_img;
int image_size;
string delta_path;
@@ -73,6 +74,9 @@
string new_kernel;
chromeos::Blob new_kernel_data;
+ string result_kernel;
+ chromeos::Blob result_kernel_data;
+
// The in-memory copy of delta file.
chromeos::Blob delta;
@@ -314,6 +318,11 @@
uint32_t minor_version) {
EXPECT_TRUE(utils::MakeTempFile("a_img.XXXXXX", &state->a_img, nullptr));
EXPECT_TRUE(utils::MakeTempFile("b_img.XXXXXX", &state->b_img, nullptr));
+
+ // result_img is used in minor version 2. Instead of applying the update
+ // in-place on A, we apply it to a new image, result_img.
+ EXPECT_TRUE(
+ utils::MakeTempFile("result_img.XXXXXX", &state->result_img, nullptr));
test_utils::CreateExtImageAtPath(state->a_img, nullptr);
state->image_size = static_cast<int>(utils::FileSize(state->a_img));
@@ -390,6 +399,18 @@
base::FilePath(state->b_img)));
old_image_info = new_image_info;
} else {
+ if (minor_version == kSourceMinorPayloadVersion) {
+ // Create a result image with image_size bytes of garbage, followed by
+ // zeroes after the rootfs, like image A and B have.
+ chromeos::Blob ones(state->image_size, 0xff);
+ ones.insert(ones.end(), 1024 * 1024, 0);
+ EXPECT_TRUE(utils::WriteFile(state->result_img.c_str(),
+ ones.data(),
+ ones.size()));
+ EXPECT_EQ(utils::FileSize(state->a_img),
+ utils::FileSize(state->result_img));
+ }
+
test_utils::CreateExtImageAtPath(state->b_img, nullptr);
EXPECT_EQ(0, System(base::StringPrintf(
"dd if=/dev/zero of=%s seek=%d bs=1 count=1 status=none",
@@ -468,10 +489,17 @@
&state->new_kernel,
nullptr));
+ string result_kernel;
+ EXPECT_TRUE(utils::MakeTempFile("result_kernel.XXXXXX",
+ &state->result_kernel,
+ nullptr));
+
state->old_kernel_data.resize(kDefaultKernelSize);
state->new_kernel_data.resize(state->old_kernel_data.size());
+ state->result_kernel_data.resize(state->old_kernel_data.size());
test_utils::FillWithData(&state->old_kernel_data);
test_utils::FillWithData(&state->new_kernel_data);
+ test_utils::FillWithData(&state->result_kernel_data);
// change the new kernel data
std::copy(std::begin(kNewData), std::end(kNewData),
@@ -488,6 +516,9 @@
EXPECT_TRUE(utils::WriteFile(state->new_kernel.c_str(),
state->new_kernel_data.data(),
state->new_kernel_data.size()));
+ EXPECT_TRUE(utils::WriteFile(state->result_kernel.c_str(),
+ state->result_kernel_data.data(),
+ state->result_kernel_data.size()));
EXPECT_TRUE(utils::MakeTempFile("delta.XXXXXX",
&state->delta_path,
@@ -722,8 +753,16 @@
EXPECT_TRUE(OmahaHashCalculator::RawHashOfData(state->old_kernel_data,
&install_plan.kernel_hash));
- EXPECT_EQ(0, (*performer)->Open(state->a_img.c_str(), 0, 0));
- EXPECT_TRUE((*performer)->OpenKernel(state->old_kernel.c_str()));
+ // With minor version 2, we want the target to be the new image, result_img,
+ // but with version 1, we want to update A in place.
+ if (minor_version == kSourceMinorPayloadVersion) {
+ EXPECT_EQ(0, (*performer)->Open(state->result_img.c_str(), 0, 0));
+ EXPECT_TRUE((*performer)->OpenKernel(state->result_kernel.c_str()));
+ } else {
+ EXPECT_EQ(0, (*performer)->Open(state->a_img.c_str(), 0, 0));
+ EXPECT_TRUE((*performer)->OpenKernel(state->old_kernel.c_str()));
+ }
+
ErrorCode expected_error, actual_error;
bool continue_writing;
@@ -747,12 +786,6 @@
break;
}
- // For now, source operations are not implemented, so we expect an error.
- if (minor_version == kSourceMinorPayloadVersion) {
- expected_error = ErrorCode::kDownloadOperationExecutionError;
- continue_writing = false;
- }
-
// Write at some number of bytes per operation. Arbitrarily chose 5.
const size_t kBytesPerWrite = 5;
for (size_t i = 0; i < state->delta.size(); i += kBytesPerWrite) {
@@ -790,7 +823,8 @@
void VerifyPayloadResult(DeltaPerformer* performer,
DeltaState* state,
- ErrorCode expected_result) {
+ ErrorCode expected_result,
+ uint32_t minor_version) {
if (!performer) {
EXPECT_TRUE(!"Skipping payload verification since performer is null.");
return;
@@ -812,11 +846,18 @@
return;
}
- CompareFilesByBlock(state->old_kernel, state->new_kernel);
- CompareFilesByBlock(state->a_img, state->b_img);
-
chromeos::Blob updated_kernel_partition;
- EXPECT_TRUE(utils::ReadFile(state->old_kernel, &updated_kernel_partition));
+ if (minor_version == kSourceMinorPayloadVersion) {
+ CompareFilesByBlock(state->result_kernel, state->new_kernel);
+ CompareFilesByBlock(state->result_img, state->b_img);
+ EXPECT_TRUE(utils::ReadFile(state->result_kernel,
+ &updated_kernel_partition));
+ } else {
+ CompareFilesByBlock(state->old_kernel, state->new_kernel);
+ CompareFilesByBlock(state->a_img, state->b_img);
+ EXPECT_TRUE(utils::ReadFile(state->old_kernel, &updated_kernel_partition));
+ }
+
ASSERT_GE(updated_kernel_partition.size(), arraysize(kNewData));
EXPECT_TRUE(std::equal(std::begin(kNewData), std::end(kNewData),
updated_kernel_partition.begin()));
@@ -845,7 +886,8 @@
void VerifyPayload(DeltaPerformer* performer,
DeltaState* state,
- SignatureTest signature_test) {
+ SignatureTest signature_test,
+ uint32_t minor_version) {
ErrorCode expected_result = ErrorCode::kSuccess;
switch (signature_test) {
case kSignatureNone:
@@ -857,7 +899,7 @@
default: break; // appease gcc
}
- VerifyPayloadResult(performer, state, expected_result);
+ VerifyPayloadResult(performer, state, expected_result, minor_version);
}
void DoSmallImageTest(bool full_kernel, bool full_rootfs, bool noop,
@@ -871,13 +913,15 @@
ScopedPathUnlinker a_img_unlinker(state.a_img);
ScopedPathUnlinker b_img_unlinker(state.b_img);
+ ScopedPathUnlinker new_img_unlinker(state.result_img);
ScopedPathUnlinker delta_unlinker(state.delta_path);
ScopedPathUnlinker old_kernel_unlinker(state.old_kernel);
ScopedPathUnlinker new_kernel_unlinker(state.new_kernel);
+ ScopedPathUnlinker result_kernel_unlinker(state.result_kernel);
ApplyDeltaFile(full_kernel, full_rootfs, noop, signature_test,
&state, hash_checks_mandatory, kValidOperationData,
&performer, minor_version);
- VerifyPayload(performer, &state, signature_test);
+ VerifyPayload(performer, &state, signature_test, minor_version);
delete performer;
}
@@ -1197,6 +1241,11 @@
false, kInPlaceMinorPayloadVersion);
}
+TEST(DeltaPerformerTest, RunAsRootSmallImageSourceOpsTest) {
+ DoSmallImageTest(false, false, false, -1, kSignatureGenerator,
+ false, kSourceMinorPayloadVersion);
+}
+
TEST(DeltaPerformerTest, BadDeltaMagicTest) {
MockPrefs prefs;
InstallPlan install_plan;
@@ -1377,24 +1426,6 @@
EXPECT_TRUE(test_utils::RecursiveUnlinkDir(temp_dir));
}
-TEST(DeltaPerformerTest, RunAsRootSourceOperationsTest) {
- // Make sure we can generate a payload with the new source ops and minor
- // version 2. For now, we expect ApplyDeltaFile to fail because the ops are
- // not yet implemented, but eventually we can verify the resulting payload.
- DeltaState state;
- DeltaPerformer* performer = nullptr;
- GenerateDeltaFile(false, false, false, -1, kSignatureNone, &state,
- kSourceMinorPayloadVersion);
- ScopedPathUnlinker a_img_unlinker(state.a_img);
- ScopedPathUnlinker b_img_unlinker(state.b_img);
- ScopedPathUnlinker delta_unlinker(state.delta_path);
- ScopedPathUnlinker old_kernel_unlinker(state.old_kernel);
- ScopedPathUnlinker new_kernel_unlinker(state.new_kernel);
- ApplyDeltaFile(false, false, false, kSignatureNone, &state, false,
- kValidOperationData, &performer, kSourceMinorPayloadVersion);
- delete performer;
-}
-
TEST(DeltaPerformerTest, MinorVersionsMatch) {
// Test that the minor version in update_engine.conf that is installed to
// the image matches the supported delta minor version in the update engine.
diff --git a/omaha_response_handler_action.cc b/omaha_response_handler_action.cc
index 9f240fb..3e6f5d2 100644
--- a/omaha_response_handler_action.cc
+++ b/omaha_response_handler_action.cc
@@ -105,6 +105,9 @@
&install_plan_.install_path));
install_plan_.kernel_install_path =
utils::KernelDeviceOfBootDevice(install_plan_.install_path);
+ install_plan_.source_path = system_state_->hardware()->BootDevice();
+ install_plan_.kernel_source_path =
+ utils::KernelDeviceOfBootDevice(install_plan_.source_path);
if (params->to_more_stable_channel() && params->is_powerwash_allowed())
install_plan_.powerwash_required = true;
diff --git a/payload_generator/delta_diff_generator.cc b/payload_generator/delta_diff_generator.cc
index fca6064..50238e3 100644
--- a/payload_generator/delta_diff_generator.cc
+++ b/payload_generator/delta_diff_generator.cc
@@ -713,7 +713,8 @@
off_t* blobs_length,
const string& old_image_path,
const string& new_image_path,
- Vertex* vertex) {
+ Vertex* vertex,
+ uint32_t minor_version) {
vertex->file_name = "<rootfs-non-file-data>";
DeltaArchiveManifest_InstallOperation* out_op = &vertex->op;
@@ -789,7 +790,8 @@
int buf_offset = 0;
for (int i = 0; i < copy_block_cnt; ++i) {
int buf_end_offset = buf_offset + kBlockSize;
- if (!std::equal(new_buf.begin() + buf_offset,
+ if (minor_version == kSourceMinorPayloadVersion ||
+ !std::equal(new_buf.begin() + buf_offset,
new_buf.begin() + buf_end_offset,
old_buf.begin() + buf_offset)) {
BZ2_bzWrite(&err, bz_file, &new_buf[buf_offset], kBlockSize);
@@ -987,7 +989,7 @@
vector<AnnotatedOperation>* kernel_ops) {
// List of blocks in the target partition, with the operation that needs to
// write it and the operation that needs to read it. This is used here to
- // keep track of the blocks that no operation is writting it.
+ // keep track of the blocks that no operation is writing it.
vector<Block> blocks(config.target.rootfs_size / config.block_size);
// TODO(deymo): DeltaReadFiles() should not use a graph to generate the
@@ -1030,7 +1032,8 @@
data_file_size,
config.source.rootfs_part,
config.target.rootfs_part,
- &unwritten_vertex));
+ &unwritten_vertex,
+ config.minor_version));
if (unwritten_vertex.op.data_length() == 0) {
LOG(INFO) << "No unwritten blocks to write, omitting operation";
} else {
diff --git a/payload_generator/delta_diff_generator.h b/payload_generator/delta_diff_generator.h
index 9a35a61..45039db 100644
--- a/payload_generator/delta_diff_generator.h
+++ b/payload_generator/delta_diff_generator.h
@@ -153,7 +153,8 @@
off_t* blobs_length,
const std::string& old_image_path,
const std::string& new_image_path,
- Vertex* vertex);
+ Vertex* vertex,
+ uint32_t minor_version);
// Stores all Extents in 'extents' into 'out'.
static void StoreExtents(const std::vector<Extent>& extents,
diff --git a/payload_generator/inplace_generator.cc b/payload_generator/inplace_generator.cc
index 23617f3..869535c 100644
--- a/payload_generator/inplace_generator.cc
+++ b/payload_generator/inplace_generator.cc
@@ -723,7 +723,8 @@
data_file_size,
config.source.rootfs_part,
config.target.rootfs_part,
- &graph.back()));
+ &graph.back(),
+ config.minor_version));
if (graph.back().op.data_length() == 0) {
LOG(INFO) << "No unwritten blocks to write, omitting operation";
graph.pop_back();
diff --git a/update_metadata.proto b/update_metadata.proto
index f99c912..bf0cb45 100644
--- a/update_metadata.proto
+++ b/update_metadata.proto
@@ -133,11 +133,12 @@
// Byte length of src, equal to the number of blocks in src_extents *
// block_size. It is used for BSDIFF, because we need to pass that
// external program the number of bytes to read from the blocks we pass it.
+ // This is not used in any other operation.
optional uint64 src_length = 5;
repeated Extent dst_extents = 6;
// Byte length of dst, equal to the number of blocks in dst_extents *
- // block_size. Used for BSDIFF.
+ // block_size. Used for BSDIFF, but not in any other operation.
optional uint64 dst_length = 7;
// Optional SHA 256 hash of the blob associated with this operation.