update_engine: Payload generator emits SOURCE_COPY and SOURCE_BSDIFF ops.
When generating delta updates, payload generator will now emit
SOURCE_COPY and SOURCE_BSDIFF operations instead of MOVE and BSDIFF when
the minor version is 2.
BUG=chromium:461167
TEST=`FEATURES=test emerge-link update_engine`
Change-Id: I1e7553312db7362d44d320d457e4a59756bfe5b9
Reviewed-on: https://chromium-review.googlesource.com/254780
Reviewed-by: Alex Deymo <deymo@chromium.org>
Commit-Queue: Allie Wood <alliewood@chromium.org>
Trybot-Ready: Allie Wood <alliewood@chromium.org>
Tested-by: Allie Wood <alliewood@chromium.org>
diff --git a/payload_generator/delta_diff_generator.cc b/payload_generator/delta_diff_generator.cc
index 77444e1..3683860 100644
--- a/payload_generator/delta_diff_generator.cc
+++ b/payload_generator/delta_diff_generator.cc
@@ -86,6 +86,9 @@
const char* const kEmptyPath = "";
const char* const kBsdiffPath = "bsdiff";
+const uint32_t kInPlaceMinorPayloadVersion = 1;
+const uint32_t kSourceMinorPayloadVersion = 2;
+
// Needed for testing purposes, in case we can't use actual filesystem objects.
// TODO(garnold) (chromium:331965) Replace this hack with a properly injected
// parameter in form of a mockable abstract class.
@@ -116,7 +119,8 @@
const string& new_root,
off_t chunk_size,
int data_fd,
- off_t* data_file_size) {
+ off_t* data_file_size,
+ bool src_ops_allowed) {
set<ino_t> visited_inodes;
set<ino_t> visited_src_inodes;
for (FilesystemIterator fs_iter(new_root,
@@ -171,7 +175,8 @@
offset,
size,
data_fd,
- data_file_size));
+ data_file_size,
+ src_ops_allowed));
}
}
return true;
@@ -384,7 +389,8 @@
const string& new_kernel_part,
vector<DeltaArchiveManifest_InstallOperation>* ops,
int blobs_fd,
- off_t* blobs_length) {
+ off_t* blobs_length,
+ bool src_ops_allowed) {
LOG(INFO) << "Delta compressing kernel partition...";
LOG_IF(INFO, old_kernel_part.empty()) << "Generating full kernel update...";
@@ -398,7 +404,8 @@
true, // bsdiff_allowed
&data,
&op,
- false));
+ false,
+ src_ops_allowed));
// Check if the operation writes nothing.
if (op.dst_extents_size() == 0) {
@@ -412,7 +419,8 @@
}
// Write the data.
- if (op.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE) {
+ if (op.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE &&
+ op.type() != DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY) {
op.set_data_offset(*blobs_length);
op.set_data_length(data.size());
}
@@ -603,7 +611,8 @@
off_t chunk_offset,
off_t chunk_size,
int data_fd,
- off_t* data_file_size) {
+ off_t* data_file_size,
+ bool src_ops_allowed) {
chromeos::Blob data;
DeltaArchiveManifest_InstallOperation operation;
@@ -629,12 +638,14 @@
bsdiff_allowed,
&data,
&operation,
- true));
+ true,
+ src_ops_allowed));
// Check if the operation writes nothing.
if (operation.dst_extents_size() == 0) {
if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) {
- LOG(INFO) << "Empty MOVE operation (" << new_root + path << "), skipping";
+ LOG(INFO) << "Empty MOVE operation ("
+ << new_root + path << "), skipping";
return true;
} else {
LOG(ERROR) << "Empty non-MOVE operation";
@@ -643,7 +654,9 @@
}
// Write the data
- if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE) {
+ if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE &&
+ operation.type() !=
+ DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY) {
operation.set_data_offset(*data_file_size);
operation.set_data_length(data.size());
}
@@ -681,7 +694,8 @@
bool bsdiff_allowed,
chromeos::Blob* out_data,
DeltaArchiveManifest_InstallOperation* out_op,
- bool gather_extents) {
+ bool gather_extents,
+ bool src_ops_allowed) {
// Read new data in
chromeos::Blob new_data;
TEST_AND_RETURN_FALSE(
@@ -726,7 +740,12 @@
old_filename, chunk_offset, chunk_size, &old_data));
if (old_data == new_data) {
// No change in data.
- operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE);
+ if (src_ops_allowed) {
+ operation.set_type(
+ DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY);
+ } else {
+ operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE);
+ }
current_best_size = 0;
data.clear();
} else if (!old_data.empty() && bsdiff_allowed) {
@@ -750,7 +769,12 @@
BsdiffFiles(old_chunk.value(), new_chunk.value(), &bsdiff_delta));
CHECK_GT(bsdiff_delta.size(), static_cast<chromeos::Blob::size_type>(0));
if (bsdiff_delta.size() < current_best_size) {
- operation.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF);
+ if (src_ops_allowed) {
+ operation.set_type(
+ DeltaArchiveManifest_InstallOperation_Type_SOURCE_BSDIFF);
+ } else {
+ operation.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF);
+ }
current_best_size = bsdiff_delta.size();
data = bsdiff_delta;
}
@@ -762,7 +786,11 @@
vector<Extent> src_extents, dst_extents;
if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE ||
- operation.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF) {
+ operation.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF ||
+ operation.type() ==
+ DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY ||
+ operation.type() ==
+ DeltaArchiveManifest_InstallOperation_Type_SOURCE_BSDIFF) {
if (gather_extents) {
TEST_AND_RETURN_FALSE(
GatherExtents(old_filename,
@@ -941,6 +969,12 @@
return true;
}
+vector<Vertex::Index> DeltaDiffGenerator::OrderIndices(const Graph& graph) {
+ vector<Vertex::Index> order(graph.size());
+ std::iota(order.begin(), order.end(), 0);
+ return order;
+}
+
bool DeltaDiffGenerator::GenerateDeltaUpdateFile(
const string& old_root,
const string& old_image,
@@ -1025,71 +1059,116 @@
// Set the minor version for this payload.
LOG(INFO) << "Adding Delta Minor Version.";
manifest.set_minor_version(minor_version);
+ if (minor_version == kInPlaceMinorPayloadVersion) {
+ TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph,
+ &blocks,
+ old_root,
+ new_root,
+ chunk_size,
+ fd,
+ &data_file_size,
+ false)); // src_ops_allowed
+ LOG(INFO) << "done reading normal files";
+ CheckGraph(graph);
- TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph,
- &blocks,
- old_root,
- new_root,
- chunk_size,
- fd,
- &data_file_size));
- LOG(INFO) << "done reading normal files";
- CheckGraph(graph);
+ LOG(INFO) << "Starting metadata processing";
+ TEST_AND_RETURN_FALSE(Metadata::DeltaReadMetadata(&graph,
+ &blocks,
+ old_image,
+ new_image,
+ fd,
+ &data_file_size));
+ LOG(INFO) << "Done metadata processing";
+ CheckGraph(graph);
- LOG(INFO) << "Starting metadata processing";
- TEST_AND_RETURN_FALSE(Metadata::DeltaReadMetadata(&graph,
- &blocks,
- old_image,
- new_image,
- fd,
- &data_file_size));
- LOG(INFO) << "Done metadata processing";
- CheckGraph(graph);
-
- graph.resize(graph.size() + 1);
- TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks,
- fd,
- &data_file_size,
- old_image,
- new_image,
- &graph.back()));
- if (graph.back().op.data_length() == 0) {
- LOG(INFO) << "No unwritten blocks to write, omitting operation";
- graph.pop_back();
- }
-
- // Final scratch block (if there's space)
- if (blocks.size() < (rootfs_partition_size / kBlockSize)) {
- scratch_vertex = graph.size();
graph.resize(graph.size() + 1);
- InplaceGenerator::CreateScratchNode(
- blocks.size(),
- (rootfs_partition_size / kBlockSize) - blocks.size(),
- &graph.back());
+ TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks,
+ fd,
+ &data_file_size,
+ old_image,
+ new_image,
+ &graph.back()));
+ if (graph.back().op.data_length() == 0) {
+ LOG(INFO) << "No unwritten blocks to write, omitting operation";
+ graph.pop_back();
+ }
+
+ // Final scratch block (if there's space)
+ if (blocks.size() < (rootfs_partition_size / kBlockSize)) {
+ scratch_vertex = graph.size();
+ graph.resize(graph.size() + 1);
+ InplaceGenerator::CreateScratchNode(
+ blocks.size(),
+ (rootfs_partition_size / kBlockSize) - blocks.size(),
+ &graph.back());
+ }
+
+ // Read kernel partition
+ TEST_AND_RETURN_FALSE(
+ DeltaCompressKernelPartition(old_kernel_part,
+ new_kernel_part,
+ &kernel_ops,
+ fd,
+ &data_file_size,
+ false)); // src_ops_allowed
+
+ LOG(INFO) << "done reading kernel";
+ CheckGraph(graph);
+
+ LOG(INFO) << "Creating edges...";
+ InplaceGenerator::CreateEdges(&graph, blocks);
+ LOG(INFO) << "Done creating edges";
+ CheckGraph(graph);
+
+ TEST_AND_RETURN_FALSE(InplaceGenerator::ConvertGraphToDag(
+ &graph,
+ new_root,
+ fd,
+ &data_file_size,
+ &final_order,
+ scratch_vertex));
+ } else if (minor_version == kSourceMinorPayloadVersion) {
+ TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph,
+ &blocks,
+ old_root,
+ new_root,
+ chunk_size,
+ fd,
+ &data_file_size,
+ true)); // src_ops_allowed
+
+ LOG(INFO) << "done reading normal files";
+ CheckGraph(graph);
+
+ // Read kernel partition
+ TEST_AND_RETURN_FALSE(
+ DeltaCompressKernelPartition(old_kernel_part,
+ new_kernel_part,
+ &kernel_ops,
+ fd,
+ &data_file_size,
+ true)); // src_ops_allowed
+ LOG(INFO) << "done reading kernel";
+ CheckGraph(graph);
+
+ graph.resize(graph.size() + 1);
+ TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks,
+ fd,
+ &data_file_size,
+ old_image,
+ new_image,
+ &graph.back()));
+ if (graph.back().op.data_length() == 0) {
+ LOG(INFO) << "No unwritten blocks to write, omitting operation";
+ graph.pop_back();
+ }
+
+ final_order = OrderIndices(graph);
+ } else {
+ LOG(ERROR) << "Unsupported minor version given for delta payload: "
+ << minor_version;
+ return false;
}
-
- // Read kernel partition
- TEST_AND_RETURN_FALSE(DeltaCompressKernelPartition(old_kernel_part,
- new_kernel_part,
- &kernel_ops,
- fd,
- &data_file_size));
-
- LOG(INFO) << "done reading kernel";
- CheckGraph(graph);
-
- LOG(INFO) << "Creating edges...";
- InplaceGenerator::CreateEdges(&graph, blocks);
- LOG(INFO) << "Done creating edges";
- CheckGraph(graph);
-
- TEST_AND_RETURN_FALSE(InplaceGenerator::ConvertGraphToDag(
- &graph,
- new_root,
- fd,
- &data_file_size,
- &final_order,
- scratch_vertex));
} else {
// Full update
off_t new_image_size =
diff --git a/payload_generator/delta_diff_generator.h b/payload_generator/delta_diff_generator.h
index a28e1a8..2f1b356 100644
--- a/payload_generator/delta_diff_generator.h
+++ b/payload_generator/delta_diff_generator.h
@@ -30,6 +30,12 @@
extern const size_t kBlockSize;
extern const size_t kRootFSPartitionSize;
+// The minor version used by the in-place delta generator algorithm.
+extern const uint32_t kInPlaceMinorPayloadVersion;
+
+// The minor version used by the A to B delta generator algorithm.
+extern const uint32_t kSourceMinorPayloadVersion;
+
// This struct stores all relevant info for an edge that is cut between
// nodes old_src -> old_dst by creating new vertex new_vertex. The new
// relationship is:
@@ -111,7 +117,8 @@
off_t chunk_offset,
off_t chunk_size,
int data_fd,
- off_t* data_file_size);
+ off_t* data_file_size,
+ bool src_ops_allowed);
// Reads old_filename (if it exists) and a new_filename and determines
// the smallest way to encode this file for the diff. It stores
@@ -122,6 +129,8 @@
// new_filename must contain at least one byte.
// |new_filename| is read starting at |chunk_offset|.
// If |chunk_size| is not -1, only up to |chunk_size| bytes are diffed.
+ // If |src_ops_allowed| is true, it will emit SOURCE_COPY and SOURCE_BSDIFF
+ // operations instead of MOVE and BSDIFF, respectively.
// Returns true on success.
static bool ReadFileToDiff(const std::string& old_filename,
const std::string& new_filename,
@@ -130,7 +139,8 @@
bool bsdiff_allowed,
chromeos::Blob* out_data,
DeltaArchiveManifest_InstallOperation* out_op,
- bool gather_extents);
+ bool gather_extents,
+ bool src_ops_allowed);
// Stores all Extents in 'extents' into 'out'.
static void StoreExtents(const std::vector<Extent>& extents,
@@ -177,6 +187,10 @@
uint64_t signature_blob_length,
DeltaArchiveManifest* manifest);
+ // Takes |graph| and returns a vertex order with the vertex indices of
+ // |graph|, in the order they appear. Returns |true| on success.
+ static std::vector<Vertex::Index> OrderIndices(const Graph& graph);
+
// Takes a collection (vector or RepeatedPtrField) of Extent and
// returns a vector of the blocks referenced, in order.
template<typename T>
diff --git a/payload_generator/delta_diff_generator_unittest.cc b/payload_generator/delta_diff_generator_unittest.cc
index d60aa79..c8b104d 100644
--- a/payload_generator/delta_diff_generator_unittest.cc
+++ b/payload_generator/delta_diff_generator_unittest.cc
@@ -93,7 +93,8 @@
true, // bsdiff_allowed
&data,
&op,
- true));
+ true,
+ false)); // src_ops_allowed
EXPECT_TRUE(data.empty());
EXPECT_TRUE(op.has_type());
@@ -183,7 +184,8 @@
true, // bsdiff_allowed
&data,
&op,
- true));
+ true,
+ false)); // src_ops_allowed
// Adjust the old/new extents to remove duplicates.
old_extents[0].set_num_blocks(1);
@@ -247,7 +249,8 @@
true, // bsdiff_allowed
&data,
&op,
- true));
+ true,
+ false)); // src_ops_allowed
EXPECT_FALSE(data.empty());
EXPECT_TRUE(op.has_type());
@@ -280,7 +283,8 @@
false, // bsdiff_allowed
&data,
&op,
- true));
+ true,
+ false)); // src_ops_allowed
EXPECT_FALSE(data.empty());
// The point of this test is that we don't use BSDIFF the way the above
@@ -306,7 +310,8 @@
false, // bsdiff_allowed
&data,
&op,
- true));
+ true,
+ false)); // src_ops_allowed
EXPECT_TRUE(data.empty());
// The point of this test is that we can still use a MOVE for a file
@@ -332,7 +337,8 @@
true, // bsdiff_allowed
&data,
&op,
- true));
+ true,
+ false)); // src_ops_allowed
EXPECT_FALSE(data.empty());
EXPECT_TRUE(op.has_type());
@@ -366,7 +372,8 @@
true, // bsdiff_allowed
&data,
&op,
- false));
+ false,
+ false)); // src_ops_allowed
EXPECT_FALSE(data.empty());
EXPECT_TRUE(op.has_type());
@@ -383,6 +390,60 @@
EXPECT_EQ(sizeof(kRandomString), op.dst_length());
}
+TEST_F(DeltaDiffGeneratorTest, RunAsRootSourceCopyTest) {
+ // Makes sure SOURCE_COPY operations are emitted whenever src_ops_allowed
+ // is true. It is the same setup as RunAsRootMoveSmallTest, which checks that
+ // the operation is well-formed.
+ EXPECT_TRUE(utils::WriteFile(old_path().c_str(),
+ reinterpret_cast<const char*>(kRandomString),
+ sizeof(kRandomString)));
+ EXPECT_TRUE(utils::WriteFile(new_path().c_str(),
+ reinterpret_cast<const char*>(kRandomString),
+ sizeof(kRandomString)));
+ chromeos::Blob data;
+ DeltaArchiveManifest_InstallOperation op;
+ EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_path(),
+ new_path(),
+ 0, // chunk_offset
+ -1, // chunk_size
+ true, // bsdiff_allowed
+ &data,
+ &op,
+ true,
+ true)); // src_ops_allowed
+ EXPECT_TRUE(data.empty());
+
+ EXPECT_TRUE(op.has_type());
+ EXPECT_EQ(DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY, op.type());
+}
+
+TEST_F(DeltaDiffGeneratorTest, RunAsRootSourceBsdiffTest) {
+ // Makes sure SOURCE_BSDIFF operations are emitted whenever src_ops_allowed
+ // is true. It is the same setup as RunAsRootBsdiffSmallTest, which checks
+ // that the operation is well-formed.
+ EXPECT_TRUE(utils::WriteFile(old_path().c_str(),
+ reinterpret_cast<const char*>(kRandomString),
+ sizeof(kRandomString) - 1));
+ EXPECT_TRUE(utils::WriteFile(new_path().c_str(),
+ reinterpret_cast<const char*>(kRandomString),
+ sizeof(kRandomString)));
+ chromeos::Blob data;
+ DeltaArchiveManifest_InstallOperation op;
+ EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_path(),
+ new_path(),
+ 0, // chunk_offset
+ -1, // chunk_size
+ true, // bsdiff_allowed
+ &data,
+ &op,
+ true,
+ true)); // src_ops_allowed
+ EXPECT_FALSE(data.empty());
+ EXPECT_TRUE(op.has_type());
+ EXPECT_EQ(DeltaArchiveManifest_InstallOperation_Type_SOURCE_BSDIFF,
+ op.type());
+}
+
TEST_F(DeltaDiffGeneratorTest, ReorderBlobsTest) {
string orig_blobs;
EXPECT_TRUE(utils::MakeTempFile("ReorderBlobsTest.orig.XXXXXX", &orig_blobs,
@@ -422,7 +483,6 @@
unlink(new_blobs.c_str());
}
-
TEST_F(DeltaDiffGeneratorTest, IsNoopOperationTest) {
DeltaArchiveManifest_InstallOperation op;
op.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ);
@@ -448,4 +508,14 @@
EXPECT_FALSE(DeltaDiffGenerator::IsNoopOperation(op));
}
+TEST_F(DeltaDiffGeneratorTest, OrderIndicesTest) {
+ Graph graph(3);
+ vector<Vertex::Index> order;
+ order = DeltaDiffGenerator::OrderIndices(graph);
+ EXPECT_EQ(order.size(), 3);
+ EXPECT_EQ(order[0], 0);
+ EXPECT_EQ(order[1], 1);
+ EXPECT_EQ(order[2], 2);
+}
+
} // namespace chromeos_update_engine
diff --git a/payload_generator/generate_delta_main.cc b/payload_generator/generate_delta_main.cc
index 3fb6654..9f7c6ae 100644
--- a/payload_generator/generate_delta_main.cc
+++ b/payload_generator/generate_delta_main.cc
@@ -40,9 +40,6 @@
namespace {
-// The minor version used by the in-place delta generator algorithm.
-const uint32_t kInPlaceMinorPayloadVersion = 1;
-
void ParseSignatureSizes(const string& signature_sizes_flag,
vector<int>* signature_sizes) {
signature_sizes->clear();
@@ -398,6 +395,8 @@
}
if (static_cast<uint32_t>(FLAGS_minor_version) ==
+ kSourceMinorPayloadVersion ||
+ static_cast<uint32_t>(FLAGS_minor_version) ==
kInPlaceMinorPayloadVersion ||
static_cast<uint32_t>(FLAGS_minor_version) ==
DeltaPerformer::kFullPayloadMinorVersion) {
diff --git a/payload_generator/inplace_generator.cc b/payload_generator/inplace_generator.cc
index 0567b62..766dcda 100644
--- a/payload_generator/inplace_generator.cc
+++ b/payload_generator/inplace_generator.cc
@@ -534,7 +534,8 @@
(*graph)[cut.old_dst].chunk_offset,
(*graph)[cut.old_dst].chunk_size,
data_fd,
- data_file_size));
+ data_file_size,
+ false)); // src_ops_allowed
(*graph)[cut.old_dst].out_edges = out_edges;