AU: support for generating new style full updates.
New style full updates are full updates that use the delta upload file
format. A full update consists of sequential operations to write the
full kernel/rootfs in chunks (currently 128KiB).
Because these full updates use the new file format, they get benefits
over the old type of full update:
- support for embedded signatures
- (eventual) support for resuming installation across reboot
- help reduce code needed in the updater client software
BUG=7248
TEST=generated/applied full update on host; unittests
Review URL: http://codereview.chromium.org/3550020
diff --git a/delta_diff_generator.cc b/delta_diff_generator.cc
index 676629a..9a5219d 100644
--- a/delta_diff_generator.cc
+++ b/delta_diff_generator.cc
@@ -6,6 +6,7 @@
#include <errno.h>
#include <fcntl.h>
+#include <inttypes.h>
#include <sys/stat.h>
#include <sys/types.h>
@@ -47,9 +48,10 @@
typedef DeltaDiffGenerator::Block Block;
namespace {
-const size_t kBlockSize = 4096;
+const size_t kBlockSize = 4096; // bytes
const size_t kRootFSPartitionSize = 1 * 1024 * 1024 * 1024; // 1 GiB
const uint64_t kVersionNumber = 1;
+const uint64_t kFullUpdateChunkSize = 128 * 1024; // bytes
// Stores all Extents for a file into 'out'. Returns true on success.
bool GatherExtents(const string& path,
@@ -1119,6 +1121,82 @@
return true;
}
+bool DeltaDiffGenerator::ReadFullUpdateFromDisk(
+ Graph* graph,
+ const std::string& new_kernel_part,
+ const std::string& new_image,
+ int fd,
+ off_t* data_file_size,
+ off_t chunk_size,
+ vector<DeltaArchiveManifest_InstallOperation>* kernel_ops,
+ std::vector<Vertex::Index>* final_order) {
+ TEST_AND_RETURN_FALSE(chunk_size > 0);
+ TEST_AND_RETURN_FALSE((chunk_size % kBlockSize) == 0);
+
+ // Get the sizes early in the function, so we can fail fast if the user
+ // passed us bad paths.
+ const off_t image_size = utils::FileSize(new_image);
+ TEST_AND_RETURN_FALSE(image_size >= 0);
+ const off_t kernel_size = utils::FileSize(new_kernel_part);
+ TEST_AND_RETURN_FALSE(kernel_size >= 0);
+
+ off_t part_sizes[] = { image_size, kernel_size };
+ string paths[] = { new_image, new_kernel_part };
+
+ for (int partition = 0; partition < 2; ++partition) {
+ const string& path = paths[partition];
+ LOG(INFO) << "compressing " << path;
+
+ int in_fd = open(path.c_str(), O_RDONLY, 0);
+ TEST_AND_RETURN_FALSE(in_fd >= 0);
+ ScopedFdCloser in_fd_closer(&in_fd);
+
+ for (off_t bytes_left = part_sizes[partition], counter = 0, offset = 0;
+ bytes_left > 0;
+ bytes_left -= chunk_size, ++counter, offset += chunk_size) {
+ LOG(INFO) << "offset = " << offset;
+ DeltaArchiveManifest_InstallOperation* op = NULL;
+ if (partition == 0) {
+ graph->resize(graph->size() + 1);
+ graph->back().file_name = path + StringPrintf("-%" PRIi64, counter);
+ op = &graph->back().op;
+ final_order->push_back(graph->size() - 1);
+ } else {
+ kernel_ops->resize(kernel_ops->size() + 1);
+ op = &kernel_ops->back();
+ }
+ LOG(INFO) << "have an op";
+
+ vector<char> buf(min(bytes_left, chunk_size));
+ LOG(INFO) << "buf size: " << buf.size();
+ ssize_t bytes_read = -1;
+
+ TEST_AND_RETURN_FALSE(utils::PReadAll(
+ in_fd, &buf[0], buf.size(), offset, &bytes_read));
+ TEST_AND_RETURN_FALSE(bytes_read == static_cast<ssize_t>(buf.size()));
+
+ vector<char> buf_compressed;
+
+ TEST_AND_RETURN_FALSE(BzipCompress(buf, &buf_compressed));
+ const bool compress = buf_compressed.size() < buf.size();
+ const vector<char>& use_buf = compress ? buf_compressed : buf;
+ if (compress) {
+ op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ);
+ } else {
+ op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE);
+ }
+ op->set_data_offset(*data_file_size);
+ *data_file_size += use_buf.size();
+ op->set_data_length(use_buf.size());
+ Extent* dst_extent = op->add_dst_extents();
+ dst_extent->set_start_block(offset / kBlockSize);
+ dst_extent->set_num_blocks(chunk_size / kBlockSize);
+ }
+ }
+
+ return true;
+}
+
bool DeltaDiffGenerator::GenerateDeltaUpdateFile(
const string& old_root,
const string& old_image,
@@ -1129,18 +1207,23 @@
const string& output_path,
const string& private_key_path) {
struct stat old_image_stbuf;
- TEST_AND_RETURN_FALSE_ERRNO(stat(old_image.c_str(), &old_image_stbuf) == 0);
struct stat new_image_stbuf;
TEST_AND_RETURN_FALSE_ERRNO(stat(new_image.c_str(), &new_image_stbuf) == 0);
- LOG_IF(WARNING, new_image_stbuf.st_size != old_image_stbuf.st_size)
- << "Old and new images are different sizes.";
+ if (!old_image.empty()) {
+ TEST_AND_RETURN_FALSE_ERRNO(stat(old_image.c_str(), &old_image_stbuf) == 0);
+ LOG_IF(WARNING, new_image_stbuf.st_size != old_image_stbuf.st_size)
+ << "Old and new images are different sizes.";
+ LOG_IF(FATAL, old_image_stbuf.st_size % kBlockSize)
+ << "Old image not a multiple of block size " << kBlockSize;
+ // Sanity check kernel partition arg
+ TEST_AND_RETURN_FALSE(utils::FileSize(old_kernel_part) >= 0);
+ } else {
+ old_image_stbuf.st_size = 0;
+ }
LOG_IF(FATAL, new_image_stbuf.st_size % kBlockSize)
<< "New image not a multiple of block size " << kBlockSize;
- LOG_IF(FATAL, old_image_stbuf.st_size % kBlockSize)
- << "Old image not a multiple of block size " << kBlockSize;
- // Sanity check kernel partition args
- TEST_AND_RETURN_FALSE(utils::FileSize(old_kernel_part) >= 0);
+ // Sanity check kernel partition arg
TEST_AND_RETURN_FALSE(utils::FileSize(new_kernel_part) >= 0);
vector<Block> blocks(max(old_image_stbuf.st_size / kBlockSize,
@@ -1163,7 +1246,8 @@
vector<DeltaArchiveManifest_InstallOperation> kernel_ops;
vector<Vertex::Index> final_order;
- {
+ if (!old_image.empty()) {
+ // Delta update
int fd;
TEST_AND_RETURN_FALSE(
utils::MakeTempFile(kTempFileTemplate, &temp_file_path, &fd));
@@ -1206,6 +1290,17 @@
fd,
&data_file_size,
&final_order));
+ } else {
+ // Full update
+ int fd = 0;
+ TEST_AND_RETURN_FALSE(ReadFullUpdateFromDisk(&graph,
+ new_kernel_part,
+ new_image,
+ fd,
+ &data_file_size,
+ kFullUpdateChunkSize,
+ &kernel_ops,
+ &final_order));
}
// Convert to protobuf Manifest object
diff --git a/delta_diff_generator.h b/delta_diff_generator.h
index 9362b55..bebe122 100644
--- a/delta_diff_generator.h
+++ b/delta_diff_generator.h
@@ -193,6 +193,21 @@
std::vector<std::vector<Vertex::Index>::size_type>* reverse_op_indexes,
std::vector<CutEdgeVertexes>& cuts);
+ // Given a new rootfs and kernel (|new_image|, |new_kernel_part|),
+ // Reads them sequentially, creating a full update of chunk_size chunks.
+ // Populates |graph|, |kernel_ops|, and |final_order|, with data
+ // about the update operations, and writes relevant data to |fd|,
+ // updating |data_file_size| as it does.
+ static bool ReadFullUpdateFromDisk(
+ Graph* graph,
+ const std::string& new_kernel_part,
+ const std::string& new_image,
+ int fd,
+ off_t* data_file_size,
+ off_t chunk_size,
+ std::vector<DeltaArchiveManifest_InstallOperation>* kernel_ops,
+ std::vector<Vertex::Index>* final_order);
+
private:
// This should never be constructed
DISALLOW_IMPLICIT_CONSTRUCTORS(DeltaDiffGenerator);
diff --git a/delta_performer_unittest.cc b/delta_performer_unittest.cc
index 12de8a9..4a11c69 100755
--- a/delta_performer_unittest.cc
+++ b/delta_performer_unittest.cc
@@ -34,6 +34,11 @@
extern const char* kUnittestPrivateKeyPath;
extern const char* kUnittestPublicKeyPath;
+namespace {
+ const size_t kBlockSize = 4096;
+} // namespace {}
+
+
class DeltaPerformerTest : public ::testing::Test { };
TEST(DeltaPerformerTest, ExtentsToByteStringTest) {
@@ -85,7 +90,6 @@
EXPECT_TRUE(utils::ReadFile(b_file, &b_data)) << "file failed: " << b_file;
EXPECT_EQ(a_data.size(), b_data.size());
- size_t kBlockSize = 4096;
EXPECT_EQ(0, a_data.size() % kBlockSize);
for (size_t i = 0; i < a_data.size(); i += kBlockSize) {
EXPECT_EQ(0, i % kBlockSize);
@@ -277,4 +281,65 @@
EXPECT_TRUE(performer.VerifyPayload(kUnittestPublicKeyPath));
}
+TEST(DeltaPerformerTest, NewFullUpdateTest) {
+ vector<char> new_root(20 * 1024 * 1024);
+ vector<char> new_kern(16 * 1024 * 1024);
+ const off_t kChunkSize = 128 * 1024;
+ FillWithData(&new_root);
+ FillWithData(&new_kern);
+
+ string new_root_path;
+ EXPECT_TRUE(utils::MakeTempFile("/tmp/NewFullUpdateTest_R.XXXXXX",
+ &new_root_path,
+ NULL));
+ ScopedPathUnlinker new_root_path_unlinker(new_root_path);
+ EXPECT_TRUE(WriteFileVector(new_root_path, new_root));
+
+ string new_kern_path;
+ EXPECT_TRUE(utils::MakeTempFile("/tmp/NewFullUpdateTest_K.XXXXXX",
+ &new_kern_path,
+ NULL));
+ ScopedPathUnlinker new_kern_path_unlinker(new_kern_path);
+ EXPECT_TRUE(WriteFileVector(new_kern_path, new_kern));
+
+ string out_blobs_path;
+ int out_blobs_fd;
+ EXPECT_TRUE(utils::MakeTempFile("/tmp/NewFullUpdateTest_D.XXXXXX",
+ &out_blobs_path,
+ &out_blobs_fd));
+ ScopedPathUnlinker out_blobs_path_unlinker(out_blobs_path);
+ ScopedFdCloser out_blobs_fd_closer(&out_blobs_fd);
+
+ off_t out_blobs_length = 0;
+
+ Graph graph;
+ vector<DeltaArchiveManifest_InstallOperation> kernel_ops;
+ vector<Vertex::Index> final_order;
+
+ EXPECT_TRUE(DeltaDiffGenerator::ReadFullUpdateFromDisk(&graph,
+ new_kern_path,
+ new_root_path,
+ out_blobs_fd,
+ &out_blobs_length,
+ kChunkSize,
+ &kernel_ops,
+ &final_order));
+ EXPECT_EQ(new_root.size() / kChunkSize, graph.size());
+ EXPECT_EQ(new_root.size() / kChunkSize, final_order.size());
+ EXPECT_EQ(new_kern.size() / kChunkSize, kernel_ops.size());
+ for (size_t i = 0; i < (new_root.size() / kChunkSize); ++i) {
+ EXPECT_EQ(i, final_order[i]);
+ EXPECT_EQ(1, graph[i].op.dst_extents_size());
+ EXPECT_EQ(i * kChunkSize / kBlockSize,
+ graph[i].op.dst_extents(0).start_block()) << "i = " << i;
+ EXPECT_EQ(kChunkSize / kBlockSize,
+ graph[i].op.dst_extents(0).num_blocks());
+ if (graph[i].op.type() !=
+ DeltaArchiveManifest_InstallOperation_Type_REPLACE) {
+ EXPECT_EQ(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ,
+ graph[i].op.type());
+ }
+ }
+}
+
} // namespace chromeos_update_engine
diff --git a/generate_delta_main.cc b/generate_delta_main.cc
index ab05c77..c66d9d1 100644
--- a/generate_delta_main.cc
+++ b/generate_delta_main.cc
@@ -92,17 +92,20 @@
LOG(INFO) << "done applying delta.";
return 0;
}
- CHECK(!FLAGS_old_dir.empty());
- CHECK(!FLAGS_new_dir.empty());
- CHECK(!FLAGS_old_image.empty());
CHECK(!FLAGS_new_image.empty());
CHECK(!FLAGS_out_file.empty());
- CHECK(!FLAGS_old_kernel.empty());
CHECK(!FLAGS_new_kernel.empty());
- if ((!IsDir(FLAGS_old_dir.c_str())) || (!IsDir(FLAGS_new_dir.c_str()))) {
- LOG(FATAL) << "old_dir or new_dir not directory";
+ if (FLAGS_old_image.empty()) {
+ LOG(INFO) << "Generating full update";
+ } else {
+ LOG(INFO) << "Generating delta update";
+ CHECK(!FLAGS_old_kernel.empty());
+ CHECK(!FLAGS_old_dir.empty());
+ CHECK(!FLAGS_new_dir.empty());
+ if ((!IsDir(FLAGS_old_dir.c_str())) || (!IsDir(FLAGS_new_dir.c_str()))) {
+ LOG(FATAL) << "old_dir or new_dir not directory";
+ }
}
-
DeltaDiffGenerator::GenerateDeltaUpdateFile(FLAGS_old_dir,
FLAGS_old_image,
FLAGS_new_dir,