AU: Optionally limit the size of delta update operations.
Add a --chunk_size flag to delta_generator. If it's not -1, files will
be split into chunks of this size when generating delta payloads. This
effectively limits the size of each delta operation.
BUG=chromium:229797
TEST=unit tests; generated delta payloads and checked them through
paycheck.py.
Change-Id: I21502118088bfbac75aa8009eb144f6aaf23a83a
Reviewed-on: https://gerrit.chromium.org/gerrit/48357
Commit-Queue: Darin Petkov <petkov@chromium.org>
Reviewed-by: Darin Petkov <petkov@chromium.org>
Tested-by: Darin Petkov <petkov@chromium.org>
diff --git a/delta_diff_generator.cc b/delta_diff_generator.cc
index 6aea2df..6279537 100644
--- a/delta_diff_generator.cc
+++ b/delta_diff_generator.cc
@@ -17,8 +17,11 @@
#include <utility>
#include <vector>
+#include <base/file_path.h>
+#include <base/file_util.h>
#include <base/logging.h>
#include <base/memory/scoped_ptr.h>
+#include <base/string_number_conversions.h>
#include <base/string_util.h>
#include <base/stringprintf.h>
#include <bzlib.h>
@@ -53,7 +56,7 @@
typedef DeltaDiffGenerator::Block Block;
typedef map<const DeltaArchiveManifest_InstallOperation*,
- const string*> OperationNameMap;
+ string> OperationNameMap;
namespace {
const size_t kBlockSize = 4096; // bytes
@@ -74,9 +77,13 @@
// Stores all Extents for a file into 'out'. Returns true on success.
bool GatherExtents(const string& path,
+ off_t chunk_offset,
+ off_t chunk_size,
google::protobuf::RepeatedPtrField<Extent>* out) {
vector<Extent> extents;
- TEST_AND_RETURN_FALSE(extent_mapper::ExtentsForFileFibmap(path, &extents));
+ TEST_AND_RETURN_FALSE(
+ extent_mapper::ExtentsForFileChunkFibmap(
+ path, chunk_offset, chunk_size, &extents));
DeltaDiffGenerator::StoreExtents(extents, out);
return true;
}
@@ -95,6 +102,8 @@
const string& old_root,
const string& new_root,
const string& path, // within new_root
+ off_t chunk_offset,
+ off_t chunk_size,
int data_fd,
off_t* data_file_size) {
vector<char> data;
@@ -114,6 +123,8 @@
TEST_AND_RETURN_FALSE(DeltaDiffGenerator::ReadFileToDiff(old_path,
new_root + path,
+ chunk_offset,
+ chunk_size,
bsdiff_allowed,
&data,
&operation,
@@ -137,6 +148,8 @@
(*graph)[vertex].op = operation;
CHECK((*graph)[vertex].op.has_type());
(*graph)[vertex].file_name = path;
+ (*graph)[vertex].chunk_offset = chunk_offset;
+ (*graph)[vertex].chunk_size = chunk_size;
if (blocks)
TEST_AND_RETURN_FALSE(DeltaDiffGenerator::AddInstallOpToBlocksVector(
@@ -154,6 +167,7 @@
vector<Block>* blocks,
const string& old_root,
const string& new_root,
+ off_t chunk_size,
int data_fd,
off_t* data_file_size) {
set<ino_t> visited_inodes;
@@ -169,7 +183,8 @@
if (utils::SetContainsKey(visited_inodes, fs_iter.GetStat().st_ino))
continue;
visited_inodes.insert(fs_iter.GetStat().st_ino);
- if (fs_iter.GetStat().st_size == 0)
+ off_t dst_size = fs_iter.GetStat().st_size;
+ if (dst_size == 0)
continue;
LOG(INFO) << "Encoding file " << fs_iter.GetPartialPath();
@@ -193,16 +208,25 @@
visited_src_inodes.insert(src_stbuf.st_ino);
}
- TEST_AND_RETURN_FALSE(DeltaReadFile(graph,
- Vertex::kInvalidIndex,
- blocks,
- (should_diff_from_source ?
- old_root :
- kNonexistentPath),
- new_root,
- fs_iter.GetPartialPath(),
- data_fd,
- data_file_size));
+ off_t size = chunk_size == -1 ? dst_size : chunk_size;
+ off_t step = size;
+ for (off_t offset = 0; offset < dst_size; offset += step) {
+ if (offset + size >= dst_size) {
+ size = -1; // Read through the end of the file.
+ }
+ TEST_AND_RETURN_FALSE(DeltaReadFile(graph,
+ Vertex::kInvalidIndex,
+ blocks,
+ (should_diff_from_source ?
+ old_root :
+ kNonexistentPath),
+ new_root,
+ fs_iter.GetPartialPath(),
+ offset,
+ size,
+ data_fd,
+ data_file_size));
+ }
}
return true;
}
@@ -372,7 +396,18 @@
DeltaArchiveManifest_InstallOperation* op =
out_manifest->add_install_operations();
*op = add_op;
- (*out_op_name_map)[op] = &vertex.file_name;
+ string name = vertex.file_name;
+ if (vertex.chunk_offset || vertex.chunk_size != -1) {
+ string offset = base::Int64ToString(vertex.chunk_offset);
+ if (vertex.chunk_size != -1) {
+ name += " [" + offset + ", " +
+ base::Int64ToString(vertex.chunk_offset + vertex.chunk_size - 1) +
+ "]";
+ } else {
+ name += " [" + offset + ", end]";
+ }
+ }
+ (*out_op_name_map)[op] = name;
}
for (vector<DeltaArchiveManifest_InstallOperation>::const_iterator it =
kernel_ops.begin(); it != kernel_ops.end(); ++it) {
@@ -412,6 +447,8 @@
TEST_AND_RETURN_FALSE(
DeltaDiffGenerator::ReadFileToDiff(old_kernel_part,
new_kernel_part,
+ 0, // chunk_offset
+ -1, // chunk_size
true, // bsdiff_allowed
&data,
op,
@@ -454,7 +491,7 @@
for (int i = 0; i < manifest.install_operations_size(); ++i) {
const DeltaArchiveManifest_InstallOperation& op =
manifest.install_operations(i);
- objects.push_back(DeltaObject(*op_name_map.find(&op)->second,
+ objects.push_back(DeltaObject(op_name_map.find(&op)->second,
op.type(),
op.data_length()));
total_size += op.data_length();
@@ -495,15 +532,20 @@
bool DeltaDiffGenerator::ReadFileToDiff(
const string& old_filename,
const string& new_filename,
+ off_t chunk_offset,
+ off_t chunk_size,
bool bsdiff_allowed,
vector<char>* out_data,
DeltaArchiveManifest_InstallOperation* out_op,
bool gather_extents) {
// Read new data in
vector<char> new_data;
- TEST_AND_RETURN_FALSE(utils::ReadFile(new_filename, &new_data));
+ TEST_AND_RETURN_FALSE(
+ utils::ReadFileChunk(new_filename, chunk_offset, chunk_size, &new_data));
TEST_AND_RETURN_FALSE(!new_data.empty());
+ TEST_AND_RETURN_FALSE(chunk_size == -1 ||
+ static_cast<off_t>(new_data.size()) <= chunk_size);
vector<char> new_data_bz;
TEST_AND_RETURN_FALSE(BzipCompress(new_data, &new_data_bz));
@@ -532,21 +574,36 @@
original = false;
}
+ vector<char> old_data;
if (original) {
// Read old data
- vector<char> old_data;
- TEST_AND_RETURN_FALSE(utils::ReadFile(old_filename, &old_data));
+ TEST_AND_RETURN_FALSE(
+ utils::ReadFileChunk(
+ old_filename, chunk_offset, chunk_size, &old_data));
if (old_data == new_data) {
// No change in data.
operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE);
current_best_size = 0;
data.clear();
- } else if (bsdiff_allowed) {
+ } else if (!old_data.empty() && bsdiff_allowed) {
// If the source file is considered bsdiff safe (no bsdiff bugs
// triggered), see if BSDIFF encoding is smaller.
+ FilePath old_chunk;
+ TEST_AND_RETURN_FALSE(file_util::CreateTemporaryFile(&old_chunk));
+ ScopedPathUnlinker old_unlinker(old_chunk.value());
+ TEST_AND_RETURN_FALSE(
+ utils::WriteFile(old_chunk.value().c_str(),
+ &old_data[0], old_data.size()));
+ FilePath new_chunk;
+ TEST_AND_RETURN_FALSE(file_util::CreateTemporaryFile(&new_chunk));
+ ScopedPathUnlinker new_unlinker(new_chunk.value());
+ TEST_AND_RETURN_FALSE(
+ utils::WriteFile(new_chunk.value().c_str(),
+ &new_data[0], new_data.size()));
+
vector<char> bsdiff_delta;
TEST_AND_RETURN_FALSE(
- BsdiffFiles(old_filename, new_filename, &bsdiff_delta));
+ BsdiffFiles(old_chunk.value(), new_chunk.value(), &bsdiff_delta));
CHECK_GT(bsdiff_delta.size(), static_cast<vector<char>::size_type>(0));
if (bsdiff_delta.size() < current_best_size) {
operation.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF);
@@ -563,19 +620,25 @@
operation.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF) {
if (gather_extents) {
TEST_AND_RETURN_FALSE(
- GatherExtents(old_filename, operation.mutable_src_extents()));
+ GatherExtents(old_filename,
+ chunk_offset,
+ chunk_size,
+ operation.mutable_src_extents()));
} else {
Extent* src_extent = operation.add_src_extents();
src_extent->set_start_block(0);
src_extent->set_num_blocks(
(old_stbuf.st_size + kBlockSize - 1) / kBlockSize);
}
- operation.set_src_length(old_stbuf.st_size);
+ operation.set_src_length(old_data.size());
}
if (gather_extents) {
TEST_AND_RETURN_FALSE(
- GatherExtents(new_filename, operation.mutable_dst_extents()));
+ GatherExtents(new_filename,
+ chunk_offset,
+ chunk_size,
+ operation.mutable_dst_extents()));
} else {
Extent* dst_extent = operation.add_dst_extents();
dst_extent->set_start_block(0);
@@ -1221,6 +1284,8 @@
kNonexistentPath,
new_root,
(*graph)[cut.old_dst].file_name,
+ (*graph)[cut.old_dst].chunk_offset,
+ (*graph)[cut.old_dst].chunk_size,
data_fd,
data_file_size));
@@ -1321,7 +1386,9 @@
const string& new_kernel_part,
const string& output_path,
const string& private_key_path,
+ off_t chunk_size,
uint64_t* metadata_size) {
+ TEST_AND_RETURN_FALSE(chunk_size == -1 || chunk_size % kBlockSize == 0);
int old_image_block_count = 0, old_image_block_size = 0;
int new_image_block_count = 0, new_image_block_size = 0;
TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(new_image,
@@ -1373,6 +1440,7 @@
&blocks,
old_root,
new_root,
+ chunk_size,
fd,
&data_file_size));
LOG(INFO) << "done reading normal files";
diff --git a/delta_diff_generator.h b/delta_diff_generator.h
index 8ecac63..5dfa364 100644
--- a/delta_diff_generator.h
+++ b/delta_diff_generator.h
@@ -61,6 +61,8 @@
// private_key_path points to a private key used to sign the update.
// Pass empty string to not sign the update.
// output_path is the filename where the delta update should be written.
+ // If |chunk_size| is not -1, the delta payload is generated based on
+ // |chunk_size| chunks rather than whole files.
// Returns true on success. Also writes the size of the metadata into
// |metadata_size|.
static bool GenerateDeltaUpdateFile(const std::string& old_root,
@@ -71,6 +73,7 @@
const std::string& new_kernel_part,
const std::string& output_path,
const std::string& private_key_path,
+ off_t chunk_size,
uint64_t* metadata_size);
// These functions are public so that the unit tests can access them:
@@ -98,9 +101,13 @@
// operation. If there is a change, or the old file doesn't exist,
// the smallest of REPLACE, REPLACE_BZ, or BSDIFF wins.
// new_filename must contain at least one byte.
+ // |new_filename| is read starting at |chunk_offset|.
+ // If |chunk_size| is not -1, only up to |chunk_size| bytes are diffed.
// Returns true on success.
static bool ReadFileToDiff(const std::string& old_filename,
const std::string& new_filename,
+ off_t chunk_offset,
+ off_t chunk_size,
bool bsdiff_allowed,
std::vector<char>* out_data,
DeltaArchiveManifest_InstallOperation* out_op,
diff --git a/delta_diff_generator_unittest.cc b/delta_diff_generator_unittest.cc
index 349065f..7fa28c3 100644
--- a/delta_diff_generator_unittest.cc
+++ b/delta_diff_generator_unittest.cc
@@ -70,6 +70,8 @@
DeltaArchiveManifest_InstallOperation op;
EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_path(),
new_path(),
+ 0, // chunk_offset
+ -1, // chunk_size
true, // bsdiff_allowed
&data,
&op,
@@ -100,6 +102,8 @@
DeltaArchiveManifest_InstallOperation op;
EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_path(),
new_path(),
+ 0, // chunk_offset
+ -1, // chunk_size
true, // bsdiff_allowed
&data,
&op,
@@ -131,6 +135,8 @@
EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_path(),
new_path(),
+ 0, // chunk_offset
+ -1, // chunk_size
false, // bsdiff_allowed
&data,
&op,
@@ -155,6 +161,8 @@
EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_path(),
new_path(),
+ 0, // chunk_offset
+ -1, // chunk_size
false, // bsdiff_allowed
&data,
&op,
@@ -180,6 +188,8 @@
DeltaArchiveManifest_InstallOperation op;
EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_path(),
new_path(),
+ 0, // chunk_offset
+ -1, // chunk_size
true, // bsdiff_allowed
&data,
&op,
@@ -212,6 +222,8 @@
DeltaArchiveManifest_InstallOperation op;
EXPECT_TRUE(DeltaDiffGenerator::ReadFileToDiff(old_path(),
new_path(),
+ 0, // chunk_offset
+ -1, // chunk_size
true, // bsdiff_allowed
&data,
&op,
diff --git a/delta_performer_unittest.cc b/delta_performer_unittest.cc
index 1c0c2b5..0c2c588 100644
--- a/delta_performer_unittest.cc
+++ b/delta_performer_unittest.cc
@@ -253,6 +253,7 @@
static void GenerateDeltaFile(bool full_kernel,
bool full_rootfs,
bool noop,
+ off_t chunk_size,
SignatureTest signature_test,
DeltaState *state) {
EXPECT_TRUE(utils::MakeTempFile("/tmp/a_img.XXXXXX", &state->a_img, NULL));
@@ -273,10 +274,17 @@
string a_mnt;
ScopedLoopMounter b_mounter(state->a_img, &a_mnt, 0);
+ vector<char> hardtocompress;
+ while (hardtocompress.size() < 3 * kBlockSize) {
+ hardtocompress.insert(hardtocompress.end(),
+ kRandomString,
+ kRandomString + sizeof(kRandomString) - 1);
+ }
EXPECT_TRUE(utils::WriteFile(StringPrintf("%s/hardtocompress",
a_mnt.c_str()).c_str(),
- reinterpret_cast<const char*>(kRandomString),
- sizeof(kRandomString) - 1));
+ &hardtocompress[0],
+ hardtocompress.size()));
+
// Write 1 MiB of 0xff to try to catch the case where writing a bsdiff
// patch fails to zero out the final block.
vector<char> ones(1024 * 1024, 0xff);
@@ -322,10 +330,17 @@
EXPECT_EQ(0, system(StringPrintf("rm %s/boguslink && "
"echo foobar > %s/boguslink",
b_mnt.c_str(), b_mnt.c_str()).c_str()));
+
+ vector<char> hardtocompress;
+ while (hardtocompress.size() < 3 * kBlockSize) {
+ hardtocompress.insert(hardtocompress.end(),
+ kRandomString,
+ kRandomString + sizeof(kRandomString));
+ }
EXPECT_TRUE(utils::WriteFile(StringPrintf("%s/hardtocompress",
b_mnt.c_str()).c_str(),
- reinterpret_cast<const char*>(kRandomString),
- sizeof(kRandomString)));
+ &hardtocompress[0],
+ hardtocompress.size()));
}
string old_kernel;
@@ -378,6 +393,7 @@
state->new_kernel,
state->delta_path,
private_key,
+ chunk_size,
&state->metadata_size));
}
@@ -642,11 +658,13 @@
}
void DoSmallImageTest(bool full_kernel, bool full_rootfs, bool noop,
+ off_t chunk_size,
SignatureTest signature_test,
bool hash_checks_mandatory) {
DeltaState state;
DeltaPerformer *performer;
- GenerateDeltaFile(full_kernel, full_rootfs, noop, signature_test, &state);
+ GenerateDeltaFile(full_kernel, full_rootfs, noop, chunk_size,
+ signature_test, &state);
ScopedPathUnlinker a_img_unlinker(state.a_img);
ScopedPathUnlinker b_img_unlinker(state.b_img);
ScopedPathUnlinker delta_unlinker(state.delta_path);
@@ -706,7 +724,7 @@
// Using kSignatureNone since it doesn't affect the results of our test.
// If we've to use other signature options, then we'd have to get the
// metadata size again after adding the signing operation to the manifest.
- GenerateDeltaFile(true, true, false, signature_test, &state);
+ GenerateDeltaFile(true, true, false, -1, signature_test, &state);
ScopedPathUnlinker a_img_unlinker(state.a_img);
ScopedPathUnlinker b_img_unlinker(state.b_img);
@@ -794,7 +812,7 @@
void DoOperationHashMismatchTest(OperationHashTest op_hash_test,
bool hash_checks_mandatory) {
DeltaState state;
- GenerateDeltaFile(true, true, false, kSignatureGenerated, &state);
+ GenerateDeltaFile(true, true, false, -1, kSignatureGenerated, &state);
ScopedPathUnlinker a_img_unlinker(state.a_img);
ScopedPathUnlinker b_img_unlinker(state.b_img);
ScopedPathUnlinker delta_unlinker(state.delta_path);
@@ -831,61 +849,67 @@
TEST(DeltaPerformerTest, RunAsRootSmallImageTest) {
bool hash_checks_mandatory = false;
- DoSmallImageTest(false, false, false, kSignatureGenerator,
+ DoSmallImageTest(false, false, false, -1, kSignatureGenerator,
+ hash_checks_mandatory);
+}
+
+TEST(DeltaPerformerTest, RunAsRootSmallImageChunksTest) {
+ bool hash_checks_mandatory = false;
+ DoSmallImageTest(false, false, false, kBlockSize, kSignatureGenerator,
hash_checks_mandatory);
}
TEST(DeltaPerformerTest, RunAsRootFullKernelSmallImageTest) {
bool hash_checks_mandatory = false;
- DoSmallImageTest(true, false, false, kSignatureGenerator,
+ DoSmallImageTest(true, false, false, -1, kSignatureGenerator,
hash_checks_mandatory);
}
TEST(DeltaPerformerTest, RunAsRootFullSmallImageTest) {
bool hash_checks_mandatory = true;
- DoSmallImageTest(true, true, false, kSignatureGenerator,
+ DoSmallImageTest(true, true, false, -1, kSignatureGenerator,
hash_checks_mandatory);
}
TEST(DeltaPerformerTest, RunAsRootNoopSmallImageTest) {
bool hash_checks_mandatory = false;
- DoSmallImageTest(false, false, true, kSignatureGenerator,
+ DoSmallImageTest(false, false, true, -1, kSignatureGenerator,
hash_checks_mandatory);
}
TEST(DeltaPerformerTest, RunAsRootSmallImageSignNoneTest) {
bool hash_checks_mandatory = false;
- DoSmallImageTest(false, false, false, kSignatureNone,
+ DoSmallImageTest(false, false, false, -1, kSignatureNone,
hash_checks_mandatory);
}
TEST(DeltaPerformerTest, RunAsRootSmallImageSignGeneratedTest) {
bool hash_checks_mandatory = true;
- DoSmallImageTest(false, false, false, kSignatureGenerated,
+ DoSmallImageTest(false, false, false, -1, kSignatureGenerated,
hash_checks_mandatory);
}
TEST(DeltaPerformerTest, RunAsRootSmallImageSignGeneratedShellTest) {
bool hash_checks_mandatory = false;
- DoSmallImageTest(false, false, false, kSignatureGeneratedShell,
+ DoSmallImageTest(false, false, false, -1, kSignatureGeneratedShell,
hash_checks_mandatory);
}
TEST(DeltaPerformerTest, RunAsRootSmallImageSignGeneratedShellBadKeyTest) {
bool hash_checks_mandatory = false;
- DoSmallImageTest(false, false, false, kSignatureGeneratedShellBadKey,
+ DoSmallImageTest(false, false, false, -1, kSignatureGeneratedShellBadKey,
hash_checks_mandatory);
}
TEST(DeltaPerformerTest, RunAsRootSmallImageSignGeneratedShellRotateCl1Test) {
bool hash_checks_mandatory = false;
- DoSmallImageTest(false, false, false, kSignatureGeneratedShellRotateCl1,
+ DoSmallImageTest(false, false, false, -1, kSignatureGeneratedShellRotateCl1,
hash_checks_mandatory);
}
TEST(DeltaPerformerTest, RunAsRootSmallImageSignGeneratedShellRotateCl2Test) {
bool hash_checks_mandatory = false;
- DoSmallImageTest(false, false, false, kSignatureGeneratedShellRotateCl2,
+ DoSmallImageTest(false, false, false, -1, kSignatureGeneratedShellRotateCl2,
hash_checks_mandatory);
}
diff --git a/extent_mapper.cc b/extent_mapper.cc
index e02f5a2..1cfd8bb 100644
--- a/extent_mapper.cc
+++ b/extent_mapper.cc
@@ -31,40 +31,55 @@
const int kBlockSize = 4096;
}
-bool ExtentsForFileFibmap(const std::string& path, std::vector<Extent>* out) {
+bool ExtentsForFileChunkFibmap(const std::string& path,
+ off_t chunk_offset,
+ off_t chunk_size,
+ std::vector<Extent>* out) {
CHECK(out);
+ CHECK_EQ(0, chunk_offset % kBlockSize);
+ CHECK(chunk_size == -1 || chunk_size >= 0);
struct stat stbuf;
int rc = stat(path.c_str(), &stbuf);
TEST_AND_RETURN_FALSE_ERRNO(rc == 0);
TEST_AND_RETURN_FALSE(S_ISREG(stbuf.st_mode));
-
+
int fd = open(path.c_str(), O_RDONLY, 0);
TEST_AND_RETURN_FALSE_ERRNO(fd >= 0);
ScopedFdCloser fd_closer(&fd);
-
+
// Get file size in blocks
rc = fstat(fd, &stbuf);
if (rc < 0) {
perror("fstat");
return false;
}
- const int block_count = (stbuf.st_size + kBlockSize - 1) / kBlockSize;
+ CHECK_LE(chunk_offset, stbuf.st_size);
+ off_t size = stbuf.st_size - chunk_offset;
+ if (chunk_size != -1) {
+ size = std::min(size, chunk_size);
+ }
+ const int block_count = (size + kBlockSize - 1) / kBlockSize;
+ const int start_block = chunk_offset / kBlockSize;
Extent current;
current.set_start_block(0);
current.set_num_blocks(0);
- for (int i = 0; i < block_count; i++) {
+ for (int i = start_block; i < start_block + block_count; i++) {
unsigned int block32 = i;
rc = ioctl(fd, FIBMAP, &block32);
TEST_AND_RETURN_FALSE_ERRNO(rc == 0);
-
+
const uint64_t block = (block32 == 0 ? kSparseHole : block32);
-
+
graph_utils::AppendBlockToExtents(out, block);
}
return true;
}
+bool ExtentsForFileFibmap(const std::string& path, std::vector<Extent>* out) {
+ return ExtentsForFileChunkFibmap(path, 0, -1, out);
+}
+
bool GetFilesystemBlockSize(const std::string& path, uint32_t* out_blocksize) {
int fd = open(path.c_str(), O_RDONLY, 0);
TEST_AND_RETURN_FALSE_ERRNO(fd >= 0);
diff --git a/extent_mapper.h b/extent_mapper.h
index 8805393..e03ef21 100644
--- a/extent_mapper.h
+++ b/extent_mapper.h
@@ -22,7 +22,16 @@
// the blocksize of a filesystem is often 4096 bytes, that is not always
// the case, so one should consult GetFilesystemBlockSize(), too.
// Returns true on success.
+//
+// ExtentsForFileChunkFibmap gets the blocks starting from
+// |chunk_offset|. |chunk_offset| must be a multiple of the block size. If
+// |chunk_size| is not -1, only blocks covering up to |chunk_size| bytes are
+// returned.
bool ExtentsForFileFibmap(const std::string& path, std::vector<Extent>* out);
+bool ExtentsForFileChunkFibmap(const std::string& path,
+ off_t chunk_offset,
+ off_t chunk_size,
+ std::vector<Extent>* out);
// Puts the blocksize of the filesystem, as used by the FIBMAP ioctl, into
// out_blocksize by using the FIGETBSZ ioctl. Returns true on success.
diff --git a/extent_mapper_unittest.cc b/extent_mapper_unittest.cc
index 7a5e598..3797b7a 100644
--- a/extent_mapper_unittest.cc
+++ b/extent_mapper_unittest.cc
@@ -28,18 +28,18 @@
// In lieu of this, we do a weak test: make sure the extents of the unittest
// executable are consistent and they match with the size of the file.
const string kFilename = "/proc/self/exe";
-
+
uint32_t block_size = 0;
EXPECT_TRUE(extent_mapper::GetFilesystemBlockSize(kFilename, &block_size));
EXPECT_GT(block_size, 0);
-
+
vector<Extent> extents;
-
+
ASSERT_TRUE(extent_mapper::ExtentsForFileFibmap(kFilename, &extents));
-
+
EXPECT_FALSE(extents.empty());
set<uint64_t> blocks;
-
+
for (vector<Extent>::const_iterator it = extents.begin();
it != extents.end(); ++it) {
for (uint64_t block = it->start_block();
@@ -49,10 +49,25 @@
blocks.insert(block);
}
}
-
+
struct stat stbuf;
EXPECT_EQ(0, stat(kFilename.c_str(), &stbuf));
EXPECT_EQ(blocks.size(), (stbuf.st_size + block_size - 1)/block_size);
+
+ // Map a 2-block chunk at offset |block_size|.
+ vector<Extent> chunk_extents;
+ ASSERT_TRUE(
+ extent_mapper::ExtentsForFileChunkFibmap(kFilename,
+ block_size,
+ block_size + 1,
+ &chunk_extents));
+ EXPECT_FALSE(chunk_extents.empty());
+ int chunk_blocks = 0;
+ for (vector<Extent>::const_iterator it = chunk_extents.begin();
+ it != chunk_extents.end(); ++it) {
+ chunk_blocks += it->num_blocks();
+ }
+ EXPECT_EQ(2, chunk_blocks);
}
TEST(ExtentMapperTest, RunAsRootSparseFileTest) {
diff --git a/generate_delta_main.cc b/generate_delta_main.cc
index aa2b9cb..bde08d4 100644
--- a/generate_delta_main.cc
+++ b/generate_delta_main.cc
@@ -60,6 +60,7 @@
"e.g. /path/to/sig:/path/to/next:/path/to/last_sig . Each "
"signature will be assigned a client version, starting from "
"kSignatureOriginalVersion.");
+DEFINE_int32(chunk_size, -1, "Payload chunk size (-1 -- no limit/default)");
// This file contains a simple program that takes an old path, a new path,
// and an output file as arguments and the path to an output file and
@@ -272,6 +273,7 @@
FLAGS_new_kernel,
FLAGS_out_file,
FLAGS_private_key,
+ FLAGS_chunk_size,
&metadata_size)) {
return 1;
}
diff --git a/graph_types.h b/graph_types.h
index e3220c0..f8d5afa 100644
--- a/graph_types.h
+++ b/graph_types.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@@ -25,21 +25,26 @@
// node pointed to before the pointing node runs (presumably b/c it
// overwrites these blocks).
std::vector<Extent> extents;
-
+
// Write before extents. I.e., blocks in |write_extents| must be written
// by the node pointed to before the pointing node runs (presumably
// b/c it reads the data written by the other node).
std::vector<Extent> write_extents;
-
+
bool operator==(const EdgeProperties& that) const {
return extents == that.extents && write_extents == that.write_extents;
}
};
struct Vertex {
- Vertex() : valid(true), index(-1), lowlink(-1) {}
+ Vertex() :
+ valid(true),
+ index(-1),
+ lowlink(-1),
+ chunk_offset(0),
+ chunk_size(-1) {}
bool valid;
-
+
typedef std::map<std::vector<Vertex>::size_type, EdgeProperties> EdgeMap;
EdgeMap out_edges;
@@ -57,6 +62,8 @@
// Other Vertex properties:
DeltaArchiveManifest_InstallOperation op;
std::string file_name;
+ off_t chunk_offset;
+ off_t chunk_size;
typedef std::vector<Vertex>::size_type Index;
static const Vertex::Index kInvalidIndex = -1;
diff --git a/graph_utils.cc b/graph_utils.cc
index 47d0c53..02b49c3 100644
--- a/graph_utils.cc
+++ b/graph_utils.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@@ -155,9 +155,10 @@
type_str = "REPLACE_BZ";
break;
}
- LOG(INFO) << i
+ LOG(INFO) << i
<< (graph[i].valid ? "" : "-INV")
<< ": " << graph[i].file_name
+ << " " << graph[i].chunk_size << "@" << graph[i].chunk_offset
<< ": " << type_str;
LOG(INFO) << " src_extents:";
DumpExtents(graph[i].op.src_extents(), 4);
diff --git a/utils.cc b/utils.cc
index 6e14f22..3853b94 100644
--- a/utils.cc
+++ b/utils.cc
@@ -177,25 +177,56 @@
// Reads from an open file |fp|, appending the read content to the container
// pointer to by |out_p|. Returns true upon successful reading all of the
-// file's content, false otherwise.
+// file's content, false otherwise. If |size| is not -1, reads up to |size|
+// bytes.
template <class T>
-static bool Read(FILE* fp, T* out_p) {
+static bool Read(FILE* fp, off_t size, T* out_p) {
CHECK(fp);
+ CHECK(size == -1 || size >= 0);
char buf[1024];
- while (size_t nbytes = fread(buf, 1, sizeof(buf), fp))
+ while (size == -1 || size > 0) {
+ off_t bytes_to_read = sizeof(buf);
+ if (size > 0 && bytes_to_read > size) {
+ bytes_to_read = size;
+ }
+ size_t nbytes = fread(buf, 1, bytes_to_read, fp);
+ if (!nbytes) {
+ break;
+ }
AppendBytes(buf, nbytes, out_p);
- return feof(fp) && !ferror(fp);
+ if (size != -1) {
+ CHECK(size >= static_cast<off_t>(nbytes));
+ size -= nbytes;
+ }
+ }
+ if (ferror(fp)) {
+ return false;
+ }
+ return size == 0 || feof(fp);
}
-// Opens a file |path| for reading, then uses |append_func| to append its
-// content to a container |out_p|.
+// Opens a file |path| for reading and appends its the contents to a container
+// |out_p|. Starts reading the file from |offset|. If |offset| is beyond the end
+// of the file, returns success. If |size| is not -1, reads up to |size| bytes.
template <class T>
-static bool ReadFileAndAppend(const std::string& path, T* out_p) {
- FILE* fp = fopen(path.c_str(), "r");
- if (!fp)
+static bool ReadFileChunkAndAppend(const std::string& path,
+ off_t offset,
+ off_t size,
+ T* out_p) {
+ CHECK_GE(offset, 0);
+ CHECK(size == -1 || size >= 0);
+ file_util::ScopedFILE fp(fopen(path.c_str(), "r"));
+ if (!fp.get())
return false;
- bool success = Read(fp, out_p);
- return (success && !fclose(fp));
+ if (offset) {
+ // Return success without appending any data if a chunk beyond the end of
+ // the file is requested.
+ if (offset >= FileSize(path)) {
+ return true;
+ }
+ TEST_AND_RETURN_FALSE_ERRNO(fseek(fp.get(), offset, SEEK_SET) == 0);
+ }
+ return Read(fp.get(), size, out_p);
}
// Invokes a pipe |cmd|, then uses |append_func| to append its stdout to a
@@ -205,24 +236,29 @@
FILE* fp = popen(cmd.c_str(), "r");
if (!fp)
return false;
- bool success = Read(fp, out_p);
+ bool success = Read(fp, -1, out_p);
return (success && pclose(fp) >= 0);
}
-bool ReadFile(const std::string& path, std::vector<char>* out_p) {
- return ReadFileAndAppend(path, out_p);
+bool ReadFile(const string& path, vector<char>* out_p) {
+ return ReadFileChunkAndAppend(path, 0, -1, out_p);
}
-bool ReadFile(const std::string& path, std::string* out_p) {
- return ReadFileAndAppend(path, out_p);
+bool ReadFile(const string& path, string* out_p) {
+ return ReadFileChunkAndAppend(path, 0, -1, out_p);
}
-bool ReadPipe(const std::string& cmd, std::vector<char>* out_p) {
+bool ReadFileChunk(const string& path, off_t offset, off_t size,
+ vector<char>* out_p) {
+ return ReadFileChunkAndAppend(path, offset, size, out_p);
+}
+
+bool ReadPipe(const string& cmd, vector<char>* out_p) {
return ReadPipeAndAppend(cmd, out_p);
}
-bool ReadPipe(const std::string& cmd, std::string* out_p) {
+bool ReadPipe(const string& cmd, string* out_p) {
return ReadPipeAndAppend(cmd, out_p);
}
diff --git a/utils.h b/utils.h
index 53e58fe..ef42745 100644
--- a/utils.h
+++ b/utils.h
@@ -62,9 +62,12 @@
// Opens |path| for reading and appends its entire content to the container
// pointed to by |out_p|. Returns true upon successfully reading all of the
// file's content, false otherwise, in which case the state of the output
-// container is unknown.
+// container is unknown. ReadFileChunk starts reading the file from |offset|; if
+// |size| is not -1, only up to |size| bytes are read in.
bool ReadFile(const std::string& path, std::vector<char>* out_p);
bool ReadFile(const std::string& path, std::string* out_p);
+bool ReadFileChunk(const std::string& path, off_t offset, off_t size,
+ std::vector<char>* out_p);
// Invokes |cmd| in a pipe and appends its stdout to the container pointed to by
// |out_p|. Returns true upon successfully reading all of the output, false
diff --git a/utils_unittest.cc b/utils_unittest.cc
index 39c8ad7..ec58046 100644
--- a/utils_unittest.cc
+++ b/utils_unittest.cc
@@ -10,6 +10,8 @@
#include <string>
#include <vector>
+#include <base/file_path.h>
+#include <base/file_util.h>
#include <base/string_util.h>
#include <base/stringprintf.h>
#include <gtest/gtest.h>
@@ -69,6 +71,27 @@
EXPECT_FALSE(utils::ReadFile("/this/doesn't/exist", &empty));
}
+TEST(UtilsTest, ReadFileChunk) {
+ FilePath file;
+ EXPECT_TRUE(file_util::CreateTemporaryFile(&file));
+ ScopedPathUnlinker unlinker(file.value());
+ vector<char> data;
+ const size_t kSize = 1024 * 1024;
+ for (size_t i = 0; i < kSize; i++) {
+ data.push_back(i % 255);
+ }
+ EXPECT_TRUE(utils::WriteFile(file.value().c_str(), &data[0], data.size()));
+ vector<char> in_data;
+ EXPECT_TRUE(utils::ReadFileChunk(file.value().c_str(), kSize, 10, &in_data));
+ EXPECT_TRUE(in_data.empty());
+ EXPECT_TRUE(utils::ReadFileChunk(file.value().c_str(), 0, -1, &in_data));
+ EXPECT_TRUE(data == in_data);
+ in_data.clear();
+ EXPECT_TRUE(utils::ReadFileChunk(file.value().c_str(), 10, 20, &in_data));
+ EXPECT_TRUE(vector<char>(data.begin() + 10, data.begin() + 10 + 20) ==
+ in_data);
+}
+
TEST(UtilsTest, ErrnoNumberAsStringTest) {
EXPECT_EQ("No such file or directory", utils::ErrnoNumberAsString(ENOENT));
}