Parse deflates in gzip files.
The diff for NOTICE.xml.gz is reduced from 200+K to 17K.
Also we are now honoring puffin::LocateDeflatesInGzip() failures,
since if won't complain about padding data anymore, and
puffin::LocateDeflateSubBlocksInZipArchive() is updated to
puffin::LocateDeflatesInZipArchive().
Test: generate a delta payload extracting deflates in gzip
Change-Id: I44b816cff8ac99ce78f758eef5875fa99e36e369
diff --git a/payload_generator/boot_img_filesystem.cc b/payload_generator/boot_img_filesystem.cc
index 90be966..19de410 100644
--- a/payload_generator/boot_img_filesystem.cc
+++ b/payload_generator/boot_img_filesystem.cc
@@ -70,15 +70,10 @@
// Check GZip header magic.
if (data.size() > kGZipHeaderSize && data[0] == 0x1F && data[1] == 0x8B) {
if (!puffin::LocateDeflatesInGzip(data, &file.deflates)) {
- // We still use the deflates found even if LocateDeflatesInGzip() fails,
- // if any deflates are returned, they should be correct, it's possible
- // something went wrong later but it shouldn't stop us from using the
- // previous deflates. Another common case is if there's more data after
- // the gzip, the function will try to parse that as another gzip and
- // will fail, but we still want the deflates from the first gzip.
- LOG(WARNING) << "Error occurred parsing gzip " << name << " at offset "
- << offset << " of " << filename_ << ", found "
- << file.deflates.size() << " deflates.";
+ LOG(ERROR) << "Error occurred parsing gzip " << name << " at offset "
+ << offset << " of " << filename_ << ", found "
+ << file.deflates.size() << " deflates.";
+ return file;
}
for (auto& deflate : file.deflates) {
deflate.offset += offset * 8;
diff --git a/payload_generator/deflate_utils.cc b/payload_generator/deflate_utils.cc
index 2719048..6f47488 100644
--- a/payload_generator/deflate_utils.cc
+++ b/payload_generator/deflate_utils.cc
@@ -102,6 +102,15 @@
((extent.start_block() + extent.num_blocks()) * kBlockSize);
}
+// Returns whether the given file |name| has an extension listed in
+// |extensions|.
+bool IsFileExtensions(const string& name,
+ const std::initializer_list<string>& extensions) {
+ return any_of(extensions.begin(), extensions.end(), [&name](const auto& ext) {
+ return base::EndsWith(name, ext, base::CompareCase::INSENSITIVE_ASCII);
+ });
+}
+
} // namespace
ByteExtent ExpandToByteExtent(const BitExtent& extent) {
@@ -286,35 +295,35 @@
}
}
- // Search for deflates if the file is in zip format.
- // .zvoice files may eventually move out of rootfs. If that happens, remove
- // ".zvoice" (crbug.com/782918).
- const string zip_file_extensions[] = {".apk", ".zip", ".jar", ".zvoice"};
- bool is_zip =
- any_of(zip_file_extensions,
- std::end(zip_file_extensions),
- [&file](const string& ext) {
- return base::EndsWith(
- file.name, ext, base::CompareCase::INSENSITIVE_ASCII);
- });
-
- if (is_zip && extract_deflates) {
- brillo::Blob data;
- TEST_AND_RETURN_FALSE(
- utils::ReadExtents(part.path,
- file.extents,
- &data,
- kBlockSize * utils::BlocksInExtents(file.extents),
- kBlockSize));
- std::vector<puffin::BitExtent> deflates_sub_blocks;
- TEST_AND_RETURN_FALSE(puffin::LocateDeflateSubBlocksInZipArchive(
- data, &deflates_sub_blocks));
- // Shift the deflate's extent to the offset starting from the beginning
- // of the current partition; and the delta processor will align the
- // extents in a continuous buffer later.
- TEST_AND_RETURN_FALSE(
- ShiftBitExtentsOverExtents(file.extents, &deflates_sub_blocks));
- file.deflates = std::move(deflates_sub_blocks);
+ if (extract_deflates) {
+ // Search for deflates if the file is in zip or gzip format.
+ // .zvoice files may eventually move out of rootfs. If that happens,
+ // remove ".zvoice" (crbug.com/782918).
+ bool is_zip =
+ IsFileExtensions(file.name, {".apk", ".zip", ".jar", ".zvoice"});
+ bool is_gzip = IsFileExtensions(file.name, {".gz", ".gzip", ".tgz"});
+ if (is_zip || is_gzip) {
+ brillo::Blob data;
+ TEST_AND_RETURN_FALSE(utils::ReadExtents(
+ part.path,
+ file.extents,
+ &data,
+ kBlockSize * utils::BlocksInExtents(file.extents),
+ kBlockSize));
+ vector<puffin::BitExtent> deflates;
+ if (is_zip) {
+ TEST_AND_RETURN_FALSE(
+ puffin::LocateDeflatesInZipArchive(data, &deflates));
+ } else if (is_gzip) {
+ TEST_AND_RETURN_FALSE(puffin::LocateDeflatesInGzip(data, &deflates));
+ }
+ // Shift the deflate's extent to the offset starting from the beginning
+ // of the current partition; and the delta processor will align the
+ // extents in a continuous buffer later.
+ TEST_AND_RETURN_FALSE(
+ ShiftBitExtentsOverExtents(file.extents, &deflates));
+ file.deflates = std::move(deflates);
+ }
}
result_files->push_back(file);
diff --git a/payload_generator/deflate_utils.h b/payload_generator/deflate_utils.h
index 798ce25..9b40a47 100644
--- a/payload_generator/deflate_utils.h
+++ b/payload_generator/deflate_utils.h
@@ -29,6 +29,7 @@
// Gets the files from the partition and processes all its files. Processing
// includes:
// - splitting large Squashfs containers into its smaller files.
+// - extracting deflates in zip and gzip files.
bool PreprocessParitionFiles(const PartitionConfig& part,
std::vector<FilesystemInterface::File>* result,
bool extract_deflates);