update_engine: Add Squashfs (phase 2)
This patch replaces the Android container related Squashfs images with
their files in the update engine. The location of each new file is
defined as:
'Squashfs image file path' + / + 'the file path inside the Squashfs'
This works even if the files inside the Squashfs are not 4k aligned. But
we need 4k alignment before we add Puffin support. In some experiments
this could even reduce the delta payload size by like 1MB.
BUG=chromium:767120
TEST=cros_workon_make --board=amd64-generic --test update_engine; brillo_update_payload generate --payload=payload.delta --source_image=veyron_minnie-R61-9765.13.0.bin --target_image=veyron-minnie-R62-9804.0.0.bin
Change-Id: I3fbcc5cd0d724435955ed1bd1204dee0e41944be
Reviewed-on: https://chromium-review.googlesource.com/636106
Commit-Ready: Amin Hassani <ahassani@chromium.org>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Ben Chan <benchan@chromium.org>
diff --git a/payload_generator/deflate_utils.cc b/payload_generator/deflate_utils.cc
new file mode 100644
index 0000000..9dd3fa4
--- /dev/null
+++ b/payload_generator/deflate_utils.cc
@@ -0,0 +1,178 @@
+//
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "update_engine/payload_generator/deflate_utils.h"
+
+#include <algorithm>
+#include <string>
+#include <utility>
+
+#include <base/files/file_util.h>
+#include <base/logging.h>
+#include <base/strings/string_util.h>
+
+#include "update_engine/common/utils.h"
+#include "update_engine/payload_generator/delta_diff_generator.h"
+#include "update_engine/payload_generator/extent_ranges.h"
+#include "update_engine/payload_generator/extent_utils.h"
+#include "update_engine/payload_generator/squashfs_filesystem.h"
+#include "update_engine/update_metadata.pb.h"
+
+using std::string;
+using std::vector;
+
+namespace chromeos_update_engine {
+namespace deflate_utils {
+namespace {
+
+// The minimum size for a squashfs image to be processed.
+const uint64_t kMinimumSquashfsImageSize = 1 * 1024 * 1024; // bytes
+
+// TODO(*): Optimize this so we don't have to read all extents into memory in
+// case it is large.
+bool CopyExtentsToFile(const string& in_path,
+ const vector<Extent> extents,
+ const string& out_path,
+ size_t block_size) {
+ brillo::Blob data(BlocksInExtents(extents) * block_size);
+ TEST_AND_RETURN_FALSE(
+ utils::ReadExtents(in_path, extents, &data, data.size(), block_size));
+ TEST_AND_RETURN_FALSE(
+ utils::WriteFile(out_path.c_str(), data.data(), data.size()));
+ return true;
+}
+
+bool IsSquashfsImage(const string& part_path,
+ const FilesystemInterface::File& file) {
+ // Only check for files with img postfix.
+ if (base::EndsWith(file.name, ".img", base::CompareCase::SENSITIVE) &&
+ BlocksInExtents(file.extents) >= kMinimumSquashfsImageSize / kBlockSize) {
+ brillo::Blob super_block;
+ TEST_AND_RETURN_FALSE(
+ utils::ReadFileChunk(part_path,
+ file.extents[0].start_block() * kBlockSize,
+ 100,
+ &super_block));
+ return SquashfsFilesystem::IsSquashfsImage(super_block);
+ }
+ return false;
+}
+
+// Realigns subfiles |files| of a splitted file |file| into its correct
+// positions. This can be used for squashfs, zip, apk, etc.
+bool RealignSplittedFiles(const FilesystemInterface::File& file,
+ vector<FilesystemInterface::File>* files) {
+ // We have to shift all the Extents in |files|, based on the Extents of the
+ // |file| itself.
+ size_t num_blocks = 0;
+ for (auto& in_file : *files) { // We need to modify so no constant.
+ TEST_AND_RETURN_FALSE(
+ ShiftExtentsOverExtents(file.extents, &in_file.extents));
+ in_file.name = file.name + "/" + in_file.name;
+ num_blocks += BlocksInExtents(in_file.extents);
+ }
+
+ // Check that all files in |in_files| cover the entire image.
+ TEST_AND_RETURN_FALSE(BlocksInExtents(file.extents) == num_blocks);
+ return true;
+}
+
+} // namespace
+
+bool ShiftExtentsOverExtents(const vector<Extent>& base_extents,
+ vector<Extent>* over_extents) {
+ if (BlocksInExtents(base_extents) < BlocksInExtents(*over_extents)) {
+ LOG(ERROR) << "over_extents have more blocks than base_extents! Invalid!";
+ return false;
+ }
+ for (size_t idx = 0; idx < over_extents->size(); idx++) {
+ auto over_ext = &over_extents->at(idx);
+ auto gap_blocks = base_extents[0].start_block();
+ auto last_end_block = base_extents[0].start_block();
+ for (auto base_ext : base_extents) { // We need to modify |base_ext|, so we
+ // use copy.
+ gap_blocks += base_ext.start_block() - last_end_block;
+ last_end_block = base_ext.start_block() + base_ext.num_blocks();
+ base_ext.set_start_block(base_ext.start_block() - gap_blocks);
+ if (over_ext->start_block() >= base_ext.start_block() &&
+ over_ext->start_block() <
+ base_ext.start_block() + base_ext.num_blocks()) {
+ if (over_ext->start_block() + over_ext->num_blocks() <=
+ base_ext.start_block() + base_ext.num_blocks()) {
+ // |over_ext| is inside |base_ext|, increase its start block.
+ over_ext->set_start_block(over_ext->start_block() + gap_blocks);
+ } else {
+ // |over_ext| spills over this |base_ext|, split it into two.
+ auto new_blocks = base_ext.start_block() + base_ext.num_blocks() -
+ over_ext->start_block();
+ vector<Extent> new_extents = {
+ ExtentForRange(gap_blocks + over_ext->start_block(), new_blocks),
+ ExtentForRange(over_ext->start_block() + new_blocks,
+ over_ext->num_blocks() - new_blocks)};
+ *over_ext = new_extents[0];
+ over_extents->insert(std::next(over_extents->begin(), idx + 1),
+ new_extents[1]);
+ }
+ break; // We processed |over_ext|, so break the loop;
+ }
+ }
+ }
+ return true;
+}
+
+bool PreprocessParitionFiles(const PartitionConfig& part,
+ vector<FilesystemInterface::File>* result_files) {
+ // Get the file system files.
+ vector<FilesystemInterface::File> tmp_files;
+ part.fs_interface->GetFiles(&tmp_files);
+ result_files->reserve(tmp_files.size());
+
+ for (const auto& file : tmp_files) {
+ if (IsSquashfsImage(part.path, file)) {
+ // Read the image into a file.
+ base::FilePath path;
+ TEST_AND_RETURN_FALSE(base::CreateTemporaryFile(&path));
+ ScopedPathUnlinker old_unlinker(path.value());
+ TEST_AND_RETURN_FALSE(
+ CopyExtentsToFile(part.path, file.extents, path.value(), kBlockSize));
+ // Test if it is actually a Squashfs file.
+ auto sqfs = SquashfsFilesystem::CreateFromFile(path.value());
+ if (sqfs) {
+ // It is an squashfs file. Get its files to replace with itself.
+ vector<FilesystemInterface::File> files;
+ sqfs->GetFiles(&files);
+
+ // Replace squashfs file with its files only if |files| has at
+ // least two files.
+ if (files.size() > 1) {
+ TEST_AND_RETURN_FALSE(RealignSplittedFiles(file, &files));
+ result_files->insert(result_files->end(), files.begin(), files.end());
+ continue;
+ }
+ } else {
+ LOG(WARNING) << "We thought file: " << file.name
+ << " was a Squashfs file, but it was not.";
+ }
+ }
+ // TODO(ahassani): Process other types of files like apk, zip, etc.
+ result_files->push_back(file);
+ }
+
+ return true;
+}
+
+} // namespace deflate_utils
+} // namespace chromeos_update_engine