Add erofs compression param
Test: th
Bug: 206729162
Change-Id: I774918693262c8e48a2656d82d5acaeb4e3aaed2
diff --git a/payload_generator/delta_diff_utils.h b/payload_generator/delta_diff_utils.h
index 1fd1f46..dcb6867 100644
--- a/payload_generator/delta_diff_utils.h
+++ b/payload_generator/delta_diff_utils.h
@@ -227,8 +227,8 @@
bool TryZucchiniAndUpdateOperation(AnnotatedOperation* aop,
brillo::Blob* data_blob);
- brillo::Blob old_data_;
- brillo::Blob new_data_;
+ const brillo::Blob& old_data_;
+ const brillo::Blob& new_data_;
const std::vector<Extent>& src_extents_;
const std::vector<Extent>& dst_extents_;
std::vector<puffin::BitExtent> old_deflates_;
diff --git a/payload_generator/erofs_filesystem.cc b/payload_generator/erofs_filesystem.cc
index 9ab37fd..677b473 100644
--- a/payload_generator/erofs_filesystem.cc
+++ b/payload_generator/erofs_filesystem.cc
@@ -27,11 +27,13 @@
#include "erofs_iterate.h"
#include "lz4diff/lz4diff.pb.h"
-#include "payload_generator/filesystem_interface.h"
+#include "lz4diff/lz4patch.h"
+#include "lz4diff/lz4diff.h"
#include "update_engine/common/utils.h"
#include "update_engine/payload_generator/delta_diff_generator.h"
#include "update_engine/payload_generator/extent_ranges.h"
#include "update_engine/payload_generator/extent_utils.h"
+#include "update_engine/payload_generator/filesystem_interface.h"
namespace chromeos_update_engine {
@@ -81,10 +83,6 @@
if (!file.is_compressed) {
return;
}
- // TODO(b/206729162) Fill in compression algorithm info from input target
- // files
- file.compressed_file_info.algo.set_type(CompressionAlgorithm::LZ4HC);
- file.compressed_file_info.algo.set_level(9);
struct erofs_map_blocks block {};
block.m_la = 0;
@@ -142,7 +140,7 @@
static_assert(kBlockSize == EROFS_BLKSIZ);
std::unique_ptr<ErofsFilesystem> ErofsFilesystem::CreateFromFile(
- const std::string& filename) {
+ const std::string& filename, const CompressionAlgorithm& algo) {
// erofs-utils makes heavy use of global variables. Hence its functions aren't
// thread safe. For example, it stores a global int holding file descriptors
// to the opened EROFS image. It doesn't even support opening more than 1
@@ -171,9 +169,10 @@
LOG(INFO) << "Parsed EROFS image of size " << st.st_size << " built in "
<< ctime(&time) << " " << filename;
std::vector<File> files;
- if (!ErofsFilesystem::GetFiles(filename, &files)) {
+ if (!ErofsFilesystem::GetFiles(filename, &files, algo)) {
return nullptr;
}
+ LOG(INFO) << "Using compression algo " << algo << " for " << filename;
// private ctor doesn't work with make_unique
return std::unique_ptr<ErofsFilesystem>(
new ErofsFilesystem(filename, st.st_size, std::move(files)));
@@ -185,7 +184,8 @@
}
bool ErofsFilesystem::GetFiles(const std::string& filename,
- std::vector<File>* files) {
+ std::vector<File>* files,
+ const CompressionAlgorithm& algo) {
erofs_iterate_root_dir(&sbi, [&](struct erofs_iterate_dir_context* p_info) {
const auto& info = *p_info;
if (info.ctx.de_ftype != EROFS_FT_REG_FILE) {
@@ -226,6 +226,7 @@
file.file_stat.st_size = uncompressed_size;
file.file_stat.st_ino = inode.nid;
FillCompressedBlockInfo(&file, filename, &inode);
+ file.compressed_file_info.algo = algo;
files->emplace_back(std::move(file));
return 0;
diff --git a/payload_generator/erofs_filesystem.h b/payload_generator/erofs_filesystem.h
index 473c609..0863b50 100644
--- a/payload_generator/erofs_filesystem.h
+++ b/payload_generator/erofs_filesystem.h
@@ -24,10 +24,15 @@
class ErofsFilesystem final : public FilesystemInterface {
public:
- // Creates an Ext2Filesystem from a ext2 formatted filesystem stored in a
- // file. The file doesn't need to be loop-back mounted.
+ // Creates an ErofsFilesystem from a erofs formatted filesystem stored in a
+ // file. The file doesn't need to be loop-back mounted. Since erofs-utils
+ // library functions are not concurrency safe(can't be used in multi-threaded
+ // context, can't even work with multiple EROFS images concurrently on 1
+ // thread), this function takes a global mutex.
static std::unique_ptr<ErofsFilesystem> CreateFromFile(
- const std::string& filename);
+ const std::string& filename,
+ const CompressionAlgorithm& algo =
+ PartitionConfig::GetDefaultCompressionParam());
virtual ~ErofsFilesystem() = default;
// FilesystemInterface overrides.
@@ -45,7 +50,9 @@
// space.
// <metadata>: With the rest of ext2 metadata blocks, such as superblocks
// and bitmap tables.
- static bool GetFiles(const std::string& filename, std::vector<File>* files);
+ static bool GetFiles(const std::string& filename,
+ std::vector<File>* files,
+ const CompressionAlgorithm& algo);
bool GetFiles(std::vector<File>* files) const override;
diff --git a/payload_generator/generate_delta_main.cc b/payload_generator/generate_delta_main.cc
index 09fb837..ef36a6d 100644
--- a/payload_generator/generate_delta_main.cc
+++ b/payload_generator/generate_delta_main.cc
@@ -444,6 +444,11 @@
true,
"Whether to enable zucchini feature when processing executable files.");
+ DEFINE_string(erofs_compression_param,
+ "",
+ "Compression parameter passed to mkfs.erofs's -z option. "
+ "Example: lz4 lz4hc,9");
+
brillo::FlagHelper::Init(
argc,
argv,
@@ -594,6 +599,10 @@
payload_config.target.partitions.back().path = new_partitions[i];
payload_config.target.partitions.back().disable_fec_computation =
FLAGS_disable_fec_computation;
+ if (!FLAGS_erofs_compression_param.empty()) {
+ payload_config.target.partitions.back().erofs_compression_param =
+ PartitionConfig::ParseCompressionParam(FLAGS_erofs_compression_param);
+ }
if (i < new_mapfiles.size())
payload_config.target.partitions.back().mapfile_path = new_mapfiles[i];
}
diff --git a/payload_generator/payload_generation_config.cc b/payload_generator/payload_generation_config.cc
index 8ff4999..7971f28 100644
--- a/payload_generator/payload_generation_config.cc
+++ b/payload_generator/payload_generation_config.cc
@@ -17,6 +17,7 @@
#include "update_engine/payload_generator/payload_generation_config.h"
#include <algorithm>
+#include <charconv>
#include <map>
#include <utility>
@@ -76,7 +77,7 @@
return true;
}
}
- fs_interface = ErofsFilesystem::CreateFromFile(path);
+ fs_interface = ErofsFilesystem::CreateFromFile(path, erofs_compression_param);
if (fs_interface) {
TEST_AND_RETURN_FALSE(fs_interface->GetBlockSize() == kBlockSize);
return true;
@@ -373,4 +374,35 @@
}
}
+CompressionAlgorithm PartitionConfig::ParseCompressionParam(
+ std::string_view param) {
+ CompressionAlgorithm algo;
+ auto algo_name = param;
+ const auto pos = param.find_first_of(',');
+ if (pos != std::string::npos) {
+ algo_name = param.substr(0, pos);
+ }
+ if (algo_name == "lz4") {
+ algo.set_type(CompressionAlgorithm::LZ4);
+ CHECK_EQ(pos, std::string::npos)
+ << "Invalid compression param " << param
+ << ", compression level not supported for lz4";
+ } else if (algo_name == "lz4hc") {
+ algo.set_type(CompressionAlgorithm::LZ4HC);
+ if (pos != std::string::npos) {
+ const auto level = param.substr(pos + 1);
+ int level_num = 0;
+ const auto [ptr, ec] =
+ std::from_chars(level.data(), level.data() + level.size(), level_num);
+ CHECK_EQ(ec, std::errc()) << "Failed to parse compression level " << level
+ << ", compression param: " << param;
+ algo.set_level(level_num);
+ } else {
+ LOG(FATAL) << "Unrecognized compression type: " << algo_name
+ << ", param: " << param;
+ }
+ }
+ return algo;
+}
+
} // namespace chromeos_update_engine
diff --git a/payload_generator/payload_generation_config.h b/payload_generator/payload_generation_config.h
index d71649b..a7ddee4 100644
--- a/payload_generator/payload_generation_config.h
+++ b/payload_generator/payload_generation_config.h
@@ -25,6 +25,7 @@
#include <brillo/key_value_store.h>
#include <brillo/secure_blob.h>
+#include <lz4diff/lz4diff.pb.h>
#include "bsdiff/constants.h"
#include "update_engine/payload_consumer/payload_constants.h"
@@ -83,6 +84,13 @@
struct PartitionConfig {
explicit PartitionConfig(std::string name) : name(name) {}
+ static CompressionAlgorithm ParseCompressionParam(std::string_view param);
+ static CompressionAlgorithm GetDefaultCompressionParam() {
+ CompressionAlgorithm algo;
+ algo.set_type(CompressionAlgorithm::LZ4HC);
+ algo.set_level(9);
+ return algo;
+ }
// Returns whether the PartitionConfig is not an empty image and all the
// fields are set correctly to a valid image file.
@@ -123,6 +131,12 @@
// Per-partition version, usually a number representing timestamp.
std::string version;
+
+ // parameter passed to mkfs.erofs's -z option.
+ // In the format of "compressor,compression_level"
+ // Examples: lz4 lz4hc,9
+ // The default is usually lz4hc,9 for mkfs.erofs
+ CompressionAlgorithm erofs_compression_param = GetDefaultCompressionParam();
};
// The ImageConfig struct describes a pair of binaries kernel and rootfs and the