update_engine: Adds BROTLI_BSDIFF operation
Brotli compression creates on average 10%-20% smaller output than bzip2
in addition to having faster decompressor. With recent changes in bsdiff
to compress the its patch with brotli, we can use it in the
update_engine as a new operation BROTLI_BSDIFF. This operation will be
turned on in minor version 4. However, this CL only adds support for it
in the client. It will not generate BROTLI_BSDIFF operations yet.
BUG=chromium:783437
TEST=unittests pass for both update_engine and update_payload;
'brillo_update_payload {generate|verify}' passes;
'scripts/paycheck.py payload.delta' passes;
Change-Id: Ie791ba5431561c95de6fbc031a8196dbfd912288
Reviewed-on: https://chromium-review.googlesource.com/764791
Commit-Ready: Amin Hassani <ahassani@chromium.org>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Ben Chan <benchan@chromium.org>
diff --git a/scripts/update_payload/checker.py b/scripts/update_payload/checker.py
index c613642..c710c47 100644
--- a/scripts/update_payload/checker.py
+++ b/scripts/update_payload/checker.py
@@ -833,10 +833,12 @@
if op.data_offset:
raise error.PayloadError('%s: contains data_offset.' % op_name)
- def _CheckAnyDiffOperation(self, data_length, total_dst_blocks, op_name):
- """Specific checks for BSDIFF, SOURCE_BSDIFF and PUFFDIFF operations.
+ def _CheckAnyDiffOperation(self, op, data_length, total_dst_blocks, op_name):
+ """Specific checks for BSDIFF, SOURCE_BSDIFF, PUFFDIFF and BROTLI_BSDIFF
+ operations.
Args:
+ op: The operation.
data_length: The length of the data blob associated with the operation.
total_dst_blocks: Total number of blocks in dst_extents.
op_name: Operation name for error reporting.
@@ -856,6 +858,15 @@
(op_name, data_length, total_dst_blocks, self.block_size,
total_dst_blocks * self.block_size))
+ # Check the existence of src_length and dst_length for legacy bsdiffs.
+ if (op.type == common.OpType.BSDIFF or
+ (op.type == common.OpType.SOURCE_BSDIFF and self.minor_version <= 3)):
+ if not op.HasField('src_length') or not op.HasField('dst_length'):
+ raise error.PayloadError('%s: require {src,dst}_length.' % op_name)
+ else:
+ if op.HasField('src_length') or op.HasField('dst_length'):
+ raise error.PayloadError('%s: unneeded {src,dst}_length.' % op_name)
+
def _CheckSourceCopyOperation(self, data_offset, total_src_blocks,
total_dst_blocks, op_name):
"""Specific checks for SOURCE_COPY.
@@ -993,16 +1004,17 @@
elif op.type == common.OpType.ZERO and self.minor_version >= 4:
self._CheckZeroOperation(op, op_name)
elif op.type == common.OpType.BSDIFF and self.minor_version == 1:
- self._CheckAnyDiffOperation(data_length, total_dst_blocks, op_name)
+ self._CheckAnyDiffOperation(op, data_length, total_dst_blocks, op_name)
elif op.type == common.OpType.SOURCE_COPY and self.minor_version >= 2:
self._CheckSourceCopyOperation(data_offset, total_src_blocks,
total_dst_blocks, op_name)
self._CheckAnySourceOperation(op, total_src_blocks, op_name)
elif op.type == common.OpType.SOURCE_BSDIFF and self.minor_version >= 2:
- self._CheckAnyDiffOperation(data_length, total_dst_blocks, op_name)
+ self._CheckAnyDiffOperation(op, data_length, total_dst_blocks, op_name)
self._CheckAnySourceOperation(op, total_src_blocks, op_name)
- elif op.type == common.OpType.PUFFDIFF and self.minor_version >= 4:
- self._CheckAnyDiffOperation(data_length, total_dst_blocks, op_name)
+ elif (op.type in (common.OpType.PUFFDIFF, common.OpType.BROTLI_BSDIFF) and
+ self.minor_version >= 4):
+ self._CheckAnyDiffOperation(op, data_length, total_dst_blocks, op_name)
self._CheckAnySourceOperation(op, total_src_blocks, op_name)
else:
raise error.PayloadError(
@@ -1064,6 +1076,7 @@
common.OpType.SOURCE_COPY: 0,
common.OpType.SOURCE_BSDIFF: 0,
common.OpType.PUFFDIFF: 0,
+ common.OpType.BROTLI_BSDIFF: 0,
}
# Total blob sizes for each operation type.
op_blob_totals = {
@@ -1074,6 +1087,7 @@
# SOURCE_COPY operations don't have blobs.
common.OpType.SOURCE_BSDIFF: 0,
common.OpType.PUFFDIFF: 0,
+ common.OpType.BROTLI_BSDIFF: 0,
}
# Counts of hashed vs unhashed operations.
blob_hash_counts = {