update_engine: Adds BROTLI_BSDIFF operation
Brotli compression creates on average 10%-20% smaller output than bzip2
in addition to having faster decompressor. With recent changes in bsdiff
to compress the its patch with brotli, we can use it in the
update_engine as a new operation BROTLI_BSDIFF. This operation will be
turned on in minor version 4. However, this CL only adds support for it
in the client. It will not generate BROTLI_BSDIFF operations yet.
BUG=chromium:783437
TEST=unittests pass for both update_engine and update_payload;
'brillo_update_payload {generate|verify}' passes;
'scripts/paycheck.py payload.delta' passes;
Change-Id: Ie791ba5431561c95de6fbc031a8196dbfd912288
Reviewed-on: https://chromium-review.googlesource.com/764791
Commit-Ready: Amin Hassani <ahassani@chromium.org>
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Ben Chan <benchan@chromium.org>
diff --git a/scripts/update_payload/checker_unittest.py b/scripts/update_payload/checker_unittest.py
index eecd456..245da55 100755
--- a/scripts/update_payload/checker_unittest.py
+++ b/scripts/update_payload/checker_unittest.py
@@ -39,6 +39,7 @@
'DISCARD': common.OpType.DISCARD,
'REPLACE_XZ': common.OpType.REPLACE_XZ,
'PUFFDIFF': common.OpType.PUFFDIFF,
+ 'BROTLI_BSDIFF': common.OpType.BROTLI_BSDIFF,
}
return op_name_to_type[op_name]
@@ -771,22 +772,23 @@
def testCheckAnyDiff(self):
"""Tests _CheckAnyDiffOperation()."""
payload_checker = checker.PayloadChecker(self.MockPayload())
+ op = update_metadata_pb2.InstallOperation()
# Pass.
self.assertIsNone(
- payload_checker._CheckAnyDiffOperation(10000, 3, 'foo'))
+ payload_checker._CheckAnyDiffOperation(op, 10000, 3, 'foo'))
# Fail, missing data blob.
self.assertRaises(
update_payload.PayloadError,
payload_checker._CheckAnyDiffOperation,
- None, 3, 'foo')
+ op, None, 3, 'foo')
# Fail, too big of a diff blob (unjustified).
self.assertRaises(
update_payload.PayloadError,
payload_checker._CheckAnyDiffOperation,
- 10000, 2, 'foo')
+ op, 10000, 2, 'foo')
def testCheckSourceCopyOperation_Pass(self):
"""Tests _CheckSourceCopyOperation(); pass case."""
@@ -818,7 +820,7 @@
Args:
op_type_name: 'REPLACE', 'REPLACE_BZ', 'MOVE', 'BSDIFF', 'SOURCE_COPY',
- 'SOURCE_BSDIFF' or 'PUFFDIFF'.
+ 'SOURCE_BSDIFF', BROTLI_BSDIFF or 'PUFFDIFF'.
is_last: Whether we're testing the last operation in a sequence.
allow_signature: Whether we're testing a signature-capable operation.
allow_unhashed: Whether we're allowing to not hash the data.
@@ -858,7 +860,7 @@
total_src_blocks = 0
if op_type in (common.OpType.MOVE, common.OpType.BSDIFF,
common.OpType.SOURCE_COPY, common.OpType.SOURCE_BSDIFF,
- common.OpType.PUFFDIFF):
+ common.OpType.PUFFDIFF, common.OpType.BROTLI_BSDIFF):
if fail_src_extents:
self.AddToMessage(op.src_extents,
self.NewExtentList((1, 0)))
@@ -874,7 +876,7 @@
elif op_type in (common.OpType.SOURCE_COPY, common.OpType.SOURCE_BSDIFF):
payload_checker.minor_version = 1 if fail_bad_minor_version else 2
elif op_type in (common.OpType.ZERO, common.OpType.DISCARD,
- common.OpType.PUFFDIFF):
+ common.OpType.PUFFDIFF, common.OpType.BROTLI_BSDIFF):
payload_checker.minor_version = 3 if fail_bad_minor_version else 4
if op_type not in (common.OpType.MOVE, common.OpType.SOURCE_COPY):
@@ -893,6 +895,7 @@
op.data_sha256_hash = hashlib.sha256(fake_data).digest()
payload.ReadDataBlob(op.data_offset, op.data_length).AndReturn(
fake_data)
+
elif fail_data_hash:
# Create an invalid data blob hash.
op.data_sha256_hash = hashlib.sha256(
@@ -913,7 +916,9 @@
if total_src_blocks:
if fail_src_length:
op.src_length = total_src_blocks * block_size + 8
- else:
+ elif (op_type in (common.OpType.MOVE, common.OpType.BSDIFF,
+ common.OpType.SOURCE_BSDIFF) and
+ payload_checker.minor_version <= 3):
op.src_length = total_src_blocks * block_size
elif fail_src_length:
# Add an orphaned src_length.
@@ -922,7 +927,9 @@
if total_dst_blocks:
if fail_dst_length:
op.dst_length = total_dst_blocks * block_size + 8
- else:
+ elif (op_type in (common.OpType.MOVE, common.OpType.BSDIFF,
+ common.OpType.SOURCE_BSDIFF) and
+ payload_checker.minor_version <= 3):
op.dst_length = total_dst_blocks * block_size
self.mox.ReplayAll()
@@ -1279,7 +1286,8 @@
AddParametricTests('CheckOperation',
{'op_type_name': ('REPLACE', 'REPLACE_BZ', 'MOVE',
'BSDIFF', 'SOURCE_COPY',
- 'SOURCE_BSDIFF', 'PUFFDIFF'),
+ 'SOURCE_BSDIFF', 'PUFFDIFF',
+ 'BROTLI_BSDIFF'),
'is_last': (True, False),
'allow_signature': (True, False),
'allow_unhashed': (True, False),