releasetools: Support verifying files with non-monotonic ranges.
Fixes: 79951650
Test: Run validate_target_files on target_files.zip with files in
non-monotonic ranges.
Test: python -m unittest test_validate_target_files
Test: python3 -m unittest test_validate_target_files
Change-Id: I82571d3358598775de4cdeb5e64035689fea6487
diff --git a/tools/releasetools/sparse_img.py b/tools/releasetools/sparse_img.py
index 3367896..a249081 100644
--- a/tools/releasetools/sparse_img.py
+++ b/tools/releasetools/sparse_img.py
@@ -249,8 +249,9 @@
with open(fn) as f:
for line in f:
- fn, ranges = line.split(None, 1)
- ranges = rangelib.RangeSet.parse(ranges)
+ fn, ranges_text = line.rstrip().split(None, 1)
+ ranges = rangelib.RangeSet.parse(ranges_text)
+ ranges.extra['text_str'] = ranges_text
if allow_shared_blocks:
# Find the shared blocks that have been claimed by others. If so, tag
@@ -261,9 +262,6 @@
if not non_shared:
continue
- # There shouldn't anything in the extra dict yet.
- assert not ranges.extra, "Non-empty RangeSet.extra"
-
# Put the non-shared RangeSet as the value in the block map, which
# has a copy of the original RangeSet.
non_shared.extra['uses_shared_blocks'] = ranges
diff --git a/tools/releasetools/test_validate_target_files.py b/tools/releasetools/test_validate_target_files.py
index 9c816eb..6504515 100644
--- a/tools/releasetools/test_validate_target_files.py
+++ b/tools/releasetools/test_validate_target_files.py
@@ -238,14 +238,14 @@
system_root = os.path.join(input_tmp, "SYSTEM")
os.mkdir(system_root)
- # Write the test file that contain multiple blocks of zeros, and these
- # zero blocks will be omitted by kernel. And the test files will occupy one
- # block range each in the final system image.
+ # Write test files that contain multiple blocks of zeros, and these zero
+ # blocks will be omitted by kernel. Each test file will occupy one block in
+ # the final system image.
with open(os.path.join(system_root, 'a'), 'w') as f:
- f.write("aaa")
+ f.write('aaa')
f.write('\0' * 4096 * 3)
with open(os.path.join(system_root, 'b'), 'w') as f:
- f.write("bbb")
+ f.write('bbb')
f.write('\0' * 4096 * 3)
raw_file_map = os.path.join(input_tmp, 'IMAGES', 'raw_system.map')
@@ -254,7 +254,7 @@
# Parse the generated file map and update the block ranges for each file.
file_map_list = {}
image_ranges = RangeSet()
- with open(raw_file_map, 'r') as f:
+ with open(raw_file_map) as f:
for line in f.readlines():
info = line.split()
self.assertEqual(2, len(info))
@@ -265,7 +265,7 @@
mock_shared_block = RangeSet("10-20").subtract(image_ranges).first(1)
with open(os.path.join(input_tmp, 'IMAGES', 'system.map'), 'w') as f:
for key in sorted(file_map_list.keys()):
- line = "{} {}\n".format(
+ line = '{} {}\n'.format(
key, file_map_list[key].union(mock_shared_block))
f.write(line)
@@ -277,9 +277,55 @@
for name in all_entries:
input_zip.write(os.path.join(input_tmp, name), arcname=name)
- input_zip = zipfile.ZipFile(input_file, 'r')
- info_dict = {'extfs_sparse_flag': '-s'}
-
# Expect the validation to pass and both files are skipped due to
# 'incomplete' block range.
- ValidateFileConsistency(input_zip, input_tmp, info_dict)
+ with zipfile.ZipFile(input_file) as input_zip:
+ info_dict = {'extfs_sparse_flag': '-s'}
+ ValidateFileConsistency(input_zip, input_tmp, info_dict)
+
+ @test_utils.SkipIfExternalToolsUnavailable()
+ def test_ValidateFileConsistency_nonMonotonicRanges(self):
+ input_tmp = common.MakeTempDir()
+ os.mkdir(os.path.join(input_tmp, 'IMAGES'))
+ system_image = os.path.join(input_tmp, 'IMAGES', 'system.img')
+ system_root = os.path.join(input_tmp, "SYSTEM")
+ os.mkdir(system_root)
+
+ # Write the test file that contain three blocks of 'a', 'b', 'c'.
+ with open(os.path.join(system_root, 'abc'), 'w') as f:
+ f.write('a' * 4096 + 'b' * 4096 + 'c' * 4096)
+ raw_file_map = os.path.join(input_tmp, 'IMAGES', 'raw_system.map')
+ self._generate_system_image(system_image, system_root, raw_file_map)
+
+ # Parse the generated file map and manipulate the block ranges of 'abc' to
+ # be 'cba'.
+ file_map_list = {}
+ with open(raw_file_map) as f:
+ for line in f.readlines():
+ info = line.split()
+ self.assertEqual(2, len(info))
+ ranges = RangeSet(info[1])
+ self.assertTrue(ranges.monotonic)
+ blocks = reversed(list(ranges.next_item()))
+ file_map_list[info[0]] = ' '.join([str(block) for block in blocks])
+
+ # Update the contents of 'abc' to be 'cba'.
+ with open(os.path.join(system_root, 'abc'), 'w') as f:
+ f.write('c' * 4096 + 'b' * 4096 + 'a' * 4096)
+
+ # Update the system.map.
+ with open(os.path.join(input_tmp, 'IMAGES', 'system.map'), 'w') as f:
+ for key in sorted(file_map_list.keys()):
+ f.write('{} {}\n'.format(key, file_map_list[key]))
+
+ # Get the target zip file.
+ input_file = common.MakeTempFile()
+ all_entries = ['SYSTEM/', 'SYSTEM/abc', 'IMAGES/',
+ 'IMAGES/system.map', 'IMAGES/system.img']
+ with zipfile.ZipFile(input_file, 'w') as input_zip:
+ for name in all_entries:
+ input_zip.write(os.path.join(input_tmp, name), arcname=name)
+
+ with zipfile.ZipFile(input_file) as input_zip:
+ info_dict = {'extfs_sparse_flag': '-s'}
+ ValidateFileConsistency(input_zip, input_tmp, info_dict)
diff --git a/tools/releasetools/validate_target_files.py b/tools/releasetools/validate_target_files.py
index c299a48..d88ccc0 100755
--- a/tools/releasetools/validate_target_files.py
+++ b/tools/releasetools/validate_target_files.py
@@ -36,20 +36,21 @@
import os.path
import re
import zipfile
+from hashlib import sha1
import common
+import rangelib
def _ReadFile(file_name, unpacked_name, round_up=False):
"""Constructs and returns a File object. Rounds up its size if needed."""
-
assert os.path.exists(unpacked_name)
with open(unpacked_name, 'rb') as f:
file_data = f.read()
file_size = len(file_data)
if round_up:
file_size_rounded_up = common.RoundUpTo4K(file_size)
- file_data += '\0' * (file_size_rounded_up - file_size)
+ file_data += b'\0' * (file_size_rounded_up - file_size)
return common.File(file_name, file_data)
@@ -96,13 +97,15 @@
logging.warning('Skipping %s that has incomplete block list', entry)
continue
- # TODO(b/79951650): Handle files with non-monotonic ranges.
+ # If the file has non-monotonic ranges, read each range in order.
if not file_ranges.monotonic:
- logging.warning(
- 'Skipping %s that has non-monotonic ranges: %s', entry, file_ranges)
- continue
-
- blocks_sha1 = image.RangeSha1(file_ranges)
+ h = sha1()
+ for file_range in file_ranges.extra['text_str'].split(' '):
+ for data in image.ReadRangeSet(rangelib.RangeSet(file_range)):
+ h.update(data)
+ blocks_sha1 = h.hexdigest()
+ else:
+ blocks_sha1 = image.RangeSha1(file_ranges)
# The filename under unpacked directory, such as SYSTEM/bin/sh.
unpacked_name = os.path.join(