Merge changes from topic "merge_target_files_new_flags"
* changes:
Adds --output-ota flag to enable building the OTA package.
Adds --output-super-empty flag to enable building super_empty.img.
Adds output-dir and output-item-list for copying only certain files.
diff --git a/tools/releasetools/merge_target_files.py b/tools/releasetools/merge_target_files.py
index 3b72551..77c774a 100755
--- a/tools/releasetools/merge_target_files.py
+++ b/tools/releasetools/merge_target_files.py
@@ -42,7 +42,25 @@
contents of default_other_item_list if provided.
--output-target-files output-target-files-package
- The output merged target files package. Also a zip archive.
+ If provided, the output merged target files package. Also a zip archive.
+
+ --output-dir output-directory
+ If provided, the destination directory for saving merged files. Requires
+ the --output-item-list flag.
+ Can be provided alongside --output-target-files, or by itself.
+
+ --output-item-list output-item-list-file.
+ The optional path to a newline-separated config file that specifies the
+ file patterns to copy into the --output-dir. Required if providing
+ the --output-dir flag.
+
+ --output-ota output-ota-package
+ The output ota package. This is a zip archive. Use of this flag may
+ require passing the --path common flag; see common.py.
+
+ --output-super-empty output-super-empty-image
+ If provided, creates a super_empty.img file from the merged target
+ files package and saves it at this path.
--rebuild_recovery
Rebuild the recovery patch used by non-A/B devices and write it to the
@@ -57,11 +75,14 @@
import fnmatch
import logging
import os
+import shutil
import sys
import zipfile
-import common
import add_img_to_target_files
+import build_super_image
+import common
+import ota_from_target_files
logger = logging.getLogger(__name__)
OPTIONS = common.OPTIONS
@@ -72,6 +93,10 @@
OPTIONS.other_target_files = None
OPTIONS.other_item_list = None
OPTIONS.output_target_files = None
+OPTIONS.output_dir = None
+OPTIONS.output_item_list = None
+OPTIONS.output_ota = None
+OPTIONS.output_super_empty = None
OPTIONS.rebuild_recovery = False
OPTIONS.keep_tmp = False
@@ -195,6 +220,29 @@
filtered_extract_item_list)
+def copy_items(from_dir, to_dir, patterns):
+ """Similar to extract_items() except uses an input dir instead of zip."""
+ file_paths = []
+ for dirpath, _, filenames in os.walk(from_dir):
+ file_paths.extend(os.path.relpath(path=os.path.join(dirpath, filename),
+ start=from_dir) for filename in filenames)
+
+ filtered_file_paths = set()
+ for pattern in patterns:
+ filtered_file_paths.update(fnmatch.filter(file_paths, pattern))
+
+ for file_path in filtered_file_paths:
+ original_file_path = os.path.join(from_dir, file_path)
+ copied_file_path = os.path.join(to_dir, file_path)
+ copied_file_dir = os.path.dirname(copied_file_path)
+ if not os.path.exists(copied_file_dir):
+ os.makedirs(copied_file_dir)
+ if os.path.islink(original_file_path):
+ os.symlink(os.readlink(original_file_path), copied_file_path)
+ else:
+ shutil.copyfile(original_file_path, copied_file_path)
+
+
def read_config_list(config_file_path):
"""Reads a config file into a list of strings.
@@ -388,7 +436,7 @@
# a shared system image.
for partition_group in merged_info_dict['super_partition_groups'].split(' '):
if ('super_%s_group_size' % partition_group) not in merged_info_dict:
- raise common.ExternalError(
+ raise ValueError(
'Other META/misc_info.txt does not contain required key '
'super_%s_group_size.' % partition_group)
key = 'super_%s_partition_list' % partition_group
@@ -546,6 +594,10 @@
other_target_files,
other_item_list,
output_target_files,
+ output_dir,
+ output_item_list,
+ output_ota,
+ output_super_empty,
rebuild_recovery):
"""Merge two target files packages together.
@@ -580,6 +632,11 @@
output_target_files: The name of the output zip archive target files
package created by merging system and other.
+ output_ota: The name of the output zip archive ota package.
+
+ output_super_empty: If provided, creates a super_empty.img file from the
+ merged target files package and saves it at this path.
+
rebuild_recovery: If true, rebuild the recovery patch used by non-A/B
devices and write it to the system image.
"""
@@ -646,6 +703,27 @@
system_misc_info_keys=system_misc_info_keys,
rebuild_recovery=rebuild_recovery)
+ # Create super_empty.img using the merged misc_info.txt.
+
+ if output_super_empty:
+ misc_info_txt = os.path.join(output_target_files_temp_dir,
+ 'META', 'misc_info.txt')
+ def read_helper():
+ with open(misc_info_txt) as f:
+ return list(f.read().splitlines())
+
+ misc_info_dict = common.LoadDictionaryFromLines(read_helper())
+ if misc_info_dict.get('use_dynamic_partitions') != 'true':
+ raise ValueError(
+ 'Building super_empty.img requires use_dynamic_partitions=true.')
+
+ build_super_image_args = [
+ '--verbose',
+ misc_info_txt,
+ output_super_empty,
+ ]
+ build_super_image.main(build_super_image_args)
+
# Regenerate IMAGES in the temporary directory.
add_img_args = ['--verbose']
@@ -655,7 +733,14 @@
add_img_to_target_files.main(add_img_args)
- # Finally, create the output target files zip archive.
+ # Finally, create the output target files zip archive and/or copy the
+ # output items to the output target files directory.
+
+ if output_dir:
+ copy_items(output_target_files_temp_dir, output_dir, output_item_list)
+
+ if not output_target_files:
+ return
output_zip = os.path.abspath(output_target_files)
output_target_files_list = os.path.join(temp_dir, 'output.list')
@@ -694,6 +779,15 @@
logger.info('creating %s', output_target_files)
common.RunAndWait(command, verbose=True)
+ # Create the OTA package from the merged target files package.
+
+ if output_ota:
+ ota_from_target_files_args = [
+ output_zip,
+ output_ota,
+ ]
+ ota_from_target_files.main(ota_from_target_files_args)
+
def call_func_with_temp_dir(func, keep_tmp):
"""Manage the creation and cleanup of the temporary directory.
@@ -747,6 +841,14 @@
OPTIONS.other_item_list = a
elif o == '--output-target-files':
OPTIONS.output_target_files = a
+ elif o == '--output-dir':
+ OPTIONS.output_dir = a
+ elif o == '--output-item-list':
+ OPTIONS.output_item_list = a
+ elif o == '--output-ota':
+ OPTIONS.output_ota = a
+ elif o == '--output-super-empty':
+ OPTIONS.output_super_empty = a
elif o == '--rebuild_recovery':
OPTIONS.rebuild_recovery = True
elif o == '--keep-tmp':
@@ -764,6 +866,10 @@
'other-target-files=',
'other-item-list=',
'output-target-files=',
+ 'output-dir=',
+ 'output-item-list=',
+ 'output-ota=',
+ 'output-super-empty=',
'rebuild_recovery',
'keep-tmp',
],
@@ -771,8 +877,11 @@
if (len(args) != 0 or
OPTIONS.system_target_files is None or
- OPTIONS.other_target_files is None or
- OPTIONS.output_target_files is None):
+ OPTIONS.other_target_files is None or (
+ OPTIONS.output_target_files is None and
+ OPTIONS.output_dir is None) or (
+ OPTIONS.output_dir is not None and
+ OPTIONS.output_item_list is None)):
common.Usage(__doc__)
sys.exit(1)
@@ -791,6 +900,11 @@
else:
other_item_list = default_other_item_list
+ if OPTIONS.output_item_list:
+ output_item_list = read_config_list(OPTIONS.output_item_list)
+ else:
+ output_item_list = None
+
if not validate_config_lists(
system_item_list=system_item_list,
system_misc_info_keys=system_misc_info_keys,
@@ -806,6 +920,10 @@
other_target_files=OPTIONS.other_target_files,
other_item_list=other_item_list,
output_target_files=OPTIONS.output_target_files,
+ output_dir=OPTIONS.output_dir,
+ output_item_list=output_item_list,
+ output_ota=OPTIONS.output_ota,
+ output_super_empty=OPTIONS.output_super_empty,
rebuild_recovery=OPTIONS.rebuild_recovery),
OPTIONS.keep_tmp)
diff --git a/tools/releasetools/test_merge_target_files.py b/tools/releasetools/test_merge_target_files.py
index ec452a8..7e18a34 100644
--- a/tools/releasetools/test_merge_target_files.py
+++ b/tools/releasetools/test_merge_target_files.py
@@ -16,10 +16,12 @@
import os.path
+import common
import test_utils
-from merge_target_files import (
- read_config_list, validate_config_lists, default_system_item_list,
- default_other_item_list, default_system_misc_info_keys)
+from merge_target_files import (read_config_list, validate_config_lists,
+ default_system_item_list,
+ default_other_item_list,
+ default_system_misc_info_keys, copy_items)
class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase):
@@ -27,6 +29,58 @@
def setUp(self):
self.testdata_dir = test_utils.get_testdata_dir()
+ def test_copy_items_CopiesItemsMatchingPatterns(self):
+
+ def createEmptyFile(path):
+ if not os.path.exists(os.path.dirname(path)):
+ os.makedirs(os.path.dirname(path))
+ open(path, 'a').close()
+ return path
+
+ def createSymLink(source, dest):
+ os.symlink(source, dest)
+ return dest
+
+ def getRelPaths(start, filepaths):
+ return set(
+ os.path.relpath(path=filepath, start=start) for filepath in filepaths)
+
+ input_dir = common.MakeTempDir()
+ output_dir = common.MakeTempDir()
+ expected_copied_items = []
+ actual_copied_items = []
+ patterns = ['*.cpp', 'subdir/*.txt']
+
+ # Create various files that we expect to get copied because they
+ # match one of the patterns.
+ expected_copied_items.extend([
+ createEmptyFile(os.path.join(input_dir, 'a.cpp')),
+ createEmptyFile(os.path.join(input_dir, 'b.cpp')),
+ createEmptyFile(os.path.join(input_dir, 'subdir', 'c.txt')),
+ createEmptyFile(os.path.join(input_dir, 'subdir', 'd.txt')),
+ createEmptyFile(
+ os.path.join(input_dir, 'subdir', 'subsubdir', 'e.txt')),
+ createSymLink('a.cpp', os.path.join(input_dir, 'a_link.cpp')),
+ ])
+ # Create some more files that we expect to not get copied.
+ createEmptyFile(os.path.join(input_dir, 'a.h'))
+ createEmptyFile(os.path.join(input_dir, 'b.h'))
+ createEmptyFile(os.path.join(input_dir, 'subdir', 'subsubdir', 'f.gif'))
+ createSymLink('a.h', os.path.join(input_dir, 'a_link.h'))
+
+ # Copy items.
+ copy_items(input_dir, output_dir, patterns)
+
+ # Assert the actual copied items match the ones we expected.
+ for dirpath, _, filenames in os.walk(output_dir):
+ actual_copied_items.extend(
+ os.path.join(dirpath, filename) for filename in filenames)
+ self.assertEqual(
+ getRelPaths(output_dir, actual_copied_items),
+ getRelPaths(input_dir, expected_copied_items))
+ self.assertEqual(
+ os.readlink(os.path.join(output_dir, 'a_link.cpp')), 'a.cpp')
+
def test_read_config_list(self):
system_item_list_file = os.path.join(self.testdata_dir,
'merge_config_system_item_list')