Split the huge merge_target_files script into multiple files.
Bug: 221858722
Test: m otatools; Use to create merged builds
Test: atest --host releasetools_test
Change-Id: I5f932f160d3f6405b41a7721b1c75cc96749e77b
diff --git a/tools/releasetools/merge/merge_target_files.py b/tools/releasetools/merge/merge_target_files.py
index 25ed60e..67dd6f1 100755
--- a/tools/releasetools/merge/merge_target_files.py
+++ b/tools/releasetools/merge/merge_target_files.py
@@ -102,29 +102,22 @@
If provided, the location of vendor's dexpreopt_config.zip.
"""
-import fnmatch
-import glob
-import json
import logging
import os
-import re
import shutil
import subprocess
import sys
-import zipfile
-from xml.etree import ElementTree
import add_img_to_target_files
-import apex_utils
import build_image
import build_super_image
-import check_target_files_vintf
import common
import img_from_target_files
-import find_shareduid_violation
+import merge_compatibility_checks
+import merge_dexopt
+import merge_meta
+import merge_utils
import ota_from_target_files
-import sparse_img
-import verity_utils
from common import ExternalError
@@ -154,28 +147,6 @@
OPTIONS.framework_dexpreopt_tools = None
OPTIONS.vendor_dexpreopt_config = None
-# In an item list (framework or vendor), we may see entries that select whole
-# partitions. Such an entry might look like this 'SYSTEM/*' (e.g., for the
-# system partition). The following regex matches this and extracts the
-# partition name.
-
-PARTITION_ITEM_PATTERN = re.compile(r'^([A-Z_]+)/\*$')
-
-# In apexkeys.txt or apkcerts.txt, we will find partition tags on each entry in
-# the file. We use these partition tags to filter the entries in those files
-# from the two different target files packages to produce a merged apexkeys.txt
-# or apkcerts.txt file. A partition tag (e.g., for the product partition) looks
-# like this: 'partition="product"'. We use the group syntax grab the value of
-# the tag. We use non-greedy matching in case there are other fields on the
-# same line.
-
-PARTITION_TAG_PATTERN = re.compile(r'partition="(.*?)"')
-
-# The sorting algorithm for apexkeys.txt and apkcerts.txt does not include the
-# ".apex" or ".apk" suffix, so we use the following pattern to extract a key.
-
-MODULE_KEY_PATTERN = re.compile(r'name="(.+)\.(apex|apk)"')
-
# DEFAULT_FRAMEWORK_ITEM_LIST is a list of items to extract from the partial
# framework target files package as is, meaning these items will land in the
# output target files package exactly as they appear in the input partial
@@ -234,801 +205,6 @@
'VENDOR/*',
)
-# The merge config lists should not attempt to extract items from both
-# builds for any of the following partitions. The partitions in
-# SINGLE_BUILD_PARTITIONS should come entirely from a single build (either
-# framework or vendor, but not both).
-
-SINGLE_BUILD_PARTITIONS = (
- 'BOOT/',
- 'DATA/',
- 'ODM/',
- 'PRODUCT/',
- 'SYSTEM_EXT/',
- 'RADIO/',
- 'RECOVERY/',
- 'ROOT/',
- 'SYSTEM/',
- 'SYSTEM_OTHER/',
- 'VENDOR/',
- 'VENDOR_DLKM/',
- 'ODM_DLKM/',
- 'SYSTEM_DLKM/',
-)
-
-
-def write_sorted_data(data, path):
- """Writes the sorted contents of either a list or dict to file.
-
- This function sorts the contents of the list or dict and then writes the
- resulting sorted contents to a file specified by path.
-
- Args:
- data: The list or dict to sort and write.
- path: Path to the file to write the sorted values to. The file at path will
- be overridden if it exists.
- """
- with open(path, 'w') as output:
- for entry in sorted(data):
- out_str = '{}={}\n'.format(entry, data[entry]) if isinstance(
- data, dict) else '{}\n'.format(entry)
- output.write(out_str)
-
-
-def extract_items(input_zip, output_dir, extract_item_list):
- """Extracts items in extra_item_list from a zip to a dir."""
-
- logger.info('extracting from %s', input_zip)
-
- # Filter the extract_item_list to remove any items that do not exist in the
- # zip file. Otherwise, the extraction step will fail.
-
- with zipfile.ZipFile(input_zip, allowZip64=True) as input_zipfile:
- input_namelist = input_zipfile.namelist()
-
- filtered_extract_item_list = []
- for pattern in extract_item_list:
- matching_namelist = fnmatch.filter(input_namelist, pattern)
- if not matching_namelist:
- logger.warning('no match for %s', pattern)
- else:
- filtered_extract_item_list.append(pattern)
-
- common.UnzipToDir(input_zip, output_dir, filtered_extract_item_list)
-
-
-def copy_items(from_dir, to_dir, patterns):
- """Similar to extract_items() except uses an input dir instead of zip."""
- file_paths = []
- for dirpath, _, filenames in os.walk(from_dir):
- file_paths.extend(
- os.path.relpath(path=os.path.join(dirpath, filename), start=from_dir)
- for filename in filenames)
-
- filtered_file_paths = set()
- for pattern in patterns:
- filtered_file_paths.update(fnmatch.filter(file_paths, pattern))
-
- for file_path in filtered_file_paths:
- original_file_path = os.path.join(from_dir, file_path)
- copied_file_path = os.path.join(to_dir, file_path)
- copied_file_dir = os.path.dirname(copied_file_path)
- if not os.path.exists(copied_file_dir):
- os.makedirs(copied_file_dir)
- if os.path.islink(original_file_path):
- os.symlink(os.readlink(original_file_path), copied_file_path)
- else:
- shutil.copyfile(original_file_path, copied_file_path)
-
-
-def validate_config_lists():
- """Performs validations on the merge config lists.
-
- Returns:
- False if a validation fails, otherwise true.
- """
- has_error = False
-
- default_combined_item_set = set(DEFAULT_FRAMEWORK_ITEM_LIST)
- default_combined_item_set.update(DEFAULT_VENDOR_ITEM_LIST)
-
- combined_item_set = set(OPTIONS.framework_item_list)
- combined_item_set.update(OPTIONS.vendor_item_list)
-
- # Check that the merge config lists are not missing any item specified
- # by the default config lists.
- difference = default_combined_item_set.difference(combined_item_set)
- if difference:
- logger.error('Missing merge config items: %s', list(difference))
- logger.error('Please ensure missing items are in either the '
- 'framework-item-list or vendor-item-list files provided to '
- 'this script.')
- has_error = True
-
- # Check that partitions only come from one input.
- for partition in SINGLE_BUILD_PARTITIONS:
- image_path = 'IMAGES/{}.img'.format(partition.lower().replace('/', ''))
- in_framework = (
- any(item.startswith(partition) for item in OPTIONS.framework_item_list)
- or image_path in OPTIONS.framework_item_list)
- in_vendor = (
- any(item.startswith(partition) for item in OPTIONS.vendor_item_list) or
- image_path in OPTIONS.vendor_item_list)
- if in_framework and in_vendor:
- logger.error(
- 'Cannot extract items from %s for both the framework and vendor'
- ' builds. Please ensure only one merge config item list'
- ' includes %s.', partition, partition)
- has_error = True
-
- if ('dynamic_partition_list' in OPTIONS.framework_misc_info_keys) or (
- 'super_partition_groups' in OPTIONS.framework_misc_info_keys):
- logger.error('Dynamic partition misc info keys should come from '
- 'the vendor instance of META/misc_info.txt.')
- has_error = True
-
- return not has_error
-
-
-def merge_ab_partitions_txt(framework_meta_dir, vendor_meta_dir,
- merged_meta_dir):
- """Merges META/ab_partitions.txt.
-
- The output contains the union of the partition names.
- """
- with open(os.path.join(framework_meta_dir, 'ab_partitions.txt')) as f:
- framework_ab_partitions = f.read().splitlines()
-
- with open(os.path.join(vendor_meta_dir, 'ab_partitions.txt')) as f:
- vendor_ab_partitions = f.read().splitlines()
-
- write_sorted_data(
- data=set(framework_ab_partitions + vendor_ab_partitions),
- path=os.path.join(merged_meta_dir, 'ab_partitions.txt'))
-
-
-def merge_misc_info_txt(framework_meta_dir, vendor_meta_dir, merged_meta_dir):
- """Merges META/misc_info.txt.
-
- The output contains a combination of key=value pairs from both inputs.
- Most pairs are taken from the vendor input, while some are taken from
- the framework input.
- """
-
- OPTIONS.framework_misc_info = common.LoadDictionaryFromFile(
- os.path.join(framework_meta_dir, 'misc_info.txt'))
- OPTIONS.vendor_misc_info = common.LoadDictionaryFromFile(
- os.path.join(vendor_meta_dir, 'misc_info.txt'))
-
- # Merged misc info is a combination of vendor misc info plus certain values
- # from the framework misc info.
-
- merged_dict = OPTIONS.vendor_misc_info
- for key in OPTIONS.framework_misc_info_keys:
- merged_dict[key] = OPTIONS.framework_misc_info[key]
-
- # If AVB is enabled then ensure that we build vbmeta.img.
- # Partial builds with AVB enabled may set PRODUCT_BUILD_VBMETA_IMAGE=false to
- # skip building an incomplete vbmeta.img.
- if merged_dict.get('avb_enable') == 'true':
- merged_dict['avb_building_vbmeta_image'] = 'true'
-
- return merged_dict
-
-
-def merge_dynamic_partitions_info_txt(framework_meta_dir, vendor_meta_dir,
- merged_meta_dir):
- """Merge META/dynamic_partitions_info.txt."""
- framework_dynamic_partitions_dict = common.LoadDictionaryFromFile(
- os.path.join(framework_meta_dir, 'dynamic_partitions_info.txt'))
- vendor_dynamic_partitions_dict = common.LoadDictionaryFromFile(
- os.path.join(vendor_meta_dir, 'dynamic_partitions_info.txt'))
-
- merged_dynamic_partitions_dict = common.MergeDynamicPartitionInfoDicts(
- framework_dict=framework_dynamic_partitions_dict,
- vendor_dict=vendor_dynamic_partitions_dict)
-
- write_sorted_data(
- data=merged_dynamic_partitions_dict,
- path=os.path.join(merged_meta_dir, 'dynamic_partitions_info.txt'))
-
- # Merge misc info keys used for Dynamic Partitions.
- OPTIONS.merged_misc_info.update(merged_dynamic_partitions_dict)
- # Ensure that add_img_to_target_files rebuilds super split images for
- # devices that retrofit dynamic partitions. This flag may have been set to
- # false in the partial builds to prevent duplicate building of super.img.
- OPTIONS.merged_misc_info['build_super_partition'] = 'true'
-
-
-def item_list_to_partition_set(item_list):
- """Converts a target files item list to a partition set.
-
- The item list contains items that might look like 'SYSTEM/*' or 'VENDOR/*' or
- 'OTA/android-info.txt'. Items that end in '/*' are assumed to match entire
- directories where 'SYSTEM' or 'VENDOR' is a directory name that identifies the
- contents of a partition of the same name. Other items in the list, such as the
- 'OTA' example contain metadata. This function iterates such a list, returning
- a set that contains the partition entries.
-
- Args:
- item_list: A list of items in a target files package.
-
- Returns:
- A set of partitions extracted from the list of items.
- """
-
- partition_set = set()
-
- for item in item_list:
- match = PARTITION_ITEM_PATTERN.search(item.strip())
- partition_tag = match.group(1).lower() if match else None
-
- if partition_tag:
- partition_set.add(partition_tag)
-
- return partition_set
-
-
-def merge_package_keys_txt(framework_meta_dir, vendor_meta_dir, merged_meta_dir,
- file_name):
- """Merges APK/APEX key list files."""
-
- if file_name not in ('apkcerts.txt', 'apexkeys.txt'):
- raise ExternalError(
- 'Unexpected file_name provided to merge_package_keys_txt: %s',
- file_name)
-
- def read_helper(d):
- temp = {}
- with open(os.path.join(d, file_name)) as f:
- for line in f.read().splitlines():
- line = line.strip()
- if line:
- name_search = MODULE_KEY_PATTERN.search(line.split()[0])
- temp[name_search.group(1)] = line
- return temp
-
- framework_dict = read_helper(framework_meta_dir)
- vendor_dict = read_helper(vendor_meta_dir)
- merged_dict = {}
-
- def filter_into_merged_dict(item_dict, partition_set):
- for key, value in item_dict.items():
- tag_search = PARTITION_TAG_PATTERN.search(value)
-
- if tag_search is None:
- raise ValueError('Entry missing partition tag: %s' % value)
-
- partition_tag = tag_search.group(1)
-
- if partition_tag in partition_set:
- if key in merged_dict:
- if OPTIONS.allow_duplicate_apkapex_keys:
- # TODO(b/150582573) Always raise on duplicates.
- logger.warning('Duplicate key %s' % key)
- continue
- else:
- raise ValueError('Duplicate key %s' % key)
-
- merged_dict[key] = value
-
- # Prioritize framework keys first.
- # Duplicate keys from vendor are an error, or ignored.
- filter_into_merged_dict(framework_dict, OPTIONS.framework_partition_set)
- filter_into_merged_dict(vendor_dict, OPTIONS.vendor_partition_set)
-
- # The following code is similar to write_sorted_data, but different enough
- # that we couldn't use that function. We need the output to be sorted by the
- # basename of the apex/apk (without the ".apex" or ".apk" suffix). This
- # allows the sort to be consistent with the framework/vendor input data and
- # eases comparison of input data with merged data.
- with open(os.path.join(merged_meta_dir, file_name), 'w') as output:
- for key, value in sorted(merged_dict.items()):
- output.write(value + '\n')
-
-
-def create_file_contexts_copies(framework_meta_dir, vendor_meta_dir,
- merged_meta_dir):
- """Creates named copies of each partial build's file_contexts.bin.
-
- Used when regenerating images from the partial build.
- """
-
- def copy_fc_file(source_dir, file_name):
- for name in (file_name, 'file_contexts.bin'):
- fc_path = os.path.join(source_dir, name)
- if os.path.exists(fc_path):
- shutil.copyfile(fc_path, os.path.join(merged_meta_dir, file_name))
- return
- raise ValueError('Missing file_contexts file from %s: %s', source_dir,
- file_name)
-
- copy_fc_file(framework_meta_dir, 'framework_file_contexts.bin')
- copy_fc_file(vendor_meta_dir, 'vendor_file_contexts.bin')
-
- # Replace <image>_selinux_fc values with framework or vendor file_contexts.bin
- # depending on which dictionary the key came from.
- # Only the file basename is required because all selinux_fc properties are
- # replaced with the full path to the file under META/ when misc_info.txt is
- # loaded from target files for repacking. See common.py LoadInfoDict().
- for key in OPTIONS.vendor_misc_info:
- if key.endswith('_selinux_fc'):
- OPTIONS.merged_misc_info[key] = 'vendor_file_contexts.bin'
- for key in OPTIONS.framework_misc_info:
- if key.endswith('_selinux_fc'):
- OPTIONS.merged_misc_info[key] = 'framework_file_contexts.bin'
-
-
-def compile_split_sepolicy(target_files_dir, partition_map):
- """Uses secilc to compile a split sepolicy file.
-
- Depends on various */etc/selinux/* and */etc/vintf/* files within partitions.
-
- Args:
- target_files_dir: Extracted directory of target_files, containing partition
- directories.
- partition_map: A map of partition name -> relative path within
- target_files_dir.
-
- Returns:
- A command list that can be executed to create the compiled sepolicy.
- """
-
- def get_file(partition, path):
- if partition not in partition_map:
- logger.warning('Cannot load SEPolicy files for missing partition %s',
- partition)
- return None
- return os.path.join(target_files_dir, partition_map[partition], path)
-
- # Load the kernel sepolicy version from the FCM. This is normally provided
- # directly to selinux.cpp as a build flag, but is also available in this file.
- fcm_file = get_file('system', 'etc/vintf/compatibility_matrix.device.xml')
- if not fcm_file or not os.path.exists(fcm_file):
- raise ExternalError('Missing required file for loading sepolicy: %s', fcm)
- kernel_sepolicy_version = ElementTree.parse(fcm_file).getroot().find(
- 'sepolicy/kernel-sepolicy-version').text
-
- # Load the vendor's plat sepolicy version. This is the version used for
- # locating sepolicy mapping files.
- vendor_plat_version_file = get_file('vendor',
- 'etc/selinux/plat_sepolicy_vers.txt')
- if not vendor_plat_version_file or not os.path.exists(
- vendor_plat_version_file):
- raise ExternalError('Missing required sepolicy file %s',
- vendor_plat_version_file)
- with open(vendor_plat_version_file) as f:
- vendor_plat_version = f.read().strip()
-
- # Use the same flags and arguments as selinux.cpp OpenSplitPolicy().
- cmd = ['secilc', '-m', '-M', 'true', '-G', '-N']
- cmd.extend(['-c', kernel_sepolicy_version])
- cmd.extend(['-o', os.path.join(target_files_dir, 'META/combined_sepolicy')])
- cmd.extend(['-f', '/dev/null'])
-
- required_policy_files = (
- ('system', 'etc/selinux/plat_sepolicy.cil'),
- ('system', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
- ('vendor', 'etc/selinux/vendor_sepolicy.cil'),
- ('vendor', 'etc/selinux/plat_pub_versioned.cil'),
- )
- for policy in (map(lambda partition_and_path: get_file(*partition_and_path),
- required_policy_files)):
- if not policy or not os.path.exists(policy):
- raise ExternalError('Missing required sepolicy file %s', policy)
- cmd.append(policy)
-
- optional_policy_files = (
- ('system', 'etc/selinux/mapping/%s.compat.cil' % vendor_plat_version),
- ('system_ext', 'etc/selinux/system_ext_sepolicy.cil'),
- ('system_ext', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
- ('product', 'etc/selinux/product_sepolicy.cil'),
- ('product', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
- ('odm', 'etc/selinux/odm_sepolicy.cil'),
- )
- for policy in (map(lambda partition_and_path: get_file(*partition_and_path),
- optional_policy_files)):
- if policy and os.path.exists(policy):
- cmd.append(policy)
-
- return cmd
-
-
-def validate_merged_apex_info(target_files_dir, partitions):
- """Validates the APEX files in the merged target files directory.
-
- Checks the APEX files in all possible preinstalled APEX directories.
- Depends on the <partition>/apex/* APEX files within partitions.
-
- Args:
- target_files_dir: Extracted directory of target_files, containing partition
- directories.
- partitions: A list of all the partitions in the output directory.
-
- Raises:
- RuntimeError: if apex_utils fails to parse any APEX file.
- ExternalError: if the same APEX package is provided by multiple partitions.
- """
- apex_packages = set()
-
- apex_partitions = ('system', 'system_ext', 'product', 'vendor', 'odm')
- for partition in filter(lambda p: p in apex_partitions, partitions):
- apex_info = apex_utils.GetApexInfoFromTargetFiles(
- target_files_dir, partition, compressed_only=False)
- partition_apex_packages = set([info.package_name for info in apex_info])
- duplicates = apex_packages.intersection(partition_apex_packages)
- if duplicates:
- raise ExternalError(
- 'Duplicate APEX packages found in multiple partitions: %s' %
- ' '.join(duplicates))
- apex_packages.update(partition_apex_packages)
-
-
-def update_care_map_image_size_props(images_dir):
- """Sets <partition>_image_size props in misc_info.
-
- add_images_to_target_files uses these props to generate META/care_map.pb.
- Regenerated images will have this property set during regeneration.
-
- However, images copied directly from input partial target files packages
- need this value calculated here.
- """
- for partition in common.PARTITIONS_WITH_CARE_MAP:
- image_path = os.path.join(images_dir, '{}.img'.format(partition))
- if os.path.exists(image_path):
- partition_size = sparse_img.GetImagePartitionSize(image_path)
- image_props = build_image.ImagePropFromGlobalDict(
- OPTIONS.merged_misc_info, partition)
- verity_image_builder = verity_utils.CreateVerityImageBuilder(image_props)
- image_size = verity_image_builder.CalculateMaxImageSize(partition_size)
- OPTIONS.merged_misc_info['{}_image_size'.format(partition)] = image_size
-
-
-def merge_meta_files(temp_dir, merged_dir):
- """Merges various files in META/*."""
-
- framework_meta_dir = os.path.join(temp_dir, 'framework_meta', 'META')
- extract_items(
- input_zip=OPTIONS.framework_target_files,
- output_dir=os.path.dirname(framework_meta_dir),
- extract_item_list=('META/*',))
-
- vendor_meta_dir = os.path.join(temp_dir, 'vendor_meta', 'META')
- extract_items(
- input_zip=OPTIONS.vendor_target_files,
- output_dir=os.path.dirname(vendor_meta_dir),
- extract_item_list=('META/*',))
-
- merged_meta_dir = os.path.join(merged_dir, 'META')
-
- # Merge META/misc_info.txt into OPTIONS.merged_misc_info,
- # but do not write it yet. The following functions may further
- # modify this dict.
- OPTIONS.merged_misc_info = merge_misc_info_txt(
- framework_meta_dir=framework_meta_dir,
- vendor_meta_dir=vendor_meta_dir,
- merged_meta_dir=merged_meta_dir)
-
- create_file_contexts_copies(
- framework_meta_dir=framework_meta_dir,
- vendor_meta_dir=vendor_meta_dir,
- merged_meta_dir=merged_meta_dir)
-
- if OPTIONS.merged_misc_info.get('use_dynamic_partitions') == 'true':
- merge_dynamic_partitions_info_txt(
- framework_meta_dir=framework_meta_dir,
- vendor_meta_dir=vendor_meta_dir,
- merged_meta_dir=merged_meta_dir)
-
- if OPTIONS.merged_misc_info.get('ab_update') == 'true':
- merge_ab_partitions_txt(
- framework_meta_dir=framework_meta_dir,
- vendor_meta_dir=vendor_meta_dir,
- merged_meta_dir=merged_meta_dir)
- update_care_map_image_size_props(images_dir=os.path.join(merged_dir, 'IMAGES'))
-
- for file_name in ('apkcerts.txt', 'apexkeys.txt'):
- merge_package_keys_txt(
- framework_meta_dir=framework_meta_dir,
- vendor_meta_dir=vendor_meta_dir,
- merged_meta_dir=merged_meta_dir,
- file_name=file_name)
-
- # Write the now-finalized OPTIONS.merged_misc_info.
- write_sorted_data(
- data=OPTIONS.merged_misc_info,
- path=os.path.join(merged_meta_dir, 'misc_info.txt'))
-
-
-def process_dexopt(temp_dir, output_target_files_dir):
- """If needed, generates dexopt files for vendor apps.
-
- Args:
- temp_dir: Location containing an 'output' directory where target files have
- been extracted, e.g. <temp_dir>/output/SYSTEM, <temp_dir>/output/IMAGES,
- etc.
- output_target_files_dir: The name of a directory that will be used to create
- the output target files package after all the special cases are processed.
- """
- # Load vendor and framework META/misc_info.txt.
- if (OPTIONS.vendor_misc_info.get('building_with_vsdk') != 'true' or
- OPTIONS.framework_dexpreopt_tools is None or
- OPTIONS.framework_dexpreopt_config is None or
- OPTIONS.vendor_dexpreopt_config is None):
- return
-
- logger.info('applying dexpreopt')
-
- # The directory structure to apply dexpreopt is:
- #
- # <temp_dir>/
- # framework_meta/
- # META/
- # vendor_meta/
- # META/
- # output/
- # SYSTEM/
- # VENDOR/
- # IMAGES/
- # <other items extracted from system and vendor target files>
- # tools/
- # <contents of dexpreopt_tools.zip>
- # system_config/
- # <contents of system dexpreopt_config.zip>
- # vendor_config/
- # <contents of vendor dexpreopt_config.zip>
- # system -> output/SYSTEM
- # vendor -> output/VENDOR
- # apex -> output/SYSTEM/apex (only for flattened APEX builds)
- # apex/ (extracted updatable APEX)
- # <apex 1>/
- # ...
- # <apex 2>/
- # ...
- # ...
- # out/dex2oat_result/vendor/
- # <app>
- # oat/arm64/
- # package.vdex
- # package.odex
- # <priv-app>
- # oat/arm64/
- # package.vdex
- # package.odex
- dexpreopt_tools_files_temp_dir = os.path.join(temp_dir, 'tools')
- dexpreopt_framework_config_files_temp_dir = os.path.join(
- temp_dir, 'system_config')
- dexpreopt_vendor_config_files_temp_dir = os.path.join(temp_dir,
- 'vendor_config')
-
- extract_items(
- input_zip=OPTIONS.framework_dexpreopt_tools,
- output_dir=dexpreopt_tools_files_temp_dir,
- extract_item_list=('*',))
- extract_items(
- input_zip=OPTIONS.framework_dexpreopt_config,
- output_dir=dexpreopt_framework_config_files_temp_dir,
- extract_item_list=('*',))
- extract_items(
- input_zip=OPTIONS.vendor_dexpreopt_config,
- output_dir=dexpreopt_vendor_config_files_temp_dir,
- extract_item_list=('*',))
-
- os.symlink(
- os.path.join(output_target_files_dir, 'SYSTEM'),
- os.path.join(temp_dir, 'system'))
- os.symlink(
- os.path.join(output_target_files_dir, 'VENDOR'),
- os.path.join(temp_dir, 'vendor'))
-
- # The directory structure for flatteded APEXes is:
- #
- # SYSTEM
- # apex
- # <APEX name, e.g., com.android.wifi>
- # apex_manifest.pb
- # apex_pubkey
- # etc/
- # javalib/
- # lib/
- # lib64/
- # priv-app/
- #
- # The directory structure for updatable APEXes is:
- #
- # SYSTEM
- # apex
- # com.android.adbd.apex
- # com.android.appsearch.apex
- # com.android.art.apex
- # ...
- apex_root = os.path.join(output_target_files_dir, 'SYSTEM', 'apex')
-
- # Check for flattended versus updatable APEX.
- if OPTIONS.framework_misc_info.get('target_flatten_apex') == 'false':
- # Extract APEX.
- logging.info('extracting APEX')
-
- apex_extract_root_dir = os.path.join(temp_dir, 'apex')
- os.makedirs(apex_extract_root_dir)
-
- for apex in (glob.glob(os.path.join(apex_root, '*.apex')) +
- glob.glob(os.path.join(apex_root, '*.capex'))):
- logging.info(' apex: %s', apex)
- # deapexer is in the same directory as the merge_target_files binary extracted
- # from otatools.zip.
- apex_json_info = subprocess.check_output(['deapexer', 'info', apex])
- logging.info(' info: %s', apex_json_info)
- apex_info = json.loads(apex_json_info)
- apex_name = apex_info['name']
- logging.info(' name: %s', apex_name)
-
- apex_extract_dir = os.path.join(apex_extract_root_dir, apex_name)
- os.makedirs(apex_extract_dir)
-
- # deapexer uses debugfs_static, which is part of otatools.zip.
- command = [
- 'deapexer',
- '--debugfs_path',
- 'debugfs_static',
- 'extract',
- apex,
- apex_extract_dir,
- ]
- logging.info(' running %s', command)
- subprocess.check_call(command)
- else:
- # Flattened APEXes don't need to be extracted since they have the necessary
- # directory structure.
- os.symlink(os.path.join(apex_root), os.path.join(temp_dir, 'apex'))
-
- # Modify system config to point to the tools that have been extracted.
- # Absolute or .. paths are not allowed by the dexpreopt_gen tool in
- # dexpreopt_soong.config.
- dexpreopt_framework_soon_config = os.path.join(
- dexpreopt_framework_config_files_temp_dir, 'dexpreopt_soong.config')
- with open(dexpreopt_framework_soon_config, 'w') as f:
- dexpreopt_soong_config = {
- 'Profman': 'tools/profman',
- 'Dex2oat': 'tools/dex2oatd',
- 'Aapt': 'tools/aapt2',
- 'SoongZip': 'tools/soong_zip',
- 'Zip2zip': 'tools/zip2zip',
- 'ManifestCheck': 'tools/manifest_check',
- 'ConstructContext': 'tools/construct_context',
- }
- json.dump(dexpreopt_soong_config, f)
-
- # TODO(b/188179859): Make *dex location configurable to vendor or system_other.
- use_system_other_odex = False
-
- if use_system_other_odex:
- dex_img = 'SYSTEM_OTHER'
- else:
- dex_img = 'VENDOR'
- # Open vendor_filesystem_config to append the items generated by dexopt.
- vendor_file_system_config = open(
- os.path.join(temp_dir, 'output', 'META',
- 'vendor_filesystem_config.txt'), 'a')
-
- # Dexpreopt vendor apps.
- dexpreopt_config_suffix = '_dexpreopt.config'
- for config in glob.glob(
- os.path.join(dexpreopt_vendor_config_files_temp_dir,
- '*' + dexpreopt_config_suffix)):
- app = os.path.basename(config)[:-len(dexpreopt_config_suffix)]
- logging.info('dexpreopt config: %s %s', config, app)
-
- apk_dir = 'app'
- apk_path = os.path.join(temp_dir, 'vendor', apk_dir, app, app + '.apk')
- if not os.path.exists(apk_path):
- apk_dir = 'priv-app'
- apk_path = os.path.join(temp_dir, 'vendor', apk_dir, app, app + '.apk')
- if not os.path.exists(apk_path):
- logging.warning(
- 'skipping dexpreopt for %s, no apk found in vendor/app '
- 'or vendor/priv-app', app)
- continue
-
- # Generate dexpreopting script. Note 'out_dir' is not the output directory
- # where the script is generated, but the OUT_DIR at build time referenced
- # in the dexpreot config files, e.g., "out/.../core-oj.jar", so the tool knows
- # how to adjust the path.
- command = [
- os.path.join(dexpreopt_tools_files_temp_dir, 'dexpreopt_gen'),
- '-global',
- os.path.join(dexpreopt_framework_config_files_temp_dir,
- 'dexpreopt.config'),
- '-global_soong',
- os.path.join(dexpreopt_framework_config_files_temp_dir,
- 'dexpreopt_soong.config'),
- '-module',
- config,
- '-dexpreopt_script',
- 'dexpreopt_app.sh',
- '-out_dir',
- 'out',
- '-base_path',
- '.',
- '--uses_target_files',
- ]
-
- # Run the command from temp_dir so all tool paths are its descendants.
- logging.info('running %s', command)
- subprocess.check_call(command, cwd=temp_dir)
-
- # Call the generated script.
- command = ['sh', 'dexpreopt_app.sh', apk_path]
- logging.info('running %s', command)
- subprocess.check_call(command, cwd=temp_dir)
-
- # Output files are in:
- #
- # <temp_dir>/out/dex2oat_result/vendor/priv-app/<app>/oat/arm64/package.vdex
- # <temp_dir>/out/dex2oat_result/vendor/priv-app/<app>/oat/arm64/package.odex
- # <temp_dir>/out/dex2oat_result/vendor/app/<app>/oat/arm64/package.vdex
- # <temp_dir>/out/dex2oat_result/vendor/app/<app>/oat/arm64/package.odex
- #
- # Copy the files to their destination. The structure of system_other is:
- #
- # system_other/
- # system-other-odex-marker
- # system/
- # app/
- # <app>/oat/arm64/
- # <app>.odex
- # <app>.vdex
- # ...
- # priv-app/
- # <app>/oat/arm64/
- # <app>.odex
- # <app>.vdex
- # ...
-
- # TODO(b/188179859): Support for other architectures.
- arch = 'arm64'
-
- dex_destination = os.path.join(temp_dir, 'output', dex_img, apk_dir, app,
- 'oat', arch)
- os.makedirs(dex_destination)
- dex2oat_path = os.path.join(temp_dir, 'out', 'dex2oat_result', 'vendor',
- apk_dir, app, 'oat', arch)
- shutil.copy(
- os.path.join(dex2oat_path, 'package.vdex'),
- os.path.join(dex_destination, app + '.vdex'))
- shutil.copy(
- os.path.join(dex2oat_path, 'package.odex'),
- os.path.join(dex_destination, app + '.odex'))
-
- # Append entries to vendor_file_system_config.txt, such as:
- #
- # vendor/app/<app>/oat 0 2000 755 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
- # vendor/app/<app>/oat/arm64 0 2000 755 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
- # vendor/app/<app>/oat/arm64/<app>.odex 0 0 644 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
- # vendor/app/<app>/oat/arm64/<app>.vdex 0 0 644 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
- if not use_system_other_odex:
- vendor_app_prefix = 'vendor/' + apk_dir + '/' + app + '/oat'
- selabel = 'selabel=u:object_r:vendor_app_file:s0 capabilities=0x0'
- vendor_file_system_config.writelines([
- vendor_app_prefix + ' 0 2000 755 ' + selabel + '\n',
- vendor_app_prefix + '/' + arch + ' 0 2000 755 ' + selabel + '\n',
- vendor_app_prefix + '/' + arch + '/' + app + '.odex 0 0 644 ' +
- selabel + '\n',
- vendor_app_prefix + '/' + arch + '/' + app + '.vdex 0 0 644 ' +
- selabel + '\n',
- ])
-
- if not use_system_other_odex:
- vendor_file_system_config.close()
- # Delete vendor.img so that it will be regenerated.
- # TODO(b/188179859): Rebuilding a vendor image in GRF mode (e.g., T(framework)
- # and S(vendor) may require logic similar to that in
- # rebuild_image_with_sepolicy.
- vendor_img = os.path.join(output_target_files_dir, 'IMAGES', 'vendor.img')
- if os.path.exists(vendor_img):
- logging.info('Deleting %s', vendor_img)
- os.remove(vendor_img)
-
def create_merged_package(temp_dir):
"""Merges two target files packages into one target files structure.
@@ -1041,11 +217,11 @@
# do not need special case processing.
output_target_files_temp_dir = os.path.join(temp_dir, 'output')
- extract_items(
+ merge_utils.ExtractItems(
input_zip=OPTIONS.framework_target_files,
output_dir=output_target_files_temp_dir,
extract_item_list=OPTIONS.framework_item_list)
- extract_items(
+ merge_utils.ExtractItems(
input_zip=OPTIONS.vendor_target_files,
output_dir=output_target_files_temp_dir,
extract_item_list=OPTIONS.vendor_item_list)
@@ -1053,9 +229,10 @@
# Perform special case processing on META/* items.
# After this function completes successfully, all the files we need to create
# the output target files package are in place.
- merge_meta_files(temp_dir=temp_dir, merged_dir=output_target_files_temp_dir)
+ merge_meta.MergeMetaFiles(
+ temp_dir=temp_dir, merged_dir=output_target_files_temp_dir)
- process_dexopt(
+ merge_dexopt.MergeDexopt(
temp_dir=temp_dir, output_target_files_dir=output_target_files_temp_dir)
return output_target_files_temp_dir
@@ -1284,58 +461,22 @@
output_target_files_temp_dir = create_merged_package(temp_dir)
- if not check_target_files_vintf.CheckVintf(output_target_files_temp_dir):
- raise RuntimeError('Incompatible VINTF metadata')
-
partition_map = common.PartitionMapFromTargetFiles(
output_target_files_temp_dir)
- # Generate and check for cross-partition violations of sharedUserId
- # values in APKs. This requires the input target-files packages to contain
- # *.apk files.
- shareduid_violation_modules = os.path.join(
- output_target_files_temp_dir, 'META', 'shareduid_violation_modules.json')
- with open(shareduid_violation_modules, 'w') as f:
- violation = find_shareduid_violation.FindShareduidViolation(
- output_target_files_temp_dir, partition_map)
+ compatibility_errors = merge_compatibility_checks.CheckCompatibility(
+ target_files_dir=output_target_files_temp_dir,
+ partition_map=partition_map)
+ if compatibility_errors:
+ for error in compatibility_errors:
+ logger.error(error)
+ raise ExternalError(
+ 'Found incompatibilities in the merged target files package.')
- # Write the output to a file to enable debugging.
- f.write(violation)
-
- # Check for violations across the input builds' partition groups.
- shareduid_errors = common.SharedUidPartitionViolations(
- json.loads(violation),
- [OPTIONS.framework_partition_set, OPTIONS.vendor_partition_set])
- if shareduid_errors:
- for error in shareduid_errors:
- logger.error(error)
- raise ValueError('sharedUserId APK error. See %s' %
- shareduid_violation_modules)
-
- # host_init_verifier and secilc check only the following partitions:
- filtered_partitions = {
- partition: path
- for partition, path in partition_map.items()
- if partition in ['system', 'system_ext', 'product', 'vendor', 'odm']
- }
-
- # Run host_init_verifier on the combined init rc files.
- common.RunHostInitVerifier(
- product_out=output_target_files_temp_dir,
- partition_map=filtered_partitions)
-
- # Check that the split sepolicy from the multiple builds can compile.
- split_sepolicy_cmd = compile_split_sepolicy(output_target_files_temp_dir,
- filtered_partitions)
- logger.info('Compiling split sepolicy: %s', ' '.join(split_sepolicy_cmd))
- common.RunAndCheckOutput(split_sepolicy_cmd)
# Include the compiled policy in an image if requested.
if OPTIONS.rebuild_sepolicy:
rebuild_image_with_sepolicy(output_target_files_temp_dir)
- # Run validation checks on the pre-installed APEX files.
- validate_merged_apex_info(output_target_files_temp_dir, partition_map.keys())
-
generate_missing_images(output_target_files_temp_dir)
generate_super_empty_image(output_target_files_temp_dir,
@@ -1345,8 +486,8 @@
# output items to the output target files directory.
if OPTIONS.output_dir:
- copy_items(output_target_files_temp_dir, OPTIONS.output_dir,
- OPTIONS.output_item_list)
+ merge_utils.CopyItems(output_target_files_temp_dir, OPTIONS.output_dir,
+ OPTIONS.output_item_list)
if not OPTIONS.output_target_files:
return
@@ -1366,33 +507,6 @@
[OPTIONS.output_target_files, OPTIONS.output_ota])
-def call_func_with_temp_dir(func, keep_tmp):
- """Manages the creation and cleanup of the temporary directory.
-
- This function calls the given function after first creating a temporary
- directory. It also cleans up the temporary directory.
-
- Args:
- func: The function to call. Should accept one parameter, the path to the
- temporary directory.
- keep_tmp: Keep the temporary directory after processing is complete.
- """
-
- # Create a temporary directory. This will serve as the parent of directories
- # we use when we extract items from the input target files packages, and also
- # a scratch directory that we use for temporary files.
-
- temp_dir = common.MakeTempDir(prefix='merge_target_files_')
-
- try:
- func(temp_dir)
- finally:
- if keep_tmp:
- logger.info('keeping %s', temp_dir)
- else:
- common.Cleanup()
-
-
def main():
"""The main function.
@@ -1444,7 +558,7 @@
OPTIONS.output_img = a
elif o == '--output-super-empty':
OPTIONS.output_super_empty = a
- elif o == '--rebuild_recovery':
+ elif o == '--rebuild_recovery' or o == '--rebuild-recovery':
OPTIONS.rebuild_recovery = True
elif o == '--allow-duplicate-apkapex-keys':
OPTIONS.allow_duplicate_apkapex_keys = True
@@ -1488,6 +602,7 @@
'framework-dexpreopt-tools=',
'vendor-dexpreopt-config=',
'rebuild_recovery',
+ 'rebuild-recovery',
'allow-duplicate-apkapex-keys',
'vendor-otatools=',
'rebuild-sepolicy',
@@ -1509,7 +624,7 @@
OPTIONS.framework_item_list)
else:
OPTIONS.framework_item_list = DEFAULT_FRAMEWORK_ITEM_LIST
- OPTIONS.framework_partition_set = item_list_to_partition_set(
+ OPTIONS.framework_partition_set = merge_utils.ItemListToPartitionSet(
OPTIONS.framework_item_list)
if OPTIONS.framework_misc_info_keys:
@@ -1522,7 +637,7 @@
OPTIONS.vendor_item_list = common.LoadListFromFile(OPTIONS.vendor_item_list)
else:
OPTIONS.vendor_item_list = DEFAULT_VENDOR_ITEM_LIST
- OPTIONS.vendor_partition_set = item_list_to_partition_set(
+ OPTIONS.vendor_partition_set = merge_utils.ItemListToPartitionSet(
OPTIONS.vendor_item_list)
if OPTIONS.output_item_list:
@@ -1530,11 +645,17 @@
else:
OPTIONS.output_item_list = None
- if not validate_config_lists():
+ if not merge_utils.ValidateConfigLists():
sys.exit(1)
- call_func_with_temp_dir(lambda temp_dir: merge_target_files(temp_dir),
- OPTIONS.keep_tmp)
+ temp_dir = common.MakeTempDir(prefix='merge_target_files_')
+ try:
+ merge_target_files(temp_dir)
+ finally:
+ if OPTIONS.keep_tmp:
+ logger.info('Keeping temp_dir %s', temp_dir)
+ else:
+ common.Cleanup()
if __name__ == '__main__':