Moves merge scripts into a new merge folder.
This will be used in the following change to split
merge_target_files into a collection of smaller
more-focused scripts.
Bug: 221858722
Test: m otatools; Use to create merged builds
Change-Id: Ie01dac81c5f9f28f1e0fe037259eabd2478e60b6
diff --git a/tools/releasetools/merge/Android.bp b/tools/releasetools/merge/Android.bp
new file mode 100644
index 0000000..c07d8ee
--- /dev/null
+++ b/tools/releasetools/merge/Android.bp
@@ -0,0 +1,69 @@
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+filegroup {
+ name: "releasetools_merge_sources",
+ srcs: [
+ "merge_target_files.py",
+ ],
+}
+
+filegroup {
+ name: "releasetools_merge_tests",
+ srcs: [
+ "test_merge_target_files.py",
+ ],
+}
+
+python_binary_host {
+ name: "merge_target_files",
+ defaults: ["releasetools_binary_defaults"],
+ srcs: [":releasetools_merge_sources"],
+ libs: [
+ "releasetools_add_img_to_target_files",
+ "releasetools_build_super_image",
+ "releasetools_check_target_files_vintf",
+ "releasetools_common",
+ "releasetools_find_shareduid_violation",
+ "releasetools_img_from_target_files",
+ "releasetools_ota_from_target_files",
+ ],
+ required: [
+ "checkvintf",
+ "host_init_verifier",
+ "secilc",
+ ],
+ target: {
+ darwin: {
+ // libs dep "releasetools_ota_from_target_files" is disabled on darwin
+ enabled: false,
+ },
+ },
+}
+
+python_binary_host {
+ name: "merge_builds",
+ defaults: ["releasetools_binary_defaults"],
+ srcs: [
+ "merge_builds.py",
+ ],
+ libs: [
+ "releasetools_build_super_image",
+ "releasetools_common",
+ ],
+}
diff --git a/tools/releasetools/merge/OWNERS b/tools/releasetools/merge/OWNERS
new file mode 100644
index 0000000..9012e3a
--- /dev/null
+++ b/tools/releasetools/merge/OWNERS
@@ -0,0 +1,3 @@
+danielnorman@google.com
+jgalmes@google.com
+rseymour@google.com
diff --git a/tools/releasetools/merge/merge_builds.py b/tools/releasetools/merge/merge_builds.py
new file mode 100644
index 0000000..3ac4ec4
--- /dev/null
+++ b/tools/releasetools/merge/merge_builds.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+"""Merges two non-dist partial builds together.
+
+Given two partial builds, a framework build and a vendor build, merge the builds
+together so that the images can be flashed using 'fastboot flashall'.
+
+To support both DAP and non-DAP vendor builds with a single framework partial
+build, the framework partial build should always be built with DAP enabled. The
+vendor partial build determines whether the merged result supports DAP.
+
+This script does not require builds to be built with 'make dist'.
+This script regenerates super_empty.img and vbmeta.img if necessary. Other
+images are assumed to not require regeneration.
+
+Usage: merge_builds.py [args]
+
+ --framework_images comma_separated_image_list
+ Comma-separated list of image names that should come from the framework
+ build.
+
+ --product_out_framework product_out_framework_path
+ Path to out/target/product/<framework build>.
+
+ --product_out_vendor product_out_vendor_path
+ Path to out/target/product/<vendor build>.
+
+ --build_vbmeta
+ If provided, vbmeta.img will be regenerated in out/target/product/<vendor
+ build>.
+
+ --framework_misc_info_keys
+ The optional path to a newline-separated config file containing keys to
+ obtain from the framework instance of misc_info.txt, used for creating
+ vbmeta.img. The remaining keys come from the vendor instance.
+"""
+from __future__ import print_function
+
+import logging
+import os
+import sys
+
+import build_super_image
+import common
+
+logger = logging.getLogger(__name__)
+
+OPTIONS = common.OPTIONS
+OPTIONS.framework_images = ("system",)
+OPTIONS.product_out_framework = None
+OPTIONS.product_out_vendor = None
+OPTIONS.build_vbmeta = False
+OPTIONS.framework_misc_info_keys = None
+
+
+def CreateImageSymlinks():
+ for image in OPTIONS.framework_images:
+ image_path = os.path.join(OPTIONS.product_out_framework, "%s.img" % image)
+ symlink_path = os.path.join(OPTIONS.product_out_vendor, "%s.img" % image)
+ if os.path.exists(symlink_path):
+ if os.path.islink(symlink_path):
+ os.remove(symlink_path)
+ else:
+ raise ValueError("Attempting to overwrite built image: %s" %
+ symlink_path)
+ os.symlink(image_path, symlink_path)
+
+
+def BuildSuperEmpty():
+ framework_dict = common.LoadDictionaryFromFile(
+ os.path.join(OPTIONS.product_out_framework, "misc_info.txt"))
+ vendor_dict = common.LoadDictionaryFromFile(
+ os.path.join(OPTIONS.product_out_vendor, "misc_info.txt"))
+ # Regenerate super_empty.img if both partial builds enable DAP. If only the
+ # the vendor build enables DAP, the vendor build's existing super_empty.img
+ # will be reused. If only the framework build should enable DAP, super_empty
+ # should be included in the --framework_images flag to copy the existing
+ # super_empty.img from the framework build.
+ if (framework_dict.get("use_dynamic_partitions") == "true") and (
+ vendor_dict.get("use_dynamic_partitions") == "true"):
+ logger.info("Building super_empty.img.")
+ merged_dict = dict(vendor_dict)
+ merged_dict.update(
+ common.MergeDynamicPartitionInfoDicts(
+ framework_dict=framework_dict, vendor_dict=vendor_dict))
+ output_super_empty_path = os.path.join(OPTIONS.product_out_vendor,
+ "super_empty.img")
+ build_super_image.BuildSuperImage(merged_dict, output_super_empty_path)
+
+
+def BuildVBMeta():
+ logger.info("Building vbmeta.img.")
+
+ framework_dict = common.LoadDictionaryFromFile(
+ os.path.join(OPTIONS.product_out_framework, "misc_info.txt"))
+ vendor_dict = common.LoadDictionaryFromFile(
+ os.path.join(OPTIONS.product_out_vendor, "misc_info.txt"))
+ merged_dict = dict(vendor_dict)
+ if OPTIONS.framework_misc_info_keys:
+ for key in common.LoadListFromFile(OPTIONS.framework_misc_info_keys):
+ merged_dict[key] = framework_dict[key]
+
+ # Build vbmeta.img using partitions in product_out_vendor.
+ partitions = {}
+ for partition in common.AVB_PARTITIONS:
+ partition_path = os.path.join(OPTIONS.product_out_vendor,
+ "%s.img" % partition)
+ if os.path.exists(partition_path):
+ partitions[partition] = partition_path
+
+ # vbmeta_partitions includes the partitions that should be included into
+ # top-level vbmeta.img, which are the ones that are not included in any
+ # chained VBMeta image plus the chained VBMeta images themselves.
+ vbmeta_partitions = common.AVB_PARTITIONS[:]
+ for partition in common.AVB_VBMETA_PARTITIONS:
+ chained_partitions = merged_dict.get("avb_%s" % partition, "").strip()
+ if chained_partitions:
+ partitions[partition] = os.path.join(OPTIONS.product_out_vendor,
+ "%s.img" % partition)
+ vbmeta_partitions = [
+ item for item in vbmeta_partitions
+ if item not in chained_partitions.split()
+ ]
+ vbmeta_partitions.append(partition)
+
+ output_vbmeta_path = os.path.join(OPTIONS.product_out_vendor, "vbmeta.img")
+ OPTIONS.info_dict = merged_dict
+ common.BuildVBMeta(output_vbmeta_path, partitions, "vbmeta",
+ vbmeta_partitions)
+
+
+def MergeBuilds():
+ CreateImageSymlinks()
+ BuildSuperEmpty()
+ if OPTIONS.build_vbmeta:
+ BuildVBMeta()
+
+
+def main():
+ common.InitLogging()
+
+ def option_handler(o, a):
+ if o == "--framework_images":
+ OPTIONS.framework_images = [i.strip() for i in a.split(",")]
+ elif o == "--product_out_framework":
+ OPTIONS.product_out_framework = a
+ elif o == "--product_out_vendor":
+ OPTIONS.product_out_vendor = a
+ elif o == "--build_vbmeta":
+ OPTIONS.build_vbmeta = True
+ elif o == "--framework_misc_info_keys":
+ OPTIONS.framework_misc_info_keys = a
+ else:
+ return False
+ return True
+
+ args = common.ParseOptions(
+ sys.argv[1:],
+ __doc__,
+ extra_long_opts=[
+ "framework_images=",
+ "product_out_framework=",
+ "product_out_vendor=",
+ "build_vbmeta",
+ "framework_misc_info_keys=",
+ ],
+ extra_option_handler=option_handler)
+
+ if (args or OPTIONS.product_out_framework is None or
+ OPTIONS.product_out_vendor is None):
+ common.Usage(__doc__)
+ sys.exit(1)
+
+ MergeBuilds()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/releasetools/merge/merge_target_files.py b/tools/releasetools/merge/merge_target_files.py
new file mode 100755
index 0000000..25ed60e
--- /dev/null
+++ b/tools/releasetools/merge/merge_target_files.py
@@ -0,0 +1,1541 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+"""This script merges two partial target files packages.
+
+One input package contains framework files, and the other contains vendor files.
+
+This script produces a complete, merged target files package:
+ - This package can be used to generate a flashable IMG package.
+ See --output-img.
+ - This package can be used to generate an OTA package. See --output-ota.
+ - The merged package is checked for compatibility between the two inputs.
+
+Usage: merge_target_files [args]
+
+ --framework-target-files framework-target-files-zip-archive
+ The input target files package containing framework bits. This is a zip
+ archive.
+
+ --framework-item-list framework-item-list-file
+ The optional path to a newline-separated config file that replaces the
+ contents of DEFAULT_FRAMEWORK_ITEM_LIST if provided.
+
+ --framework-misc-info-keys framework-misc-info-keys-file
+ The optional path to a newline-separated config file that replaces the
+ contents of DEFAULT_FRAMEWORK_MISC_INFO_KEYS if provided.
+
+ --vendor-target-files vendor-target-files-zip-archive
+ The input target files package containing vendor bits. This is a zip
+ archive.
+
+ --vendor-item-list vendor-item-list-file
+ The optional path to a newline-separated config file that replaces the
+ contents of DEFAULT_VENDOR_ITEM_LIST if provided.
+
+ --output-target-files output-target-files-package
+ If provided, the output merged target files package. Also a zip archive.
+
+ --output-dir output-directory
+ If provided, the destination directory for saving merged files. Requires
+ the --output-item-list flag.
+ Can be provided alongside --output-target-files, or by itself.
+
+ --output-item-list output-item-list-file.
+ The optional path to a newline-separated config file that specifies the
+ file patterns to copy into the --output-dir. Required if providing
+ the --output-dir flag.
+
+ --output-ota output-ota-package
+ The output ota package. This is a zip archive. Use of this flag may
+ require passing the --path common flag; see common.py.
+
+ --output-img output-img-package
+ The output img package, suitable for use with 'fastboot update'. Use of
+ this flag may require passing the --path common flag; see common.py.
+
+ --output-super-empty output-super-empty-image
+ If provided, creates a super_empty.img file from the merged target
+ files package and saves it at this path.
+
+ --rebuild_recovery
+ Copy the recovery image used by non-A/B devices, used when
+ regenerating vendor images with --rebuild-sepolicy.
+
+ --allow-duplicate-apkapex-keys
+ If provided, duplicate APK/APEX keys are ignored and the value from the
+ framework is used.
+
+ --rebuild-sepolicy
+ If provided, rebuilds odm.img or vendor.img to include merged sepolicy
+ files. If odm is present then odm is preferred.
+
+ --vendor-otatools otatools.zip
+ If provided, use this otatools.zip when recompiling the odm or vendor
+ image to include sepolicy.
+
+ --keep-tmp
+ Keep tempoary files for debugging purposes.
+
+ The following only apply when using the VSDK to perform dexopt on vendor apps:
+
+ --framework-dexpreopt-config
+ If provided, the location of framwework's dexpreopt_config.zip.
+
+ --framework-dexpreopt-tools
+ if provided, the location of framework's dexpreopt_tools.zip.
+
+ --vendor-dexpreopt-config
+ If provided, the location of vendor's dexpreopt_config.zip.
+"""
+
+import fnmatch
+import glob
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+import zipfile
+from xml.etree import ElementTree
+
+import add_img_to_target_files
+import apex_utils
+import build_image
+import build_super_image
+import check_target_files_vintf
+import common
+import img_from_target_files
+import find_shareduid_violation
+import ota_from_target_files
+import sparse_img
+import verity_utils
+
+from common import ExternalError
+
+logger = logging.getLogger(__name__)
+
+OPTIONS = common.OPTIONS
+# Always turn on verbose logging.
+OPTIONS.verbose = True
+OPTIONS.framework_target_files = None
+OPTIONS.framework_item_list = None
+OPTIONS.framework_misc_info_keys = None
+OPTIONS.vendor_target_files = None
+OPTIONS.vendor_item_list = None
+OPTIONS.output_target_files = None
+OPTIONS.output_dir = None
+OPTIONS.output_item_list = None
+OPTIONS.output_ota = None
+OPTIONS.output_img = None
+OPTIONS.output_super_empty = None
+OPTIONS.rebuild_recovery = False
+# TODO(b/150582573): Remove this option.
+OPTIONS.allow_duplicate_apkapex_keys = False
+OPTIONS.vendor_otatools = None
+OPTIONS.rebuild_sepolicy = False
+OPTIONS.keep_tmp = False
+OPTIONS.framework_dexpreopt_config = None
+OPTIONS.framework_dexpreopt_tools = None
+OPTIONS.vendor_dexpreopt_config = None
+
+# In an item list (framework or vendor), we may see entries that select whole
+# partitions. Such an entry might look like this 'SYSTEM/*' (e.g., for the
+# system partition). The following regex matches this and extracts the
+# partition name.
+
+PARTITION_ITEM_PATTERN = re.compile(r'^([A-Z_]+)/\*$')
+
+# In apexkeys.txt or apkcerts.txt, we will find partition tags on each entry in
+# the file. We use these partition tags to filter the entries in those files
+# from the two different target files packages to produce a merged apexkeys.txt
+# or apkcerts.txt file. A partition tag (e.g., for the product partition) looks
+# like this: 'partition="product"'. We use the group syntax grab the value of
+# the tag. We use non-greedy matching in case there are other fields on the
+# same line.
+
+PARTITION_TAG_PATTERN = re.compile(r'partition="(.*?)"')
+
+# The sorting algorithm for apexkeys.txt and apkcerts.txt does not include the
+# ".apex" or ".apk" suffix, so we use the following pattern to extract a key.
+
+MODULE_KEY_PATTERN = re.compile(r'name="(.+)\.(apex|apk)"')
+
+# DEFAULT_FRAMEWORK_ITEM_LIST is a list of items to extract from the partial
+# framework target files package as is, meaning these items will land in the
+# output target files package exactly as they appear in the input partial
+# framework target files package.
+
+DEFAULT_FRAMEWORK_ITEM_LIST = (
+ 'META/apkcerts.txt',
+ 'META/filesystem_config.txt',
+ 'META/root_filesystem_config.txt',
+ 'META/update_engine_config.txt',
+ 'PRODUCT/*',
+ 'ROOT/*',
+ 'SYSTEM/*',
+)
+
+# DEFAULT_FRAMEWORK_MISC_INFO_KEYS is a list of keys to obtain from the
+# framework instance of META/misc_info.txt. The remaining keys should come
+# from the vendor instance.
+
+DEFAULT_FRAMEWORK_MISC_INFO_KEYS = (
+ 'avb_system_hashtree_enable',
+ 'avb_system_add_hashtree_footer_args',
+ 'avb_system_key_path',
+ 'avb_system_algorithm',
+ 'avb_system_rollback_index_location',
+ 'avb_product_hashtree_enable',
+ 'avb_product_add_hashtree_footer_args',
+ 'avb_system_ext_hashtree_enable',
+ 'avb_system_ext_add_hashtree_footer_args',
+ 'system_root_image',
+ 'root_dir',
+ 'ab_update',
+ 'default_system_dev_certificate',
+ 'system_size',
+ 'building_system_image',
+ 'building_system_ext_image',
+ 'building_product_image',
+)
+
+# DEFAULT_VENDOR_ITEM_LIST is a list of items to extract from the partial
+# vendor target files package as is, meaning these items will land in the output
+# target files package exactly as they appear in the input partial vendor target
+# files package.
+
+DEFAULT_VENDOR_ITEM_LIST = (
+ 'META/boot_filesystem_config.txt',
+ 'META/otakeys.txt',
+ 'META/releasetools.py',
+ 'META/vendor_filesystem_config.txt',
+ 'BOOT/*',
+ 'DATA/*',
+ 'ODM/*',
+ 'OTA/android-info.txt',
+ 'PREBUILT_IMAGES/*',
+ 'RADIO/*',
+ 'VENDOR/*',
+)
+
+# The merge config lists should not attempt to extract items from both
+# builds for any of the following partitions. The partitions in
+# SINGLE_BUILD_PARTITIONS should come entirely from a single build (either
+# framework or vendor, but not both).
+
+SINGLE_BUILD_PARTITIONS = (
+ 'BOOT/',
+ 'DATA/',
+ 'ODM/',
+ 'PRODUCT/',
+ 'SYSTEM_EXT/',
+ 'RADIO/',
+ 'RECOVERY/',
+ 'ROOT/',
+ 'SYSTEM/',
+ 'SYSTEM_OTHER/',
+ 'VENDOR/',
+ 'VENDOR_DLKM/',
+ 'ODM_DLKM/',
+ 'SYSTEM_DLKM/',
+)
+
+
+def write_sorted_data(data, path):
+ """Writes the sorted contents of either a list or dict to file.
+
+ This function sorts the contents of the list or dict and then writes the
+ resulting sorted contents to a file specified by path.
+
+ Args:
+ data: The list or dict to sort and write.
+ path: Path to the file to write the sorted values to. The file at path will
+ be overridden if it exists.
+ """
+ with open(path, 'w') as output:
+ for entry in sorted(data):
+ out_str = '{}={}\n'.format(entry, data[entry]) if isinstance(
+ data, dict) else '{}\n'.format(entry)
+ output.write(out_str)
+
+
+def extract_items(input_zip, output_dir, extract_item_list):
+ """Extracts items in extra_item_list from a zip to a dir."""
+
+ logger.info('extracting from %s', input_zip)
+
+ # Filter the extract_item_list to remove any items that do not exist in the
+ # zip file. Otherwise, the extraction step will fail.
+
+ with zipfile.ZipFile(input_zip, allowZip64=True) as input_zipfile:
+ input_namelist = input_zipfile.namelist()
+
+ filtered_extract_item_list = []
+ for pattern in extract_item_list:
+ matching_namelist = fnmatch.filter(input_namelist, pattern)
+ if not matching_namelist:
+ logger.warning('no match for %s', pattern)
+ else:
+ filtered_extract_item_list.append(pattern)
+
+ common.UnzipToDir(input_zip, output_dir, filtered_extract_item_list)
+
+
+def copy_items(from_dir, to_dir, patterns):
+ """Similar to extract_items() except uses an input dir instead of zip."""
+ file_paths = []
+ for dirpath, _, filenames in os.walk(from_dir):
+ file_paths.extend(
+ os.path.relpath(path=os.path.join(dirpath, filename), start=from_dir)
+ for filename in filenames)
+
+ filtered_file_paths = set()
+ for pattern in patterns:
+ filtered_file_paths.update(fnmatch.filter(file_paths, pattern))
+
+ for file_path in filtered_file_paths:
+ original_file_path = os.path.join(from_dir, file_path)
+ copied_file_path = os.path.join(to_dir, file_path)
+ copied_file_dir = os.path.dirname(copied_file_path)
+ if not os.path.exists(copied_file_dir):
+ os.makedirs(copied_file_dir)
+ if os.path.islink(original_file_path):
+ os.symlink(os.readlink(original_file_path), copied_file_path)
+ else:
+ shutil.copyfile(original_file_path, copied_file_path)
+
+
+def validate_config_lists():
+ """Performs validations on the merge config lists.
+
+ Returns:
+ False if a validation fails, otherwise true.
+ """
+ has_error = False
+
+ default_combined_item_set = set(DEFAULT_FRAMEWORK_ITEM_LIST)
+ default_combined_item_set.update(DEFAULT_VENDOR_ITEM_LIST)
+
+ combined_item_set = set(OPTIONS.framework_item_list)
+ combined_item_set.update(OPTIONS.vendor_item_list)
+
+ # Check that the merge config lists are not missing any item specified
+ # by the default config lists.
+ difference = default_combined_item_set.difference(combined_item_set)
+ if difference:
+ logger.error('Missing merge config items: %s', list(difference))
+ logger.error('Please ensure missing items are in either the '
+ 'framework-item-list or vendor-item-list files provided to '
+ 'this script.')
+ has_error = True
+
+ # Check that partitions only come from one input.
+ for partition in SINGLE_BUILD_PARTITIONS:
+ image_path = 'IMAGES/{}.img'.format(partition.lower().replace('/', ''))
+ in_framework = (
+ any(item.startswith(partition) for item in OPTIONS.framework_item_list)
+ or image_path in OPTIONS.framework_item_list)
+ in_vendor = (
+ any(item.startswith(partition) for item in OPTIONS.vendor_item_list) or
+ image_path in OPTIONS.vendor_item_list)
+ if in_framework and in_vendor:
+ logger.error(
+ 'Cannot extract items from %s for both the framework and vendor'
+ ' builds. Please ensure only one merge config item list'
+ ' includes %s.', partition, partition)
+ has_error = True
+
+ if ('dynamic_partition_list' in OPTIONS.framework_misc_info_keys) or (
+ 'super_partition_groups' in OPTIONS.framework_misc_info_keys):
+ logger.error('Dynamic partition misc info keys should come from '
+ 'the vendor instance of META/misc_info.txt.')
+ has_error = True
+
+ return not has_error
+
+
+def merge_ab_partitions_txt(framework_meta_dir, vendor_meta_dir,
+ merged_meta_dir):
+ """Merges META/ab_partitions.txt.
+
+ The output contains the union of the partition names.
+ """
+ with open(os.path.join(framework_meta_dir, 'ab_partitions.txt')) as f:
+ framework_ab_partitions = f.read().splitlines()
+
+ with open(os.path.join(vendor_meta_dir, 'ab_partitions.txt')) as f:
+ vendor_ab_partitions = f.read().splitlines()
+
+ write_sorted_data(
+ data=set(framework_ab_partitions + vendor_ab_partitions),
+ path=os.path.join(merged_meta_dir, 'ab_partitions.txt'))
+
+
+def merge_misc_info_txt(framework_meta_dir, vendor_meta_dir, merged_meta_dir):
+ """Merges META/misc_info.txt.
+
+ The output contains a combination of key=value pairs from both inputs.
+ Most pairs are taken from the vendor input, while some are taken from
+ the framework input.
+ """
+
+ OPTIONS.framework_misc_info = common.LoadDictionaryFromFile(
+ os.path.join(framework_meta_dir, 'misc_info.txt'))
+ OPTIONS.vendor_misc_info = common.LoadDictionaryFromFile(
+ os.path.join(vendor_meta_dir, 'misc_info.txt'))
+
+ # Merged misc info is a combination of vendor misc info plus certain values
+ # from the framework misc info.
+
+ merged_dict = OPTIONS.vendor_misc_info
+ for key in OPTIONS.framework_misc_info_keys:
+ merged_dict[key] = OPTIONS.framework_misc_info[key]
+
+ # If AVB is enabled then ensure that we build vbmeta.img.
+ # Partial builds with AVB enabled may set PRODUCT_BUILD_VBMETA_IMAGE=false to
+ # skip building an incomplete vbmeta.img.
+ if merged_dict.get('avb_enable') == 'true':
+ merged_dict['avb_building_vbmeta_image'] = 'true'
+
+ return merged_dict
+
+
+def merge_dynamic_partitions_info_txt(framework_meta_dir, vendor_meta_dir,
+ merged_meta_dir):
+ """Merge META/dynamic_partitions_info.txt."""
+ framework_dynamic_partitions_dict = common.LoadDictionaryFromFile(
+ os.path.join(framework_meta_dir, 'dynamic_partitions_info.txt'))
+ vendor_dynamic_partitions_dict = common.LoadDictionaryFromFile(
+ os.path.join(vendor_meta_dir, 'dynamic_partitions_info.txt'))
+
+ merged_dynamic_partitions_dict = common.MergeDynamicPartitionInfoDicts(
+ framework_dict=framework_dynamic_partitions_dict,
+ vendor_dict=vendor_dynamic_partitions_dict)
+
+ write_sorted_data(
+ data=merged_dynamic_partitions_dict,
+ path=os.path.join(merged_meta_dir, 'dynamic_partitions_info.txt'))
+
+ # Merge misc info keys used for Dynamic Partitions.
+ OPTIONS.merged_misc_info.update(merged_dynamic_partitions_dict)
+ # Ensure that add_img_to_target_files rebuilds super split images for
+ # devices that retrofit dynamic partitions. This flag may have been set to
+ # false in the partial builds to prevent duplicate building of super.img.
+ OPTIONS.merged_misc_info['build_super_partition'] = 'true'
+
+
+def item_list_to_partition_set(item_list):
+ """Converts a target files item list to a partition set.
+
+ The item list contains items that might look like 'SYSTEM/*' or 'VENDOR/*' or
+ 'OTA/android-info.txt'. Items that end in '/*' are assumed to match entire
+ directories where 'SYSTEM' or 'VENDOR' is a directory name that identifies the
+ contents of a partition of the same name. Other items in the list, such as the
+ 'OTA' example contain metadata. This function iterates such a list, returning
+ a set that contains the partition entries.
+
+ Args:
+ item_list: A list of items in a target files package.
+
+ Returns:
+ A set of partitions extracted from the list of items.
+ """
+
+ partition_set = set()
+
+ for item in item_list:
+ match = PARTITION_ITEM_PATTERN.search(item.strip())
+ partition_tag = match.group(1).lower() if match else None
+
+ if partition_tag:
+ partition_set.add(partition_tag)
+
+ return partition_set
+
+
+def merge_package_keys_txt(framework_meta_dir, vendor_meta_dir, merged_meta_dir,
+ file_name):
+ """Merges APK/APEX key list files."""
+
+ if file_name not in ('apkcerts.txt', 'apexkeys.txt'):
+ raise ExternalError(
+ 'Unexpected file_name provided to merge_package_keys_txt: %s',
+ file_name)
+
+ def read_helper(d):
+ temp = {}
+ with open(os.path.join(d, file_name)) as f:
+ for line in f.read().splitlines():
+ line = line.strip()
+ if line:
+ name_search = MODULE_KEY_PATTERN.search(line.split()[0])
+ temp[name_search.group(1)] = line
+ return temp
+
+ framework_dict = read_helper(framework_meta_dir)
+ vendor_dict = read_helper(vendor_meta_dir)
+ merged_dict = {}
+
+ def filter_into_merged_dict(item_dict, partition_set):
+ for key, value in item_dict.items():
+ tag_search = PARTITION_TAG_PATTERN.search(value)
+
+ if tag_search is None:
+ raise ValueError('Entry missing partition tag: %s' % value)
+
+ partition_tag = tag_search.group(1)
+
+ if partition_tag in partition_set:
+ if key in merged_dict:
+ if OPTIONS.allow_duplicate_apkapex_keys:
+ # TODO(b/150582573) Always raise on duplicates.
+ logger.warning('Duplicate key %s' % key)
+ continue
+ else:
+ raise ValueError('Duplicate key %s' % key)
+
+ merged_dict[key] = value
+
+ # Prioritize framework keys first.
+ # Duplicate keys from vendor are an error, or ignored.
+ filter_into_merged_dict(framework_dict, OPTIONS.framework_partition_set)
+ filter_into_merged_dict(vendor_dict, OPTIONS.vendor_partition_set)
+
+ # The following code is similar to write_sorted_data, but different enough
+ # that we couldn't use that function. We need the output to be sorted by the
+ # basename of the apex/apk (without the ".apex" or ".apk" suffix). This
+ # allows the sort to be consistent with the framework/vendor input data and
+ # eases comparison of input data with merged data.
+ with open(os.path.join(merged_meta_dir, file_name), 'w') as output:
+ for key, value in sorted(merged_dict.items()):
+ output.write(value + '\n')
+
+
+def create_file_contexts_copies(framework_meta_dir, vendor_meta_dir,
+ merged_meta_dir):
+ """Creates named copies of each partial build's file_contexts.bin.
+
+ Used when regenerating images from the partial build.
+ """
+
+ def copy_fc_file(source_dir, file_name):
+ for name in (file_name, 'file_contexts.bin'):
+ fc_path = os.path.join(source_dir, name)
+ if os.path.exists(fc_path):
+ shutil.copyfile(fc_path, os.path.join(merged_meta_dir, file_name))
+ return
+ raise ValueError('Missing file_contexts file from %s: %s', source_dir,
+ file_name)
+
+ copy_fc_file(framework_meta_dir, 'framework_file_contexts.bin')
+ copy_fc_file(vendor_meta_dir, 'vendor_file_contexts.bin')
+
+ # Replace <image>_selinux_fc values with framework or vendor file_contexts.bin
+ # depending on which dictionary the key came from.
+ # Only the file basename is required because all selinux_fc properties are
+ # replaced with the full path to the file under META/ when misc_info.txt is
+ # loaded from target files for repacking. See common.py LoadInfoDict().
+ for key in OPTIONS.vendor_misc_info:
+ if key.endswith('_selinux_fc'):
+ OPTIONS.merged_misc_info[key] = 'vendor_file_contexts.bin'
+ for key in OPTIONS.framework_misc_info:
+ if key.endswith('_selinux_fc'):
+ OPTIONS.merged_misc_info[key] = 'framework_file_contexts.bin'
+
+
+def compile_split_sepolicy(target_files_dir, partition_map):
+ """Uses secilc to compile a split sepolicy file.
+
+ Depends on various */etc/selinux/* and */etc/vintf/* files within partitions.
+
+ Args:
+ target_files_dir: Extracted directory of target_files, containing partition
+ directories.
+ partition_map: A map of partition name -> relative path within
+ target_files_dir.
+
+ Returns:
+ A command list that can be executed to create the compiled sepolicy.
+ """
+
+ def get_file(partition, path):
+ if partition not in partition_map:
+ logger.warning('Cannot load SEPolicy files for missing partition %s',
+ partition)
+ return None
+ return os.path.join(target_files_dir, partition_map[partition], path)
+
+ # Load the kernel sepolicy version from the FCM. This is normally provided
+ # directly to selinux.cpp as a build flag, but is also available in this file.
+ fcm_file = get_file('system', 'etc/vintf/compatibility_matrix.device.xml')
+ if not fcm_file or not os.path.exists(fcm_file):
+ raise ExternalError('Missing required file for loading sepolicy: %s', fcm)
+ kernel_sepolicy_version = ElementTree.parse(fcm_file).getroot().find(
+ 'sepolicy/kernel-sepolicy-version').text
+
+ # Load the vendor's plat sepolicy version. This is the version used for
+ # locating sepolicy mapping files.
+ vendor_plat_version_file = get_file('vendor',
+ 'etc/selinux/plat_sepolicy_vers.txt')
+ if not vendor_plat_version_file or not os.path.exists(
+ vendor_plat_version_file):
+ raise ExternalError('Missing required sepolicy file %s',
+ vendor_plat_version_file)
+ with open(vendor_plat_version_file) as f:
+ vendor_plat_version = f.read().strip()
+
+ # Use the same flags and arguments as selinux.cpp OpenSplitPolicy().
+ cmd = ['secilc', '-m', '-M', 'true', '-G', '-N']
+ cmd.extend(['-c', kernel_sepolicy_version])
+ cmd.extend(['-o', os.path.join(target_files_dir, 'META/combined_sepolicy')])
+ cmd.extend(['-f', '/dev/null'])
+
+ required_policy_files = (
+ ('system', 'etc/selinux/plat_sepolicy.cil'),
+ ('system', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
+ ('vendor', 'etc/selinux/vendor_sepolicy.cil'),
+ ('vendor', 'etc/selinux/plat_pub_versioned.cil'),
+ )
+ for policy in (map(lambda partition_and_path: get_file(*partition_and_path),
+ required_policy_files)):
+ if not policy or not os.path.exists(policy):
+ raise ExternalError('Missing required sepolicy file %s', policy)
+ cmd.append(policy)
+
+ optional_policy_files = (
+ ('system', 'etc/selinux/mapping/%s.compat.cil' % vendor_plat_version),
+ ('system_ext', 'etc/selinux/system_ext_sepolicy.cil'),
+ ('system_ext', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
+ ('product', 'etc/selinux/product_sepolicy.cil'),
+ ('product', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
+ ('odm', 'etc/selinux/odm_sepolicy.cil'),
+ )
+ for policy in (map(lambda partition_and_path: get_file(*partition_and_path),
+ optional_policy_files)):
+ if policy and os.path.exists(policy):
+ cmd.append(policy)
+
+ return cmd
+
+
+def validate_merged_apex_info(target_files_dir, partitions):
+ """Validates the APEX files in the merged target files directory.
+
+ Checks the APEX files in all possible preinstalled APEX directories.
+ Depends on the <partition>/apex/* APEX files within partitions.
+
+ Args:
+ target_files_dir: Extracted directory of target_files, containing partition
+ directories.
+ partitions: A list of all the partitions in the output directory.
+
+ Raises:
+ RuntimeError: if apex_utils fails to parse any APEX file.
+ ExternalError: if the same APEX package is provided by multiple partitions.
+ """
+ apex_packages = set()
+
+ apex_partitions = ('system', 'system_ext', 'product', 'vendor', 'odm')
+ for partition in filter(lambda p: p in apex_partitions, partitions):
+ apex_info = apex_utils.GetApexInfoFromTargetFiles(
+ target_files_dir, partition, compressed_only=False)
+ partition_apex_packages = set([info.package_name for info in apex_info])
+ duplicates = apex_packages.intersection(partition_apex_packages)
+ if duplicates:
+ raise ExternalError(
+ 'Duplicate APEX packages found in multiple partitions: %s' %
+ ' '.join(duplicates))
+ apex_packages.update(partition_apex_packages)
+
+
+def update_care_map_image_size_props(images_dir):
+ """Sets <partition>_image_size props in misc_info.
+
+ add_images_to_target_files uses these props to generate META/care_map.pb.
+ Regenerated images will have this property set during regeneration.
+
+ However, images copied directly from input partial target files packages
+ need this value calculated here.
+ """
+ for partition in common.PARTITIONS_WITH_CARE_MAP:
+ image_path = os.path.join(images_dir, '{}.img'.format(partition))
+ if os.path.exists(image_path):
+ partition_size = sparse_img.GetImagePartitionSize(image_path)
+ image_props = build_image.ImagePropFromGlobalDict(
+ OPTIONS.merged_misc_info, partition)
+ verity_image_builder = verity_utils.CreateVerityImageBuilder(image_props)
+ image_size = verity_image_builder.CalculateMaxImageSize(partition_size)
+ OPTIONS.merged_misc_info['{}_image_size'.format(partition)] = image_size
+
+
+def merge_meta_files(temp_dir, merged_dir):
+ """Merges various files in META/*."""
+
+ framework_meta_dir = os.path.join(temp_dir, 'framework_meta', 'META')
+ extract_items(
+ input_zip=OPTIONS.framework_target_files,
+ output_dir=os.path.dirname(framework_meta_dir),
+ extract_item_list=('META/*',))
+
+ vendor_meta_dir = os.path.join(temp_dir, 'vendor_meta', 'META')
+ extract_items(
+ input_zip=OPTIONS.vendor_target_files,
+ output_dir=os.path.dirname(vendor_meta_dir),
+ extract_item_list=('META/*',))
+
+ merged_meta_dir = os.path.join(merged_dir, 'META')
+
+ # Merge META/misc_info.txt into OPTIONS.merged_misc_info,
+ # but do not write it yet. The following functions may further
+ # modify this dict.
+ OPTIONS.merged_misc_info = merge_misc_info_txt(
+ framework_meta_dir=framework_meta_dir,
+ vendor_meta_dir=vendor_meta_dir,
+ merged_meta_dir=merged_meta_dir)
+
+ create_file_contexts_copies(
+ framework_meta_dir=framework_meta_dir,
+ vendor_meta_dir=vendor_meta_dir,
+ merged_meta_dir=merged_meta_dir)
+
+ if OPTIONS.merged_misc_info.get('use_dynamic_partitions') == 'true':
+ merge_dynamic_partitions_info_txt(
+ framework_meta_dir=framework_meta_dir,
+ vendor_meta_dir=vendor_meta_dir,
+ merged_meta_dir=merged_meta_dir)
+
+ if OPTIONS.merged_misc_info.get('ab_update') == 'true':
+ merge_ab_partitions_txt(
+ framework_meta_dir=framework_meta_dir,
+ vendor_meta_dir=vendor_meta_dir,
+ merged_meta_dir=merged_meta_dir)
+ update_care_map_image_size_props(images_dir=os.path.join(merged_dir, 'IMAGES'))
+
+ for file_name in ('apkcerts.txt', 'apexkeys.txt'):
+ merge_package_keys_txt(
+ framework_meta_dir=framework_meta_dir,
+ vendor_meta_dir=vendor_meta_dir,
+ merged_meta_dir=merged_meta_dir,
+ file_name=file_name)
+
+ # Write the now-finalized OPTIONS.merged_misc_info.
+ write_sorted_data(
+ data=OPTIONS.merged_misc_info,
+ path=os.path.join(merged_meta_dir, 'misc_info.txt'))
+
+
+def process_dexopt(temp_dir, output_target_files_dir):
+ """If needed, generates dexopt files for vendor apps.
+
+ Args:
+ temp_dir: Location containing an 'output' directory where target files have
+ been extracted, e.g. <temp_dir>/output/SYSTEM, <temp_dir>/output/IMAGES,
+ etc.
+ output_target_files_dir: The name of a directory that will be used to create
+ the output target files package after all the special cases are processed.
+ """
+ # Load vendor and framework META/misc_info.txt.
+ if (OPTIONS.vendor_misc_info.get('building_with_vsdk') != 'true' or
+ OPTIONS.framework_dexpreopt_tools is None or
+ OPTIONS.framework_dexpreopt_config is None or
+ OPTIONS.vendor_dexpreopt_config is None):
+ return
+
+ logger.info('applying dexpreopt')
+
+ # The directory structure to apply dexpreopt is:
+ #
+ # <temp_dir>/
+ # framework_meta/
+ # META/
+ # vendor_meta/
+ # META/
+ # output/
+ # SYSTEM/
+ # VENDOR/
+ # IMAGES/
+ # <other items extracted from system and vendor target files>
+ # tools/
+ # <contents of dexpreopt_tools.zip>
+ # system_config/
+ # <contents of system dexpreopt_config.zip>
+ # vendor_config/
+ # <contents of vendor dexpreopt_config.zip>
+ # system -> output/SYSTEM
+ # vendor -> output/VENDOR
+ # apex -> output/SYSTEM/apex (only for flattened APEX builds)
+ # apex/ (extracted updatable APEX)
+ # <apex 1>/
+ # ...
+ # <apex 2>/
+ # ...
+ # ...
+ # out/dex2oat_result/vendor/
+ # <app>
+ # oat/arm64/
+ # package.vdex
+ # package.odex
+ # <priv-app>
+ # oat/arm64/
+ # package.vdex
+ # package.odex
+ dexpreopt_tools_files_temp_dir = os.path.join(temp_dir, 'tools')
+ dexpreopt_framework_config_files_temp_dir = os.path.join(
+ temp_dir, 'system_config')
+ dexpreopt_vendor_config_files_temp_dir = os.path.join(temp_dir,
+ 'vendor_config')
+
+ extract_items(
+ input_zip=OPTIONS.framework_dexpreopt_tools,
+ output_dir=dexpreopt_tools_files_temp_dir,
+ extract_item_list=('*',))
+ extract_items(
+ input_zip=OPTIONS.framework_dexpreopt_config,
+ output_dir=dexpreopt_framework_config_files_temp_dir,
+ extract_item_list=('*',))
+ extract_items(
+ input_zip=OPTIONS.vendor_dexpreopt_config,
+ output_dir=dexpreopt_vendor_config_files_temp_dir,
+ extract_item_list=('*',))
+
+ os.symlink(
+ os.path.join(output_target_files_dir, 'SYSTEM'),
+ os.path.join(temp_dir, 'system'))
+ os.symlink(
+ os.path.join(output_target_files_dir, 'VENDOR'),
+ os.path.join(temp_dir, 'vendor'))
+
+ # The directory structure for flatteded APEXes is:
+ #
+ # SYSTEM
+ # apex
+ # <APEX name, e.g., com.android.wifi>
+ # apex_manifest.pb
+ # apex_pubkey
+ # etc/
+ # javalib/
+ # lib/
+ # lib64/
+ # priv-app/
+ #
+ # The directory structure for updatable APEXes is:
+ #
+ # SYSTEM
+ # apex
+ # com.android.adbd.apex
+ # com.android.appsearch.apex
+ # com.android.art.apex
+ # ...
+ apex_root = os.path.join(output_target_files_dir, 'SYSTEM', 'apex')
+
+ # Check for flattended versus updatable APEX.
+ if OPTIONS.framework_misc_info.get('target_flatten_apex') == 'false':
+ # Extract APEX.
+ logging.info('extracting APEX')
+
+ apex_extract_root_dir = os.path.join(temp_dir, 'apex')
+ os.makedirs(apex_extract_root_dir)
+
+ for apex in (glob.glob(os.path.join(apex_root, '*.apex')) +
+ glob.glob(os.path.join(apex_root, '*.capex'))):
+ logging.info(' apex: %s', apex)
+ # deapexer is in the same directory as the merge_target_files binary extracted
+ # from otatools.zip.
+ apex_json_info = subprocess.check_output(['deapexer', 'info', apex])
+ logging.info(' info: %s', apex_json_info)
+ apex_info = json.loads(apex_json_info)
+ apex_name = apex_info['name']
+ logging.info(' name: %s', apex_name)
+
+ apex_extract_dir = os.path.join(apex_extract_root_dir, apex_name)
+ os.makedirs(apex_extract_dir)
+
+ # deapexer uses debugfs_static, which is part of otatools.zip.
+ command = [
+ 'deapexer',
+ '--debugfs_path',
+ 'debugfs_static',
+ 'extract',
+ apex,
+ apex_extract_dir,
+ ]
+ logging.info(' running %s', command)
+ subprocess.check_call(command)
+ else:
+ # Flattened APEXes don't need to be extracted since they have the necessary
+ # directory structure.
+ os.symlink(os.path.join(apex_root), os.path.join(temp_dir, 'apex'))
+
+ # Modify system config to point to the tools that have been extracted.
+ # Absolute or .. paths are not allowed by the dexpreopt_gen tool in
+ # dexpreopt_soong.config.
+ dexpreopt_framework_soon_config = os.path.join(
+ dexpreopt_framework_config_files_temp_dir, 'dexpreopt_soong.config')
+ with open(dexpreopt_framework_soon_config, 'w') as f:
+ dexpreopt_soong_config = {
+ 'Profman': 'tools/profman',
+ 'Dex2oat': 'tools/dex2oatd',
+ 'Aapt': 'tools/aapt2',
+ 'SoongZip': 'tools/soong_zip',
+ 'Zip2zip': 'tools/zip2zip',
+ 'ManifestCheck': 'tools/manifest_check',
+ 'ConstructContext': 'tools/construct_context',
+ }
+ json.dump(dexpreopt_soong_config, f)
+
+ # TODO(b/188179859): Make *dex location configurable to vendor or system_other.
+ use_system_other_odex = False
+
+ if use_system_other_odex:
+ dex_img = 'SYSTEM_OTHER'
+ else:
+ dex_img = 'VENDOR'
+ # Open vendor_filesystem_config to append the items generated by dexopt.
+ vendor_file_system_config = open(
+ os.path.join(temp_dir, 'output', 'META',
+ 'vendor_filesystem_config.txt'), 'a')
+
+ # Dexpreopt vendor apps.
+ dexpreopt_config_suffix = '_dexpreopt.config'
+ for config in glob.glob(
+ os.path.join(dexpreopt_vendor_config_files_temp_dir,
+ '*' + dexpreopt_config_suffix)):
+ app = os.path.basename(config)[:-len(dexpreopt_config_suffix)]
+ logging.info('dexpreopt config: %s %s', config, app)
+
+ apk_dir = 'app'
+ apk_path = os.path.join(temp_dir, 'vendor', apk_dir, app, app + '.apk')
+ if not os.path.exists(apk_path):
+ apk_dir = 'priv-app'
+ apk_path = os.path.join(temp_dir, 'vendor', apk_dir, app, app + '.apk')
+ if not os.path.exists(apk_path):
+ logging.warning(
+ 'skipping dexpreopt for %s, no apk found in vendor/app '
+ 'or vendor/priv-app', app)
+ continue
+
+ # Generate dexpreopting script. Note 'out_dir' is not the output directory
+ # where the script is generated, but the OUT_DIR at build time referenced
+ # in the dexpreot config files, e.g., "out/.../core-oj.jar", so the tool knows
+ # how to adjust the path.
+ command = [
+ os.path.join(dexpreopt_tools_files_temp_dir, 'dexpreopt_gen'),
+ '-global',
+ os.path.join(dexpreopt_framework_config_files_temp_dir,
+ 'dexpreopt.config'),
+ '-global_soong',
+ os.path.join(dexpreopt_framework_config_files_temp_dir,
+ 'dexpreopt_soong.config'),
+ '-module',
+ config,
+ '-dexpreopt_script',
+ 'dexpreopt_app.sh',
+ '-out_dir',
+ 'out',
+ '-base_path',
+ '.',
+ '--uses_target_files',
+ ]
+
+ # Run the command from temp_dir so all tool paths are its descendants.
+ logging.info('running %s', command)
+ subprocess.check_call(command, cwd=temp_dir)
+
+ # Call the generated script.
+ command = ['sh', 'dexpreopt_app.sh', apk_path]
+ logging.info('running %s', command)
+ subprocess.check_call(command, cwd=temp_dir)
+
+ # Output files are in:
+ #
+ # <temp_dir>/out/dex2oat_result/vendor/priv-app/<app>/oat/arm64/package.vdex
+ # <temp_dir>/out/dex2oat_result/vendor/priv-app/<app>/oat/arm64/package.odex
+ # <temp_dir>/out/dex2oat_result/vendor/app/<app>/oat/arm64/package.vdex
+ # <temp_dir>/out/dex2oat_result/vendor/app/<app>/oat/arm64/package.odex
+ #
+ # Copy the files to their destination. The structure of system_other is:
+ #
+ # system_other/
+ # system-other-odex-marker
+ # system/
+ # app/
+ # <app>/oat/arm64/
+ # <app>.odex
+ # <app>.vdex
+ # ...
+ # priv-app/
+ # <app>/oat/arm64/
+ # <app>.odex
+ # <app>.vdex
+ # ...
+
+ # TODO(b/188179859): Support for other architectures.
+ arch = 'arm64'
+
+ dex_destination = os.path.join(temp_dir, 'output', dex_img, apk_dir, app,
+ 'oat', arch)
+ os.makedirs(dex_destination)
+ dex2oat_path = os.path.join(temp_dir, 'out', 'dex2oat_result', 'vendor',
+ apk_dir, app, 'oat', arch)
+ shutil.copy(
+ os.path.join(dex2oat_path, 'package.vdex'),
+ os.path.join(dex_destination, app + '.vdex'))
+ shutil.copy(
+ os.path.join(dex2oat_path, 'package.odex'),
+ os.path.join(dex_destination, app + '.odex'))
+
+ # Append entries to vendor_file_system_config.txt, such as:
+ #
+ # vendor/app/<app>/oat 0 2000 755 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
+ # vendor/app/<app>/oat/arm64 0 2000 755 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
+ # vendor/app/<app>/oat/arm64/<app>.odex 0 0 644 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
+ # vendor/app/<app>/oat/arm64/<app>.vdex 0 0 644 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
+ if not use_system_other_odex:
+ vendor_app_prefix = 'vendor/' + apk_dir + '/' + app + '/oat'
+ selabel = 'selabel=u:object_r:vendor_app_file:s0 capabilities=0x0'
+ vendor_file_system_config.writelines([
+ vendor_app_prefix + ' 0 2000 755 ' + selabel + '\n',
+ vendor_app_prefix + '/' + arch + ' 0 2000 755 ' + selabel + '\n',
+ vendor_app_prefix + '/' + arch + '/' + app + '.odex 0 0 644 ' +
+ selabel + '\n',
+ vendor_app_prefix + '/' + arch + '/' + app + '.vdex 0 0 644 ' +
+ selabel + '\n',
+ ])
+
+ if not use_system_other_odex:
+ vendor_file_system_config.close()
+ # Delete vendor.img so that it will be regenerated.
+ # TODO(b/188179859): Rebuilding a vendor image in GRF mode (e.g., T(framework)
+ # and S(vendor) may require logic similar to that in
+ # rebuild_image_with_sepolicy.
+ vendor_img = os.path.join(output_target_files_dir, 'IMAGES', 'vendor.img')
+ if os.path.exists(vendor_img):
+ logging.info('Deleting %s', vendor_img)
+ os.remove(vendor_img)
+
+
+def create_merged_package(temp_dir):
+ """Merges two target files packages into one target files structure.
+
+ Returns:
+ Path to merged package under temp directory.
+ """
+ # Extract "as is" items from the input framework and vendor partial target
+ # files packages directly into the output temporary directory, since these items
+ # do not need special case processing.
+
+ output_target_files_temp_dir = os.path.join(temp_dir, 'output')
+ extract_items(
+ input_zip=OPTIONS.framework_target_files,
+ output_dir=output_target_files_temp_dir,
+ extract_item_list=OPTIONS.framework_item_list)
+ extract_items(
+ input_zip=OPTIONS.vendor_target_files,
+ output_dir=output_target_files_temp_dir,
+ extract_item_list=OPTIONS.vendor_item_list)
+
+ # Perform special case processing on META/* items.
+ # After this function completes successfully, all the files we need to create
+ # the output target files package are in place.
+ merge_meta_files(temp_dir=temp_dir, merged_dir=output_target_files_temp_dir)
+
+ process_dexopt(
+ temp_dir=temp_dir, output_target_files_dir=output_target_files_temp_dir)
+
+ return output_target_files_temp_dir
+
+
+def generate_missing_images(target_files_dir):
+ """Generate any missing images from target files."""
+
+ # Regenerate IMAGES in the target directory.
+
+ add_img_args = [
+ '--verbose',
+ '--add_missing',
+ ]
+ if OPTIONS.rebuild_recovery:
+ add_img_args.append('--rebuild_recovery')
+ add_img_args.append(target_files_dir)
+
+ add_img_to_target_files.main(add_img_args)
+
+
+def rebuild_image_with_sepolicy(target_files_dir):
+ """Rebuilds odm.img or vendor.img to include merged sepolicy files.
+
+ If odm is present then odm is preferred -- otherwise vendor is used.
+ """
+ partition = 'vendor'
+ if os.path.exists(os.path.join(target_files_dir, 'ODM')) or os.path.exists(
+ os.path.join(target_files_dir, 'IMAGES/odm.img')):
+ partition = 'odm'
+ partition_img = '{}.img'.format(partition)
+ partition_map = '{}.map'.format(partition)
+
+ logger.info('Recompiling %s using the merged sepolicy files.', partition_img)
+
+ # Copy the combined SEPolicy file and framework hashes to the image that is
+ # being rebuilt.
+ def copy_selinux_file(input_path, output_filename):
+ input_filename = os.path.join(target_files_dir, input_path)
+ if not os.path.exists(input_filename):
+ input_filename = input_filename.replace('SYSTEM_EXT/', 'SYSTEM/system_ext/') \
+ .replace('PRODUCT/', 'SYSTEM/product/')
+ if not os.path.exists(input_filename):
+ logger.info('Skipping copy_selinux_file for %s', input_filename)
+ return
+ shutil.copy(
+ input_filename,
+ os.path.join(target_files_dir, partition.upper(), 'etc/selinux',
+ output_filename))
+
+ copy_selinux_file('META/combined_sepolicy', 'precompiled_sepolicy')
+ copy_selinux_file('SYSTEM/etc/selinux/plat_sepolicy_and_mapping.sha256',
+ 'precompiled_sepolicy.plat_sepolicy_and_mapping.sha256')
+ copy_selinux_file(
+ 'SYSTEM_EXT/etc/selinux/system_ext_sepolicy_and_mapping.sha256',
+ 'precompiled_sepolicy.system_ext_sepolicy_and_mapping.sha256')
+ copy_selinux_file('PRODUCT/etc/selinux/product_sepolicy_and_mapping.sha256',
+ 'precompiled_sepolicy.product_sepolicy_and_mapping.sha256')
+
+ if not OPTIONS.vendor_otatools:
+ # Remove the partition from the merged target-files archive. It will be
+ # rebuilt later automatically by generate_missing_images().
+ os.remove(os.path.join(target_files_dir, 'IMAGES', partition_img))
+ return
+
+ # TODO(b/192253131): Remove the need for vendor_otatools by fixing
+ # backwards-compatibility issues when compiling images across releases.
+ if not OPTIONS.vendor_target_files:
+ raise ValueError(
+ 'Expected vendor_target_files if vendor_otatools is not None.')
+ logger.info(
+ '%s recompilation will be performed using the vendor otatools.zip',
+ partition_img)
+
+ # Unzip the vendor build's otatools.zip and target-files archive.
+ vendor_otatools_dir = common.MakeTempDir(
+ prefix='merge_target_files_vendor_otatools_')
+ vendor_target_files_dir = common.MakeTempDir(
+ prefix='merge_target_files_vendor_target_files_')
+ common.UnzipToDir(OPTIONS.vendor_otatools, vendor_otatools_dir)
+ common.UnzipToDir(OPTIONS.vendor_target_files, vendor_target_files_dir)
+
+ # Copy the partition contents from the merged target-files archive to the
+ # vendor target-files archive.
+ shutil.rmtree(os.path.join(vendor_target_files_dir, partition.upper()))
+ shutil.copytree(
+ os.path.join(target_files_dir, partition.upper()),
+ os.path.join(vendor_target_files_dir, partition.upper()),
+ symlinks=True)
+
+ # Delete then rebuild the partition.
+ os.remove(os.path.join(vendor_target_files_dir, 'IMAGES', partition_img))
+ rebuild_partition_command = [
+ os.path.join(vendor_otatools_dir, 'bin', 'add_img_to_target_files'),
+ '--verbose',
+ '--add_missing',
+ ]
+ if OPTIONS.rebuild_recovery:
+ rebuild_partition_command.append('--rebuild_recovery')
+ rebuild_partition_command.append(vendor_target_files_dir)
+ logger.info('Recompiling %s: %s', partition_img,
+ ' '.join(rebuild_partition_command))
+ common.RunAndCheckOutput(rebuild_partition_command, verbose=True)
+
+ # Move the newly-created image to the merged target files dir.
+ if not os.path.exists(os.path.join(target_files_dir, 'IMAGES')):
+ os.makedirs(os.path.join(target_files_dir, 'IMAGES'))
+ shutil.move(
+ os.path.join(vendor_target_files_dir, 'IMAGES', partition_img),
+ os.path.join(target_files_dir, 'IMAGES', partition_img))
+ shutil.move(
+ os.path.join(vendor_target_files_dir, 'IMAGES', partition_map),
+ os.path.join(target_files_dir, 'IMAGES', partition_map))
+
+ def copy_recovery_file(filename):
+ for subdir in ('VENDOR', 'SYSTEM/vendor'):
+ source = os.path.join(vendor_target_files_dir, subdir, filename)
+ if os.path.exists(source):
+ dest = os.path.join(target_files_dir, subdir, filename)
+ shutil.copy(source, dest)
+ return
+ logger.info('Skipping copy_recovery_file for %s, file not found', filename)
+
+ if OPTIONS.rebuild_recovery:
+ copy_recovery_file('etc/recovery.img')
+ copy_recovery_file('bin/install-recovery.sh')
+ copy_recovery_file('recovery-from-boot.p')
+
+
+def generate_super_empty_image(target_dir, output_super_empty):
+ """Generates super_empty image from target package.
+
+ Args:
+ target_dir: Path to the target file package which contains misc_info.txt for
+ detailed information for super image.
+ output_super_empty: If provided, copies a super_empty.img file from the
+ target files package to this path.
+ """
+ # Create super_empty.img using the merged misc_info.txt.
+
+ misc_info_txt = os.path.join(target_dir, 'META', 'misc_info.txt')
+
+ use_dynamic_partitions = common.LoadDictionaryFromFile(misc_info_txt).get(
+ 'use_dynamic_partitions')
+
+ if use_dynamic_partitions != 'true' and output_super_empty:
+ raise ValueError(
+ 'Building super_empty.img requires use_dynamic_partitions=true.')
+ elif use_dynamic_partitions == 'true':
+ super_empty_img = os.path.join(target_dir, 'IMAGES', 'super_empty.img')
+ build_super_image_args = [
+ misc_info_txt,
+ super_empty_img,
+ ]
+ build_super_image.main(build_super_image_args)
+
+ # Copy super_empty.img to the user-provided output_super_empty location.
+ if output_super_empty:
+ shutil.copyfile(super_empty_img, output_super_empty)
+
+
+def create_target_files_archive(output_zip, source_dir, temp_dir):
+ """Creates a target_files zip archive from the input source dir.
+
+ Args:
+ output_zip: The name of the zip archive target files package.
+ source_dir: The target directory contains package to be archived.
+ temp_dir: Path to temporary directory for any intermediate files.
+ """
+ output_target_files_list = os.path.join(temp_dir, 'output.list')
+ output_target_files_meta_dir = os.path.join(source_dir, 'META')
+
+ def files_from_path(target_path, extra_args=None):
+ """Gets files under the given path and return a sorted list."""
+ find_command = ['find', target_path] + (extra_args or [])
+ find_process = common.Run(
+ find_command, stdout=subprocess.PIPE, verbose=False)
+ return common.RunAndCheckOutput(['sort'],
+ stdin=find_process.stdout,
+ verbose=False)
+
+ # META content appears first in the zip. This is done by the
+ # standard build system for optimized extraction of those files,
+ # so we do the same step for merged target_files.zips here too.
+ meta_content = files_from_path(output_target_files_meta_dir)
+ other_content = files_from_path(
+ source_dir,
+ ['-path', output_target_files_meta_dir, '-prune', '-o', '-print'])
+
+ with open(output_target_files_list, 'w') as f:
+ f.write(meta_content)
+ f.write(other_content)
+
+ command = [
+ 'soong_zip',
+ '-d',
+ '-o',
+ os.path.abspath(output_zip),
+ '-C',
+ source_dir,
+ '-r',
+ output_target_files_list,
+ ]
+
+ logger.info('creating %s', output_zip)
+ common.RunAndCheckOutput(command, verbose=True)
+ logger.info('finished creating %s', output_zip)
+
+
+def merge_target_files(temp_dir):
+ """Merges two target files packages together.
+
+ This function uses framework and vendor target files packages as input,
+ performs various file extractions, special case processing, and finally
+ creates a merged zip archive as output.
+
+ Args:
+ temp_dir: The name of a directory we use when we extract items from the
+ input target files packages, and also a scratch directory that we use for
+ temporary files.
+ """
+
+ logger.info('starting: merge framework %s and vendor %s into output %s',
+ OPTIONS.framework_target_files, OPTIONS.vendor_target_files,
+ OPTIONS.output_target_files)
+
+ output_target_files_temp_dir = create_merged_package(temp_dir)
+
+ if not check_target_files_vintf.CheckVintf(output_target_files_temp_dir):
+ raise RuntimeError('Incompatible VINTF metadata')
+
+ partition_map = common.PartitionMapFromTargetFiles(
+ output_target_files_temp_dir)
+
+ # Generate and check for cross-partition violations of sharedUserId
+ # values in APKs. This requires the input target-files packages to contain
+ # *.apk files.
+ shareduid_violation_modules = os.path.join(
+ output_target_files_temp_dir, 'META', 'shareduid_violation_modules.json')
+ with open(shareduid_violation_modules, 'w') as f:
+ violation = find_shareduid_violation.FindShareduidViolation(
+ output_target_files_temp_dir, partition_map)
+
+ # Write the output to a file to enable debugging.
+ f.write(violation)
+
+ # Check for violations across the input builds' partition groups.
+ shareduid_errors = common.SharedUidPartitionViolations(
+ json.loads(violation),
+ [OPTIONS.framework_partition_set, OPTIONS.vendor_partition_set])
+ if shareduid_errors:
+ for error in shareduid_errors:
+ logger.error(error)
+ raise ValueError('sharedUserId APK error. See %s' %
+ shareduid_violation_modules)
+
+ # host_init_verifier and secilc check only the following partitions:
+ filtered_partitions = {
+ partition: path
+ for partition, path in partition_map.items()
+ if partition in ['system', 'system_ext', 'product', 'vendor', 'odm']
+ }
+
+ # Run host_init_verifier on the combined init rc files.
+ common.RunHostInitVerifier(
+ product_out=output_target_files_temp_dir,
+ partition_map=filtered_partitions)
+
+ # Check that the split sepolicy from the multiple builds can compile.
+ split_sepolicy_cmd = compile_split_sepolicy(output_target_files_temp_dir,
+ filtered_partitions)
+ logger.info('Compiling split sepolicy: %s', ' '.join(split_sepolicy_cmd))
+ common.RunAndCheckOutput(split_sepolicy_cmd)
+ # Include the compiled policy in an image if requested.
+ if OPTIONS.rebuild_sepolicy:
+ rebuild_image_with_sepolicy(output_target_files_temp_dir)
+
+ # Run validation checks on the pre-installed APEX files.
+ validate_merged_apex_info(output_target_files_temp_dir, partition_map.keys())
+
+ generate_missing_images(output_target_files_temp_dir)
+
+ generate_super_empty_image(output_target_files_temp_dir,
+ OPTIONS.output_super_empty)
+
+ # Finally, create the output target files zip archive and/or copy the
+ # output items to the output target files directory.
+
+ if OPTIONS.output_dir:
+ copy_items(output_target_files_temp_dir, OPTIONS.output_dir,
+ OPTIONS.output_item_list)
+
+ if not OPTIONS.output_target_files:
+ return
+
+ create_target_files_archive(OPTIONS.output_target_files,
+ output_target_files_temp_dir, temp_dir)
+
+ # Create the IMG package from the merged target files package.
+ if OPTIONS.output_img:
+ img_from_target_files.main(
+ [OPTIONS.output_target_files, OPTIONS.output_img])
+
+ # Create the OTA package from the merged target files package.
+
+ if OPTIONS.output_ota:
+ ota_from_target_files.main(
+ [OPTIONS.output_target_files, OPTIONS.output_ota])
+
+
+def call_func_with_temp_dir(func, keep_tmp):
+ """Manages the creation and cleanup of the temporary directory.
+
+ This function calls the given function after first creating a temporary
+ directory. It also cleans up the temporary directory.
+
+ Args:
+ func: The function to call. Should accept one parameter, the path to the
+ temporary directory.
+ keep_tmp: Keep the temporary directory after processing is complete.
+ """
+
+ # Create a temporary directory. This will serve as the parent of directories
+ # we use when we extract items from the input target files packages, and also
+ # a scratch directory that we use for temporary files.
+
+ temp_dir = common.MakeTempDir(prefix='merge_target_files_')
+
+ try:
+ func(temp_dir)
+ finally:
+ if keep_tmp:
+ logger.info('keeping %s', temp_dir)
+ else:
+ common.Cleanup()
+
+
+def main():
+ """The main function.
+
+ Process command line arguments, then call merge_target_files to
+ perform the heavy lifting.
+ """
+
+ common.InitLogging()
+
+ def option_handler(o, a):
+ if o == '--system-target-files':
+ logger.warning(
+ '--system-target-files has been renamed to --framework-target-files')
+ OPTIONS.framework_target_files = a
+ elif o == '--framework-target-files':
+ OPTIONS.framework_target_files = a
+ elif o == '--system-item-list':
+ logger.warning(
+ '--system-item-list has been renamed to --framework-item-list')
+ OPTIONS.framework_item_list = a
+ elif o == '--framework-item-list':
+ OPTIONS.framework_item_list = a
+ elif o == '--system-misc-info-keys':
+ logger.warning('--system-misc-info-keys has been renamed to '
+ '--framework-misc-info-keys')
+ OPTIONS.framework_misc_info_keys = a
+ elif o == '--framework-misc-info-keys':
+ OPTIONS.framework_misc_info_keys = a
+ elif o == '--other-target-files':
+ logger.warning(
+ '--other-target-files has been renamed to --vendor-target-files')
+ OPTIONS.vendor_target_files = a
+ elif o == '--vendor-target-files':
+ OPTIONS.vendor_target_files = a
+ elif o == '--other-item-list':
+ logger.warning('--other-item-list has been renamed to --vendor-item-list')
+ OPTIONS.vendor_item_list = a
+ elif o == '--vendor-item-list':
+ OPTIONS.vendor_item_list = a
+ elif o == '--output-target-files':
+ OPTIONS.output_target_files = a
+ elif o == '--output-dir':
+ OPTIONS.output_dir = a
+ elif o == '--output-item-list':
+ OPTIONS.output_item_list = a
+ elif o == '--output-ota':
+ OPTIONS.output_ota = a
+ elif o == '--output-img':
+ OPTIONS.output_img = a
+ elif o == '--output-super-empty':
+ OPTIONS.output_super_empty = a
+ elif o == '--rebuild_recovery':
+ OPTIONS.rebuild_recovery = True
+ elif o == '--allow-duplicate-apkapex-keys':
+ OPTIONS.allow_duplicate_apkapex_keys = True
+ elif o == '--vendor-otatools':
+ OPTIONS.vendor_otatools = a
+ elif o == '--rebuild-sepolicy':
+ OPTIONS.rebuild_sepolicy = True
+ elif o == '--keep-tmp':
+ OPTIONS.keep_tmp = True
+ elif o == '--framework-dexpreopt-config':
+ OPTIONS.framework_dexpreopt_config = a
+ elif o == '--framework-dexpreopt-tools':
+ OPTIONS.framework_dexpreopt_tools = a
+ elif o == '--vendor-dexpreopt-config':
+ OPTIONS.vendor_dexpreopt_config = a
+ else:
+ return False
+ return True
+
+ args = common.ParseOptions(
+ sys.argv[1:],
+ __doc__,
+ extra_long_opts=[
+ 'system-target-files=',
+ 'framework-target-files=',
+ 'system-item-list=',
+ 'framework-item-list=',
+ 'system-misc-info-keys=',
+ 'framework-misc-info-keys=',
+ 'other-target-files=',
+ 'vendor-target-files=',
+ 'other-item-list=',
+ 'vendor-item-list=',
+ 'output-target-files=',
+ 'output-dir=',
+ 'output-item-list=',
+ 'output-ota=',
+ 'output-img=',
+ 'output-super-empty=',
+ 'framework-dexpreopt-config=',
+ 'framework-dexpreopt-tools=',
+ 'vendor-dexpreopt-config=',
+ 'rebuild_recovery',
+ 'allow-duplicate-apkapex-keys',
+ 'vendor-otatools=',
+ 'rebuild-sepolicy',
+ 'keep-tmp',
+ ],
+ extra_option_handler=option_handler)
+
+ # pylint: disable=too-many-boolean-expressions
+ if (args or OPTIONS.framework_target_files is None or
+ OPTIONS.vendor_target_files is None or
+ (OPTIONS.output_target_files is None and OPTIONS.output_dir is None) or
+ (OPTIONS.output_dir is not None and OPTIONS.output_item_list is None) or
+ (OPTIONS.rebuild_recovery and not OPTIONS.rebuild_sepolicy)):
+ common.Usage(__doc__)
+ sys.exit(1)
+
+ if OPTIONS.framework_item_list:
+ OPTIONS.framework_item_list = common.LoadListFromFile(
+ OPTIONS.framework_item_list)
+ else:
+ OPTIONS.framework_item_list = DEFAULT_FRAMEWORK_ITEM_LIST
+ OPTIONS.framework_partition_set = item_list_to_partition_set(
+ OPTIONS.framework_item_list)
+
+ if OPTIONS.framework_misc_info_keys:
+ OPTIONS.framework_misc_info_keys = common.LoadListFromFile(
+ OPTIONS.framework_misc_info_keys)
+ else:
+ OPTIONS.framework_misc_info_keys = DEFAULT_FRAMEWORK_MISC_INFO_KEYS
+
+ if OPTIONS.vendor_item_list:
+ OPTIONS.vendor_item_list = common.LoadListFromFile(OPTIONS.vendor_item_list)
+ else:
+ OPTIONS.vendor_item_list = DEFAULT_VENDOR_ITEM_LIST
+ OPTIONS.vendor_partition_set = item_list_to_partition_set(
+ OPTIONS.vendor_item_list)
+
+ if OPTIONS.output_item_list:
+ OPTIONS.output_item_list = common.LoadListFromFile(OPTIONS.output_item_list)
+ else:
+ OPTIONS.output_item_list = None
+
+ if not validate_config_lists():
+ sys.exit(1)
+
+ call_func_with_temp_dir(lambda temp_dir: merge_target_files(temp_dir),
+ OPTIONS.keep_tmp)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/releasetools/merge/test_merge_target_files.py b/tools/releasetools/merge/test_merge_target_files.py
new file mode 100644
index 0000000..088ebee
--- /dev/null
+++ b/tools/releasetools/merge/test_merge_target_files.py
@@ -0,0 +1,288 @@
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os.path
+import shutil
+
+import common
+import merge_target_files
+import test_utils
+from merge_target_files import (
+ validate_config_lists, DEFAULT_FRAMEWORK_ITEM_LIST,
+ DEFAULT_VENDOR_ITEM_LIST, DEFAULT_FRAMEWORK_MISC_INFO_KEYS, copy_items,
+ item_list_to_partition_set, merge_package_keys_txt, compile_split_sepolicy,
+ validate_merged_apex_info)
+
+
+class MergeTargetFilesTest(test_utils.ReleaseToolsTestCase):
+
+ def setUp(self):
+ self.testdata_dir = test_utils.get_testdata_dir()
+ self.OPTIONS = merge_target_files.OPTIONS
+ self.OPTIONS.framework_item_list = DEFAULT_FRAMEWORK_ITEM_LIST
+ self.OPTIONS.framework_misc_info_keys = DEFAULT_FRAMEWORK_MISC_INFO_KEYS
+ self.OPTIONS.vendor_item_list = DEFAULT_VENDOR_ITEM_LIST
+ self.OPTIONS.framework_partition_set = set(
+ ['product', 'system', 'system_ext'])
+ self.OPTIONS.vendor_partition_set = set(['odm', 'vendor'])
+
+ def test_copy_items_CopiesItemsMatchingPatterns(self):
+
+ def createEmptyFile(path):
+ if not os.path.exists(os.path.dirname(path)):
+ os.makedirs(os.path.dirname(path))
+ open(path, 'a').close()
+ return path
+
+ def createSymLink(source, dest):
+ os.symlink(source, dest)
+ return dest
+
+ def getRelPaths(start, filepaths):
+ return set(
+ os.path.relpath(path=filepath, start=start) for filepath in filepaths)
+
+ input_dir = common.MakeTempDir()
+ output_dir = common.MakeTempDir()
+ expected_copied_items = []
+ actual_copied_items = []
+ patterns = ['*.cpp', 'subdir/*.txt']
+
+ # Create various files that we expect to get copied because they
+ # match one of the patterns.
+ expected_copied_items.extend([
+ createEmptyFile(os.path.join(input_dir, 'a.cpp')),
+ createEmptyFile(os.path.join(input_dir, 'b.cpp')),
+ createEmptyFile(os.path.join(input_dir, 'subdir', 'c.txt')),
+ createEmptyFile(os.path.join(input_dir, 'subdir', 'd.txt')),
+ createEmptyFile(
+ os.path.join(input_dir, 'subdir', 'subsubdir', 'e.txt')),
+ createSymLink('a.cpp', os.path.join(input_dir, 'a_link.cpp')),
+ ])
+ # Create some more files that we expect to not get copied.
+ createEmptyFile(os.path.join(input_dir, 'a.h'))
+ createEmptyFile(os.path.join(input_dir, 'b.h'))
+ createEmptyFile(os.path.join(input_dir, 'subdir', 'subsubdir', 'f.gif'))
+ createSymLink('a.h', os.path.join(input_dir, 'a_link.h'))
+
+ # Copy items.
+ copy_items(input_dir, output_dir, patterns)
+
+ # Assert the actual copied items match the ones we expected.
+ for dirpath, _, filenames in os.walk(output_dir):
+ actual_copied_items.extend(
+ os.path.join(dirpath, filename) for filename in filenames)
+ self.assertEqual(
+ getRelPaths(output_dir, actual_copied_items),
+ getRelPaths(input_dir, expected_copied_items))
+ self.assertEqual(
+ os.readlink(os.path.join(output_dir, 'a_link.cpp')), 'a.cpp')
+
+ def test_validate_config_lists_ReturnsFalseIfMissingDefaultItem(self):
+ self.OPTIONS.framework_item_list = list(DEFAULT_FRAMEWORK_ITEM_LIST)
+ self.OPTIONS.framework_item_list.remove('SYSTEM/*')
+ self.assertFalse(validate_config_lists())
+
+ def test_validate_config_lists_ReturnsTrueIfDefaultItemInDifferentList(self):
+ self.OPTIONS.framework_item_list = list(DEFAULT_FRAMEWORK_ITEM_LIST)
+ self.OPTIONS.framework_item_list.remove('ROOT/*')
+ self.OPTIONS.vendor_item_list = list(DEFAULT_VENDOR_ITEM_LIST)
+ self.OPTIONS.vendor_item_list.append('ROOT/*')
+ self.assertTrue(validate_config_lists())
+
+ def test_validate_config_lists_ReturnsTrueIfExtraItem(self):
+ self.OPTIONS.framework_item_list = list(DEFAULT_FRAMEWORK_ITEM_LIST)
+ self.OPTIONS.framework_item_list.append('MY_NEW_PARTITION/*')
+ self.assertTrue(validate_config_lists())
+
+ def test_validate_config_lists_ReturnsFalseIfSharedExtractedPartition(self):
+ self.OPTIONS.vendor_item_list = list(DEFAULT_VENDOR_ITEM_LIST)
+ self.OPTIONS.vendor_item_list.append('SYSTEM/my_system_file')
+ self.assertFalse(validate_config_lists())
+
+ def test_validate_config_lists_ReturnsFalseIfSharedExtractedPartitionImage(
+ self):
+ self.OPTIONS.vendor_item_list = list(DEFAULT_VENDOR_ITEM_LIST)
+ self.OPTIONS.vendor_item_list.append('IMAGES/system.img')
+ self.assertFalse(validate_config_lists())
+
+ def test_validate_config_lists_ReturnsFalseIfBadSystemMiscInfoKeys(self):
+ for bad_key in ['dynamic_partition_list', 'super_partition_groups']:
+ self.OPTIONS.framework_misc_info_keys = list(
+ DEFAULT_FRAMEWORK_MISC_INFO_KEYS)
+ self.OPTIONS.framework_misc_info_keys.append(bad_key)
+ self.assertFalse(validate_config_lists())
+
+ def test_merge_package_keys_txt_ReturnsTrueIfNoConflicts(self):
+ output_meta_dir = common.MakeTempDir()
+
+ framework_meta_dir = common.MakeTempDir()
+ os.symlink(
+ os.path.join(self.testdata_dir, 'apexkeys_framework.txt'),
+ os.path.join(framework_meta_dir, 'apexkeys.txt'))
+
+ vendor_meta_dir = common.MakeTempDir()
+ os.symlink(
+ os.path.join(self.testdata_dir, 'apexkeys_vendor.txt'),
+ os.path.join(vendor_meta_dir, 'apexkeys.txt'))
+
+ merge_package_keys_txt(framework_meta_dir, vendor_meta_dir, output_meta_dir,
+ 'apexkeys.txt')
+
+ merged_entries = []
+ merged_path = os.path.join(self.testdata_dir, 'apexkeys_merge.txt')
+
+ with open(merged_path) as f:
+ merged_entries = f.read().split('\n')
+
+ output_entries = []
+ output_path = os.path.join(output_meta_dir, 'apexkeys.txt')
+
+ with open(output_path) as f:
+ output_entries = f.read().split('\n')
+
+ return self.assertEqual(merged_entries, output_entries)
+
+ def test_process_apex_keys_apk_certs_ReturnsFalseIfConflictsPresent(self):
+ output_meta_dir = common.MakeTempDir()
+
+ framework_meta_dir = common.MakeTempDir()
+ os.symlink(
+ os.path.join(self.testdata_dir, 'apexkeys_framework.txt'),
+ os.path.join(framework_meta_dir, 'apexkeys.txt'))
+
+ conflict_meta_dir = common.MakeTempDir()
+ os.symlink(
+ os.path.join(self.testdata_dir, 'apexkeys_framework_conflict.txt'),
+ os.path.join(conflict_meta_dir, 'apexkeys.txt'))
+
+ self.assertRaises(ValueError, merge_package_keys_txt, framework_meta_dir,
+ conflict_meta_dir, output_meta_dir, 'apexkeys.txt')
+
+ def test_process_apex_keys_apk_certs_HandlesApkCertsSyntax(self):
+ output_meta_dir = common.MakeTempDir()
+
+ framework_meta_dir = common.MakeTempDir()
+ os.symlink(
+ os.path.join(self.testdata_dir, 'apkcerts_framework.txt'),
+ os.path.join(framework_meta_dir, 'apkcerts.txt'))
+
+ vendor_meta_dir = common.MakeTempDir()
+ os.symlink(
+ os.path.join(self.testdata_dir, 'apkcerts_vendor.txt'),
+ os.path.join(vendor_meta_dir, 'apkcerts.txt'))
+
+ merge_package_keys_txt(framework_meta_dir, vendor_meta_dir, output_meta_dir,
+ 'apkcerts.txt')
+
+ merged_entries = []
+ merged_path = os.path.join(self.testdata_dir, 'apkcerts_merge.txt')
+
+ with open(merged_path) as f:
+ merged_entries = f.read().split('\n')
+
+ output_entries = []
+ output_path = os.path.join(output_meta_dir, 'apkcerts.txt')
+
+ with open(output_path) as f:
+ output_entries = f.read().split('\n')
+
+ return self.assertEqual(merged_entries, output_entries)
+
+ def test_item_list_to_partition_set(self):
+ item_list = [
+ 'META/apexkeys.txt',
+ 'META/apkcerts.txt',
+ 'META/filesystem_config.txt',
+ 'PRODUCT/*',
+ 'SYSTEM/*',
+ 'SYSTEM_EXT/*',
+ ]
+ partition_set = item_list_to_partition_set(item_list)
+ self.assertEqual(set(['product', 'system', 'system_ext']), partition_set)
+
+ def test_compile_split_sepolicy(self):
+ product_out_dir = common.MakeTempDir()
+
+ def write_temp_file(path, data=''):
+ full_path = os.path.join(product_out_dir, path)
+ if not os.path.exists(os.path.dirname(full_path)):
+ os.makedirs(os.path.dirname(full_path))
+ with open(full_path, 'w') as f:
+ f.write(data)
+
+ write_temp_file(
+ 'system/etc/vintf/compatibility_matrix.device.xml', """
+ <compatibility-matrix>
+ <sepolicy>
+ <kernel-sepolicy-version>30</kernel-sepolicy-version>
+ </sepolicy>
+ </compatibility-matrix>""")
+ write_temp_file('vendor/etc/selinux/plat_sepolicy_vers.txt', '30.0')
+
+ write_temp_file('system/etc/selinux/plat_sepolicy.cil')
+ write_temp_file('system/etc/selinux/mapping/30.0.cil')
+ write_temp_file('product/etc/selinux/mapping/30.0.cil')
+ write_temp_file('vendor/etc/selinux/vendor_sepolicy.cil')
+ write_temp_file('vendor/etc/selinux/plat_pub_versioned.cil')
+
+ cmd = compile_split_sepolicy(product_out_dir, {
+ 'system': 'system',
+ 'product': 'product',
+ 'vendor': 'vendor',
+ })
+ self.assertEqual(' '.join(cmd),
+ ('secilc -m -M true -G -N -c 30 '
+ '-o {OTP}/META/combined_sepolicy -f /dev/null '
+ '{OTP}/system/etc/selinux/plat_sepolicy.cil '
+ '{OTP}/system/etc/selinux/mapping/30.0.cil '
+ '{OTP}/vendor/etc/selinux/vendor_sepolicy.cil '
+ '{OTP}/vendor/etc/selinux/plat_pub_versioned.cil '
+ '{OTP}/product/etc/selinux/mapping/30.0.cil').format(
+ OTP=product_out_dir))
+
+ def _copy_apex(self, source, output_dir, partition):
+ shutil.copy(
+ source,
+ os.path.join(output_dir, partition, 'apex', os.path.basename(source)))
+
+ @test_utils.SkipIfExternalToolsUnavailable()
+ def test_validate_merged_apex_info(self):
+ output_dir = common.MakeTempDir()
+ os.makedirs(os.path.join(output_dir, 'SYSTEM/apex'))
+ os.makedirs(os.path.join(output_dir, 'VENDOR/apex'))
+
+ self._copy_apex(
+ os.path.join(self.testdata_dir, 'has_apk.apex'), output_dir, 'SYSTEM')
+ self._copy_apex(
+ os.path.join(test_utils.get_current_dir(),
+ 'com.android.apex.compressed.v1.capex'), output_dir,
+ 'VENDOR')
+ validate_merged_apex_info(output_dir, ('system', 'vendor'))
+
+ @test_utils.SkipIfExternalToolsUnavailable()
+ def test_validate_merged_apex_info_RaisesOnPackageInMultiplePartitions(self):
+ output_dir = common.MakeTempDir()
+ os.makedirs(os.path.join(output_dir, 'SYSTEM/apex'))
+ os.makedirs(os.path.join(output_dir, 'VENDOR/apex'))
+
+ same_apex_package = os.path.join(self.testdata_dir, 'has_apk.apex')
+ self._copy_apex(same_apex_package, output_dir, 'SYSTEM')
+ self._copy_apex(same_apex_package, output_dir, 'VENDOR')
+ self.assertRaisesRegexp(
+ common.ExternalError,
+ 'Duplicate APEX packages found in multiple partitions: com.android.wifi',
+ validate_merged_apex_info, output_dir, ('system', 'vendor'))