Merge "aconfig: create flag value file" into main
diff --git a/ci/build_test_suites b/ci/build_test_suites
new file mode 100755
index 0000000..861065a
--- /dev/null
+++ b/ci/build_test_suites
@@ -0,0 +1,22 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import build_test_suites
+
+if __name__ == '__main__':
+ sys.dont_write_bytecode = True
+
+ build_test_suites.main(sys.argv)
diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py
new file mode 100644
index 0000000..e88b420
--- /dev/null
+++ b/ci/build_test_suites.py
@@ -0,0 +1,282 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script to build only the necessary modules for general-tests along
+
+with whatever other targets are passed in.
+"""
+
+import argparse
+from collections.abc import Sequence
+import json
+import os
+import pathlib
+import re
+import subprocess
+import sys
+from typing import Any, Dict, Set, Text
+
+import test_mapping_module_retriever
+
+
+# List of modules that are always required to be in general-tests.zip
+REQUIRED_MODULES = frozenset(
+ ['cts-tradefed', 'vts-tradefed', 'compatibility-host-util', 'soong_zip']
+)
+
+
+def build_test_suites(argv):
+ args = parse_args(argv)
+
+ if not args.change_info:
+ build_everything(args)
+ return
+
+ # Call the class to map changed files to modules to build.
+ # TODO(lucafarsi): Move this into a replaceable class.
+ build_affected_modules(args)
+
+
+def parse_args(argv):
+ argparser = argparse.ArgumentParser()
+ argparser.add_argument(
+ 'extra_targets', nargs='*', help='Extra test suites to build.'
+ )
+ argparser.add_argument('--target_product')
+ argparser.add_argument('--target_release')
+ argparser.add_argument(
+ '--with_dexpreopt_boot_img_and_system_server_only', action='store_true'
+ )
+ argparser.add_argument('--dist_dir')
+ argparser.add_argument('--change_info', nargs='?')
+ argparser.add_argument('--extra_required_modules', nargs='*')
+
+ return argparser.parse_args()
+
+
+def build_everything(args: argparse.Namespace):
+ build_command = base_build_command(args)
+ build_command.append('general-tests')
+
+ run_command(build_command)
+
+
+def build_affected_modules(args: argparse.Namespace):
+ modules_to_build = find_modules_to_build(
+ pathlib.Path(args.change_info), args.extra_required_modules
+ )
+
+ # Call the build command with everything.
+ build_command = base_build_command(args)
+ build_command.extend(modules_to_build)
+
+ run_command(build_command)
+
+ zip_build_outputs(modules_to_build, args.dist_dir)
+
+
+def base_build_command(args: argparse.Namespace) -> list:
+ build_command = []
+ build_command.append('time')
+ build_command.append('./build/soong/soong_ui.bash')
+ build_command.append('--make-mode')
+ build_command.append('dist')
+ build_command.append('DIST_DIR=' + args.dist_dir)
+ build_command.append('TARGET_PRODUCT=' + args.target_product)
+ build_command.append('TARGET_RELEASE=' + args.target_release)
+ build_command.extend(args.extra_targets)
+
+ return build_command
+
+
+def run_command(args: list[str]) -> str:
+ result = subprocess.run(
+ args=args,
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ # If the process failed, print its stdout and propagate the exception.
+ if not result.returncode == 0:
+ print('Build command failed! output:')
+ print('stdout: ' + result.stdout)
+ print('stderr: ' + result.stderr)
+
+ result.check_returncode()
+ return result.stdout
+
+
+def find_modules_to_build(
+ change_info: pathlib.Path, extra_required_modules: list[Text]
+) -> Set[Text]:
+ changed_files = find_changed_files(change_info)
+
+ test_mappings = test_mapping_module_retriever.GetTestMappings(
+ changed_files, set()
+ )
+
+ # Soong_zip is required to generate the output zip so always build it.
+ modules_to_build = set(REQUIRED_MODULES)
+ if extra_required_modules:
+ modules_to_build.update(extra_required_modules)
+
+ modules_to_build.update(find_affected_modules(test_mappings, changed_files))
+
+ return modules_to_build
+
+
+def find_changed_files(change_info: pathlib.Path) -> Set[Text]:
+ with open(change_info) as change_info_file:
+ change_info_contents = json.load(change_info_file)
+
+ changed_files = set()
+
+ for change in change_info_contents['changes']:
+ project_path = change.get('projectPath') + '/'
+
+ for revision in change.get('revisions'):
+ for file_info in revision.get('fileInfos'):
+ changed_files.add(project_path + file_info.get('path'))
+
+ return changed_files
+
+
+def find_affected_modules(
+ test_mappings: Dict[str, Any], changed_files: Set[Text]
+) -> Set[Text]:
+ modules = set()
+
+ # The test_mappings object returned by GetTestMappings is organized as
+ # follows:
+ # {
+ # 'test_mapping_file_path': {
+ # 'group_name' : [
+ # 'name': 'module_name',
+ # ],
+ # }
+ # }
+ for test_mapping in test_mappings.values():
+ for group in test_mapping.values():
+ for entry in group:
+ module_name = entry.get('name', None)
+
+ if not module_name:
+ continue
+
+ file_patterns = entry.get('file_patterns')
+ if not file_patterns:
+ modules.add(module_name)
+ continue
+
+ if matches_file_patterns(file_patterns, changed_files):
+ modules.add(module_name)
+ continue
+
+ return modules
+
+
+# TODO(lucafarsi): Share this logic with the original logic in
+# test_mapping_test_retriever.py
+def matches_file_patterns(
+ file_patterns: list[Text], changed_files: Set[Text]
+) -> bool:
+ for changed_file in changed_files:
+ for pattern in file_patterns:
+ if re.search(pattern, changed_file):
+ return True
+
+ return False
+
+
+def zip_build_outputs(modules_to_build: Set[Text], dist_dir: Text):
+ src_top = os.environ.get('TOP', os.getcwd())
+
+ # Call dumpvars to get the necessary things.
+ # TODO(lucafarsi): Don't call soong_ui 4 times for this, --dumpvars-mode can
+ # do it but it requires parsing.
+ host_out_testcases = get_soong_var('HOST_OUT_TESTCASES')
+ target_out_testcases = get_soong_var('TARGET_OUT_TESTCASES')
+ product_out = get_soong_var('PRODUCT_OUT')
+ soong_host_out = get_soong_var('SOONG_HOST_OUT')
+ host_out = get_soong_var('HOST_OUT')
+
+ # Call the class to package the outputs.
+ # TODO(lucafarsi): Move this code into a replaceable class.
+ host_paths = []
+ target_paths = []
+ for module in modules_to_build:
+ host_path = os.path.join(host_out_testcases, module)
+ if os.path.exists(host_path):
+ host_paths.append(host_path)
+
+ target_path = os.path.join(target_out_testcases, module)
+ if os.path.exists(target_path):
+ target_paths.append(target_path)
+
+ zip_command = ['time', os.path.join(host_out, 'bin', 'soong_zip')]
+
+ # Add host testcases.
+ zip_command.append('-C')
+ zip_command.append(os.path.join(src_top, soong_host_out))
+ zip_command.append('-P')
+ zip_command.append('host/')
+ for path in host_paths:
+ zip_command.append('-D')
+ zip_command.append(path)
+
+ # Add target testcases.
+ zip_command.append('-C')
+ zip_command.append(os.path.join(src_top, product_out))
+ zip_command.append('-P')
+ zip_command.append('target')
+ for path in target_paths:
+ zip_command.append('-D')
+ zip_command.append(path)
+
+ # TODO(lucafarsi): Push this logic into a general-tests-minimal build command
+ # Add necessary tools. These are also hardcoded in general-tests.mk.
+ framework_path = os.path.join(soong_host_out, 'framework')
+
+ zip_command.append('-C')
+ zip_command.append(framework_path)
+ zip_command.append('-P')
+ zip_command.append('host/tools')
+ zip_command.append('-f')
+ zip_command.append(os.path.join(framework_path, 'cts-tradefed.jar'))
+ zip_command.append('-f')
+ zip_command.append(
+ os.path.join(framework_path, 'compatibility-host-util.jar')
+ )
+ zip_command.append('-f')
+ zip_command.append(os.path.join(framework_path, 'vts-tradefed.jar'))
+
+ # Zip to the DIST dir.
+ zip_command.append('-o')
+ zip_command.append(os.path.join(dist_dir, 'general-tests.zip'))
+
+ run_command(zip_command)
+
+
+def get_soong_var(var: str) -> str:
+ value = run_command(
+ ['./build/soong/soong_ui.bash', '--dumpvar-mode', '--abs', var]
+ ).strip()
+ if not value:
+ raise RuntimeError('Necessary soong variable ' + var + ' not found.')
+
+ return value
+
+
+def main(argv):
+ build_test_suites(sys.argv)
diff --git a/ci/test_mapping_module_retriever.py b/ci/test_mapping_module_retriever.py
new file mode 100644
index 0000000..d2c13c0
--- /dev/null
+++ b/ci/test_mapping_module_retriever.py
@@ -0,0 +1,125 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Simple parsing code to scan test_mapping files and determine which
+modules are needed to build for the given list of changed files.
+TODO(lucafarsi): Deduplicate from artifact_helper.py
+"""
+
+from typing import Any, Dict, Set, Text
+import json
+import os
+import re
+
+# Regex to extra test name from the path of test config file.
+TEST_NAME_REGEX = r'(?:^|.*/)([^/]+)\.config'
+
+# Key name for TEST_MAPPING imports
+KEY_IMPORTS = 'imports'
+KEY_IMPORT_PATH = 'path'
+
+# Name of TEST_MAPPING file.
+TEST_MAPPING = 'TEST_MAPPING'
+
+# Pattern used to identify double-quoted strings and '//'-format comments in
+# TEST_MAPPING file, but only double-quoted strings are included within the
+# matching group.
+_COMMENTS_RE = re.compile(r'(\"(?:[^\"\\]|\\.)*\"|(?=//))(?://.*)?')
+
+
+def FilterComments(test_mapping_file: Text) -> Text:
+ """Remove comments in TEST_MAPPING file to valid format.
+
+ Only '//' is regarded as comments.
+
+ Args:
+ test_mapping_file: Path to a TEST_MAPPING file.
+
+ Returns:
+ Valid json string without comments.
+ """
+ return re.sub(_COMMENTS_RE, r'\1', test_mapping_file)
+
+def GetTestMappings(paths: Set[Text],
+ checked_paths: Set[Text]) -> Dict[Text, Dict[Text, Any]]:
+ """Get the affected TEST_MAPPING files.
+
+ TEST_MAPPING files in source code are packaged into a build artifact
+ `test_mappings.zip`. Inside the zip file, the path of each TEST_MAPPING file
+ is preserved. From all TEST_MAPPING files in the source code, this method
+ locates the affected TEST_MAPPING files based on the given paths list.
+
+ A TEST_MAPPING file may also contain `imports` that import TEST_MAPPING files
+ from a different location, e.g.,
+ "imports": [
+ {
+ "path": "../folder2"
+ }
+ ]
+ In that example, TEST_MAPPING files inside ../folder2 (relative to the
+ TEST_MAPPING file containing that imports section) and its parent directories
+ will also be included.
+
+ Args:
+ paths: A set of paths with related TEST_MAPPING files for given changes.
+ checked_paths: A set of paths that have been checked for TEST_MAPPING file
+ already. The set is updated after processing each TEST_MAPPING file. It's
+ used to prevent infinite loop when the method is called recursively.
+
+ Returns:
+ A dictionary of Test Mapping containing the content of the affected
+ TEST_MAPPING files, indexed by the path containing the TEST_MAPPING file.
+ """
+ test_mappings = {}
+
+ # Search for TEST_MAPPING files in each modified path and its parent
+ # directories.
+ all_paths = set()
+ for path in paths:
+ dir_names = path.split(os.path.sep)
+ all_paths |= set(
+ [os.path.sep.join(dir_names[:i + 1]) for i in range(len(dir_names))])
+ # Add root directory to the paths to search for TEST_MAPPING file.
+ all_paths.add('')
+
+ all_paths.difference_update(checked_paths)
+ checked_paths |= all_paths
+ # Try to load TEST_MAPPING file in each possible path.
+ for path in all_paths:
+ try:
+ test_mapping_file = os.path.join(os.path.join(os.getcwd(), path), 'TEST_MAPPING')
+ # Read content of TEST_MAPPING file.
+ content = FilterComments(open(test_mapping_file, "r").read())
+ test_mapping = json.loads(content)
+ test_mappings[path] = test_mapping
+
+ import_paths = set()
+ for import_detail in test_mapping.get(KEY_IMPORTS, []):
+ import_path = import_detail[KEY_IMPORT_PATH]
+ # Try the import path as absolute path.
+ import_paths.add(import_path)
+ # Try the import path as relative path based on the test mapping file
+ # containing the import.
+ norm_import_path = os.path.normpath(os.path.join(path, import_path))
+ import_paths.add(norm_import_path)
+ import_paths.difference_update(checked_paths)
+ if import_paths:
+ import_test_mappings = GetTestMappings(import_paths, checked_paths)
+ test_mappings.update(import_test_mappings)
+ except (KeyError, FileNotFoundError, NotADirectoryError):
+ # TEST_MAPPING file doesn't exist in path
+ pass
+
+ return test_mappings
diff --git a/core/main.mk b/core/main.mk
index 348a964..649c75c 100644
--- a/core/main.mk
+++ b/core/main.mk
@@ -1721,10 +1721,8 @@
# dist_files only for putting your library into the dist directory with a full build.
.PHONY: dist_files
-ifeq ($(SOONG_COLLECT_JAVA_DEPS), true)
- $(call dist-for-goals, dist_files, $(SOONG_OUT_DIR)/module_bp_java_deps.json)
- $(call dist-for-goals, dist_files, $(PRODUCT_OUT)/module-info.json)
-endif
+$(call dist-for-goals, dist_files, $(SOONG_OUT_DIR)/module_bp_java_deps.json)
+$(call dist-for-goals, dist_files, $(PRODUCT_OUT)/module-info.json)
.PHONY: apps_only
ifeq ($(HOST_OS),darwin)
diff --git a/tools/aconfig/printflags/src/main.rs b/tools/aconfig/printflags/src/main.rs
index 4110317..ae9b83a 100644
--- a/tools/aconfig/printflags/src/main.rs
+++ b/tools/aconfig/printflags/src/main.rs
@@ -20,6 +20,7 @@
use aconfig_protos::aconfig::Parsed_flags as ProtoParsedFlags;
use anyhow::{bail, Context, Result};
use regex::Regex;
+use std::collections::BTreeMap;
use std::collections::HashMap;
use std::process::Command;
use std::{fs, str};
@@ -66,7 +67,7 @@
let device_config_flags = parse_device_config(dc_stdout);
// read aconfig_flags.pb files
- let mut flags: HashMap<String, Vec<String>> = HashMap::new();
+ let mut flags: BTreeMap<String, Vec<String>> = BTreeMap::new();
for partition in ["system", "system_ext", "product", "vendor"] {
let path = format!("/{}/etc/aconfig_flags.pb", partition);
let Ok(bytes) = fs::read(&path) else {
@@ -86,11 +87,10 @@
// print flags
for (key, mut value) in flags {
- let (_, package_and_name) = key.split_once('/').unwrap();
if let Some(dc_value) = device_config_flags.get(&key) {
value.push(dc_value.to_string());
}
- println!("{}: {}", package_and_name, value.join(", "));
+ println!("{}: {}", key, value.join(", "));
}
Ok(())
diff --git a/tools/aconfig/src/main.rs b/tools/aconfig/src/main.rs
index 6c4e241..7d719f0 100644
--- a/tools/aconfig/src/main.rs
+++ b/tools/aconfig/src/main.rs
@@ -135,7 +135,7 @@
.required(true)
.help("The target container for the generated storage file."),
)
- .arg(Arg::new("cache").long("cache").required(true))
+ .arg(Arg::new("cache").long("cache").action(ArgAction::Append).required(true))
.arg(Arg::new("out").long("out").required(true)),
)
}