Implement get_build_targets_impl in GeneralTestsOptimizer
Implement functionality in GeneralTestsOptimizer to find what targets to
build.
This logic is fairly complex and involves checking for test
configs that download general-tests.zip. Then the configs are checked to
see if they're proper test mapping tests (if they use the
'test-mapping-test-group' option). If they are, then TEST_MAPPING
modules are scanned to see if the list of changed files would cause any
test mapping modules to run. The tests are then further filtered by
test-mapping-test-groups used in the test configs.
In case that a test uses general-tests.zip but does not specify
'test-mapping-test-group' then all bets are off and general-tests.zip
is built in its entirety.
package_outputs is still unimplemented so this will need to be
implemented before the optimization can be enabled.
Test: atest build_test_suites_test && atest optimized_targets_test
Bug: 358215235
Change-Id: I6a7eebfd1b06b380799292eb2019ac17c9af5367
diff --git a/ci/optimized_targets.py b/ci/optimized_targets.py
index 116d6f8..fddde17 100644
--- a/ci/optimized_targets.py
+++ b/ci/optimized_targets.py
@@ -16,8 +16,13 @@
from abc import ABC
import argparse
import functools
-from typing import Self
from build_context import BuildContext
+import json
+import logging
+import os
+from typing import Self
+
+import test_mapping_module_retriever
class OptimizedBuildTarget(ABC):
@@ -41,7 +46,10 @@
def get_build_targets(self) -> set[str]:
features = self.build_context.enabled_build_features
if self.get_enabled_flag() in features:
- return self.get_build_targets_impl()
+ self.modules_to_build = self.get_build_targets_impl()
+ return self.modules_to_build
+
+ self.modules_to_build = {self.target}
return {self.target}
def package_outputs(self):
@@ -82,6 +90,30 @@
pass
+class ChangeInfo:
+
+ def __init__(self, change_info_file_path):
+ try:
+ with open(change_info_file_path) as change_info_file:
+ change_info_contents = json.load(change_info_file)
+ except json.decoder.JSONDecodeError:
+ logging.error(f'Failed to load CHANGE_INFO: {change_info_file_path}')
+ raise
+
+ self._change_info_contents = change_info_contents
+
+ def find_changed_files(self) -> set[str]:
+ changed_files = set()
+
+ for change in self._change_info_contents['changes']:
+ project_path = change.get('projectPath') + '/'
+
+ for revision in change.get('revisions'):
+ for file_info in revision.get('fileInfos'):
+ changed_files.add(project_path + file_info.get('path'))
+
+ return changed_files
+
class GeneralTestsOptimizer(OptimizedBuildTarget):
"""general-tests optimizer
@@ -94,8 +126,55 @@
normally built.
"""
+ # List of modules that are always required to be in general-tests.zip.
+ _REQUIRED_MODULES = frozenset(
+ ['cts-tradefed', 'vts-tradefed', 'compatibility-host-util']
+ )
+
+ def get_build_targets_impl(self) -> set[str]:
+ change_info_file_path = os.environ.get('CHANGE_INFO')
+ if not change_info_file_path:
+ logging.info(
+ 'No CHANGE_INFO env var found, general-tests optimization disabled.'
+ )
+ return {'general-tests'}
+
+ test_infos = self.build_context.test_infos
+ test_mapping_test_groups = set()
+ for test_info in test_infos:
+ is_test_mapping = test_info.is_test_mapping
+ current_test_mapping_test_groups = test_info.test_mapping_test_groups
+ uses_general_tests = test_info.build_target_used('general-tests')
+
+ if uses_general_tests and not is_test_mapping:
+ logging.info(
+ 'Test uses general-tests.zip but is not test-mapping, general-tests'
+ ' optimization disabled.'
+ )
+ return {'general-tests'}
+
+ if is_test_mapping:
+ test_mapping_test_groups.update(current_test_mapping_test_groups)
+
+ change_info = ChangeInfo(change_info_file_path)
+ changed_files = change_info.find_changed_files()
+
+ test_mappings = test_mapping_module_retriever.GetTestMappings(
+ changed_files, set()
+ )
+
+ modules_to_build = set(self._REQUIRED_MODULES)
+
+ modules_to_build.update(
+ test_mapping_module_retriever.FindAffectedModules(
+ test_mappings, changed_files, test_mapping_test_groups
+ )
+ )
+
+ return modules_to_build
+
def get_enabled_flag(self):
- return 'general-tests-optimized'
+ return 'general_tests_optimized'
@classmethod
def get_optimized_targets(cls) -> dict[str, OptimizedBuildTarget]: