Merge "Don't build targets if they're not used." into main
diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py
index ac5023f..75dd9f2 100644
--- a/ci/build_test_suites.py
+++ b/ci/build_test_suites.py
@@ -20,6 +20,7 @@
import logging
import os
import pathlib
+import re
import subprocess
import sys
from typing import Callable
@@ -52,6 +53,15 @@
any output zip files needed by the build.
"""
+ _DOWNLOAD_OPTS = {
+ 'test-config-only-zip',
+ 'test-zip-file-filter',
+ 'extra-host-shared-lib-zip',
+ 'sandbox-tests-zips',
+ 'additional-files-filter',
+ 'cts-package-name',
+ }
+
def __init__(
self,
build_context: dict[str, any],
@@ -64,12 +74,19 @@
def create_build_plan(self):
- if 'optimized_build' not in self.build_context['enabled_build_features']:
+ if 'optimized_build' not in self.build_context.get(
+ 'enabledBuildFeatures', []
+ ):
return BuildPlan(set(self.args.extra_targets), set())
build_targets = set()
packaging_functions = set()
for target in self.args.extra_targets:
+ if self._unused_target_exclusion_enabled(
+ target
+ ) and not self._build_target_used(target):
+ continue
+
target_optimizer_getter = self.target_optimizations.get(target, None)
if not target_optimizer_getter:
build_targets.add(target)
@@ -83,6 +100,60 @@
return BuildPlan(build_targets, packaging_functions)
+ def _unused_target_exclusion_enabled(self, target: str) -> bool:
+ return f'{target}_unused_exclusion' in self.build_context.get(
+ 'enabledBuildFeatures', []
+ )
+
+ def _build_target_used(self, target: str) -> bool:
+ """Determines whether this target's outputs are used by the test configurations listed in the build context."""
+ file_download_regexes = self._aggregate_file_download_regexes()
+ # For all of a targets' outputs, check if any of the regexes used by tests
+ # to download artifacts would match it. If any of them do then this target
+ # is necessary.
+ for artifact in self._get_target_potential_outputs(target):
+ for regex in file_download_regexes:
+ if re.match(regex, artifact):
+ return True
+ return False
+
+ def _get_target_potential_outputs(self, target: str) -> set[str]:
+ tests_suffix = '-tests'
+ if target.endswith('tests'):
+ tests_suffix = ''
+ # This is a list of all the potential zips output by the test suite targets.
+ # If the test downloads artifacts from any of these zips, we will be
+ # conservative and avoid skipping the tests.
+ return {
+ f'{target}.zip',
+ f'android-{target}.zip',
+ f'android-{target}-verifier.zip',
+ f'{target}{tests_suffix}_list.zip',
+ f'android-{target}{tests_suffix}_list.zip',
+ f'{target}{tests_suffix}_host-shared-libs.zip',
+ f'android-{target}{tests_suffix}_host-shared-libs.zip',
+ f'{target}{tests_suffix}_configs.zip',
+ f'android-{target}{tests_suffix}_configs.zip',
+ }
+
+ def _aggregate_file_download_regexes(self) -> set[re.Pattern]:
+ """Lists out all test config options to specify targets to download.
+
+ These come in the form of regexes.
+ """
+ all_regexes = set()
+ for test_info in self._get_test_infos():
+ for opt in test_info.get('extraOptions', []):
+ # check the known list of options for downloading files.
+ if opt.get('key') in self._DOWNLOAD_OPTS:
+ all_regexes.update(
+ re.compile(value) for value in opt.get('values', [])
+ )
+ return all_regexes
+
+ def _get_test_infos(self):
+ return self.build_context.get('testContext', dict()).get('testInfos', [])
+
@dataclass(frozen=True)
class BuildPlan:
@@ -155,7 +226,7 @@
def empty_build_context():
- return {'enabled_build_features': []}
+ return {'enabledBuildFeatures': []}
def execute_build_plan(build_plan: BuildPlan):
diff --git a/ci/build_test_suites_test.py b/ci/build_test_suites_test.py
index a9ff3fb..7efb154 100644
--- a/ci/build_test_suites_test.py
+++ b/ci/build_test_suites_test.py
@@ -324,6 +324,55 @@
self.run_packaging_functions(build_plan),
)
+ def test_target_output_used_target_built(self):
+ build_target = 'test_target'
+ build_planner = self.create_build_planner(
+ build_targets={build_target},
+ build_context=self.create_build_context(
+ test_context=self.get_test_context(build_target),
+ enabled_build_features={'test_target_unused_exclusion'},
+ ),
+ )
+
+ build_plan = build_planner.create_build_plan()
+
+ self.assertSetEqual(build_plan.build_targets, {build_target})
+
+ def test_target_regex_used_target_built(self):
+ build_target = 'test_target'
+ test_context = self.get_test_context(build_target)
+ test_context['testInfos'][0]['extraOptions'] = [{
+ 'key': 'additional-files-filter',
+ 'values': [f'.*{build_target}.*\.zip'],
+ }]
+ build_planner = self.create_build_planner(
+ build_targets={build_target},
+ build_context=self.create_build_context(
+ test_context=test_context,
+ enabled_build_features={'test_target_unused_exclusion'},
+ ),
+ )
+
+ build_plan = build_planner.create_build_plan()
+
+ self.assertSetEqual(build_plan.build_targets, {build_target})
+
+ def test_target_output_not_used_target_not_built(self):
+ build_target = 'test_target'
+ test_context = self.get_test_context(build_target)
+ test_context['testInfos'][0]['extraOptions'] = []
+ build_planner = self.create_build_planner(
+ build_targets={build_target},
+ build_context=self.create_build_context(
+ test_context=test_context,
+ enabled_build_features={'test_target_unused_exclusion'},
+ ),
+ )
+
+ build_plan = build_planner.create_build_plan()
+
+ self.assertSetEqual(build_plan.build_targets, set())
+
def create_build_planner(
self,
build_targets: set[str],
@@ -352,10 +401,10 @@
test_context: dict[str, any] = {},
) -> dict[str, any]:
build_context = {}
- build_context['enabled_build_features'] = enabled_build_features
+ build_context['enabledBuildFeatures'] = enabled_build_features
if optimized_build_enabled:
- build_context['enabled_build_features'].add('optimized_build')
- build_context['test_context'] = test_context
+ build_context['enabledBuildFeatures'].add('optimized_build')
+ build_context['testContext'] = test_context
return build_context
def create_args(
@@ -398,6 +447,25 @@
return output
+ def get_test_context(self, target: str):
+ return {
+ 'testInfos': [
+ {
+ 'name': 'atp_test',
+ 'target': 'test_target',
+ 'branch': 'branch',
+ 'extraOptions': [{
+ 'key': 'additional-files-filter',
+ 'values': [f'{target}.zip'],
+ }],
+ 'command': '/tf/command',
+ 'extraBuildTargets': [
+ 'extra_build_target',
+ ],
+ },
+ ],
+ }
+
def wait_until(
condition_function: Callable[[], bool],
diff --git a/ci/optimized_targets.py b/ci/optimized_targets.py
index 224c8c0..6321719 100644
--- a/ci/optimized_targets.py
+++ b/ci/optimized_targets.py
@@ -53,7 +53,7 @@
def get_target_optimizer(target, enabled_flag, build_context, optimizer):
- if enabled_flag in build_context['enabled_build_features']:
+ if enabled_flag in build_context['enabledBuildFeatures']:
return optimizer
return NullOptimizer(target)