Merge "Return list of packages" into main
diff --git a/ci/build_device_and_tests b/ci/build_device_and_tests
index 9d11268..63d3ce3 100755
--- a/ci/build_device_and_tests
+++ b/ci/build_device_and_tests
@@ -13,6 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+set -euo pipefail
-build/soong/soong_ui.bash --make-mode build_test_suites || exit $?
-$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites $@ || exit $?
+build/soong/soong_ui.bash --make-mode build_test_suites
+$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites --device-build $@
diff --git a/ci/build_test_suites b/ci/build_test_suites
index 9d11268..74470a8 100755
--- a/ci/build_test_suites
+++ b/ci/build_test_suites
@@ -13,6 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+set -euo pipefail
-build/soong/soong_ui.bash --make-mode build_test_suites || exit $?
-$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites $@ || exit $?
+build/soong/soong_ui.bash --make-mode build_test_suites
+$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites $@
diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py
index 623d256..b67ecec 100644
--- a/ci/build_test_suites.py
+++ b/ci/build_test_suites.py
@@ -30,9 +30,10 @@
import test_discovery_agent
-REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP'])
+REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP', 'DIST_DIR'])
SOONG_UI_EXE_REL_PATH = 'build/soong/soong_ui.bash'
LOG_PATH = 'logs/build_test_suites.log'
+REQUIRED_BUILD_TARGETS = frozenset(['dist'])
class Error(Exception):
@@ -73,34 +74,46 @@
build_targets = set()
packaging_commands_getters = []
- test_discovery_zip_regexes = set()
- optimization_rationale = ''
- try:
- # Do not use these regexes for now, only run this to collect data on what
- # would be optimized.
- test_discovery_zip_regexes = self._get_test_discovery_zip_regexes()
- logging.info(f'Discovered test discovery regexes: {test_discovery_zip_regexes}')
- except test_discovery_agent.TestDiscoveryError as e:
- optimization_rationale = e.message
- logging.warning(f'Unable to perform test discovery: {optimization_rationale}')
- for target in self.args.extra_targets:
- if optimization_rationale:
- get_metrics_agent().report_unoptimized_target(target, optimization_rationale)
- else:
+ # In order to roll optimizations out differently between test suites and
+ # device builds, we have separate flags.
+ if (
+ 'test_suites_zip_test_discovery'
+ in self.build_context.enabled_build_features
+ and not self.args.device_build
+ ) or (
+ 'device_zip_test_discovery'
+ in self.build_context.enabled_build_features
+ and self.args.device_build
+ ):
+ preliminary_build_targets = self._collect_preliminary_build_targets()
+ else:
+ preliminary_build_targets = self._legacy_collect_preliminary_build_targets()
+
+ # Keep reporting metrics when test discovery is disabled.
+ # To be removed once test discovery is fully rolled out.
+ optimization_rationale = ''
+ test_discovery_zip_regexes = set()
+ try:
+ test_discovery_zip_regexes = self._get_test_discovery_zip_regexes()
+ logging.info(f'Discovered test discovery regexes: {test_discovery_zip_regexes}')
+ except test_discovery_agent.TestDiscoveryError as e:
+ optimization_rationale = e.message
+ logging.warning(f'Unable to perform test discovery: {optimization_rationale}')
+
+ for target in self.args.extra_targets:
+ if optimization_rationale:
+ get_metrics_agent().report_unoptimized_target(target, optimization_rationale)
+ continue
try:
- regex = r'\b(%s)\b' % re.escape(target)
+ regex = r'\b(%s.*)\b' % re.escape(target)
if any(re.search(regex, opt) for opt in test_discovery_zip_regexes):
get_metrics_agent().report_unoptimized_target(target, 'Test artifact used.')
- else:
- get_metrics_agent().report_optimized_target(target)
+ continue
+ get_metrics_agent().report_optimized_target(target)
except Exception as e:
logging.error(f'unable to parse test discovery output: {repr(e)}')
- if self._unused_target_exclusion_enabled(
- target
- ) and not self.build_context.build_target_used(target):
- continue
-
+ for target in preliminary_build_targets:
target_optimizer_getter = self.target_optimizations.get(target, None)
if not target_optimizer_getter:
build_targets.add(target)
@@ -116,6 +129,51 @@
return BuildPlan(build_targets, packaging_commands_getters)
+ def _collect_preliminary_build_targets(self):
+ build_targets = set()
+ try:
+ test_discovery_zip_regexes = self._get_test_discovery_zip_regexes()
+ logging.info(f'Discovered test discovery regexes: {test_discovery_zip_regexes}')
+ except test_discovery_agent.TestDiscoveryError as e:
+ optimization_rationale = e.message
+ logging.warning(f'Unable to perform test discovery: {optimization_rationale}')
+
+ for target in self.args.extra_targets:
+ get_metrics_agent().report_unoptimized_target(target, optimization_rationale)
+ return self._legacy_collect_preliminary_build_targets()
+
+ for target in self.args.extra_targets:
+ if target in REQUIRED_BUILD_TARGETS:
+ build_targets.add(target)
+ continue
+
+ regex = r'\b(%s.*)\b' % re.escape(target)
+ for opt in test_discovery_zip_regexes:
+ try:
+ if re.search(regex, opt):
+ get_metrics_agent().report_unoptimized_target(target, 'Test artifact used.')
+ build_targets.add(target)
+ continue
+ get_metrics_agent().report_optimized_target(target)
+ except Exception as e:
+ # In case of exception report as unoptimized
+ build_targets.add(target)
+ get_metrics_agent().report_unoptimized_target(target, f'Error in parsing test discovery output for {target}: {repr(e)}')
+ logging.error(f'unable to parse test discovery output: {repr(e)}')
+
+ return build_targets
+
+ def _legacy_collect_preliminary_build_targets(self):
+ build_targets = set()
+ for target in self.args.extra_targets:
+ if self._unused_target_exclusion_enabled(
+ target
+ ) and not self.build_context.build_target_used(target):
+ continue
+
+ build_targets.add(target)
+ return build_targets
+
def _unused_target_exclusion_enabled(self, target: str) -> bool:
return (
f'{target}_unused_exclusion'
@@ -197,6 +255,11 @@
argparser.add_argument(
'extra_targets', nargs='*', help='Extra test suites to build.'
)
+ argparser.add_argument(
+ '--device-build',
+ action='store_true',
+ help='Flag to indicate running a device build.',
+ )
return argparser.parse_args(argv)
diff --git a/ci/build_test_suites_test.py b/ci/build_test_suites_test.py
index 26f4316..29d268e 100644
--- a/ci/build_test_suites_test.py
+++ b/ci/build_test_suites_test.py
@@ -78,6 +78,12 @@
with self.assert_raises_word(build_test_suites.Error, 'TOP'):
build_test_suites.main([])
+ def test_missing_dist_dir_env_var_raises(self):
+ del os.environ['DIST_DIR']
+
+ with self.assert_raises_word(build_test_suites.Error, 'DIST_DIR'):
+ build_test_suites.main([])
+
def test_invalid_arg_raises(self):
invalid_args = ['--invalid_arg']
@@ -114,6 +120,9 @@
self.soong_ui_dir = self.fake_top.joinpath('build/soong')
self.soong_ui_dir.mkdir(parents=True, exist_ok=True)
+ self.logs_dir = self.fake_top.joinpath('dist/logs')
+ self.logs_dir.mkdir(parents=True, exist_ok=True)
+
self.soong_ui = self.soong_ui_dir.joinpath('soong_ui.bash')
self.soong_ui.touch()
@@ -121,6 +130,7 @@
'TARGET_RELEASE': 'release',
'TARGET_PRODUCT': 'product',
'TOP': str(self.fake_top),
+ 'DIST_DIR': str(self.fake_top.joinpath('dist')),
})
self.mock_subprocess_run.return_value = 0
diff --git a/core/product_config.mk b/core/product_config.mk
index 68614ae..f93b63c 100644
--- a/core/product_config.mk
+++ b/core/product_config.mk
@@ -468,10 +468,17 @@
$(eval SANITIZER.$(TARGET_PRODUCT).$(m).CONFIG := $(cf))))
_psmc_modules :=
-# Reset ADB keys for non-debuggable builds
-ifeq (,$(filter eng userdebug,$(TARGET_BUILD_VARIANT)))
+# Reset ADB keys. If RELEASE_BUILD_USE_VARIANT_FLAGS is set look for
+# the value of a dedicated flag. Otherwise check if build variant is
+# non-debuggable.
+ifneq (,$(RELEASE_BUILD_USE_VARIANT_FLAGS))
+ifneq (,$(RELEASE_BUILD_PURGE_PRODUCT_ADB_KEYS))
PRODUCT_ADB_KEYS :=
endif
+else ifeq (,$(filter eng userdebug,$(TARGET_BUILD_VARIANT)))
+ PRODUCT_ADB_KEYS :=
+endif
+
ifneq ($(filter-out 0 1,$(words $(PRODUCT_ADB_KEYS))),)
$(error Only one file may be in PRODUCT_ADB_KEYS: $(PRODUCT_ADB_KEYS))
endif
diff --git a/core/soong_extra_config.mk b/core/soong_extra_config.mk
index 2ff83a1..8eee50a 100644
--- a/core/soong_extra_config.mk
+++ b/core/soong_extra_config.mk
@@ -80,7 +80,7 @@
$(call add_json_str, ScreenDensity, $(TARGET_SCREEN_DENSITY))
-$(call add_json_bool, UsesVulkan, $(filter true,$(TARGET_USES_VULKAN)))
+$(call add_json_str, UsesVulkan, $(TARGET_USES_VULKAN))
$(call add_json_bool, ZygoteForce64, $(filter true,$(ZYGOTE_FORCE_64)))
diff --git a/target/product/gsi/current.txt b/target/product/gsi/current.txt
index f771916..cbb8a0e 100644
--- a/target/product/gsi/current.txt
+++ b/target/product/gsi/current.txt
@@ -24,7 +24,7 @@
VNDK-SP: android.hardware.common-V2-ndk.so
VNDK-SP: android.hardware.common.fmq-V1-ndk.so
VNDK-SP: android.hardware.graphics.allocator-V2-ndk.so
-VNDK-SP: android.hardware.graphics.common-V5-ndk.so
+VNDK-SP: android.hardware.graphics.common-V6-ndk.so
VNDK-SP: android.hardware.graphics.common@1.0.so
VNDK-SP: android.hardware.graphics.common@1.1.so
VNDK-SP: android.hardware.graphics.common@1.2.so
diff --git a/tools/edit_monitor/daemon_manager.py b/tools/edit_monitor/daemon_manager.py
index a352e30..7d666fe 100644
--- a/tools/edit_monitor/daemon_manager.py
+++ b/tools/edit_monitor/daemon_manager.py
@@ -13,6 +13,8 @@
# limitations under the License.
+import errno
+import fcntl
import getpass
import hashlib
import logging
@@ -100,16 +102,32 @@
logging.warning("Edit monitor for cog is not supported, exiting...")
return
- try:
- self._stop_any_existing_instance()
- self._write_pid_to_pidfile()
- self._start_daemon_process()
- except Exception as e:
- logging.exception("Failed to start daemon manager with error %s", e)
- self._send_error_event_to_clearcut(
- edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
- )
- raise e
+ setup_lock_file = pathlib.Path(tempfile.gettempdir()).joinpath(
+ self.pid_file_path.name + ".setup"
+ )
+ logging.info("setup lock file: %s", setup_lock_file)
+ with open(setup_lock_file, "w") as f:
+ try:
+ # Acquire an exclusive lock
+ fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ self._stop_any_existing_instance()
+ self._write_pid_to_pidfile()
+ self._start_daemon_process()
+ except Exception as e:
+ if (
+ isinstance(e, IOError) and e.errno == errno.EAGAIN
+ ): # Failed to acquire the file lock.
+ logging.warning("Another edit monitor is starting, exitinng...")
+ return
+ else:
+ logging.exception("Failed to start daemon manager with error %s", e)
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
+ )
+ raise e
+ finally:
+ # Release the lock
+ fcntl.flock(f, fcntl.LOCK_UN)
def monitor_daemon(
self,
@@ -413,13 +431,19 @@
def _find_all_instances_pids(self) -> list[int]:
pids = []
- for file in os.listdir(self.pid_file_path.parent):
- if file.endswith(".lock"):
- try:
- with open(self.pid_file_path.parent.joinpath(file), "r") as f:
- pids.append(int(f.read().strip()))
- except (FileNotFoundError, IOError, ValueError, TypeError):
- logging.exception("Failed to get pid from file path: %s", file)
+ try:
+ output = subprocess.check_output(["ps", "-ef", "--no-headers"], text=True)
+ for line in output.splitlines():
+ parts = line.split()
+ process_path = parts[7]
+ if pathlib.Path(process_path).name == "edit_monitor":
+ pid = int(parts[1])
+ if pid != self.pid: # exclude the current process
+ pids.append(pid)
+ except Exception:
+ logging.exception(
+ "Failed to get pids of existing edit monitors from ps command."
+ )
return pids
diff --git a/tools/edit_monitor/daemon_manager_test.py b/tools/edit_monitor/daemon_manager_test.py
index 350739d..be28965 100644
--- a/tools/edit_monitor/daemon_manager_test.py
+++ b/tools/edit_monitor/daemon_manager_test.py
@@ -14,6 +14,7 @@
"""Unittests for DaemonManager."""
+import fcntl
import logging
import multiprocessing
import os
@@ -82,7 +83,8 @@
# tests will be cleaned.
tempfile.tempdir = self.working_dir.name
self.patch = mock.patch.dict(
- os.environ, {'ENABLE_ANDROID_EDIT_MONITOR': 'true'})
+ os.environ, {'ENABLE_ANDROID_EDIT_MONITOR': 'true'}
+ )
self.patch.start()
def tearDown(self):
@@ -102,6 +104,7 @@
p = self._create_fake_deamon_process()
self.assert_run_simple_daemon_success()
+ self.assert_no_subprocess_running()
def test_start_success_with_existing_instance_already_dead(self):
# Create a pidfile with pid that does not exist.
@@ -137,7 +140,9 @@
# Verify no daemon process is started.
self.assertIsNone(dm.daemon_process)
- @mock.patch.dict(os.environ, {'ENABLE_ANDROID_EDIT_MONITOR': 'false'}, clear=True)
+ @mock.patch.dict(
+ os.environ, {'ENABLE_ANDROID_EDIT_MONITOR': 'false'}, clear=True
+ )
def test_start_return_directly_if_disabled(self):
dm = daemon_manager.DaemonManager(TEST_BINARY_FILE)
dm.start()
@@ -154,6 +159,25 @@
# Verify no daemon process is started.
self.assertIsNone(dm.daemon_process)
+ def test_start_failed_other_instance_is_starting(self):
+ f = open(
+ pathlib.Path(self.working_dir.name).joinpath(
+ TEST_PID_FILE_PATH + '.setup'
+ ),
+ 'w',
+ )
+ # Acquire an exclusive lock
+ fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+ dm = daemon_manager.DaemonManager(TEST_BINARY_FILE)
+ dm.start()
+
+ # Release the lock
+ fcntl.flock(f, fcntl.LOCK_UN)
+ f.close()
+ # Verify no daemon process is started.
+ self.assertIsNone(dm.daemon_process)
+
@mock.patch('os.kill')
def test_start_failed_to_kill_existing_instance(self, mock_kill):
mock_kill.side_effect = OSError('Unknown OSError')
@@ -177,6 +201,7 @@
'edit_monitor'
)
pid_file_path_dir.mkdir(parents=True, exist_ok=True)
+
# Makes the directory read-only so write pidfile will fail.
os.chmod(pid_file_path_dir, 0o555)
@@ -216,7 +241,7 @@
cclient=fake_cclient,
)
# set the fake total_memory_size
- dm.total_memory_size = 100 * 1024 *1024
+ dm.total_memory_size = 100 * 1024 * 1024
dm.start()
dm.monitor_daemon(interval=1)
@@ -367,6 +392,26 @@
fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_REBOOT_EDIT_MONITOR
)
+ @mock.patch('subprocess.check_output')
+ def test_cleanup_success(self, mock_check_output):
+ p = self._create_fake_deamon_process()
+ fake_cclient = FakeClearcutClient()
+ mock_check_output.return_value = f'user {p.pid} 1 1 1 1 1 edit_monitor arg'
+
+ dm = daemon_manager.DaemonManager(
+ TEST_BINARY_FILE,
+ daemon_target=long_running_daemon,
+ cclient=fake_cclient,
+ )
+ dm.cleanup()
+
+ self.assertFalse(p.is_alive())
+ self.assertTrue(
+ pathlib.Path(self.working_dir.name)
+ .joinpath(daemon_manager.BLOCK_SIGN_FILE)
+ .exists()
+ )
+
def assert_run_simple_daemon_success(self):
damone_output_file = tempfile.NamedTemporaryFile(
dir=self.working_dir.name, delete=False
@@ -432,7 +477,7 @@
pass
def _create_fake_deamon_process(
- self, name: str = ''
+ self, name: str = TEST_PID_FILE_PATH
) -> multiprocessing.Process:
# Create a long running subprocess
p = multiprocessing.Process(target=long_running_daemon)
@@ -443,7 +488,7 @@
'edit_monitor'
)
pid_file_path_dir.mkdir(parents=True, exist_ok=True)
- with open(pid_file_path_dir.joinpath(name + 'pid.lock'), 'w') as f:
+ with open(pid_file_path_dir.joinpath(name), 'w') as f:
f.write(str(p.pid))
return p
diff --git a/tools/edit_monitor/main.py b/tools/edit_monitor/main.py
index 7ca0daa..3c2d183 100644
--- a/tools/edit_monitor/main.py
+++ b/tools/edit_monitor/main.py
@@ -102,12 +102,12 @@
daemon_args=(args.path, args.dry_run),
)
- if args.force_cleanup:
- dm.cleanup()
-
try:
- dm.start()
- dm.monitor_daemon()
+ if args.force_cleanup:
+ dm.cleanup()
+ else:
+ dm.start()
+ dm.monitor_daemon()
except Exception:
logging.exception('Unexpected exception raised when run daemon.')
finally: