Merge "Export super image related variables to Soong" into main
diff --git a/core/soong_config.mk b/core/soong_config.mk
index 20edb9b..a007888 100644
--- a/core/soong_config.mk
+++ b/core/soong_config.mk
@@ -404,6 +404,7 @@
$(call add_json_str, BoardAvbAlgorithm, $(BOARD_AVB_$(image_type)_ALGORITHM)) \
$(call add_json_str, BoardAvbRollbackIndex, $(BOARD_AVB_$(image_type)_ROLLBACK_INDEX)) \
$(call add_json_str, BoardAvbRollbackIndexLocation, $(BOARD_AVB_$(image_type)_ROLLBACK_INDEX_LOCATION)) \
+ $(call add_json_str, BoardAvbAddHashtreeFooterArgs, $(BOARD_AVB_$(image_type)_ADD_HASHTREE_FOOTER_ARGS)) \
$(call add_json_str, ProductBaseFsPath, $(PRODUCT_$(image_type)_BASE_FS_PATH)) \
$(call add_json_str, ProductHeadroom, $(PRODUCT_$(image_type)_HEADROOM)) \
$(call add_json_str, ProductVerityPartition, $(PRODUCT_$(image_type)_VERITY_PARTITION)) \
diff --git a/core/sysprop.mk b/core/sysprop.mk
index 255b699..e5f1267 100644
--- a/core/sysprop.mk
+++ b/core/sysprop.mk
@@ -183,7 +183,7 @@
endif
BUILD_FINGERPRINT_FILE := $(PRODUCT_OUT)/build_fingerprint.txt
-ifneq (,$(shell mkdir -p $(PRODUCT_OUT) && echo $(BUILD_FINGERPRINT) >$(BUILD_FINGERPRINT_FILE) && grep " " $(BUILD_FINGERPRINT_FILE)))
+ifneq (,$(shell mkdir -p $(PRODUCT_OUT) && echo $(BUILD_FINGERPRINT) >$(BUILD_FINGERPRINT_FILE).tmp && (if ! cmp -s $(BUILD_FINGERPRINT_FILE).tmp $(BUILD_FINGERPRINT_FILE); then mv $(BUILD_FINGERPRINT_FILE).tmp $(BUILD_FINGERPRINT_FILE); else rm $(BUILD_FINGERPRINT_FILE).tmp; fi) && grep " " $(BUILD_FINGERPRINT_FILE)))
$(error BUILD_FINGERPRINT cannot contain spaces: "$(file <$(BUILD_FINGERPRINT_FILE))")
endif
BUILD_FINGERPRINT_FROM_FILE := $$(cat $(BUILD_FINGERPRINT_FILE))
diff --git a/core/tasks/general-tests-shared-libs.mk b/core/tasks/general-tests-shared-libs.mk
new file mode 100644
index 0000000..2405140
--- /dev/null
+++ b/core/tasks/general-tests-shared-libs.mk
@@ -0,0 +1,52 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+.PHONY: general-tests-shared-libs
+
+intermediates_dir := $(call intermediates-dir-for,PACKAGING,general-tests-shared-libs)
+
+general_tests_shared_libs_zip := $(PRODUCT_OUT)/general-tests_host-shared-libs.zip
+
+# Filter shared entries between general-tests and device-tests's HOST_SHARED_LIBRARY.FILES,
+# to avoid warning about overriding commands.
+my_host_shared_lib_for_general_tests := \
+ $(foreach m,$(filter $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\
+ $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES)),$(call word-colon,2,$(m)))
+my_general_tests_shared_lib_files := \
+ $(filter-out $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\
+ $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES))
+
+my_host_shared_lib_for_general_tests += $(call copy-many-files,$(my_general_tests_shared_lib_files))
+
+$(general_tests_shared_libs_zip) : PRIVATE_INTERMEDIATES_DIR := $(intermediates_dir)
+$(general_tests_shared_libs_zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_general_tests)
+$(general_tests_shared_libs_zip) : PRIVATE_general_host_shared_libs_zip := $(general_tests_shared_libs_zip)
+$(general_tests_shared_libs_zip) : $(my_host_shared_lib_for_general_tests) $(SOONG_ZIP)
+ rm -rf $(PRIVATE_INTERMEDIATES_DIR)
+ mkdir -p $(PRIVATE_INTERMEDIATES_DIR) $(PRIVATE_INTERMEDIATES_DIR)/tools
+ $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \
+ echo $$shared_lib >> $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list; \
+ done
+ grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list > $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list || true
+ $(SOONG_ZIP) -d -o $(PRIVATE_general_host_shared_libs_zip) \
+ -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list
+
+general-tests-shared-libs: $(general_tests_shared_libs_zip)
+$(call dist-for-goals, general-tests-shared-libs, $(general_tests_shared_libs_zip))
+
+$(call declare-1p-container,$(general_tests_shared_libs_zip),)
+$(call declare-container-license-deps,$(general_tests_shared_libs_zip),$(my_host_shared_lib_for_general_tests),$(PRODUCT_OUT)/:/)
+
+intermediates_dir :=
+general_tests_shared_libs_zip :=
diff --git a/core/tasks/general-tests.mk b/core/tasks/general-tests.mk
index dcfcfad..1901ed5 100644
--- a/core/tasks/general-tests.mk
+++ b/core/tasks/general-tests.mk
@@ -27,61 +27,19 @@
# Create an artifact to include all test config files in general-tests.
general_tests_configs_zip := $(PRODUCT_OUT)/general-tests_configs.zip
-# Filter shared entries between general-tests and device-tests's HOST_SHARED_LIBRARY.FILES,
-# to avoid warning about overriding commands.
-my_host_shared_lib_for_general_tests := \
- $(foreach m,$(filter $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\
- $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES)),$(call word-colon,2,$(m)))
-my_general_tests_shared_lib_files := \
- $(filter-out $(COMPATIBILITY.device-tests.HOST_SHARED_LIBRARY.FILES),\
- $(COMPATIBILITY.general-tests.HOST_SHARED_LIBRARY.FILES))
+general_tests_shared_libs_zip := $(PRODUCT_OUT)/general-tests_host-shared-libs.zip
-my_host_shared_lib_for_general_tests += $(call copy-many-files,$(my_general_tests_shared_lib_files))
-
-my_host_shared_lib_symlinks := \
- $(filter $(COMPATIBILITY.host-unit-tests.SYMLINKS),\
- $(COMPATIBILITY.general-tests.SYMLINKS))
-
-my_general_tests_symlinks := \
- $(filter-out $(COMPATIBILITY.camera-hal-tests.SYMLINKS),\
- $(filter-out $(COMPATIBILITY.host-unit-tests.SYMLINKS),\
- $(COMPATIBILITY.general-tests.SYMLINKS)))
-
-my_symlinks_for_general_tests := $(foreach f,$(my_general_tests_symlinks),\
- $(strip $(eval _cmf_tuple := $(subst :, ,$(f))) \
- $(eval _cmf_dep := $(word 1,$(_cmf_tuple))) \
- $(eval _cmf_src := $(word 2,$(_cmf_tuple))) \
- $(eval _cmf_dest := $(word 3,$(_cmf_tuple))) \
- $(call symlink-file,$(_cmf_dep),$(_cmf_src),$(_cmf_dest)) \
- $(_cmf_dest)))
-
-# In this one directly take the overlap into the zip since we can't rewrite rules
-my_symlinks_for_general_tests += $(foreach f,$(my_host_shared_lib_symlinks),\
- $(strip $(eval _cmf_tuple := $(subst :, ,$(f))) \
- $(eval _cmf_dep := $(word 1,$(_cmf_tuple))) \
- $(eval _cmf_src := $(word 2,$(_cmf_tuple))) \
- $(eval _cmf_dest := $(word 3,$(_cmf_tuple))) \
- $(_cmf_dest)))
-
+$(general_tests_zip) : $(general_tests_shared_libs_zip)
$(general_tests_zip) : PRIVATE_general_tests_list_zip := $(general_tests_list_zip)
$(general_tests_zip) : .KATI_IMPLICIT_OUTPUTS := $(general_tests_list_zip) $(general_tests_configs_zip)
$(general_tests_zip) : PRIVATE_TOOLS := $(general_tests_tools)
$(general_tests_zip) : PRIVATE_INTERMEDIATES_DIR := $(intermediates_dir)
-$(general_tests_zip) : PRIVATE_HOST_SHARED_LIBS := $(my_host_shared_lib_for_general_tests)
-$(general_tests_zip) : PRIVATE_SYMLINKS := $(my_symlinks_for_general_tests)
$(general_tests_zip) : PRIVATE_general_tests_configs_zip := $(general_tests_configs_zip)
-$(general_tests_zip) : $(COMPATIBILITY.general-tests.FILES) $(my_host_shared_lib_for_general_tests) $(COMPATIBILITY.general-tests.SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES) $(general_tests_tools) $(my_symlinks_for_general_tests) $(SOONG_ZIP)
+$(general_tests_zip) : $(COMPATIBILITY.general-tests.FILES) $(COMPATIBILITY.general-tests.SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES) $(general_tests_tools) $(SOONG_ZIP)
rm -rf $(PRIVATE_INTERMEDIATES_DIR)
rm -f $@ $(PRIVATE_general_tests_list_zip)
mkdir -p $(PRIVATE_INTERMEDIATES_DIR) $(PRIVATE_INTERMEDIATES_DIR)/tools
echo $(sort $(COMPATIBILITY.general-tests.FILES) $(COMPATIBILITY.general-tests.SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES)) | tr " " "\n" > $(PRIVATE_INTERMEDIATES_DIR)/list
- for symlink in $(PRIVATE_SYMLINKS); do \
- echo $$symlink >> $(PRIVATE_INTERMEDIATES_DIR)/list; \
- done
- $(hide) for shared_lib in $(PRIVATE_HOST_SHARED_LIBS); do \
- echo $$shared_lib >> $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list; \
- done
- grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/shared-libs.list > $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list || true
grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/host.list || true
grep $(TARGET_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/target.list || true
grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/host.list > $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list || true
@@ -91,7 +49,6 @@
-P host -C $(PRIVATE_INTERMEDIATES_DIR) -D $(PRIVATE_INTERMEDIATES_DIR)/tools \
-P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host.list \
-P target -C $(PRODUCT_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/target.list \
- -P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-shared-libs.list \
-sha256
$(SOONG_ZIP) -d -o $(PRIVATE_general_tests_configs_zip) \
-P host -C $(HOST_OUT) -l $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list \
@@ -112,8 +69,3 @@
general_tests_list_zip :=
general_tests_configs_zip :=
general_tests_shared_libs_zip :=
-my_host_shared_lib_for_general_tests :=
-my_symlinks_for_general_tests :=
-my_general_tests_shared_lib_files :=
-my_general_tests_symlinks :=
-my_host_shared_lib_symlinks :=
diff --git a/tools/edit_monitor/daemon_manager.py b/tools/edit_monitor/daemon_manager.py
index 0c31ab8..a352e30 100644
--- a/tools/edit_monitor/daemon_manager.py
+++ b/tools/edit_monitor/daemon_manager.py
@@ -149,15 +149,15 @@
edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE
)
logging.error(
- "Daemon process is consuming too much memory, rebooting...")
+ "Daemon process is consuming too much memory, rebooting..."
+ )
self.reboot()
if self.max_cpu_usage >= cpu_threshold:
self._send_error_event_to_clearcut(
edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE
)
- logging.error(
- "Daemon process is consuming too much cpu, killing...")
+ logging.error("Daemon process is consuming too much cpu, killing...")
self._terminate_process(self.daemon_process.pid)
logging.info(
@@ -179,7 +179,7 @@
self._wait_for_process_terminate(self.daemon_process.pid, 1)
if self.daemon_process.is_alive():
self._terminate_process(self.daemon_process.pid)
- self._remove_pidfile()
+ self._remove_pidfile(self.pid)
logging.info("Successfully stopped daemon manager.")
except Exception as e:
logging.exception("Failed to stop daemon manager with error %s", e)
@@ -253,11 +253,15 @@
if ex_pid:
logging.info("Found another instance with pid %d.", ex_pid)
self._terminate_process(ex_pid)
- self._remove_pidfile()
+ self._remove_pidfile(ex_pid)
- def _read_pid_from_pidfile(self):
- with open(self.pid_file_path, "r") as f:
- return int(f.read().strip())
+ def _read_pid_from_pidfile(self) -> int | None:
+ try:
+ with open(self.pid_file_path, "r") as f:
+ return int(f.read().strip())
+ except FileNotFoundError as e:
+ logging.warning("pidfile %s does not exist.", self.pid_file_path)
+ return None
def _write_pid_to_pidfile(self):
"""Creates a pidfile and writes the current pid to the file.
@@ -333,7 +337,23 @@
)
return True
- def _remove_pidfile(self):
+ def _remove_pidfile(self, expected_pid: int):
+ recorded_pid = self._read_pid_from_pidfile()
+
+ if recorded_pid is None:
+ logging.info("pid file %s already removed.", self.pid_file_path)
+ return
+
+ if recorded_pid != expected_pid:
+ logging.warning(
+ "pid file contains pid from a different process, expected pid: %d,"
+ " actual pid: %d.",
+ expected_pid,
+ recorded_pid,
+ )
+ return
+
+ logging.debug("removing pidfile written by process %s", expected_pid)
try:
os.remove(self.pid_file_path)
except FileNotFoundError:
@@ -378,9 +398,7 @@
uptime_end = float(f.readline().split()[0])
return (
- (total_end_time - total_start_time)
- / (uptime_end - uptime_start)
- * 100
+ (total_end_time - total_start_time) / (uptime_end - uptime_start) * 100
)
def _get_total_cpu_time(self, pid: int) -> float:
diff --git a/tools/edit_monitor/edit_monitor_integration_test.py b/tools/edit_monitor/edit_monitor_integration_test.py
index 3d28274..f39b936 100644
--- a/tools/edit_monitor/edit_monitor_integration_test.py
+++ b/tools/edit_monitor/edit_monitor_integration_test.py
@@ -15,6 +15,7 @@
"""Integration tests for Edit Monitor."""
import glob
+from importlib import resources
import logging
import os
import pathlib
@@ -25,8 +26,6 @@
import tempfile
import time
import unittest
-
-from importlib import resources
from unittest import mock
@@ -49,7 +48,8 @@
self.root_monitoring_path.mkdir()
self.edit_monitor_binary_path = self._import_executable("edit_monitor")
self.patch = mock.patch.dict(
- os.environ, {'ENABLE_ANDROID_EDIT_MONITOR': 'true'})
+ os.environ, {"ENABLE_ANDROID_EDIT_MONITOR": "true"}
+ )
self.patch.start()
def tearDown(self):
@@ -83,7 +83,21 @@
self.assertEqual(self._get_logged_events_num(), 4)
- def _start_edit_monitor_process(self):
+ def test_start_multiple_edit_monitor_only_one_started(self):
+ p1 = self._start_edit_monitor_process(wait_for_observer_start=False)
+ p2 = self._start_edit_monitor_process(wait_for_observer_start=False)
+ p3 = self._start_edit_monitor_process(wait_for_observer_start=False)
+
+ live_processes = self._get_live_processes([p1, p2, p3])
+
+ # Cleanup all live processes.
+ for p in live_processes:
+ os.kill(p.pid, signal.SIGINT)
+ p.communicate()
+
+ self.assertEqual(len(live_processes), 1)
+
+ def _start_edit_monitor_process(self, wait_for_observer_start=True):
command = f"""
export TMPDIR="{self.working_dir.name}"
{self.edit_monitor_binary_path} --path={self.root_monitoring_path} --dry_run"""
@@ -94,7 +108,9 @@
start_new_session=True,
executable="/bin/bash",
)
- self._wait_for_observer_start(time_out=5)
+ if wait_for_observer_start:
+ self._wait_for_observer_start(time_out=5)
+
return p
def _wait_for_observer_start(self, time_out):
@@ -125,6 +141,18 @@
return 0
+ def _get_live_processes(self, processes):
+ live_processes = []
+ for p in processes:
+ try:
+ p.wait(timeout=5)
+ except subprocess.TimeoutExpired as e:
+ live_processes.append(p)
+ logging.info("process: %d still alive.", p.pid)
+ else:
+ logging.info("process: %d stopped.", p.pid)
+ return live_processes
+
def _import_executable(self, executable_name: str) -> pathlib.Path:
binary_dir = pathlib.Path(self.working_dir.name).joinpath("binary")
binary_dir.mkdir()
diff --git a/tools/edit_monitor/main.py b/tools/edit_monitor/main.py
index 49385f1..7ca0daa 100644
--- a/tools/edit_monitor/main.py
+++ b/tools/edit_monitor/main.py
@@ -72,7 +72,8 @@
root_logging_dir = tempfile.mkdtemp(prefix='edit_monitor_')
_, log_path = tempfile.mkstemp(dir=root_logging_dir, suffix='.log')
- log_fmt = '%(asctime)s %(filename)s:%(lineno)s:%(levelname)s: %(message)s'
+
+ log_fmt = '%(asctime)s.%(msecs)03d %(filename)s:%(lineno)s:%(levelname)s: %(message)s'
date_fmt = '%Y-%m-%d %H:%M:%S'
log_level = logging.DEBUG if verbose else logging.INFO
diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py
index 6446e1f..76d168c 100755
--- a/tools/releasetools/ota_from_target_files.py
+++ b/tools/releasetools/ota_from_target_files.py
@@ -1039,6 +1039,9 @@
# Prepare custom images.
if OPTIONS.custom_images:
+ if source_file is not None:
+ source_file = GetTargetFilesZipForCustomImagesUpdates(
+ source_file, OPTIONS.custom_images)
target_file = GetTargetFilesZipForCustomImagesUpdates(
target_file, OPTIONS.custom_images)
@@ -1121,17 +1124,18 @@
additional_args += ["--enable_lz4diff=" +
str(OPTIONS.enable_lz4diff).lower()]
+ env_override = {}
if source_file and OPTIONS.enable_lz4diff:
- input_tmp = common.UnzipTemp(source_file, ["META/liblz4.so"])
- liblz4_path = os.path.join(input_tmp, "META", "liblz4.so")
+ liblz4_path = os.path.join(source_file, "META", "liblz4.so")
assert os.path.exists(
liblz4_path), "liblz4.so not found in META/ dir of target file {}".format(liblz4_path)
logger.info("Enabling lz4diff %s", liblz4_path)
- additional_args += ["--liblz4_path", liblz4_path]
erofs_compression_param = OPTIONS.target_info_dict.get(
"erofs_default_compressor")
assert erofs_compression_param is not None, "'erofs_default_compressor' not found in META/misc_info.txt of target build. This is required to enable lz4diff."
additional_args += ["--erofs_compression_param", erofs_compression_param]
+ env_override["LD_PRELOAD"] = liblz4_path + \
+ ":" + os.environ.get("LD_PRELOAD", "")
if OPTIONS.disable_vabc:
additional_args += ["--disable_vabc=true"]
@@ -1141,10 +1145,15 @@
additional_args += ["--compressor_types", OPTIONS.compressor_types]
additional_args += ["--max_timestamp", max_timestamp]
+ env = dict(os.environ)
+ if env_override:
+ logger.info("Using environment variables %s", env_override)
+ env.update(env_override)
payload.Generate(
target_file,
source_file,
- additional_args + partition_timestamps_flags
+ additional_args + partition_timestamps_flags,
+ env=env
)
# Sign the payload.
diff --git a/tools/releasetools/ota_utils.py b/tools/releasetools/ota_utils.py
index 81b53dc..852d62b 100644
--- a/tools/releasetools/ota_utils.py
+++ b/tools/releasetools/ota_utils.py
@@ -845,16 +845,16 @@
self.is_partial_update = is_partial_update
self.spl_downgrade = spl_downgrade
- def _Run(self, cmd): # pylint: disable=no-self-use
+ def _Run(self, cmd, **kwargs): # pylint: disable=no-self-use
# Don't pipe (buffer) the output if verbose is set. Let
# brillo_update_payload write to stdout/stderr directly, so its progress can
# be monitored.
if OPTIONS.verbose:
- common.RunAndCheckOutput(cmd, stdout=None, stderr=None)
+ common.RunAndCheckOutput(cmd, stdout=None, stderr=None, **kwargs)
else:
- common.RunAndCheckOutput(cmd)
+ common.RunAndCheckOutput(cmd, **kwargs)
- def Generate(self, target_file, source_file=None, additional_args=None):
+ def Generate(self, target_file, source_file=None, additional_args=None, **kwargs):
"""Generates a payload from the given target-files zip(s).
Args:
@@ -863,6 +863,7 @@
generating a full OTA.
additional_args: A list of additional args that should be passed to
delta_generator binary; or None.
+ kwargs: Any additional args to pass to subprocess.Popen
"""
if additional_args is None:
additional_args = []
@@ -918,7 +919,7 @@
if self.is_partial_update:
cmd.extend(["--is_partial_update=true"])
cmd.extend(additional_args)
- self._Run(cmd)
+ self._Run(cmd, **kwargs)
self.payload_file = payload_file
self.payload_properties = None