Merge "Revert "remove file check"" into main
diff --git a/ci/build_device_and_tests b/ci/build_device_and_tests
new file mode 100755
index 0000000..9d11268
--- /dev/null
+++ b/ci/build_device_and_tests
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+build/soong/soong_ui.bash --make-mode build_test_suites || exit $?
+$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites $@ || exit $?
diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py
index cd9d76d..4d3b546 100644
--- a/ci/build_test_suites.py
+++ b/ci/build_test_suites.py
@@ -89,6 +89,8 @@
else:
regex = r'\b(%s)\b' % re.escape(target)
if any(re.search(regex, opt) for opt in test_discovery_zip_regexes):
+ get_metrics_agent().report_unoptimized_target(target, 'Test artifact used.')
+ else:
get_metrics_agent().report_optimized_target(target)
if self._unused_target_exclusion_enabled(
diff --git a/core/Makefile b/core/Makefile
index 5e0d4df..92dd86d 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -1654,6 +1654,8 @@
$(ALL_DEFAULT_INSTALLED_MODULES))
INTERNAL_VENDOR_RAMDISK_TARGET := $(call intermediates-dir-for,PACKAGING,vendor_boot)/vendor_ramdisk.cpio$(RAMDISK_EXT)
+vendor_ramdisk_intermediates :=$= $(call intermediates-dir-for,PACKAGING,vendor_ramdisk)
+$(eval $(call write-partition-file-list,$(vendor_ramdisk_intermediates)/file_list.txt,$(TARGET_VENDOR_RAMDISK_OUT),$(INTERNAL_VENDOR_RAMDISK_FILES)))
# Exclude recovery files in the default vendor ramdisk if including a standalone
# recovery ramdisk in vendor_boot.
diff --git a/core/config.mk b/core/config.mk
index 485e8cc..b546ea1 100644
--- a/core/config.mk
+++ b/core/config.mk
@@ -1307,10 +1307,6 @@
SOONG_VARIABLES :=
SOONG_EXTRA_VARIABLES :=
--include external/ltp/android/ltp_package_list.mk
-DEFAULT_DATA_OUT_MODULES := ltp $(ltp_packages)
-.KATI_READONLY := DEFAULT_DATA_OUT_MODULES
-
include $(BUILD_SYSTEM)/dumpvar.mk
ifdef BOARD_VNDK_VERSION
diff --git a/core/definitions.mk b/core/definitions.mk
index cd1b36e..7a32464 100644
--- a/core/definitions.mk
+++ b/core/definitions.mk
@@ -2605,7 +2605,17 @@
@$(call emit-line,$(wordlist 108501,109000,$(1)),$(2))
@$(call emit-line,$(wordlist 109001,109500,$(1)),$(2))
@$(call emit-line,$(wordlist 109501,110000,$(1)),$(2))
- @$(if $(wordlist 110001,110002,$(1)),$(error dump-words-to-file: Too many words ($(words $(1)))))
+ @$(call emit-line,$(wordlist 110001,110500,$(1)),$(2))
+ @$(call emit-line,$(wordlist 110501,111000,$(1)),$(2))
+ @$(call emit-line,$(wordlist 111001,111500,$(1)),$(2))
+ @$(call emit-line,$(wordlist 111501,112000,$(1)),$(2))
+ @$(call emit-line,$(wordlist 112001,112500,$(1)),$(2))
+ @$(call emit-line,$(wordlist 112501,113000,$(1)),$(2))
+ @$(call emit-line,$(wordlist 113001,113500,$(1)),$(2))
+ @$(call emit-line,$(wordlist 113501,114000,$(1)),$(2))
+ @$(call emit-line,$(wordlist 114001,114500,$(1)),$(2))
+ @$(call emit-line,$(wordlist 114501,115000,$(1)),$(2))
+ @$(if $(wordlist 115001,115002,$(1)),$(error dump-words-to-file: Too many words ($(words $(1)))))
endef
# Return jar arguments to compress files in a given directory
# $(1): directory
diff --git a/core/java_common.mk b/core/java_common.mk
index a21f062..f574b76 100644
--- a/core/java_common.mk
+++ b/core/java_common.mk
@@ -32,7 +32,7 @@
else ifneq (,$(LOCAL_SDK_VERSION)$(TARGET_BUILD_USE_PREBUILT_SDKS))
# TODO(ccross): allow 1.9 for current and unbundled once we have SDK system modules
LOCAL_JAVA_LANGUAGE_VERSION := 1.8
- else ifeq ($(EXPERIMENTAL_TARGET_JAVA_VERSION_21),true)
+ else ifeq ($(RELEASE_TARGET_JAVA_21),true)
LOCAL_JAVA_LANGUAGE_VERSION := 21
else
LOCAL_JAVA_LANGUAGE_VERSION := 17
diff --git a/core/product.mk b/core/product.mk
index 9c567c3..1b336b0 100644
--- a/core/product.mk
+++ b/core/product.mk
@@ -284,6 +284,9 @@
# Whether APEX should be compressed or not
_product_single_value_vars += PRODUCT_COMPRESSED_APEX
+# Default fs type for APEX payload image (apex_payload.img)
+_product_single_value_vars += PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE
+
# VNDK version of product partition. It can be 'current' if the product
# partitions uses PLATFORM_VNDK_VERSION.
_product_single_value_vars += PRODUCT_PRODUCT_VNDK_VERSION
diff --git a/core/product_config.mk b/core/product_config.mk
index abe6e38..692e375 100644
--- a/core/product_config.mk
+++ b/core/product_config.mk
@@ -534,6 +534,17 @@
PRODUCT_COMPRESSED_APEX := $(OVERRIDE_PRODUCT_COMPRESSED_APEX)
endif
+ifdef OVERRIDE_PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE
+ PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE := $(OVERRIDE_PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE)
+else ifeq ($(PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE),)
+ # Use ext4 as a default payload fs type
+ PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE := ext4
+endif
+ifeq ($(filter ext4 erofs,$(PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE)),)
+ $(error PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE should be either erofs or ext4,\
+ not $(PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE).)
+endif
+
$(KATI_obsolete_var OVERRIDE_PRODUCT_EXTRA_VNDK_VERSIONS \
,Use PRODUCT_EXTRA_VNDK_VERSIONS instead)
diff --git a/core/soong_config.mk b/core/soong_config.mk
index c5dd320..32ca660 100644
--- a/core/soong_config.mk
+++ b/core/soong_config.mk
@@ -274,6 +274,7 @@
$(call add_json_str, DeviceCurrentApiLevelForVendorModules, $(BOARD_CURRENT_API_LEVEL_FOR_VENDOR_MODULES))
$(call add_json_bool, CompressedApex, $(filter true,$(PRODUCT_COMPRESSED_APEX)))
+$(call add_json_str, DefaultApexPayloadType, $(PRODUCT_DEFAULT_APEX_PAYLOAD_TYPE))
$(call add_json_bool, BoardUsesRecoveryAsBoot, $(filter true,$(BOARD_USES_RECOVERY_AS_BOOT)))
@@ -430,14 +431,16 @@
# boot image stuff
$(call add_json_bool, BuildingRamdiskImage, $(filter true,$(BUILDING_RAMDISK_IMAGE)))
$(call add_json_bool, ProductBuildBootImage, $(filter true,$(PRODUCT_BUILD_BOOT_IMAGE)))
+ $(call add_json_str, ProductBuildVendorBootImage, $(PRODUCT_BUILD_VENDOR_BOOT_IMAGE))
$(call add_json_bool, ProductBuildInitBootImage, $(filter true,$(PRODUCT_BUILD_INIT_BOOT_IMAGE)))
$(call add_json_bool, BoardUsesRecoveryAsBoot, $(filter true,$(BOARD_USES_RECOVERY_AS_BOOT)))
$(call add_json_str, BoardPrebuiltBootimage, $(BOARD_PREBUILT_BOOT_IMAGE))
$(call add_json_str, BoardPrebuiltInitBootimage, $(BOARD_PREBUILT_INIT_BOOT_IMAGE))
$(call add_json_str, BoardBootimagePartitionSize, $(BOARD_BOOTIMAGE_PARTITION_SIZE))
- $(call add_json_str, BoardInitBootimagePartitionSize, $(BOARD_INIT_BOOTIMAGE_PARTITION_SIZE))
+ $(call add_json_str, BoardInitBootimagePartitionSize, $(BOARD_INIT_BOOT_IMAGE_PARTITION_SIZE))
$(call add_json_str, BoardBootHeaderVersion, $(BOARD_BOOT_HEADER_VERSION))
$(call add_json_str, TargetKernelPath, $(TARGET_KERNEL_PATH))
+ $(call add_json_bool, BoardUsesGenericKernelImage, $(BOARD_USES_GENERIC_KERNEL_IMAGE))
# Avb (android verified boot) stuff
$(call add_json_bool, BoardAvbEnable, $(filter true,$(BOARD_AVB_ENABLE)))
diff --git a/core/tasks/general-tests.mk b/core/tasks/general-tests.mk
index d6fc072..1901ed5 100644
--- a/core/tasks/general-tests.mk
+++ b/core/tasks/general-tests.mk
@@ -27,21 +27,9 @@
# Create an artifact to include all test config files in general-tests.
general_tests_configs_zip := $(PRODUCT_OUT)/general-tests_configs.zip
-# Copy kernel test modules to testcases directories
-include $(BUILD_SYSTEM)/tasks/tools/vts-kernel-tests.mk
-ltp_copy_pairs := \
- $(call target-native-copy-pairs,$(kernel_ltp_modules),$(kernel_ltp_host_out))
-copy_ltp_tests := $(call copy-many-files,$(ltp_copy_pairs))
-
-# PHONY target to be used to build and test `vts_ltp_tests` without building full vts
-.PHONY: vts_kernel_ltp_tests
-vts_kernel_ltp_tests: $(copy_ltp_tests)
-
general_tests_shared_libs_zip := $(PRODUCT_OUT)/general-tests_host-shared-libs.zip
$(general_tests_zip) : $(general_tests_shared_libs_zip)
-$(general_tests_zip) : $(copy_ltp_tests)
-$(general_tests_zip) : PRIVATE_KERNEL_LTP_HOST_OUT := $(kernel_ltp_host_out)
$(general_tests_zip) : PRIVATE_general_tests_list_zip := $(general_tests_list_zip)
$(general_tests_zip) : .KATI_IMPLICIT_OUTPUTS := $(general_tests_list_zip) $(general_tests_configs_zip)
$(general_tests_zip) : PRIVATE_TOOLS := $(general_tests_tools)
@@ -52,7 +40,6 @@
rm -f $@ $(PRIVATE_general_tests_list_zip)
mkdir -p $(PRIVATE_INTERMEDIATES_DIR) $(PRIVATE_INTERMEDIATES_DIR)/tools
echo $(sort $(COMPATIBILITY.general-tests.FILES) $(COMPATIBILITY.general-tests.SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES)) | tr " " "\n" > $(PRIVATE_INTERMEDIATES_DIR)/list
- find $(PRIVATE_KERNEL_LTP_HOST_OUT) >> $(PRIVATE_INTERMEDIATES_DIR)/list
grep $(HOST_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/host.list || true
grep $(TARGET_OUT_TESTCASES) $(PRIVATE_INTERMEDIATES_DIR)/list > $(PRIVATE_INTERMEDIATES_DIR)/target.list || true
grep -e .*\\.config$$ $(PRIVATE_INTERMEDIATES_DIR)/host.list > $(PRIVATE_INTERMEDIATES_DIR)/host-test-configs.list || true
diff --git a/core/tasks/tools/vts-kernel-tests.mk b/core/tasks/tools/vts-kernel-tests.mk
deleted file mode 100644
index e727dc1..0000000
--- a/core/tasks/tools/vts-kernel-tests.mk
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright (C) 2022 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
--include external/ltp/android/ltp_package_list.mk
-
-include $(BUILD_SYSTEM)/tasks/tools/vts_package_utils.mk
-
-# Copy kernel test modules to testcases directories
-kernel_ltp_host_out := $(HOST_OUT_TESTCASES)/vts_kernel_ltp_tests
-kernel_ltp_vts_out := $(HOST_OUT)/$(test_suite_name)/android-$(test_suite_name)/testcases/vts_kernel_ltp_tests
-kernel_ltp_modules := \
- ltp \
- $(ltp_packages)
diff --git a/core/tasks/tools/vts_package_utils.mk b/core/tasks/tools/vts_package_utils.mk
deleted file mode 100644
index 1a819f2..0000000
--- a/core/tasks/tools/vts_package_utils.mk
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-# Copyright (C) 2020 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# $(1): List of target native files to copy.
-# $(2): Copy destination directory.
-# Evaluates to a list of ":"-separated pairs src:dst.
-define target-native-copy-pairs
-$(foreach m,$(1),\
- $(eval _built_files := $(strip $(ALL_MODULES.$(m).BUILT_INSTALLED)\
- $(ALL_MODULES.$(m)$(TARGET_2ND_ARCH_MODULE_SUFFIX).BUILT_INSTALLED)))\
- $(foreach i, $(sort $(_built_files)),\
- $(eval bui_ins := $(subst :,$(space),$(i)))\
- $(eval ins := $(word 2,$(bui_ins)))\
- $(if $(filter $(TARGET_OUT_ROOT)/%,$(ins)),\
- $(eval bui := $(word 1,$(bui_ins)))\
- $(eval my_copy_dest := $(patsubst data/%,DATA/%,\
- $(patsubst system/%,DATA/%,\
- $(patsubst $(PRODUCT_OUT)/%,%,$(ins)))))\
- $(call declare-copy-target-license-metadata,$(2)/$(my_copy_dest),$(bui))\
- $(bui):$(2)/$(my_copy_dest))))
-endef
diff --git a/core/tasks/vts-core-tests.mk b/core/tasks/vts-core-tests.mk
index 1eeb078..11bb932 100644
--- a/core/tasks/vts-core-tests.mk
+++ b/core/tasks/vts-core-tests.mk
@@ -16,15 +16,6 @@
test_suite_tradefed := vts-tradefed
test_suite_readme := test/vts/tools/vts-core-tradefed/README
-include $(BUILD_SYSTEM)/tasks/tools/vts-kernel-tests.mk
-
-ltp_copy_pairs := \
- $(call target-native-copy-pairs,$(kernel_ltp_modules),$(kernel_ltp_vts_out))
-
-copy_ltp_tests := $(call copy-many-files,$(ltp_copy_pairs))
-
-test_suite_extra_deps := $(copy_ltp_tests)
-
include $(BUILD_SYSTEM)/tasks/tools/compatibility.mk
.PHONY: vts
diff --git a/target/product/generic/Android.bp b/target/product/generic/Android.bp
index ea49af4..5451042 100644
--- a/target/product/generic/Android.bp
+++ b/target/product/generic/Android.bp
@@ -56,10 +56,6 @@
name: "bugreports",
},
{
- target: "/data/cache",
- name: "cache",
- },
- {
target: "/sys/kernel/debug",
name: "d",
},
@@ -125,6 +121,10 @@
target: "/system_ext",
name: "system/system_ext",
},
+ {
+ target: "/data/cache",
+ name: "cache",
+ },
]
filegroup {
@@ -346,11 +346,6 @@
],
}
-filegroup {
- name: "system_image_erofs_compress_hints.txt",
- srcs: ["erofs_compress_hints.txt"],
-}
-
android_filesystem_defaults {
name: "system_image_defaults",
partition_name: "system",
@@ -374,11 +369,6 @@
],
libs: [":framework-res{.export-package.apk}"],
},
- type: "erofs",
- erofs: {
- compressor: "lz4hc,9",
- compress_hints: ":system_image_erofs_compress_hints.txt",
- },
build_logtags: true,
gen_aconfig_flags_pb: true,
@@ -885,8 +875,13 @@
}
android_system_image {
- name: "generic_system_image",
+ name: "aosp_shared_system_image",
defaults: ["system_image_defaults"],
dirs: android_rootdirs,
symlinks: android_symlinks,
+ type: "erofs",
+ erofs: {
+ compressor: "lz4hc,9",
+ compress_hints: "erofs_compress_hints.txt",
+ },
}
diff --git a/target/product/gsi/Android.bp b/target/product/gsi/Android.bp
index 1b8f2d7..9e8946d 100644
--- a/target/product/gsi/Android.bp
+++ b/target/product/gsi/Android.bp
@@ -71,12 +71,21 @@
target: "/system/product",
name: "product",
},
+ {
+ target: "/odm/odm_dlkm/etc",
+ name: "odm_dlkm/etc",
+ },
+ {
+ target: "/vendor/vendor_dlkm/etc",
+ name: "vendor_dlkm/etc",
+ },
]
android_system_image {
name: "android_gsi",
defaults: ["system_image_defaults"],
symlinks: gsi_symlinks,
+ dirs: ["cache"],
deps: [
///////////////////////////////////////////
// gsi_system_ext
@@ -196,4 +205,5 @@
"true": true,
default: false,
}),
+ type: "ext4",
}
diff --git a/tools/aconfig/aconfig/src/storage/mod.rs b/tools/aconfig/aconfig/src/storage/mod.rs
index ef1b4f6..4bc72f7 100644
--- a/tools/aconfig/aconfig/src/storage/mod.rs
+++ b/tools/aconfig/aconfig/src/storage/mod.rs
@@ -83,7 +83,7 @@
p.boolean_start_index = boolean_start_index;
boolean_start_index += p.boolean_flags.len() as u32;
- if version > 2 {
+ if version >= 2 {
let mut flag_names_vec =
p.flag_names.clone().into_iter().map(String::from).collect::<Vec<_>>();
let fingerprint = compute_flags_fingerprint(&mut flag_names_vec);
@@ -202,6 +202,7 @@
assert!(packages[0].flag_names.contains("disabled_rw"));
assert!(packages[0].flag_names.contains("enabled_ro"));
assert_eq!(packages[0].boolean_start_index, 0);
+ assert_eq!(packages[0].fingerprint, 0);
assert_eq!(packages[1].package_name, "com.android.aconfig.storage.test_2");
assert_eq!(packages[1].package_id, 1);
@@ -210,6 +211,7 @@
assert!(packages[1].flag_names.contains("disabled_rw"));
assert!(packages[1].flag_names.contains("enabled_fixed_ro"));
assert_eq!(packages[1].boolean_start_index, 3);
+ assert_eq!(packages[0].fingerprint, 0);
assert_eq!(packages[2].package_name, "com.android.aconfig.storage.test_4");
assert_eq!(packages[2].package_id, 2);
@@ -217,5 +219,49 @@
assert!(packages[2].flag_names.contains("enabled_rw"));
assert!(packages[2].flag_names.contains("enabled_fixed_ro"));
assert_eq!(packages[2].boolean_start_index, 6);
+ assert_eq!(packages[2].fingerprint, 0);
+ }
+
+ #[test]
+ fn test_flag_package_with_fingerprint() {
+ let caches = parse_all_test_flags();
+ let packages = group_flags_by_package(caches.iter(), 2);
+
+ for pkg in packages.iter() {
+ let pkg_name = pkg.package_name;
+ assert_eq!(pkg.flag_names.len(), pkg.boolean_flags.len());
+ for pf in pkg.boolean_flags.iter() {
+ assert!(pkg.flag_names.contains(pf.name()));
+ assert_eq!(pf.package(), pkg_name);
+ }
+ }
+
+ assert_eq!(packages.len(), 3);
+
+ assert_eq!(packages[0].package_name, "com.android.aconfig.storage.test_1");
+ assert_eq!(packages[0].package_id, 0);
+ assert_eq!(packages[0].flag_names.len(), 3);
+ assert!(packages[0].flag_names.contains("enabled_rw"));
+ assert!(packages[0].flag_names.contains("disabled_rw"));
+ assert!(packages[0].flag_names.contains("enabled_ro"));
+ assert_eq!(packages[0].boolean_start_index, 0);
+ assert_eq!(packages[0].fingerprint, 15248948510590158086u64);
+
+ assert_eq!(packages[1].package_name, "com.android.aconfig.storage.test_2");
+ assert_eq!(packages[1].package_id, 1);
+ assert_eq!(packages[1].flag_names.len(), 3);
+ assert!(packages[1].flag_names.contains("enabled_ro"));
+ assert!(packages[1].flag_names.contains("disabled_rw"));
+ assert!(packages[1].flag_names.contains("enabled_fixed_ro"));
+ assert_eq!(packages[1].boolean_start_index, 3);
+ assert_eq!(packages[1].fingerprint, 4431940502274857964u64);
+
+ assert_eq!(packages[2].package_name, "com.android.aconfig.storage.test_4");
+ assert_eq!(packages[2].package_id, 2);
+ assert_eq!(packages[2].flag_names.len(), 2);
+ assert!(packages[2].flag_names.contains("enabled_rw"));
+ assert!(packages[2].flag_names.contains("enabled_fixed_ro"));
+ assert_eq!(packages[2].boolean_start_index, 6);
+ assert_eq!(packages[2].fingerprint, 16233229917711622375u64);
}
}
diff --git a/tools/edit_monitor/daemon_manager.py b/tools/edit_monitor/daemon_manager.py
index ef2a309..22782f7 100644
--- a/tools/edit_monitor/daemon_manager.py
+++ b/tools/edit_monitor/daemon_manager.py
@@ -33,7 +33,7 @@
DEFAULT_PROCESS_TERMINATION_TIMEOUT_SECONDS = 5
DEFAULT_MONITOR_INTERVAL_SECONDS = 5
-DEFAULT_MEMORY_USAGE_THRESHOLD = 3 * 1024 # 3GB
+DEFAULT_MEMORY_USAGE_THRESHOLD = 0.02 # 2% of total memory
DEFAULT_CPU_USAGE_THRESHOLD = 200
DEFAULT_REBOOT_TIMEOUT_SECONDS = 60 * 60 * 24
BLOCK_SIGN_FILE = "edit_monitor_block_sign"
@@ -70,6 +70,9 @@
self.max_memory_usage = 0
self.max_cpu_usage = 0
+ self.total_memory_size = os.sysconf("SC_PAGE_SIZE") * os.sysconf(
+ "SC_PHYS_PAGES"
+ )
pid_file_dir = pathlib.Path(tempfile.gettempdir()).joinpath("edit_monitor")
pid_file_dir.mkdir(parents=True, exist_ok=True)
@@ -142,10 +145,20 @@
logging.warning("Failed to monitor daemon process with error: %s", e)
if self.max_memory_usage >= memory_threshold:
- self._handle_resource_exhausted_error("memory")
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE
+ )
+ logging.error(
+ "Daemon process is consuming too much memory, rebooting...")
+ self.reboot()
if self.max_cpu_usage >= cpu_threshold:
- self._handle_resource_exhausted_error("cpu")
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE
+ )
+ logging.error(
+ "Daemon process is consuming too much cpu, killing...")
+ self._terminate_process(self.daemon_process.pid)
logging.info(
"Daemon process %d terminated. Max memory usage: %f, Max cpu"
@@ -345,7 +358,13 @@
stat_data = f.readline().split()
# RSS is the 24th field in /proc/[pid]/stat
rss_pages = int(stat_data[23])
- return rss_pages * 4 / 1024 # Covert to MB
+ process_memory = rss_pages * 4 * 1024 # Convert to bytes
+
+ return (
+ process_memory / self.total_memory_size
+ if self.total_memory_size
+ else 0.0
+ )
def _get_process_cpu_percent(self, pid: int, interval: int = 1) -> float:
total_start_time = self._get_total_cpu_time(pid)
@@ -386,20 +405,6 @@
return pids
- def _handle_resource_exhausted_error(self, resource_type:str):
- if resource_type == "memory":
- self._send_error_event_to_clearcut(
- edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE
- )
- else:
- self._send_error_event_to_clearcut(
- edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE
- )
- logging.error(
- "Daemon process is consuming too much %s, killing...", resource_type
- ),
- self._terminate_process(self.daemon_process.pid)
-
def _send_error_event_to_clearcut(self, error_type):
edit_monitor_error_event_proto = edit_event_pb2.EditEvent(
user_name=self.user_name,
diff --git a/tools/edit_monitor/daemon_manager_test.py b/tools/edit_monitor/daemon_manager_test.py
index 12aaab3..350739d 100644
--- a/tools/edit_monitor/daemon_manager_test.py
+++ b/tools/edit_monitor/daemon_manager_test.py
@@ -202,23 +202,31 @@
fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
)
- def test_monitor_daemon_subprocess_killed_high_memory_usage(self):
+ @mock.patch('os.execv')
+ def test_monitor_reboot_with_high_memory_usage(self, mock_execv):
fake_cclient = FakeClearcutClient()
+ binary_file = tempfile.NamedTemporaryFile(
+ dir=self.working_dir.name, delete=False
+ )
+
dm = daemon_manager.DaemonManager(
- TEST_BINARY_FILE,
+ binary_file.name,
daemon_target=memory_consume_daemon_target,
daemon_args=(2,),
cclient=fake_cclient,
)
+ # set the fake total_memory_size
+ dm.total_memory_size = 100 * 1024 *1024
dm.start()
- dm.monitor_daemon(interval=1, memory_threshold=2)
+ dm.monitor_daemon(interval=1)
- self.assertTrue(dm.max_memory_usage >= 2)
+ self.assertTrue(dm.max_memory_usage >= 0.02)
self.assert_no_subprocess_running()
self._assert_error_event_logged(
fake_cclient,
edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE,
)
+ mock_execv.assert_called_once()
def test_monitor_daemon_subprocess_killed_high_cpu_usage(self):
fake_cclient = FakeClearcutClient()