Reboot edit monitor when memory exhausted
Instead of teminating, reboot the edit monitor when memory usage exceeds
the threshold
Test: atest daemon_manager_test
Bug: 379747487
Change-Id: If90309e720aa0d095bc1eca93521a01ed63575da
diff --git a/tools/edit_monitor/daemon_manager.py b/tools/edit_monitor/daemon_manager.py
index c73603c..22782f7 100644
--- a/tools/edit_monitor/daemon_manager.py
+++ b/tools/edit_monitor/daemon_manager.py
@@ -145,10 +145,20 @@
logging.warning("Failed to monitor daemon process with error: %s", e)
if self.max_memory_usage >= memory_threshold:
- self._handle_resource_exhausted_error("memory")
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE
+ )
+ logging.error(
+ "Daemon process is consuming too much memory, rebooting...")
+ self.reboot()
if self.max_cpu_usage >= cpu_threshold:
- self._handle_resource_exhausted_error("cpu")
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE
+ )
+ logging.error(
+ "Daemon process is consuming too much cpu, killing...")
+ self._terminate_process(self.daemon_process.pid)
logging.info(
"Daemon process %d terminated. Max memory usage: %f, Max cpu"
@@ -395,20 +405,6 @@
return pids
- def _handle_resource_exhausted_error(self, resource_type:str):
- if resource_type == "memory":
- self._send_error_event_to_clearcut(
- edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE
- )
- else:
- self._send_error_event_to_clearcut(
- edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE
- )
- logging.error(
- "Daemon process is consuming too much %s, killing...", resource_type
- ),
- self._terminate_process(self.daemon_process.pid)
-
def _send_error_event_to_clearcut(self, error_type):
edit_monitor_error_event_proto = edit_event_pb2.EditEvent(
user_name=self.user_name,