commit 78fd076d999151f5880aaf7228b5bb513c7d2354
author Zhuoyao Zhang <zhuoyao@google.com> Mon Nov 18 22:15:42 2024 +0000
committer Zhuoyao Zhang <zhuoyao@google.com> Mon Nov 18 23:36:14 2024 +0000
tree aacab34428c4f1734a40b00251b100abc3292aa0
parent 6988272f77745b8cacdf58814ec609fbf47401cb

Reboot edit monitor when memory exhausted

Instead of teminating, reboot the edit monitor when memory usage exceeds
the threshold

Test: atest daemon_manager_test
Bug: 379747487
Change-Id: If90309e720aa0d095bc1eca93521a01ed63575da

tools/edit_monitor/daemon_manager.py

diff --git a/tools/edit_monitor/daemon_manager.py b/tools/edit_monitor/daemon_manager.py
index c73603c..22782f7 100644
--- a/tools/edit_monitor/daemon_manager.py
+++ b/tools/edit_monitor/daemon_manager.py
@@ -145,10 +145,20 @@
         logging.warning("Failed to monitor daemon process with error: %s", e)
 
       if self.max_memory_usage >= memory_threshold:
-        self._handle_resource_exhausted_error("memory")
+        self._send_error_event_to_clearcut(
+            edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE
+        )
+        logging.error(
+            "Daemon process is consuming too much memory, rebooting...")
+        self.reboot()
 
       if self.max_cpu_usage >= cpu_threshold:
-        self._handle_resource_exhausted_error("cpu")
+        self._send_error_event_to_clearcut(
+            edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE
+        )
+        logging.error(
+            "Daemon process is consuming too much cpu, killing...")
+        self._terminate_process(self.daemon_process.pid)
 
     logging.info(
         "Daemon process %d terminated. Max memory usage: %f, Max cpu"
@@ -395,20 +405,6 @@
 
     return pids
 
-  def _handle_resource_exhausted_error(self, resource_type:str):
-    if resource_type == "memory":
-      self._send_error_event_to_clearcut(
-          edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE
-      )
-    else:
-      self._send_error_event_to_clearcut(
-          edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE
-      )
-    logging.error(
-        "Daemon process is consuming too much %s, killing...", resource_type
-    ),
-    self._terminate_process(self.daemon_process.pid)
-
   def _send_error_event_to_clearcut(self, error_type):
     edit_monitor_error_event_proto = edit_event_pb2.EditEvent(
         user_name=self.user_name,

tools/edit_monitor/daemon_manager_test.py

diff --git a/tools/edit_monitor/daemon_manager_test.py b/tools/edit_monitor/daemon_manager_test.py
index e2b3f22..350739d 100644
--- a/tools/edit_monitor/daemon_manager_test.py
+++ b/tools/edit_monitor/daemon_manager_test.py
@@ -202,11 +202,15 @@
         fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
     )
 
-  def test_monitor_daemon_subprocess_killed_high_memory_usage(self):
+  @mock.patch('os.execv')
+  def test_monitor_reboot_with_high_memory_usage(self, mock_execv):
     fake_cclient = FakeClearcutClient()
+    binary_file = tempfile.NamedTemporaryFile(
+        dir=self.working_dir.name, delete=False
+    )
 
     dm = daemon_manager.DaemonManager(
-        TEST_BINARY_FILE,
+        binary_file.name,
         daemon_target=memory_consume_daemon_target,
         daemon_args=(2,),
         cclient=fake_cclient,
@@ -222,6 +226,7 @@
         fake_cclient,
         edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE,
     )
+    mock_execv.assert_called_once()
 
   def test_monitor_daemon_subprocess_killed_high_cpu_usage(self):
     fake_cclient = FakeClearcutClient()