Make edit monitor error type clearer
Before this change, edit monitor will send a EXCEEDED_RESOURCE_USAGE
error to clearcut, however, it does not tell whether it is due to memory
or CPU usage overload. This CL separate the errory type to
EXCEEDED_MEMORY_USAGE and EXCEEDED_CPU_USAGE to make it easier to debug
Test: atest daemon_manager_test
Bug: 365617369
Change-Id: Ib0605249d87544c0a2ec82d72f05b2cc9aaa17c1
diff --git a/tools/edit_monitor/daemon_manager.py b/tools/edit_monitor/daemon_manager.py
index afdefc5..328b6ee 100644
--- a/tools/edit_monitor/daemon_manager.py
+++ b/tools/edit_monitor/daemon_manager.py
@@ -141,17 +141,11 @@
# Logging the error and continue.
logging.warning("Failed to monitor daemon process with error: %s", e)
- if (
- self.max_memory_usage >= memory_threshold
- or self.max_cpu_usage >= cpu_threshold
- ):
- logging.error(
- "Daemon process is consuming too much resource, killing..."
- ),
- self._send_error_event_to_clearcut(
- edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE
- )
- self._terminate_process(self.daemon_process.pid)
+ if self.max_memory_usage >= memory_threshold:
+ self._handle_resource_exhausted_error("memory")
+
+ if self.max_cpu_usage >= cpu_threshold:
+ self._handle_resource_exhausted_error("cpu")
logging.info(
"Daemon process %d terminated. Max memory usage: %f, Max cpu"
@@ -392,6 +386,20 @@
return pids
+ def _handle_resource_exhausted_error(self, resource_type:str):
+ if resource_type == "memory":
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE
+ )
+ else:
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE
+ )
+ logging.error(
+ "Daemon process is consuming too much %s, killing...", resource_type
+ ),
+ self._terminate_process(self.daemon_process.pid)
+
def _send_error_event_to_clearcut(self, error_type):
edit_monitor_error_event_proto = edit_event_pb2.EditEvent(
user_name=self.user_name,
diff --git a/tools/edit_monitor/daemon_manager_test.py b/tools/edit_monitor/daemon_manager_test.py
index 8d18388..12aaab3 100644
--- a/tools/edit_monitor/daemon_manager_test.py
+++ b/tools/edit_monitor/daemon_manager_test.py
@@ -217,7 +217,7 @@
self.assert_no_subprocess_running()
self._assert_error_event_logged(
fake_cclient,
- edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE,
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE,
)
def test_monitor_daemon_subprocess_killed_high_cpu_usage(self):
@@ -235,7 +235,7 @@
self.assert_no_subprocess_running()
self._assert_error_event_logged(
fake_cclient,
- edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE,
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE,
)
@mock.patch('subprocess.check_output')
diff --git a/tools/edit_monitor/proto/edit_event.proto b/tools/edit_monitor/proto/edit_event.proto
index dc3d3f6..9acc2e7 100644
--- a/tools/edit_monitor/proto/edit_event.proto
+++ b/tools/edit_monitor/proto/edit_event.proto
@@ -16,8 +16,9 @@
FAILED_TO_START_EDIT_MONITOR = 1;
FAILED_TO_STOP_EDIT_MONITOR = 2;
FAILED_TO_REBOOT_EDIT_MONITOR = 3;
- KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE = 4;
+ KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE = 4;
FORCE_CLEANUP = 5;
+ KILLED_DUE_TO_EXCEEDED_CPU_USAGE = 6;
}
// Event that logs a single edit