Synchronize the setup process of edit monitor
This cl ensures only one edit monitor instance is in the setup proecess
by requesting exclusive file lock on a setup lock file. If there is
already an edit monitor in setup process, other instances trying to
setup will fail. This prevents race conditions when multiple edit
monitor instance are starting.
Test: atest daemon_manager_test
bug: 382135550
Change-Id: Ia8bd2f35318ae81fc002ccce62e2fde3b8cae26b
diff --git a/tools/edit_monitor/daemon_manager.py b/tools/edit_monitor/daemon_manager.py
index 36a7593..7d666fe 100644
--- a/tools/edit_monitor/daemon_manager.py
+++ b/tools/edit_monitor/daemon_manager.py
@@ -13,6 +13,8 @@
# limitations under the License.
+import errno
+import fcntl
import getpass
import hashlib
import logging
@@ -100,16 +102,32 @@
logging.warning("Edit monitor for cog is not supported, exiting...")
return
- try:
- self._stop_any_existing_instance()
- self._write_pid_to_pidfile()
- self._start_daemon_process()
- except Exception as e:
- logging.exception("Failed to start daemon manager with error %s", e)
- self._send_error_event_to_clearcut(
- edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
- )
- raise e
+ setup_lock_file = pathlib.Path(tempfile.gettempdir()).joinpath(
+ self.pid_file_path.name + ".setup"
+ )
+ logging.info("setup lock file: %s", setup_lock_file)
+ with open(setup_lock_file, "w") as f:
+ try:
+ # Acquire an exclusive lock
+ fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ self._stop_any_existing_instance()
+ self._write_pid_to_pidfile()
+ self._start_daemon_process()
+ except Exception as e:
+ if (
+ isinstance(e, IOError) and e.errno == errno.EAGAIN
+ ): # Failed to acquire the file lock.
+ logging.warning("Another edit monitor is starting, exitinng...")
+ return
+ else:
+ logging.exception("Failed to start daemon manager with error %s", e)
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
+ )
+ raise e
+ finally:
+ # Release the lock
+ fcntl.flock(f, fcntl.LOCK_UN)
def monitor_daemon(
self,
@@ -414,18 +432,18 @@
pids = []
try:
- output = subprocess.check_output(
- ["ps", "-ef", "--no-headers"], text=True)
+ output = subprocess.check_output(["ps", "-ef", "--no-headers"], text=True)
for line in output.splitlines():
- parts = line.split()
- process_path = parts[7]
- if pathlib.Path(process_path).name == 'edit_monitor':
- pid = int(parts[1])
- if pid != self.pid: # exclude the current process
- pids.append(pid)
+ parts = line.split()
+ process_path = parts[7]
+ if pathlib.Path(process_path).name == "edit_monitor":
+ pid = int(parts[1])
+ if pid != self.pid: # exclude the current process
+ pids.append(pid)
except Exception:
logging.exception(
- "Failed to get pids of existing edit monitors from ps command.")
+ "Failed to get pids of existing edit monitors from ps command."
+ )
return pids