From 3640694a056e7958d005f7d98d2470497eb3be47 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 2 Oct 2026 15:33:02 -0600 Subject: [PATCH] api: forked workers die on SIGTERM before Sanic installs their handlers A Sanic worker is forked from the manager and carries the manager's Python SIGTERM/SIGINT handlers until `_setup_system_signals` installs its own, which happens after the startup listeners (about two seconds with sixteen workers starting at once). When auto-reload fires a second time inside that window the manager terminates the half-started generation, each child runs `WorkerManager.shutdown_signal` instead of exiting, marks itself TERMINATED, and then finishes starting. The manager has already spawned replacements, so the old generation lives on: untracked, holding the listening socket, serving stale code, and holding ~220 MB each. Register an at-fork hook in the main process that resets both signals to their default action in the child. The perpetual process already did this by hand for itself; server workers now get the same treatment. Co-Authored-By: Claude Fable 5.1 --- wrolpi/api_utils.py | 13 +++++++++++++ wrolpi/test/test_perpetual.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/wrolpi/api_utils.py b/wrolpi/api_utils.py index c07eebf6..c23cc2b2 100644 --- a/wrolpi/api_utils.py +++ b/wrolpi/api_utils.py @@ -2,6 +2,7 @@ import asyncio import json import logging import os +import signal from datetime import datetime, timezone, date from decimal import Decimal from functools import wraps @@ -23,6 +24,18 @@ logger = logger.getChild(__name__) Sanic.start_method = "fork" Sanic.START_METHOD_SET = True + +def _reset_inherited_signal_handlers(): + """A forked worker carries the manager's SIGTERM/SIGINT handlers until Sanic installs the worker's own. In + that window (startup listeners, ~2s with many workers) a signal must do the default thing, exit, rather than + run the manager's handler inside the child. Otherwise a second auto-reload leaves the half-started + generation alive, untracked, holding the listening socket.""" + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, signal.SIG_DFL) + + +os.register_at_fork(after_in_child=_reset_inherited_signal_handlers) + # The only Sanic App, this is imported all over. api_app = Sanic(name='api_app', log_config=LOGGING_CONFIG) diff --git a/wrolpi/test/test_perpetual.py b/wrolpi/test/test_perpetual.py index 233704c8..647f1ae6 100644 --- a/wrolpi/test/test_perpetual.py +++ b/wrolpi/test/test_perpetual.py @@ -369,3 +369,38 @@ def test_flag_context_manager_records_its_holder(async_client, flags_lock): assert api_app.shared_ctx.flag_holders['file_worker_modeling'] == os.getpid() assert flags.file_worker_modeling.holder_pid() is None assert not flags.file_worker_modeling.is_set() + + +def _sleep_then_exit_zero(): + # Stands in for a Sanic worker between fork and `_setup_system_signals`: no handlers of its own yet. + time.sleep(3) + + +def _parent_handler_that_returns(signum, frame): + # Stands in for `WorkerManager.shutdown_signal`: it returns, it does not exit. + pass + + +def test_forked_child_dies_on_sigterm_before_installing_handlers(): + """A forked worker inherits the manager's Python signal handlers. Until Sanic installs the worker's own, + SIGTERM must kill the child (the default action) rather than run the manager's handler inside it, which lets a + half-started worker survive a restart as an untracked orphan.""" + saved = {s: signal.getsignal(s) for s in (signal.SIGINT, signal.SIGTERM)} + try: + for sig in saved: + signal.signal(sig, _parent_handler_that_returns) + process = multiprocessing.get_context('fork').Process(target=_sleep_then_exit_zero) + process.start() + try: + time.sleep(0.3) + process.terminate() # What `WorkerProcess._terminate_now` sends on restart. + process.join(5) + assert process.exitcode == -signal.SIGTERM, \ + f'child survived SIGTERM (exitcode={process.exitcode}); it ran the inherited handler' + finally: + if process.is_alive(): + process.kill() + process.join(5) + finally: + for sig, handler in saved.items(): + signal.signal(sig, handler)