mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 23:16:17 -05:00
api: forked workers die on SIGTERM before Sanic installs their handlers
A Sanic worker is forked from the manager and carries the manager's Python SIGTERM/SIGINT handlers until `_setup_system_signals` installs its own, which happens after the startup listeners (about two seconds with sixteen workers starting at once). When auto-reload fires a second time inside that window the manager terminates the half-started generation, each child runs `WorkerManager.shutdown_signal` instead of exiting, marks itself TERMINATED, and then finishes starting. The manager has already spawned replacements, so the old generation lives on: untracked, holding the listening socket, serving stale code, and holding ~220 MB each. Register an at-fork hook in the main process that resets both signals to their default action in the child. The perpetual process already did this by hand for itself; server workers now get the same treatment. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
5c78aa332d
commit
3640694a05
2 changed files with 48 additions and 0 deletions
|
|
@ -2,6 +2,7 @@ import asyncio
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
from datetime import datetime, timezone, date
|
||||
from decimal import Decimal
|
||||
from functools import wraps
|
||||
|
|
@ -23,6 +24,18 @@ logger = logger.getChild(__name__)
|
|||
Sanic.start_method = "fork"
|
||||
Sanic.START_METHOD_SET = True
|
||||
|
||||
|
||||
def _reset_inherited_signal_handlers():
|
||||
"""A forked worker carries the manager's SIGTERM/SIGINT handlers until Sanic installs the worker's own. In
|
||||
that window (startup listeners, ~2s with many workers) a signal must do the default thing, exit, rather than
|
||||
run the manager's handler inside the child. Otherwise a second auto-reload leaves the half-started
|
||||
generation alive, untracked, holding the listening socket."""
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
signal.signal(sig, signal.SIG_DFL)
|
||||
|
||||
|
||||
os.register_at_fork(after_in_child=_reset_inherited_signal_handlers)
|
||||
|
||||
# The only Sanic App, this is imported all over.
|
||||
api_app = Sanic(name='api_app', log_config=LOGGING_CONFIG)
|
||||
|
||||
|
|
|
|||
|
|
@ -369,3 +369,38 @@ def test_flag_context_manager_records_its_holder(async_client, flags_lock):
|
|||
assert api_app.shared_ctx.flag_holders['file_worker_modeling'] == os.getpid()
|
||||
assert flags.file_worker_modeling.holder_pid() is None
|
||||
assert not flags.file_worker_modeling.is_set()
|
||||
|
||||
|
||||
def _sleep_then_exit_zero():
|
||||
# Stands in for a Sanic worker between fork and `_setup_system_signals`: no handlers of its own yet.
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
def _parent_handler_that_returns(signum, frame):
|
||||
# Stands in for `WorkerManager.shutdown_signal`: it returns, it does not exit.
|
||||
pass
|
||||
|
||||
|
||||
def test_forked_child_dies_on_sigterm_before_installing_handlers():
|
||||
"""A forked worker inherits the manager's Python signal handlers. Until Sanic installs the worker's own,
|
||||
SIGTERM must kill the child (the default action) rather than run the manager's handler inside it, which lets a
|
||||
half-started worker survive a restart as an untracked orphan."""
|
||||
saved = {s: signal.getsignal(s) for s in (signal.SIGINT, signal.SIGTERM)}
|
||||
try:
|
||||
for sig in saved:
|
||||
signal.signal(sig, _parent_handler_that_returns)
|
||||
process = multiprocessing.get_context('fork').Process(target=_sleep_then_exit_zero)
|
||||
process.start()
|
||||
try:
|
||||
time.sleep(0.3)
|
||||
process.terminate() # What `WorkerProcess._terminate_now` sends on restart.
|
||||
process.join(5)
|
||||
assert process.exitcode == -signal.SIGTERM, \
|
||||
f'child survived SIGTERM (exitcode={process.exitcode}); it ran the inherited handler'
|
||||
finally:
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join(5)
|
||||
finally:
|
||||
for sig, handler in saved.items():
|
||||
signal.signal(sig, handler)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue