api: forked workers die on SIGTERM before Sanic installs their handlers

A Sanic worker is forked from the manager and carries the manager's
Python SIGTERM/SIGINT handlers until `_setup_system_signals` installs its
own, which happens after the startup listeners (about two seconds with
sixteen workers starting at once).  When auto-reload fires a second time
inside that window the manager terminates the half-started generation,
each child runs `WorkerManager.shutdown_signal` instead of exiting, marks
itself TERMINATED, and then finishes starting.  The manager has already
spawned replacements, so the old generation lives on: untracked, holding
the listening socket, serving stale code, and holding ~220 MB each.

Register an at-fork hook in the main process that resets both signals to
their default action in the child.  The perpetual process already did
this by hand for itself; server workers now get the same treatment.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-10-02 15:33:02 -06:00
parent 5c78aa332d
commit 3640694a05
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
2 changed files with 48 additions and 0 deletions

View file

@ -2,6 +2,7 @@ import asyncio
import json
import logging
import os
import signal
from datetime import datetime, timezone, date
from decimal import Decimal
from functools import wraps
@ -23,6 +24,18 @@ logger = logger.getChild(__name__)
Sanic.start_method = "fork"
Sanic.START_METHOD_SET = True
def _reset_inherited_signal_handlers():
"""A forked worker carries the manager's SIGTERM/SIGINT handlers until Sanic installs the worker's own. In
that window (startup listeners, ~2s with many workers) a signal must do the default thing, exit, rather than
run the manager's handler inside the child. Otherwise a second auto-reload leaves the half-started
generation alive, untracked, holding the listening socket."""
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, signal.SIG_DFL)
os.register_at_fork(after_in_child=_reset_inherited_signal_handlers)
# The only Sanic App, this is imported all over.
api_app = Sanic(name='api_app', log_config=LOGGING_CONFIG)

View file

@ -369,3 +369,38 @@ def test_flag_context_manager_records_its_holder(async_client, flags_lock):
assert api_app.shared_ctx.flag_holders['file_worker_modeling'] == os.getpid()
assert flags.file_worker_modeling.holder_pid() is None
assert not flags.file_worker_modeling.is_set()
def _sleep_then_exit_zero():
# Stands in for a Sanic worker between fork and `_setup_system_signals`: no handlers of its own yet.
time.sleep(3)
def _parent_handler_that_returns(signum, frame):
# Stands in for `WorkerManager.shutdown_signal`: it returns, it does not exit.
pass
def test_forked_child_dies_on_sigterm_before_installing_handlers():
"""A forked worker inherits the manager's Python signal handlers. Until Sanic installs the worker's own,
SIGTERM must kill the child (the default action) rather than run the manager's handler inside it, which lets a
half-started worker survive a restart as an untracked orphan."""
saved = {s: signal.getsignal(s) for s in (signal.SIGINT, signal.SIGTERM)}
try:
for sig in saved:
signal.signal(sig, _parent_handler_that_returns)
process = multiprocessing.get_context('fork').Process(target=_sleep_then_exit_zero)
process.start()
try:
time.sleep(0.3)
process.terminate() # What `WorkerProcess._terminate_now` sends on restart.
process.join(5)
assert process.exitcode == -signal.SIGTERM, \
f'child survived SIGTERM (exitcode={process.exitcode}); it ran the inherited handler'
finally:
if process.is_alive():
process.kill()
process.join(5)
finally:
for sig, handler in saved.items():
signal.signal(sig, handler)