mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
* api: run the perpetual loops in one Sanic-managed process The singleton background loops (file worker, downloads, switches, jobs, bulk tag, flag checks, update check) now run in a dedicated process that Sanic's Worker Manager owns: `run_perpetual_process` is registered with `app.manager.manage()` from `main_process_ready`. The manager guarantees a single instance and SIGTERMs it on shutdown. No server worker runs the loops, so the owner-pid / heartbeat / watchdog claim scheme is deleted rather than hardened. `perpetual_signal` keeps its name and is now a plain registrar. A new `per_worker_task` registrar covers loops every server worker must run for itself: syncing its log level (previously only the owner did, so the other workers never picked up level changes) and a health check that asks the manager to restart the perpetual process when it is FAILED (Sanic marks a dead managed process but does not restart it). Downloads are started as background tasks instead of Sanic signals; the perpetual process has no server, so no finalized signal router. The update check and the upgrade-check endpoint run git in a thread via `refresh_update_status`. /api/status lists the perpetual process next to the servers. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * perpetual: survive a predecessor's death, exit cleanly, coordinate restarts Review fixes for the managed perpetual process. Shutdown: cancel_background_tasks / cancel_refresh_tasks gather with return_exceptions, so a cancelled child's CancelledError no longer aborts the shutdown that called them (the process exited 1 with a download in flight). Restart after a crash: reconcile_after_predecessor heals shared locks a dead process left held, fails RUNNING Jobs owned by a dead pid, clears processing_domains, renews pending Downloads, and resets the file-worker status before the loops start. N server workers issue one restart request per cooldown through a shared timestamp in shared_ctx. The log-level sync now also runs in the perpetual process. Manage() is registered through register_perpetual_process so it is testable; the entry point refuses to run with no loops registered (spawn). Tests exercise the real entry point in a forked process with a live background task under SIGINT and SIGTERM, Sanic's real WorkerManager marking a crashed managed process FAILED and reviving it on restart, Sanic's process naming and ProcessState enum, the listener registration, and the reconcile step. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * perpetual: import the jobs helper inside its reconcile step Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
212 lines
7.7 KiB
Python
212 lines
7.7 KiB
Python
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
from datetime import datetime, timezone, date
|
|
from decimal import Decimal
|
|
from functools import wraps
|
|
from http import HTTPStatus
|
|
from pathlib import Path
|
|
|
|
from sanic import response, HTTPResponse, Request, Sanic, SanicException
|
|
|
|
from wrolpi.common import Base, get_media_directory, logger, LOGGING_CONFIG
|
|
from wrolpi.errors import APIError
|
|
from wrolpi.perpetual import PERPETUAL_LOOPS, PER_WORKER_TASKS, PerpetualLoop, run_loop
|
|
from wrolpi.vars import PYTEST
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
# Sanic 25.x wants to use 'spawn' as the multiprocessing start method, but our code
|
|
# uses 'fork' extensively (via multiprocessing.Manager, Event, Queue, etc.).
|
|
# Tell Sanic to use 'fork' and that the start method is already configured.
|
|
Sanic.start_method = "fork"
|
|
Sanic.START_METHOD_SET = True
|
|
|
|
# The only Sanic App, this is imported all over.
|
|
api_app = Sanic(name='api_app', log_config=LOGGING_CONFIG)
|
|
|
|
|
|
@wraps(response.json)
|
|
def json_response(*a, **kwargs) -> HTTPResponse:
|
|
"""
|
|
Handles encoding date/datetime in JSON.
|
|
"""
|
|
resp = response.json(*a, **kwargs, cls=CustomJSONEncoder, dumps=json.dumps)
|
|
return resp
|
|
|
|
|
|
class CustomJSONEncoder(json.JSONEncoder):
|
|
|
|
def default(self, obj):
|
|
try:
|
|
if hasattr(obj, '__json__'):
|
|
# Get __json__ before others.
|
|
return obj.__json__()
|
|
elif isinstance(obj, datetime):
|
|
# API always returns dates in UTC.
|
|
if obj.tzinfo:
|
|
obj = obj.astimezone(timezone.utc)
|
|
else:
|
|
# A datetime with no timezone is UTC.
|
|
obj = obj.replace(tzinfo=timezone.utc)
|
|
obj = obj.isoformat()
|
|
return obj
|
|
elif isinstance(obj, date):
|
|
# API always returns dates in UTC.
|
|
obj = datetime(obj.year, obj.month, obj.day, tzinfo=timezone.utc)
|
|
return obj.isoformat()
|
|
elif isinstance(obj, Decimal):
|
|
return str(obj)
|
|
elif isinstance(obj, Base):
|
|
if hasattr(obj, 'dict'):
|
|
return obj.dict()
|
|
elif isinstance(obj, Path):
|
|
media_directory = get_media_directory()
|
|
try:
|
|
path = obj.relative_to(media_directory)
|
|
except ValueError:
|
|
# Path may not be absolute.
|
|
path = obj
|
|
if str(path) == '.':
|
|
return ''
|
|
return str(path)
|
|
return super(CustomJSONEncoder, self).default(obj)
|
|
except Exception as e:
|
|
logger.fatal(f'Failed to JSON encode {obj}', exc_info=e)
|
|
raise
|
|
|
|
|
|
def get_error_json(exception: BaseException):
|
|
"""Return a JSON representation of the Exception instance."""
|
|
if isinstance(exception, APIError):
|
|
# An exception from WROLPi.
|
|
body = dict(error=str(exception), message=exception.summary, code=exception.code)
|
|
elif isinstance(exception, SanicException):
|
|
# An exception from Sanic.
|
|
body = dict(error=str(exception), message=exception.message, code=type(exception).__name__)
|
|
else:
|
|
# Not a WROLPi APIError error.
|
|
body = dict(
|
|
error=str(exception),
|
|
message=None,
|
|
code=type(exception).__name__,
|
|
)
|
|
if exception.__cause__:
|
|
# This exception was caused by another, follow the stack.
|
|
body['cause'] = get_error_json(exception.__cause__)
|
|
return body
|
|
|
|
|
|
def json_error_handler(request: Request, exception: Exception):
|
|
"""Converts all API APIError/SanicException to more informative json object."""
|
|
try:
|
|
body = get_error_json(exception)
|
|
except Exception as e:
|
|
logger.error('Failed to create error json', exc_info=e)
|
|
raise
|
|
|
|
error = repr(str(body["error"]))
|
|
message = repr(str(body["message"]))
|
|
code = body['code']
|
|
if logger.isEnabledFor(logging.DEBUG):
|
|
logger.error(f'API returning JSON error {type(exception).__name__} {error=} {message=} {code=}',
|
|
exc_info=exception)
|
|
else:
|
|
logger.error(f'API returning JSON error {type(exception).__name__} {error=} {message=} {code=}')
|
|
if isinstance(exception, SanicException):
|
|
return json_response(body, exception.status_code)
|
|
|
|
logger.error('Unexpected error', exc_info=exception)
|
|
|
|
# Some unknown error, use internal error code.
|
|
return json_response(body, HTTPStatus.INTERNAL_SERVER_ERROR)
|
|
|
|
|
|
api_app.error_handler.add(Exception, json_error_handler)
|
|
|
|
|
|
@api_app.middleware('request')
|
|
async def inject_session(request: Request):
|
|
"""Inject a database session into the request context.
|
|
|
|
This session is deliberately *deferred*, even for a POST: it is committed by the response
|
|
middleware, so a writer would hold the SQLite write lock for the whole handler -- including the
|
|
slow parts (a chunk being written to disk, a subprocess) and including any background task the
|
|
handler fires before returning. A handler that writes should use `get_db_session(commit=True)`,
|
|
which begins as a writer and commits before the handler returns.
|
|
"""
|
|
from wrolpi.db import get_db_context
|
|
engine, session = get_db_context()
|
|
request.ctx.session = session
|
|
request.ctx._db_engine = engine
|
|
|
|
|
|
@api_app.middleware('response')
|
|
async def cleanup_session(request: Request, response_: HTTPResponse):
|
|
"""Cleanup session after request completes."""
|
|
if hasattr(request.ctx, 'session'):
|
|
session = request.ctx.session
|
|
try:
|
|
if response_.status < 400:
|
|
session.commit()
|
|
else:
|
|
session.rollback()
|
|
except Exception:
|
|
session.rollback()
|
|
raise
|
|
finally:
|
|
# Don't close session during tests - the test_session fixture manages it.
|
|
if not PYTEST:
|
|
session.close()
|
|
return response_
|
|
|
|
|
|
def perpetual_signal(sleep: int | float = 1, run_while_testing: bool = False):
|
|
"""Register a singleton background loop. The wrapped function is called forever, `sleep` seconds after each
|
|
call returns, in the ONE perpetual process that Sanic's Worker Manager owns (see wrolpi/perpetual.py). Errors
|
|
are logged and the loop continues; a long call is not re-entered until it returns.
|
|
|
|
The function is returned unchanged so tests can call it directly. Under pytest nothing is registered unless
|
|
`run_while_testing` is set."""
|
|
|
|
def wrapper(func: callable):
|
|
if PYTEST and not run_while_testing:
|
|
return func
|
|
PERPETUAL_LOOPS.append(PerpetualLoop(func.__name__, func, sleep))
|
|
return func
|
|
|
|
return wrapper
|
|
|
|
|
|
def per_worker_task(sleep: int | float = 1):
|
|
"""Register a loop that EVERY Sanic server worker runs for itself (e.g. syncing its own log level). Anything
|
|
that must happen exactly once belongs in `perpetual_signal` instead."""
|
|
|
|
def wrapper(func: callable):
|
|
if PYTEST:
|
|
return func
|
|
PER_WORKER_TASKS.append(PerpetualLoop(func.__name__, func, sleep))
|
|
return func
|
|
|
|
return wrapper
|
|
|
|
|
|
@api_app.after_server_start
|
|
async def start_per_worker_tasks(app: Sanic):
|
|
stop = asyncio.Event()
|
|
app.ctx.per_worker_stop = stop
|
|
app.ctx.per_worker_tasks = [asyncio.create_task(run_loop(loop, stop), name=loop.name) for loop in PER_WORKER_TASKS]
|
|
logger.info(f'start_per_worker_tasks pid={os.getpid()} tasks={[i.name for i in PER_WORKER_TASKS]}')
|
|
|
|
|
|
@api_app.listener('before_server_stop')
|
|
async def stop_per_worker_tasks(app: Sanic):
|
|
stop = getattr(app.ctx, 'per_worker_stop', None)
|
|
if stop is not None:
|
|
stop.set()
|
|
tasks = getattr(app.ctx, 'per_worker_tasks', None) or []
|
|
for task in tasks:
|
|
task.cancel()
|
|
if tasks:
|
|
await asyncio.gather(*tasks, return_exceptions=True)
|