mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
* db: one request-scoped database handle per API request Every API request now gets `request.ctx.db` (`wrolpi.db.RequestDB`): a Session plus raw cursors on that Session's connection, so a request needs at most one SQLite connection. Creating it does no I/O; requests that never touch the database never connect. `request.ctx.session` remains the handle's Session. A `RequestSession` binds itself to one lazily-opened Connection. An engine-bound Session releases its connection at every commit, and NullPool closes it, so a request that read and then wrote reconnected. Handlers write in `request.ctx.db.write()`: BEGIN IMMEDIATE, committed when the block exits, so the write lock is not held for the rest of the handler or by background work it starts. This replaces the handler-level `get_db_session(commit=True)` blocks (download create, collection item add/remove/reorder, channel create). The zim search handlers use the request's Session in their worker thread. Tests count the connections a request opens with production's per-call sessions (`count_request_connections`). Endpoints whose library functions still open their own connection are strict xfails, to be removed as those functions take the request's session. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * tests: name the real extra connections in the request connection xfails Channel detail: get_channel already uses the request session; the extra connection is Channel.get_statistics opening a raw cursor (get_db_curs). Search: count_file_groups opens a cursor for the total, then handle_file_group_search_results opens a cursor and a session. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
215 lines
8.2 KiB
Python
215 lines
8.2 KiB
Python
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import signal
|
|
from datetime import datetime, timezone, date
|
|
from decimal import Decimal
|
|
from functools import wraps
|
|
from http import HTTPStatus
|
|
from pathlib import Path
|
|
|
|
from sanic import response, HTTPResponse, Request, Sanic, SanicException
|
|
|
|
from wrolpi.common import Base, get_media_directory, logger, LOGGING_CONFIG
|
|
from wrolpi.db import RequestDB
|
|
from wrolpi.errors import APIError
|
|
from wrolpi.perpetual import PERPETUAL_LOOPS, PER_WORKER_TASKS, PerpetualLoop, run_loop
|
|
from wrolpi.vars import PYTEST
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
# Sanic 25.x wants to use 'spawn' as the multiprocessing start method, but our code
|
|
# uses 'fork' extensively (via multiprocessing.Manager, Event, Queue, etc.).
|
|
# Tell Sanic to use 'fork' and that the start method is already configured.
|
|
Sanic.start_method = "fork"
|
|
Sanic.START_METHOD_SET = True
|
|
|
|
|
|
def _reset_inherited_signal_handlers():
|
|
"""A forked worker carries the manager's SIGTERM/SIGINT handlers until Sanic installs the worker's own. In
|
|
that window (startup listeners, ~2s with many workers) a signal must do the default thing, exit, rather than
|
|
run the manager's handler inside the child. Otherwise a second auto-reload leaves the half-started
|
|
generation alive, untracked, holding the listening socket."""
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
signal.signal(sig, signal.SIG_DFL)
|
|
|
|
|
|
os.register_at_fork(after_in_child=_reset_inherited_signal_handlers)
|
|
|
|
# The only Sanic App, this is imported all over.
|
|
api_app = Sanic(name='api_app', log_config=LOGGING_CONFIG)
|
|
|
|
|
|
@wraps(response.json)
|
|
def json_response(*a, **kwargs) -> HTTPResponse:
|
|
"""
|
|
Handles encoding date/datetime in JSON.
|
|
"""
|
|
resp = response.json(*a, **kwargs, cls=CustomJSONEncoder, dumps=json.dumps)
|
|
return resp
|
|
|
|
|
|
class CustomJSONEncoder(json.JSONEncoder):
|
|
|
|
def default(self, obj):
|
|
try:
|
|
if hasattr(obj, '__json__'):
|
|
# Get __json__ before others.
|
|
return obj.__json__()
|
|
elif isinstance(obj, datetime):
|
|
# API always returns dates in UTC.
|
|
if obj.tzinfo:
|
|
obj = obj.astimezone(timezone.utc)
|
|
else:
|
|
# A datetime with no timezone is UTC.
|
|
obj = obj.replace(tzinfo=timezone.utc)
|
|
obj = obj.isoformat()
|
|
return obj
|
|
elif isinstance(obj, date):
|
|
# API always returns dates in UTC.
|
|
obj = datetime(obj.year, obj.month, obj.day, tzinfo=timezone.utc)
|
|
return obj.isoformat()
|
|
elif isinstance(obj, Decimal):
|
|
return str(obj)
|
|
elif isinstance(obj, Base):
|
|
if hasattr(obj, 'dict'):
|
|
return obj.dict()
|
|
elif isinstance(obj, Path):
|
|
media_directory = get_media_directory()
|
|
try:
|
|
path = obj.relative_to(media_directory)
|
|
except ValueError:
|
|
# Path may not be absolute.
|
|
path = obj
|
|
if str(path) == '.':
|
|
return ''
|
|
return str(path)
|
|
return super(CustomJSONEncoder, self).default(obj)
|
|
except Exception as e:
|
|
logger.fatal(f'Failed to JSON encode {obj}', exc_info=e)
|
|
raise
|
|
|
|
|
|
def get_error_json(exception: BaseException):
|
|
"""Return a JSON representation of the Exception instance."""
|
|
if isinstance(exception, APIError):
|
|
# An exception from WROLPi.
|
|
body = dict(error=str(exception), message=exception.summary, code=exception.code)
|
|
elif isinstance(exception, SanicException):
|
|
# An exception from Sanic.
|
|
body = dict(error=str(exception), message=exception.message, code=type(exception).__name__)
|
|
else:
|
|
# Not a WROLPi APIError error.
|
|
body = dict(
|
|
error=str(exception),
|
|
message=None,
|
|
code=type(exception).__name__,
|
|
)
|
|
if exception.__cause__:
|
|
# This exception was caused by another, follow the stack.
|
|
body['cause'] = get_error_json(exception.__cause__)
|
|
return body
|
|
|
|
|
|
def json_error_handler(request: Request, exception: Exception):
|
|
"""Converts all API APIError/SanicException to more informative json object."""
|
|
try:
|
|
body = get_error_json(exception)
|
|
except Exception as e:
|
|
logger.error('Failed to create error json', exc_info=e)
|
|
raise
|
|
|
|
error = repr(str(body["error"]))
|
|
message = repr(str(body["message"]))
|
|
code = body['code']
|
|
if logger.isEnabledFor(logging.DEBUG):
|
|
logger.error(f'API returning JSON error {type(exception).__name__} {error=} {message=} {code=}',
|
|
exc_info=exception)
|
|
else:
|
|
logger.error(f'API returning JSON error {type(exception).__name__} {error=} {message=} {code=}')
|
|
if isinstance(exception, SanicException):
|
|
return json_response(body, exception.status_code)
|
|
|
|
logger.error('Unexpected error', exc_info=exception)
|
|
|
|
# Some unknown error, use internal error code.
|
|
return json_response(body, HTTPStatus.INTERNAL_SERVER_ERROR)
|
|
|
|
|
|
api_app.error_handler.add(Exception, json_error_handler)
|
|
|
|
|
|
@api_app.middleware('request')
|
|
async def inject_session(request: Request):
|
|
"""Give the request its database handle, `request.ctx.db` (see `wrolpi.db.RequestDB`).
|
|
|
|
`request.ctx.session` is the handle's Session. It begins *deferred*, even for a POST, because it
|
|
is committed by the response middleware: a writer would hold the SQLite write lock for the whole
|
|
handler -- including the slow parts (a chunk being written to disk, a subprocess) and any
|
|
background task the handler fires before returning. A handler that writes does so in
|
|
`request.ctx.db.write()`, which begins as a writer and commits when its block exits.
|
|
|
|
Nothing connects until the handler's first statement, so requests that never touch the database
|
|
cost nothing here.
|
|
"""
|
|
request.ctx.db = RequestDB()
|
|
request.ctx.session = request.ctx.db.session
|
|
|
|
|
|
@api_app.middleware('response')
|
|
async def cleanup_session(request: Request, response_: HTTPResponse):
|
|
"""Commit (or, for an error response, roll back) the request's transaction and release its connection."""
|
|
if hasattr(request.ctx, 'db'):
|
|
request.ctx.db.finish(ok=response_.status < 400)
|
|
return response_
|
|
|
|
|
|
def perpetual_signal(sleep: int | float = 1, run_while_testing: bool = False):
|
|
"""Register a singleton background loop. The wrapped function is called forever, `sleep` seconds after each
|
|
call returns, in the ONE perpetual process that Sanic's Worker Manager owns (see wrolpi/perpetual.py). Errors
|
|
are logged and the loop continues; a long call is not re-entered until it returns.
|
|
|
|
The function is returned unchanged so tests can call it directly. Under pytest nothing is registered unless
|
|
`run_while_testing` is set."""
|
|
|
|
def wrapper(func: callable):
|
|
if PYTEST and not run_while_testing:
|
|
return func
|
|
PERPETUAL_LOOPS.append(PerpetualLoop(func.__name__, func, sleep))
|
|
return func
|
|
|
|
return wrapper
|
|
|
|
|
|
def per_worker_task(sleep: int | float = 1):
|
|
"""Register a loop that EVERY Sanic server worker runs for itself (e.g. syncing its own log level). Anything
|
|
that must happen exactly once belongs in `perpetual_signal` instead."""
|
|
|
|
def wrapper(func: callable):
|
|
if PYTEST:
|
|
return func
|
|
PER_WORKER_TASKS.append(PerpetualLoop(func.__name__, func, sleep))
|
|
return func
|
|
|
|
return wrapper
|
|
|
|
|
|
@api_app.after_server_start
|
|
async def start_per_worker_tasks(app: Sanic):
|
|
stop = asyncio.Event()
|
|
app.ctx.per_worker_stop = stop
|
|
app.ctx.per_worker_tasks = [asyncio.create_task(run_loop(loop, stop), name=loop.name) for loop in PER_WORKER_TASKS]
|
|
logger.info(f'start_per_worker_tasks pid={os.getpid()} tasks={[i.name for i in PER_WORKER_TASKS]}')
|
|
|
|
|
|
@api_app.listener('before_server_stop')
|
|
async def stop_per_worker_tasks(app: Sanic):
|
|
stop = getattr(app.ctx, 'per_worker_stop', None)
|
|
if stop is not None:
|
|
stop.set()
|
|
tasks = getattr(app.ctx, 'per_worker_tasks', None) or []
|
|
for task in tasks:
|
|
task.cancel()
|
|
if tasks:
|
|
await asyncio.gather(*tasks, return_exceptions=True)
|