wrolpi/modules/zim/api.py
lrnselfreliance bfe7b8afbd
Zim estimates and searches run in a libzim worker process so a cold Zim cannot stall the API worker (#928)
* Zim search estimates run in a separate process so a cold estimate cannot stall the API worker

The global search estimates every auto-search Zim on each keystroke.  On a
large library that estimate took tens of seconds when the Zims' pages were not
cached, and every other request routed to the same Sanic worker, such as the
channel suggestions, waited behind it.  Two things held the worker: opening a
large Zim cold takes seconds (ten for a 75 GB Zim) and `Zims.get_all` did that
on the event loop; and python-libzim never releases the GIL, so the estimate
offloaded with asyncio.to_thread still froze the worker.

Everything libzim does for an estimate now runs in a standalone script started
as a subprocess (Sanic workers are daemonic, so a multiprocessing pool cannot
be forked from them).  It reads the Zim paths and search string on stdin and
writes the counts on stdout.  A Zim it cannot read is reported and left out, as
`Zims.get_all` would leave it out.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Zim searches run in the libzim worker process too

The /api/zim/search endpoints had the same stall as the estimate: they opened
every Zim through Zims.get_all and searched them with libzim, which holds the
GIL, so a thread offload still froze the Sanic worker.

The estimate script is now a general libzim worker with estimate, search and
entries jobs.  A search of all Zims sends every Zim's job to one worker
process; the API worker only headlines the returned HTML and looks up tags.
The worker also returns each Zim's metadata so Zim.__json__ does not open the
Zim again on the event loop.  Zim.entries_with_tags is split so the tagged
paths come from the database without opening the Zim.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Tagged Zim estimates count from the database, and a cancelled request kills the libzim worker

Review of the earlier commits found two gaps.  Estimates with tags selected
still went through Zims.entries_with_tags, which opened every Zim in the API
process, and after the thread offload was removed it did so on the event loop;
they now count the tagged paths in SQL and ask the worker only for metadata.
And Sanic cancels a handler at its response timeout or on disconnect, which
left the worker process running until its stdout pipe filled; the runner now
kills and reaps it on cancellation.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-01 13:02:26 -07:00

215 lines
7.2 KiB
Python

import urllib.parse
from http import HTTPStatus
from sanic import Request, response, Blueprint
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi import lang
from wrolpi.api_utils import json_response
from wrolpi.common import logger
from wrolpi.downloader import download_manager
from wrolpi.events import Events
from . import lib, schema
zim_bp = Blueprint('Zim', '/api/zim')
logger = logger.getChild(__name__)
@zim_bp.get('/')
@openapi.definition(
summary='List all known Zim files',
response=schema.GetZimsResponse,
)
async def get_zims(request: Request):
session = request.ctx.session
zims = lib.get_zims(session)
resp = {
'zims': [i.__json__() for i in zims],
}
return json_response(resp)
@zim_bp.delete('/<zim_ids:[0-9,]+>')
@openapi.definition(
summary='Delete all Zim files',
)
async def delete_zims(request: Request, zim_ids: str):
zim_ids = [int(i) for i in zim_ids.split(',')]
with request.ctx.db.write() as session:
lib.delete_zims(session, zim_ids)
return response.empty()
@zim_bp.post('/<zim_id:[0-9]+>/auto_search')
@openapi.definition(
summary='Change if a Zim file will be searched by default.',
body=schema.ZimAutoSearchRequest,
)
@validate(schema.ZimAutoSearchRequest)
async def post_set_zim_auto_search(request: Request, zim_id: int, body: schema.ZimAutoSearchRequest):
zim_id = int(zim_id)
with request.ctx.db.write() as session:
lib.set_zim_auto_search(session, zim_id, body.auto_search)
return response.empty()
@zim_bp.post('/search')
@openapi.definition(
summary='Search all Zim files that have auto_search enabled',
body=schema.ZimSearchRequest,
)
@validate(schema.ZimSearchRequest)
async def search_all_zims(request: Request, body: schema.ZimSearchRequest):
results = await lib.search_all_zims(request.ctx.session, body.search_str, tag_names=body.tag_names,
offset=body.offset, limit=body.limit)
return json_response({'zims': results})
@zim_bp.post('/search/<zim_id:[0-9]+>')
@openapi.definition(
summary='Search all entries of a Zim',
body=schema.ZimSearchRequest,
response=schema.ZimSearchResponse,
)
@validate(schema.ZimSearchRequest)
async def search_zim(request: Request, zim_id: int, body: schema.ZimSearchRequest):
headlines = await lib.headline_zim(request.ctx.session, body.search_str, zim_id, tag_names=body.tag_names,
offset=body.offset, limit=body.limit)
return json_response({'zim': headlines})
@zim_bp.get('/<zim_id:[0-9]+>/entry/<zim_path:[ -~/]*>')
@openapi.definition(
summary='Read the entry at `zim_path` from the Zim file',
)
async def get_zim_entry(_: Request, zim_id: int, zim_path: str):
zim_path = urllib.parse.unquote(zim_path)
entry = lib.get_entry(zim_path, zim_id=zim_id)
content = bytes(entry.get_item().content)
try:
resp = response.html(content.decode('UTF-8'))
logger.debug(f'Returning ZIM HTML response {repr(str(zim_path))}')
except UnicodeDecodeError:
resp = response.raw(content)
logger.debug(f'Returning ZIM raw response {repr(str(zim_path))}')
return resp
@zim_bp.post('/tag')
@openapi.definition(
summary='Tag a Zim entry',
body=schema.TagZimEntry,
)
@validate(schema.TagZimEntry)
async def post_zim_tag(request: Request, body: schema.TagZimEntry):
with request.ctx.db.write() as session:
await lib.add_tag(session, body.tag_name, body.zim_id, body.zim_entry)
return response.empty(HTTPStatus.CREATED)
@zim_bp.post('/untag')
@openapi.definition(
summary='Untag a Zim entry',
body=schema.TagZimEntry,
)
@validate(schema.TagZimEntry)
async def post_zim_untag(request: Request, body: schema.TagZimEntry):
with request.ctx.db.write() as session:
await lib.untag(session, body.tag_name, body.zim_id, body.zim_entry)
return response.empty(HTTPStatus.NO_CONTENT)
@zim_bp.get('/subscribe')
@openapi.definition(
summary='Retrieve Zim subscriptions',
response=schema.ZimSubscriptions,
)
async def get_zim_subscriptions(request: Request):
catalog = lib.get_kiwix_catalog()
session = request.ctx.session
subscriptions = {i: j.__json__() for i, j in lib.get_kiwix_subscriptions(session).items()}
resp = dict(
catalog=catalog,
iso_639_codes=lang.ISO_639_CODES,
subscriptions=subscriptions,
)
return json_response(resp)
@zim_bp.post('/subscribe')
@openapi.definition(
summary='Subscribe to a particular Kiwix Zim',
body=schema.ZimSubscribeRequest,
)
@validate(schema.ZimSubscribeRequest)
async def post_zim_subscribe(request: Request, body: schema.ZimSubscribeRequest):
with request.ctx.db.write() as session:
await lib.subscribe(session, body.name, body.language)
if download_manager.disabled or download_manager.stopped:
# Downloads are disabled, warn the user.
Events.send_downloads_disabled('Download created. But, downloads are disabled.')
Events.send_created(f'Zim subscription created')
return response.empty(HTTPStatus.CREATED)
@zim_bp.delete('/subscribe/<subscription_id:[0-9]+>')
@openapi.definition(
summary='Unsubscribe to a particular Kiwix Zim',
)
async def delete_zim_subscription(request: Request, subscription_id: int):
with request.ctx.db.write() as session:
await lib.unsubscribe(session, subscription_id)
Events.send_deleted(f'Zim subscription deleted')
return response.empty(HTTPStatus.NO_CONTENT)
@zim_bp.get('/outdated')
@openapi.definition(
summary='Returns the outdated and current Zim files',
response=schema.OutdatedZims,
)
async def get_outdated_zims(_: Request):
outdated, current = lib.find_outdated_zim_files()
d = dict(outdated=outdated, current=current)
return json_response(d)
@zim_bp.delete('/outdated')
@openapi.definition(
summary='Remove all outdated Zims, if any.'
)
async def delete_outdated_zims(_: Request):
try:
deleted_count = await lib.remove_outdated_zim_files()
finally:
# The flag must match disk whether or not the delete succeeded.
lib.flag_outdated_zim_files()
if deleted_count:
Events.send_deleted(f'Deleted {deleted_count} outdated Zims')
return response.empty(HTTPStatus.NO_CONTENT)
@zim_bp.post('/search_estimates')
@validate(json=schema.SearchEstimateRequest)
async def post_search_estimates(request: Request, body: schema.SearchEstimateRequest):
"""Get an estimated count of FileGroups/Zims which may or may not have been tagged."""
if not body.search_str and not body.tag_names:
return response.empty(HTTPStatus.BAD_REQUEST)
session = request.ctx.session
if body.tag_names:
# Count the entries tagged with the tag names (from the database; see `lib.count_tagged_zim_entries`).
counts = await lib.count_tagged_zim_entries(session, body.tag_names)
else:
# Get estimates using libzim, in the estimate process (see `lib.estimate_zims`).
counts = await lib.estimate_zims(session, body.search_str)
zims_estimates = [dict(estimate=count, **zim.__json__()) for zim, count in counts.items()]
ret = dict(
zims_estimates=zims_estimates
)
return json_response(ret)