mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
* Zim search estimates run in a separate process so a cold estimate cannot stall the API worker The global search estimates every auto-search Zim on each keystroke. On a large library that estimate took tens of seconds when the Zims' pages were not cached, and every other request routed to the same Sanic worker, such as the channel suggestions, waited behind it. Two things held the worker: opening a large Zim cold takes seconds (ten for a 75 GB Zim) and `Zims.get_all` did that on the event loop; and python-libzim never releases the GIL, so the estimate offloaded with asyncio.to_thread still froze the worker. Everything libzim does for an estimate now runs in a standalone script started as a subprocess (Sanic workers are daemonic, so a multiprocessing pool cannot be forked from them). It reads the Zim paths and search string on stdin and writes the counts on stdout. A Zim it cannot read is reported and left out, as `Zims.get_all` would leave it out. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Zim searches run in the libzim worker process too The /api/zim/search endpoints had the same stall as the estimate: they opened every Zim through Zims.get_all and searched them with libzim, which holds the GIL, so a thread offload still froze the Sanic worker. The estimate script is now a general libzim worker with estimate, search and entries jobs. A search of all Zims sends every Zim's job to one worker process; the API worker only headlines the returned HTML and looks up tags. The worker also returns each Zim's metadata so Zim.__json__ does not open the Zim again on the event loop. Zim.entries_with_tags is split so the tagged paths come from the database without opening the Zim. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Tagged Zim estimates count from the database, and a cancelled request kills the libzim worker Review of the earlier commits found two gaps. Estimates with tags selected still went through Zims.entries_with_tags, which opened every Zim in the API process, and after the thread offload was removed it did so on the event loop; they now count the tagged paths in SQL and ask the worker only for metadata. And Sanic cancels a handler at its response timeout or on disconnect, which left the worker process running until its stdout pipe filled; the runner now kills and reaps it on cancellation. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
99 lines
3.8 KiB
Python
99 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Run libzim jobs (estimate, search, fetch entries) for the API.
|
|
|
|
Run by `modules.zim.lib.run_libzim_jobs` as a separate process: python-libzim never releases the GIL, and merely
|
|
opening a large Zim whose pages are not cached takes seconds (ten for a 75 GB Zim), so doing this in the API
|
|
worker would freeze every other request that worker is handling.
|
|
|
|
Reads `{"jobs": [...]}` from stdin and writes one result per job, in order, to stdout. Every job names the Zim
|
|
`path` and an `op`:
|
|
|
|
{"op": "estimate", "search_str": ...} -> {"estimate": int}
|
|
{"op": "search", "search_str": ..., "offset": ..., "limit": ...}
|
|
-> {"estimate": int, "entries": [{"path", "title", "content"}]}
|
|
{"op": "entries", "paths": [...]} -> {"entries": [{"path", "title", "content"}]}
|
|
{"op": "metadata"} -> {} (with the metadata below)
|
|
|
|
Any job may add `"metadata": true` to also get the Zim's metadata. A job that fails returns `{"error": str}`
|
|
so one bad Zim does not hide the rest. Imports only the standard library and libzim; it is run with
|
|
`python -I` from any working directory.
|
|
"""
|
|
import json
|
|
import sys
|
|
|
|
from libzim.reader import Archive
|
|
from libzim.search import Query, Searcher
|
|
from libzim.suggestion import SuggestionSearcher
|
|
|
|
METADATA_KEYS = ('Date', 'Creator', 'Description', 'Name', 'Publisher', 'Tags', 'Title')
|
|
|
|
|
|
def get_metadata(archive: Archive) -> dict:
|
|
metadata = dict()
|
|
for key in METADATA_KEYS:
|
|
try:
|
|
metadata[key.lower()] = archive.get_metadata(key).decode()
|
|
except RuntimeError:
|
|
metadata[key.lower()] = None
|
|
return metadata
|
|
|
|
|
|
def unique_paths(*paths: str) -> list:
|
|
"""The first of any paths that differ only in case, dashes, underscores or slashes (disambiguation or
|
|
redirect entries)."""
|
|
seen, unique = set(), list()
|
|
for path in paths:
|
|
key = path.replace('-', '').replace('_', '').replace('/', '').lower()
|
|
if key not in seen:
|
|
seen.add(key)
|
|
unique.append(path)
|
|
return unique
|
|
|
|
|
|
def entry_dict(archive: Archive, path: str) -> dict:
|
|
entry = archive.get_entry_by_path(path)
|
|
return dict(path=entry.path, title=entry.title, content=bytes(entry.get_item().content).decode('UTF-8'))
|
|
|
|
|
|
def run_job(job: dict) -> dict:
|
|
archive = Archive(job['path'])
|
|
op = job['op']
|
|
result = dict()
|
|
if op == 'estimate':
|
|
query = Query().set_query(job['search_str'])
|
|
result['estimate'] = Searcher(archive).search(query).getEstimatedMatches()
|
|
elif op == 'search':
|
|
search_str, offset, limit = job['search_str'], job['offset'], job['limit']
|
|
query = Query().set_query(search_str)
|
|
# Suggestions search the titles.
|
|
suggested = list(SuggestionSearcher(archive).suggest(search_str).getResults(offset, limit))
|
|
search = Searcher(archive).search(query)
|
|
found = list(search.getResults(offset, limit))
|
|
paths = unique_paths(*suggested, *found)[:limit]
|
|
result['estimate'] = search.getEstimatedMatches()
|
|
result['entries'] = [entry_dict(archive, i) for i in paths]
|
|
elif op == 'entries':
|
|
result['entries'] = [entry_dict(archive, i) for i in job['paths']]
|
|
elif op == 'metadata':
|
|
job['metadata'] = True
|
|
else:
|
|
raise ValueError(f'Unknown op {op!r}')
|
|
if job.get('metadata'):
|
|
result['metadata'] = get_metadata(archive)
|
|
return result
|
|
|
|
|
|
def main():
|
|
request = json.load(sys.stdin)
|
|
results = list()
|
|
for job in request['jobs']:
|
|
try:
|
|
results.append(run_job(job))
|
|
except Exception as e:
|
|
print(f'{job.get("path")}: {e}', file=sys.stderr)
|
|
results.append(dict(error=str(e)))
|
|
json.dump(results, sys.stdout)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|