From bfe7b8afbd0d0c3a5decd5bc379a74e6e24113e3 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:02:26 -0700 Subject: [PATCH] Zim estimates and searches run in a libzim worker process so a cold Zim cannot stall the API worker (#928) * Zim search estimates run in a separate process so a cold estimate cannot stall the API worker The global search estimates every auto-search Zim on each keystroke. On a large library that estimate took tens of seconds when the Zims' pages were not cached, and every other request routed to the same Sanic worker, such as the channel suggestions, waited behind it. Two things held the worker: opening a large Zim cold takes seconds (ten for a 75 GB Zim) and `Zims.get_all` did that on the event loop; and python-libzim never releases the GIL, so the estimate offloaded with asyncio.to_thread still froze the worker. Everything libzim does for an estimate now runs in a standalone script started as a subprocess (Sanic workers are daemonic, so a multiprocessing pool cannot be forked from them). It reads the Zim paths and search string on stdin and writes the counts on stdout. A Zim it cannot read is reported and left out, as `Zims.get_all` would leave it out. Co-Authored-By: Claude Fable 5.1 * Zim searches run in the libzim worker process too The /api/zim/search endpoints had the same stall as the estimate: they opened every Zim through Zims.get_all and searched them with libzim, which holds the GIL, so a thread offload still froze the Sanic worker. The estimate script is now a general libzim worker with estimate, search and entries jobs. A search of all Zims sends every Zim's job to one worker process; the API worker only headlines the returned HTML and looks up tags. The worker also returns each Zim's metadata so Zim.__json__ does not open the Zim again on the event loop. Zim.entries_with_tags is split so the tagged paths come from the database without opening the Zim. Co-Authored-By: Claude Fable 5.1 * Tagged Zim estimates count from the database, and a cancelled request kills the libzim worker Review of the earlier commits found two gaps. Estimates with tags selected still went through Zims.entries_with_tags, which opened every Zim in the API process, and after the thread offload was removed it did so on the event loop; they now count the tagged paths in SQL and ask the worker only for metadata. And Sanic cancels a handler at its response timeout or on disconnect, which left the worker process running until its stdout pipe filled; the runner now kills and reaps it on cancellation. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- modules/zim/api.py | 37 +++----- modules/zim/lib.py | 177 +++++++++++++++++++++++++---------- modules/zim/libzim_worker.py | 99 ++++++++++++++++++++ modules/zim/models.py | 73 +++------------ modules/zim/test/test_api.py | 108 +++++++++++++++++++++ modules/zim/test/test_lib.py | 36 +++---- 6 files changed, 376 insertions(+), 154 deletions(-) create mode 100644 modules/zim/libzim_worker.py diff --git a/modules/zim/api.py b/modules/zim/api.py index 46b53692..891c1bfc 100644 --- a/modules/zim/api.py +++ b/modules/zim/api.py @@ -1,4 +1,3 @@ -import asyncio import urllib.parse from http import HTTPStatus @@ -12,7 +11,6 @@ from wrolpi.common import logger from wrolpi.downloader import download_manager from wrolpi.events import Events from . import lib, schema -from .models import Zims zim_bp = Blueprint('Zim', '/api/zim') @@ -64,13 +62,8 @@ async def post_set_zim_auto_search(request: Request, zim_id: int, body: schema.Z ) @validate(schema.ZimSearchRequest) async def search_all_zims(request: Request, body: schema.ZimSearchRequest): - # libzim searches are synchronous and slow; run off the event loop. The request's Session is used - # in the worker thread; the handler awaits the thread, so the Session has one user at a time. - def search(): - return lib.search_all_zims(request.ctx.session, body.search_str, tag_names=body.tag_names, - offset=body.offset, limit=body.limit) - - results = await asyncio.to_thread(search) + results = await lib.search_all_zims(request.ctx.session, body.search_str, tag_names=body.tag_names, + offset=body.offset, limit=body.limit) return json_response({'zims': results}) @@ -82,11 +75,8 @@ async def search_all_zims(request: Request, body: schema.ZimSearchRequest): ) @validate(schema.ZimSearchRequest) async def search_zim(request: Request, zim_id: int, body: schema.ZimSearchRequest): - def search(): - return lib.headline_zim(request.ctx.session, body.search_str, zim_id, tag_names=body.tag_names, - offset=body.offset, limit=body.limit) - - headlines = await asyncio.to_thread(search) + headlines = await lib.headline_zim(request.ctx.session, body.search_str, zim_id, tag_names=body.tag_names, + offset=body.offset, limit=body.limit) return json_response({'zim': headlines}) @@ -210,17 +200,14 @@ async def post_search_estimates(request: Request, body: schema.SearchEstimateReq if not body.search_str and not body.tag_names: return response.empty(HTTPStatus.BAD_REQUEST) - def estimate(): - session = request.ctx.session - if body.tag_names: - # Get actual count of entries tagged with the tag names. - counts = Zims.entries_with_tags(session, body.tag_names) - else: - # Get estimates using libzim. - counts = Zims.estimate(session, body.search_str) - return [dict(estimate=count, **zim.__json__()) for zim, count in counts.items()] - - zims_estimates = await asyncio.to_thread(estimate) + session = request.ctx.session + if body.tag_names: + # Count the entries tagged with the tag names (from the database; see `lib.count_tagged_zim_entries`). + counts = await lib.count_tagged_zim_entries(session, body.tag_names) + else: + # Get estimates using libzim, in the estimate process (see `lib.estimate_zims`). + counts = await lib.estimate_zims(session, body.search_str) + zims_estimates = [dict(estimate=count, **zim.__json__()) for zim, count in counts.items()] ret = dict( zims_estimates=zims_estimates diff --git a/modules/zim/lib.py b/modules/zim/lib.py index e99456da..95221ab4 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -1,9 +1,12 @@ +import asyncio import functools import json import pathlib import re +import sys from datetime import datetime -from typing import Callable, List, Tuple, Dict +from collections import OrderedDict +from typing import Callable, List, Tuple, Dict, OrderedDict as OrderedDictType from libzim import Entry from sqlalchemy import or_ @@ -13,11 +16,11 @@ from sqlalchemy.orm.exc import NoResultFound # noqa from modules.zim import kiwix from modules.zim.errors import UnknownZim, UnknownZimSubscription from modules.zim.kiwix import KIWIX_CATALOG -from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription +from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription, ZimMetadata from wrolpi import flags, tags from wrolpi.cmd import run_command from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \ - register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate, aiohttp_post + register_refresh_cleanup, background_task, get_wrolpi_config, aiohttp_post from wrolpi.switches import register_switch_handler from wrolpi.dates import Seconds from wrolpi.events import Events @@ -39,7 +42,6 @@ __all__ = [ 'get_entries_tags', 'get_entry', 'get_kiwix_catalog', - 'get_unique_paths', 'get_zim', 'get_zims', 'model_zim', @@ -153,40 +155,67 @@ def get_entries_tags(session: Session, paths: List[str], zim_id: int): return entries -def get_unique_paths(*paths: str) -> List[str]: - """Return a new list which contains unique Zim paths.""" - unique_paths = unique_by_predicate( - paths, - lambda i: i.replace('-', '').replace('_', '').replace('/', '').lower() +LIBZIM_WORKER = pathlib.Path(__file__).with_name('libzim_worker.py') + + +async def run_libzim_jobs(jobs: List[Dict]) -> List[Dict]: + """Run libzim jobs (see `libzim_worker.py`) in a separate process, return one result per job. + + python-libzim never releases the GIL, and opening a large Zim cold takes seconds, so nothing libzim does may + run in the API worker: every other request on that worker would wait behind it. (A plain subprocess: + Sanic workers are daemonic, so they cannot fork multiprocessing children.) A failed job's result is + `{'error': ...}`, so one bad Zim does not fail the rest.""" + request = json.dumps(dict(jobs=jobs)).encode() + proc = await asyncio.create_subprocess_exec( + sys.executable, '-I', str(LIBZIM_WORKER), + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - return list(unique_paths) + try: + stdout, stderr = await proc.communicate(request) + except asyncio.CancelledError: + # Sanic cancels the handler at its response timeout or when the client disconnects; cancelling + # `communicate` leaves the process running (and, once its stdout pipe fills, stuck). Kill and reap it. + try: + proc.kill() + except ProcessLookupError: + pass + await proc.wait() + raise + if proc.returncode != 0: + raise RuntimeError(f'libzim worker failed ({proc.returncode}): {stderr.decode(errors="replace").strip()}') + if stderr: + logger.error(f'libzim worker reported errors:\n{stderr.decode(errors="replace").strip()}') + return json.loads(stdout) -def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List[str] = None, offset: int = 0, - limit: int = 10) -> Dict: - zim_id = int(zim_id) - zim = get_zim(session, zim_id) +def _apply_metadata(zim: Zim, result: Dict): + """Fill the Zim's cached `zim_metadata` from a worker result so `Zim.__json__` does not open the Zim.""" + if metadata := result.get('metadata'): + zim.__dict__['zim_metadata'] = ZimMetadata(**metadata) + +def _search_job(zim: Zim, search_str: str, tag_names: List[str], session: Session, offset: int, limit: int) -> Dict: + """The worker job that fetches what `_headline_results` needs for one Zim.""" if tag_names: # TODO what is the count goes over the limit? - entries = zim.entries_with_tags(session, tag_names, offset=offset, limit=1_000) - estimate = 0 + paths = zim.entry_paths_with_tags(session, tag_names, offset=offset, limit=1_000) + return dict(op='entries', path=str(zim.path), paths=paths, metadata=True) elif search_str: - # Get suggested results (this searches titles). - results = zim.suggest(search_str, offset, limit) - search_results = zim.search(search_str, offset, limit) - # Remove duplicate entries (disambiguation or redirect). - results = get_unique_paths(*results, *search_results) - results = results[:limit] - entries = [zim.find_entry(i) for i in results] - estimate = zim.estimate(search_str) - else: - raise RuntimeError('Must provide search_str or tag_names.') + return dict(op='search', path=str(zim.path), search_str=search_str, offset=offset, limit=limit, metadata=True) + raise RuntimeError('Must provide search_str or tag_names.') - articles = [bytes(i.get_item().content).decode('UTF-8') for i in entries] - articles = [extract_html_text(i) for i in articles] - entries_tag_names = get_entries_tags(session, [i.path for i in entries], zim_id) +def _headline_results(session: Session, zim: Zim, search_str: str, tag_names: List[str], result: Dict) -> Dict: + """Headline the entries a search job returned for one Zim.""" + if 'error' in result: + raise RuntimeError(f'Failed to search {zim}: {result["error"]}') + _apply_metadata(zim, result) + entries = result['entries'] + estimate = result.get('estimate', 0) + + articles = [extract_html_text(i['content']) for i in entries] + + entries_tag_names = get_entries_tags(session, [i['path'] for i in entries], zim.id) # Always get headlines even if there is no search_str. Non-matches get the leading text of each article. headlines = extract_headlines(articles, search_str) @@ -194,7 +223,7 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List search = list() if search_str: # `search_str` allows us to headline the title. - titles = [i.title for i in entries] + titles = [i['title'] for i in entries] headline_titles = extract_headlines(titles, search_str) for entry in entries: @@ -202,7 +231,7 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List headline, rank = headlines.pop(0) if rank > 0 or title_rank > 0: # The result for each Entry. - path = entry.path + path = entry['path'] entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list() search.append(dict( headline=headline, @@ -217,14 +246,14 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List # No `search_str`, we return all entries. for entry in entries: headline, _ = headlines.pop(0) - path = entry.path + path = entry['path'] entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list() search.append(dict( headline=headline, path=path, rank=0, tag_names=entry_tag_names, - title=entry.title, + title=entry['title'], )) estimate = len(entries) @@ -237,26 +266,76 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List return results -def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0, - limit: int = 10) -> List[Dict]: - """Search all Zim files that have auto_search enabled. +async def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List[str] = None, offset: int = 0, + limit: int = 10) -> Dict: + zim = get_zim(session, int(zim_id)) + job = _search_job(zim, search_str, tag_names, session, offset, limit) + result, = await run_libzim_jobs([job]) + return await asyncio.to_thread(_headline_results, session, zim, search_str, tag_names, result) + + +def auto_search_zims(session: Session) -> List[Zim]: + """The Zims that the global search searches, without opening them.""" + zims = session.query(Zim).options(*Zim.json_options()).filter_by(auto_search=True).order_by(Zim.path).all() + return [i for i in zims if i.path and i.path.is_file()] + + +async def estimate_zims(session: Session, search_str: str) -> OrderedDictType[Zim, int]: + """Estimate how many entries of each auto-search Zim match `search_str`. + + `Zims.get_all` opens every Zim and is not used here for that reason; a Zim the worker cannot read is + reported and left out, as `Zims.get_all` would.""" + zims = auto_search_zims(session) + jobs = [dict(op='estimate', path=str(i.path), search_str=search_str, metadata=True) for i in zims] + results = await run_libzim_jobs(jobs) + counts = OrderedDict() + for zim, result in zip(zims, results): + if 'error' not in result: + _apply_metadata(zim, result) + counts[zim] = result['estimate'] + return counts + + +async def count_tagged_zim_entries(session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]: + """Count the entries of every Zim tagged with all of `tag_names`. The counts come from the database; the + worker only reads each Zim's metadata, and a Zim it cannot read is left out.""" + zims = session.query(Zim).options(*Zim.json_options()).order_by(Zim.path).all() + zims = [i for i in zims if i.path and i.path.is_file()] + counts = {i: len(i.entry_paths_with_tags(session, tag_names, limit=1_000_000)) for i in zims} + results = await run_libzim_jobs([dict(op='metadata', path=str(i.path)) for i in zims]) + readable = OrderedDict() + for zim, result in zip(zims, results): + if 'error' not in result: + _apply_metadata(zim, result) + readable[zim] = counts[zim] + return readable + + +async def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0, + limit: int = 10) -> List[Dict]: + """Search all Zim files that have auto_search enabled, in one libzim worker process. Returns a list of per-zim result dicts, each containing metadata and search results. Each search entry includes `zim_id` so callers can fetch the full entry afterward.""" - zims = Zims.get_all(session) - all_results = [] - for zim in zims: - if not zim.auto_search: - continue - try: - result = headline_zim(session, search_str, zim.id, tag_names=tag_names, offset=offset, limit=limit) - if result.get('search'): - for entry in result['search']: + zims = auto_search_zims(session) + jobs = [_search_job(zim, search_str, tag_names, session, offset, limit) for zim in zims] + results = await run_libzim_jobs(jobs) + + def headline_all(): + all_results = [] + for zim, result in zip(zims, results): + try: + headlined = _headline_results(session, zim, search_str, tag_names, result) + except Exception as e: + logger.error(f'Failed to search Zim {zim}', exc_info=e) + continue + if headlined.get('search'): + for entry in headlined['search']: entry['zim_id'] = zim.id - all_results.append(result) - except Exception as e: - logger.error(f'Failed to search Zim {zim}', exc_info=e) - return all_results + all_results.append(headlined) + return all_results + + return await asyncio.to_thread(headline_all) def get_zim(session: Session, zim_id: int) -> Zim: diff --git a/modules/zim/libzim_worker.py b/modules/zim/libzim_worker.py new file mode 100644 index 00000000..f48466d6 --- /dev/null +++ b/modules/zim/libzim_worker.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Run libzim jobs (estimate, search, fetch entries) for the API. + +Run by `modules.zim.lib.run_libzim_jobs` as a separate process: python-libzim never releases the GIL, and merely +opening a large Zim whose pages are not cached takes seconds (ten for a 75 GB Zim), so doing this in the API +worker would freeze every other request that worker is handling. + +Reads `{"jobs": [...]}` from stdin and writes one result per job, in order, to stdout. Every job names the Zim +`path` and an `op`: + + {"op": "estimate", "search_str": ...} -> {"estimate": int} + {"op": "search", "search_str": ..., "offset": ..., "limit": ...} + -> {"estimate": int, "entries": [{"path", "title", "content"}]} + {"op": "entries", "paths": [...]} -> {"entries": [{"path", "title", "content"}]} + {"op": "metadata"} -> {} (with the metadata below) + +Any job may add `"metadata": true` to also get the Zim's metadata. A job that fails returns `{"error": str}` +so one bad Zim does not hide the rest. Imports only the standard library and libzim; it is run with +`python -I` from any working directory. +""" +import json +import sys + +from libzim.reader import Archive +from libzim.search import Query, Searcher +from libzim.suggestion import SuggestionSearcher + +METADATA_KEYS = ('Date', 'Creator', 'Description', 'Name', 'Publisher', 'Tags', 'Title') + + +def get_metadata(archive: Archive) -> dict: + metadata = dict() + for key in METADATA_KEYS: + try: + metadata[key.lower()] = archive.get_metadata(key).decode() + except RuntimeError: + metadata[key.lower()] = None + return metadata + + +def unique_paths(*paths: str) -> list: + """The first of any paths that differ only in case, dashes, underscores or slashes (disambiguation or + redirect entries).""" + seen, unique = set(), list() + for path in paths: + key = path.replace('-', '').replace('_', '').replace('/', '').lower() + if key not in seen: + seen.add(key) + unique.append(path) + return unique + + +def entry_dict(archive: Archive, path: str) -> dict: + entry = archive.get_entry_by_path(path) + return dict(path=entry.path, title=entry.title, content=bytes(entry.get_item().content).decode('UTF-8')) + + +def run_job(job: dict) -> dict: + archive = Archive(job['path']) + op = job['op'] + result = dict() + if op == 'estimate': + query = Query().set_query(job['search_str']) + result['estimate'] = Searcher(archive).search(query).getEstimatedMatches() + elif op == 'search': + search_str, offset, limit = job['search_str'], job['offset'], job['limit'] + query = Query().set_query(search_str) + # Suggestions search the titles. + suggested = list(SuggestionSearcher(archive).suggest(search_str).getResults(offset, limit)) + search = Searcher(archive).search(query) + found = list(search.getResults(offset, limit)) + paths = unique_paths(*suggested, *found)[:limit] + result['estimate'] = search.getEstimatedMatches() + result['entries'] = [entry_dict(archive, i) for i in paths] + elif op == 'entries': + result['entries'] = [entry_dict(archive, i) for i in job['paths']] + elif op == 'metadata': + job['metadata'] = True + else: + raise ValueError(f'Unknown op {op!r}') + if job.get('metadata'): + result['metadata'] = get_metadata(archive) + return result + + +def main(): + request = json.load(sys.stdin) + results = list() + for job in request['jobs']: + try: + results.append(run_job(job)) + except Exception as e: + print(f'{job.get("path")}: {e}', file=sys.stderr) + results.append(dict(error=str(e))) + json.dump(results, sys.stdout) + + +if __name__ == '__main__': + main() diff --git a/modules/zim/models.py b/modules/zim/models.py index 5afeb50d..5aa98ebb 100644 --- a/modules/zim/models.py +++ b/modules/zim/models.py @@ -5,7 +5,7 @@ from collections import OrderedDict from datetime import datetime from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional -from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher +from libzim import Archive, Entry from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean, UniqueConstraint from sqlalchemy.orm import Mapped, relationship, Session, joinedload from sqlalchemy.orm.exc import NoResultFound # noqa @@ -165,39 +165,6 @@ class Zim(Base, ModelHelper): ) return metadata - def get_searcher(self) -> Searcher: - searcher = Searcher(self.get_zim()) - return searcher - - def get_search(self, search_str: str): - query = Query().set_query(search_str) - searcher = self.get_searcher() - search = searcher.search(query) - return search - - def get_suggestion_searcher(self) -> SuggestionSearcher: - searcher = SuggestionSearcher(self.get_zim()) - return searcher - - def estimate(self, search_str: str) -> int: - """Count the number of Entry(s) that contain the `search_str`.""" - search = self.get_search(search_str) - count = search.getEstimatedMatches() - return count - - def search(self, search_str: str, offset: int = 0, limit: int = 10) -> List[str]: - """Return the paths of the Entry(s) that contain the `search_str`.""" - search = self.get_search(search_str) - results = list(search.getResults(offset, limit)) - return results - - def suggest(self, search_str: str, offset: int = 0, limit: int = 10) -> List[str]: - """Return the paths of the Entry(s) that contain the `search_str`. Also searches the titles.""" - searcher = self.get_suggestion_searcher() - suggestions = searcher.suggest(search_str) - results = list(suggestions.getResults(offset, limit)) - return results - def get_entry(self, path: str) -> Optional[Entry]: """Search this Zim file for the provided Entry at the `path`.""" zim = self.get_zim() @@ -256,22 +223,21 @@ class Zim(Base, ModelHelper): .order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry) \ .offset(offset).limit(limit) - entries = list() - for tag_zim_entry, zim in tag_zim_entries: - entry = zim.find_entry(tag_zim_entry.zim_entry) - entries.append(entry) + return [(zim, tag_zim_entry.zim_entry) for tag_zim_entry, zim in tag_zim_entries] - return entries - - def entries_with_tags(self, session: Session, tag_names: List[str], limit: int = 10, offset: int = 0) -> List[ - Entry]: - """Return Zim Entries tagged with the provided names.""" + def entry_paths_with_tags(self, session: Session, tag_names: List[str], limit: int = 10, offset: int = 0) \ + -> List[str]: + """Return the paths of the Zim Entries tagged with the provided names. Does not open the Zim.""" if limit < 1: raise RuntimeError('Limit must be greater than zero') tags_sub_select, params = tag_names_to_zim_sub_select(tag_names, zim_id=self.id) - entries = Zim._entries_with_tags_process(limit, offset, params, session, tags_sub_select) - return entries + return [path for _, path in Zim._entries_with_tags_process(limit, offset, params, session, tags_sub_select)] + + def entries_with_tags(self, session: Session, tag_names: List[str], limit: int = 10, offset: int = 0) -> List[ + Entry]: + """Return Zim Entries tagged with the provided names.""" + return [self.find_entry(i) for i in self.entry_paths_with_tags(session, tag_names, limit, offset)] class Zims: @@ -290,23 +256,6 @@ class Zims: logger.error(f'Failed to read Zim: {zim}', exc_info=e) return zims - @classmethod - def estimate(cls, session: Session, search_str: str) -> OrderedDictType[Zim, int]: - zims = cls.get_all(session) - results = OrderedDict() - for zim in zims: - if zim.auto_search: - results[zim] = zim.estimate(search_str) - return results - - @classmethod - def entries_with_tags(cls, session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]: - zims = cls.get_all(session) - results = OrderedDict() - for zim in zims: - results[zim] = len(zim.entries_with_tags(session, tag_names)) - return results - class TagZimEntry(Base): __tablename__ = 'tag_zim' diff --git a/modules/zim/test/test_api.py b/modules/zim/test/test_api.py index 630e2acc..06a4b6ae 100644 --- a/modules/zim/test/test_api.py +++ b/modules/zim/test/test_api.py @@ -1,4 +1,7 @@ +import asyncio import json +import sys +from unittest import mock from http import HTTPStatus import pytest @@ -366,3 +369,108 @@ async def test_delete_zim_waits_for_a_concurrent_writer(async_client, test_sessi assert not zim_path.exists() with maker() as session: assert session.query(Zim).filter_by(id=zim_id).one_or_none() is None + + +@pytest.mark.asyncio +async def test_estimate_zims_runs_outside_the_api_process(test_session, zim_factory): + """libzim holds the GIL, so estimates run in a separate process rather than a thread of the API worker.""" + zim = zim_factory('test zim') + test_session.commit() + + with mock.patch('modules.zim.lib.asyncio.create_subprocess_exec', wraps=asyncio.create_subprocess_exec) as m: + counts = await lib.estimate_zims(test_session, 'one') + assert counts == {zim: 2} + assert m.call_args.args[:2] == (sys.executable, '-I') + + +@pytest.mark.asyncio +async def test_estimate_zims_skips_a_zim_it_cannot_read(test_session, zim_factory): + """A Zim the estimate process cannot open is left out of the estimates, like `Zims.get_all` leaves it out.""" + good = zim_factory('good zim') + bad = zim_factory('bad zim') + test_session.commit() + bad.path.write_bytes(b'not a zim file') + + counts = await lib.estimate_zims(test_session, 'one') + assert counts == {good: 2} + + bad.path.unlink() + counts = await lib.estimate_zims(test_session, 'one') + assert counts == {good: 2} + + +@pytest.mark.asyncio +async def test_zim_search_runs_outside_the_api_process(async_client, test_session, test_zim): + """Zim searches, like estimates, do all their libzim work in the worker script's process.""" + with mock.patch('modules.zim.lib.asyncio.create_subprocess_exec', wraps=asyncio.create_subprocess_exec) as m: + request, response = await async_client.post(f'/api/zim/search/{test_zim.id}', json=dict(search_str='one')) + assert response.status_code == HTTPStatus.OK + assert [i['path'] for i in response.json['zim']['search']] == ['one', 'home'] + assert response.json['zim']['metadata']['title'], 'Metadata should come from the worker' + + request, response = await async_client.post('/api/zim/search', json=dict(search_str='one')) + assert response.status_code == HTTPStatus.OK + assert [i['path'] for i in response.json['zims'][0]['search']] == ['one', 'home'] + assert m.call_count == 2 + assert all(call.args[:2] == (sys.executable, '-I') for call in m.call_args_list) + + +@pytest.mark.asyncio +async def test_search_all_zims_skips_a_zim_it_cannot_read(async_client, test_session, zim_factory): + """One unreadable Zim does not fail the search of all Zims.""" + good = zim_factory('good zim') + bad = zim_factory('bad zim') + test_session.commit() + bad.path.write_bytes(b'not a zim file') + + request, response = await async_client.post('/api/zim/search', json=dict(search_str='one')) + assert response.status_code == HTTPStatus.OK + zims = response.json['zims'] + assert len(zims) == 1 and zims[0]['path'].endswith('good zim') + assert zims[0]['estimate'] == 2 + + +@pytest.mark.asyncio +async def test_tagged_search_estimates_do_not_open_zims(async_client, test_session, test_zim, tag_factory): + """Estimates with tags count from the database; the API process never opens a Zim, even for its metadata.""" + tag1, tag2 = await tag_factory(), await tag_factory() + test_zim.tag_entry(test_session, tag1.name, 'one') + test_zim.tag_entry(test_session, tag1.name, 'two') + test_zim.tag_entry(test_session, tag2.name, 'one') + test_session.commit() + + with mock.patch('modules.zim.models.Zim.get_zim', side_effect=AssertionError('opened a Zim in the API')): + body = dict(search_str='one', tag_names=[tag1.name]) + request, response = await async_client.post('/api/zim/search_estimates', json=body) + assert response.status_code == HTTPStatus.OK + assert [(i['estimate'], bool(i['metadata']['title'])) for i in response.json['zims_estimates']] \ + == [(2, True)] + + body = dict(search_str='', tag_names=[tag1.name, tag2.name]) + request, response = await async_client.post('/api/zim/search_estimates', json=body) + assert [i['estimate'] for i in response.json['zims_estimates']] == [1] + + +@pytest.mark.asyncio +async def test_cancelled_libzim_jobs_kill_the_worker(test_directory): + """Sanic cancels a handler at its response timeout or on disconnect; the worker process must not outlive it.""" + slow_worker = test_directory / 'slow_worker.py' + slow_worker.write_text('import time; time.sleep(60)') + procs = [] + create_subprocess_exec = asyncio.create_subprocess_exec + + async def spy(*args, **kwargs): + proc = await create_subprocess_exec(*args, **kwargs) + procs.append(proc) + return proc + + with mock.patch('modules.zim.lib.LIBZIM_WORKER', slow_worker), \ + mock.patch('modules.zim.lib.asyncio.create_subprocess_exec', side_effect=spy): + task = asyncio.create_task(lib.run_libzim_jobs([])) + while not procs: + await asyncio.sleep(0.01) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert procs[0].returncode is not None, 'The worker was killed and reaped' diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index 814d97a9..06f67b70 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -4,7 +4,7 @@ from http import HTTPStatus import pytest from libzim import Entry -from modules.zim import lib +from modules.zim import lib, libzim_worker from modules.zim.errors import UnknownZim, UnknownZimEntry from modules.zim.models import TagZimEntry, ZimSubscription from wrolpi import tags, flags @@ -96,29 +96,29 @@ async def test_zim_entries_with_tags(async_client, test_session, zim_factory, ta assert entries[0].path == 'two' -def test_get_unique_paths(): +def test_unique_paths(): paths = ['A/Prickly_Pear', 'A/Prickly_pear', 'A/Prickly-pear', 'A/Arborescent_Prickly_Pear', 'A/Arborescent_prickly_pear', 'A/Beavertail_prickly_pear', 'A/Brittle_prickly-pear', 'A/Chenille_prickly-pear', 'A/Coastal_Prickly_Pear', 'A/Coastal_prickly_pear', 'A/Prickly_pears_in_South_Africa', 'A/Prickly_Pear_Cays', 'A/Prickly_pears_in_Australia', 'A/Opuntia', 'A/Prickly_Pear_(British_Virgin_Islands)', 'A/Prickly_Pear_Island', 'A/Prickly_Pears_(film)', 'A/Invasive_succulent_plants_in_South_Africa', 'A/Cactoblastis_Memorial_Hall', 'A/Thomas_Harvey_Johnston'] - assert lib.get_unique_paths(*paths) == ['A/Prickly_Pear', - 'A/Arborescent_Prickly_Pear', - 'A/Beavertail_prickly_pear', - 'A/Brittle_prickly-pear', - 'A/Chenille_prickly-pear', - 'A/Coastal_Prickly_Pear', - 'A/Prickly_pears_in_South_Africa', - 'A/Prickly_Pear_Cays', - 'A/Prickly_pears_in_Australia', - 'A/Opuntia', - 'A/Prickly_Pear_(British_Virgin_Islands)', - 'A/Prickly_Pear_Island', - 'A/Prickly_Pears_(film)', - 'A/Invasive_succulent_plants_in_South_Africa', - 'A/Cactoblastis_Memorial_Hall', - 'A/Thomas_Harvey_Johnston'] + assert libzim_worker.unique_paths(*paths) == ['A/Prickly_Pear', + 'A/Arborescent_Prickly_Pear', + 'A/Beavertail_prickly_pear', + 'A/Brittle_prickly-pear', + 'A/Chenille_prickly-pear', + 'A/Coastal_Prickly_Pear', + 'A/Prickly_pears_in_South_Africa', + 'A/Prickly_Pear_Cays', + 'A/Prickly_pears_in_Australia', + 'A/Opuntia', + 'A/Prickly_Pear_(British_Virgin_Islands)', + 'A/Prickly_Pear_Island', + 'A/Prickly_Pears_(film)', + 'A/Invasive_succulent_plants_in_South_Africa', + 'A/Cactoblastis_Memorial_Hall', + 'A/Thomas_Harvey_Johnston'] @pytest.mark.asyncio