diff --git a/modules/zim/api.py b/modules/zim/api.py index 46b53692..891c1bfc 100644 --- a/modules/zim/api.py +++ b/modules/zim/api.py @@ -1,4 +1,3 @@ -import asyncio import urllib.parse from http import HTTPStatus @@ -12,7 +11,6 @@ from wrolpi.common import logger from wrolpi.downloader import download_manager from wrolpi.events import Events from . import lib, schema -from .models import Zims zim_bp = Blueprint('Zim', '/api/zim') @@ -64,13 +62,8 @@ async def post_set_zim_auto_search(request: Request, zim_id: int, body: schema.Z ) @validate(schema.ZimSearchRequest) async def search_all_zims(request: Request, body: schema.ZimSearchRequest): - # libzim searches are synchronous and slow; run off the event loop. The request's Session is used - # in the worker thread; the handler awaits the thread, so the Session has one user at a time. - def search(): - return lib.search_all_zims(request.ctx.session, body.search_str, tag_names=body.tag_names, - offset=body.offset, limit=body.limit) - - results = await asyncio.to_thread(search) + results = await lib.search_all_zims(request.ctx.session, body.search_str, tag_names=body.tag_names, + offset=body.offset, limit=body.limit) return json_response({'zims': results}) @@ -82,11 +75,8 @@ async def search_all_zims(request: Request, body: schema.ZimSearchRequest): ) @validate(schema.ZimSearchRequest) async def search_zim(request: Request, zim_id: int, body: schema.ZimSearchRequest): - def search(): - return lib.headline_zim(request.ctx.session, body.search_str, zim_id, tag_names=body.tag_names, - offset=body.offset, limit=body.limit) - - headlines = await asyncio.to_thread(search) + headlines = await lib.headline_zim(request.ctx.session, body.search_str, zim_id, tag_names=body.tag_names, + offset=body.offset, limit=body.limit) return json_response({'zim': headlines}) @@ -210,17 +200,14 @@ async def post_search_estimates(request: Request, body: schema.SearchEstimateReq if not body.search_str and not body.tag_names: return response.empty(HTTPStatus.BAD_REQUEST) - def estimate(): - session = request.ctx.session - if body.tag_names: - # Get actual count of entries tagged with the tag names. - counts = Zims.entries_with_tags(session, body.tag_names) - else: - # Get estimates using libzim. - counts = Zims.estimate(session, body.search_str) - return [dict(estimate=count, **zim.__json__()) for zim, count in counts.items()] - - zims_estimates = await asyncio.to_thread(estimate) + session = request.ctx.session + if body.tag_names: + # Count the entries tagged with the tag names (from the database; see `lib.count_tagged_zim_entries`). + counts = await lib.count_tagged_zim_entries(session, body.tag_names) + else: + # Get estimates using libzim, in the estimate process (see `lib.estimate_zims`). + counts = await lib.estimate_zims(session, body.search_str) + zims_estimates = [dict(estimate=count, **zim.__json__()) for zim, count in counts.items()] ret = dict( zims_estimates=zims_estimates diff --git a/modules/zim/lib.py b/modules/zim/lib.py index e99456da..95221ab4 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -1,9 +1,12 @@ +import asyncio import functools import json import pathlib import re +import sys from datetime import datetime -from typing import Callable, List, Tuple, Dict +from collections import OrderedDict +from typing import Callable, List, Tuple, Dict, OrderedDict as OrderedDictType from libzim import Entry from sqlalchemy import or_ @@ -13,11 +16,11 @@ from sqlalchemy.orm.exc import NoResultFound # noqa from modules.zim import kiwix from modules.zim.errors import UnknownZim, UnknownZimSubscription from modules.zim.kiwix import KIWIX_CATALOG -from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription +from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription, ZimMetadata from wrolpi import flags, tags from wrolpi.cmd import run_command from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \ - register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate, aiohttp_post + register_refresh_cleanup, background_task, get_wrolpi_config, aiohttp_post from wrolpi.switches import register_switch_handler from wrolpi.dates import Seconds from wrolpi.events import Events @@ -39,7 +42,6 @@ __all__ = [ 'get_entries_tags', 'get_entry', 'get_kiwix_catalog', - 'get_unique_paths', 'get_zim', 'get_zims', 'model_zim', @@ -153,40 +155,67 @@ def get_entries_tags(session: Session, paths: List[str], zim_id: int): return entries -def get_unique_paths(*paths: str) -> List[str]: - """Return a new list which contains unique Zim paths.""" - unique_paths = unique_by_predicate( - paths, - lambda i: i.replace('-', '').replace('_', '').replace('/', '').lower() +LIBZIM_WORKER = pathlib.Path(__file__).with_name('libzim_worker.py') + + +async def run_libzim_jobs(jobs: List[Dict]) -> List[Dict]: + """Run libzim jobs (see `libzim_worker.py`) in a separate process, return one result per job. + + python-libzim never releases the GIL, and opening a large Zim cold takes seconds, so nothing libzim does may + run in the API worker: every other request on that worker would wait behind it. (A plain subprocess: + Sanic workers are daemonic, so they cannot fork multiprocessing children.) A failed job's result is + `{'error': ...}`, so one bad Zim does not fail the rest.""" + request = json.dumps(dict(jobs=jobs)).encode() + proc = await asyncio.create_subprocess_exec( + sys.executable, '-I', str(LIBZIM_WORKER), + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - return list(unique_paths) + try: + stdout, stderr = await proc.communicate(request) + except asyncio.CancelledError: + # Sanic cancels the handler at its response timeout or when the client disconnects; cancelling + # `communicate` leaves the process running (and, once its stdout pipe fills, stuck). Kill and reap it. + try: + proc.kill() + except ProcessLookupError: + pass + await proc.wait() + raise + if proc.returncode != 0: + raise RuntimeError(f'libzim worker failed ({proc.returncode}): {stderr.decode(errors="replace").strip()}') + if stderr: + logger.error(f'libzim worker reported errors:\n{stderr.decode(errors="replace").strip()}') + return json.loads(stdout) -def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List[str] = None, offset: int = 0, - limit: int = 10) -> Dict: - zim_id = int(zim_id) - zim = get_zim(session, zim_id) +def _apply_metadata(zim: Zim, result: Dict): + """Fill the Zim's cached `zim_metadata` from a worker result so `Zim.__json__` does not open the Zim.""" + if metadata := result.get('metadata'): + zim.__dict__['zim_metadata'] = ZimMetadata(**metadata) + +def _search_job(zim: Zim, search_str: str, tag_names: List[str], session: Session, offset: int, limit: int) -> Dict: + """The worker job that fetches what `_headline_results` needs for one Zim.""" if tag_names: # TODO what is the count goes over the limit? - entries = zim.entries_with_tags(session, tag_names, offset=offset, limit=1_000) - estimate = 0 + paths = zim.entry_paths_with_tags(session, tag_names, offset=offset, limit=1_000) + return dict(op='entries', path=str(zim.path), paths=paths, metadata=True) elif search_str: - # Get suggested results (this searches titles). - results = zim.suggest(search_str, offset, limit) - search_results = zim.search(search_str, offset, limit) - # Remove duplicate entries (disambiguation or redirect). - results = get_unique_paths(*results, *search_results) - results = results[:limit] - entries = [zim.find_entry(i) for i in results] - estimate = zim.estimate(search_str) - else: - raise RuntimeError('Must provide search_str or tag_names.') + return dict(op='search', path=str(zim.path), search_str=search_str, offset=offset, limit=limit, metadata=True) + raise RuntimeError('Must provide search_str or tag_names.') - articles = [bytes(i.get_item().content).decode('UTF-8') for i in entries] - articles = [extract_html_text(i) for i in articles] - entries_tag_names = get_entries_tags(session, [i.path for i in entries], zim_id) +def _headline_results(session: Session, zim: Zim, search_str: str, tag_names: List[str], result: Dict) -> Dict: + """Headline the entries a search job returned for one Zim.""" + if 'error' in result: + raise RuntimeError(f'Failed to search {zim}: {result["error"]}') + _apply_metadata(zim, result) + entries = result['entries'] + estimate = result.get('estimate', 0) + + articles = [extract_html_text(i['content']) for i in entries] + + entries_tag_names = get_entries_tags(session, [i['path'] for i in entries], zim.id) # Always get headlines even if there is no search_str. Non-matches get the leading text of each article. headlines = extract_headlines(articles, search_str) @@ -194,7 +223,7 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List search = list() if search_str: # `search_str` allows us to headline the title. - titles = [i.title for i in entries] + titles = [i['title'] for i in entries] headline_titles = extract_headlines(titles, search_str) for entry in entries: @@ -202,7 +231,7 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List headline, rank = headlines.pop(0) if rank > 0 or title_rank > 0: # The result for each Entry. - path = entry.path + path = entry['path'] entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list() search.append(dict( headline=headline, @@ -217,14 +246,14 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List # No `search_str`, we return all entries. for entry in entries: headline, _ = headlines.pop(0) - path = entry.path + path = entry['path'] entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list() search.append(dict( headline=headline, path=path, rank=0, tag_names=entry_tag_names, - title=entry.title, + title=entry['title'], )) estimate = len(entries) @@ -237,26 +266,76 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List return results -def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0, - limit: int = 10) -> List[Dict]: - """Search all Zim files that have auto_search enabled. +async def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List[str] = None, offset: int = 0, + limit: int = 10) -> Dict: + zim = get_zim(session, int(zim_id)) + job = _search_job(zim, search_str, tag_names, session, offset, limit) + result, = await run_libzim_jobs([job]) + return await asyncio.to_thread(_headline_results, session, zim, search_str, tag_names, result) + + +def auto_search_zims(session: Session) -> List[Zim]: + """The Zims that the global search searches, without opening them.""" + zims = session.query(Zim).options(*Zim.json_options()).filter_by(auto_search=True).order_by(Zim.path).all() + return [i for i in zims if i.path and i.path.is_file()] + + +async def estimate_zims(session: Session, search_str: str) -> OrderedDictType[Zim, int]: + """Estimate how many entries of each auto-search Zim match `search_str`. + + `Zims.get_all` opens every Zim and is not used here for that reason; a Zim the worker cannot read is + reported and left out, as `Zims.get_all` would.""" + zims = auto_search_zims(session) + jobs = [dict(op='estimate', path=str(i.path), search_str=search_str, metadata=True) for i in zims] + results = await run_libzim_jobs(jobs) + counts = OrderedDict() + for zim, result in zip(zims, results): + if 'error' not in result: + _apply_metadata(zim, result) + counts[zim] = result['estimate'] + return counts + + +async def count_tagged_zim_entries(session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]: + """Count the entries of every Zim tagged with all of `tag_names`. The counts come from the database; the + worker only reads each Zim's metadata, and a Zim it cannot read is left out.""" + zims = session.query(Zim).options(*Zim.json_options()).order_by(Zim.path).all() + zims = [i for i in zims if i.path and i.path.is_file()] + counts = {i: len(i.entry_paths_with_tags(session, tag_names, limit=1_000_000)) for i in zims} + results = await run_libzim_jobs([dict(op='metadata', path=str(i.path)) for i in zims]) + readable = OrderedDict() + for zim, result in zip(zims, results): + if 'error' not in result: + _apply_metadata(zim, result) + readable[zim] = counts[zim] + return readable + + +async def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0, + limit: int = 10) -> List[Dict]: + """Search all Zim files that have auto_search enabled, in one libzim worker process. Returns a list of per-zim result dicts, each containing metadata and search results. Each search entry includes `zim_id` so callers can fetch the full entry afterward.""" - zims = Zims.get_all(session) - all_results = [] - for zim in zims: - if not zim.auto_search: - continue - try: - result = headline_zim(session, search_str, zim.id, tag_names=tag_names, offset=offset, limit=limit) - if result.get('search'): - for entry in result['search']: + zims = auto_search_zims(session) + jobs = [_search_job(zim, search_str, tag_names, session, offset, limit) for zim in zims] + results = await run_libzim_jobs(jobs) + + def headline_all(): + all_results = [] + for zim, result in zip(zims, results): + try: + headlined = _headline_results(session, zim, search_str, tag_names, result) + except Exception as e: + logger.error(f'Failed to search Zim {zim}', exc_info=e) + continue + if headlined.get('search'): + for entry in headlined['search']: entry['zim_id'] = zim.id - all_results.append(result) - except Exception as e: - logger.error(f'Failed to search Zim {zim}', exc_info=e) - return all_results + all_results.append(headlined) + return all_results + + return await asyncio.to_thread(headline_all) def get_zim(session: Session, zim_id: int) -> Zim: diff --git a/modules/zim/libzim_worker.py b/modules/zim/libzim_worker.py new file mode 100644 index 00000000..f48466d6 --- /dev/null +++ b/modules/zim/libzim_worker.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Run libzim jobs (estimate, search, fetch entries) for the API. + +Run by `modules.zim.lib.run_libzim_jobs` as a separate process: python-libzim never releases the GIL, and merely +opening a large Zim whose pages are not cached takes seconds (ten for a 75 GB Zim), so doing this in the API +worker would freeze every other request that worker is handling. + +Reads `{"jobs": [...]}` from stdin and writes one result per job, in order, to stdout. Every job names the Zim +`path` and an `op`: + + {"op": "estimate", "search_str": ...} -> {"estimate": int} + {"op": "search", "search_str": ..., "offset": ..., "limit": ...} + -> {"estimate": int, "entries": [{"path", "title", "content"}]} + {"op": "entries", "paths": [...]} -> {"entries": [{"path", "title", "content"}]} + {"op": "metadata"} -> {} (with the metadata below) + +Any job may add `"metadata": true` to also get the Zim's metadata. A job that fails returns `{"error": str}` +so one bad Zim does not hide the rest. Imports only the standard library and libzim; it is run with +`python -I` from any working directory. +""" +import json +import sys + +from libzim.reader import Archive +from libzim.search import Query, Searcher +from libzim.suggestion import SuggestionSearcher + +METADATA_KEYS = ('Date', 'Creator', 'Description', 'Name', 'Publisher', 'Tags', 'Title') + + +def get_metadata(archive: Archive) -> dict: + metadata = dict() + for key in METADATA_KEYS: + try: + metadata[key.lower()] = archive.get_metadata(key).decode() + except RuntimeError: + metadata[key.lower()] = None + return metadata + + +def unique_paths(*paths: str) -> list: + """The first of any paths that differ only in case, dashes, underscores or slashes (disambiguation or + redirect entries).""" + seen, unique = set(), list() + for path in paths: + key = path.replace('-', '').replace('_', '').replace('/', '').lower() + if key not in seen: + seen.add(key) + unique.append(path) + return unique + + +def entry_dict(archive: Archive, path: str) -> dict: + entry = archive.get_entry_by_path(path) + return dict(path=entry.path, title=entry.title, content=bytes(entry.get_item().content).decode('UTF-8')) + + +def run_job(job: dict) -> dict: + archive = Archive(job['path']) + op = job['op'] + result = dict() + if op == 'estimate': + query = Query().set_query(job['search_str']) + result['estimate'] = Searcher(archive).search(query).getEstimatedMatches() + elif op == 'search': + search_str, offset, limit = job['search_str'], job['offset'], job['limit'] + query = Query().set_query(search_str) + # Suggestions search the titles. + suggested = list(SuggestionSearcher(archive).suggest(search_str).getResults(offset, limit)) + search = Searcher(archive).search(query) + found = list(search.getResults(offset, limit)) + paths = unique_paths(*suggested, *found)[:limit] + result['estimate'] = search.getEstimatedMatches() + result['entries'] = [entry_dict(archive, i) for i in paths] + elif op == 'entries': + result['entries'] = [entry_dict(archive, i) for i in job['paths']] + elif op == 'metadata': + job['metadata'] = True + else: + raise ValueError(f'Unknown op {op!r}') + if job.get('metadata'): + result['metadata'] = get_metadata(archive) + return result + + +def main(): + request = json.load(sys.stdin) + results = list() + for job in request['jobs']: + try: + results.append(run_job(job)) + except Exception as e: + print(f'{job.get("path")}: {e}', file=sys.stderr) + results.append(dict(error=str(e))) + json.dump(results, sys.stdout) + + +if __name__ == '__main__': + main() diff --git a/modules/zim/models.py b/modules/zim/models.py index 5afeb50d..5aa98ebb 100644 --- a/modules/zim/models.py +++ b/modules/zim/models.py @@ -5,7 +5,7 @@ from collections import OrderedDict from datetime import datetime from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional -from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher +from libzim import Archive, Entry from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean, UniqueConstraint from sqlalchemy.orm import Mapped, relationship, Session, joinedload from sqlalchemy.orm.exc import NoResultFound # noqa @@ -165,39 +165,6 @@ class Zim(Base, ModelHelper): ) return metadata - def get_searcher(self) -> Searcher: - searcher = Searcher(self.get_zim()) - return searcher - - def get_search(self, search_str: str): - query = Query().set_query(search_str) - searcher = self.get_searcher() - search = searcher.search(query) - return search - - def get_suggestion_searcher(self) -> SuggestionSearcher: - searcher = SuggestionSearcher(self.get_zim()) - return searcher - - def estimate(self, search_str: str) -> int: - """Count the number of Entry(s) that contain the `search_str`.""" - search = self.get_search(search_str) - count = search.getEstimatedMatches() - return count - - def search(self, search_str: str, offset: int = 0, limit: int = 10) -> List[str]: - """Return the paths of the Entry(s) that contain the `search_str`.""" - search = self.get_search(search_str) - results = list(search.getResults(offset, limit)) - return results - - def suggest(self, search_str: str, offset: int = 0, limit: int = 10) -> List[str]: - """Return the paths of the Entry(s) that contain the `search_str`. Also searches the titles.""" - searcher = self.get_suggestion_searcher() - suggestions = searcher.suggest(search_str) - results = list(suggestions.getResults(offset, limit)) - return results - def get_entry(self, path: str) -> Optional[Entry]: """Search this Zim file for the provided Entry at the `path`.""" zim = self.get_zim() @@ -256,22 +223,21 @@ class Zim(Base, ModelHelper): .order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry) \ .offset(offset).limit(limit) - entries = list() - for tag_zim_entry, zim in tag_zim_entries: - entry = zim.find_entry(tag_zim_entry.zim_entry) - entries.append(entry) + return [(zim, tag_zim_entry.zim_entry) for tag_zim_entry, zim in tag_zim_entries] - return entries - - def entries_with_tags(self, session: Session, tag_names: List[str], limit: int = 10, offset: int = 0) -> List[ - Entry]: - """Return Zim Entries tagged with the provided names.""" + def entry_paths_with_tags(self, session: Session, tag_names: List[str], limit: int = 10, offset: int = 0) \ + -> List[str]: + """Return the paths of the Zim Entries tagged with the provided names. Does not open the Zim.""" if limit < 1: raise RuntimeError('Limit must be greater than zero') tags_sub_select, params = tag_names_to_zim_sub_select(tag_names, zim_id=self.id) - entries = Zim._entries_with_tags_process(limit, offset, params, session, tags_sub_select) - return entries + return [path for _, path in Zim._entries_with_tags_process(limit, offset, params, session, tags_sub_select)] + + def entries_with_tags(self, session: Session, tag_names: List[str], limit: int = 10, offset: int = 0) -> List[ + Entry]: + """Return Zim Entries tagged with the provided names.""" + return [self.find_entry(i) for i in self.entry_paths_with_tags(session, tag_names, limit, offset)] class Zims: @@ -290,23 +256,6 @@ class Zims: logger.error(f'Failed to read Zim: {zim}', exc_info=e) return zims - @classmethod - def estimate(cls, session: Session, search_str: str) -> OrderedDictType[Zim, int]: - zims = cls.get_all(session) - results = OrderedDict() - for zim in zims: - if zim.auto_search: - results[zim] = zim.estimate(search_str) - return results - - @classmethod - def entries_with_tags(cls, session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]: - zims = cls.get_all(session) - results = OrderedDict() - for zim in zims: - results[zim] = len(zim.entries_with_tags(session, tag_names)) - return results - class TagZimEntry(Base): __tablename__ = 'tag_zim' diff --git a/modules/zim/test/test_api.py b/modules/zim/test/test_api.py index 630e2acc..06a4b6ae 100644 --- a/modules/zim/test/test_api.py +++ b/modules/zim/test/test_api.py @@ -1,4 +1,7 @@ +import asyncio import json +import sys +from unittest import mock from http import HTTPStatus import pytest @@ -366,3 +369,108 @@ async def test_delete_zim_waits_for_a_concurrent_writer(async_client, test_sessi assert not zim_path.exists() with maker() as session: assert session.query(Zim).filter_by(id=zim_id).one_or_none() is None + + +@pytest.mark.asyncio +async def test_estimate_zims_runs_outside_the_api_process(test_session, zim_factory): + """libzim holds the GIL, so estimates run in a separate process rather than a thread of the API worker.""" + zim = zim_factory('test zim') + test_session.commit() + + with mock.patch('modules.zim.lib.asyncio.create_subprocess_exec', wraps=asyncio.create_subprocess_exec) as m: + counts = await lib.estimate_zims(test_session, 'one') + assert counts == {zim: 2} + assert m.call_args.args[:2] == (sys.executable, '-I') + + +@pytest.mark.asyncio +async def test_estimate_zims_skips_a_zim_it_cannot_read(test_session, zim_factory): + """A Zim the estimate process cannot open is left out of the estimates, like `Zims.get_all` leaves it out.""" + good = zim_factory('good zim') + bad = zim_factory('bad zim') + test_session.commit() + bad.path.write_bytes(b'not a zim file') + + counts = await lib.estimate_zims(test_session, 'one') + assert counts == {good: 2} + + bad.path.unlink() + counts = await lib.estimate_zims(test_session, 'one') + assert counts == {good: 2} + + +@pytest.mark.asyncio +async def test_zim_search_runs_outside_the_api_process(async_client, test_session, test_zim): + """Zim searches, like estimates, do all their libzim work in the worker script's process.""" + with mock.patch('modules.zim.lib.asyncio.create_subprocess_exec', wraps=asyncio.create_subprocess_exec) as m: + request, response = await async_client.post(f'/api/zim/search/{test_zim.id}', json=dict(search_str='one')) + assert response.status_code == HTTPStatus.OK + assert [i['path'] for i in response.json['zim']['search']] == ['one', 'home'] + assert response.json['zim']['metadata']['title'], 'Metadata should come from the worker' + + request, response = await async_client.post('/api/zim/search', json=dict(search_str='one')) + assert response.status_code == HTTPStatus.OK + assert [i['path'] for i in response.json['zims'][0]['search']] == ['one', 'home'] + assert m.call_count == 2 + assert all(call.args[:2] == (sys.executable, '-I') for call in m.call_args_list) + + +@pytest.mark.asyncio +async def test_search_all_zims_skips_a_zim_it_cannot_read(async_client, test_session, zim_factory): + """One unreadable Zim does not fail the search of all Zims.""" + good = zim_factory('good zim') + bad = zim_factory('bad zim') + test_session.commit() + bad.path.write_bytes(b'not a zim file') + + request, response = await async_client.post('/api/zim/search', json=dict(search_str='one')) + assert response.status_code == HTTPStatus.OK + zims = response.json['zims'] + assert len(zims) == 1 and zims[0]['path'].endswith('good zim') + assert zims[0]['estimate'] == 2 + + +@pytest.mark.asyncio +async def test_tagged_search_estimates_do_not_open_zims(async_client, test_session, test_zim, tag_factory): + """Estimates with tags count from the database; the API process never opens a Zim, even for its metadata.""" + tag1, tag2 = await tag_factory(), await tag_factory() + test_zim.tag_entry(test_session, tag1.name, 'one') + test_zim.tag_entry(test_session, tag1.name, 'two') + test_zim.tag_entry(test_session, tag2.name, 'one') + test_session.commit() + + with mock.patch('modules.zim.models.Zim.get_zim', side_effect=AssertionError('opened a Zim in the API')): + body = dict(search_str='one', tag_names=[tag1.name]) + request, response = await async_client.post('/api/zim/search_estimates', json=body) + assert response.status_code == HTTPStatus.OK + assert [(i['estimate'], bool(i['metadata']['title'])) for i in response.json['zims_estimates']] \ + == [(2, True)] + + body = dict(search_str='', tag_names=[tag1.name, tag2.name]) + request, response = await async_client.post('/api/zim/search_estimates', json=body) + assert [i['estimate'] for i in response.json['zims_estimates']] == [1] + + +@pytest.mark.asyncio +async def test_cancelled_libzim_jobs_kill_the_worker(test_directory): + """Sanic cancels a handler at its response timeout or on disconnect; the worker process must not outlive it.""" + slow_worker = test_directory / 'slow_worker.py' + slow_worker.write_text('import time; time.sleep(60)') + procs = [] + create_subprocess_exec = asyncio.create_subprocess_exec + + async def spy(*args, **kwargs): + proc = await create_subprocess_exec(*args, **kwargs) + procs.append(proc) + return proc + + with mock.patch('modules.zim.lib.LIBZIM_WORKER', slow_worker), \ + mock.patch('modules.zim.lib.asyncio.create_subprocess_exec', side_effect=spy): + task = asyncio.create_task(lib.run_libzim_jobs([])) + while not procs: + await asyncio.sleep(0.01) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert procs[0].returncode is not None, 'The worker was killed and reaped' diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index 814d97a9..06f67b70 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -4,7 +4,7 @@ from http import HTTPStatus import pytest from libzim import Entry -from modules.zim import lib +from modules.zim import lib, libzim_worker from modules.zim.errors import UnknownZim, UnknownZimEntry from modules.zim.models import TagZimEntry, ZimSubscription from wrolpi import tags, flags @@ -96,29 +96,29 @@ async def test_zim_entries_with_tags(async_client, test_session, zim_factory, ta assert entries[0].path == 'two' -def test_get_unique_paths(): +def test_unique_paths(): paths = ['A/Prickly_Pear', 'A/Prickly_pear', 'A/Prickly-pear', 'A/Arborescent_Prickly_Pear', 'A/Arborescent_prickly_pear', 'A/Beavertail_prickly_pear', 'A/Brittle_prickly-pear', 'A/Chenille_prickly-pear', 'A/Coastal_Prickly_Pear', 'A/Coastal_prickly_pear', 'A/Prickly_pears_in_South_Africa', 'A/Prickly_Pear_Cays', 'A/Prickly_pears_in_Australia', 'A/Opuntia', 'A/Prickly_Pear_(British_Virgin_Islands)', 'A/Prickly_Pear_Island', 'A/Prickly_Pears_(film)', 'A/Invasive_succulent_plants_in_South_Africa', 'A/Cactoblastis_Memorial_Hall', 'A/Thomas_Harvey_Johnston'] - assert lib.get_unique_paths(*paths) == ['A/Prickly_Pear', - 'A/Arborescent_Prickly_Pear', - 'A/Beavertail_prickly_pear', - 'A/Brittle_prickly-pear', - 'A/Chenille_prickly-pear', - 'A/Coastal_Prickly_Pear', - 'A/Prickly_pears_in_South_Africa', - 'A/Prickly_Pear_Cays', - 'A/Prickly_pears_in_Australia', - 'A/Opuntia', - 'A/Prickly_Pear_(British_Virgin_Islands)', - 'A/Prickly_Pear_Island', - 'A/Prickly_Pears_(film)', - 'A/Invasive_succulent_plants_in_South_Africa', - 'A/Cactoblastis_Memorial_Hall', - 'A/Thomas_Harvey_Johnston'] + assert libzim_worker.unique_paths(*paths) == ['A/Prickly_Pear', + 'A/Arborescent_Prickly_Pear', + 'A/Beavertail_prickly_pear', + 'A/Brittle_prickly-pear', + 'A/Chenille_prickly-pear', + 'A/Coastal_Prickly_Pear', + 'A/Prickly_pears_in_South_Africa', + 'A/Prickly_Pear_Cays', + 'A/Prickly_pears_in_Australia', + 'A/Opuntia', + 'A/Prickly_Pear_(British_Virgin_Islands)', + 'A/Prickly_Pear_Island', + 'A/Prickly_Pears_(film)', + 'A/Invasive_succulent_plants_in_South_Africa', + 'A/Cactoblastis_Memorial_Hall', + 'A/Thomas_Harvey_Johnston'] @pytest.mark.asyncio