Zim estimates and searches run in a libzim worker process so a cold Zim cannot stall the API worker (#928)

* Zim search estimates run in a separate process so a cold estimate cannot stall the API worker

The global search estimates every auto-search Zim on each keystroke.  On a
large library that estimate took tens of seconds when the Zims' pages were not
cached, and every other request routed to the same Sanic worker, such as the
channel suggestions, waited behind it.  Two things held the worker: opening a
large Zim cold takes seconds (ten for a 75 GB Zim) and `Zims.get_all` did that
on the event loop; and python-libzim never releases the GIL, so the estimate
offloaded with asyncio.to_thread still froze the worker.

Everything libzim does for an estimate now runs in a standalone script started
as a subprocess (Sanic workers are daemonic, so a multiprocessing pool cannot
be forked from them).  It reads the Zim paths and search string on stdin and
writes the counts on stdout.  A Zim it cannot read is reported and left out, as
`Zims.get_all` would leave it out.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Zim searches run in the libzim worker process too

The /api/zim/search endpoints had the same stall as the estimate: they opened
every Zim through Zims.get_all and searched them with libzim, which holds the
GIL, so a thread offload still froze the Sanic worker.

The estimate script is now a general libzim worker with estimate, search and
entries jobs.  A search of all Zims sends every Zim's job to one worker
process; the API worker only headlines the returned HTML and looks up tags.
The worker also returns each Zim's metadata so Zim.__json__ does not open the
Zim again on the event loop.  Zim.entries_with_tags is split so the tagged
paths come from the database without opening the Zim.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Tagged Zim estimates count from the database, and a cancelled request kills the libzim worker

Review of the earlier commits found two gaps.  Estimates with tags selected
still went through Zims.entries_with_tags, which opened every Zim in the API
process, and after the thread offload was removed it did so on the event loop;
they now count the tagged paths in SQL and ask the worker only for metadata.
And Sanic cancels a handler at its response timeout or on disconnect, which
left the worker process running until its stdout pipe filled; the runner now
kills and reaps it on cancellation.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-11-01 13:02:26 -07:00 • committed by GitHub
parent 9c258b41ee
commit bfe7b8afbd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 376 additions and 154 deletions

View file

@ -1,4 +1,3 @@
import asyncio
import urllib.parse
from http import HTTPStatus
@ -12,7 +11,6 @@ from wrolpi.common import logger
from wrolpi.downloader import download_manager
from wrolpi.events import Events
from . import lib, schema
from .models import Zims
zim_bp = Blueprint('Zim', '/api/zim')
@ -64,13 +62,8 @@ async def post_set_zim_auto_search(request: Request, zim_id: int, body: schema.Z
)
@validate(schema.ZimSearchRequest)
async def search_all_zims(request: Request, body: schema.ZimSearchRequest):
# libzim searches are synchronous and slow; run off the event loop. The request's Session is used
# in the worker thread; the handler awaits the thread, so the Session has one user at a time.
def search():
return lib.search_all_zims(request.ctx.session, body.search_str, tag_names=body.tag_names,
offset=body.offset, limit=body.limit)
results = await asyncio.to_thread(search)
results = await lib.search_all_zims(request.ctx.session, body.search_str, tag_names=body.tag_names,
offset=body.offset, limit=body.limit)
return json_response({'zims': results})
@ -82,11 +75,8 @@ async def search_all_zims(request: Request, body: schema.ZimSearchRequest):
)
@validate(schema.ZimSearchRequest)
async def search_zim(request: Request, zim_id: int, body: schema.ZimSearchRequest):
def search():
return lib.headline_zim(request.ctx.session, body.search_str, zim_id, tag_names=body.tag_names,
offset=body.offset, limit=body.limit)
headlines = await asyncio.to_thread(search)
headlines = await lib.headline_zim(request.ctx.session, body.search_str, zim_id, tag_names=body.tag_names,
offset=body.offset, limit=body.limit)
return json_response({'zim': headlines})
@ -210,17 +200,14 @@ async def post_search_estimates(request: Request, body: schema.SearchEstimateReq
if not body.search_str and not body.tag_names:
return response.empty(HTTPStatus.BAD_REQUEST)
def estimate():
session = request.ctx.session
if body.tag_names:
# Get actual count of entries tagged with the tag names.
counts = Zims.entries_with_tags(session, body.tag_names)
else:
# Get estimates using libzim.
counts = Zims.estimate(session, body.search_str)
return [dict(estimate=count, **zim.__json__()) for zim, count in counts.items()]
zims_estimates = await asyncio.to_thread(estimate)
session = request.ctx.session
if body.tag_names:
# Count the entries tagged with the tag names (from the database; see `lib.count_tagged_zim_entries`).
counts = await lib.count_tagged_zim_entries(session, body.tag_names)
else:
# Get estimates using libzim, in the estimate process (see `lib.estimate_zims`).
counts = await lib.estimate_zims(session, body.search_str)
zims_estimates = [dict(estimate=count, **zim.__json__()) for zim, count in counts.items()]
ret = dict(
zims_estimates=zims_estimates

View file

@ -1,9 +1,12 @@
import asyncio
import functools
import json
import pathlib
import re
import sys
from datetime import datetime
from typing import Callable, List, Tuple, Dict
from collections import OrderedDict
from typing import Callable, List, Tuple, Dict, OrderedDict as OrderedDictType
from libzim import Entry
from sqlalchemy import or_
@ -13,11 +16,11 @@ from sqlalchemy.orm.exc import NoResultFound # noqa
from modules.zim import kiwix
from modules.zim.errors import UnknownZim, UnknownZimSubscription
from modules.zim.kiwix import KIWIX_CATALOG
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription, ZimMetadata
from wrolpi import flags, tags
from wrolpi.cmd import run_command
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate, aiohttp_post
register_refresh_cleanup, background_task, get_wrolpi_config, aiohttp_post
from wrolpi.switches import register_switch_handler
from wrolpi.dates import Seconds
from wrolpi.events import Events
@ -39,7 +42,6 @@ __all__ = [
'get_entries_tags',
'get_entry',
'get_kiwix_catalog',
'get_unique_paths',
'get_zim',
'get_zims',
'model_zim',
@ -153,40 +155,67 @@ def get_entries_tags(session: Session, paths: List[str], zim_id: int):
return entries
def get_unique_paths(*paths: str) -> List[str]:
"""Return a new list which contains unique Zim paths."""
unique_paths = unique_by_predicate(
paths,
lambda i: i.replace('-', '').replace('_', '').replace('/', '').lower()
LIBZIM_WORKER = pathlib.Path(__file__).with_name('libzim_worker.py')
async def run_libzim_jobs(jobs: List[Dict]) -> List[Dict]:
"""Run libzim jobs (see `libzim_worker.py`) in a separate process, return one result per job.
python-libzim never releases the GIL, and opening a large Zim cold takes seconds, so nothing libzim does may
run in the API worker: every other request on that worker would wait behind it. (A plain subprocess:
Sanic workers are daemonic, so they cannot fork multiprocessing children.) A failed job's result is
`{'error': ...}`, so one bad Zim does not fail the rest."""
request = json.dumps(dict(jobs=jobs)).encode()
proc = await asyncio.create_subprocess_exec(
sys.executable, '-I', str(LIBZIM_WORKER),
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
)
return list(unique_paths)
try:
stdout, stderr = await proc.communicate(request)
except asyncio.CancelledError:
# Sanic cancels the handler at its response timeout or when the client disconnects; cancelling
# `communicate` leaves the process running (and, once its stdout pipe fills, stuck). Kill and reap it.
try:
proc.kill()
except ProcessLookupError:
pass
await proc.wait()
raise
if proc.returncode != 0:
raise RuntimeError(f'libzim worker failed ({proc.returncode}): {stderr.decode(errors="replace").strip()}')
if stderr:
logger.error(f'libzim worker reported errors:\n{stderr.decode(errors="replace").strip()}')
return json.loads(stdout)
def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List[str] = None, offset: int = 0,
limit: int = 10) -> Dict:
zim_id = int(zim_id)
zim = get_zim(session, zim_id)
def _apply_metadata(zim: Zim, result: Dict):
"""Fill the Zim's cached `zim_metadata` from a worker result so `Zim.__json__` does not open the Zim."""
if metadata := result.get('metadata'):
zim.__dict__['zim_metadata'] = ZimMetadata(**metadata)
def _search_job(zim: Zim, search_str: str, tag_names: List[str], session: Session, offset: int, limit: int) -> Dict:
"""The worker job that fetches what `_headline_results` needs for one Zim."""
if tag_names:
# TODO what is the count goes over the limit?
entries = zim.entries_with_tags(session, tag_names, offset=offset, limit=1_000)
estimate = 0
paths = zim.entry_paths_with_tags(session, tag_names, offset=offset, limit=1_000)
return dict(op='entries', path=str(zim.path), paths=paths, metadata=True)
elif search_str:
# Get suggested results (this searches titles).
results = zim.suggest(search_str, offset, limit)
search_results = zim.search(search_str, offset, limit)
# Remove duplicate entries (disambiguation or redirect).
results = get_unique_paths(*results, *search_results)
results = results[:limit]
entries = [zim.find_entry(i) for i in results]
estimate = zim.estimate(search_str)
else:
raise RuntimeError('Must provide search_str or tag_names.')
return dict(op='search', path=str(zim.path), search_str=search_str, offset=offset, limit=limit, metadata=True)
raise RuntimeError('Must provide search_str or tag_names.')
articles = [bytes(i.get_item().content).decode('UTF-8') for i in entries]
articles = [extract_html_text(i) for i in articles]
entries_tag_names = get_entries_tags(session, [i.path for i in entries], zim_id)
def _headline_results(session: Session, zim: Zim, search_str: str, tag_names: List[str], result: Dict) -> Dict:
"""Headline the entries a search job returned for one Zim."""
if 'error' in result:
raise RuntimeError(f'Failed to search {zim}: {result["error"]}')
_apply_metadata(zim, result)
entries = result['entries']
estimate = result.get('estimate', 0)
articles = [extract_html_text(i['content']) for i in entries]
entries_tag_names = get_entries_tags(session, [i['path'] for i in entries], zim.id)
# Always get headlines even if there is no search_str. Non-matches get the leading text of each article.
headlines = extract_headlines(articles, search_str)
@ -194,7 +223,7 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List
search = list()
if search_str:
# `search_str` allows us to headline the title.
titles = [i.title for i in entries]
titles = [i['title'] for i in entries]
headline_titles = extract_headlines(titles, search_str)
for entry in entries:
@ -202,7 +231,7 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List
headline, rank = headlines.pop(0)
if rank > 0 or title_rank > 0:
# The result for each Entry.
path = entry.path
path = entry['path']
entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list()
search.append(dict(
headline=headline,
@ -217,14 +246,14 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List
# No `search_str`, we return all entries.
for entry in entries:
headline, _ = headlines.pop(0)
path = entry.path
path = entry['path']
entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list()
search.append(dict(
headline=headline,
path=path,
rank=0,
tag_names=entry_tag_names,
title=entry.title,
title=entry['title'],
))
estimate = len(entries)
@ -237,26 +266,76 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List
return results
def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0,
limit: int = 10) -> List[Dict]:
"""Search all Zim files that have auto_search enabled.
async def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List[str] = None, offset: int = 0,
limit: int = 10) -> Dict:
zim = get_zim(session, int(zim_id))
job = _search_job(zim, search_str, tag_names, session, offset, limit)
result, = await run_libzim_jobs([job])
return await asyncio.to_thread(_headline_results, session, zim, search_str, tag_names, result)
def auto_search_zims(session: Session) -> List[Zim]:
"""The Zims that the global search searches, without opening them."""
zims = session.query(Zim).options(*Zim.json_options()).filter_by(auto_search=True).order_by(Zim.path).all()
return [i for i in zims if i.path and i.path.is_file()]
async def estimate_zims(session: Session, search_str: str) -> OrderedDictType[Zim, int]:
"""Estimate how many entries of each auto-search Zim match `search_str`.
`Zims.get_all` opens every Zim and is not used here for that reason; a Zim the worker cannot read is
reported and left out, as `Zims.get_all` would."""
zims = auto_search_zims(session)
jobs = [dict(op='estimate', path=str(i.path), search_str=search_str, metadata=True) for i in zims]
results = await run_libzim_jobs(jobs)
counts = OrderedDict()
for zim, result in zip(zims, results):
if 'error' not in result:
_apply_metadata(zim, result)
counts[zim] = result['estimate']
return counts
async def count_tagged_zim_entries(session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]:
"""Count the entries of every Zim tagged with all of `tag_names`. The counts come from the database; the
worker only reads each Zim's metadata, and a Zim it cannot read is left out."""
zims = session.query(Zim).options(*Zim.json_options()).order_by(Zim.path).all()
zims = [i for i in zims if i.path and i.path.is_file()]
counts = {i: len(i.entry_paths_with_tags(session, tag_names, limit=1_000_000)) for i in zims}
results = await run_libzim_jobs([dict(op='metadata', path=str(i.path)) for i in zims])
readable = OrderedDict()
for zim, result in zip(zims, results):
if 'error' not in result:
_apply_metadata(zim, result)
readable[zim] = counts[zim]
return readable
async def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0,
limit: int = 10) -> List[Dict]:
"""Search all Zim files that have auto_search enabled, in one libzim worker process.
Returns a list of per-zim result dicts, each containing metadata and search results.
Each search entry includes `zim_id` so callers can fetch the full entry afterward."""
zims = Zims.get_all(session)
all_results = []
for zim in zims:
if not zim.auto_search:
continue
try:
result = headline_zim(session, search_str, zim.id, tag_names=tag_names, offset=offset, limit=limit)
if result.get('search'):
for entry in result['search']:
zims = auto_search_zims(session)
jobs = [_search_job(zim, search_str, tag_names, session, offset, limit) for zim in zims]
results = await run_libzim_jobs(jobs)
def headline_all():
all_results = []
for zim, result in zip(zims, results):
try:
headlined = _headline_results(session, zim, search_str, tag_names, result)
except Exception as e:
logger.error(f'Failed to search Zim {zim}', exc_info=e)
continue
if headlined.get('search'):
for entry in headlined['search']:
entry['zim_id'] = zim.id
all_results.append(result)
except Exception as e:
logger.error(f'Failed to search Zim {zim}', exc_info=e)
return all_results
all_results.append(headlined)
return all_results
return await asyncio.to_thread(headline_all)
def get_zim(session: Session, zim_id: int) -> Zim:

View file

@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""Run libzim jobs (estimate, search, fetch entries) for the API.
Run by `modules.zim.lib.run_libzim_jobs` as a separate process: python-libzim never releases the GIL, and merely
opening a large Zim whose pages are not cached takes seconds (ten for a 75 GB Zim), so doing this in the API
worker would freeze every other request that worker is handling.
Reads `{"jobs": [...]}` from stdin and writes one result per job, in order, to stdout. Every job names the Zim
`path` and an `op`:
{"op": "estimate", "search_str": ...} -> {"estimate": int}
{"op": "search", "search_str": ..., "offset": ..., "limit": ...}
-> {"estimate": int, "entries": [{"path", "title", "content"}]}
{"op": "entries", "paths": [...]} -> {"entries": [{"path", "title", "content"}]}
{"op": "metadata"} -> {} (with the metadata below)
Any job may add `"metadata": true` to also get the Zim's metadata. A job that fails returns `{"error": str}`
so one bad Zim does not hide the rest. Imports only the standard library and libzim; it is run with
`python -I` from any working directory.
"""
import json
import sys
from libzim.reader import Archive
from libzim.search import Query, Searcher
from libzim.suggestion import SuggestionSearcher
METADATA_KEYS = ('Date', 'Creator', 'Description', 'Name', 'Publisher', 'Tags', 'Title')
def get_metadata(archive: Archive) -> dict:
metadata = dict()
for key in METADATA_KEYS:
try:
metadata[key.lower()] = archive.get_metadata(key).decode()
except RuntimeError:
metadata[key.lower()] = None
return metadata
def unique_paths(*paths: str) -> list:
"""The first of any paths that differ only in case, dashes, underscores or slashes (disambiguation or
redirect entries)."""
seen, unique = set(), list()
for path in paths:
key = path.replace('-', '').replace('_', '').replace('/', '').lower()
if key not in seen:
seen.add(key)
unique.append(path)
return unique
def entry_dict(archive: Archive, path: str) -> dict:
entry = archive.get_entry_by_path(path)
return dict(path=entry.path, title=entry.title, content=bytes(entry.get_item().content).decode('UTF-8'))
def run_job(job: dict) -> dict:
archive = Archive(job['path'])
op = job['op']
result = dict()
if op == 'estimate':
query = Query().set_query(job['search_str'])
result['estimate'] = Searcher(archive).search(query).getEstimatedMatches()
elif op == 'search':
search_str, offset, limit = job['search_str'], job['offset'], job['limit']
query = Query().set_query(search_str)
# Suggestions search the titles.
suggested = list(SuggestionSearcher(archive).suggest(search_str).getResults(offset, limit))
search = Searcher(archive).search(query)
found = list(search.getResults(offset, limit))
paths = unique_paths(*suggested, *found)[:limit]
result['estimate'] = search.getEstimatedMatches()
result['entries'] = [entry_dict(archive, i) for i in paths]
elif op == 'entries':
result['entries'] = [entry_dict(archive, i) for i in job['paths']]
elif op == 'metadata':
job['metadata'] = True
else:
raise ValueError(f'Unknown op {op!r}')
if job.get('metadata'):
result['metadata'] = get_metadata(archive)
return result
def main():
request = json.load(sys.stdin)
results = list()
for job in request['jobs']:
try:
results.append(run_job(job))
except Exception as e:
print(f'{job.get("path")}: {e}', file=sys.stderr)
results.append(dict(error=str(e)))
json.dump(results, sys.stdout)
if __name__ == '__main__':
main()

View file

@ -5,7 +5,7 @@ from collections import OrderedDict
from datetime import datetime
from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional
from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher
from libzim import Archive, Entry
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean, UniqueConstraint
from sqlalchemy.orm import Mapped, relationship, Session, joinedload
from sqlalchemy.orm.exc import NoResultFound # noqa
@ -165,39 +165,6 @@ class Zim(Base, ModelHelper):
)
return metadata
def get_searcher(self) -> Searcher:
searcher = Searcher(self.get_zim())
return searcher
def get_search(self, search_str: str):
query = Query().set_query(search_str)
searcher = self.get_searcher()
search = searcher.search(query)
return search
def get_suggestion_searcher(self) -> SuggestionSearcher:
searcher = SuggestionSearcher(self.get_zim())
return searcher
def estimate(self, search_str: str) -> int:
"""Count the number of Entry(s) that contain the `search_str`."""
search = self.get_search(search_str)
count = search.getEstimatedMatches()
return count
def search(self, search_str: str, offset: int = 0, limit: int = 10) -> List[str]:
"""Return the paths of the Entry(s) that contain the `search_str`."""
search = self.get_search(search_str)
results = list(search.getResults(offset, limit))
return results
def suggest(self, search_str: str, offset: int = 0, limit: int = 10) -> List[str]:
"""Return the paths of the Entry(s) that contain the `search_str`. Also searches the titles."""
searcher = self.get_suggestion_searcher()
suggestions = searcher.suggest(search_str)
results = list(suggestions.getResults(offset, limit))
return results
def get_entry(self, path: str) -> Optional[Entry]:
"""Search this Zim file for the provided Entry at the `path`."""
zim = self.get_zim()
@ -256,22 +223,21 @@ class Zim(Base, ModelHelper):
.order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry) \
.offset(offset).limit(limit)
entries = list()
for tag_zim_entry, zim in tag_zim_entries:
entry = zim.find_entry(tag_zim_entry.zim_entry)
entries.append(entry)
return [(zim, tag_zim_entry.zim_entry) for tag_zim_entry, zim in tag_zim_entries]
return entries
def entries_with_tags(self, session: Session, tag_names: List[str], limit: int = 10, offset: int = 0) -> List[
Entry]:
"""Return Zim Entries tagged with the provided names."""
def entry_paths_with_tags(self, session: Session, tag_names: List[str], limit: int = 10, offset: int = 0) \
-> List[str]:
"""Return the paths of the Zim Entries tagged with the provided names. Does not open the Zim."""
if limit < 1:
raise RuntimeError('Limit must be greater than zero')
tags_sub_select, params = tag_names_to_zim_sub_select(tag_names, zim_id=self.id)
entries = Zim._entries_with_tags_process(limit, offset, params, session, tags_sub_select)
return entries
return [path for _, path in Zim._entries_with_tags_process(limit, offset, params, session, tags_sub_select)]
def entries_with_tags(self, session: Session, tag_names: List[str], limit: int = 10, offset: int = 0) -> List[
Entry]:
"""Return Zim Entries tagged with the provided names."""
return [self.find_entry(i) for i in self.entry_paths_with_tags(session, tag_names, limit, offset)]
class Zims:
@ -290,23 +256,6 @@ class Zims:
logger.error(f'Failed to read Zim: {zim}', exc_info=e)
return zims
@classmethod
def estimate(cls, session: Session, search_str: str) -> OrderedDictType[Zim, int]:
zims = cls.get_all(session)
results = OrderedDict()
for zim in zims:
if zim.auto_search:
results[zim] = zim.estimate(search_str)
return results
@classmethod
def entries_with_tags(cls, session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]:
zims = cls.get_all(session)
results = OrderedDict()
for zim in zims:
results[zim] = len(zim.entries_with_tags(session, tag_names))
return results
class TagZimEntry(Base):
__tablename__ = 'tag_zim'

View file

@ -1,4 +1,7 @@
import asyncio
import json
import sys
from unittest import mock
from http import HTTPStatus
import pytest
@ -366,3 +369,108 @@ async def test_delete_zim_waits_for_a_concurrent_writer(async_client, test_sessi
assert not zim_path.exists()
with maker() as session:
assert session.query(Zim).filter_by(id=zim_id).one_or_none() is None
@pytest.mark.asyncio
async def test_estimate_zims_runs_outside_the_api_process(test_session, zim_factory):
"""libzim holds the GIL, so estimates run in a separate process rather than a thread of the API worker."""
zim = zim_factory('test zim')
test_session.commit()
with mock.patch('modules.zim.lib.asyncio.create_subprocess_exec', wraps=asyncio.create_subprocess_exec) as m:
counts = await lib.estimate_zims(test_session, 'one')
assert counts == {zim: 2}
assert m.call_args.args[:2] == (sys.executable, '-I')
@pytest.mark.asyncio
async def test_estimate_zims_skips_a_zim_it_cannot_read(test_session, zim_factory):
"""A Zim the estimate process cannot open is left out of the estimates, like `Zims.get_all` leaves it out."""
good = zim_factory('good zim')
bad = zim_factory('bad zim')
test_session.commit()
bad.path.write_bytes(b'not a zim file')
counts = await lib.estimate_zims(test_session, 'one')
assert counts == {good: 2}
bad.path.unlink()
counts = await lib.estimate_zims(test_session, 'one')
assert counts == {good: 2}
@pytest.mark.asyncio
async def test_zim_search_runs_outside_the_api_process(async_client, test_session, test_zim):
"""Zim searches, like estimates, do all their libzim work in the worker script's process."""
with mock.patch('modules.zim.lib.asyncio.create_subprocess_exec', wraps=asyncio.create_subprocess_exec) as m:
request, response = await async_client.post(f'/api/zim/search/{test_zim.id}', json=dict(search_str='one'))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['zim']['search']] == ['one', 'home']
assert response.json['zim']['metadata']['title'], 'Metadata should come from the worker'
request, response = await async_client.post('/api/zim/search', json=dict(search_str='one'))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['zims'][0]['search']] == ['one', 'home']
assert m.call_count == 2
assert all(call.args[:2] == (sys.executable, '-I') for call in m.call_args_list)
@pytest.mark.asyncio
async def test_search_all_zims_skips_a_zim_it_cannot_read(async_client, test_session, zim_factory):
"""One unreadable Zim does not fail the search of all Zims."""
good = zim_factory('good zim')
bad = zim_factory('bad zim')
test_session.commit()
bad.path.write_bytes(b'not a zim file')
request, response = await async_client.post('/api/zim/search', json=dict(search_str='one'))
assert response.status_code == HTTPStatus.OK
zims = response.json['zims']
assert len(zims) == 1 and zims[0]['path'].endswith('good zim')
assert zims[0]['estimate'] == 2
@pytest.mark.asyncio
async def test_tagged_search_estimates_do_not_open_zims(async_client, test_session, test_zim, tag_factory):
"""Estimates with tags count from the database; the API process never opens a Zim, even for its metadata."""
tag1, tag2 = await tag_factory(), await tag_factory()
test_zim.tag_entry(test_session, tag1.name, 'one')
test_zim.tag_entry(test_session, tag1.name, 'two')
test_zim.tag_entry(test_session, tag2.name, 'one')
test_session.commit()
with mock.patch('modules.zim.models.Zim.get_zim', side_effect=AssertionError('opened a Zim in the API')):
body = dict(search_str='one', tag_names=[tag1.name])
request, response = await async_client.post('/api/zim/search_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert [(i['estimate'], bool(i['metadata']['title'])) for i in response.json['zims_estimates']] \
== [(2, True)]
body = dict(search_str='', tag_names=[tag1.name, tag2.name])
request, response = await async_client.post('/api/zim/search_estimates', json=body)
assert [i['estimate'] for i in response.json['zims_estimates']] == [1]
@pytest.mark.asyncio
async def test_cancelled_libzim_jobs_kill_the_worker(test_directory):
"""Sanic cancels a handler at its response timeout or on disconnect; the worker process must not outlive it."""
slow_worker = test_directory / 'slow_worker.py'
slow_worker.write_text('import time; time.sleep(60)')
procs = []
create_subprocess_exec = asyncio.create_subprocess_exec
async def spy(*args, **kwargs):
proc = await create_subprocess_exec(*args, **kwargs)
procs.append(proc)
return proc
with mock.patch('modules.zim.lib.LIBZIM_WORKER', slow_worker), \
mock.patch('modules.zim.lib.asyncio.create_subprocess_exec', side_effect=spy):
task = asyncio.create_task(lib.run_libzim_jobs([]))
while not procs:
await asyncio.sleep(0.01)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert procs[0].returncode is not None, 'The worker was killed and reaped'

View file

@ -4,7 +4,7 @@ from http import HTTPStatus
import pytest
from libzim import Entry
from modules.zim import lib
from modules.zim import lib, libzim_worker
from modules.zim.errors import UnknownZim, UnknownZimEntry
from modules.zim.models import TagZimEntry, ZimSubscription
from wrolpi import tags, flags
@ -96,29 +96,29 @@ async def test_zim_entries_with_tags(async_client, test_session, zim_factory, ta
assert entries[0].path == 'two'
def test_get_unique_paths():
def test_unique_paths():
paths = ['A/Prickly_Pear', 'A/Prickly_pear', 'A/Prickly-pear', 'A/Arborescent_Prickly_Pear',
'A/Arborescent_prickly_pear', 'A/Beavertail_prickly_pear', 'A/Brittle_prickly-pear',
'A/Chenille_prickly-pear', 'A/Coastal_Prickly_Pear', 'A/Coastal_prickly_pear',
'A/Prickly_pears_in_South_Africa', 'A/Prickly_Pear_Cays', 'A/Prickly_pears_in_Australia', 'A/Opuntia',
'A/Prickly_Pear_(British_Virgin_Islands)', 'A/Prickly_Pear_Island', 'A/Prickly_Pears_(film)',
'A/Invasive_succulent_plants_in_South_Africa', 'A/Cactoblastis_Memorial_Hall', 'A/Thomas_Harvey_Johnston']
assert lib.get_unique_paths(*paths) == ['A/Prickly_Pear',
'A/Arborescent_Prickly_Pear',
'A/Beavertail_prickly_pear',
'A/Brittle_prickly-pear',
'A/Chenille_prickly-pear',
'A/Coastal_Prickly_Pear',
'A/Prickly_pears_in_South_Africa',
'A/Prickly_Pear_Cays',
'A/Prickly_pears_in_Australia',
'A/Opuntia',
'A/Prickly_Pear_(British_Virgin_Islands)',
'A/Prickly_Pear_Island',
'A/Prickly_Pears_(film)',
'A/Invasive_succulent_plants_in_South_Africa',
'A/Cactoblastis_Memorial_Hall',
'A/Thomas_Harvey_Johnston']
assert libzim_worker.unique_paths(*paths) == ['A/Prickly_Pear',
'A/Arborescent_Prickly_Pear',
'A/Beavertail_prickly_pear',
'A/Brittle_prickly-pear',
'A/Chenille_prickly-pear',
'A/Coastal_Prickly_Pear',
'A/Prickly_pears_in_South_Africa',
'A/Prickly_Pear_Cays',
'A/Prickly_pears_in_Australia',
'A/Opuntia',
'A/Prickly_Pear_(British_Virgin_Islands)',
'A/Prickly_Pear_Island',
'A/Prickly_Pears_(film)',
'A/Invasive_succulent_plants_in_South_Africa',
'A/Cactoblastis_Memorial_Hall',
'A/Thomas_Harvey_Johnston']
@pytest.mark.asyncio