From c78378d4f54e6111405c3539340d598d89b682c8 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:19:22 -0700 Subject: [PATCH] The Zim list reads each Zim's metadata in the libzim worker instead of opening it in the API (#939) GET /api/zim went through Zims.get_all, which opened every Zim's Archive in the API process, and then Zim.__json__ opened each one again for its metadata. python-libzim holds the GIL and a cold open of a large Zim takes seconds, so loading the Zims page stalled that API worker for every other request (27.8 s cold against 0.75 s warm for 44 Zims). #928 moved estimates and searches to the worker process; the list now uses the same worker for metadata, and a Zim the worker cannot read is left out as before. count_tagged_zim_entries shares the new readable_zims helper. Zims.get_all had no other callers and is removed. Co-authored-by: Claude Fable 5.1 --- modules/zim/api.py | 2 +- modules/zim/lib.py | 38 ++++++++++++++++++------------------ modules/zim/models.py | 17 ---------------- modules/zim/test/test_api.py | 38 +++++++++++++++++++++++++++++++++++- 4 files changed, 57 insertions(+), 38 deletions(-) diff --git a/modules/zim/api.py b/modules/zim/api.py index 891c1bfc..7debc19c 100644 --- a/modules/zim/api.py +++ b/modules/zim/api.py @@ -24,7 +24,7 @@ logger = logger.getChild(__name__) ) async def get_zims(request: Request): session = request.ctx.session - zims = lib.get_zims(session) + zims = await lib.readable_zims(session) resp = { 'zims': [i.__json__() for i in zims], } diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 2d42edd2..af054794 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -17,7 +17,7 @@ from sqlalchemy.orm.exc import NoResultFound # noqa from modules.zim import kiwix from modules.zim.errors import UnknownZim, UnknownZimSubscription from modules.zim.kiwix import KIWIX_CATALOG -from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription, ZimMetadata +from modules.zim.models import Zim, TagZimEntry, ZimSubscription, ZimMetadata from wrolpi import flags, tags from wrolpi.cmd import run_command from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \ @@ -292,8 +292,8 @@ def auto_search_zims(session: Session) -> List[Zim]: async def estimate_zims(session: Session, search_str: str) -> OrderedDictType[Zim, int]: """Estimate how many entries of each auto-search Zim match `search_str`. - `Zims.get_all` opens every Zim and is not used here for that reason; a Zim the worker cannot read is - reported and left out, as `Zims.get_all` would.""" + Only the auto-search Zims are estimated, so `readable_zims` is not used; a Zim the worker cannot read is + reported and left out all the same.""" zims = auto_search_zims(session) jobs = [dict(op='estimate', path=str(i.path), search_str=search_str, metadata=True) for i in zims] results = await run_libzim_jobs(jobs) @@ -305,21 +305,30 @@ async def estimate_zims(session: Session, search_str: str) -> OrderedDictType[Zi return counts -async def count_tagged_zim_entries(session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]: - """Count the entries of every Zim tagged with all of `tag_names`. The counts come from the database; the - worker only reads each Zim's metadata, and a Zim it cannot read is left out.""" +async def readable_zims(session: Session) -> List[Zim]: + """Every Zim whose file the libzim worker can read, in path order, with its metadata filled so that + `Zim.__json__` does not open it. The API process never opens a Zim here: a cold open takes seconds and libzim + holds the GIL, so the Zims page used to stall its API worker for every other request.""" zims = session.query(Zim).options(*Zim.json_options()).order_by(Zim.path).all() zims = [i for i in zims if i.path and i.path.is_file()] - counts = {i: len(i.entry_paths_with_tags(session, tag_names, limit=1_000_000)) for i in zims} + if not zims: + return zims results = await run_libzim_jobs([dict(op='metadata', path=str(i.path)) for i in zims]) - readable = OrderedDict() + readable = list() for zim, result in zip(zims, results): if 'error' not in result: _apply_metadata(zim, result) - readable[zim] = counts[zim] + readable.append(zim) return readable +async def count_tagged_zim_entries(session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]: + """Count the entries of every Zim tagged with all of `tag_names`. The counts come from the database; the + worker only reads each Zim's metadata, and a Zim it cannot read is left out.""" + zims = await readable_zims(session) + return OrderedDict((i, len(i.entry_paths_with_tags(session, tag_names, limit=1_000_000))) for i in zims) + + async def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0, limit: int = 10) -> List[Dict]: """Search all Zim files that have auto_search enabled, in one libzim worker process. @@ -370,17 +379,8 @@ def get_entry(path: str, zim_id: int) -> Entry: return entry -def get_zims(session: Session, ids: List[int] = None) -> List[Zim]: - if ids: - zims = session.query(Zim).filter(Zim.id.in_(ids)).all() - else: - zims = Zims.get_all(session) - - return zims - - def delete_zims(session: Session, ids: List[int]): - zims = get_zims(session, ids) + zims = session.query(Zim).filter(Zim.id.in_(ids)).all() for zim in zims: zim.delete() diff --git a/modules/zim/models.py b/modules/zim/models.py index 5aa98ebb..cbca8e9d 100644 --- a/modules/zim/models.py +++ b/modules/zim/models.py @@ -240,23 +240,6 @@ class Zim(Base, ModelHelper): return [self.find_entry(i) for i in self.entry_paths_with_tags(session, tag_names, limit, offset)] -class Zims: - """Convenience class which searches all known Zim files.""" - - @staticmethod - def get_all(session: Session) -> List[Zim]: - """Returns all Zim records.""" - zims = list() - for zim in session.query(Zim).options(*Zim.json_options()).order_by(Zim.path): - try: - # It may not be possible to read the Zim file (drive not mounted), report and ignore. - zim.get_zim() - zims.append(zim) - except Exception as e: - logger.error(f'Failed to read Zim: {zim}', exc_info=e) - return zims - - class TagZimEntry(Base): __tablename__ = 'tag_zim' __table_args__ = ( diff --git a/modules/zim/test/test_api.py b/modules/zim/test/test_api.py index 06a4b6ae..75d93bd0 100644 --- a/modules/zim/test/test_api.py +++ b/modules/zim/test/test_api.py @@ -385,7 +385,7 @@ async def test_estimate_zims_runs_outside_the_api_process(test_session, zim_fact @pytest.mark.asyncio async def test_estimate_zims_skips_a_zim_it_cannot_read(test_session, zim_factory): - """A Zim the estimate process cannot open is left out of the estimates, like `Zims.get_all` leaves it out.""" + """A Zim the estimate process cannot open is left out of the estimates, as the Zim list leaves it out.""" good = zim_factory('good zim') bad = zim_factory('bad zim') test_session.commit() @@ -451,6 +451,42 @@ async def test_tagged_search_estimates_do_not_open_zims(async_client, test_sessi assert [i['estimate'] for i in response.json['zims_estimates']] == [1] +@pytest.mark.asyncio +async def test_zim_list_does_not_open_zims(async_client, test_session, zim_factory): + """Listing Zims reads their metadata in the worker; the API process never opens a Zim. A cold open takes + seconds per Zim and libzim holds the GIL, so the Zims page used to stall its API worker.""" + zim_factory('one zim') + zim_factory('two zim') + test_session.commit() + + with mock.patch('modules.zim.models.Zim.get_zim', side_effect=AssertionError('opened a Zim in the API')): + request, response = await async_client.get('/api/zim') + assert response.status_code == HTTPStatus.OK + zims = response.json['zims'] + assert [i['path'].endswith(('one zim', 'two zim')) for i in zims] == [True, True] + assert all(i['metadata']['title'] for i in zims), 'Metadata should come from the worker' + assert all(i['size'] for i in zims) + + +@pytest.mark.asyncio +async def test_zim_list_skips_a_zim_it_cannot_read(async_client, test_session, zim_factory): + """A Zim the worker cannot read, or whose file is gone, is left out of the list; the rest are listed.""" + good = zim_factory('good zim') + bad = zim_factory('bad zim') + test_session.commit() + bad.path.write_bytes(b'not a zim file') + + request, response = await async_client.get('/api/zim') + assert response.status_code == HTTPStatus.OK + assert [i['path'].endswith('good zim') for i in response.json['zims']] == [True] + + bad.path.unlink() + request, response = await async_client.get('/api/zim') + assert response.status_code == HTTPStatus.OK + assert [i['path'].endswith('good zim') for i in response.json['zims']] == [True] + assert good.path.is_file() + + @pytest.mark.asyncio async def test_cancelled_libzim_jobs_kill_the_worker(test_directory): """Sanic cancels a handler at its response timeout or on disconnect; the worker process must not outlive it."""