mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
The Zim list reads each Zim's metadata in the libzim worker instead of opening it in the API (#939)
GET /api/zim went through Zims.get_all, which opened every Zim's Archive in the API process, and then Zim.__json__ opened each one again for its metadata. python-libzim holds the GIL and a cold open of a large Zim takes seconds, so loading the Zims page stalled that API worker for every other request (27.8 s cold against 0.75 s warm for 44 Zims). #928 moved estimates and searches to the worker process; the list now uses the same worker for metadata, and a Zim the worker cannot read is left out as before. count_tagged_zim_entries shares the new readable_zims helper. Zims.get_all had no other callers and is removed. Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
b8122a4637
commit
c78378d4f5
4 changed files with 57 additions and 38 deletions
|
|
@ -24,7 +24,7 @@ logger = logger.getChild(__name__)
|
|||
)
|
||||
async def get_zims(request: Request):
|
||||
session = request.ctx.session
|
||||
zims = lib.get_zims(session)
|
||||
zims = await lib.readable_zims(session)
|
||||
resp = {
|
||||
'zims': [i.__json__() for i in zims],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from sqlalchemy.orm.exc import NoResultFound # noqa
|
|||
from modules.zim import kiwix
|
||||
from modules.zim.errors import UnknownZim, UnknownZimSubscription
|
||||
from modules.zim.kiwix import KIWIX_CATALOG
|
||||
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription, ZimMetadata
|
||||
from modules.zim.models import Zim, TagZimEntry, ZimSubscription, ZimMetadata
|
||||
from wrolpi import flags, tags
|
||||
from wrolpi.cmd import run_command
|
||||
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
|
||||
|
|
@ -292,8 +292,8 @@ def auto_search_zims(session: Session) -> List[Zim]:
|
|||
async def estimate_zims(session: Session, search_str: str) -> OrderedDictType[Zim, int]:
|
||||
"""Estimate how many entries of each auto-search Zim match `search_str`.
|
||||
|
||||
`Zims.get_all` opens every Zim and is not used here for that reason; a Zim the worker cannot read is
|
||||
reported and left out, as `Zims.get_all` would."""
|
||||
Only the auto-search Zims are estimated, so `readable_zims` is not used; a Zim the worker cannot read is
|
||||
reported and left out all the same."""
|
||||
zims = auto_search_zims(session)
|
||||
jobs = [dict(op='estimate', path=str(i.path), search_str=search_str, metadata=True) for i in zims]
|
||||
results = await run_libzim_jobs(jobs)
|
||||
|
|
@ -305,21 +305,30 @@ async def estimate_zims(session: Session, search_str: str) -> OrderedDictType[Zi
|
|||
return counts
|
||||
|
||||
|
||||
async def count_tagged_zim_entries(session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]:
|
||||
"""Count the entries of every Zim tagged with all of `tag_names`. The counts come from the database; the
|
||||
worker only reads each Zim's metadata, and a Zim it cannot read is left out."""
|
||||
async def readable_zims(session: Session) -> List[Zim]:
|
||||
"""Every Zim whose file the libzim worker can read, in path order, with its metadata filled so that
|
||||
`Zim.__json__` does not open it. The API process never opens a Zim here: a cold open takes seconds and libzim
|
||||
holds the GIL, so the Zims page used to stall its API worker for every other request."""
|
||||
zims = session.query(Zim).options(*Zim.json_options()).order_by(Zim.path).all()
|
||||
zims = [i for i in zims if i.path and i.path.is_file()]
|
||||
counts = {i: len(i.entry_paths_with_tags(session, tag_names, limit=1_000_000)) for i in zims}
|
||||
if not zims:
|
||||
return zims
|
||||
results = await run_libzim_jobs([dict(op='metadata', path=str(i.path)) for i in zims])
|
||||
readable = OrderedDict()
|
||||
readable = list()
|
||||
for zim, result in zip(zims, results):
|
||||
if 'error' not in result:
|
||||
_apply_metadata(zim, result)
|
||||
readable[zim] = counts[zim]
|
||||
readable.append(zim)
|
||||
return readable
|
||||
|
||||
|
||||
async def count_tagged_zim_entries(session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]:
|
||||
"""Count the entries of every Zim tagged with all of `tag_names`. The counts come from the database; the
|
||||
worker only reads each Zim's metadata, and a Zim it cannot read is left out."""
|
||||
zims = await readable_zims(session)
|
||||
return OrderedDict((i, len(i.entry_paths_with_tags(session, tag_names, limit=1_000_000))) for i in zims)
|
||||
|
||||
|
||||
async def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0,
|
||||
limit: int = 10) -> List[Dict]:
|
||||
"""Search all Zim files that have auto_search enabled, in one libzim worker process.
|
||||
|
|
@ -370,17 +379,8 @@ def get_entry(path: str, zim_id: int) -> Entry:
|
|||
return entry
|
||||
|
||||
|
||||
def get_zims(session: Session, ids: List[int] = None) -> List[Zim]:
|
||||
if ids:
|
||||
zims = session.query(Zim).filter(Zim.id.in_(ids)).all()
|
||||
else:
|
||||
zims = Zims.get_all(session)
|
||||
|
||||
return zims
|
||||
|
||||
|
||||
def delete_zims(session: Session, ids: List[int]):
|
||||
zims = get_zims(session, ids)
|
||||
zims = session.query(Zim).filter(Zim.id.in_(ids)).all()
|
||||
for zim in zims:
|
||||
zim.delete()
|
||||
|
||||
|
|
|
|||
|
|
@ -240,23 +240,6 @@ class Zim(Base, ModelHelper):
|
|||
return [self.find_entry(i) for i in self.entry_paths_with_tags(session, tag_names, limit, offset)]
|
||||
|
||||
|
||||
class Zims:
|
||||
"""Convenience class which searches all known Zim files."""
|
||||
|
||||
@staticmethod
|
||||
def get_all(session: Session) -> List[Zim]:
|
||||
"""Returns all Zim records."""
|
||||
zims = list()
|
||||
for zim in session.query(Zim).options(*Zim.json_options()).order_by(Zim.path):
|
||||
try:
|
||||
# It may not be possible to read the Zim file (drive not mounted), report and ignore.
|
||||
zim.get_zim()
|
||||
zims.append(zim)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to read Zim: {zim}', exc_info=e)
|
||||
return zims
|
||||
|
||||
|
||||
class TagZimEntry(Base):
|
||||
__tablename__ = 'tag_zim'
|
||||
__table_args__ = (
|
||||
|
|
|
|||
|
|
@ -385,7 +385,7 @@ async def test_estimate_zims_runs_outside_the_api_process(test_session, zim_fact
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_estimate_zims_skips_a_zim_it_cannot_read(test_session, zim_factory):
|
||||
"""A Zim the estimate process cannot open is left out of the estimates, like `Zims.get_all` leaves it out."""
|
||||
"""A Zim the estimate process cannot open is left out of the estimates, as the Zim list leaves it out."""
|
||||
good = zim_factory('good zim')
|
||||
bad = zim_factory('bad zim')
|
||||
test_session.commit()
|
||||
|
|
@ -451,6 +451,42 @@ async def test_tagged_search_estimates_do_not_open_zims(async_client, test_sessi
|
|||
assert [i['estimate'] for i in response.json['zims_estimates']] == [1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_list_does_not_open_zims(async_client, test_session, zim_factory):
|
||||
"""Listing Zims reads their metadata in the worker; the API process never opens a Zim. A cold open takes
|
||||
seconds per Zim and libzim holds the GIL, so the Zims page used to stall its API worker."""
|
||||
zim_factory('one zim')
|
||||
zim_factory('two zim')
|
||||
test_session.commit()
|
||||
|
||||
with mock.patch('modules.zim.models.Zim.get_zim', side_effect=AssertionError('opened a Zim in the API')):
|
||||
request, response = await async_client.get('/api/zim')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
zims = response.json['zims']
|
||||
assert [i['path'].endswith(('one zim', 'two zim')) for i in zims] == [True, True]
|
||||
assert all(i['metadata']['title'] for i in zims), 'Metadata should come from the worker'
|
||||
assert all(i['size'] for i in zims)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_list_skips_a_zim_it_cannot_read(async_client, test_session, zim_factory):
|
||||
"""A Zim the worker cannot read, or whose file is gone, is left out of the list; the rest are listed."""
|
||||
good = zim_factory('good zim')
|
||||
bad = zim_factory('bad zim')
|
||||
test_session.commit()
|
||||
bad.path.write_bytes(b'not a zim file')
|
||||
|
||||
request, response = await async_client.get('/api/zim')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert [i['path'].endswith('good zim') for i in response.json['zims']] == [True]
|
||||
|
||||
bad.path.unlink()
|
||||
request, response = await async_client.get('/api/zim')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert [i['path'].endswith('good zim') for i in response.json['zims']] == [True]
|
||||
assert good.path.is_file()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_libzim_jobs_kill_the_worker(test_directory):
|
||||
"""Sanic cancels a handler at its response timeout or on disconnect; the worker process must not outlive it."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue