wrolpi/modules/zim/lib.py
lrnselfreliance 5be899bf1f
Saving zims.yaml keeps the entries of Zims the database does not have yet (#935)
A rebuilt database imports zims.yaml before the refresh has modeled a single
Zim.  That import has no database version to compare with, so it saves the
config afterward, and the save wrote only the Zims in the database: none.  The
file's choices were erased and every Zim was then modeled with the default.

The dump now starts from the file's entries and overlays the database's, so a
choice survives until its Zim is modeled again, and a Zim in the database still
decides its own line.

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-02 11:11:41 -07:00

888 lines
37 KiB
Python

import asyncio
import functools
import json
import pathlib
import re
import sys
from dataclasses import dataclass, field
from datetime import datetime
from collections import OrderedDict
from typing import Callable, List, Tuple, Dict, OrderedDict as OrderedDictType
from libzim import Entry
from sqlalchemy import or_
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound # noqa
from modules.zim import kiwix
from modules.zim.errors import UnknownZim, UnknownZimSubscription
from modules.zim.kiwix import KIWIX_CATALOG
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription, ZimMetadata
from wrolpi import flags, tags
from wrolpi.cmd import run_command
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
register_refresh_cleanup, background_task, get_wrolpi_config, aiohttp_post, ConfigFile, \
get_relative_to_media_directory
from wrolpi.config_version import register_ownership, IMPORT, SAVE, set_config_version
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.dates import Seconds
from wrolpi.events import Events
from wrolpi.db import get_db_session, session_curs
from wrolpi.downloader import DownloadFrequency
from wrolpi.files.lib import split_file_name_words
from wrolpi.files.modeler import run_modeler_loop, skip_clause
from wrolpi.files.worker import file_worker
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST, DOCKERIZED
logger = logger.getChild(__name__)
CONTROLLER_SERVICE = 'http://controller'
__all__ = [
'flag_outdated_zim_files',
'get_all_entries_tags',
'get_entries_tags',
'get_entry',
'get_kiwix_catalog',
'get_zim',
'get_zims',
'get_zims_config',
'import_zims_config',
'model_zim',
'save_zims_config',
'zims_config',
'restart_kiwix_handler',
'zim_download_url_to_name',
'zim_modeler',
]
@register_switch_handler('restart_kiwix')
async def restart_kiwix_handler():
"""Switch handler that restarts the Kiwix service after zim files are added."""
await restart_kiwix()
def model_zim(file_group: FileGroup, session: Session) -> Zim:
"""Create a Zim record for the provided FileGroup."""
# Check if a Zim already exists for this FileGroup
zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none()
if not zim:
# Create new Zim if it doesn't exist. Its settings come from zims.yaml: a rebuilt database, or a newer
# version of a Zim, keeps what the user chose.
zim = Zim(file_group=file_group, path=file_group.primary_path,
auto_search=zims_config.auto_search_of(file_group.primary_path))
session.add(zim)
# Trigger Kiwix restart so the new zim file is available for browsing.
restart_kiwix_handler.activate_switch()
file_group.title = file_group.primary_path.name
file_group.a_text = split_file_name_words(file_group.primary_path.name)
file_group.indexed = True
file_group.model = 'zim'
return zim
ZIM_PROCESSING_LIMIT = 10
def _select_zim_ids(session, skip_ids: set) -> list[int]:
query = session.query(FileGroup.id) \
.outerjoin(Zim, Zim.file_group_id == FileGroup.id) \
.filter(FileGroup.primary_path.ilike('%.zim')) \
.filter(
# Exclusive-mimetype modeler: claim unmodeled rows even if apply_indexers
# already set indexed=True, and re-model when files changed.
or_(Zim.id.is_(None), FileGroup.indexed != True),
)
query = skip_clause(query, FileGroup.id, skip_ids)
return [row[0] for row in query.limit(ZIM_PROCESSING_LIMIT).all()]
def _apply_zim(session: Session, file_group: FileGroup, prepared):
zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none()
if not zim:
zim = model_zim(file_group, session)
zim.flush()
else:
file_group.title = file_group.primary_path.name
file_group.a_text = split_file_name_words(file_group.primary_path.name)
file_group.model = 'zim'
@register_modeler
async def zim_modeler(progress_callback: Callable[[int], None] = None):
await run_modeler_loop(
name='zim_modeler',
batch_size=ZIM_PROCESSING_LIMIT,
select_ids=_select_zim_ids,
apply=_apply_zim,
txn='batch',
progress_callback=progress_callback,
)
def get_all_entries_tags(session: Session):
"""Returns a dict of dicts which contain the names of all Tags of each Entry's path.
Example:
{
1: {
'path1': ['tag1', 'tag2'],
},
2: {
'path2': ['tag1'],
},
"""
stmt = '''
SELECT tz.zim_id, tz.zim_entry, json_group_array(t.name)
FROM tag t
LEFT JOIN tag_zim tz on t.id = tz.tag_id
GROUP BY 1, 2 \
'''
with session_curs(session) as curs:
curs.execute(stmt)
entries = dict()
for zim_id, zim_entry, tag_names in curs.fetchall():
if not zim_id:
continue
tag_names = json.loads(tag_names)
try:
entries[zim_id][zim_entry] = tag_names
except KeyError:
entries[zim_id] = {zim_entry: tag_names}
return entries
def get_entries_tags(session: Session, paths: List[str], zim_id: int):
all_entries_tags = get_all_entries_tags(session)
entries = dict()
for path in paths:
zim_tag_entries = all_entries_tags.get(zim_id, dict())
entries[path] = zim_tag_entries.get(path, list())
return entries
LIBZIM_WORKER = pathlib.Path(__file__).with_name('libzim_worker.py')
async def run_libzim_jobs(jobs: List[Dict]) -> List[Dict]:
"""Run libzim jobs (see `libzim_worker.py`) in a separate process, return one result per job.
python-libzim never releases the GIL, and opening a large Zim cold takes seconds, so nothing libzim does may
run in the API worker: every other request on that worker would wait behind it. (A plain subprocess:
Sanic workers are daemonic, so they cannot fork multiprocessing children.) A failed job's result is
`{'error': ...}`, so one bad Zim does not fail the rest."""
request = json.dumps(dict(jobs=jobs)).encode()
proc = await asyncio.create_subprocess_exec(
sys.executable, '-I', str(LIBZIM_WORKER),
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
)
try:
stdout, stderr = await proc.communicate(request)
except asyncio.CancelledError:
# Sanic cancels the handler at its response timeout or when the client disconnects; cancelling
# `communicate` leaves the process running (and, once its stdout pipe fills, stuck). Kill and reap it.
try:
proc.kill()
except ProcessLookupError:
pass
await proc.wait()
raise
if proc.returncode != 0:
raise RuntimeError(f'libzim worker failed ({proc.returncode}): {stderr.decode(errors="replace").strip()}')
if stderr:
logger.error(f'libzim worker reported errors:\n{stderr.decode(errors="replace").strip()}')
return json.loads(stdout)
def _apply_metadata(zim: Zim, result: Dict):
"""Fill the Zim's cached `zim_metadata` from a worker result so `Zim.__json__` does not open the Zim."""
if metadata := result.get('metadata'):
zim.__dict__['zim_metadata'] = ZimMetadata(**metadata)
def _search_job(zim: Zim, search_str: str, tag_names: List[str], session: Session, offset: int, limit: int) -> Dict:
"""The worker job that fetches what `_headline_results` needs for one Zim."""
if tag_names:
# TODO what is the count goes over the limit?
paths = zim.entry_paths_with_tags(session, tag_names, offset=offset, limit=1_000)
return dict(op='entries', path=str(zim.path), paths=paths, metadata=True)
elif search_str:
return dict(op='search', path=str(zim.path), search_str=search_str, offset=offset, limit=limit, metadata=True)
raise RuntimeError('Must provide search_str or tag_names.')
def _headline_results(session: Session, zim: Zim, search_str: str, tag_names: List[str], result: Dict) -> Dict:
"""Headline the entries a search job returned for one Zim."""
if 'error' in result:
raise RuntimeError(f'Failed to search {zim}: {result["error"]}')
_apply_metadata(zim, result)
entries = result['entries']
estimate = result.get('estimate', 0)
articles = [extract_html_text(i['content']) for i in entries]
entries_tag_names = get_entries_tags(session, [i['path'] for i in entries], zim.id)
# Always get headlines even if there is no search_str. Non-matches get the leading text of each article.
headlines = extract_headlines(articles, search_str)
search = list()
if search_str:
# `search_str` allows us to headline the title.
titles = [i['title'] for i in entries]
headline_titles = extract_headlines(titles, search_str)
for entry in entries:
title_headline, title_rank = headline_titles.pop(0)
headline, rank = headlines.pop(0)
if rank > 0 or title_rank > 0:
# The result for each Entry.
path = entry['path']
entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list()
search.append(dict(
headline=headline,
path=path,
rank=max(rank, title_rank),
tag_names=entry_tag_names,
title=title_headline,
))
if tag_names:
estimate += 1
else:
# No `search_str`, we return all entries.
for entry in entries:
headline, _ = headlines.pop(0)
path = entry['path']
entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list()
search.append(dict(
headline=headline,
path=path,
rank=0,
tag_names=entry_tag_names,
title=entry['title'],
))
estimate = len(entries)
results = dict(
path=zim.path,
metadata=zim.zim_metadata,
search=search,
estimate=estimate,
)
return results
async def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List[str] = None, offset: int = 0,
limit: int = 10) -> Dict:
zim = get_zim(session, int(zim_id))
job = _search_job(zim, search_str, tag_names, session, offset, limit)
result, = await run_libzim_jobs([job])
return await asyncio.to_thread(_headline_results, session, zim, search_str, tag_names, result)
def auto_search_zims(session: Session) -> List[Zim]:
"""The Zims that the global search searches, without opening them."""
zims = session.query(Zim).options(*Zim.json_options()).filter_by(auto_search=True).order_by(Zim.path).all()
return [i for i in zims if i.path and i.path.is_file()]
async def estimate_zims(session: Session, search_str: str) -> OrderedDictType[Zim, int]:
"""Estimate how many entries of each auto-search Zim match `search_str`.
`Zims.get_all` opens every Zim and is not used here for that reason; a Zim the worker cannot read is
reported and left out, as `Zims.get_all` would."""
zims = auto_search_zims(session)
jobs = [dict(op='estimate', path=str(i.path), search_str=search_str, metadata=True) for i in zims]
results = await run_libzim_jobs(jobs)
counts = OrderedDict()
for zim, result in zip(zims, results):
if 'error' not in result:
_apply_metadata(zim, result)
counts[zim] = result['estimate']
return counts
async def count_tagged_zim_entries(session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]:
"""Count the entries of every Zim tagged with all of `tag_names`. The counts come from the database; the
worker only reads each Zim's metadata, and a Zim it cannot read is left out."""
zims = session.query(Zim).options(*Zim.json_options()).order_by(Zim.path).all()
zims = [i for i in zims if i.path and i.path.is_file()]
counts = {i: len(i.entry_paths_with_tags(session, tag_names, limit=1_000_000)) for i in zims}
results = await run_libzim_jobs([dict(op='metadata', path=str(i.path)) for i in zims])
readable = OrderedDict()
for zim, result in zip(zims, results):
if 'error' not in result:
_apply_metadata(zim, result)
readable[zim] = counts[zim]
return readable
async def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0,
limit: int = 10) -> List[Dict]:
"""Search all Zim files that have auto_search enabled, in one libzim worker process.
Returns a list of per-zim result dicts, each containing metadata and search results.
Each search entry includes `zim_id` so callers can fetch the full entry afterward."""
zims = auto_search_zims(session)
jobs = [_search_job(zim, search_str, tag_names, session, offset, limit) for zim in zims]
results = await run_libzim_jobs(jobs)
def headline_all():
all_results = []
for zim, result in zip(zims, results):
try:
headlined = _headline_results(session, zim, search_str, tag_names, result)
except Exception as e:
logger.error(f'Failed to search Zim {zim}', exc_info=e)
continue
if headlined.get('search'):
for entry in headlined['search']:
entry['zim_id'] = zim.id
all_results.append(headlined)
return all_results
return await asyncio.to_thread(headline_all)
def get_zim(session: Session, zim_id: int) -> Zim:
"""Return the Zim record of the provided path.
@raise UnknownZim: When no record exists with the id.
@warning: Results are cached."""
try:
zim: Zim = session.query(Zim) \
.join(FileGroup, FileGroup.id == Zim.file_group_id) \
.filter(Zim.id == zim_id) \
.one()
return zim
except Exception as e:
raise UnknownZim() from e
@functools.lru_cache(maxsize=1_000)
def get_entry(path: str, zim_id: int) -> Entry:
with get_db_session() as session:
zim = get_zim(session, zim_id)
entry = zim.find_entry(path)
return entry
def get_zims(session: Session, ids: List[int] = None) -> List[Zim]:
if ids:
zims = session.query(Zim).filter(Zim.id.in_(ids)).all()
else:
zims = Zims.get_all(session)
return zims
def delete_zims(session: Session, ids: List[int]):
zims = get_zims(session, ids)
for zim in zims:
zim.delete()
async def add_tag(session: Session, tag_name: str, zim_id: int, zim_entry: str) -> TagZimEntry:
zim = session.query(Zim).filter_by(id=zim_id).one()
tag_zim_entry = zim.tag_entry(session, tag_name, zim_entry)
session.commit()
return tag_zim_entry
async def untag(session: Session, tag_name: str, zim_id: int, zim_entry: str):
zim: Zim = session.query(Zim).filter_by(id=zim_id).one()
zim.untag_entry(session, tag_name, zim_entry)
session.commit()
def get_kiwix_catalog():
return KIWIX_CATALOG.copy()
def get_kiwix_subscriptions(session: Session) -> Dict[str, ZimSubscription]:
subscriptions = session.query(ZimSubscription).options(*ZimSubscription.json_options())
subscriptions_by_name = {i.name: i for i in subscriptions}
return subscriptions_by_name
async def subscribe(session: Session, name: str, language: str,
frequency: int = DownloadFrequency.days180) -> ZimSubscription:
subscription = ZimSubscription.get_or_create(session, name)
subscription.language = language
if name == 'Wikipedia (with images)':
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_maxi_'
elif name == 'Wikipedia (mini)':
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_mini_'
elif name == 'Wikipedia (no images)':
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_nopic_'
elif name == 'Wikiversity (with images)':
url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_maxi_'
elif name == 'Wikiversity (no images)':
url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_nopic_'
elif name == 'Wikitionary (no images)':
url = f'https://download.kiwix.org/zim/wiktionary/wiktionary_{language}_all_nopic_'
elif name == 'Wikitionary (with images)':
url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_maxi_'
elif name == 'Wikibooks (no images)':
url = f'https://download.kiwix.org/zim/wikibooks/wikibooks_{language}_all_nopic_'
elif name == 'Wikibooks (with images)':
url = f'https://download.kiwix.org/zim/wikibooks/wikibooks_{language}_all_maxi_'
elif name == 'Wikisource':
url = f'https://download.kiwix.org/zim/wikisource/wikisource_{language}_all_maxi_'
elif name == 'Raspberry Pi (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/raspberrypi.stackexchange.com_{language}_all_'
elif name == 'Vikidia':
url = f'https://download.kiwix.org/zim/vikidia/vikidia_{language}_all_maxi_'
elif name == 'Stackoverflow (Stack Exchange)':
if language == 'en':
url = f'https://download.kiwix.org/zim/stack_exchange/stackoverflow.com_{language}_all_'
else:
url = f'https://download.kiwix.org/zim/stack_exchange/{language}.stackoverflow.com_{language}_all_'
elif name == 'Superuser (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/superuser.com_{language}_all_'
elif name == 'iFixit':
url = f'https://download.kiwix.org/zim/ifixit/ifixit_{language}_all_'
elif name == 'Gutenberg':
url = f'https://download.kiwix.org/zim/gutenberg/gutenberg_{language}_all_'
elif name == 'Amateur Radio (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/ham.stackexchange.com_{language}_all_'
elif name == 'DIY (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/diy.stackexchange.com_{language}_all_'
elif name == 'Electronics (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/electronics.stackexchange.com_{language}_all_'
elif name == 'Unix (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/unix.stackexchange.com_{language}_all_'
elif name == 'Ask Ubuntu (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/askubuntu.com_{language}_all_'
elif name == 'Bicycles (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/bicycles.stackexchange.com_{language}_all_'
elif name == 'Biology (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/biology.stackexchange.com_{language}_all_'
elif name == 'Arduino (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/arduino.stackexchange.com_{language}_all_'
elif name == '3D Printing (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/3dprinting.stackexchange.com_{language}_all_'
elif name == 'MD Wiki (Wiki Project Med Foundation)':
url = f'https://download.kiwix.org/zim/other/mdwiki_{language}_all_'
elif name == 'WikiMed Medical Encyclopedia':
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_medicine_'
elif name == 'Post Disaster Resource Library':
url = f'https://download.kiwix.org/zim/other/zimgit-post-disaster_{language}_'
elif name == 'Military Medicine':
url = f'https://download.kiwix.org/zim/zimit/fas-military-medicine_{language}_'
elif name == 'WikEm (Global Emergency Medicine Wiki)':
url = f'https://download.kiwix.org/zim/other/wikem_{language}_all_maxi_'
elif name == 'Health (Stack Exchange)':
url = f'https://download.kiwix.org/zim/stack_exchange/health.stackexchange.com_{language}_all_'
elif name == 'Khan Academy':
url = f'https://download.kiwix.org/zim/other/khanacademy_{language}_all_'
else:
raise ValueError(f'{name} not a valid Kiwix subscription name!')
if language not in kiwix.KIWIX_CATALOG_BY_NAME[name]['languages']:
raise ValueError(f'Language {repr(str(language))} is invalid for {name}')
subscription.change_download(session, url, frequency)
session.add(subscription)
session.flush([subscription])
session.commit()
return subscription
async def unsubscribe(session: Session, subscription_id: int):
subscription: ZimSubscription = session.query(ZimSubscription).filter_by(id=subscription_id).one_or_none()
if subscription:
session.delete(subscription)
if subscription.download:
session.delete(subscription.download)
session.commit()
else:
raise UnknownZimSubscription(f'{subscription_id=}')
KIWIX_URL_PARSER = re.compile(r'https:\/\/download\.kiwix\.org\/zim\/(.+?)\/(.+?)[_-](\w{2,3})([_-].+)')
KIWIX_URL_NO_FLAVOR_PARSER = re.compile(r'https:\/\/download\.kiwix\.org\/zim\/(.+?)\/(.+)[_-]([a-zA-Z]{2,3})')
def zim_download_url_to_name(url: str) -> Tuple[str, str]:
try:
_, project, language, flavor = KIWIX_URL_PARSER.match(url).groups()
except AttributeError:
# May not have a "flavor".
_, project, language = KIWIX_URL_NO_FLAVOR_PARSER.match(url).groups()
flavor = None
if project == 'wikipedia' and flavor == '_all_maxi_':
name = 'Wikipedia (with images)'
elif project == 'wikipedia' and flavor == '_all_nopic_':
name = 'Wikipedia (no images)'
elif project == 'wikipedia' and flavor == '_all_mini_':
name = 'Wikipedia (mini)'
elif project == 'wikiversity' and flavor == '_all_maxi_':
name = 'Wikiversity (with images)'
elif project == 'wikiversity' and flavor == '_all_nopic_':
name = 'Wikiversity (no images)'
elif project == 'wiktionary' and flavor == '_all_maxi_':
name = 'Wikitionary (with images)'
elif project == 'wiktionary' and flavor == '_all_nopic_':
name = 'Wikitionary (no images)'
elif project == 'wikibooks' and flavor == '_all_maxi_':
name = 'Wikibooks (with images)'
elif project == 'wikibooks' and flavor == '_all_nopic_':
name = 'Wikibooks (no images)'
elif project == 'wikisource' and flavor == '_all_maxi_':
name = 'Wikisource'
elif project == 'raspberrypi.stackexchange.com' and flavor == '_all_':
name = 'Raspberry Pi (Stack Exchange)'
elif project == 'vikidia' and flavor == '_all_maxi_':
name = 'Vikidia'
elif 'stackoverflow.com' in project and flavor == '_all_':
name = 'Stackoverflow (Stack Exchange)'
elif project == 'superuser.com' and flavor == '_all_':
name = 'Superuser (Stack Exchange)'
elif project == 'ifixit' and flavor == '_all_':
name = 'iFixit'
elif project == 'gutenberg' and flavor == '_all_':
name = 'Gutenberg'
elif project == 'ham.stackexchange.com' and flavor == '_all_':
name = 'Amateur Radio (Stack Exchange)'
elif project == 'diy.stackexchange.com' and flavor == '_all_':
name = 'DIY (Stack Exchange)'
elif project == 'electronics.stackexchange.com' and flavor == '_all_':
name = 'Electronics (Stack Exchange)'
elif project == 'unix.stackexchange.com' and flavor == '_all_':
name = 'Unix (Stack Exchange)'
elif project == 'askubuntu.com' and flavor == '_all_':
name = 'Ask Ubuntu (Stack Exchange)'
elif project == 'bicycles.stackexchange.com' and flavor == '_all_':
name = 'Bicycles (Stack Exchange)'
elif project == 'biology.stackexchange.com' and flavor == '_all_':
name = 'Biology (Stack Exchange)'
elif project == 'arduino.stackexchange.com' and flavor == '_all_':
name = 'Arduino (Stack Exchange)'
elif project == '3dprinting.stackexchange.com' and flavor == '_all_':
name = '3D Printing (Stack Exchange)'
elif project == 'mdwiki' and flavor == '_all_':
name = 'MD Wiki (Wiki Project Med Foundation)'
elif project == 'wikipedia' and flavor == '_medicine_':
name = 'WikiMed Medical Encyclopedia'
elif project == 'zimgit-post-disaster' and flavor is None:
name = 'Post Disaster Resource Library'
elif project == 'fas-military-medicine' and flavor is None:
name = 'Military Medicine'
elif project == 'wikem' and flavor == '_all_maxi_':
name = 'WikEm (Global Emergency Medicine Wiki)'
elif project == 'health.stackexchange.com' and flavor == '_all_':
name = 'Health (Stack Exchange)'
elif project == 'khanacademy' and flavor == '_all_':
name = 'Khan Academy'
else:
raise RuntimeError(f'Could not find name for Zim URL: {url} {project=} {flavor=}')
if language not in kiwix.KIWIX_CATALOG_BY_NAME[name]['languages']:
raise ValueError(f'Language {repr(str(language))} is invalid for {name}')
return name, language
def get_zim_directory() -> pathlib.Path:
zim_directory = get_media_directory() / get_wrolpi_config().zims_destination
return zim_directory
async def check_zim(path: pathlib.Path) -> int:
"""
Runs `zimcheck` against the Zim file. Zim is valid only if this returns `0`.
@warning: Only performs the checksum check!
"""
cmd = ('zimcheck', '-C', path.absolute())
cmd_str = " ".join(str(i) for i in cmd)
logger.debug(f'check_zim: {cmd_str}')
result = await run_command(cmd)
logger.debug(f'zimcheck returned {result.return_code}')
if result.return_code != 0 and result.stderr:
logger.debug(result.stderr)
return result.return_code
ZIM_NAME_PARSER = re.compile(r'(.+?)_(\d{4}-\d{2}).zim')
def parse_name(path: pathlib.Path) -> Tuple[str, datetime]:
match = ZIM_NAME_PARSER.match(path.name)
if not match:
raise ValueError(f'Zim file name does not match expected pattern: {path.name}')
name, date = match.groups()
date = datetime.strptime(date, '%Y-%m')
return name, date
def find_outdated_zim_files(path: pathlib.Path = None) -> Tuple[List[pathlib.Path], List[pathlib.Path]]:
"""Search the Zim directory for outdated Zim files. Returns a list of outdated, and a list of current Zim files."""
path = path or get_zim_directory()
if not path.is_dir():
logger.error(f'Cannot search for outdated Zim files because the directory does not exist: {path}')
return list(), list()
# Find all non-empty Zim files.
files = [i for i in walk(path) if i.is_file() and i.suffix == '.zim' and i.stat().st_size]
zims = list()
for file in files:
try:
name, date = parse_name(file)
except Exception:
logger.warning(f'Found Zim {file} but its name does not match.')
continue
zims.append((name, date, file))
# Order by newest first.
zims = sorted(zims, key=lambda i: i[1], reverse=True)
outdated_files = list()
current_files = list()
newest_names = list()
for name, date, file in zims:
if name in newest_names:
# Already saw this name, this date is older.
outdated_files.append(file)
else:
newest_names.append(name)
current_files.append(file)
return sorted(outdated_files), sorted(current_files)
def migrate_outdated_zim_tags(outdated: List[pathlib.Path], current: List[pathlib.Path]) -> int:
"""Move the entry tags of each outdated Zim to the current Zim of the same name.
Must run before the outdated Zims are deleted; deleting a Zim cascade-deletes its TagZimEntry rows.
Returns the count of TagZimEntry rows moved."""
current_by_name = {parse_name(i)[0]: i for i in current}
moved = 0
with get_db_session(commit=True) as session:
for outdated_path in outdated:
current_path = current_by_name.get(parse_name(outdated_path)[0])
outdated_zim = Zim.get_by_path(session, outdated_path)
current_zim = Zim.get_by_path(session, current_path) if current_path else None
if not outdated_zim or not current_zim:
continue
for tag_zim_entry in session.query(TagZimEntry).filter_by(zim_id=outdated_zim.id).all():
if TagZimEntry.get_by_primary_keys(tag_zim_entry.tag_id, current_zim.id, tag_zim_entry.zim_entry,
session):
# The current Zim already has this tag on this entry.
session.delete(tag_zim_entry)
else:
tag_zim_entry.zim_id = current_zim.id
moved += 1
# A later outdated Zim may have the same tagged entry.
session.flush()
if moved:
logger.info(f'Migrated {moved} Zim entry tags from outdated Zims')
return moved
async def remove_outdated_zim_files(path: pathlib.Path = None) -> int:
"""Deletes all old Zim files. Their entry tags are moved to the current Zim of the same name."""
outdated, current = find_outdated_zim_files(path)
if outdated:
migrate_outdated_zim_tags(outdated, current)
# Save the migrated tags; the config is the source of truth.
tags.save_tags_config.activate_switch()
logger.info(f'Deleting {len(outdated)} outdated Zim files: {outdated}')
deleted_count = 0
for file in outdated:
file.unlink()
deleted_count += 1
if deleted_count:
# Refresh synchronously to clean up DB before returning. Deleting needs no modeling, and the
# global modelers would work through the whole unindexed backlog inside this request.
await file_worker.refresh_sync(list(outdated), post_processing=False)
flag_outdated_zim_files()
# Trigger Kiwix restart so the deleted zim files are no longer served.
restart_kiwix_handler.activate_switch()
return deleted_count
@register_refresh_cleanup
def flag_outdated_zim_files():
"""Set the `outdated_zims` Flag if outdated Zim files can be found, otherwise clear it."""
outdated, _ = find_outdated_zim_files()
if outdated:
logger.info('Outdated Zims were found')
flags.outdated_zims.set()
else:
logger.info('Outdated Zims were not found')
flags.outdated_zims.clear()
async def restart_kiwix():
if PYTEST:
logger.warning('Unable to restart Kiwix serve because we are testing')
return
if DOCKERIZED:
logger.info('Restarting Kiwix via controller API')
try:
async with aiohttp_post(f'{CONTROLLER_SERVICE}/api/services/zim/restart', None,
timeout=Seconds.minute) as response:
if response.status == 200:
logger.info('Successfully restarted Kiwix container')
return 0
else:
logger.error(f'Failed to restart Kiwix: {response.status}')
Events.send_user_notify(
'Failed to restart Kiwix automatically. Please restart manually: docker compose restart zim')
return 1
except Exception as e:
logger.error(f'Failed to restart Kiwix: {e}')
Events.send_user_notify(
'Failed to restart Kiwix automatically. Please restart manually: docker compose restart zim')
return 1
logger.info('Restarting Kiwix serve')
cmd = ('sudo', '/usr/bin/systemctl', 'restart', 'wrolpi-kiwix.service')
result = await run_command(cmd)
logger.debug(f'systemctl returned {result.return_code}')
if result.return_code != 0 and result.stderr:
logger.debug(result.stderr)
return result.return_code
def zim_config_name(path: pathlib.Path) -> str:
"""The name zims.yaml keys a Zim by: its file name without the date, so a newer version of the Zim is the same
Zim (wikipedia_en_all_maxi_2026-02.zim -> wikipedia_en_all_maxi). A Zim named without a date keeps its stem."""
try:
return parse_name(path)[0]
except ValueError:
return path.stem
@dataclass
class ZimsConfigValidator:
version: int = 0
auto_search: dict = field(default_factory=dict)
class ZimsConfig(ConfigFile):
"""zims.yaml: the settings of each Zim that must survive a rebuilt database, keyed by `zim_config_name`.
Format:
auto_search:
stackoverflow.com_en_all: false
wikipedia_en_all_maxi: true
`auto_search` is whether the global search searches the Zim. A Zim the file does not name is searched (the
default), so removing a line restores the default."""
file_name = 'zims.yaml'
validator = ZimsConfigValidator
default_config = dict(version=0, auto_search={})
@property
def auto_search(self) -> Dict[str, bool]:
return dict(self._config.get('auto_search') or {})
def auto_search_of(self, path: pathlib.Path) -> bool:
"""Whether the Zim at `path` should be searched by default, per the config."""
return bool(self.auto_search.get(zim_config_name(path), True))
def import_config(self, file: pathlib.Path = None, send_events=False):
"""Read the file, then apply `auto_search` to every Zim in the database."""
file = file or self.get_file()
if not file.is_file():
# A missing config is not an empty one: the defaults, and the database, stand.
logger.info(f'No config file, skipping import: {get_relative_to_media_directory(file)}')
self.successful_import = True
return
forced, self.force_import = self.force_import, False
super().import_config(file, send_events)
try:
decision, db_version = self.version_decision(self.version or 0, forced)
if decision == SAVE:
self.save_database_instead()
return
config = self.auto_search
changed = 0
with get_db_session(commit=True) as session:
for zim in session.query(Zim):
name = zim_config_name(zim.path)
if name in config:
want = bool(config[name])
elif decision == IMPORT:
# Not in the file: the default.
want = True
else:
# The file may be older than the database; only apply what it names.
continue
if bool(zim.auto_search) != want:
zim.auto_search = want
changed += 1
if decision == IMPORT:
session.flush()
set_config_version(session, self.file_name, self.version or 0)
logger.info(f'Imported {self.file_name} ({decision}): {changed} Zims changed')
if decision != IMPORT:
# The database may have Zims the file does not list; save them (see `record_saved_version`).
self.background_dump.activate_switch()
except Exception as e:
self.successful_import = False
message = f'Failed to import {self.file_name}'
logger.error(message, exc_info=e)
if send_events:
Events.send_config_import_failed(message)
raise
def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False):
"""Save every Zim's settings. Of several versions of one Zim, the newest decides.
The file's entries for Zims the database does not have are kept: a rebuilt database imports this config
(and saves it) before the refresh has modeled a single Zim, and the choices must survive until then."""
with get_db_session() as session:
version = self.db_version_for_dump(session)
auto_search = self.auto_search
# Sorted by path, so the newest version of a Zim (its date sorts last) wins.
for zim in session.query(Zim).order_by(Zim.path):
auto_search[zim_config_name(zim.path)] = bool(zim.auto_search)
self._config['auto_search'] = dict(sorted(auto_search.items()))
self._config['version'] = version
logger.info(f'Dumping {len(auto_search)} Zim settings to config')
self.save(file, send_events, overwrite)
zims_config = ZimsConfig()
register_ownership(ZimsConfig.file_name, Zim, columns={'auto_search'})
def get_zims_config() -> ZimsConfig:
return zims_config
@register_switch_handler('save_zims_config')
def save_zims_config():
"""Save the zims config when the switch is activated."""
zims_config.background_dump.activate_switch()
save_zims_config: ActivateSwitchMethod
def import_zims_config():
logger.info('Importing zims config')
zims_config.import_config()
def set_zim_auto_search(session: Session, zim_id: int, auto_search: bool):
zim = Zim.find_by_id(session, zim_id)
zim.auto_search = auto_search
session.commit()
save_zims_config.activate_switch()