mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
The toggle was only the zim.auto_search column, which defaults to on. The database is rebuilt from files and configs, and a Zim's newer version is a new row, so every rebuild and every subscription update turned search back on for every Zim. On a large library the global search then searched all of them. zims.yaml records auto_search for each Zim, keyed by the Zim's name without its date, so a newer version of a Zim is the same Zim. A Zim the file does not name is searched (the default stays on). The config is imported on startup with the usual version gating, saved when the toggle changes, and a newly modeled Zim takes its setting from it. Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
885 lines
37 KiB
Python
885 lines
37 KiB
Python
import asyncio
|
|
import functools
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from collections import OrderedDict
|
|
from typing import Callable, List, Tuple, Dict, OrderedDict as OrderedDictType
|
|
|
|
from libzim import Entry
|
|
from sqlalchemy import or_
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.orm.exc import NoResultFound # noqa
|
|
|
|
from modules.zim import kiwix
|
|
from modules.zim.errors import UnknownZim, UnknownZimSubscription
|
|
from modules.zim.kiwix import KIWIX_CATALOG
|
|
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription, ZimMetadata
|
|
from wrolpi import flags, tags
|
|
from wrolpi.cmd import run_command
|
|
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
|
|
register_refresh_cleanup, background_task, get_wrolpi_config, aiohttp_post, ConfigFile, \
|
|
get_relative_to_media_directory
|
|
from wrolpi.config_version import register_ownership, IMPORT, SAVE, set_config_version
|
|
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
|
|
from wrolpi.dates import Seconds
|
|
from wrolpi.events import Events
|
|
from wrolpi.db import get_db_session, session_curs
|
|
from wrolpi.downloader import DownloadFrequency
|
|
from wrolpi.files.lib import split_file_name_words
|
|
from wrolpi.files.modeler import run_modeler_loop, skip_clause
|
|
from wrolpi.files.worker import file_worker
|
|
from wrolpi.files.models import FileGroup
|
|
from wrolpi.vars import PYTEST, DOCKERIZED
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
CONTROLLER_SERVICE = 'http://controller'
|
|
|
|
__all__ = [
|
|
'flag_outdated_zim_files',
|
|
'get_all_entries_tags',
|
|
'get_entries_tags',
|
|
'get_entry',
|
|
'get_kiwix_catalog',
|
|
'get_zim',
|
|
'get_zims',
|
|
'get_zims_config',
|
|
'import_zims_config',
|
|
'model_zim',
|
|
'save_zims_config',
|
|
'zims_config',
|
|
'restart_kiwix_handler',
|
|
'zim_download_url_to_name',
|
|
'zim_modeler',
|
|
]
|
|
|
|
|
|
@register_switch_handler('restart_kiwix')
|
|
async def restart_kiwix_handler():
|
|
"""Switch handler that restarts the Kiwix service after zim files are added."""
|
|
await restart_kiwix()
|
|
|
|
|
|
def model_zim(file_group: FileGroup, session: Session) -> Zim:
|
|
"""Create a Zim record for the provided FileGroup."""
|
|
# Check if a Zim already exists for this FileGroup
|
|
zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none()
|
|
if not zim:
|
|
# Create new Zim if it doesn't exist. Its settings come from zims.yaml: a rebuilt database, or a newer
|
|
# version of a Zim, keeps what the user chose.
|
|
zim = Zim(file_group=file_group, path=file_group.primary_path,
|
|
auto_search=zims_config.auto_search_of(file_group.primary_path))
|
|
session.add(zim)
|
|
# Trigger Kiwix restart so the new zim file is available for browsing.
|
|
restart_kiwix_handler.activate_switch()
|
|
file_group.title = file_group.primary_path.name
|
|
file_group.a_text = split_file_name_words(file_group.primary_path.name)
|
|
file_group.indexed = True
|
|
file_group.model = 'zim'
|
|
return zim
|
|
|
|
|
|
ZIM_PROCESSING_LIMIT = 10
|
|
|
|
|
|
def _select_zim_ids(session, skip_ids: set) -> list[int]:
|
|
query = session.query(FileGroup.id) \
|
|
.outerjoin(Zim, Zim.file_group_id == FileGroup.id) \
|
|
.filter(FileGroup.primary_path.ilike('%.zim')) \
|
|
.filter(
|
|
# Exclusive-mimetype modeler: claim unmodeled rows even if apply_indexers
|
|
# already set indexed=True, and re-model when files changed.
|
|
or_(Zim.id.is_(None), FileGroup.indexed != True),
|
|
)
|
|
query = skip_clause(query, FileGroup.id, skip_ids)
|
|
return [row[0] for row in query.limit(ZIM_PROCESSING_LIMIT).all()]
|
|
|
|
|
|
def _apply_zim(session: Session, file_group: FileGroup, prepared):
|
|
zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none()
|
|
if not zim:
|
|
zim = model_zim(file_group, session)
|
|
zim.flush()
|
|
else:
|
|
file_group.title = file_group.primary_path.name
|
|
file_group.a_text = split_file_name_words(file_group.primary_path.name)
|
|
file_group.model = 'zim'
|
|
|
|
|
|
@register_modeler
|
|
async def zim_modeler(progress_callback: Callable[[int], None] = None):
|
|
await run_modeler_loop(
|
|
name='zim_modeler',
|
|
batch_size=ZIM_PROCESSING_LIMIT,
|
|
select_ids=_select_zim_ids,
|
|
apply=_apply_zim,
|
|
txn='batch',
|
|
progress_callback=progress_callback,
|
|
)
|
|
|
|
|
|
def get_all_entries_tags(session: Session):
|
|
"""Returns a dict of dicts which contain the names of all Tags of each Entry's path.
|
|
|
|
Example:
|
|
{
|
|
1: {
|
|
'path1': ['tag1', 'tag2'],
|
|
},
|
|
2: {
|
|
'path2': ['tag1'],
|
|
},
|
|
"""
|
|
stmt = '''
|
|
SELECT tz.zim_id, tz.zim_entry, json_group_array(t.name)
|
|
FROM tag t
|
|
LEFT JOIN tag_zim tz on t.id = tz.tag_id
|
|
GROUP BY 1, 2 \
|
|
'''
|
|
with session_curs(session) as curs:
|
|
curs.execute(stmt)
|
|
entries = dict()
|
|
for zim_id, zim_entry, tag_names in curs.fetchall():
|
|
if not zim_id:
|
|
continue
|
|
tag_names = json.loads(tag_names)
|
|
try:
|
|
entries[zim_id][zim_entry] = tag_names
|
|
except KeyError:
|
|
entries[zim_id] = {zim_entry: tag_names}
|
|
return entries
|
|
|
|
|
|
def get_entries_tags(session: Session, paths: List[str], zim_id: int):
|
|
all_entries_tags = get_all_entries_tags(session)
|
|
entries = dict()
|
|
for path in paths:
|
|
zim_tag_entries = all_entries_tags.get(zim_id, dict())
|
|
entries[path] = zim_tag_entries.get(path, list())
|
|
|
|
return entries
|
|
|
|
|
|
LIBZIM_WORKER = pathlib.Path(__file__).with_name('libzim_worker.py')
|
|
|
|
|
|
async def run_libzim_jobs(jobs: List[Dict]) -> List[Dict]:
|
|
"""Run libzim jobs (see `libzim_worker.py`) in a separate process, return one result per job.
|
|
|
|
python-libzim never releases the GIL, and opening a large Zim cold takes seconds, so nothing libzim does may
|
|
run in the API worker: every other request on that worker would wait behind it. (A plain subprocess:
|
|
Sanic workers are daemonic, so they cannot fork multiprocessing children.) A failed job's result is
|
|
`{'error': ...}`, so one bad Zim does not fail the rest."""
|
|
request = json.dumps(dict(jobs=jobs)).encode()
|
|
proc = await asyncio.create_subprocess_exec(
|
|
sys.executable, '-I', str(LIBZIM_WORKER),
|
|
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
try:
|
|
stdout, stderr = await proc.communicate(request)
|
|
except asyncio.CancelledError:
|
|
# Sanic cancels the handler at its response timeout or when the client disconnects; cancelling
|
|
# `communicate` leaves the process running (and, once its stdout pipe fills, stuck). Kill and reap it.
|
|
try:
|
|
proc.kill()
|
|
except ProcessLookupError:
|
|
pass
|
|
await proc.wait()
|
|
raise
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f'libzim worker failed ({proc.returncode}): {stderr.decode(errors="replace").strip()}')
|
|
if stderr:
|
|
logger.error(f'libzim worker reported errors:\n{stderr.decode(errors="replace").strip()}')
|
|
return json.loads(stdout)
|
|
|
|
|
|
def _apply_metadata(zim: Zim, result: Dict):
|
|
"""Fill the Zim's cached `zim_metadata` from a worker result so `Zim.__json__` does not open the Zim."""
|
|
if metadata := result.get('metadata'):
|
|
zim.__dict__['zim_metadata'] = ZimMetadata(**metadata)
|
|
|
|
|
|
def _search_job(zim: Zim, search_str: str, tag_names: List[str], session: Session, offset: int, limit: int) -> Dict:
|
|
"""The worker job that fetches what `_headline_results` needs for one Zim."""
|
|
if tag_names:
|
|
# TODO what is the count goes over the limit?
|
|
paths = zim.entry_paths_with_tags(session, tag_names, offset=offset, limit=1_000)
|
|
return dict(op='entries', path=str(zim.path), paths=paths, metadata=True)
|
|
elif search_str:
|
|
return dict(op='search', path=str(zim.path), search_str=search_str, offset=offset, limit=limit, metadata=True)
|
|
raise RuntimeError('Must provide search_str or tag_names.')
|
|
|
|
|
|
def _headline_results(session: Session, zim: Zim, search_str: str, tag_names: List[str], result: Dict) -> Dict:
|
|
"""Headline the entries a search job returned for one Zim."""
|
|
if 'error' in result:
|
|
raise RuntimeError(f'Failed to search {zim}: {result["error"]}')
|
|
_apply_metadata(zim, result)
|
|
entries = result['entries']
|
|
estimate = result.get('estimate', 0)
|
|
|
|
articles = [extract_html_text(i['content']) for i in entries]
|
|
|
|
entries_tag_names = get_entries_tags(session, [i['path'] for i in entries], zim.id)
|
|
|
|
# Always get headlines even if there is no search_str. Non-matches get the leading text of each article.
|
|
headlines = extract_headlines(articles, search_str)
|
|
|
|
search = list()
|
|
if search_str:
|
|
# `search_str` allows us to headline the title.
|
|
titles = [i['title'] for i in entries]
|
|
headline_titles = extract_headlines(titles, search_str)
|
|
|
|
for entry in entries:
|
|
title_headline, title_rank = headline_titles.pop(0)
|
|
headline, rank = headlines.pop(0)
|
|
if rank > 0 or title_rank > 0:
|
|
# The result for each Entry.
|
|
path = entry['path']
|
|
entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list()
|
|
search.append(dict(
|
|
headline=headline,
|
|
path=path,
|
|
rank=max(rank, title_rank),
|
|
tag_names=entry_tag_names,
|
|
title=title_headline,
|
|
))
|
|
if tag_names:
|
|
estimate += 1
|
|
else:
|
|
# No `search_str`, we return all entries.
|
|
for entry in entries:
|
|
headline, _ = headlines.pop(0)
|
|
path = entry['path']
|
|
entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list()
|
|
search.append(dict(
|
|
headline=headline,
|
|
path=path,
|
|
rank=0,
|
|
tag_names=entry_tag_names,
|
|
title=entry['title'],
|
|
))
|
|
estimate = len(entries)
|
|
|
|
results = dict(
|
|
path=zim.path,
|
|
metadata=zim.zim_metadata,
|
|
search=search,
|
|
estimate=estimate,
|
|
)
|
|
return results
|
|
|
|
|
|
async def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List[str] = None, offset: int = 0,
|
|
limit: int = 10) -> Dict:
|
|
zim = get_zim(session, int(zim_id))
|
|
job = _search_job(zim, search_str, tag_names, session, offset, limit)
|
|
result, = await run_libzim_jobs([job])
|
|
return await asyncio.to_thread(_headline_results, session, zim, search_str, tag_names, result)
|
|
|
|
|
|
def auto_search_zims(session: Session) -> List[Zim]:
|
|
"""The Zims that the global search searches, without opening them."""
|
|
zims = session.query(Zim).options(*Zim.json_options()).filter_by(auto_search=True).order_by(Zim.path).all()
|
|
return [i for i in zims if i.path and i.path.is_file()]
|
|
|
|
|
|
async def estimate_zims(session: Session, search_str: str) -> OrderedDictType[Zim, int]:
|
|
"""Estimate how many entries of each auto-search Zim match `search_str`.
|
|
|
|
`Zims.get_all` opens every Zim and is not used here for that reason; a Zim the worker cannot read is
|
|
reported and left out, as `Zims.get_all` would."""
|
|
zims = auto_search_zims(session)
|
|
jobs = [dict(op='estimate', path=str(i.path), search_str=search_str, metadata=True) for i in zims]
|
|
results = await run_libzim_jobs(jobs)
|
|
counts = OrderedDict()
|
|
for zim, result in zip(zims, results):
|
|
if 'error' not in result:
|
|
_apply_metadata(zim, result)
|
|
counts[zim] = result['estimate']
|
|
return counts
|
|
|
|
|
|
async def count_tagged_zim_entries(session: Session, tag_names: List[str]) -> OrderedDictType[Zim, int]:
|
|
"""Count the entries of every Zim tagged with all of `tag_names`. The counts come from the database; the
|
|
worker only reads each Zim's metadata, and a Zim it cannot read is left out."""
|
|
zims = session.query(Zim).options(*Zim.json_options()).order_by(Zim.path).all()
|
|
zims = [i for i in zims if i.path and i.path.is_file()]
|
|
counts = {i: len(i.entry_paths_with_tags(session, tag_names, limit=1_000_000)) for i in zims}
|
|
results = await run_libzim_jobs([dict(op='metadata', path=str(i.path)) for i in zims])
|
|
readable = OrderedDict()
|
|
for zim, result in zip(zims, results):
|
|
if 'error' not in result:
|
|
_apply_metadata(zim, result)
|
|
readable[zim] = counts[zim]
|
|
return readable
|
|
|
|
|
|
async def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0,
|
|
limit: int = 10) -> List[Dict]:
|
|
"""Search all Zim files that have auto_search enabled, in one libzim worker process.
|
|
|
|
Returns a list of per-zim result dicts, each containing metadata and search results.
|
|
Each search entry includes `zim_id` so callers can fetch the full entry afterward."""
|
|
zims = auto_search_zims(session)
|
|
jobs = [_search_job(zim, search_str, tag_names, session, offset, limit) for zim in zims]
|
|
results = await run_libzim_jobs(jobs)
|
|
|
|
def headline_all():
|
|
all_results = []
|
|
for zim, result in zip(zims, results):
|
|
try:
|
|
headlined = _headline_results(session, zim, search_str, tag_names, result)
|
|
except Exception as e:
|
|
logger.error(f'Failed to search Zim {zim}', exc_info=e)
|
|
continue
|
|
if headlined.get('search'):
|
|
for entry in headlined['search']:
|
|
entry['zim_id'] = zim.id
|
|
all_results.append(headlined)
|
|
return all_results
|
|
|
|
return await asyncio.to_thread(headline_all)
|
|
|
|
|
|
def get_zim(session: Session, zim_id: int) -> Zim:
|
|
"""Return the Zim record of the provided path.
|
|
|
|
@raise UnknownZim: When no record exists with the id.
|
|
@warning: Results are cached."""
|
|
try:
|
|
zim: Zim = session.query(Zim) \
|
|
.join(FileGroup, FileGroup.id == Zim.file_group_id) \
|
|
.filter(Zim.id == zim_id) \
|
|
.one()
|
|
return zim
|
|
except Exception as e:
|
|
raise UnknownZim() from e
|
|
|
|
|
|
@functools.lru_cache(maxsize=1_000)
|
|
def get_entry(path: str, zim_id: int) -> Entry:
|
|
with get_db_session() as session:
|
|
zim = get_zim(session, zim_id)
|
|
entry = zim.find_entry(path)
|
|
return entry
|
|
|
|
|
|
def get_zims(session: Session, ids: List[int] = None) -> List[Zim]:
|
|
if ids:
|
|
zims = session.query(Zim).filter(Zim.id.in_(ids)).all()
|
|
else:
|
|
zims = Zims.get_all(session)
|
|
|
|
return zims
|
|
|
|
|
|
def delete_zims(session: Session, ids: List[int]):
|
|
zims = get_zims(session, ids)
|
|
for zim in zims:
|
|
zim.delete()
|
|
|
|
|
|
async def add_tag(session: Session, tag_name: str, zim_id: int, zim_entry: str) -> TagZimEntry:
|
|
zim = session.query(Zim).filter_by(id=zim_id).one()
|
|
tag_zim_entry = zim.tag_entry(session, tag_name, zim_entry)
|
|
session.commit()
|
|
return tag_zim_entry
|
|
|
|
|
|
async def untag(session: Session, tag_name: str, zim_id: int, zim_entry: str):
|
|
zim: Zim = session.query(Zim).filter_by(id=zim_id).one()
|
|
zim.untag_entry(session, tag_name, zim_entry)
|
|
session.commit()
|
|
|
|
|
|
def get_kiwix_catalog():
|
|
return KIWIX_CATALOG.copy()
|
|
|
|
|
|
def get_kiwix_subscriptions(session: Session) -> Dict[str, ZimSubscription]:
|
|
subscriptions = session.query(ZimSubscription).options(*ZimSubscription.json_options())
|
|
subscriptions_by_name = {i.name: i for i in subscriptions}
|
|
return subscriptions_by_name
|
|
|
|
|
|
async def subscribe(session: Session, name: str, language: str,
|
|
frequency: int = DownloadFrequency.days180) -> ZimSubscription:
|
|
subscription = ZimSubscription.get_or_create(session, name)
|
|
subscription.language = language
|
|
|
|
if name == 'Wikipedia (with images)':
|
|
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_maxi_'
|
|
elif name == 'Wikipedia (mini)':
|
|
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_mini_'
|
|
elif name == 'Wikipedia (no images)':
|
|
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_nopic_'
|
|
elif name == 'Wikiversity (with images)':
|
|
url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_maxi_'
|
|
elif name == 'Wikiversity (no images)':
|
|
url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_nopic_'
|
|
elif name == 'Wikitionary (no images)':
|
|
url = f'https://download.kiwix.org/zim/wiktionary/wiktionary_{language}_all_nopic_'
|
|
elif name == 'Wikitionary (with images)':
|
|
url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_maxi_'
|
|
elif name == 'Wikibooks (no images)':
|
|
url = f'https://download.kiwix.org/zim/wikibooks/wikibooks_{language}_all_nopic_'
|
|
elif name == 'Wikibooks (with images)':
|
|
url = f'https://download.kiwix.org/zim/wikibooks/wikibooks_{language}_all_maxi_'
|
|
elif name == 'Wikisource':
|
|
url = f'https://download.kiwix.org/zim/wikisource/wikisource_{language}_all_maxi_'
|
|
elif name == 'Raspberry Pi (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/raspberrypi.stackexchange.com_{language}_all_'
|
|
elif name == 'Vikidia':
|
|
url = f'https://download.kiwix.org/zim/vikidia/vikidia_{language}_all_maxi_'
|
|
elif name == 'Stackoverflow (Stack Exchange)':
|
|
if language == 'en':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/stackoverflow.com_{language}_all_'
|
|
else:
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/{language}.stackoverflow.com_{language}_all_'
|
|
elif name == 'Superuser (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/superuser.com_{language}_all_'
|
|
elif name == 'iFixit':
|
|
url = f'https://download.kiwix.org/zim/ifixit/ifixit_{language}_all_'
|
|
elif name == 'Gutenberg':
|
|
url = f'https://download.kiwix.org/zim/gutenberg/gutenberg_{language}_all_'
|
|
elif name == 'Amateur Radio (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/ham.stackexchange.com_{language}_all_'
|
|
elif name == 'DIY (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/diy.stackexchange.com_{language}_all_'
|
|
elif name == 'Electronics (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/electronics.stackexchange.com_{language}_all_'
|
|
elif name == 'Unix (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/unix.stackexchange.com_{language}_all_'
|
|
elif name == 'Ask Ubuntu (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/askubuntu.com_{language}_all_'
|
|
elif name == 'Bicycles (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/bicycles.stackexchange.com_{language}_all_'
|
|
elif name == 'Biology (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/biology.stackexchange.com_{language}_all_'
|
|
elif name == 'Arduino (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/arduino.stackexchange.com_{language}_all_'
|
|
elif name == '3D Printing (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/3dprinting.stackexchange.com_{language}_all_'
|
|
elif name == 'MD Wiki (Wiki Project Med Foundation)':
|
|
url = f'https://download.kiwix.org/zim/other/mdwiki_{language}_all_'
|
|
elif name == 'WikiMed Medical Encyclopedia':
|
|
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_medicine_'
|
|
elif name == 'Post Disaster Resource Library':
|
|
url = f'https://download.kiwix.org/zim/other/zimgit-post-disaster_{language}_'
|
|
elif name == 'Military Medicine':
|
|
url = f'https://download.kiwix.org/zim/zimit/fas-military-medicine_{language}_'
|
|
elif name == 'WikEm (Global Emergency Medicine Wiki)':
|
|
url = f'https://download.kiwix.org/zim/other/wikem_{language}_all_maxi_'
|
|
elif name == 'Health (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/health.stackexchange.com_{language}_all_'
|
|
elif name == 'Khan Academy':
|
|
url = f'https://download.kiwix.org/zim/other/khanacademy_{language}_all_'
|
|
else:
|
|
raise ValueError(f'{name} not a valid Kiwix subscription name!')
|
|
|
|
if language not in kiwix.KIWIX_CATALOG_BY_NAME[name]['languages']:
|
|
raise ValueError(f'Language {repr(str(language))} is invalid for {name}')
|
|
|
|
subscription.change_download(session, url, frequency)
|
|
session.add(subscription)
|
|
session.flush([subscription])
|
|
session.commit()
|
|
return subscription
|
|
|
|
|
|
async def unsubscribe(session: Session, subscription_id: int):
|
|
subscription: ZimSubscription = session.query(ZimSubscription).filter_by(id=subscription_id).one_or_none()
|
|
if subscription:
|
|
session.delete(subscription)
|
|
if subscription.download:
|
|
session.delete(subscription.download)
|
|
session.commit()
|
|
else:
|
|
raise UnknownZimSubscription(f'{subscription_id=}')
|
|
|
|
|
|
KIWIX_URL_PARSER = re.compile(r'https:\/\/download\.kiwix\.org\/zim\/(.+?)\/(.+?)[_-](\w{2,3})([_-].+)')
|
|
KIWIX_URL_NO_FLAVOR_PARSER = re.compile(r'https:\/\/download\.kiwix\.org\/zim\/(.+?)\/(.+)[_-]([a-zA-Z]{2,3})')
|
|
|
|
|
|
def zim_download_url_to_name(url: str) -> Tuple[str, str]:
|
|
try:
|
|
_, project, language, flavor = KIWIX_URL_PARSER.match(url).groups()
|
|
except AttributeError:
|
|
# May not have a "flavor".
|
|
_, project, language = KIWIX_URL_NO_FLAVOR_PARSER.match(url).groups()
|
|
flavor = None
|
|
if project == 'wikipedia' and flavor == '_all_maxi_':
|
|
name = 'Wikipedia (with images)'
|
|
elif project == 'wikipedia' and flavor == '_all_nopic_':
|
|
name = 'Wikipedia (no images)'
|
|
elif project == 'wikipedia' and flavor == '_all_mini_':
|
|
name = 'Wikipedia (mini)'
|
|
elif project == 'wikiversity' and flavor == '_all_maxi_':
|
|
name = 'Wikiversity (with images)'
|
|
elif project == 'wikiversity' and flavor == '_all_nopic_':
|
|
name = 'Wikiversity (no images)'
|
|
elif project == 'wiktionary' and flavor == '_all_maxi_':
|
|
name = 'Wikitionary (with images)'
|
|
elif project == 'wiktionary' and flavor == '_all_nopic_':
|
|
name = 'Wikitionary (no images)'
|
|
elif project == 'wikibooks' and flavor == '_all_maxi_':
|
|
name = 'Wikibooks (with images)'
|
|
elif project == 'wikibooks' and flavor == '_all_nopic_':
|
|
name = 'Wikibooks (no images)'
|
|
elif project == 'wikisource' and flavor == '_all_maxi_':
|
|
name = 'Wikisource'
|
|
elif project == 'raspberrypi.stackexchange.com' and flavor == '_all_':
|
|
name = 'Raspberry Pi (Stack Exchange)'
|
|
elif project == 'vikidia' and flavor == '_all_maxi_':
|
|
name = 'Vikidia'
|
|
elif 'stackoverflow.com' in project and flavor == '_all_':
|
|
name = 'Stackoverflow (Stack Exchange)'
|
|
elif project == 'superuser.com' and flavor == '_all_':
|
|
name = 'Superuser (Stack Exchange)'
|
|
elif project == 'ifixit' and flavor == '_all_':
|
|
name = 'iFixit'
|
|
elif project == 'gutenberg' and flavor == '_all_':
|
|
name = 'Gutenberg'
|
|
elif project == 'ham.stackexchange.com' and flavor == '_all_':
|
|
name = 'Amateur Radio (Stack Exchange)'
|
|
elif project == 'diy.stackexchange.com' and flavor == '_all_':
|
|
name = 'DIY (Stack Exchange)'
|
|
elif project == 'electronics.stackexchange.com' and flavor == '_all_':
|
|
name = 'Electronics (Stack Exchange)'
|
|
elif project == 'unix.stackexchange.com' and flavor == '_all_':
|
|
name = 'Unix (Stack Exchange)'
|
|
elif project == 'askubuntu.com' and flavor == '_all_':
|
|
name = 'Ask Ubuntu (Stack Exchange)'
|
|
elif project == 'bicycles.stackexchange.com' and flavor == '_all_':
|
|
name = 'Bicycles (Stack Exchange)'
|
|
elif project == 'biology.stackexchange.com' and flavor == '_all_':
|
|
name = 'Biology (Stack Exchange)'
|
|
elif project == 'arduino.stackexchange.com' and flavor == '_all_':
|
|
name = 'Arduino (Stack Exchange)'
|
|
elif project == '3dprinting.stackexchange.com' and flavor == '_all_':
|
|
name = '3D Printing (Stack Exchange)'
|
|
elif project == 'mdwiki' and flavor == '_all_':
|
|
name = 'MD Wiki (Wiki Project Med Foundation)'
|
|
elif project == 'wikipedia' and flavor == '_medicine_':
|
|
name = 'WikiMed Medical Encyclopedia'
|
|
elif project == 'zimgit-post-disaster' and flavor is None:
|
|
name = 'Post Disaster Resource Library'
|
|
elif project == 'fas-military-medicine' and flavor is None:
|
|
name = 'Military Medicine'
|
|
elif project == 'wikem' and flavor == '_all_maxi_':
|
|
name = 'WikEm (Global Emergency Medicine Wiki)'
|
|
elif project == 'health.stackexchange.com' and flavor == '_all_':
|
|
name = 'Health (Stack Exchange)'
|
|
elif project == 'khanacademy' and flavor == '_all_':
|
|
name = 'Khan Academy'
|
|
else:
|
|
raise RuntimeError(f'Could not find name for Zim URL: {url} {project=} {flavor=}')
|
|
|
|
if language not in kiwix.KIWIX_CATALOG_BY_NAME[name]['languages']:
|
|
raise ValueError(f'Language {repr(str(language))} is invalid for {name}')
|
|
|
|
return name, language
|
|
|
|
|
|
def get_zim_directory() -> pathlib.Path:
|
|
zim_directory = get_media_directory() / get_wrolpi_config().zims_destination
|
|
return zim_directory
|
|
|
|
|
|
async def check_zim(path: pathlib.Path) -> int:
|
|
"""
|
|
Runs `zimcheck` against the Zim file. Zim is valid only if this returns `0`.
|
|
|
|
@warning: Only performs the checksum check!
|
|
"""
|
|
cmd = ('zimcheck', '-C', path.absolute())
|
|
cmd_str = " ".join(str(i) for i in cmd)
|
|
logger.debug(f'check_zim: {cmd_str}')
|
|
result = await run_command(cmd)
|
|
logger.debug(f'zimcheck returned {result.return_code}')
|
|
if result.return_code != 0 and result.stderr:
|
|
logger.debug(result.stderr)
|
|
return result.return_code
|
|
|
|
|
|
ZIM_NAME_PARSER = re.compile(r'(.+?)_(\d{4}-\d{2}).zim')
|
|
|
|
|
|
def parse_name(path: pathlib.Path) -> Tuple[str, datetime]:
|
|
match = ZIM_NAME_PARSER.match(path.name)
|
|
if not match:
|
|
raise ValueError(f'Zim file name does not match expected pattern: {path.name}')
|
|
name, date = match.groups()
|
|
date = datetime.strptime(date, '%Y-%m')
|
|
return name, date
|
|
|
|
|
|
def find_outdated_zim_files(path: pathlib.Path = None) -> Tuple[List[pathlib.Path], List[pathlib.Path]]:
|
|
"""Search the Zim directory for outdated Zim files. Returns a list of outdated, and a list of current Zim files."""
|
|
path = path or get_zim_directory()
|
|
if not path.is_dir():
|
|
logger.error(f'Cannot search for outdated Zim files because the directory does not exist: {path}')
|
|
return list(), list()
|
|
|
|
# Find all non-empty Zim files.
|
|
files = [i for i in walk(path) if i.is_file() and i.suffix == '.zim' and i.stat().st_size]
|
|
zims = list()
|
|
for file in files:
|
|
try:
|
|
name, date = parse_name(file)
|
|
except Exception:
|
|
logger.warning(f'Found Zim {file} but its name does not match.')
|
|
continue
|
|
|
|
zims.append((name, date, file))
|
|
|
|
# Order by newest first.
|
|
zims = sorted(zims, key=lambda i: i[1], reverse=True)
|
|
outdated_files = list()
|
|
current_files = list()
|
|
newest_names = list()
|
|
for name, date, file in zims:
|
|
if name in newest_names:
|
|
# Already saw this name, this date is older.
|
|
outdated_files.append(file)
|
|
else:
|
|
newest_names.append(name)
|
|
current_files.append(file)
|
|
|
|
return sorted(outdated_files), sorted(current_files)
|
|
|
|
|
|
def migrate_outdated_zim_tags(outdated: List[pathlib.Path], current: List[pathlib.Path]) -> int:
|
|
"""Move the entry tags of each outdated Zim to the current Zim of the same name.
|
|
|
|
Must run before the outdated Zims are deleted; deleting a Zim cascade-deletes its TagZimEntry rows.
|
|
|
|
Returns the count of TagZimEntry rows moved."""
|
|
current_by_name = {parse_name(i)[0]: i for i in current}
|
|
moved = 0
|
|
with get_db_session(commit=True) as session:
|
|
for outdated_path in outdated:
|
|
current_path = current_by_name.get(parse_name(outdated_path)[0])
|
|
outdated_zim = Zim.get_by_path(session, outdated_path)
|
|
current_zim = Zim.get_by_path(session, current_path) if current_path else None
|
|
if not outdated_zim or not current_zim:
|
|
continue
|
|
|
|
for tag_zim_entry in session.query(TagZimEntry).filter_by(zim_id=outdated_zim.id).all():
|
|
if TagZimEntry.get_by_primary_keys(tag_zim_entry.tag_id, current_zim.id, tag_zim_entry.zim_entry,
|
|
session):
|
|
# The current Zim already has this tag on this entry.
|
|
session.delete(tag_zim_entry)
|
|
else:
|
|
tag_zim_entry.zim_id = current_zim.id
|
|
moved += 1
|
|
# A later outdated Zim may have the same tagged entry.
|
|
session.flush()
|
|
|
|
if moved:
|
|
logger.info(f'Migrated {moved} Zim entry tags from outdated Zims')
|
|
return moved
|
|
|
|
|
|
async def remove_outdated_zim_files(path: pathlib.Path = None) -> int:
|
|
"""Deletes all old Zim files. Their entry tags are moved to the current Zim of the same name."""
|
|
outdated, current = find_outdated_zim_files(path)
|
|
if outdated:
|
|
migrate_outdated_zim_tags(outdated, current)
|
|
# Save the migrated tags; the config is the source of truth.
|
|
tags.save_tags_config.activate_switch()
|
|
|
|
logger.info(f'Deleting {len(outdated)} outdated Zim files: {outdated}')
|
|
deleted_count = 0
|
|
for file in outdated:
|
|
file.unlink()
|
|
deleted_count += 1
|
|
|
|
if deleted_count:
|
|
# Refresh synchronously to clean up DB before returning. Deleting needs no modeling, and the
|
|
# global modelers would work through the whole unindexed backlog inside this request.
|
|
await file_worker.refresh_sync(list(outdated), post_processing=False)
|
|
flag_outdated_zim_files()
|
|
|
|
# Trigger Kiwix restart so the deleted zim files are no longer served.
|
|
restart_kiwix_handler.activate_switch()
|
|
|
|
return deleted_count
|
|
|
|
|
|
@register_refresh_cleanup
|
|
def flag_outdated_zim_files():
|
|
"""Set the `outdated_zims` Flag if outdated Zim files can be found, otherwise clear it."""
|
|
outdated, _ = find_outdated_zim_files()
|
|
if outdated:
|
|
logger.info('Outdated Zims were found')
|
|
flags.outdated_zims.set()
|
|
else:
|
|
logger.info('Outdated Zims were not found')
|
|
flags.outdated_zims.clear()
|
|
|
|
|
|
async def restart_kiwix():
|
|
if PYTEST:
|
|
logger.warning('Unable to restart Kiwix serve because we are testing')
|
|
return
|
|
if DOCKERIZED:
|
|
logger.info('Restarting Kiwix via controller API')
|
|
try:
|
|
async with aiohttp_post(f'{CONTROLLER_SERVICE}/api/services/zim/restart', None,
|
|
timeout=Seconds.minute) as response:
|
|
if response.status == 200:
|
|
logger.info('Successfully restarted Kiwix container')
|
|
return 0
|
|
else:
|
|
logger.error(f'Failed to restart Kiwix: {response.status}')
|
|
Events.send_user_notify(
|
|
'Failed to restart Kiwix automatically. Please restart manually: docker compose restart zim')
|
|
return 1
|
|
except Exception as e:
|
|
logger.error(f'Failed to restart Kiwix: {e}')
|
|
Events.send_user_notify(
|
|
'Failed to restart Kiwix automatically. Please restart manually: docker compose restart zim')
|
|
return 1
|
|
|
|
logger.info('Restarting Kiwix serve')
|
|
|
|
cmd = ('sudo', '/usr/bin/systemctl', 'restart', 'wrolpi-kiwix.service')
|
|
result = await run_command(cmd)
|
|
logger.debug(f'systemctl returned {result.return_code}')
|
|
if result.return_code != 0 and result.stderr:
|
|
logger.debug(result.stderr)
|
|
return result.return_code
|
|
|
|
|
|
def zim_config_name(path: pathlib.Path) -> str:
|
|
"""The name zims.yaml keys a Zim by: its file name without the date, so a newer version of the Zim is the same
|
|
Zim (wikipedia_en_all_maxi_2026-02.zim -> wikipedia_en_all_maxi). A Zim named without a date keeps its stem."""
|
|
try:
|
|
return parse_name(path)[0]
|
|
except ValueError:
|
|
return path.stem
|
|
|
|
|
|
@dataclass
|
|
class ZimsConfigValidator:
|
|
version: int = 0
|
|
auto_search: dict = field(default_factory=dict)
|
|
|
|
|
|
class ZimsConfig(ConfigFile):
|
|
"""zims.yaml: the settings of each Zim that must survive a rebuilt database, keyed by `zim_config_name`.
|
|
|
|
Format:
|
|
auto_search:
|
|
stackoverflow.com_en_all: false
|
|
wikipedia_en_all_maxi: true
|
|
|
|
`auto_search` is whether the global search searches the Zim. A Zim the file does not name is searched (the
|
|
default), so removing a line restores the default."""
|
|
file_name = 'zims.yaml'
|
|
validator = ZimsConfigValidator
|
|
default_config = dict(version=0, auto_search={})
|
|
|
|
@property
|
|
def auto_search(self) -> Dict[str, bool]:
|
|
return dict(self._config.get('auto_search') or {})
|
|
|
|
def auto_search_of(self, path: pathlib.Path) -> bool:
|
|
"""Whether the Zim at `path` should be searched by default, per the config."""
|
|
return bool(self.auto_search.get(zim_config_name(path), True))
|
|
|
|
def import_config(self, file: pathlib.Path = None, send_events=False):
|
|
"""Read the file, then apply `auto_search` to every Zim in the database."""
|
|
file = file or self.get_file()
|
|
if not file.is_file():
|
|
# A missing config is not an empty one: the defaults, and the database, stand.
|
|
logger.info(f'No config file, skipping import: {get_relative_to_media_directory(file)}')
|
|
self.successful_import = True
|
|
return
|
|
forced, self.force_import = self.force_import, False
|
|
super().import_config(file, send_events)
|
|
try:
|
|
decision, db_version = self.version_decision(self.version or 0, forced)
|
|
if decision == SAVE:
|
|
self.save_database_instead()
|
|
return
|
|
config = self.auto_search
|
|
changed = 0
|
|
with get_db_session(commit=True) as session:
|
|
for zim in session.query(Zim):
|
|
name = zim_config_name(zim.path)
|
|
if name in config:
|
|
want = bool(config[name])
|
|
elif decision == IMPORT:
|
|
# Not in the file: the default.
|
|
want = True
|
|
else:
|
|
# The file may be older than the database; only apply what it names.
|
|
continue
|
|
if bool(zim.auto_search) != want:
|
|
zim.auto_search = want
|
|
changed += 1
|
|
if decision == IMPORT:
|
|
session.flush()
|
|
set_config_version(session, self.file_name, self.version or 0)
|
|
logger.info(f'Imported {self.file_name} ({decision}): {changed} Zims changed')
|
|
if decision != IMPORT:
|
|
# The database may have Zims the file does not list; save them (see `record_saved_version`).
|
|
self.background_dump.activate_switch()
|
|
except Exception as e:
|
|
self.successful_import = False
|
|
message = f'Failed to import {self.file_name}'
|
|
logger.error(message, exc_info=e)
|
|
if send_events:
|
|
Events.send_config_import_failed(message)
|
|
raise
|
|
|
|
def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False):
|
|
"""Save every Zim's settings. Of several versions of one Zim, the newest decides."""
|
|
with get_db_session() as session:
|
|
version = self.db_version_for_dump(session)
|
|
# Sorted by path, so the newest version of a Zim (its date sorts last) wins.
|
|
auto_search = dict()
|
|
for zim in session.query(Zim).order_by(Zim.path):
|
|
auto_search[zim_config_name(zim.path)] = bool(zim.auto_search)
|
|
self._config['auto_search'] = dict(sorted(auto_search.items()))
|
|
self._config['version'] = version
|
|
logger.info(f'Dumping {len(auto_search)} Zim settings to config')
|
|
self.save(file, send_events, overwrite)
|
|
|
|
|
|
zims_config = ZimsConfig()
|
|
register_ownership(ZimsConfig.file_name, Zim, columns={'auto_search'})
|
|
|
|
|
|
def get_zims_config() -> ZimsConfig:
|
|
return zims_config
|
|
|
|
|
|
@register_switch_handler('save_zims_config')
|
|
def save_zims_config():
|
|
"""Save the zims config when the switch is activated."""
|
|
zims_config.background_dump.activate_switch()
|
|
|
|
|
|
save_zims_config: ActivateSwitchMethod
|
|
|
|
|
|
def import_zims_config():
|
|
logger.info('Importing zims config')
|
|
zims_config.import_config()
|
|
|
|
|
|
def set_zim_auto_search(session: Session, zim_id: int, auto_search: bool):
|
|
zim = Zim.find_by_id(session, zim_id)
|
|
zim.auto_search = auto_search
|
|
session.commit()
|
|
save_zims_config.activate_switch()
|