mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 10:51:36 -04:00
cfbed9e4d7
* db: serializers read only loaded data Serializing a model must not query. A lazy load there is a hidden query per row (search serialized 24 rows with 55 statements), and would be a `MissingGreenlet` error under an async Session. - `wrolpi.db.no_db_access(reason)` and `@serializer` (on every model `__json__`, and around `json_response`): an engine `before_cursor_execute` listener raises `UnexpectedQuery` during tests, and in production logs once per serializer and lets the query run. Tests record each trip as well, and the autouse `no_unexpected_queries` fixture fails a test even when the code caught the error (Video's serializer was swallowing its per-row loads of the deferred `ffprobe_json`). - Relationships serializers read load with their parent: `selectin` for FileGroup.tag_files and Collection.downloads; `joined` for TagFile.tag, Collection.tag, Channel.collection, Video.channel, Archive.collection, ZimSubscription.download, CollectionItem.zim, and the file_group of Video, Archive, Doc, Zim and CollectionItem. - Search and the Video lookups the video page uses load the deferred `ffprobe_json` (`undefer`). - `wrolpi/test/test_serializers.py`: every serialized model, freshly loaded, serializes without SQL. Search: 55 -> 5 statements for 24 videos (31 -> 5 for archives), and a few ms faster locally. Not in this change: guarding the `dict()` serializers and `expire_on_commit=False`, which go together (about 50 serialize-after-commit sites, and 40 tests that read values changed elsewhere after a commit). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * db: serializer loads per query; the guard ignores transaction control The query guard reported the engine's own BEGIN (emitted through the same event as the first statement on a Session with no open transaction) instead of the lazy load after it, and in production that BEGIN used up the one warning per serializer. Transaction-control statements are now ignored. Mapper-level eager loads reached background code: every Video/Archive load in the modelers (one per file, inside the write transaction) joined file_group/channel/collection/tag and selectin-loaded the Collection's Download rows with their yt-dlp info_json. A second refresh of 100 videos took 0.40 s instead of 0.16 s. Serializer needs are now declared per model (`json_options()`) and added only by the queries whose results are serialized; Download.info_json is deferred there (no serializer reads it). Mapper defaults keep only narrow joins nearly every use needs: TagFile.tag, Collection.tag and Channel.collection. A second refresh is back to master's statement count and time; search still drops from 55 to 3 statements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
685 lines
27 KiB
Python
685 lines
27 KiB
Python
import functools
|
|
import json
|
|
import pathlib
|
|
import re
|
|
from datetime import datetime
|
|
from typing import Callable, List, Tuple, Dict
|
|
|
|
import cachetools.func
|
|
from libzim import Entry
|
|
from sqlalchemy import or_
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.orm.exc import NoResultFound # noqa
|
|
|
|
from modules.zim import kiwix
|
|
from modules.zim.errors import UnknownZim, UnknownZimSubscription
|
|
from modules.zim.kiwix import KIWIX_CATALOG
|
|
from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription
|
|
from wrolpi import flags, tags
|
|
from wrolpi.cmd import run_command
|
|
from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
|
|
register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate, aiohttp_post
|
|
from wrolpi.switches import register_switch_handler
|
|
from wrolpi.dates import Seconds
|
|
from wrolpi.events import Events
|
|
from wrolpi.db import get_db_session, session_curs
|
|
from wrolpi.downloader import DownloadFrequency
|
|
from wrolpi.files.lib import split_file_name_words
|
|
from wrolpi.files.modeler import run_modeler_loop, skip_clause
|
|
from wrolpi.files.worker import file_worker
|
|
from wrolpi.files.models import FileGroup
|
|
from wrolpi.vars import PYTEST, DOCKERIZED
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
CONTROLLER_SERVICE = 'http://controller'
|
|
|
|
__all__ = [
|
|
'flag_outdated_zim_files',
|
|
'get_all_entries_tags',
|
|
'get_entries_tags',
|
|
'get_entry',
|
|
'get_kiwix_catalog',
|
|
'get_unique_paths',
|
|
'get_zim',
|
|
'get_zims',
|
|
'model_zim',
|
|
'restart_kiwix_handler',
|
|
'zim_download_url_to_name',
|
|
'zim_modeler',
|
|
]
|
|
|
|
|
|
@register_switch_handler('restart_kiwix')
|
|
async def restart_kiwix_handler():
|
|
"""Switch handler that restarts the Kiwix service after zim files are added."""
|
|
await restart_kiwix()
|
|
|
|
|
|
def model_zim(file_group: FileGroup, session: Session) -> Zim:
|
|
"""Create a Zim record for the provided FileGroup."""
|
|
# Check if a Zim already exists for this FileGroup
|
|
zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none()
|
|
if not zim:
|
|
# Create new Zim if it doesn't exist
|
|
zim = Zim(file_group=file_group, path=file_group.primary_path)
|
|
session.add(zim)
|
|
# Trigger Kiwix restart so the new zim file is available for browsing.
|
|
restart_kiwix_handler.activate_switch()
|
|
file_group.title = file_group.primary_path.name
|
|
file_group.a_text = split_file_name_words(file_group.primary_path.name)
|
|
file_group.indexed = True
|
|
file_group.model = 'zim'
|
|
return zim
|
|
|
|
|
|
ZIM_PROCESSING_LIMIT = 10
|
|
|
|
|
|
def _select_zim_ids(session, skip_ids: set) -> list[int]:
|
|
query = session.query(FileGroup.id) \
|
|
.outerjoin(Zim, Zim.file_group_id == FileGroup.id) \
|
|
.filter(FileGroup.primary_path.ilike('%.zim')) \
|
|
.filter(
|
|
# Exclusive-mimetype modeler: claim unmodeled rows even if apply_indexers
|
|
# already set indexed=True, and re-model when files changed.
|
|
or_(Zim.id.is_(None), FileGroup.indexed != True),
|
|
)
|
|
query = skip_clause(query, FileGroup.id, skip_ids)
|
|
return [row[0] for row in query.limit(ZIM_PROCESSING_LIMIT).all()]
|
|
|
|
|
|
def _apply_zim(session: Session, file_group: FileGroup, prepared):
|
|
zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none()
|
|
if not zim:
|
|
zim = model_zim(file_group, session)
|
|
zim.flush()
|
|
else:
|
|
file_group.title = file_group.primary_path.name
|
|
file_group.a_text = split_file_name_words(file_group.primary_path.name)
|
|
file_group.model = 'zim'
|
|
|
|
|
|
@register_modeler
|
|
async def zim_modeler(progress_callback: Callable[[int], None] = None):
|
|
await run_modeler_loop(
|
|
name='zim_modeler',
|
|
batch_size=ZIM_PROCESSING_LIMIT,
|
|
select_ids=_select_zim_ids,
|
|
apply=_apply_zim,
|
|
txn='batch',
|
|
progress_callback=progress_callback,
|
|
)
|
|
|
|
|
|
def get_all_entries_tags(session: Session):
|
|
"""Returns a dict of dicts which contain the names of all Tags of each Entry's path.
|
|
|
|
Example:
|
|
{
|
|
1: {
|
|
'path1': ['tag1', 'tag2'],
|
|
},
|
|
2: {
|
|
'path2': ['tag1'],
|
|
},
|
|
"""
|
|
stmt = '''
|
|
SELECT tz.zim_id, tz.zim_entry, json_group_array(t.name)
|
|
FROM tag t
|
|
LEFT JOIN tag_zim tz on t.id = tz.tag_id
|
|
GROUP BY 1, 2 \
|
|
'''
|
|
with session_curs(session) as curs:
|
|
curs.execute(stmt)
|
|
entries = dict()
|
|
for zim_id, zim_entry, tag_names in curs.fetchall():
|
|
if not zim_id:
|
|
continue
|
|
tag_names = json.loads(tag_names)
|
|
try:
|
|
entries[zim_id][zim_entry] = tag_names
|
|
except KeyError:
|
|
entries[zim_id] = {zim_entry: tag_names}
|
|
return entries
|
|
|
|
|
|
def get_entries_tags(session: Session, paths: List[str], zim_id: int):
|
|
all_entries_tags = get_all_entries_tags(session)
|
|
entries = dict()
|
|
for path in paths:
|
|
zim_tag_entries = all_entries_tags.get(zim_id, dict())
|
|
entries[path] = zim_tag_entries.get(path, list())
|
|
|
|
return entries
|
|
|
|
|
|
def get_unique_paths(*paths: str) -> List[str]:
|
|
"""Return a new list which contains unique Zim paths."""
|
|
unique_paths = unique_by_predicate(
|
|
paths,
|
|
lambda i: i.replace('-', '').replace('_', '').replace('/', '').lower()
|
|
)
|
|
return list(unique_paths)
|
|
|
|
|
|
def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List[str] = None, offset: int = 0,
|
|
limit: int = 10) -> Dict:
|
|
zim_id = int(zim_id)
|
|
zim = get_zim(session, zim_id)
|
|
|
|
if tag_names:
|
|
# TODO what is the count goes over the limit?
|
|
entries = zim.entries_with_tags(session, tag_names, offset=offset, limit=1_000)
|
|
estimate = 0
|
|
elif search_str:
|
|
# Get suggested results (this searches titles).
|
|
results = zim.suggest(search_str, offset, limit)
|
|
search_results = zim.search(search_str, offset, limit)
|
|
# Remove duplicate entries (disambiguation or redirect).
|
|
results = get_unique_paths(*results, *search_results)
|
|
results = results[:limit]
|
|
entries = [zim.find_entry(i) for i in results]
|
|
estimate = zim.estimate(search_str)
|
|
else:
|
|
raise RuntimeError('Must provide search_str or tag_names.')
|
|
|
|
articles = [bytes(i.get_item().content).decode('UTF-8') for i in entries]
|
|
articles = [extract_html_text(i) for i in articles]
|
|
|
|
entries_tag_names = get_entries_tags(session, [i.path for i in entries], zim_id)
|
|
|
|
# Always get headlines even if there is no search_str. Non-matches get the leading text of each article.
|
|
headlines = extract_headlines(articles, search_str)
|
|
|
|
search = list()
|
|
if search_str:
|
|
# `search_str` allows us to headline the title.
|
|
titles = [i.title for i in entries]
|
|
headline_titles = extract_headlines(titles, search_str)
|
|
|
|
for entry in entries:
|
|
title_headline, title_rank = headline_titles.pop(0)
|
|
headline, rank = headlines.pop(0)
|
|
if rank > 0 or title_rank > 0:
|
|
# The result for each Entry.
|
|
path = entry.path
|
|
entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list()
|
|
search.append(dict(
|
|
headline=headline,
|
|
path=path,
|
|
rank=max(rank, title_rank),
|
|
tag_names=entry_tag_names,
|
|
title=title_headline,
|
|
))
|
|
if tag_names:
|
|
estimate += 1
|
|
else:
|
|
# No `search_str`, we return all entries.
|
|
for entry in entries:
|
|
headline, _ = headlines.pop(0)
|
|
path = entry.path
|
|
entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list()
|
|
search.append(dict(
|
|
headline=headline,
|
|
path=path,
|
|
rank=0,
|
|
tag_names=entry_tag_names,
|
|
title=entry.title,
|
|
))
|
|
estimate = len(entries)
|
|
|
|
results = dict(
|
|
path=zim.path,
|
|
metadata=zim.zim_metadata,
|
|
search=search,
|
|
estimate=estimate,
|
|
)
|
|
return results
|
|
|
|
|
|
def search_all_zims(session: Session, search_str: str, tag_names: List[str] = None, offset: int = 0,
|
|
limit: int = 10) -> List[Dict]:
|
|
"""Search all Zim files that have auto_search enabled.
|
|
|
|
Returns a list of per-zim result dicts, each containing metadata and search results.
|
|
Each search entry includes `zim_id` so callers can fetch the full entry afterward."""
|
|
zims = Zims.get_all(session)
|
|
all_results = []
|
|
for zim in zims:
|
|
if not zim.auto_search:
|
|
continue
|
|
try:
|
|
result = headline_zim(session, search_str, zim.id, tag_names=tag_names, offset=offset, limit=limit)
|
|
if result.get('search'):
|
|
for entry in result['search']:
|
|
entry['zim_id'] = zim.id
|
|
all_results.append(result)
|
|
except Exception as e:
|
|
logger.error(f'Failed to search Zim {zim}', exc_info=e)
|
|
return all_results
|
|
|
|
|
|
@cachetools.func.lru_cache(maxsize=1_000)
|
|
def get_estimates(search_str: str) -> List[int]:
|
|
zims = Zims.get_all()
|
|
estimates = [i.estimate(search_str) for i in zims]
|
|
return estimates
|
|
|
|
|
|
def get_zim(session: Session, zim_id: int) -> Zim:
|
|
"""Return the Zim record of the provided path.
|
|
|
|
@raise UnknownZim: When no record exists with the id.
|
|
@warning: Results are cached."""
|
|
try:
|
|
zim: Zim = session.query(Zim) \
|
|
.join(FileGroup, FileGroup.id == Zim.file_group_id) \
|
|
.filter(Zim.id == zim_id) \
|
|
.one()
|
|
return zim
|
|
except Exception as e:
|
|
raise UnknownZim() from e
|
|
|
|
|
|
@functools.lru_cache(maxsize=1_000)
|
|
def get_entry(path: str, zim_id: int) -> Entry:
|
|
with get_db_session() as session:
|
|
zim = get_zim(session, zim_id)
|
|
entry = zim.find_entry(path)
|
|
return entry
|
|
|
|
|
|
def get_zims(session: Session, ids: List[int] = None) -> List[Zim]:
|
|
if ids:
|
|
zims = session.query(Zim).filter(Zim.id.in_(ids)).all()
|
|
else:
|
|
zims = Zims.get_all(session)
|
|
|
|
return zims
|
|
|
|
|
|
def delete_zims(session: Session, ids: List[int]):
|
|
zims = get_zims(session, ids)
|
|
for zim in zims:
|
|
zim.delete()
|
|
|
|
|
|
async def add_tag(session: Session, tag_name: str, zim_id: int, zim_entry: str) -> TagZimEntry:
|
|
zim = session.query(Zim).filter_by(id=zim_id).one()
|
|
tag_zim_entry = zim.tag_entry(session, tag_name, zim_entry)
|
|
session.commit()
|
|
return tag_zim_entry
|
|
|
|
|
|
async def untag(session: Session, tag_name: str, zim_id: int, zim_entry: str):
|
|
zim: Zim = session.query(Zim).filter_by(id=zim_id).one()
|
|
zim.untag_entry(session, tag_name, zim_entry)
|
|
session.commit()
|
|
|
|
|
|
def get_kiwix_catalog():
|
|
return KIWIX_CATALOG.copy()
|
|
|
|
|
|
def get_kiwix_subscriptions(session: Session) -> Dict[str, ZimSubscription]:
|
|
subscriptions = session.query(ZimSubscription).options(*ZimSubscription.json_options())
|
|
subscriptions_by_name = {i.name: i for i in subscriptions}
|
|
return subscriptions_by_name
|
|
|
|
|
|
async def subscribe(session: Session, name: str, language: str,
|
|
frequency: int = DownloadFrequency.days180) -> ZimSubscription:
|
|
subscription = ZimSubscription.get_or_create(session, name)
|
|
subscription.language = language
|
|
|
|
if name == 'Wikipedia (with images)':
|
|
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_maxi_'
|
|
elif name == 'Wikipedia (mini)':
|
|
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_mini_'
|
|
elif name == 'Wikipedia (no images)':
|
|
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_nopic_'
|
|
elif name == 'Wikiversity (with images)':
|
|
url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_maxi_'
|
|
elif name == 'Wikiversity (no images)':
|
|
url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_nopic_'
|
|
elif name == 'Wikitionary (no images)':
|
|
url = f'https://download.kiwix.org/zim/wiktionary/wiktionary_{language}_all_nopic_'
|
|
elif name == 'Wikitionary (with images)':
|
|
url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_maxi_'
|
|
elif name == 'Wikibooks (no images)':
|
|
url = f'https://download.kiwix.org/zim/wikibooks/wikibooks_{language}_all_nopic_'
|
|
elif name == 'Wikibooks (with images)':
|
|
url = f'https://download.kiwix.org/zim/wikibooks/wikibooks_{language}_all_maxi_'
|
|
elif name == 'Wikisource':
|
|
url = f'https://download.kiwix.org/zim/wikisource/wikisource_{language}_all_maxi_'
|
|
elif name == 'Raspberry Pi (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/raspberrypi.stackexchange.com_{language}_all_'
|
|
elif name == 'Vikidia':
|
|
url = f'https://download.kiwix.org/zim/vikidia/vikidia_{language}_all_maxi_'
|
|
elif name == 'Stackoverflow (Stack Exchange)':
|
|
if language == 'en':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/stackoverflow.com_{language}_all_'
|
|
else:
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/{language}.stackoverflow.com_{language}_all_'
|
|
elif name == 'Superuser (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/superuser.com_{language}_all_'
|
|
elif name == 'iFixit':
|
|
url = f'https://download.kiwix.org/zim/ifixit/ifixit_{language}_all_'
|
|
elif name == 'Gutenberg':
|
|
url = f'https://download.kiwix.org/zim/gutenberg/gutenberg_{language}_all_'
|
|
elif name == 'Amateur Radio (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/ham.stackexchange.com_{language}_all_'
|
|
elif name == 'DIY (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/diy.stackexchange.com_{language}_all_'
|
|
elif name == 'Electronics (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/electronics.stackexchange.com_{language}_all_'
|
|
elif name == 'Unix (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/unix.stackexchange.com_{language}_all_'
|
|
elif name == 'Ask Ubuntu (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/askubuntu.com_{language}_all_'
|
|
elif name == 'Bicycles (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/bicycles.stackexchange.com_{language}_all_'
|
|
elif name == 'Biology (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/biology.stackexchange.com_{language}_all_'
|
|
elif name == 'Arduino (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/arduino.stackexchange.com_{language}_all_'
|
|
elif name == '3D Printing (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/3dprinting.stackexchange.com_{language}_all_'
|
|
elif name == 'MD Wiki (Wiki Project Med Foundation)':
|
|
url = f'https://download.kiwix.org/zim/other/mdwiki_{language}_all_'
|
|
elif name == 'WikiMed Medical Encyclopedia':
|
|
url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_medicine_'
|
|
elif name == 'Post Disaster Resource Library':
|
|
url = f'https://download.kiwix.org/zim/other/zimgit-post-disaster_{language}_'
|
|
elif name == 'Military Medicine':
|
|
url = f'https://download.kiwix.org/zim/zimit/fas-military-medicine_{language}_'
|
|
elif name == 'WikEm (Global Emergency Medicine Wiki)':
|
|
url = f'https://download.kiwix.org/zim/other/wikem_{language}_all_maxi_'
|
|
elif name == 'Health (Stack Exchange)':
|
|
url = f'https://download.kiwix.org/zim/stack_exchange/health.stackexchange.com_{language}_all_'
|
|
elif name == 'Khan Academy':
|
|
url = f'https://download.kiwix.org/zim/other/khanacademy_{language}_all_'
|
|
else:
|
|
raise ValueError(f'{name} not a valid Kiwix subscription name!')
|
|
|
|
if language not in kiwix.KIWIX_CATALOG_BY_NAME[name]['languages']:
|
|
raise ValueError(f'Language {repr(str(language))} is invalid for {name}')
|
|
|
|
subscription.change_download(session, url, frequency)
|
|
session.add(subscription)
|
|
session.flush([subscription])
|
|
session.commit()
|
|
return subscription
|
|
|
|
|
|
async def unsubscribe(session: Session, subscription_id: int):
|
|
subscription: ZimSubscription = session.query(ZimSubscription).filter_by(id=subscription_id).one_or_none()
|
|
if subscription:
|
|
session.delete(subscription)
|
|
if subscription.download:
|
|
session.delete(subscription.download)
|
|
session.commit()
|
|
else:
|
|
raise UnknownZimSubscription(f'{subscription_id=}')
|
|
|
|
|
|
KIWIX_URL_PARSER = re.compile(r'https:\/\/download\.kiwix\.org\/zim\/(.+?)\/(.+?)[_-](\w{2,3})([_-].+)')
|
|
KIWIX_URL_NO_FLAVOR_PARSER = re.compile(r'https:\/\/download\.kiwix\.org\/zim\/(.+?)\/(.+)[_-]([a-zA-Z]{2,3})')
|
|
|
|
|
|
def zim_download_url_to_name(url: str) -> Tuple[str, str]:
|
|
try:
|
|
_, project, language, flavor = KIWIX_URL_PARSER.match(url).groups()
|
|
except AttributeError:
|
|
# May not have a "flavor".
|
|
_, project, language = KIWIX_URL_NO_FLAVOR_PARSER.match(url).groups()
|
|
flavor = None
|
|
if project == 'wikipedia' and flavor == '_all_maxi_':
|
|
name = 'Wikipedia (with images)'
|
|
elif project == 'wikipedia' and flavor == '_all_nopic_':
|
|
name = 'Wikipedia (no images)'
|
|
elif project == 'wikipedia' and flavor == '_all_mini_':
|
|
name = 'Wikipedia (mini)'
|
|
elif project == 'wikiversity' and flavor == '_all_maxi_':
|
|
name = 'Wikiversity (with images)'
|
|
elif project == 'wikiversity' and flavor == '_all_nopic_':
|
|
name = 'Wikiversity (no images)'
|
|
elif project == 'wiktionary' and flavor == '_all_maxi_':
|
|
name = 'Wikitionary (with images)'
|
|
elif project == 'wiktionary' and flavor == '_all_nopic_':
|
|
name = 'Wikitionary (no images)'
|
|
elif project == 'wikibooks' and flavor == '_all_maxi_':
|
|
name = 'Wikibooks (with images)'
|
|
elif project == 'wikibooks' and flavor == '_all_nopic_':
|
|
name = 'Wikibooks (no images)'
|
|
elif project == 'wikisource' and flavor == '_all_maxi_':
|
|
name = 'Wikisource'
|
|
elif project == 'raspberrypi.stackexchange.com' and flavor == '_all_':
|
|
name = 'Raspberry Pi (Stack Exchange)'
|
|
elif project == 'vikidia' and flavor == '_all_maxi_':
|
|
name = 'Vikidia'
|
|
elif 'stackoverflow.com' in project and flavor == '_all_':
|
|
name = 'Stackoverflow (Stack Exchange)'
|
|
elif project == 'superuser.com' and flavor == '_all_':
|
|
name = 'Superuser (Stack Exchange)'
|
|
elif project == 'ifixit' and flavor == '_all_':
|
|
name = 'iFixit'
|
|
elif project == 'gutenberg' and flavor == '_all_':
|
|
name = 'Gutenberg'
|
|
elif project == 'ham.stackexchange.com' and flavor == '_all_':
|
|
name = 'Amateur Radio (Stack Exchange)'
|
|
elif project == 'diy.stackexchange.com' and flavor == '_all_':
|
|
name = 'DIY (Stack Exchange)'
|
|
elif project == 'electronics.stackexchange.com' and flavor == '_all_':
|
|
name = 'Electronics (Stack Exchange)'
|
|
elif project == 'unix.stackexchange.com' and flavor == '_all_':
|
|
name = 'Unix (Stack Exchange)'
|
|
elif project == 'askubuntu.com' and flavor == '_all_':
|
|
name = 'Ask Ubuntu (Stack Exchange)'
|
|
elif project == 'bicycles.stackexchange.com' and flavor == '_all_':
|
|
name = 'Bicycles (Stack Exchange)'
|
|
elif project == 'biology.stackexchange.com' and flavor == '_all_':
|
|
name = 'Biology (Stack Exchange)'
|
|
elif project == 'arduino.stackexchange.com' and flavor == '_all_':
|
|
name = 'Arduino (Stack Exchange)'
|
|
elif project == '3dprinting.stackexchange.com' and flavor == '_all_':
|
|
name = '3D Printing (Stack Exchange)'
|
|
elif project == 'mdwiki' and flavor == '_all_':
|
|
name = 'MD Wiki (Wiki Project Med Foundation)'
|
|
elif project == 'wikipedia' and flavor == '_medicine_':
|
|
name = 'WikiMed Medical Encyclopedia'
|
|
elif project == 'zimgit-post-disaster' and flavor is None:
|
|
name = 'Post Disaster Resource Library'
|
|
elif project == 'fas-military-medicine' and flavor is None:
|
|
name = 'Military Medicine'
|
|
elif project == 'wikem' and flavor == '_all_maxi_':
|
|
name = 'WikEm (Global Emergency Medicine Wiki)'
|
|
elif project == 'health.stackexchange.com' and flavor == '_all_':
|
|
name = 'Health (Stack Exchange)'
|
|
elif project == 'khanacademy' and flavor == '_all_':
|
|
name = 'Khan Academy'
|
|
else:
|
|
raise RuntimeError(f'Could not find name for Zim URL: {url} {project=} {flavor=}')
|
|
|
|
if language not in kiwix.KIWIX_CATALOG_BY_NAME[name]['languages']:
|
|
raise ValueError(f'Language {repr(str(language))} is invalid for {name}')
|
|
|
|
return name, language
|
|
|
|
|
|
def get_zim_directory() -> pathlib.Path:
|
|
zim_directory = get_media_directory() / get_wrolpi_config().zims_destination
|
|
return zim_directory
|
|
|
|
|
|
async def check_zim(path: pathlib.Path) -> int:
|
|
"""
|
|
Runs `zimcheck` against the Zim file. Zim is valid only if this returns `0`.
|
|
|
|
@warning: Only performs the checksum check!
|
|
"""
|
|
cmd = ('zimcheck', '-C', path.absolute())
|
|
cmd_str = " ".join(str(i) for i in cmd)
|
|
logger.debug(f'check_zim: {cmd_str}')
|
|
result = await run_command(cmd)
|
|
logger.debug(f'zimcheck returned {result.return_code}')
|
|
if result.return_code != 0 and result.stderr:
|
|
logger.debug(result.stderr)
|
|
return result.return_code
|
|
|
|
|
|
ZIM_NAME_PARSER = re.compile(r'(.+?)_(\d{4}-\d{2}).zim')
|
|
|
|
|
|
def parse_name(path: pathlib.Path) -> Tuple[str, datetime]:
|
|
match = ZIM_NAME_PARSER.match(path.name)
|
|
if not match:
|
|
raise ValueError(f'Zim file name does not match expected pattern: {path.name}')
|
|
name, date = match.groups()
|
|
date = datetime.strptime(date, '%Y-%m')
|
|
return name, date
|
|
|
|
|
|
def find_outdated_zim_files(path: pathlib.Path = None) -> Tuple[List[pathlib.Path], List[pathlib.Path]]:
|
|
"""Search the Zim directory for outdated Zim files. Returns a list of outdated, and a list of current Zim files."""
|
|
path = path or get_zim_directory()
|
|
if not path.is_dir():
|
|
logger.error(f'Cannot search for outdated Zim files because the directory does not exist: {path}')
|
|
return list(), list()
|
|
|
|
# Find all non-empty Zim files.
|
|
files = [i for i in walk(path) if i.is_file() and i.suffix == '.zim' and i.stat().st_size]
|
|
zims = list()
|
|
for file in files:
|
|
try:
|
|
name, date = parse_name(file)
|
|
except Exception:
|
|
logger.warning(f'Found Zim {file} but its name does not match.')
|
|
continue
|
|
|
|
zims.append((name, date, file))
|
|
|
|
# Order by newest first.
|
|
zims = sorted(zims, key=lambda i: i[1], reverse=True)
|
|
outdated_files = list()
|
|
current_files = list()
|
|
newest_names = list()
|
|
for name, date, file in zims:
|
|
if name in newest_names:
|
|
# Already saw this name, this date is older.
|
|
outdated_files.append(file)
|
|
else:
|
|
newest_names.append(name)
|
|
current_files.append(file)
|
|
|
|
return sorted(outdated_files), sorted(current_files)
|
|
|
|
|
|
def migrate_outdated_zim_tags(outdated: List[pathlib.Path], current: List[pathlib.Path]) -> int:
|
|
"""Move the entry tags of each outdated Zim to the current Zim of the same name.
|
|
|
|
Must run before the outdated Zims are deleted; deleting a Zim cascade-deletes its TagZimEntry rows.
|
|
|
|
Returns the count of TagZimEntry rows moved."""
|
|
current_by_name = {parse_name(i)[0]: i for i in current}
|
|
moved = 0
|
|
with get_db_session(commit=True) as session:
|
|
for outdated_path in outdated:
|
|
current_path = current_by_name.get(parse_name(outdated_path)[0])
|
|
outdated_zim = Zim.get_by_path(session, outdated_path)
|
|
current_zim = Zim.get_by_path(session, current_path) if current_path else None
|
|
if not outdated_zim or not current_zim:
|
|
continue
|
|
|
|
for tag_zim_entry in session.query(TagZimEntry).filter_by(zim_id=outdated_zim.id).all():
|
|
if TagZimEntry.get_by_primary_keys(tag_zim_entry.tag_id, current_zim.id, tag_zim_entry.zim_entry,
|
|
session):
|
|
# The current Zim already has this tag on this entry.
|
|
session.delete(tag_zim_entry)
|
|
else:
|
|
tag_zim_entry.zim_id = current_zim.id
|
|
moved += 1
|
|
# A later outdated Zim may have the same tagged entry.
|
|
session.flush()
|
|
|
|
if moved:
|
|
logger.info(f'Migrated {moved} Zim entry tags from outdated Zims')
|
|
return moved
|
|
|
|
|
|
async def remove_outdated_zim_files(path: pathlib.Path = None) -> int:
|
|
"""Deletes all old Zim files. Their entry tags are moved to the current Zim of the same name."""
|
|
outdated, current = find_outdated_zim_files(path)
|
|
if outdated:
|
|
migrate_outdated_zim_tags(outdated, current)
|
|
# Save the migrated tags; the config is the source of truth.
|
|
tags.save_tags_config.activate_switch()
|
|
|
|
logger.info(f'Deleting {len(outdated)} outdated Zim files: {outdated}')
|
|
deleted_count = 0
|
|
for file in outdated:
|
|
file.unlink()
|
|
deleted_count += 1
|
|
|
|
if deleted_count:
|
|
# Refresh synchronously to clean up DB before returning. Deleting needs no modeling, and the
|
|
# global modelers would work through the whole unindexed backlog inside this request.
|
|
await file_worker.refresh_sync(list(outdated), post_processing=False)
|
|
flag_outdated_zim_files()
|
|
|
|
# Trigger Kiwix restart so the deleted zim files are no longer served.
|
|
restart_kiwix_handler.activate_switch()
|
|
|
|
return deleted_count
|
|
|
|
|
|
@register_refresh_cleanup
|
|
def flag_outdated_zim_files():
|
|
"""Set the `outdated_zims` Flag if outdated Zim files can be found, otherwise clear it."""
|
|
outdated, _ = find_outdated_zim_files()
|
|
if outdated:
|
|
logger.info('Outdated Zims were found')
|
|
flags.outdated_zims.set()
|
|
else:
|
|
logger.info('Outdated Zims were not found')
|
|
flags.outdated_zims.clear()
|
|
|
|
|
|
async def restart_kiwix():
|
|
if PYTEST:
|
|
logger.warning('Unable to restart Kiwix serve because we are testing')
|
|
return
|
|
if DOCKERIZED:
|
|
logger.info('Restarting Kiwix via controller API')
|
|
try:
|
|
async with aiohttp_post(f'{CONTROLLER_SERVICE}/api/services/zim/restart', None,
|
|
timeout=Seconds.minute) as response:
|
|
if response.status == 200:
|
|
logger.info('Successfully restarted Kiwix container')
|
|
return 0
|
|
else:
|
|
logger.error(f'Failed to restart Kiwix: {response.status}')
|
|
Events.send_user_notify(
|
|
'Failed to restart Kiwix automatically. Please restart manually: docker compose restart zim')
|
|
return 1
|
|
except Exception as e:
|
|
logger.error(f'Failed to restart Kiwix: {e}')
|
|
Events.send_user_notify(
|
|
'Failed to restart Kiwix automatically. Please restart manually: docker compose restart zim')
|
|
return 1
|
|
|
|
logger.info('Restarting Kiwix serve')
|
|
|
|
cmd = ('sudo', '/usr/bin/systemctl', 'restart', 'wrolpi-kiwix.service')
|
|
result = await run_command(cmd)
|
|
logger.debug(f'systemctl returned {result.return_code}')
|
|
if result.return_code != 0 and result.stderr:
|
|
logger.debug(result.stderr)
|
|
return result.return_code
|
|
|
|
|
|
def set_zim_auto_search(session: Session, zim_id: int, auto_search: bool):
|
|
zim = Zim.find_by_id(session, zim_id)
|
|
zim.auto_search = auto_search
|
|
session.commit()
|