mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
* Zim search estimates run in a separate process so a cold estimate cannot stall the API worker The global search estimates every auto-search Zim on each keystroke. On a large library that estimate took tens of seconds when the Zims' pages were not cached, and every other request routed to the same Sanic worker, such as the channel suggestions, waited behind it. Two things held the worker: opening a large Zim cold takes seconds (ten for a 75 GB Zim) and `Zims.get_all` did that on the event loop; and python-libzim never releases the GIL, so the estimate offloaded with asyncio.to_thread still froze the worker. Everything libzim does for an estimate now runs in a standalone script started as a subprocess (Sanic workers are daemonic, so a multiprocessing pool cannot be forked from them). It reads the Zim paths and search string on stdin and writes the counts on stdout. A Zim it cannot read is reported and left out, as `Zims.get_all` would leave it out. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Zim searches run in the libzim worker process too The /api/zim/search endpoints had the same stall as the estimate: they opened every Zim through Zims.get_all and searched them with libzim, which holds the GIL, so a thread offload still froze the Sanic worker. The estimate script is now a general libzim worker with estimate, search and entries jobs. A search of all Zims sends every Zim's job to one worker process; the API worker only headlines the returned HTML and looks up tags. The worker also returns each Zim's metadata so Zim.__json__ does not open the Zim again on the event loop. Zim.entries_with_tags is split so the tagged paths come from the database without opening the Zim. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Tagged Zim estimates count from the database, and a cancelled request kills the libzim worker Review of the earlier commits found two gaps. Estimates with tags selected still went through Zims.entries_with_tags, which opened every Zim in the API process, and after the thread offload was removed it did so on the event loop; they now count the tagged paths in SQL and ask the worker only for metadata. And Sanic cancels a handler at its response timeout or on disconnect, which left the worker process running until its stdout pipe filled; the runner now kills and reaps it on cancellation. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
443 lines
22 KiB
Python
443 lines
22 KiB
Python
import datetime
|
|
from http import HTTPStatus
|
|
|
|
import pytest
|
|
from libzim import Entry
|
|
|
|
from modules.zim import lib, libzim_worker
|
|
from modules.zim.errors import UnknownZim, UnknownZimEntry
|
|
from modules.zim.models import TagZimEntry, ZimSubscription
|
|
from wrolpi import tags, flags
|
|
from wrolpi.common import DownloadFileInfo, get_wrolpi_config
|
|
from wrolpi.downloader import Download, import_downloads_config, get_download_manager_config
|
|
from wrolpi.files.models import FileGroup
|
|
from wrolpi.test.common import skip_ci
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_zim(async_client, test_session, zim_path_factory, refresh_files):
|
|
zim_path_factory()
|
|
await refresh_files()
|
|
|
|
assert lib.get_zim(test_session, 1)
|
|
|
|
with pytest.raises(UnknownZim):
|
|
assert lib.get_zim(test_session, 2)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_zim_get_entry(async_client, test_session, zim_path_factory, refresh_files):
|
|
zim_path_factory()
|
|
await refresh_files()
|
|
|
|
entry: Entry = lib.get_entry('one', 1)
|
|
assert entry.path == 'one'
|
|
assert entry.title == 'One'
|
|
assert bytes(entry.get_item().content).decode('UTF-8') == '''
|
|
<html>
|
|
<head>
|
|
<title>One</title>
|
|
</head>
|
|
<body>
|
|
<h1>This is the first item</h1>
|
|
</body>
|
|
</html>
|
|
'''
|
|
|
|
entry: Entry = lib.get_entry('home', 1)
|
|
assert entry.path == 'home'
|
|
assert entry.title == 'Homepage'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_zim_bad_entry(async_client, test_session, test_zim, tag_factory):
|
|
tag1 = await tag_factory()
|
|
with pytest.raises(UnknownZimEntry):
|
|
test_zim.tag_entry(test_session, tag1.name, 'path does not exist')
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_zim_get_entries_tags(async_client, test_session, test_zim, tag_factory):
|
|
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
|
|
test_zim.tag_entry(test_session, tag1.name, 'one')
|
|
test_zim.tag_entry(test_session, tag2.name, 'one')
|
|
test_zim.tag_entry(test_session, tag1.name, 'two')
|
|
test_session.commit()
|
|
|
|
assert lib.get_entries_tags(test_session, ['one', 'two', 'home'], 1) \
|
|
== dict(one=['tag1', 'tag2'], two=['tag1'], home=[])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tag_zim_entry_model(async_client, test_session, zim_factory, tag_factory):
|
|
tag = await tag_factory()
|
|
tze = zim_factory('zim1').tag_entry(test_session, tag.name, 'one')
|
|
# Test TagZimEntry.__repr__
|
|
assert repr(tze) == "<TagZimEntry tag='one' zim=zim1 zim_entry=one>"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_zim_entries_with_tags(async_client, test_session, zim_factory, tag_factory):
|
|
"""Can fetch the Entries of a specific Zim that have Tags."""
|
|
tag1, tag2 = await tag_factory(), await tag_factory()
|
|
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
|
|
# Tag Zim1 with both tags. Zim2 shares one of the tags, but its Entry is not returned for Zim1.
|
|
zim1.tag_entry(test_session, tag1.name, 'one')
|
|
zim1.tag_entry(test_session, tag2.name, 'two')
|
|
zim2.tag_entry(test_session, tag2.name, 'home')
|
|
test_session.commit()
|
|
|
|
# Get entries from zim1 only.
|
|
entries = zim1.entries_with_tags(test_session, [tag1.name, ])
|
|
assert len(entries) == 1
|
|
assert entries[0].path == 'one'
|
|
entries = zim1.entries_with_tags(test_session, [tag2.name, ])
|
|
assert len(entries) == 1
|
|
assert entries[0].path == 'two'
|
|
|
|
|
|
def test_unique_paths():
|
|
paths = ['A/Prickly_Pear', 'A/Prickly_pear', 'A/Prickly-pear', 'A/Arborescent_Prickly_Pear',
|
|
'A/Arborescent_prickly_pear', 'A/Beavertail_prickly_pear', 'A/Brittle_prickly-pear',
|
|
'A/Chenille_prickly-pear', 'A/Coastal_Prickly_Pear', 'A/Coastal_prickly_pear',
|
|
'A/Prickly_pears_in_South_Africa', 'A/Prickly_Pear_Cays', 'A/Prickly_pears_in_Australia', 'A/Opuntia',
|
|
'A/Prickly_Pear_(British_Virgin_Islands)', 'A/Prickly_Pear_Island', 'A/Prickly_Pears_(film)',
|
|
'A/Invasive_succulent_plants_in_South_Africa', 'A/Cactoblastis_Memorial_Hall', 'A/Thomas_Harvey_Johnston']
|
|
assert libzim_worker.unique_paths(*paths) == ['A/Prickly_Pear',
|
|
'A/Arborescent_Prickly_Pear',
|
|
'A/Beavertail_prickly_pear',
|
|
'A/Brittle_prickly-pear',
|
|
'A/Chenille_prickly-pear',
|
|
'A/Coastal_Prickly_Pear',
|
|
'A/Prickly_pears_in_South_Africa',
|
|
'A/Prickly_Pear_Cays',
|
|
'A/Prickly_pears_in_Australia',
|
|
'A/Opuntia',
|
|
'A/Prickly_Pear_(British_Virgin_Islands)',
|
|
'A/Prickly_Pear_Island',
|
|
'A/Prickly_Pears_(film)',
|
|
'A/Invasive_succulent_plants_in_South_Africa',
|
|
'A/Cactoblastis_Memorial_Hall',
|
|
'A/Thomas_Harvey_Johnston']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_zim_tags_config(async_client, test_session, test_directory, test_zim, tag_factory, test_tags_config,
|
|
fake_now, await_switches):
|
|
fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1))
|
|
config = tags.get_tags_config()
|
|
assert not config.tag_zims
|
|
|
|
# Tag three Zim entries.
|
|
tag1, tag2 = await tag_factory('Tag1'), await tag_factory('Tag2')
|
|
test_zim.tag_entry(test_session, tag1.name, 'one')
|
|
test_zim.tag_entry(test_session, tag2.name, 'one')
|
|
test_zim.tag_entry(test_session, tag1.name, 'two')
|
|
test_session.commit()
|
|
await await_switches()
|
|
# Tags are created in empty database.
|
|
assert {i[0] for i in test_session.query(tags.Tag.id)} == {1, 2}
|
|
|
|
config = tags.get_tags_config()
|
|
assert config.tag_zims == [
|
|
['Tag1', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
|
|
['Tag2', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
|
|
['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
|
|
]
|
|
|
|
# Remove a tag, the config should change.
|
|
test_zim.untag_entry(test_session, tag1.name, 'one')
|
|
test_session.commit()
|
|
await await_switches()
|
|
|
|
config = tags.get_tags_config()
|
|
assert config.tag_zims == [
|
|
['Tag2', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
|
|
['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
|
|
]
|
|
|
|
# Delete all TagZimEntry(s) first (due to FK constraint), then Tags so they are recreated.
|
|
test_session.query(TagZimEntry).delete()
|
|
test_session.query(tags.Tag).delete()
|
|
test_session.commit()
|
|
|
|
tags.import_tags_config()
|
|
|
|
tag_zim_entries = test_session.query(TagZimEntry) \
|
|
.order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry)
|
|
assert [(i.tag.name, i.zim.path, i.zim_entry) for i in tag_zim_entries] \
|
|
== [('Tag2', test_zim.path, 'one'), ('Tag1', test_zim.path, 'two')]
|
|
|
|
# Tags were recreated. (SQLite reuses row ids after DELETE, so assert by name rather than by id.)
|
|
assert {i.name for i in test_session.query(tags.Tag)} == {'Tag1', 'Tag2'}
|
|
|
|
|
|
@skip_ci
|
|
@pytest.mark.asyncio
|
|
async def test_zim_download(test_session, kiwix_download_zim, test_directory, test_zim_bytes, flags_lock,
|
|
await_switches):
|
|
await lib.subscribe(test_session, 'Wikipedia (with images)', 'es')
|
|
|
|
# Downloading the catalog should lead to a new Zim file being downloaded.
|
|
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2023-06.zim')
|
|
recurring_download, once_download = test_session.query(Download).order_by(Download.id).all()
|
|
recurring_download: Download
|
|
once_download: Download
|
|
assert recurring_download.attempts == 1 and recurring_download.status == 'complete', \
|
|
'Recurring download should have succeeded'
|
|
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').is_file()
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').stat().st_size
|
|
assert once_download.attempts == 1
|
|
# All files downloaded were indexed and are Zim files.
|
|
assert all(i.model == 'zim' and i.indexed is True for i in test_session.query(FileGroup))
|
|
|
|
# Download catalog again, should not re-download the existing file.
|
|
recurring_download.status = 'new'
|
|
test_session.commit()
|
|
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2023-06.zim',
|
|
download_file_side_effect=Exception('should not be called twice'))
|
|
recurring_download, once_download = test_session.query(Download).order_by(Download.id).all()
|
|
assert recurring_download.attempts == 2 and recurring_download.status == 'complete', \
|
|
'Recurring download should have happened again'
|
|
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
|
|
# New file was modeled into a FileGroup.
|
|
assert test_session.query(FileGroup).count() == 1
|
|
# No outdated files exist.
|
|
assert not flags.outdated_zims.is_set()
|
|
|
|
# Download catalog again, should fetch the latest file.
|
|
recurring_download.status = 'new'
|
|
test_session.commit()
|
|
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2024-01.zim',
|
|
download_info=DownloadFileInfo(
|
|
status=HTTPStatus.OK,
|
|
name='wikipedia_es_all_maxi_2024-01.zim',
|
|
size=len(test_zim_bytes),
|
|
),
|
|
hrefs=['?C=N;O=D', '?C=M;O=A', '?C=S;O=A', '?C=D;O=A', '/zim/',
|
|
'wikipedia_es_all_maxi_2023-05.zim',
|
|
'wikipedia_es_all_maxi_2023-06.zim',
|
|
'wikipedia_es_all_maxi_2024-01.zim', # The new Zim.
|
|
'wikipedia_es_all_nopic_2023-05.zim',
|
|
'wikipedia_es_all_nopic_2023-06.zim', ]
|
|
)
|
|
recurring_download, first_download, second_download = test_session.query(Download).order_by(Download.id).all()
|
|
assert recurring_download.attempts == 3 and recurring_download.status == 'complete', \
|
|
'Recurring download should have happened again'
|
|
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').is_file()
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').stat().st_size
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2024-01.zim').is_file()
|
|
assert (test_directory / 'zims/wikipedia_es_all_maxi_2024-01.zim').stat().st_size
|
|
# New file was modeled.
|
|
assert test_session.query(FileGroup).count() == 2
|
|
# But old file is now outdated.
|
|
assert flags.outdated_zims.is_set()
|
|
|
|
# All files downloaded were indexed and are Zim files.
|
|
assert all(i.model == 'zim' and i.indexed is True for i in test_session.query(FileGroup))
|
|
|
|
# Delete the outdated file.
|
|
await lib.remove_outdated_zim_files()
|
|
assert test_session.query(FileGroup).count() == 1
|
|
# No longer outdated
|
|
assert not flags.outdated_zims.is_set()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_zim_tag_migration(await_switches, test_session, test_directory, zim_factory, tag_factory,
|
|
test_tags_config):
|
|
"""When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
|
|
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim.
|
|
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')
|
|
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
|
|
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
|
|
# Tag outdated Zim entries.
|
|
zim1.tag_entry(test_session, tag1.name, 'home')
|
|
zim1.tag_entry(test_session, tag2.name, 'one')
|
|
# This should not be migrated.
|
|
zim2.tag_entry(test_session, tag2.name, 'two')
|
|
# Tag the latest Zim as well.
|
|
zim3.tag_entry(test_session, tag2.name, 'one')
|
|
test_session.commit()
|
|
assert test_session.query(TagZimEntry).count() == 4
|
|
await await_switches()
|
|
assert tags.get_tags_config().successful_import is True
|
|
|
|
# All Zims have been tagged.
|
|
tags_config = tags.get_tags_config()
|
|
tags_config.dump_config()
|
|
assert 'wikipedia_en_all_maxi_2020-01.zim' in (config_text := tags_config.get_file().read_text())
|
|
assert 'wikipedia_en_all_maxi_2020-02.zim' in config_text
|
|
assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text
|
|
|
|
# Zim was deleted, as well as it's ZimTagEntry(s).
|
|
zim1.delete()
|
|
tags.import_tags_config()
|
|
|
|
tag_zim_entries = sorted([(i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry)])
|
|
assert tag_zim_entries == [
|
|
('tag1', zim3.path, 'home'),
|
|
('tag2', zim2.path, 'two'),
|
|
('tag2', zim3.path, 'one'), # Both "one" tags were combined.
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_remove_outdated_zim_files_migrates_tags(await_switches, test_session, test_directory, zim_factory,
|
|
tag_factory, test_tags_config):
|
|
"""Deleting outdated Zims moves their entry tags to the newest Zim of the same name.
|
|
|
|
Deleting the Zim cascade-deletes its TagZimEntry rows, and the next tags save writes them out of the config,
|
|
so the tags must be moved before the files are deleted."""
|
|
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # Outdated.
|
|
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim') # Outdated.
|
|
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
|
|
other = zim_factory('wikibooks_en_all_maxi_2020-01.zim') # A different Zim, not outdated.
|
|
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
|
|
zim1.tag_entry(test_session, tag1.name, 'home')
|
|
zim1.tag_entry(test_session, tag2.name, 'one')
|
|
zim2.tag_entry(test_session, tag2.name, 'two')
|
|
zim3.tag_entry(test_session, tag2.name, 'one') # Already on the latest Zim; must not be duplicated.
|
|
other.tag_entry(test_session, tag1.name, 'home')
|
|
test_session.commit()
|
|
zim3_path, other_path = zim3.path, other.path
|
|
await await_switches()
|
|
|
|
assert await lib.remove_outdated_zim_files(test_directory) == 2
|
|
await await_switches()
|
|
|
|
test_session.expire_all()
|
|
tag_zim_entries = sorted((i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry))
|
|
assert tag_zim_entries == [
|
|
('tag1', other_path, 'home'),
|
|
('tag1', zim3_path, 'home'),
|
|
('tag2', zim3_path, 'one'),
|
|
('tag2', zim3_path, 'two'),
|
|
]
|
|
|
|
# The tags config was saved with the migrated tags, so a later import keeps them.
|
|
config_text = tags.get_tags_config().get_file().read_text()
|
|
assert 'wikipedia_en_all_maxi_2020-01.zim' not in config_text
|
|
assert 'wikipedia_en_all_maxi_2020-02.zim' not in config_text
|
|
assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text
|
|
|
|
@pytest.mark.parametrize(
|
|
'url,expected',
|
|
[
|
|
('https://download.kiwix.org/zim/wikipedia/wikipedia_arz_all_maxi_', ('Wikipedia (with images)', 'arz')),
|
|
('https://download.kiwix.org/zim/wikipedia/wikipedia_arz_all_nopic_', ('Wikipedia (no images)', 'arz')),
|
|
('https://download.kiwix.org/zim/wikipedia/wikipedia_ar_all_mini_', ('Wikipedia (mini)', 'ar')),
|
|
('https://download.kiwix.org/zim/wikiversity/wikiversity_zh_all_maxi_', ('Wikiversity (with images)', 'zh')),
|
|
('https://download.kiwix.org/zim/wikiversity/wikiversity_zh_all_nopic_', ('Wikiversity (no images)', 'zh')),
|
|
('https://download.kiwix.org/zim/wiktionary/wiktionary_eo_all_maxi_', ('Wikitionary (with images)', 'eo')),
|
|
('https://download.kiwix.org/zim/wiktionary/wiktionary_eo_all_nopic_', ('Wikitionary (no images)', 'eo')),
|
|
('https://download.kiwix.org/zim/wikibooks/wikibooks_es_all_nopic_', ('Wikibooks (no images)', 'es')),
|
|
('https://download.kiwix.org/zim/wikibooks/wikibooks_es_all_maxi_', ('Wikibooks (with images)', 'es')),
|
|
('https://download.kiwix.org/zim/wikisource/wikisource_bn_all_maxi_', ('Wikisource', 'bn')),
|
|
('https://download.kiwix.org/zim/vikidia/vikidia_ru_all_maxi_', ('Vikidia', 'ru')),
|
|
('https://download.kiwix.org/zim/stack_exchange/raspberrypi.stackexchange.com_en_all_',
|
|
('Raspberry Pi (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/stackoverflow.com_en_all_',
|
|
('Stackoverflow (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/ja.stackoverflow.com_ja_all_',
|
|
('Stackoverflow (Stack Exchange)', 'ja')),
|
|
('https://download.kiwix.org/zim/stack_exchange/superuser.com_en_all_', ('Superuser (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/ifixit/ifixit_nl_all_', ('iFixit', 'nl')),
|
|
('https://download.kiwix.org/zim/gutenberg/gutenberg_ang_all_', ('Gutenberg', 'ang')),
|
|
('https://download.kiwix.org/zim/stack_exchange/ham.stackexchange.com_en_all_',
|
|
('Amateur Radio (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/diy.stackexchange.com_en_all_',
|
|
('DIY (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/electronics.stackexchange.com_en_all_',
|
|
('Electronics (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/unix.stackexchange.com_en_all_',
|
|
('Unix (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/askubuntu.com_en_all_',
|
|
('Ask Ubuntu (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/bicycles.stackexchange.com_en_all_',
|
|
('Bicycles (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/biology.stackexchange.com_en_all_',
|
|
('Biology (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/arduino.stackexchange.com_en_all_',
|
|
('Arduino (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/3dprinting.stackexchange.com_en_all_',
|
|
('3D Printing (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/other/mdwiki_en_all_',
|
|
('MD Wiki (Wiki Project Med Foundation)', 'en')),
|
|
('https://download.kiwix.org/zim/wikipedia/wikipedia_bpy_medicine_',
|
|
('WikiMed Medical Encyclopedia', 'bpy')),
|
|
('https://download.kiwix.org/zim/other/zimgit-post-disaster_en_',
|
|
('Post Disaster Resource Library', 'en')),
|
|
('https://download.kiwix.org/zim/zimit/fas-military-medicine_en_',
|
|
('Military Medicine', 'en')),
|
|
('https://download.kiwix.org/zim/other/wikem_en_all_maxi_',
|
|
('WikEm (Global Emergency Medicine Wiki)', 'en')),
|
|
('https://download.kiwix.org/zim/stack_exchange/health.stackexchange.com_en_all_',
|
|
('Health (Stack Exchange)', 'en')),
|
|
('https://download.kiwix.org/zim/other/khanacademy_en_all_',
|
|
('Khan Academy', 'en')),
|
|
]
|
|
)
|
|
def test_zim_download_url_to_name(url, expected):
|
|
assert lib.zim_download_url_to_name(url) == expected
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_zim_subscription_download_import(async_client, test_session):
|
|
# Subscription creates a ZimSubscription and Download
|
|
await lib.subscribe(test_session, 'Wikisource', 'en')
|
|
# Add a once-download. This should not be associated with a ZimSubscription.
|
|
test_session.add(
|
|
Download(url='https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim', status='complete'))
|
|
test_session.commit()
|
|
download = test_session.query(Download).filter(Download.frequency.isnot(None)).one()
|
|
assert download.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
|
|
|
|
# Save config file synchronously (dump_config() + save()), delete all entries.
|
|
config = get_download_manager_config()
|
|
config.dump_config()
|
|
config.save()
|
|
test_session.query(Download).delete()
|
|
test_session.query(ZimSubscription).delete()
|
|
test_session.commit()
|
|
|
|
for _ in range(1): # Import twice.
|
|
# Importing the config restores the Download and ZimSubscription.
|
|
await import_downloads_config()
|
|
# NULLS LAST because SQLite sorts NULLs first (Postgres sorted them last).
|
|
recurring, once = test_session.query(Download).order_by(Download.frequency.nullslast())
|
|
assert once.url == 'https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim'
|
|
assert test_session.query(Download).count() == 2
|
|
assert recurring.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
|
|
assert recurring.frequency
|
|
subscription: ZimSubscription = test_session.query(ZimSubscription).one()
|
|
assert subscription.download_id == recurring.id
|
|
assert subscription.name == 'Wikisource'
|
|
assert subscription.language == 'en'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_zim_modeler(async_client, test_session, zim_factory, refresh_files):
|
|
"""Zim modeler sets FileGroup title and a_text."""
|
|
zim = zim_factory('the_zim_title.zim')
|
|
assert zim.file_group.indexed is False
|
|
test_session.commit()
|
|
|
|
await refresh_files()
|
|
|
|
fg: FileGroup = test_session.query(FileGroup).one()
|
|
assert fg.indexed is True
|
|
assert fg.title == 'the_zim_title.zim'
|
|
assert fg.a_text == 'the zim title zim'
|
|
assert fg.model == 'zim'
|
|
|
|
|
|
def test_get_custom_zims_directory(async_client, test_directory, test_wrolpi_config):
|
|
"""Custom directory can be used for zims directory."""
|
|
assert lib.get_zim_directory() == (test_directory / 'zims')
|
|
|
|
get_wrolpi_config().zims_destination = 'custom/zims'
|
|
|
|
assert lib.get_zim_directory() == (test_directory / 'custom/zims')
|