import datetime
from http import HTTPStatus
import pytest
from libzim import Entry
from modules.zim import lib, libzim_worker
from modules.zim.errors import UnknownZim, UnknownZimEntry
from modules.zim.models import TagZimEntry, ZimSubscription
from wrolpi import tags, flags
from wrolpi.common import DownloadFileInfo, get_wrolpi_config
from wrolpi.downloader import Download, import_downloads_config, get_download_manager_config
from wrolpi.files.models import FileGroup
from wrolpi.test.common import skip_ci
@pytest.mark.asyncio
async def test_get_zim(async_client, test_session, zim_path_factory, refresh_files):
zim_path_factory()
await refresh_files()
assert lib.get_zim(test_session, 1)
with pytest.raises(UnknownZim):
assert lib.get_zim(test_session, 2)
@pytest.mark.asyncio
async def test_zim_get_entry(async_client, test_session, zim_path_factory, refresh_files):
zim_path_factory()
await refresh_files()
entry: Entry = lib.get_entry('one', 1)
assert entry.path == 'one'
assert entry.title == 'One'
assert bytes(entry.get_item().content).decode('UTF-8') == '''
One
This is the first item
'''
entry: Entry = lib.get_entry('home', 1)
assert entry.path == 'home'
assert entry.title == 'Homepage'
@pytest.mark.asyncio
async def test_zim_bad_entry(async_client, test_session, test_zim, tag_factory):
tag1 = await tag_factory()
with pytest.raises(UnknownZimEntry):
test_zim.tag_entry(test_session, tag1.name, 'path does not exist')
@pytest.mark.asyncio
async def test_zim_get_entries_tags(async_client, test_session, test_zim, tag_factory):
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
test_zim.tag_entry(test_session, tag1.name, 'one')
test_zim.tag_entry(test_session, tag2.name, 'one')
test_zim.tag_entry(test_session, tag1.name, 'two')
test_session.commit()
assert lib.get_entries_tags(test_session, ['one', 'two', 'home'], 1) \
== dict(one=['tag1', 'tag2'], two=['tag1'], home=[])
@pytest.mark.asyncio
async def test_tag_zim_entry_model(async_client, test_session, zim_factory, tag_factory):
tag = await tag_factory()
tze = zim_factory('zim1').tag_entry(test_session, tag.name, 'one')
# Test TagZimEntry.__repr__
assert repr(tze) == ""
@pytest.mark.asyncio
async def test_zim_entries_with_tags(async_client, test_session, zim_factory, tag_factory):
"""Can fetch the Entries of a specific Zim that have Tags."""
tag1, tag2 = await tag_factory(), await tag_factory()
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
# Tag Zim1 with both tags. Zim2 shares one of the tags, but its Entry is not returned for Zim1.
zim1.tag_entry(test_session, tag1.name, 'one')
zim1.tag_entry(test_session, tag2.name, 'two')
zim2.tag_entry(test_session, tag2.name, 'home')
test_session.commit()
# Get entries from zim1 only.
entries = zim1.entries_with_tags(test_session, [tag1.name, ])
assert len(entries) == 1
assert entries[0].path == 'one'
entries = zim1.entries_with_tags(test_session, [tag2.name, ])
assert len(entries) == 1
assert entries[0].path == 'two'
def test_unique_paths():
paths = ['A/Prickly_Pear', 'A/Prickly_pear', 'A/Prickly-pear', 'A/Arborescent_Prickly_Pear',
'A/Arborescent_prickly_pear', 'A/Beavertail_prickly_pear', 'A/Brittle_prickly-pear',
'A/Chenille_prickly-pear', 'A/Coastal_Prickly_Pear', 'A/Coastal_prickly_pear',
'A/Prickly_pears_in_South_Africa', 'A/Prickly_Pear_Cays', 'A/Prickly_pears_in_Australia', 'A/Opuntia',
'A/Prickly_Pear_(British_Virgin_Islands)', 'A/Prickly_Pear_Island', 'A/Prickly_Pears_(film)',
'A/Invasive_succulent_plants_in_South_Africa', 'A/Cactoblastis_Memorial_Hall', 'A/Thomas_Harvey_Johnston']
assert libzim_worker.unique_paths(*paths) == ['A/Prickly_Pear',
'A/Arborescent_Prickly_Pear',
'A/Beavertail_prickly_pear',
'A/Brittle_prickly-pear',
'A/Chenille_prickly-pear',
'A/Coastal_Prickly_Pear',
'A/Prickly_pears_in_South_Africa',
'A/Prickly_Pear_Cays',
'A/Prickly_pears_in_Australia',
'A/Opuntia',
'A/Prickly_Pear_(British_Virgin_Islands)',
'A/Prickly_Pear_Island',
'A/Prickly_Pears_(film)',
'A/Invasive_succulent_plants_in_South_Africa',
'A/Cactoblastis_Memorial_Hall',
'A/Thomas_Harvey_Johnston']
@pytest.mark.asyncio
async def test_zim_tags_config(async_client, test_session, test_directory, test_zim, tag_factory, test_tags_config,
fake_now, await_switches):
fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1))
config = tags.get_tags_config()
assert not config.tag_zims
# Tag three Zim entries.
tag1, tag2 = await tag_factory('Tag1'), await tag_factory('Tag2')
test_zim.tag_entry(test_session, tag1.name, 'one')
test_zim.tag_entry(test_session, tag2.name, 'one')
test_zim.tag_entry(test_session, tag1.name, 'two')
test_session.commit()
await await_switches()
# Tags are created in empty database.
assert {i[0] for i in test_session.query(tags.Tag.id)} == {1, 2}
config = tags.get_tags_config()
assert config.tag_zims == [
['Tag1', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
['Tag2', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
]
# Remove a tag, the config should change.
test_zim.untag_entry(test_session, tag1.name, 'one')
test_session.commit()
await await_switches()
config = tags.get_tags_config()
assert config.tag_zims == [
['Tag2', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
]
# Delete all TagZimEntry(s) first (due to FK constraint), then Tags so they are recreated.
test_session.query(TagZimEntry).delete()
test_session.query(tags.Tag).delete()
test_session.commit()
tags.import_tags_config()
tag_zim_entries = test_session.query(TagZimEntry) \
.order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry)
assert [(i.tag.name, i.zim.path, i.zim_entry) for i in tag_zim_entries] \
== [('Tag2', test_zim.path, 'one'), ('Tag1', test_zim.path, 'two')]
# Tags were recreated. (SQLite reuses row ids after DELETE, so assert by name rather than by id.)
assert {i.name for i in test_session.query(tags.Tag)} == {'Tag1', 'Tag2'}
@skip_ci
@pytest.mark.asyncio
async def test_zim_download(test_session, kiwix_download_zim, test_directory, test_zim_bytes, flags_lock,
await_switches):
await lib.subscribe(test_session, 'Wikipedia (with images)', 'es')
# Downloading the catalog should lead to a new Zim file being downloaded.
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2023-06.zim')
recurring_download, once_download = test_session.query(Download).order_by(Download.id).all()
recurring_download: Download
once_download: Download
assert recurring_download.attempts == 1 and recurring_download.status == 'complete', \
'Recurring download should have succeeded'
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').is_file()
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').stat().st_size
assert once_download.attempts == 1
# All files downloaded were indexed and are Zim files.
assert all(i.model == 'zim' and i.indexed is True for i in test_session.query(FileGroup))
# Download catalog again, should not re-download the existing file.
recurring_download.status = 'new'
test_session.commit()
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2023-06.zim',
download_file_side_effect=Exception('should not be called twice'))
recurring_download, once_download = test_session.query(Download).order_by(Download.id).all()
assert recurring_download.attempts == 2 and recurring_download.status == 'complete', \
'Recurring download should have happened again'
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
# New file was modeled into a FileGroup.
assert test_session.query(FileGroup).count() == 1
# No outdated files exist.
assert not flags.outdated_zims.is_set()
# Download catalog again, should fetch the latest file.
recurring_download.status = 'new'
test_session.commit()
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2024-01.zim',
download_info=DownloadFileInfo(
status=HTTPStatus.OK,
name='wikipedia_es_all_maxi_2024-01.zim',
size=len(test_zim_bytes),
),
hrefs=['?C=N;O=D', '?C=M;O=A', '?C=S;O=A', '?C=D;O=A', '/zim/',
'wikipedia_es_all_maxi_2023-05.zim',
'wikipedia_es_all_maxi_2023-06.zim',
'wikipedia_es_all_maxi_2024-01.zim', # The new Zim.
'wikipedia_es_all_nopic_2023-05.zim',
'wikipedia_es_all_nopic_2023-06.zim', ]
)
recurring_download, first_download, second_download = test_session.query(Download).order_by(Download.id).all()
assert recurring_download.attempts == 3 and recurring_download.status == 'complete', \
'Recurring download should have happened again'
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').is_file()
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').stat().st_size
assert (test_directory / 'zims/wikipedia_es_all_maxi_2024-01.zim').is_file()
assert (test_directory / 'zims/wikipedia_es_all_maxi_2024-01.zim').stat().st_size
# New file was modeled.
assert test_session.query(FileGroup).count() == 2
# But old file is now outdated.
assert flags.outdated_zims.is_set()
# All files downloaded were indexed and are Zim files.
assert all(i.model == 'zim' and i.indexed is True for i in test_session.query(FileGroup))
# Delete the outdated file.
await lib.remove_outdated_zim_files()
assert test_session.query(FileGroup).count() == 1
# No longer outdated
assert not flags.outdated_zims.is_set()
@pytest.mark.asyncio
async def test_zim_tag_migration(await_switches, test_session, test_directory, zim_factory, tag_factory,
test_tags_config):
"""When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim.
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
# Tag outdated Zim entries.
zim1.tag_entry(test_session, tag1.name, 'home')
zim1.tag_entry(test_session, tag2.name, 'one')
# This should not be migrated.
zim2.tag_entry(test_session, tag2.name, 'two')
# Tag the latest Zim as well.
zim3.tag_entry(test_session, tag2.name, 'one')
test_session.commit()
assert test_session.query(TagZimEntry).count() == 4
await await_switches()
assert tags.get_tags_config().successful_import is True
# All Zims have been tagged.
tags_config = tags.get_tags_config()
tags_config.dump_config()
assert 'wikipedia_en_all_maxi_2020-01.zim' in (config_text := tags_config.get_file().read_text())
assert 'wikipedia_en_all_maxi_2020-02.zim' in config_text
assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text
# Zim was deleted, as well as it's ZimTagEntry(s).
zim1.delete()
tags.import_tags_config()
tag_zim_entries = sorted([(i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry)])
assert tag_zim_entries == [
('tag1', zim3.path, 'home'),
('tag2', zim2.path, 'two'),
('tag2', zim3.path, 'one'), # Both "one" tags were combined.
]
@pytest.mark.asyncio
async def test_remove_outdated_zim_files_migrates_tags(await_switches, test_session, test_directory, zim_factory,
tag_factory, test_tags_config):
"""Deleting outdated Zims moves their entry tags to the newest Zim of the same name.
Deleting the Zim cascade-deletes its TagZimEntry rows, and the next tags save writes them out of the config,
so the tags must be moved before the files are deleted."""
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # Outdated.
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim') # Outdated.
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
other = zim_factory('wikibooks_en_all_maxi_2020-01.zim') # A different Zim, not outdated.
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
zim1.tag_entry(test_session, tag1.name, 'home')
zim1.tag_entry(test_session, tag2.name, 'one')
zim2.tag_entry(test_session, tag2.name, 'two')
zim3.tag_entry(test_session, tag2.name, 'one') # Already on the latest Zim; must not be duplicated.
other.tag_entry(test_session, tag1.name, 'home')
test_session.commit()
zim3_path, other_path = zim3.path, other.path
await await_switches()
assert await lib.remove_outdated_zim_files(test_directory) == 2
await await_switches()
test_session.expire_all()
tag_zim_entries = sorted((i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry))
assert tag_zim_entries == [
('tag1', other_path, 'home'),
('tag1', zim3_path, 'home'),
('tag2', zim3_path, 'one'),
('tag2', zim3_path, 'two'),
]
# The tags config was saved with the migrated tags, so a later import keeps them.
config_text = tags.get_tags_config().get_file().read_text()
assert 'wikipedia_en_all_maxi_2020-01.zim' not in config_text
assert 'wikipedia_en_all_maxi_2020-02.zim' not in config_text
assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text
@pytest.mark.parametrize(
'url,expected',
[
('https://download.kiwix.org/zim/wikipedia/wikipedia_arz_all_maxi_', ('Wikipedia (with images)', 'arz')),
('https://download.kiwix.org/zim/wikipedia/wikipedia_arz_all_nopic_', ('Wikipedia (no images)', 'arz')),
('https://download.kiwix.org/zim/wikipedia/wikipedia_ar_all_mini_', ('Wikipedia (mini)', 'ar')),
('https://download.kiwix.org/zim/wikiversity/wikiversity_zh_all_maxi_', ('Wikiversity (with images)', 'zh')),
('https://download.kiwix.org/zim/wikiversity/wikiversity_zh_all_nopic_', ('Wikiversity (no images)', 'zh')),
('https://download.kiwix.org/zim/wiktionary/wiktionary_eo_all_maxi_', ('Wikitionary (with images)', 'eo')),
('https://download.kiwix.org/zim/wiktionary/wiktionary_eo_all_nopic_', ('Wikitionary (no images)', 'eo')),
('https://download.kiwix.org/zim/wikibooks/wikibooks_es_all_nopic_', ('Wikibooks (no images)', 'es')),
('https://download.kiwix.org/zim/wikibooks/wikibooks_es_all_maxi_', ('Wikibooks (with images)', 'es')),
('https://download.kiwix.org/zim/wikisource/wikisource_bn_all_maxi_', ('Wikisource', 'bn')),
('https://download.kiwix.org/zim/vikidia/vikidia_ru_all_maxi_', ('Vikidia', 'ru')),
('https://download.kiwix.org/zim/stack_exchange/raspberrypi.stackexchange.com_en_all_',
('Raspberry Pi (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/stackoverflow.com_en_all_',
('Stackoverflow (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/ja.stackoverflow.com_ja_all_',
('Stackoverflow (Stack Exchange)', 'ja')),
('https://download.kiwix.org/zim/stack_exchange/superuser.com_en_all_', ('Superuser (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/ifixit/ifixit_nl_all_', ('iFixit', 'nl')),
('https://download.kiwix.org/zim/gutenberg/gutenberg_ang_all_', ('Gutenberg', 'ang')),
('https://download.kiwix.org/zim/stack_exchange/ham.stackexchange.com_en_all_',
('Amateur Radio (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/diy.stackexchange.com_en_all_',
('DIY (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/electronics.stackexchange.com_en_all_',
('Electronics (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/unix.stackexchange.com_en_all_',
('Unix (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/askubuntu.com_en_all_',
('Ask Ubuntu (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/bicycles.stackexchange.com_en_all_',
('Bicycles (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/biology.stackexchange.com_en_all_',
('Biology (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/arduino.stackexchange.com_en_all_',
('Arduino (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/3dprinting.stackexchange.com_en_all_',
('3D Printing (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/other/mdwiki_en_all_',
('MD Wiki (Wiki Project Med Foundation)', 'en')),
('https://download.kiwix.org/zim/wikipedia/wikipedia_bpy_medicine_',
('WikiMed Medical Encyclopedia', 'bpy')),
('https://download.kiwix.org/zim/other/zimgit-post-disaster_en_',
('Post Disaster Resource Library', 'en')),
('https://download.kiwix.org/zim/zimit/fas-military-medicine_en_',
('Military Medicine', 'en')),
('https://download.kiwix.org/zim/other/wikem_en_all_maxi_',
('WikEm (Global Emergency Medicine Wiki)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/health.stackexchange.com_en_all_',
('Health (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/other/khanacademy_en_all_',
('Khan Academy', 'en')),
]
)
def test_zim_download_url_to_name(url, expected):
assert lib.zim_download_url_to_name(url) == expected
@pytest.mark.asyncio
async def test_zim_subscription_download_import(async_client, test_session):
# Subscription creates a ZimSubscription and Download
await lib.subscribe(test_session, 'Wikisource', 'en')
# Add a once-download. This should not be associated with a ZimSubscription.
test_session.add(
Download(url='https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim', status='complete'))
test_session.commit()
download = test_session.query(Download).filter(Download.frequency.isnot(None)).one()
assert download.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
# Save config file synchronously (dump_config() + save()), delete all entries.
config = get_download_manager_config()
config.dump_config()
config.save()
test_session.query(Download).delete()
test_session.query(ZimSubscription).delete()
test_session.commit()
for _ in range(1): # Import twice.
# Importing the config restores the Download and ZimSubscription.
await import_downloads_config()
# NULLS LAST because SQLite sorts NULLs first (Postgres sorted them last).
recurring, once = test_session.query(Download).order_by(Download.frequency.nullslast())
assert once.url == 'https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim'
assert test_session.query(Download).count() == 2
assert recurring.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
assert recurring.frequency
subscription: ZimSubscription = test_session.query(ZimSubscription).one()
assert subscription.download_id == recurring.id
assert subscription.name == 'Wikisource'
assert subscription.language == 'en'
@pytest.mark.asyncio
async def test_zim_modeler(async_client, test_session, zim_factory, refresh_files):
"""Zim modeler sets FileGroup title and a_text."""
zim = zim_factory('the_zim_title.zim')
assert zim.file_group.indexed is False
test_session.commit()
await refresh_files()
fg: FileGroup = test_session.query(FileGroup).one()
assert fg.indexed is True
assert fg.title == 'the_zim_title.zim'
assert fg.a_text == 'the zim title zim'
assert fg.model == 'zim'
def test_get_custom_zims_directory(async_client, test_directory, test_wrolpi_config):
"""Custom directory can be used for zims directory."""
assert lib.get_zim_directory() == (test_directory / 'zims')
get_wrolpi_config().zims_destination = 'custom/zims'
assert lib.get_zim_directory() == (test_directory / 'custom/zims')