rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 19:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions
wrolpi/modules/zim/test/test_lib.py
lrnselfreliance 1f687fa825
Some checks are pending
CodeQL / Analyze (push) Waiting to run
Details
Controller Tests / controller-tests (push) Waiting to run
Details
Jest Tests / jest-tests (push) Waiting to run
Details
Move entry tags to the current Zim when deleting outdated Zims (#776) 
Deleting a Zim cascade-deletes its TagZimEntry rows, and the next tags
save writes them out of the config, so every outdated-Zim delete lost
its entry tags.  The config-import migration only runs at startup, too
late to help.  Move each outdated Zim's tags to the current Zim of the
same name before deleting, then save the tags config.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 18:04:11 -06:00

465 lines
23 KiB
Python
Raw Blame History

import datetime
from http import HTTPStatus
import pytest
from libzim import Entry
from modules.zim import lib
from modules.zim.errors import UnknownZim, UnknownZimEntry
from modules.zim.models import TagZimEntry, ZimSubscription
from wrolpi import tags, flags
from wrolpi.common import DownloadFileInfo, get_wrolpi_config
from wrolpi.downloader import Download, import_downloads_config, get_download_manager_config
from wrolpi.files.models import FileGroup
from wrolpi.test.common import skip_circleci
@pytest.mark.asyncio
async def test_get_zim(async_client, test_session, zim_path_factory, refresh_files):
zim_path_factory()
await refresh_files()
assert lib.get_zim(test_session, 1)
with pytest.raises(UnknownZim):
assert lib.get_zim(test_session, 2)
@pytest.mark.asyncio
async def test_zim_get_entry(async_client, test_session, zim_path_factory, refresh_files):
zim_path_factory()
await refresh_files()
entry: Entry = lib.get_entry('one', 1)
assert entry.path == 'one'
assert entry.title == 'One'
assert bytes(entry.get_item().content).decode('UTF-8') == '''
<html>
<head>
<title>One</title>
</head>
<body>
<h1>This is the first item</h1>
</body>
</html>
'''
entry: Entry = lib.get_entry('home', 1)
assert entry.path == 'home'
assert entry.title == 'Homepage'
@pytest.mark.asyncio
async def test_zim_bad_entry(async_client, test_session, test_zim, tag_factory):
tag1 = await tag_factory()
with pytest.raises(UnknownZimEntry):
test_zim.tag_entry(test_session, tag1.name, 'path does not exist')
@pytest.mark.asyncio
async def test_zim_get_entries_tags(async_client, test_session, test_zim, tag_factory):
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
test_zim.tag_entry(test_session, tag1.name, 'one')
test_zim.tag_entry(test_session, tag2.name, 'one')
test_zim.tag_entry(test_session, tag1.name, 'two')
test_session.commit()
assert lib.get_entries_tags(test_session, ['one', 'two', 'home'], 1) \
== dict(one=['tag1', 'tag2'], two=['tag1'], home=[])
@pytest.mark.asyncio
async def test_tag_zim_entry_model(async_client, test_session, zim_factory, tag_factory):
tag = await tag_factory()
tze = zim_factory('zim1').tag_entry(test_session, tag.name, 'one')
# Test TagZimEntry.__repr__
assert repr(tze) == "<TagZimEntry tag='one' zim=zim1 zim_entry=one>"
@pytest.mark.asyncio
async def test_zim_entries_with_tags(async_client, test_session, zim_factory, tag_factory):
"""Can fetch Zim Entries that have Tags. Can filter only those entries that are tagged on a specific Zim."""
tag1, tag2 = await tag_factory(), await tag_factory()
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
# Tag Zim1 with both tags, but Zim2 shares one of the tags and is returned when no Zim is passed.
zim1.tag_entry(test_session, tag1.name, 'one')
zim1.tag_entry(test_session, tag2.name, 'two')
zim2.tag_entry(test_session, tag2.name, 'home')
test_session.commit()
# Get entries from zim1 only.
entries = zim1.entries_with_tags(test_session, [tag1.name, ])
assert len(entries) == 1
assert entries[0].path == 'one'
entries = zim1.entries_with_tags(test_session, [tag2.name, ])
assert len(entries) == 1
assert entries[0].path == 'two'
# Get entries from any Zim.
entries = lib.Zim.all_entries_with_tags(test_session, [tag2.name, ])
assert {i.path for i in entries} == {'two', 'home'}, 'Tag2 was applied to `two` and `home`'
def test_get_unique_paths():
paths = ['A/Prickly_Pear', 'A/Prickly_pear', 'A/Prickly-pear', 'A/Arborescent_Prickly_Pear',
'A/Arborescent_prickly_pear', 'A/Beavertail_prickly_pear', 'A/Brittle_prickly-pear',
'A/Chenille_prickly-pear', 'A/Coastal_Prickly_Pear', 'A/Coastal_prickly_pear',
'A/Prickly_pears_in_South_Africa', 'A/Prickly_Pear_Cays', 'A/Prickly_pears_in_Australia', 'A/Opuntia',
'A/Prickly_Pear_(British_Virgin_Islands)', 'A/Prickly_Pear_Island', 'A/Prickly_Pears_(film)',
'A/Invasive_succulent_plants_in_South_Africa', 'A/Cactoblastis_Memorial_Hall', 'A/Thomas_Harvey_Johnston']
assert lib.get_unique_paths(*paths) == ['A/Prickly_Pear',
'A/Arborescent_Prickly_Pear',
'A/Beavertail_prickly_pear',
'A/Brittle_prickly-pear',
'A/Chenille_prickly-pear',
'A/Coastal_Prickly_Pear',
'A/Prickly_pears_in_South_Africa',
'A/Prickly_Pear_Cays',
'A/Prickly_pears_in_Australia',
'A/Opuntia',
'A/Prickly_Pear_(British_Virgin_Islands)',
'A/Prickly_Pear_Island',
'A/Prickly_Pears_(film)',
'A/Invasive_succulent_plants_in_South_Africa',
'A/Cactoblastis_Memorial_Hall',
'A/Thomas_Harvey_Johnston']
@pytest.mark.asyncio
async def test_zim_tags_config(async_client, test_session, test_directory, test_zim, tag_factory, test_tags_config,
fake_now, await_switches):
fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1))
config = tags.get_tags_config()
assert not config.tag_zims
# Tag three Zim entries.
tag1, tag2 = await tag_factory('Tag1'), await tag_factory('Tag2')
test_zim.tag_entry(test_session, tag1.name, 'one')
test_zim.tag_entry(test_session, tag2.name, 'one')
test_zim.tag_entry(test_session, tag1.name, 'two')
test_session.commit()
await await_switches()
# Tags are created in empty database.
assert {i[0] for i in test_session.query(tags.Tag.id)} == {1, 2}
config = tags.get_tags_config()
assert config.tag_zims == [
['Tag1', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
['Tag2', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
]
# Remove a tag, the config should change.
test_zim.untag_entry(test_session, tag1.name, 'one')
test_session.commit()
await await_switches()
config = tags.get_tags_config()
assert config.tag_zims == [
['Tag2', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
]
# Delete all TagZimEntry(s) first (due to FK constraint), then Tags so they are recreated.
test_session.query(TagZimEntry).delete()
test_session.query(tags.Tag).delete()
test_session.commit()
tags.import_tags_config()
tag_zim_entries = test_session.query(TagZimEntry) \
.order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry)
assert [(i.tag.name, i.zim.path, i.zim_entry) for i in tag_zim_entries] \
== [('Tag2', test_zim.path, 'one'), ('Tag1', test_zim.path, 'two')]
# Tags were recreated. (SQLite reuses row ids after DELETE, so assert by name rather than by id.)
assert {i.name for i in test_session.query(tags.Tag)} == {'Tag1', 'Tag2'}
def test_zim_all_entries(test_session, test_zim):
entries = [i.__json__() for i in test_zim.all_entries]
assert len(entries) == 12
@skip_circleci
@pytest.mark.asyncio
async def test_zim_download(test_session, kiwix_download_zim, test_directory, test_zim_bytes, flags_lock,
await_switches):
await lib.subscribe(test_session, 'Wikipedia (with images)', 'es')
# Downloading the catalog should lead to a new Zim file being downloaded.
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2023-06.zim')
recurring_download, once_download = test_session.query(Download).order_by(Download.id).all()
recurring_download: Download
once_download: Download
assert recurring_download.attempts == 1 and recurring_download.status == 'complete', \
'Recurring download should have succeeded'
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').is_file()
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').stat().st_size
assert once_download.attempts == 1
# All files downloaded were indexed and are Zim files.
assert all(i.model == 'zim' and i.indexed is True for i in test_session.query(FileGroup))
# Download catalog again, should not re-download the existing file.
recurring_download.status = 'new'
test_session.commit()
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2023-06.zim',
download_file_side_effect=Exception('should not be called twice'))
recurring_download, once_download = test_session.query(Download).order_by(Download.id).all()
assert recurring_download.attempts == 2 and recurring_download.status == 'complete', \
'Recurring download should have happened again'
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
# New file was modeled into a FileGroup.
assert test_session.query(FileGroup).count() == 1
# No outdated files exist.
assert not flags.outdated_zims.is_set()
# Download catalog again, should fetch the latest file.
recurring_download.status = 'new'
test_session.commit()
await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2024-01.zim',
download_info=DownloadFileInfo(
status=HTTPStatus.OK,
name='wikipedia_es_all_maxi_2024-01.zim',
size=len(test_zim_bytes),
),
hrefs=['?C=N;O=D', '?C=M;O=A', '?C=S;O=A', '?C=D;O=A', '/zim/',
'wikipedia_es_all_maxi_2023-05.zim',
'wikipedia_es_all_maxi_2023-06.zim',
'wikipedia_es_all_maxi_2024-01.zim', # The new Zim.
'wikipedia_es_all_nopic_2023-05.zim',
'wikipedia_es_all_nopic_2023-06.zim', ]
)
recurring_download, first_download, second_download = test_session.query(Download).order_by(Download.id).all()
assert recurring_download.attempts == 3 and recurring_download.status == 'complete', \
'Recurring download should have happened again'
assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').is_file()
assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').stat().st_size
assert (test_directory / 'zims/wikipedia_es_all_maxi_2024-01.zim').is_file()
assert (test_directory / 'zims/wikipedia_es_all_maxi_2024-01.zim').stat().st_size
# New file was modeled.
assert test_session.query(FileGroup).count() == 2
# But old file is now outdated.
assert flags.outdated_zims.is_set()
# All files downloaded were indexed and are Zim files.
assert all(i.model == 'zim' and i.indexed is True for i in test_session.query(FileGroup))
# Delete the outdated file.
await lib.remove_outdated_zim_files()
assert test_session.query(FileGroup).count() == 1
# No longer outdated
assert not flags.outdated_zims.is_set()
@pytest.mark.asyncio
async def test_zim_tag_migration(await_switches, test_session, test_directory, zim_factory, tag_factory,
test_tags_config):
"""When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim.
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
# Tag outdated Zim entries.
zim1.tag_entry(test_session, tag1.name, 'home')
zim1.tag_entry(test_session, tag2.name, 'one')
# This should not be migrated.
zim2.tag_entry(test_session, tag2.name, 'two')
# Tag the latest Zim as well.
zim3.tag_entry(test_session, tag2.name, 'one')
test_session.commit()
assert test_session.query(TagZimEntry).count() == 4
await await_switches()
assert tags.get_tags_config().successful_import is True
# All Zims have been tagged.
tags_config = tags.get_tags_config()
tags_config.dump_config()
assert 'wikipedia_en_all_maxi_2020-01.zim' in (config_text := tags_config.get_file().read_text())
assert 'wikipedia_en_all_maxi_2020-02.zim' in config_text
assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text
# Zim was deleted, as well as it's ZimTagEntry(s).
zim1.delete()
tags.import_tags_config()
tag_zim_entries = sorted([(i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry)])
assert tag_zim_entries == [
('tag1', zim3.path, 'home'),
('tag2', zim2.path, 'two'),
('tag2', zim3.path, 'one'), # Both "one" tags were combined.
]
@pytest.mark.asyncio
async def test_remove_outdated_zim_files_migrates_tags(await_switches, test_session, test_directory, zim_factory,
tag_factory, test_tags_config):
"""Deleting outdated Zims moves their entry tags to the newest Zim of the same name.
Deleting the Zim cascade-deletes its TagZimEntry rows, and the next tags save writes them out of the config,
so the tags must be moved before the files are deleted."""
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # Outdated.
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim') # Outdated.
zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim.
other = zim_factory('wikibooks_en_all_maxi_2020-01.zim') # A different Zim, not outdated.
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
zim1.tag_entry(test_session, tag1.name, 'home')
zim1.tag_entry(test_session, tag2.name, 'one')
zim2.tag_entry(test_session, tag2.name, 'two')
zim3.tag_entry(test_session, tag2.name, 'one') # Already on the latest Zim; must not be duplicated.
other.tag_entry(test_session, tag1.name, 'home')
test_session.commit()
zim3_path, other_path = zim3.path, other.path
await await_switches()
assert await lib.remove_outdated_zim_files(test_directory) == 2
await await_switches()
test_session.expire_all()
tag_zim_entries = sorted((i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry))
assert tag_zim_entries == [
('tag1', other_path, 'home'),
('tag1', zim3_path, 'home'),
('tag2', zim3_path, 'one'),
('tag2', zim3_path, 'two'),
]
# The tags config was saved with the migrated tags, so a later import keeps them.
config_text = tags.get_tags_config().get_file().read_text()
assert 'wikipedia_en_all_maxi_2020-01.zim' not in config_text
assert 'wikipedia_en_all_maxi_2020-02.zim' not in config_text
assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text
@pytest.mark.parametrize(
'url,expected',
[
('https://download.kiwix.org/zim/wikipedia/wikipedia_arz_all_maxi_', ('Wikipedia (with images)', 'arz')),
('https://download.kiwix.org/zim/wikipedia/wikipedia_arz_all_nopic_', ('Wikipedia (no images)', 'arz')),
('https://download.kiwix.org/zim/wikipedia/wikipedia_ar_all_mini_', ('Wikipedia (mini)', 'ar')),
('https://download.kiwix.org/zim/wikiversity/wikiversity_zh_all_maxi_', ('Wikiversity (with images)', 'zh')),
('https://download.kiwix.org/zim/wikiversity/wikiversity_zh_all_nopic_', ('Wikiversity (no images)', 'zh')),
('https://download.kiwix.org/zim/wiktionary/wiktionary_eo_all_maxi_', ('Wikitionary (with images)', 'eo')),
('https://download.kiwix.org/zim/wiktionary/wiktionary_eo_all_nopic_', ('Wikitionary (no images)', 'eo')),
('https://download.kiwix.org/zim/wikibooks/wikibooks_es_all_nopic_', ('Wikibooks (no images)', 'es')),
('https://download.kiwix.org/zim/wikibooks/wikibooks_es_all_maxi_', ('Wikibooks (with images)', 'es')),
('https://download.kiwix.org/zim/wikisource/wikisource_bn_all_maxi_', ('Wikisource', 'bn')),
('https://download.kiwix.org/zim/vikidia/vikidia_ru_all_maxi_', ('Vikidia', 'ru')),
('https://download.kiwix.org/zim/stack_exchange/raspberrypi.stackexchange.com_en_all_',
('Raspberry Pi (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/stackoverflow.com_en_all_',
('Stackoverflow (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/ja.stackoverflow.com_ja_all_',
('Stackoverflow (Stack Exchange)', 'ja')),
('https://download.kiwix.org/zim/stack_exchange/superuser.com_en_all_', ('Superuser (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/ifixit/ifixit_nl_all_', ('iFixit', 'nl')),
('https://download.kiwix.org/zim/gutenberg/gutenberg_ang_all_', ('Gutenberg', 'ang')),
('https://download.kiwix.org/zim/stack_exchange/ham.stackexchange.com_en_all_',
('Amateur Radio (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/diy.stackexchange.com_en_all_',
('DIY (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/electronics.stackexchange.com_en_all_',
('Electronics (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/unix.stackexchange.com_en_all_',
('Unix (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/askubuntu.com_en_all_',
('Ask Ubuntu (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/bicycles.stackexchange.com_en_all_',
('Bicycles (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/biology.stackexchange.com_en_all_',
('Biology (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/arduino.stackexchange.com_en_all_',
('Arduino (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/3dprinting.stackexchange.com_en_all_',
('3D Printing (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/other/mdwiki_en_all_',
('MD Wiki (Wiki Project Med Foundation)', 'en')),
('https://download.kiwix.org/zim/wikipedia/wikipedia_bpy_medicine_',
('WikiMed Medical Encyclopedia', 'bpy')),
('https://download.kiwix.org/zim/other/zimgit-post-disaster_en_',
('Post Disaster Resource Library', 'en')),
('https://download.kiwix.org/zim/zimit/fas-military-medicine_en_',
('Military Medicine', 'en')),
('https://download.kiwix.org/zim/other/wikem_en_all_maxi_',
('WikEm (Global Emergency Medicine Wiki)', 'en')),
('https://download.kiwix.org/zim/stack_exchange/health.stackexchange.com_en_all_',
('Health (Stack Exchange)', 'en')),
('https://download.kiwix.org/zim/other/khanacademy_en_all_',
('Khan Academy', 'en')),
]
)
def test_zim_download_url_to_name(url, expected):
assert lib.zim_download_url_to_name(url) == expected
@pytest.mark.asyncio
async def test_zim_subscription_download_import(async_client, test_session):
# Subscription creates a ZimSubscription and Download
await lib.subscribe(test_session, 'Wikisource', 'en')
# Add a once-download. This should not be associated with a ZimSubscription.
test_session.add(
Download(url='https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim', status='complete'))
test_session.commit()
download = test_session.query(Download).filter(Download.frequency.isnot(None)).one()
assert download.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
# Save config file synchronously (dump_config() + save()), delete all entries.
config = get_download_manager_config()
config.dump_config()
config.save()
test_session.query(Download).delete()
test_session.query(ZimSubscription).delete()
test_session.commit()
for _ in range(1): # Import twice.
# Importing the config restores the Download and ZimSubscription.
await import_downloads_config()
# NULLS LAST because SQLite sorts NULLs first (Postgres sorted them last).
recurring, once = test_session.query(Download).order_by(Download.frequency.nullslast())
assert once.url == 'https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim'
assert test_session.query(Download).count() == 2
assert recurring.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
assert recurring.frequency
subscription: ZimSubscription = test_session.query(ZimSubscription).one()
assert subscription.download_id == recurring.id
assert subscription.name == 'Wikisource'
assert subscription.language == 'en'
@pytest.mark.asyncio
async def test_zim_modeler(async_client, test_session, zim_factory, refresh_files):
"""Zim modeler sets FileGroup title and a_text."""
zim = zim_factory('the_zim_title.zim')
assert zim.file_group.indexed is False
test_session.commit()
await refresh_files()
fg: FileGroup = test_session.query(FileGroup).one()
assert fg.indexed is True
assert fg.title == 'the_zim_title.zim'
assert fg.a_text == 'the zim title zim'
assert fg.model == 'zim'
def test_get_custom_zims_directory(async_client, test_directory, test_wrolpi_config):
"""Custom directory can be used for zims directory."""
assert lib.get_zim_directory() == (test_directory / 'zims')
get_wrolpi_config().zims_destination = 'custom/zims'
assert lib.get_zim_directory() == (test_directory / 'custom/zims')
def test_get_estimates_is_cached():
"""Repeated estimates for the same search string do not re-query the Zims."""
from unittest import mock
from modules.zim import lib
lib.get_estimates.cache_clear()
with mock.patch.object(lib.Zims, 'get_all', return_value=[]) as get_all:
assert lib.get_estimates('cached search') == []
assert lib.get_estimates('cached search') == []
assert get_all.call_count == 1
Reference in a new issue View git blame Copy permalink