diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 667d5359..832d0c52 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -15,7 +15,7 @@ from modules.zim import kiwix from modules.zim.errors import UnknownZim, UnknownZimSubscription from modules.zim.kiwix import KIWIX_CATALOG from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription -from wrolpi import flags +from wrolpi import flags, tags from wrolpi.cmd import run_command from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \ register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate, aiohttp_post @@ -576,9 +576,46 @@ def find_outdated_zim_files(path: pathlib.Path = None) -> Tuple[List[pathlib.Pat return sorted(outdated_files), sorted(current_files) +def migrate_outdated_zim_tags(outdated: List[pathlib.Path], current: List[pathlib.Path]) -> int: + """Move the entry tags of each outdated Zim to the current Zim of the same name. + + Must run before the outdated Zims are deleted; deleting a Zim cascade-deletes its TagZimEntry rows. + + Returns the count of TagZimEntry rows moved.""" + current_by_name = {parse_name(i)[0]: i for i in current} + moved = 0 + with get_db_session(commit=True) as session: + for outdated_path in outdated: + current_path = current_by_name.get(parse_name(outdated_path)[0]) + outdated_zim = Zim.get_by_path(session, outdated_path) + current_zim = Zim.get_by_path(session, current_path) if current_path else None + if not outdated_zim or not current_zim: + continue + + for tag_zim_entry in session.query(TagZimEntry).filter_by(zim_id=outdated_zim.id).all(): + if TagZimEntry.get_by_primary_keys(tag_zim_entry.tag_id, current_zim.id, tag_zim_entry.zim_entry, + session): + # The current Zim already has this tag on this entry. + session.delete(tag_zim_entry) + else: + tag_zim_entry.zim_id = current_zim.id + moved += 1 + # A later outdated Zim may have the same tagged entry. + session.flush() + + if moved: + logger.info(f'Migrated {moved} Zim entry tags from outdated Zims') + return moved + + async def remove_outdated_zim_files(path: pathlib.Path = None) -> int: - """Deletes all old Zim files.""" - outdated, _ = find_outdated_zim_files(path) + """Deletes all old Zim files. Their entry tags are moved to the current Zim of the same name.""" + outdated, current = find_outdated_zim_files(path) + if outdated: + migrate_outdated_zim_tags(outdated, current) + # Save the migrated tags; the config is the source of truth. + tags.save_tags_config.activate_switch() + logger.info(f'Deleting {len(outdated)} outdated Zim files: {outdated}') deleted_count = 0 for file in outdated: diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index c6b987dc..60f9df16 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -293,6 +293,46 @@ async def test_zim_tag_migration(await_switches, test_session, test_directory, z ] + +@pytest.mark.asyncio +async def test_remove_outdated_zim_files_migrates_tags(await_switches, test_session, test_directory, zim_factory, + tag_factory, test_tags_config): + """Deleting outdated Zims moves their entry tags to the newest Zim of the same name. + + Deleting the Zim cascade-deletes its TagZimEntry rows, and the next tags save writes them out of the config, + so the tags must be moved before the files are deleted.""" + zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # Outdated. + zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim') # Outdated. + zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim. + other = zim_factory('wikibooks_en_all_maxi_2020-01.zim') # A different Zim, not outdated. + tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2') + zim1.tag_entry(test_session, tag1.name, 'home') + zim1.tag_entry(test_session, tag2.name, 'one') + zim2.tag_entry(test_session, tag2.name, 'two') + zim3.tag_entry(test_session, tag2.name, 'one') # Already on the latest Zim; must not be duplicated. + other.tag_entry(test_session, tag1.name, 'home') + test_session.commit() + zim3_path, other_path = zim3.path, other.path + await await_switches() + + assert await lib.remove_outdated_zim_files(test_directory) == 2 + await await_switches() + + test_session.expire_all() + tag_zim_entries = sorted((i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry)) + assert tag_zim_entries == [ + ('tag1', other_path, 'home'), + ('tag1', zim3_path, 'home'), + ('tag2', zim3_path, 'one'), + ('tag2', zim3_path, 'two'), + ] + + # The tags config was saved with the migrated tags, so a later import keeps them. + config_text = tags.get_tags_config().get_file().read_text() + assert 'wikipedia_en_all_maxi_2020-01.zim' not in config_text + assert 'wikipedia_en_all_maxi_2020-02.zim' not in config_text + assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text + @pytest.mark.parametrize( 'url,expected', [