From ce53ae30178767f4aa91a137ec695ac3dd122587 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 4 Oct 2026 11:17:03 -0600 Subject: [PATCH] Move entry tags to the current Zim when deleting outdated Zims Deleting a Zim cascade-deletes its TagZimEntry rows, and the next tags save writes them out of the config, so every outdated-Zim delete lost its entry tags. The config-import migration only runs at startup, too late to help. Move each outdated Zim's tags to the current Zim of the same name before deleting, then save the tags config. Co-Authored-By: Claude Opus 5.5 --- modules/zim/lib.py | 43 +++++++++++++++++++++++++++++++++--- modules/zim/test/test_lib.py | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 667d5359..832d0c52 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -15,7 +15,7 @@ from modules.zim import kiwix from modules.zim.errors import UnknownZim, UnknownZimSubscription from modules.zim.kiwix import KIWIX_CATALOG from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription -from wrolpi import flags +from wrolpi import flags, tags from wrolpi.cmd import run_command from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \ register_refresh_cleanup, background_task, get_wrolpi_config, unique_by_predicate, aiohttp_post @@ -576,9 +576,46 @@ def find_outdated_zim_files(path: pathlib.Path = None) -> Tuple[List[pathlib.Pat return sorted(outdated_files), sorted(current_files) +def migrate_outdated_zim_tags(outdated: List[pathlib.Path], current: List[pathlib.Path]) -> int: + """Move the entry tags of each outdated Zim to the current Zim of the same name. + + Must run before the outdated Zims are deleted; deleting a Zim cascade-deletes its TagZimEntry rows. + + Returns the count of TagZimEntry rows moved.""" + current_by_name = {parse_name(i)[0]: i for i in current} + moved = 0 + with get_db_session(commit=True) as session: + for outdated_path in outdated: + current_path = current_by_name.get(parse_name(outdated_path)[0]) + outdated_zim = Zim.get_by_path(session, outdated_path) + current_zim = Zim.get_by_path(session, current_path) if current_path else None + if not outdated_zim or not current_zim: + continue + + for tag_zim_entry in session.query(TagZimEntry).filter_by(zim_id=outdated_zim.id).all(): + if TagZimEntry.get_by_primary_keys(tag_zim_entry.tag_id, current_zim.id, tag_zim_entry.zim_entry, + session): + # The current Zim already has this tag on this entry. + session.delete(tag_zim_entry) + else: + tag_zim_entry.zim_id = current_zim.id + moved += 1 + # A later outdated Zim may have the same tagged entry. + session.flush() + + if moved: + logger.info(f'Migrated {moved} Zim entry tags from outdated Zims') + return moved + + async def remove_outdated_zim_files(path: pathlib.Path = None) -> int: - """Deletes all old Zim files.""" - outdated, _ = find_outdated_zim_files(path) + """Deletes all old Zim files. Their entry tags are moved to the current Zim of the same name.""" + outdated, current = find_outdated_zim_files(path) + if outdated: + migrate_outdated_zim_tags(outdated, current) + # Save the migrated tags; the config is the source of truth. + tags.save_tags_config.activate_switch() + logger.info(f'Deleting {len(outdated)} outdated Zim files: {outdated}') deleted_count = 0 for file in outdated: diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index c6b987dc..60f9df16 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -293,6 +293,46 @@ async def test_zim_tag_migration(await_switches, test_session, test_directory, z ] + +@pytest.mark.asyncio +async def test_remove_outdated_zim_files_migrates_tags(await_switches, test_session, test_directory, zim_factory, + tag_factory, test_tags_config): + """Deleting outdated Zims moves their entry tags to the newest Zim of the same name. + + Deleting the Zim cascade-deletes its TagZimEntry rows, and the next tags save writes them out of the config, + so the tags must be moved before the files are deleted.""" + zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # Outdated. + zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim') # Outdated. + zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim') # The latest Zim. + other = zim_factory('wikibooks_en_all_maxi_2020-01.zim') # A different Zim, not outdated. + tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2') + zim1.tag_entry(test_session, tag1.name, 'home') + zim1.tag_entry(test_session, tag2.name, 'one') + zim2.tag_entry(test_session, tag2.name, 'two') + zim3.tag_entry(test_session, tag2.name, 'one') # Already on the latest Zim; must not be duplicated. + other.tag_entry(test_session, tag1.name, 'home') + test_session.commit() + zim3_path, other_path = zim3.path, other.path + await await_switches() + + assert await lib.remove_outdated_zim_files(test_directory) == 2 + await await_switches() + + test_session.expire_all() + tag_zim_entries = sorted((i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry)) + assert tag_zim_entries == [ + ('tag1', other_path, 'home'), + ('tag1', zim3_path, 'home'), + ('tag2', zim3_path, 'one'), + ('tag2', zim3_path, 'two'), + ] + + # The tags config was saved with the migrated tags, so a later import keeps them. + config_text = tags.get_tags_config().get_file().read_text() + assert 'wikipedia_en_all_maxi_2020-01.zim' not in config_text + assert 'wikipedia_en_all_maxi_2020-02.zim' not in config_text + assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text + @pytest.mark.parametrize( 'url,expected', [