From 2c595e6db0ca1779694abf95fe254d0db009341d Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 19 Jul 2026 13:19:30 -0600 Subject: [PATCH] Prevent the zims directory from being ignored by file refresh. Zim Manage and search depend on FileGroup modeling of files under zims/. Ignoring that directory left Kiwix serving files on disk while /api/zim stayed empty. Refuse to ignore zims (alongside other special dirs), strip it from existing configs on settings load, and share normalized ignore handling between refresh and upsert. --- wrolpi/files/lib.py | 120 +++++++++++++++++++++++++++------- wrolpi/files/test/test_api.py | 48 ++++++++++++++ wrolpi/files/worker.py | 15 +---- wrolpi/root_api.py | 4 ++ 4 files changed, 152 insertions(+), 35 deletions(-) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index b2fa0c12..d6f93f0e 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -56,7 +56,8 @@ __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics', 'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem', 'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href', - 'get_tagged_file_groups_by_ids', 'delete_file_groups'] + 'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories', + 'get_normalized_ignored_directories', 'sanitize_ignored_directories'] def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]: @@ -786,11 +787,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime, def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]: """Return a new list which does not contain any file paths that are in ignored directories.""" - ignored_directories = list(map(str, get_wrolpi_config().ignored_directories)) - for idx, ignored_directory in enumerate(ignored_directories): - ignored_directory = pathlib.Path(ignored_directory) - if not ignored_directory.is_absolute(): - ignored_directories[idx] = str(get_media_directory() / ignored_directory) + ignored_directories = get_normalized_ignored_directories() files = [i for i in files if not any(str(i).startswith(j) for j in ignored_directories)] return files @@ -1746,23 +1743,84 @@ async def rename(session: Session, path: pathlib.Path, new_name: str, send_event return await rename_file(path, new_name) +def get_special_directories() -> List[pathlib.Path]: + """Return absolute directories that must never be ignored by file refresh. + + These are module destinations that depend on FileGroup indexing (or are the media root). + Map is intentionally excluded: it lists PMTiles by walking the filesystem and benefits + from being ignored during refresh (very large files). + """ + from modules.videos.common import get_videos_directory, get_no_channel_directory + from modules.archive.lib import get_archive_directory + + media_directory = get_media_directory() + # Avoid importing modules.zim.lib here (circular import via files.worker). + zims_directory = media_directory / get_wrolpi_config().zims_destination + return [ + media_directory.resolve(), + get_videos_directory().resolve(), + get_archive_directory().resolve(), + get_no_channel_directory().resolve(), + zims_directory.resolve(), + ] + + +def get_normalized_ignored_directories() -> List[str]: + """Return absolute ignored directory paths, excluding special directories. + + Special directories (media root, videos, archives, zims, …) are never treated as ignored, + even if they appear in config from older installs or manual edits. + """ + media_directory = get_media_directory() + special = {str(p) for p in get_special_directories()} + result = [] + for d in get_wrolpi_config().ignored_directories or []: + path = pathlib.Path(d) + absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve()) + if absolute in special: + continue + result.append(absolute) + return result + + +def sanitize_ignored_directories(): + """Persistently remove special directories from the ignored list if present.""" + wrolpi_config = get_wrolpi_config() + ignored = list(wrolpi_config.ignored_directories or []) + if not ignored: + return + + media_directory = get_media_directory() + special = {str(p) for p in get_special_directories()} + cleaned = [] + removed = False + for d in ignored: + path = pathlib.Path(d) + absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve()) + if absolute in special: + removed = True + logger.warning(f'Removing special directory from ignored_directories: {d}') + continue + cleaned.append(d) + + if removed: + wrolpi_config.ignored_directories = cleaned + + def add_ignore_directory(directory: Union[pathlib.Path, str]): """Add a directory to the `ignored_directories` in the WROLPi config. This directory will be ignored when refreshing.""" - media_directory = get_media_directory() + media_directory = get_media_directory().resolve() directory = pathlib.Path(directory) + if not directory.is_absolute(): + directory = media_directory / directory + directory = directory.resolve() - from modules.videos.common import get_videos_directory - from modules.archive.lib import get_archive_directory - from modules.videos.common import get_no_channel_directory - special_directories = [ - get_media_directory(), - get_videos_directory(), - get_archive_directory(), - get_no_channel_directory(), - ] + # Drop any special directories that were previously saved as ignored. + sanitize_ignored_directories() + special_directories = set(get_special_directories()) if directory in special_directories: raise InvalidDirectory('Refusing to ignore special directory') @@ -1772,7 +1830,9 @@ def add_ignore_directory(directory: Union[pathlib.Path, str]): wrolpi_config = get_wrolpi_config() ignored_directories = wrolpi_config.ignored_directories if wrolpi_config.ignored_directories else list() - if str(directory) in ignored_directories: + # Accept either absolute or relative forms already present in config. + relative = str(get_relative_to_media_directory(directory)) + if str(directory) in ignored_directories or relative in ignored_directories: logger.warning(f'Directory is already ignored: {directory}') return @@ -1782,13 +1842,27 @@ def add_ignore_directory(directory: Union[pathlib.Path, str]): def remove_ignored_directory(directory: Union[pathlib.Path, str]): """Remove a directory from the `ignored_directories` in the WROLPi config.""" - directory = str(pathlib.Path(directory)).rstrip('/') - ignored_directories = get_wrolpi_config().ignored_directories - if directory in ignored_directories: - ignored_directories.remove(directory) - get_wrolpi_config().ignored_directories = ignored_directories - else: + media_directory = get_media_directory() + path = pathlib.Path(directory) + if not path.is_absolute(): + path = media_directory / path + path = path.resolve() + absolute = str(path) + + ignored_directories = list(get_wrolpi_config().ignored_directories or []) + kept = [] + removed = False + for d in ignored_directories: + entry = pathlib.Path(d) + entry_abs = str((media_directory / entry).resolve() if not entry.is_absolute() else entry.resolve()) + if entry_abs == absolute: + removed = True + continue + kept.append(d) + + if not removed: raise UnknownDirectory('Directory is not ignored') + get_wrolpi_config().ignored_directories = kept async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> FileGroup: diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index ae999fe6..da15bbeb 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -1,5 +1,6 @@ import hashlib import json +import pathlib from datetime import datetime, timezone from http import HTTPStatus from unittest import mock @@ -1370,6 +1371,53 @@ async def test_ignore_directory(test_session, async_client, test_directory, make assert response.status_code == HTTPStatus.BAD_REQUEST assert len(get_wrolpi_config().ignored_directories) == 0 + # Zims directory depends on FileGroup indexing for Manage/search — cannot ignore. + content = dict(path=str(test_directory / 'zims')) + request, response = await async_client.post('/api/files/ignore_directory', json=content) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert len(get_wrolpi_config().ignored_directories) == 0 + + # Relative form of zims is also refused. + content = dict(path='zims') + request, response = await async_client.post('/api/files/ignore_directory', json=content) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert len(get_wrolpi_config().ignored_directories) == 0 + + # Map is allowed to be ignored (lists PMTiles from the filesystem, not FileGroups). + content = dict(path=str(test_directory / 'map')) + request, response = await async_client.post('/api/files/ignore_directory', json=content) + assert response.status_code == HTTPStatus.OK + assert any(pathlib.Path(i).name == 'map' or str(i).endswith('/map') + for i in get_wrolpi_config().ignored_directories) + + +@pytest.mark.asyncio +async def test_sanitize_ignored_zims_directory(test_session, async_client, test_directory, test_wrolpi_config): + """Zims previously saved as ignored is stripped so refresh can index them again.""" + from wrolpi.files.lib import ( + get_normalized_ignored_directories, + sanitize_ignored_directories, + ) + + # Simulate an older config that incorrectly ignored zims (and intentionally ignored map). + get_wrolpi_config().ignored_directories = ['config', 'tags', 'map', 'zims'] + + # Normalized list used by refresh must never exclude zims. + normalized = get_normalized_ignored_directories() + assert not any(pathlib.Path(i).name == 'zims' for i in normalized) + assert any(pathlib.Path(i).name == 'map' for i in normalized) + + sanitize_ignored_directories() + assert 'zims' not in get_wrolpi_config().ignored_directories + assert 'map' in get_wrolpi_config().ignored_directories + + # Settings endpoint also sanitizes so the UI eye-icon matches. + request, response = await async_client.get('/api/settings') + assert response.status_code == HTTPStatus.OK + ignored = [str(i) for i in response.json['ignored_directories']] + assert 'zims' not in ignored + assert 'map' in ignored + # --- Bulk Tagging API Tests --- diff --git a/wrolpi/files/worker.py b/wrolpi/files/worker.py index c4510217..18400f99 100644 --- a/wrolpi/files/worker.py +++ b/wrolpi/files/worker.py @@ -32,7 +32,7 @@ from wrolpi.files.lib import ( split_path_stem_and_suffix, _upsert_files, get_unique_files_by_stem, glob_shared_stem, group_files_by_stem, get_primary_file, delete_directory, apply_indexers, _move_file_group_files, _bulk_update_file_groups_db, MOVE_CHUNK_SIZE, - _bulk_update_file_groups_reorganize, + _bulk_update_file_groups_reorganize, get_normalized_ignored_directories, ) logger = logger.getChild(__name__) @@ -138,18 +138,9 @@ def _diff_to_paths(diff: FileGroupDiff) -> list[pathlib.Path]: def _get_normalized_ignored_directories() -> list[str]: """Get ignored directories as absolute paths. - Matches the normalization logic in remove_files_in_ignored_directories(). + Delegates to files.lib so special directories (zims, videos, …) are never excluded. """ - ignored = list(map(str, get_wrolpi_config().ignored_directories)) - media_dir = get_media_directory() - result = [] - for d in ignored: - p = pathlib.Path(d) - if not p.is_absolute(): - result.append(str(media_dir / d)) - else: - result.append(d) - return result + return get_normalized_ignored_directories() async def count_files(directories: list[Path]) -> int: diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index cb5cdcc0..631638be 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -152,6 +152,10 @@ async def echo(request: Request): def get_settings(_: Request): wrolpi_config = get_wrolpi_config() + # Drop special directories (e.g. zims) that must never be ignored, if present from older configs. + from wrolpi.files.lib import sanitize_ignored_directories + sanitize_ignored_directories() + ignored_directories = [get_relative_to_media_directory(i) for i in wrolpi_config.ignored_directories] settings = {