Prevent the zims directory from being ignored by file refresh.

Zim Manage and search depend on FileGroup modeling of files under
zims/. Ignoring that directory left Kiwix serving files on disk while
/api/zim stayed empty. Refuse to ignore zims (alongside other special
dirs), strip it from existing configs on settings load, and share
normalized ignore handling between refresh and upsert.

wrolpi/files/lib.py

@@ -56,7 +56,8 @@ __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix',
            'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics',
            'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
            'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href',
-           'get_tagged_file_groups_by_ids', 'delete_file_groups']
+           'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories',
+           'get_normalized_ignored_directories', 'sanitize_ignored_directories']
 
 
 def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
@@ -786,11 +787,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime,
 
 def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]:
     """Return a new list which does not contain any file paths that are in ignored directories."""
-    ignored_directories = list(map(str, get_wrolpi_config().ignored_directories))
-    for idx, ignored_directory in enumerate(ignored_directories):
-        ignored_directory = pathlib.Path(ignored_directory)
-        if not ignored_directory.is_absolute():
-            ignored_directories[idx] = str(get_media_directory() / ignored_directory)
+    ignored_directories = get_normalized_ignored_directories()
     files = [i for i in files if not any(str(i).startswith(j) for j in ignored_directories)]
     return files
 
@@ -1746,23 +1743,84 @@ async def rename(session: Session, path: pathlib.Path, new_name: str, send_event
     return await rename_file(path, new_name)
 
 
+def get_special_directories() -> List[pathlib.Path]:
+    """Return absolute directories that must never be ignored by file refresh.
+
+    These are module destinations that depend on FileGroup indexing (or are the media root).
+    Map is intentionally excluded: it lists PMTiles by walking the filesystem and benefits
+    from being ignored during refresh (very large files).
+    """
+    from modules.videos.common import get_videos_directory, get_no_channel_directory
+    from modules.archive.lib import get_archive_directory
+
+    media_directory = get_media_directory()
+    # Avoid importing modules.zim.lib here (circular import via files.worker).
+    zims_directory = media_directory / get_wrolpi_config().zims_destination
+    return [
+        media_directory.resolve(),
+        get_videos_directory().resolve(),
+        get_archive_directory().resolve(),
+        get_no_channel_directory().resolve(),
+        zims_directory.resolve(),
+    ]
+
+
+def get_normalized_ignored_directories() -> List[str]:
+    """Return absolute ignored directory paths, excluding special directories.
+
+    Special directories (media root, videos, archives, zims, …) are never treated as ignored,
+    even if they appear in config from older installs or manual edits.
+    """
+    media_directory = get_media_directory()
+    special = {str(p) for p in get_special_directories()}
+    result = []
+    for d in get_wrolpi_config().ignored_directories or []:
+        path = pathlib.Path(d)
+        absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve())
+        if absolute in special:
+            continue
+        result.append(absolute)
+    return result
+
+
+def sanitize_ignored_directories():
+    """Persistently remove special directories from the ignored list if present."""
+    wrolpi_config = get_wrolpi_config()
+    ignored = list(wrolpi_config.ignored_directories or [])
+    if not ignored:
+        return
+
+    media_directory = get_media_directory()
+    special = {str(p) for p in get_special_directories()}
+    cleaned = []
+    removed = False
+    for d in ignored:
+        path = pathlib.Path(d)
+        absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve())
+        if absolute in special:
+            removed = True
+            logger.warning(f'Removing special directory from ignored_directories: {d}')
+            continue
+        cleaned.append(d)
+
+    if removed:
+        wrolpi_config.ignored_directories = cleaned
+
+
 def add_ignore_directory(directory: Union[pathlib.Path, str]):
     """Add a directory to the `ignored_directories` in the WROLPi config.  This directory will be ignored when
     refreshing."""
-    media_directory = get_media_directory()
+    media_directory = get_media_directory().resolve()
 
     directory = pathlib.Path(directory)
+    if not directory.is_absolute():
+        directory = media_directory / directory
+    directory = directory.resolve()
 
-    from modules.videos.common import get_videos_directory
-    from modules.archive.lib import get_archive_directory
-    from modules.videos.common import get_no_channel_directory
-    special_directories = [
-        get_media_directory(),
-        get_videos_directory(),
-        get_archive_directory(),
-        get_no_channel_directory(),
-    ]
+    # Drop any special directories that were previously saved as ignored.
+    sanitize_ignored_directories()
 
+    special_directories = set(get_special_directories())
     if directory in special_directories:
         raise InvalidDirectory('Refusing to ignore special directory')
 
@@ -1772,7 +1830,9 @@ def add_ignore_directory(directory: Union[pathlib.Path, str]):
     wrolpi_config = get_wrolpi_config()
     ignored_directories = wrolpi_config.ignored_directories if wrolpi_config.ignored_directories else list()
 
-    if str(directory) in ignored_directories:
+    # Accept either absolute or relative forms already present in config.
+    relative = str(get_relative_to_media_directory(directory))
+    if str(directory) in ignored_directories or relative in ignored_directories:
         logger.warning(f'Directory is already ignored: {directory}')
         return
 
@@ -1782,13 +1842,27 @@ def add_ignore_directory(directory: Union[pathlib.Path, str]):
 
 def remove_ignored_directory(directory: Union[pathlib.Path, str]):
     """Remove a directory from the `ignored_directories` in the WROLPi config."""
-    directory = str(pathlib.Path(directory)).rstrip('/')
-    ignored_directories = get_wrolpi_config().ignored_directories
-    if directory in ignored_directories:
-        ignored_directories.remove(directory)
-        get_wrolpi_config().ignored_directories = ignored_directories
-    else:
+    media_directory = get_media_directory()
+    path = pathlib.Path(directory)
+    if not path.is_absolute():
+        path = media_directory / path
+    path = path.resolve()
+    absolute = str(path)
+
+    ignored_directories = list(get_wrolpi_config().ignored_directories or [])
+    kept = []
+    removed = False
+    for d in ignored_directories:
+        entry = pathlib.Path(d)
+        entry_abs = str((media_directory / entry).resolve() if not entry.is_absolute() else entry.resolve())
+        if entry_abs == absolute:
+            removed = True
+            continue
+        kept.append(d)
+
+    if not removed:
         raise UnknownDirectory('Directory is not ignored')
+    get_wrolpi_config().ignored_directories = kept
 
 
 async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> FileGroup:

wrolpi/files/test/test_api.py

@@ -1,5 +1,6 @@
 import hashlib
 import json
+import pathlib
 from datetime import datetime, timezone
 from http import HTTPStatus
 from unittest import mock
@@ -1370,6 +1371,53 @@ async def test_ignore_directory(test_session, async_client, test_directory, make
     assert response.status_code == HTTPStatus.BAD_REQUEST
     assert len(get_wrolpi_config().ignored_directories) == 0
 
+    # Zims directory depends on FileGroup indexing for Manage/search — cannot ignore.
+    content = dict(path=str(test_directory / 'zims'))
+    request, response = await async_client.post('/api/files/ignore_directory', json=content)
+    assert response.status_code == HTTPStatus.BAD_REQUEST
+    assert len(get_wrolpi_config().ignored_directories) == 0
+
+    # Relative form of zims is also refused.
+    content = dict(path='zims')
+    request, response = await async_client.post('/api/files/ignore_directory', json=content)
+    assert response.status_code == HTTPStatus.BAD_REQUEST
+    assert len(get_wrolpi_config().ignored_directories) == 0
+
+    # Map is allowed to be ignored (lists PMTiles from the filesystem, not FileGroups).
+    content = dict(path=str(test_directory / 'map'))
+    request, response = await async_client.post('/api/files/ignore_directory', json=content)
+    assert response.status_code == HTTPStatus.OK
+    assert any(pathlib.Path(i).name == 'map' or str(i).endswith('/map')
+               for i in get_wrolpi_config().ignored_directories)
+
+
+@pytest.mark.asyncio
+async def test_sanitize_ignored_zims_directory(test_session, async_client, test_directory, test_wrolpi_config):
+    """Zims previously saved as ignored is stripped so refresh can index them again."""
+    from wrolpi.files.lib import (
+        get_normalized_ignored_directories,
+        sanitize_ignored_directories,
+    )
+
+    # Simulate an older config that incorrectly ignored zims (and intentionally ignored map).
+    get_wrolpi_config().ignored_directories = ['config', 'tags', 'map', 'zims']
+
+    # Normalized list used by refresh must never exclude zims.
+    normalized = get_normalized_ignored_directories()
+    assert not any(pathlib.Path(i).name == 'zims' for i in normalized)
+    assert any(pathlib.Path(i).name == 'map' for i in normalized)
+
+    sanitize_ignored_directories()
+    assert 'zims' not in get_wrolpi_config().ignored_directories
+    assert 'map' in get_wrolpi_config().ignored_directories
+
+    # Settings endpoint also sanitizes so the UI eye-icon matches.
+    request, response = await async_client.get('/api/settings')
+    assert response.status_code == HTTPStatus.OK
+    ignored = [str(i) for i in response.json['ignored_directories']]
+    assert 'zims' not in ignored
+    assert 'map' in ignored
+
 
 # --- Bulk Tagging API Tests ---
 

wrolpi/files/worker.py

@@ -32,7 +32,7 @@ from wrolpi.files.lib import (
     split_path_stem_and_suffix, _upsert_files, get_unique_files_by_stem, glob_shared_stem,
     group_files_by_stem, get_primary_file, delete_directory, apply_indexers,
     _move_file_group_files, _bulk_update_file_groups_db, MOVE_CHUNK_SIZE,
-    _bulk_update_file_groups_reorganize,
+    _bulk_update_file_groups_reorganize, get_normalized_ignored_directories,
 )
 
 logger = logger.getChild(__name__)
@@ -138,18 +138,9 @@ def _diff_to_paths(diff: FileGroupDiff) -> list[pathlib.Path]:
 def _get_normalized_ignored_directories() -> list[str]:
     """Get ignored directories as absolute paths.
 
-    Matches the normalization logic in remove_files_in_ignored_directories().
+    Delegates to files.lib so special directories (zims, videos, …) are never excluded.
     """
-    ignored = list(map(str, get_wrolpi_config().ignored_directories))
-    media_dir = get_media_directory()
-    result = []
-    for d in ignored:
-        p = pathlib.Path(d)
-        if not p.is_absolute():
-            result.append(str(media_dir / d))
-        else:
-            result.append(d)
-    return result
+    return get_normalized_ignored_directories()
 
 
 async def count_files(directories: list[Path]) -> int:

wrolpi/root_api.py

@@ -152,6 +152,10 @@ async def echo(request: Request):
 def get_settings(_: Request):
     wrolpi_config = get_wrolpi_config()
 
+    # Drop special directories (e.g. zims) that must never be ignored, if present from older configs.
+    from wrolpi.files.lib import sanitize_ignored_directories
+    sanitize_ignored_directories()
+
     ignored_directories = [get_relative_to_media_directory(i) for i in wrolpi_config.ignored_directories]
 
     settings = {