Adding toggle to disable the tags directory feature.

app/src/api.js

@@ -312,6 +312,7 @@ export async function getSettings() {
             map_destination: content.map_destination,
             nav_color: content.nav_color,
             media_directory: content.media_directory,
+            tags_directory: content.tags_directory,
             throttle_on_startup: content.throttle_on_startup,
             throttle_status: content.throttle_status,
             videos_destination: content.videos_destination,

app/src/components/Archive.js

@@ -61,6 +61,7 @@ import {InputForm, useForm} from "../hooks/useForm";
 import {CollectionTagModal} from "./collections/CollectionTagModal";
 import {CollectionReorganizeModal} from "./collections/CollectionReorganizeModal";
 import {BatchReorganizeModal} from "./collections/BatchReorganizeModal";
+import {useReorganizationStatus} from "../contexts/FileWorkerStatusContext";
 import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router";
 import Message from "semantic-ui-react/dist/commonjs/collections/Message";
 import {
@@ -868,8 +869,17 @@ function ArchiveSettingsPage() {
     const [browsers, setBrowsers] = useState([]);
     const [browsersAvailable, setBrowsersAvailable] = useState(false);
 
-    // Check how many domains need reorganization on mount
+    // Check if batch reorganization is currently active for domains
+    const {isReorganizing, taskType, collectionKind} = useReorganizationStatus();
+    const isBatchReorganizingDomains = isReorganizing && taskType === 'batch_reorganize' && collectionKind === 'domain';
+
+    // Check how many domains need reorganization on mount (skip if batch is in progress)
     React.useEffect(() => {
+        if (isBatchReorganizingDomains) {
+            // Skip fetching preview while batch reorganization is in progress
+            setFetchingReorgCount(false);
+            return;
+        }
         setFetchingReorgCount(true);
         previewBatchReorganization('domain')
             .then(data => {
@@ -881,7 +891,7 @@ function ArchiveSettingsPage() {
             .finally(() => {
                 setFetchingReorgCount(false);
             });
-    }, []);
+    }, [isBatchReorganizingDomains]);
 
     // Fetch available browsers on mount (only if not dockerized)
     React.useEffect(() => {

app/src/components/Common.js

@@ -845,7 +845,7 @@ export function ThrottleToggle() {
     </div>;
 }
 
-export function Toggle({label, checked, disabled, onChange, icon, popupContent = null}) {
+export function Toggle({label, checked, disabled, onChange, icon, popupContent = null, info = null}) {
     // Custom toggle because Semantic UI does not handle inverted labels correctly.
     const {t, inverted} = useContext(ThemeContext);
 
@@ -892,6 +892,7 @@ export function Toggle({label, checked, disabled, onChange, icon, popupContent =
         <span style={style} data-testid='toggle-label'>
             {icon}
             {label}
+            {info && <InfoPopup content={info}/>}
         </span>
     </span>
 

app/src/components/admin/Settings.js

@@ -458,6 +458,7 @@ export function SettingsPage() {
             map_destination: settings.map_destination,
             nav_color: settings.nav_color,
             media_directory: settings.media_directory,
+            tags_directory: settings.tags_directory,
             throttle_on_startup: settings.throttle_on_startup,
             videos_destination: settings.videos_destination,
             zims_destination: settings.zims_destination,
@@ -542,6 +543,16 @@ export function SettingsPage() {
                     />
                 </div>
 
+                <div style={{margin: '0.5em'}}>
+                    <Toggle
+                        label='Tags Directory'
+                        disabled={disabled || state.tags_directory === null}
+                        checked={state.tags_directory === true}
+                        onChange={checked => handleInputChange(null, 'tags_directory', checked)}
+                        info='When enabled, WROLPi creates a "tags" directory with hardlinks to all tagged files, organized by tag name.'
+                    />
+                </div>
+
                 <Form.Group inline>
                     <Form.Input
                         label={<>

modules/archive/__init__.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 import pathlib
 from abc import ABC
-from typing import List, Tuple, Iterable
+from typing import Callable, List, Tuple, Iterable
 
 from sqlalchemy import not_
 from sqlalchemy.orm import Session
@@ -260,10 +260,11 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive:
 
 
 @register_modeler
-async def archive_modeler():
+async def archive_modeler(progress_callback: Callable[[int], None] = None):
     """Searches DB for FileGroups that contain an HTML file.  If the HTML file is a SingleFile, we model it as an
     Archive."""
     invalid_archives = set()
+    total_processed = 0
 
     while True:
         with get_db_session(commit=True) as session:
@@ -276,9 +277,10 @@ async def archive_modeler():
                 .outerjoin(Archive, Archive.file_group_id == FileGroup.id) \
                 .limit(20)
             results: Iterable[Tuple[FileGroup, Archive]]
+            results_list = list(results)
 
             processed = 0
-            for processed, (file_group, archive) in enumerate(results):
+            for processed, (file_group, archive) in enumerate(results_list):
                 with slow_logger(1, f'Modeling archive took %(elapsed)s seconds: {file_group}',
                                  logger__=logger):
                     if archive:
@@ -309,6 +311,12 @@ async def archive_modeler():
 
             session.commit()
 
+            # Report batch progress
+            batch_count = len(results_list)
+            total_processed += batch_count
+            if progress_callback and batch_count > 0:
+                progress_callback(total_processed)
+
             if processed < 19:
                 # Did not reach limit (enumerate is 0-indexed, so 19 = 20 items), do not query again.
                 if __debug__ and logger.isEnabledFor(TRACE_LEVEL):

modules/videos/__init__.py

@@ -1,5 +1,5 @@
 import asyncio
-from typing import List, Tuple
+from typing import Callable, List, Tuple
 
 from sqlalchemy.orm import Session
 
@@ -18,7 +18,8 @@ VIDEO_PROCESSING_LIMIT = 20
 
 
 @register_modeler
-async def video_modeler():
+async def video_modeler(progress_callback: Callable[[int], None] = None):
+    total_processed = 0
     while True:
         with get_db_session(commit=True) as session:
             file_groups = session.query(FileGroup, Video).filter(
@@ -58,6 +59,11 @@ async def video_modeler():
 
             session.commit()
 
+            # Report batch progress
+            total_processed += len(file_groups)
+            if progress_callback and len(file_groups) > 0:
+                progress_callback(total_processed)
+
             if processed < VIDEO_PROCESSING_LIMIT:
                 # Did not reach limit, do not query again.
                 break

modules/zim/lib.py

@@ -3,7 +3,7 @@ import functools
 import pathlib
 import re
 from datetime import datetime
-from typing import List, Tuple, Dict
+from typing import Callable, List, Tuple, Dict
 
 import cachetools.func
 from libzim import Entry
@@ -62,7 +62,8 @@ def model_zim(file_group: FileGroup, session: Session) -> Zim:
 
 
 @register_modeler
-async def zim_modeler():
+async def zim_modeler(progress_callback: Callable[[int], None] = None):
+    total_processed = 0
     while True:
         with get_db_session(commit=True) as session:
             file_groups = session.query(FileGroup, Zim) \
@@ -95,6 +96,11 @@ async def zim_modeler():
 
             logger.debug(f'Modeled {processed} zim files')
 
+            # Report batch progress
+            total_processed += len(file_groups)
+            if progress_callback and len(file_groups) > 0:
+                progress_callback(total_processed)
+
             if processed < 10:
                 # Did not reach limit, do not query again.
                 break

wrolpi/common.py

@@ -731,6 +731,7 @@ class WROLPiConfigValidator:
     log_level: str = None
     map_destination: str = None
     nav_color: str = None
+    tags_directory: bool = None
     throttle_on_startup: bool = None
     version: int = None
     videos_destination: str = None
@@ -798,6 +799,7 @@ class WROLPiConfig(ConfigFile):
         log_level='info',
         map_destination='map',
         nav_color='violet',
+        tags_directory=True,
         throttle_on_startup=False,
         version=0,
         videos_destination='videos/%(channel_tag)s/%(channel_name)s',
@@ -984,6 +986,14 @@ class WROLPiConfig(ConfigFile):
     def nav_color(self, value: str):
         self.update({'nav_color': value})
 
+    @property
+    def tags_directory(self) -> bool:
+        return self._config.get('tags_directory', True)
+
+    @tags_directory.setter
+    def tags_directory(self, value: bool):
+        self.update({'tags_directory': value})
+
 
 WROLPI_CONFIG: WROLPiConfig = WROLPiConfig()
 TEST_WROLPI_CONFIG: WROLPiConfig = None
@@ -1595,7 +1605,8 @@ async def apply_modelers(progress_callback: callable = None):
     """Run all registered modelers.
 
     Args:
-        progress_callback: Optional callback(processed, total) called after each modeler completes.
+        progress_callback: Optional callback(processed, total) called after each modeler completes,
+                          or after each batch if the modeler supports progress_callback.
     """
     from wrolpi.db import get_db_session
     from wrolpi.files.models import FileGroup
@@ -1607,21 +1618,38 @@ async def apply_modelers(progress_callback: callable = None):
             initial_unindexed = session.query(FileGroup).filter(FileGroup.indexed != True).count()
         progress_callback(0, initial_unindexed)
 
+    # Track cumulative progress across all modelers
+    cumulative_processed = 0
+
     for modeler in modelers:
         logger_.info(f'Applying modeler {modeler.__name__}')
         try:
-            await modeler()
+            # Check if this modeler accepts a progress_callback parameter
+            sig = inspect.signature(modeler)
+            if 'progress_callback' in sig.parameters and progress_callback and initial_unindexed > 0:
+                # Create a callback that updates the cumulative progress
+                def make_modeler_callback(current_cumulative):
+                    def modeler_progress_callback(batch_processed: int):
+                        nonlocal cumulative_processed
+                        cumulative_processed = current_cumulative + batch_processed
+                        progress_callback(cumulative_processed, initial_unindexed)
+
+                    return modeler_progress_callback
+
+                await modeler(progress_callback=make_modeler_callback(cumulative_processed))
+            else:
+                await modeler()
         except Exception as e:
             logger_.error(f'Modeler {modeler.__name__} raised error', exc_info=e)
             if PYTEST:
                 raise
 
-        # Update progress after each modeler
+        # Update progress after each modeler completes (fallback for modelers without progress_callback)
         if progress_callback and initial_unindexed > 0:
             with get_db_session() as session:
                 remaining = session.query(FileGroup).filter(FileGroup.indexed != True).count()
-            processed = initial_unindexed - remaining
-            progress_callback(processed, initial_unindexed)
+            cumulative_processed = initial_unindexed - remaining
+            progress_callback(cumulative_processed, initial_unindexed)
 
         # Sleep to catch cancel.
         await asyncio.sleep(0)

wrolpi/files/ebooks.py

@@ -2,7 +2,7 @@ import asyncio
 import dataclasses
 import logging
 import pathlib
-from typing import List, Optional, Tuple
+from typing import Callable, List, Optional, Tuple
 
 from sqlalchemy import Column, Integer, BigInteger, ForeignKey, or_, and_, Index
 from sqlalchemy.orm import relationship, Session
@@ -304,8 +304,9 @@ def find_ebook_files_in_group(group: List):
 
 
 @register_modeler
-async def ebook_modeler():
+async def ebook_modeler(progress_callback: Callable[[int], None] = None):
     """Searches for ebook files and models them into the ebook table."""
+    total_processed = 0
     while True:
         with get_db_session(commit=True) as session:
             file_groups = session.query(FileGroup, EBook).filter(
@@ -336,6 +337,11 @@ async def ebook_modeler():
 
             session.commit()
 
+            # Report batch progress
+            total_processed += len(file_groups)
+            if progress_callback and len(file_groups) > 0:
+                progress_callback(total_processed)
+
             if processed < 10:
                 # Did not reach limit, no more books.
                 break

wrolpi/files/pdfs.py

@@ -1,7 +1,7 @@
 import asyncio
 import dataclasses
 import pathlib
-from typing import Generator
+from typing import Callable, Generator
 
 import pytz
 
@@ -93,12 +93,13 @@ PDF_PROCESSING_LIMIT = 10
 
 
 @register_modeler
-async def pdf_modeler():
+async def pdf_modeler(progress_callback: Callable[[int], None] = None):
     """Queries for any PDF files that have not been indexed.  Each PDF found will be indexed."""
     if not PdfReader:
         logger.warning(f'Cannot index PDF without PyPDF2')
         return
 
+    total_processed = 0
     while True:
         # Continually query for PDFs that have not been indexed.
         with get_db_session(commit=True) as session:
@@ -108,6 +109,7 @@ async def pdf_modeler():
                 FileGroup.indexed != True,
                 FileGroup.mimetype == 'application/pdf',
             ).limit(PDF_PROCESSING_LIMIT)
+            file_groups = list(file_groups)
 
             processed = 0
             for file_group in file_groups:
@@ -172,6 +174,11 @@ async def pdf_modeler():
 
             session.commit()
 
+            # Report batch progress
+            total_processed += len(file_groups)
+            if progress_callback and len(file_groups) > 0:
+                progress_callback(total_processed)
+
             if processed < PDF_PROCESSING_LIMIT:
                 # Did not reach limit, do not query again.
                 break

wrolpi/root_api.py

@@ -168,6 +168,7 @@ def get_settings(_: Request):
         'map_destination': wrolpi_config.map_destination,
         'nav_color': wrolpi_config.nav_color,
         'media_directory': str(get_media_directory()),  # Convert to string to avoid conversion to relative.
+        'tags_directory': wrolpi_config.tags_directory,
         'throttle_on_startup': wrolpi_config.throttle_on_startup,
         'version': __version__,
         'videos_destination': wrolpi_config.videos_destination,

wrolpi/schema.py

@@ -28,6 +28,7 @@ class SettingsResponse:
     log_level: str
     nav_color: str
     media_directory: str
+    tags_directory: bool
     throttle_on_startup: bool
     throttle_status: str
     version: str
@@ -69,6 +70,7 @@ class SettingsRequest:
     map_destination: Optional[str] = None
     nav_color: Optional[str] = None
     media_directory: Optional[str] = None
+    tags_directory: Optional[bool] = None
     throttle_on: Optional[bool] = None
     throttle_on_startup: Optional[bool] = None
     videos_destination: Optional[str] = None

wrolpi/tags.py

@@ -13,7 +13,7 @@ from sqlalchemy.orm import relationship, Session
 
 from wrolpi import dates, flags
 from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory, background_task, \
-    get_relative_to_media_directory, is_valid_hex_color, walk, INVALID_FILE_CHARS
+    get_relative_to_media_directory, is_valid_hex_color, walk, INVALID_FILE_CHARS, get_wrolpi_config
 from wrolpi.dates import TZDateTime
 from wrolpi.db import get_db_curs, get_db_session
 from wrolpi.downloader import save_downloads_config
@@ -629,6 +629,11 @@ be AUTOMATICALLY DELETED!
 def sync_tags_directory():
     """Synchronizes database Tags with the Tags directory (typically /media/wrolpi/tags).  Removes any
     files that do not belong in the Tags Directory."""
+    wrolpi_config = get_wrolpi_config()
+    if not wrolpi_config.tags_directory:
+        logger.debug('Skipping tags directory sync because tags_directory setting is disabled')
+        return
+
     with get_db_session() as session:
         try:
             tags_directory = get_tags_directory()

wrolpi/test/test_tags.py

@@ -5,7 +5,7 @@ import pytest
 import yaml
 
 from wrolpi import tags
-from wrolpi.common import is_hardlinked, walk
+from wrolpi.common import is_hardlinked, walk, get_wrolpi_config
 from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, UsedTag
 from wrolpi.files.models import FileGroup
 from wrolpi.tags import TagFile, Tag
@@ -507,3 +507,31 @@ async def test_tag_rename_with_channel(test_session, test_directory, video_facto
     # Cannot delete Tag when used by Channel.  This prevents the need to move Channel directories when Tags are deleted.
     with pytest.raises(UsedTag):
         tag.delete()
+
+
+@pytest.mark.asyncio
+async def test_tags_directory_disabled(test_session, test_directory, tag_factory, video_factory, await_switches):
+    """Test that Tag Directory is NOT synchronized when tags_directory setting is disabled."""
+    tags_dir = test_directory / 'tags'
+
+    # Disable the tags_directory setting.
+    wrolpi_config = get_wrolpi_config()
+    wrolpi_config.tags_directory = False
+
+    # Create a tag and tag a video.
+    tag = await tag_factory()
+    video = video_factory(with_video_file=test_directory / 'video.mp4')
+    video.add_tag(test_session, tag.id)
+    test_session.commit()
+    await await_switches()
+
+    # Tags directory should NOT be created because the setting is disabled.
+    assert not tags_dir.exists(), 'Tags directory should not exist when tags_directory setting is disabled'
+
+    # Enable the setting and sync again.
+    wrolpi_config.tags_directory = True
+    tags.sync_tags_directory()
+
+    # Now the tags directory should exist with the linked file.
+    assert tags_dir.is_dir(), 'Tags directory should exist after enabling setting'
+    assert (tags_dir / tag.name / 'video.mp4').is_file(), 'Video should be linked in tags directory'