diff --git a/app/src/api.js b/app/src/api.js index b49fc296..cb49930a 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -312,6 +312,7 @@ export async function getSettings() { map_destination: content.map_destination, nav_color: content.nav_color, media_directory: content.media_directory, + tags_directory: content.tags_directory, throttle_on_startup: content.throttle_on_startup, throttle_status: content.throttle_status, videos_destination: content.videos_destination, diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index 8c9486da..c6684879 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -61,6 +61,7 @@ import {InputForm, useForm} from "../hooks/useForm"; import {CollectionTagModal} from "./collections/CollectionTagModal"; import {CollectionReorganizeModal} from "./collections/CollectionReorganizeModal"; import {BatchReorganizeModal} from "./collections/BatchReorganizeModal"; +import {useReorganizationStatus} from "../contexts/FileWorkerStatusContext"; import {Link, Route, Routes, useLocation, useNavigate, useParams} from "react-router"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; import { @@ -868,8 +869,17 @@ function ArchiveSettingsPage() { const [browsers, setBrowsers] = useState([]); const [browsersAvailable, setBrowsersAvailable] = useState(false); - // Check how many domains need reorganization on mount + // Check if batch reorganization is currently active for domains + const {isReorganizing, taskType, collectionKind} = useReorganizationStatus(); + const isBatchReorganizingDomains = isReorganizing && taskType === 'batch_reorganize' && collectionKind === 'domain'; + + // Check how many domains need reorganization on mount (skip if batch is in progress) React.useEffect(() => { + if (isBatchReorganizingDomains) { + // Skip fetching preview while batch reorganization is in progress + setFetchingReorgCount(false); + return; + } setFetchingReorgCount(true); previewBatchReorganization('domain') .then(data => { @@ -881,7 +891,7 @@ function ArchiveSettingsPage() { .finally(() => { setFetchingReorgCount(false); }); - }, []); + }, [isBatchReorganizingDomains]); // Fetch available browsers on mount (only if not dockerized) React.useEffect(() => { diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 4eeb148e..5d4971a6 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -845,7 +845,7 @@ export function ThrottleToggle() { ; } -export function Toggle({label, checked, disabled, onChange, icon, popupContent = null}) { +export function Toggle({label, checked, disabled, onChange, icon, popupContent = null, info = null}) { // Custom toggle because Semantic UI does not handle inverted labels correctly. const {t, inverted} = useContext(ThemeContext); @@ -892,6 +892,7 @@ export function Toggle({label, checked, disabled, onChange, icon, popupContent = {icon} {label} + {info && } diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js index b783c1d6..b2ba6b28 100644 --- a/app/src/components/admin/Settings.js +++ b/app/src/components/admin/Settings.js @@ -458,6 +458,7 @@ export function SettingsPage() { map_destination: settings.map_destination, nav_color: settings.nav_color, media_directory: settings.media_directory, + tags_directory: settings.tags_directory, throttle_on_startup: settings.throttle_on_startup, videos_destination: settings.videos_destination, zims_destination: settings.zims_destination, @@ -542,6 +543,16 @@ export function SettingsPage() { /> +
+ handleInputChange(null, 'tags_directory', checked)} + info='When enabled, WROLPi creates a "tags" directory with hardlinks to all tagged files, organized by tag name.' + /> +
+ diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 645d6c20..c15ce597 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -2,7 +2,7 @@ import asyncio import json import pathlib from abc import ABC -from typing import List, Tuple, Iterable +from typing import Callable, List, Tuple, Iterable from sqlalchemy import not_ from sqlalchemy.orm import Session @@ -260,10 +260,11 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive: @register_modeler -async def archive_modeler(): +async def archive_modeler(progress_callback: Callable[[int], None] = None): """Searches DB for FileGroups that contain an HTML file. If the HTML file is a SingleFile, we model it as an Archive.""" invalid_archives = set() + total_processed = 0 while True: with get_db_session(commit=True) as session: @@ -276,9 +277,10 @@ async def archive_modeler(): .outerjoin(Archive, Archive.file_group_id == FileGroup.id) \ .limit(20) results: Iterable[Tuple[FileGroup, Archive]] + results_list = list(results) processed = 0 - for processed, (file_group, archive) in enumerate(results): + for processed, (file_group, archive) in enumerate(results_list): with slow_logger(1, f'Modeling archive took %(elapsed)s seconds: {file_group}', logger__=logger): if archive: @@ -309,6 +311,12 @@ async def archive_modeler(): session.commit() + # Report batch progress + batch_count = len(results_list) + total_processed += batch_count + if progress_callback and batch_count > 0: + progress_callback(total_processed) + if processed < 19: # Did not reach limit (enumerate is 0-indexed, so 19 = 20 items), do not query again. if __debug__ and logger.isEnabledFor(TRACE_LEVEL): diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py index e4598696..3318dade 100644 --- a/modules/videos/__init__.py +++ b/modules/videos/__init__.py @@ -1,5 +1,5 @@ import asyncio -from typing import List, Tuple +from typing import Callable, List, Tuple from sqlalchemy.orm import Session @@ -18,7 +18,8 @@ VIDEO_PROCESSING_LIMIT = 20 @register_modeler -async def video_modeler(): +async def video_modeler(progress_callback: Callable[[int], None] = None): + total_processed = 0 while True: with get_db_session(commit=True) as session: file_groups = session.query(FileGroup, Video).filter( @@ -58,6 +59,11 @@ async def video_modeler(): session.commit() + # Report batch progress + total_processed += len(file_groups) + if progress_callback and len(file_groups) > 0: + progress_callback(total_processed) + if processed < VIDEO_PROCESSING_LIMIT: # Did not reach limit, do not query again. break diff --git a/modules/zim/lib.py b/modules/zim/lib.py index e322ab19..5ce169ca 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -3,7 +3,7 @@ import functools import pathlib import re from datetime import datetime -from typing import List, Tuple, Dict +from typing import Callable, List, Tuple, Dict import cachetools.func from libzim import Entry @@ -62,7 +62,8 @@ def model_zim(file_group: FileGroup, session: Session) -> Zim: @register_modeler -async def zim_modeler(): +async def zim_modeler(progress_callback: Callable[[int], None] = None): + total_processed = 0 while True: with get_db_session(commit=True) as session: file_groups = session.query(FileGroup, Zim) \ @@ -95,6 +96,11 @@ async def zim_modeler(): logger.debug(f'Modeled {processed} zim files') + # Report batch progress + total_processed += len(file_groups) + if progress_callback and len(file_groups) > 0: + progress_callback(total_processed) + if processed < 10: # Did not reach limit, do not query again. break diff --git a/wrolpi/common.py b/wrolpi/common.py index fe18f30a..3513b6a8 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -731,6 +731,7 @@ class WROLPiConfigValidator: log_level: str = None map_destination: str = None nav_color: str = None + tags_directory: bool = None throttle_on_startup: bool = None version: int = None videos_destination: str = None @@ -798,6 +799,7 @@ class WROLPiConfig(ConfigFile): log_level='info', map_destination='map', nav_color='violet', + tags_directory=True, throttle_on_startup=False, version=0, videos_destination='videos/%(channel_tag)s/%(channel_name)s', @@ -984,6 +986,14 @@ class WROLPiConfig(ConfigFile): def nav_color(self, value: str): self.update({'nav_color': value}) + @property + def tags_directory(self) -> bool: + return self._config.get('tags_directory', True) + + @tags_directory.setter + def tags_directory(self, value: bool): + self.update({'tags_directory': value}) + WROLPI_CONFIG: WROLPiConfig = WROLPiConfig() TEST_WROLPI_CONFIG: WROLPiConfig = None @@ -1595,7 +1605,8 @@ async def apply_modelers(progress_callback: callable = None): """Run all registered modelers. Args: - progress_callback: Optional callback(processed, total) called after each modeler completes. + progress_callback: Optional callback(processed, total) called after each modeler completes, + or after each batch if the modeler supports progress_callback. """ from wrolpi.db import get_db_session from wrolpi.files.models import FileGroup @@ -1607,21 +1618,38 @@ async def apply_modelers(progress_callback: callable = None): initial_unindexed = session.query(FileGroup).filter(FileGroup.indexed != True).count() progress_callback(0, initial_unindexed) + # Track cumulative progress across all modelers + cumulative_processed = 0 + for modeler in modelers: logger_.info(f'Applying modeler {modeler.__name__}') try: - await modeler() + # Check if this modeler accepts a progress_callback parameter + sig = inspect.signature(modeler) + if 'progress_callback' in sig.parameters and progress_callback and initial_unindexed > 0: + # Create a callback that updates the cumulative progress + def make_modeler_callback(current_cumulative): + def modeler_progress_callback(batch_processed: int): + nonlocal cumulative_processed + cumulative_processed = current_cumulative + batch_processed + progress_callback(cumulative_processed, initial_unindexed) + + return modeler_progress_callback + + await modeler(progress_callback=make_modeler_callback(cumulative_processed)) + else: + await modeler() except Exception as e: logger_.error(f'Modeler {modeler.__name__} raised error', exc_info=e) if PYTEST: raise - # Update progress after each modeler + # Update progress after each modeler completes (fallback for modelers without progress_callback) if progress_callback and initial_unindexed > 0: with get_db_session() as session: remaining = session.query(FileGroup).filter(FileGroup.indexed != True).count() - processed = initial_unindexed - remaining - progress_callback(processed, initial_unindexed) + cumulative_processed = initial_unindexed - remaining + progress_callback(cumulative_processed, initial_unindexed) # Sleep to catch cancel. await asyncio.sleep(0) diff --git a/wrolpi/files/ebooks.py b/wrolpi/files/ebooks.py index 3fe91fdd..4b48973f 100644 --- a/wrolpi/files/ebooks.py +++ b/wrolpi/files/ebooks.py @@ -2,7 +2,7 @@ import asyncio import dataclasses import logging import pathlib -from typing import List, Optional, Tuple +from typing import Callable, List, Optional, Tuple from sqlalchemy import Column, Integer, BigInteger, ForeignKey, or_, and_, Index from sqlalchemy.orm import relationship, Session @@ -304,8 +304,9 @@ def find_ebook_files_in_group(group: List): @register_modeler -async def ebook_modeler(): +async def ebook_modeler(progress_callback: Callable[[int], None] = None): """Searches for ebook files and models them into the ebook table.""" + total_processed = 0 while True: with get_db_session(commit=True) as session: file_groups = session.query(FileGroup, EBook).filter( @@ -336,6 +337,11 @@ async def ebook_modeler(): session.commit() + # Report batch progress + total_processed += len(file_groups) + if progress_callback and len(file_groups) > 0: + progress_callback(total_processed) + if processed < 10: # Did not reach limit, no more books. break diff --git a/wrolpi/files/pdfs.py b/wrolpi/files/pdfs.py index f8f38047..0ad5a177 100644 --- a/wrolpi/files/pdfs.py +++ b/wrolpi/files/pdfs.py @@ -1,7 +1,7 @@ import asyncio import dataclasses import pathlib -from typing import Generator +from typing import Callable, Generator import pytz @@ -93,12 +93,13 @@ PDF_PROCESSING_LIMIT = 10 @register_modeler -async def pdf_modeler(): +async def pdf_modeler(progress_callback: Callable[[int], None] = None): """Queries for any PDF files that have not been indexed. Each PDF found will be indexed.""" if not PdfReader: logger.warning(f'Cannot index PDF without PyPDF2') return + total_processed = 0 while True: # Continually query for PDFs that have not been indexed. with get_db_session(commit=True) as session: @@ -108,6 +109,7 @@ async def pdf_modeler(): FileGroup.indexed != True, FileGroup.mimetype == 'application/pdf', ).limit(PDF_PROCESSING_LIMIT) + file_groups = list(file_groups) processed = 0 for file_group in file_groups: @@ -172,6 +174,11 @@ async def pdf_modeler(): session.commit() + # Report batch progress + total_processed += len(file_groups) + if progress_callback and len(file_groups) > 0: + progress_callback(total_processed) + if processed < PDF_PROCESSING_LIMIT: # Did not reach limit, do not query again. break diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 569338d0..41721370 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -168,6 +168,7 @@ def get_settings(_: Request): 'map_destination': wrolpi_config.map_destination, 'nav_color': wrolpi_config.nav_color, 'media_directory': str(get_media_directory()), # Convert to string to avoid conversion to relative. + 'tags_directory': wrolpi_config.tags_directory, 'throttle_on_startup': wrolpi_config.throttle_on_startup, 'version': __version__, 'videos_destination': wrolpi_config.videos_destination, diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 899d40e0..bbfefbcf 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -28,6 +28,7 @@ class SettingsResponse: log_level: str nav_color: str media_directory: str + tags_directory: bool throttle_on_startup: bool throttle_status: str version: str @@ -69,6 +70,7 @@ class SettingsRequest: map_destination: Optional[str] = None nav_color: Optional[str] = None media_directory: Optional[str] = None + tags_directory: Optional[bool] = None throttle_on: Optional[bool] = None throttle_on_startup: Optional[bool] = None videos_destination: Optional[str] = None diff --git a/wrolpi/tags.py b/wrolpi/tags.py index 21654581..4840cf05 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -13,7 +13,7 @@ from sqlalchemy.orm import relationship, Session from wrolpi import dates, flags from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory, background_task, \ - get_relative_to_media_directory, is_valid_hex_color, walk, INVALID_FILE_CHARS + get_relative_to_media_directory, is_valid_hex_color, walk, INVALID_FILE_CHARS, get_wrolpi_config from wrolpi.dates import TZDateTime from wrolpi.db import get_db_curs, get_db_session from wrolpi.downloader import save_downloads_config @@ -629,6 +629,11 @@ be AUTOMATICALLY DELETED! def sync_tags_directory(): """Synchronizes database Tags with the Tags directory (typically /media/wrolpi/tags). Removes any files that do not belong in the Tags Directory.""" + wrolpi_config = get_wrolpi_config() + if not wrolpi_config.tags_directory: + logger.debug('Skipping tags directory sync because tags_directory setting is disabled') + return + with get_db_session() as session: try: tags_directory = get_tags_directory() diff --git a/wrolpi/test/test_tags.py b/wrolpi/test/test_tags.py index 11b60d66..90b7e972 100644 --- a/wrolpi/test/test_tags.py +++ b/wrolpi/test/test_tags.py @@ -5,7 +5,7 @@ import pytest import yaml from wrolpi import tags -from wrolpi.common import is_hardlinked, walk +from wrolpi.common import is_hardlinked, walk, get_wrolpi_config from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, UsedTag from wrolpi.files.models import FileGroup from wrolpi.tags import TagFile, Tag @@ -507,3 +507,31 @@ async def test_tag_rename_with_channel(test_session, test_directory, video_facto # Cannot delete Tag when used by Channel. This prevents the need to move Channel directories when Tags are deleted. with pytest.raises(UsedTag): tag.delete() + + +@pytest.mark.asyncio +async def test_tags_directory_disabled(test_session, test_directory, tag_factory, video_factory, await_switches): + """Test that Tag Directory is NOT synchronized when tags_directory setting is disabled.""" + tags_dir = test_directory / 'tags' + + # Disable the tags_directory setting. + wrolpi_config = get_wrolpi_config() + wrolpi_config.tags_directory = False + + # Create a tag and tag a video. + tag = await tag_factory() + video = video_factory(with_video_file=test_directory / 'video.mp4') + video.add_tag(test_session, tag.id) + test_session.commit() + await await_switches() + + # Tags directory should NOT be created because the setting is disabled. + assert not tags_dir.exists(), 'Tags directory should not exist when tags_directory setting is disabled' + + # Enable the setting and sync again. + wrolpi_config.tags_directory = True + tags.sync_tags_directory() + + # Now the tags directory should exist with the linked file. + assert tags_dir.is_dir(), 'Tags directory should exist after enabling setting' + assert (tags_dir / tag.name / 'video.mp4').is_file(), 'Video should be linked in tags directory'