wrolpi/wrolpi/files/lib.py
lrnselfreliance aff5f1eb1f
A directory rename is finished by the file worker, not the request (#950)
* A directory rename is finished by the file worker, not the request

A directory rename queued a move, waited for it, then removed the old
directory and updated nested Collections itself.  A rename that outlasted
Sanic's RESPONSE_TIMEOUT (60 s) was cancelled while waiting: the move still
finished, but the old directory and its emptied subdirectories were left
behind, nested Collections kept their old directories, and the browser was
told the rename failed.

The move job now finishes the rename (nested Collections, then removing the
old directory, empty directories only), and the request returns once the
move is queued.  A failure is reported by the job's event.  An empty
directory can be renamed (it raised before).  The worker status carries
`rename_from`, so the Files page progress bar labels the rename, and the
file browser fetches again when the file worker is done.

A directory rename no longer queues a refresh of its parent: the file
worker runs the newest task first, so it scanned the directory before the
rename.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* A queued rename returns its job, and the file browser waits for it

Review fixes:

- The cancellation test patched `wait_for_job`, which this path no longer
  calls, so it cancelled nothing.  It now asserts the invariant: the
  request only queues the rename (it never waits, and nothing has moved
  when it returns), and the file worker alone finishes it.

- The file browser inferred the end of a rename from the file worker's
  polled status.  A short job can start and finish between two polls, and
  the first idle poll (before the job started) spent the "fetch once"
  token, so the table could keep the old name.  POST /api/files/rename now
  returns `{"job_id": ...}` (null when the rename is already done), GET
  /api/files/jobs/<job_id> reports pending/complete/failed, and the file
  browser polls that job before fetching.

- useBrowseFiles ignores a fetch that settles after a newer one, so a
  listing requested before a rename cannot replace the one after it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 13:09:33 -07:00

2970 lines
126 KiB
Python

import asyncio
import contextlib
import dataclasses
import datetime
import functools
import glob
import hashlib
import json
import mimetypes
import os
import pathlib
import re
import shutil
import subprocess
import tarfile
import threading
import time
import urllib.parse
import zipfile
from itertools import zip_longest
from pathlib import Path
from typing import Callable, ContextManager, List, Optional, Tuple, Union, Dict, Generator, Iterable, Set
import cachetools.func
from sqlalchemy import and_, asc, or_, text as sa_text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, undefer, joinedload
from wrolpi.cmd import which
from wrolpi.common import get_media_directory, wrol_mode_check, logger, \
chunks, chunks_by_stem, walk, \
get_wrolpi_config, \
unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory, strip_surrogates
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
from wrolpi.config_version import count_config_change_curs
from wrolpi.db import get_db_session, get_db_curs, values_clause, session_curs
from wrolpi.downloader import download_manager, Download
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
NoPrimaryFile, InvalidDirectory, IgnoredDirectoryError, UnsupportedArchive, InvalidArchiveMember
from wrolpi.events import Events
from wrolpi.files.models import FileGroup, Directory
from wrolpi.files.progress import normalize_progress
from wrolpi.lang import ISO_639_CODES, ISO_3166_CODES
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config, sync_tags_directory
from wrolpi.vars import PYTEST, IS_MACOS
try:
import magic
mime = magic.Magic(mime=True)
except ImportError:
# Magic is not installed
magic = None
mime = None
logger = logger.getChild(__name__)
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'search_files',
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'choose_primary_file', 'get_file_statistics',
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href',
'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories',
'get_normalized_ignored_directories', 'sanitize_ignored_directories',
'cached_search_total', 'search_filter_cache_key', 'count_file_groups',
'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems', 'get_kv_value', 'set_kv_value',
'backfill_null_file_group_stems', 'dedupe_file_groups_by_directory_stem', 'merge_file_group_relations',
'merge_into_surviving_duplicates']
def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
"""Returns all Tag names for the provided file path."""
tags = session.query(Tag) \
.join(TagFile, Tag.id == TagFile.tag_id) \
.join(FileGroup, FileGroup.id == TagFile.file_group_id) \
.filter(FileGroup.primary_path == str(file))
names = sorted([tag.name for tag in tags], key=lambda i: i.lower())
return names
def sanitize_filename_surrogates(path: pathlib.Path) -> pathlib.Path:
"""
Check if a path contains invalid UTF-8 surrogates. If so, rename the file
on disk to replace them with underscores.
Linux filesystems store filenames as bytes. Python represents invalid UTF-8
sequences as surrogates. These surrogates cannot be stored in the database.
Returns the (possibly new) path.
"""
path_str = str(path)
# Check for surrogates by trying to encode as UTF-8
try:
path_str.encode('utf-8')
return path # Path is valid
except UnicodeEncodeError:
pass
# Get raw bytes from filesystem and decode, replacing invalid sequences
raw_bytes = os.fsencode(path)
# Replace invalid UTF-8 sequences with underscore
sanitized_name = raw_bytes.decode('utf-8', errors='replace').replace('\ufffd', '_')
sanitized_path = pathlib.Path(sanitized_name)
if path.exists() and sanitized_path != path:
if sanitized_path.exists():
# Target already exists, add a suffix
stem = sanitized_path.stem
suffix = sanitized_path.suffix
counter = 1
while sanitized_path.exists():
sanitized_path = sanitized_path.with_name(f'{stem}_{counter}{suffix}')
counter += 1
logger.warning(f'Renaming file with invalid UTF-8 characters: {path!r} -> {sanitized_path}')
path.rename(sanitized_path)
return sanitized_path
return path
def _get_file_dict(session: Session, file: pathlib.Path) -> Dict:
media_directory = get_media_directory()
try:
size = file.stat().st_size
except PermissionError:
size = None
try:
mimetype = get_mimetype(file)
except PermissionError:
mimetype = None
file_group = FileGroup.get_by_path(session, file)
return dict(
path=file.relative_to(media_directory),
size=size,
mimetype=mimetype,
tags=get_file_tag_names(session, file),
progress=file_group.progress if file_group else None,
position=file_group.position if file_group else None,
# When a finished file (no position) was finished; the preview compares it with its own position.
viewed=file_group.viewed if file_group else None,
)
def get_file_dict(session: Session, file: str) -> Dict:
media_directory = get_media_directory()
return _get_file_dict(session, media_directory / file)
async def set_file_viewed(file: pathlib.Path):
"""Change FileGroup.viewed to the current datetime.
Files in ignored directories are not tracked. Runs after the request that viewed the file, so it writes in a
session of its own, not the request's.
"""
if is_ignored_file(file):
return
with get_db_session(commit=True) as session:
try:
fg = FileGroup.find_by_path(session, file)
except UnknownFile:
fg = FileGroup.from_paths(session, file)
fg.do_model(session)
fg.set_viewed()
def _trackable_path(file: pathlib.Path) -> pathlib.Path | None:
"""Return `file` with any `..` collapsed, or None if it is in an ignored directory.
`..` is collapsed first so it cannot step into an ignored directory past the check.
@raise InvalidFile: The file is not in the media directory.
"""
file = pathlib.Path(os.path.normpath(file))
if not get_paths_in_media_directory([file]):
raise InvalidFile('File must be in the media directory')
if is_ignored_file(file):
return None
return file
def set_file_progress(session: Session, file: pathlib.Path, progress: float, position: dict = None) \
-> Tuple[float | None, dict | None]:
"""Record how far the User is through a file (see wrolpi.files.progress). This is also a view.
The caller owns the (write) transaction, and saves the recently viewed config after it commits.
@return: The (progress, position) that was stored, (None, None) if the file is not tracked.
"""
progress, position = normalize_progress(progress, position)
if not (file := _trackable_path(file)):
return None, None
try:
fg = FileGroup.find_by_path(session, file)
except UnknownFile:
if not file.is_file():
raise InvalidFile(f'File does not exist: {get_relative_to_media_directory(file)}')
fg = FileGroup.from_paths(session, file)
fg.do_model(session)
# Not `set_viewed`, which saves the config; a heartbeat during playback only writes the DB.
fg.viewed = now()
fg.progress = progress
fg.position = position
return progress, position
def clear_file_progress(session: Session, file: pathlib.Path) -> bool:
"""Forget how far the User is through a file, so it starts from the beginning. It remains viewed.
A file that was never tracked is left alone. The caller owns the (write) transaction.
@return: True if a FileGroup was changed.
"""
if not (file := _trackable_path(file)):
return False
if not (fg := FileGroup.get_by_path(session, file)):
return False
fg.progress = None
fg.position = None
return True
@cachetools.func.ttl_cache(10_000, 30.0)
def _get_directory_dict(directory: pathlib.Path,
directories_cache: str, # Used to cache by requested directories.
) -> Dict:
media_directory = get_media_directory()
try:
is_empty = not next(directory.iterdir(), False)
except PermissionError:
is_empty = False
return dict(
path=f'{directory.relative_to(media_directory)}/',
is_empty=is_empty,
)
def _get_recursive_directory_dict(session: Session, directory: pathlib.Path, directories: List[pathlib.Path]) -> Dict:
directories_cache = str(sorted(directories))
d = _get_directory_dict(directory, directories_cache)
if directory in directories:
children = dict()
for path in directory.iterdir():
if not path.is_dir() and path.name in HIDDEN_FILES:
# Never show ignored files.
continue
if path.is_dir() and path in directories:
children[f'{path.name}/'] = _get_recursive_directory_dict(session, path, directories)
elif path.is_dir():
children[f'{path.name}/'] = _get_directory_dict(path, directories_cache)
else:
children[path.name] = _get_file_dict(session, path)
d['children'] = children
return d
HIDDEN_DIRECTORIES = ('lost+found',)
# macOS metadata files which should never be shown in the FileBrowser.
HIDDEN_FILES = ('.DS_Store', '._.DS_Store')
def list_directories_contents(session: Session, directories_: List[str]) -> Dict:
"""List all files down to (and within) the directories provided.
This should only be used by the frontend for it's FileBrowser."""
media_directory = get_media_directory()
# Convert all directories to pathlib.Path.
directories = {media_directory / i for i in directories_}
if invalid_directories := [i for i in directories if not i.is_dir()]:
raise FileNotFoundError(f'Invalid directories: {invalid_directories}')
# Get all unique parents for each directory, but not if they are above the media directory.
parents = {j for i in directories for j in i.parents if j not in media_directory.parents}
# Add unique parents onto the requested directories.
directories = list(directories | parents)
paths = dict()
for path in media_directory.iterdir():
if path.is_dir() and path.name in HIDDEN_DIRECTORIES:
# Never show ignored directories.
continue
if not path.is_dir() and path.name in HIDDEN_FILES:
# Never show ignored files.
continue
if path.is_dir():
paths[f'{path.name}/'] = _get_recursive_directory_dict(session, path, directories)
else:
paths[path.name] = _get_file_dict(session, path)
return paths
def get_tagged_file_groups_by_ids(session: Session, file_group_ids: List[int]) -> List[dict]:
"""Return serialized FileGroups (subset of `file_group_ids`) that have at least one TagFile.
Used by the unified delete endpoint to require force-confirmation before deleting tagged files."""
if not file_group_ids:
return []
rows = session.query(FileGroup) \
.options(*FileGroup.json_options()) \
.join(TagFile, TagFile.file_group_id == FileGroup.id) \
.filter(FileGroup.id.in_(file_group_ids)) \
.distinct() \
.all()
return [fg.__json__() for fg in rows]
def directory_or_descendant(column, directory: Union[str, pathlib.Path]):
"""A filter matching `directory` and every directory beneath it.
A range is used instead of `LIKE 'directory/%'` so SQLite can use the column's index (it cannot for a
case-insensitive LIKE), and so `_` and `%` in a path are not wildcards. '0' is the character after '/'."""
directory = str(directory)
return or_(column == directory, and_(column >= f'{directory}/', column < f'{directory}0'))
def get_tagged_file_groups_by_paths(session: Session, paths: Iterable[pathlib.Path]) -> List[FileGroup]:
"""Return FileGroup objects under any of the given paths that have at least one TagFile.
For a file path, matches FileGroups whose primary_path shares the file's stem in the same
directory. For a directory path, matches FileGroups whose `directory` is the path or a
descendant. Used by the path-based delete endpoint to require force-confirmation before
deleting tagged files."""
paths = list(paths)
if not paths:
return []
tagged: List[FileGroup] = []
seen_ids = set()
for path in paths:
query = session.query(FileGroup) \
.options(*FileGroup.json_options()) \
.join(TagFile, TagFile.file_group_id == FileGroup.id)
if path.is_file():
# Could be deleting a file in a FileGroup that has been tagged.
stem, _ = split_path_stem_and_suffix(path)
query = query.filter(FileGroup.primary_path.like(f'{path.parent}/{stem}%'))
else:
# Search for any FileGroups that have been tagged under this directory.
# Use indexed directory column for efficient lookup.
query = query.filter(directory_or_descendant(FileGroup.directory, path))
for file_group in query:
if path.is_dir() or any(p for p in paths if p in file_group.my_paths()):
if file_group.id not in seen_ids:
seen_ids.add(file_group.id)
tagged.append(file_group)
return tagged
def delete_file_groups(session: Session, file_group_ids: List[int], force: bool = False) -> Set[str]:
"""Delete FileGroups (and their model rows via FK CASCADE) by ID.
Returns the set of `FileGroup.model` values that were deleted, so the caller can fire any
model-specific post-delete switches (e.g., `save_channels_config` for videos).
The caller is expected to have already gated this on tag confirmation via
`get_tagged_file_groups_by_ids`. `force` propagates to FileGroup.delete so the safety net
at FileGroup.delete (raises FileGroupIsTagged if force is False and tags exist) still
protects callers that skipped the pre-check."""
files = FileGroup.get_by_ids(session, file_group_ids)
if not files:
raise UnknownFile(f'No FileGroups found for ids: {file_group_ids}')
affected_models: Set[str] = {fg.model for fg in files if fg.model}
had_tagged = any(fg.tag_names for fg in files)
# Capture domain Collection IDs for any Archive FileGroups so we can prune empty ones after
# the cascade removes the Archive rows. (Archive.delete used to do this per-archive; with
# cascade-based deletion we do it as a single post-pass.)
archive_collection_ids: Set[int] = set()
if 'archive' in affected_models:
from modules.archive.models import Archive
archive_collection_ids = {
collection_id for (collection_id,) in session.query(Archive.collection_id)
.filter(Archive.file_group_id.in_([fg.id for fg in files]))
if collection_id is not None
}
for fg in files:
fg.delete(force=force)
if archive_collection_ids:
# Flush so the FK CASCADE has removed the Archive rows before we count remaining.
session.flush()
from modules.archive.models import Archive
from wrolpi.collections.models import Collection
for collection_id in archive_collection_ids:
collection = session.query(Collection).filter_by(id=collection_id, kind='domain').one_or_none()
if collection is None:
continue
remaining = session.query(Archive).filter_by(collection_id=collection_id).count()
if remaining == 0:
session.delete(collection)
if had_tagged:
save_tags_config.activate_switch()
return affected_models
@wrol_mode_check
async def delete(*paths: Union[str, pathlib.Path], force: bool = False, post_processing: bool = True):
"""
Delete a file or directory in the media directory.
If any files are tagged and force is False, returns a list of tagged FileGroups without deleting.
If force is True, deletes even tagged files.
The DB cleanup of the deleted FileGroups finishes before this returns; the global modelers/indexers are
queued in the FileWorker. If post_processing is False they are not queued at all — use this when the caller will trigger modeling itself (e.g. the upload endpoint
follows up with `upsert_file`)."""
media_directory = get_media_directory()
paths = [media_directory / i for i in paths]
if any(i == media_directory for i in paths):
raise InvalidFile(f'Cannot delete the media directory')
for path in paths:
if not path.is_dir() and not path.is_file():
logger.warning(f'Cannot delete {path} because it is not a file or a directory.')
if len(paths) > 1:
for p1 in paths:
for p2 in paths:
if p1 == p2:
continue
if p1 != p2 and p1 in p2.parents:
raise InvalidFile(f'Cannot deleted nested paths')
# Discover any tagged FileGroups under these paths so we can either gate the deletion (when
# !force) or fire `save_tags_config` afterwards (when force=True and any were tagged).
with get_db_session() as session:
tagged_file_groups = get_tagged_file_groups_by_paths(session, paths)
if tagged_file_groups and not force:
# Serialize while session is still active.
return [i.__json__() for i in tagged_file_groups]
had_tagged = bool(tagged_file_groups)
# Remove the Tags Directory links of the files about to be deleted, while they can still be matched to
# their sources. A deleted directory takes every file of a FileGroup inside it.
directories = [i for i in paths if i.is_dir()]
files = [i for i in paths if i.is_file()]
for file_group in tagged_file_groups:
if any(i == file_group.directory or i in file_group.directory.parents for i in directories):
file_group.unlink_tag_directory_links()
elif files:
file_group.unlink_tag_directory_links(*files)
for path in paths:
ignored_directories = get_wrolpi_config().ignored_directories
if ignored_directories and str(path) in ignored_directories:
remove_ignored_directory(path)
if path.is_dir():
delete_directory(path, recursive=True, force=force)
elif path.exists():
path.unlink()
# Refresh synchronously to clean up DB before returning.
# This ensures deleted FileGroups are removed before the caller continues
# (e.g., upload API recreating the file).
from wrolpi.files.worker import file_worker
await file_worker.refresh_sync(list(paths), post_processing=False)
if post_processing:
# The modelers and indexers cover the whole library, so the request must not wait on them. A queued
# refresh of these paths finds nothing left to change and then runs them in the FileWorker.
file_worker.queue_refresh(list(paths), send_events=False)
if had_tagged:
# `tags.yaml` references TagFiles that the refresh just removed; rewrite it.
save_tags_config.activate_switch()
def _mimetype_suffix_map(path: Path, mimetype: str):
"""Special handling for mimetypes. Python Magic may not return the correct mimetype."""
from modules.docs.models import MOBI_MIMETYPE
suffix = path.suffix.lower()
if mimetype == 'application/octet-stream':
if suffix.endswith('.mobi'):
return MOBI_MIMETYPE
if suffix.endswith('.stl'):
return 'model/stl'
if suffix.endswith('.3mf'):
return 'model/3mf'
if suffix.endswith('.mp4'):
return 'video/mp4'
if suffix.endswith('.html') or suffix.endswith('.htm'):
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
return 'text/html'
if suffix.endswith('.hgt'):
return 'application/octet-stream'
if mimetype.startswith('text/') and suffix in ('.md', '.markdown'):
# libmagic reports markdown as text/plain, or as a programming language when it sniffs code.
return 'text/markdown'
if mimetype == 'text/plain':
if suffix.endswith('.json'):
return 'application/json'
if suffix.endswith('.vtt'):
return 'text/vtt'
if suffix.endswith('.csv'):
return 'text/csv'
if suffix.endswith('.html') or suffix.endswith('.htm'):
return 'text/html'
if suffix.endswith('.stl'):
return 'model/stl'
if suffix.endswith('.scad'):
return 'application/x-openscad'
if suffix.endswith('.js'):
return 'application/javascript'
if suffix.endswith('.css'):
return 'text/css'
if suffix.endswith('.srt'):
return 'text/srt'
if suffix.endswith('.yaml') or suffix.endswith('.yml'):
return 'text/yaml'
if suffix.endswith('.azw3'):
return 'application/vnd.amazon.mobi8-ebook'
if suffix.endswith('.obj'):
return 'model/obj'
if mimetype == 'application/x-subrip':
# Fallback to old mimetype.
return 'text/srt'
if mimetype == 'application/zip' and (suffix == '.html' or suffix == '.htm'):
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
return 'text/html'
if mimetype == 'application/zip' and suffix == '.docx':
return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
if mimetype == 'application/zip' and suffix == '.cbz':
return 'application/x-cbz'
if mimetype == 'application/zip' and suffix == '.3mf':
return 'model/3mf'
if mimetype == 'application/zip' and suffix == '.odt':
return 'application/vnd.oasis.opendocument.text'
if mimetype in ('application/x-rar', 'application/x-rar-compressed') and suffix == '.cbr':
return 'application/x-cbr'
if mimetype == 'video/x-ms-asf' and suffix == '.wma':
return 'audio/x-ms-wma'
return mimetype
FILE_BIN = which('file', '/usr/bin/file')
@functools.lru_cache(maxsize=10_000)
def get_mimetype(path: Path) -> str:
"""Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command."""
if magic is None:
# This method is slow, prefer the speedier `magic` module.
cmd = (FILE_BIN, '--mime-type', str(path.absolute()))
output = subprocess.check_output(cmd)
output = output.decode()
mimetype = output.split(' ')[-1].strip()
else:
mimetype = mime.from_file(path)
mimetype = _mimetype_suffix_map(path, mimetype)
return mimetype
# Special suffixes within WROLPi.
SUFFIXES = {
'.info.json',
'.ffprobe.json',
'.live_chat.json',
'.readability.html',
'.readability.json',
'.readability.txt',
}
SUFFIXES |= {f'.{i}.srt' for i in ISO_639_CODES}
SUFFIXES |= {f'.{i}.vtt' for i in ISO_639_CODES}
# Convert to lowercase frozensets for O(1) case-insensitive lookup
_SUFFIXES_LOWER = frozenset(s.lower() for s in SUFFIXES)
_LANGUAGES_LOWER = frozenset(i.lower() for i in ISO_639_CODES)
_REGIONS_LOWER = frozenset(i.lower() for i in ISO_3166_CODES)
def _is_language_region_suffix(suffix_lower: str) -> bool:
"""Is this lowercase .srt/.vtt suffix a language with a region or "auto", like ".en-us.srt" or ".en-auto.vtt"?
Never build the set of every combination: 336,000 strings cost ~2s and ~65MiB in each process on a Pi 4. A
region code never contains "-"; a language code may ("en-gb")."""
language, _, region = suffix_lower[1:-4].rpartition('-')
return language in _LANGUAGES_LOWER and (region == 'auto' or region in _REGIONS_LOWER)
_HARDCODED_SUFFIXES = frozenset(
{'.info.json', '.ffprobe.json', '.live_chat.json', '.readability.html', '.readability.json', '.readability.txt'})
# Bump when SUFFIXES / split_path_stem_and_suffix grouping rules change. A stored
# file_group.stem is a cache of this algorithm; compare joins on it, so a mismatch
# looks like delete+insert and would drop tags. ensure_file_group_stems() must run
# before any compare when this version does not match the value in wrolpi_kv.
STEM_ALGORITHM_VERSION = 1
STEM_ALGORITHM_VERSION_KEY = 'stem_algorithm_version'
# Cancelled compare_file_groups leaves ensure_file_group_stems running in a thread.
# Tests (and overlapping refreshes) share one SQLite writer; serialize the backfill.
_stem_backfill_lock = threading.Lock()
PART_PARSER = re.compile(r'(.+?)(\.f[\d]{2,3})?(\.info)?(\.\w{3,4})(\.part)', re.IGNORECASE)
def _extract_candidate_suffixes(name_lower: str) -> List[str]:
"""Extract potential suffixes from filename (last 1, 2, 3 dot-parts).
Returns suffixes in order from shortest to longest.
"""
parts = name_lower.rsplit('.', 3)
if len(parts) == 1:
return []
candidates = []
append_candidate = candidates.append
suffix = ''
for i in range(len(parts) - 1, 0, -1):
suffix = '.' + parts[i] + suffix
append_candidate(suffix)
return candidates
@functools.lru_cache(maxsize=5_000)
def _get_suffix_length(name_lower: str) -> int:
"""Get length of matching suffix using O(1) set lookups.
Returns 0 if no suffix matches.
"""
# Fast path: check hardcoded multi-part suffixes first
for suffix in _HARDCODED_SUFFIXES:
if name_lower.endswith(suffix):
return len(suffix)
candidates = _extract_candidate_suffixes(name_lower)
if not candidates:
return 0
# Check SUFFIXES (longest first for most specific match)
for suffix in reversed(candidates):
if suffix in _SUFFIXES_LOWER:
return len(suffix)
# Only .srt/.vtt files have language-region suffixes
simple_suffix = candidates[0] if candidates else ''
if simple_suffix in {'.srt', '.vtt'}:
for suffix in reversed(candidates):
if _is_language_region_suffix(suffix):
return len(suffix)
return len(simple_suffix)
@functools.lru_cache(maxsize=10_000)
def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = False) -> Tuple[str, str]:
"""Get the path's stem and suffix.
This function handles WROLPi suffixes like .info.json.
>>> split_path_stem_and_suffix('/foo/bar.txt')
('bar', '.txt')
>>> split_path_stem_and_suffix('/foo/bar.txt', full=True)
('/foo/bar', '.txt')
>>> split_path_stem_and_suffix('/foo/bar.info.json', full=True)
('/foo/bar', '.info.json')
"""
path = pathlib.Path(path) if isinstance(path, str) else path
if path.suffix == '.part':
# yt-dlp uses part files while downloading, include those in a FileGroup.
stem, info, format_num, suffix, part = PART_PARSER.match(path.name).groups()
info = info or ''
format_num = format_num or ''
stem = f'{path.parent}/{stem}' if full else stem
return stem, f'{info}{format_num}{suffix}{part}'
name = path.name
name_lower = name.lower()
# Get suffix length using O(1) set lookups instead of O(n) linear search
suffix_len = _get_suffix_length(name_lower)
if suffix_len > 0:
idx = -suffix_len
if full:
return f'{path.parent}/{name[:idx]}', name[idx:]
else:
return name[:idx], name[idx:]
# Path has no suffix.
if full:
return f'{path.parent}/{path.name}', ''
else:
return path.name, ''
def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ...] | Set[pathlib.Path]) \
-> List[pathlib.Path]:
"""Returns the first of each Path with a unique stem. Used to detect if a group of files share a stem."""
results = unique_by_predicate(files, lambda i: split_path_stem_and_suffix(i)[0])
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'get_unique_files_by_stem {files=}')
logger.trace(f'get_unique_files_by_stem {results=}')
return results
# The metadata JSON holds the sums, so it cannot hold its own.
METADATA_JSON_SUFFIXES = ('.info.json', '.readability.json')
# A file's (modification time in nanoseconds, hex sha256), keyed by its path.
FileSums = Dict[pathlib.Path, Tuple[int, str]]
def sha256_files(paths: Iterable[pathlib.Path], known: FileSums = None) -> FileSums:
"""Hash each file, skipping the metadata JSON and files that do not exist.
A `known` sum is reused while its file's modification time is unchanged. This reads whole files; call it in a
thread."""
known = known or dict()
sums = dict()
for path in paths:
if path.name.endswith(METADATA_JSON_SUFFIXES) or not path.is_file():
continue
mtime_ns = path.stat().st_mtime_ns
if (known_sum := known.get(path)) and known_sum[0] == mtime_ns:
sums[path] = known_sum
continue
with path.open('rb') as fh:
sums[path] = (mtime_ns, hashlib.file_digest(fh, 'sha256').hexdigest())
return sums
def sha256_by_suffix(sums: FileSums) -> Dict[str, str]:
"""The hex sha256 of each file keyed by its suffix. The files of a FileGroup share a stem, so a suffix names one
file even after the group is renamed."""
return {split_path_stem_and_suffix(path)[1]: sha256 for path, (_, sha256) in sums.items()}
refresh_logger = logger.getChild('refresh')
def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]:
"""This generates `FileGroup.files` from a list of files.
Paths are stored as filenames only (relative to FileGroup.directory).
"""
files = list()
for file in group:
mimetype = get_mimetype(file)
stat = file.stat()
modification_datetime = str(from_timestamp(stat.st_mtime))
_, suffix = split_path_stem_and_suffix(file)
size = stat.st_size
files.append(dict(
path=file.name, # Store filename only, not full path
mimetype=mimetype,
modification_datetime=modification_datetime,
size=size,
suffix=suffix,
))
return files
# The stem of a yt-dlp per-format file, which is merged into the final media file. Example: The Title.f313.webm
YTDLP_FORMAT_STEM = re.compile(r'\.f\d{2,4}$')
def get_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path:
"""Given a list of files, return the file that we can model or index.
@raise NoPrimaryFile: If not primary file can be found."""
from modules.archive.lib import is_singlefile_file
if len(files) == 0:
raise ValueError(f'Cannot find primary file without any files')
elif len(files) == 1:
# Only one file is always the primary.
return files[0]
file_mimetypes = list()
for idx, i in enumerate(files):
try:
file_mimetypes.append((i, get_mimetype(i)))
except FileNotFoundError:
# File was deleted.
pass
if not file_mimetypes:
raise FileNotFoundError(f'Cannot find primary file. All files are no longer accessible: {files[0]}')
# EPUB has high priority.
mimetypes = {i[1] for i in file_mimetypes}
has_epub = any(i.startswith('application/epub') for i in mimetypes)
# A yt-dlp per-format file (video-only or audio-only) is primary only when no merged media file is present.
has_merged_media = any(m.startswith(('video/', 'audio/')) and not YTDLP_FORMAT_STEM.search(f.stem)
for f, m in file_mimetypes)
for file, mimetype in file_mimetypes:
# These are the files that can be modeled.
if is_singlefile_file(file):
return file
if mimetype.startswith(('video/', 'audio/')):
if has_merged_media and YTDLP_FORMAT_STEM.search(file.stem):
continue
return file
if mimetype.startswith('application/epub'):
return file
if mimetype.startswith('application/pdf') and not has_epub:
return file
if mimetype.startswith('application/x-mobipocket-ebook') and not has_epub:
# Only return a MOBI if no EPUB is present.
return file
if mimetype in ('application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.oasis.opendocument.text',
'application/x-cbz', 'application/x-cbr',
'application/vnd.comicbook+zip', 'application/vnd.comicbook-rar'):
return file
if file.suffix.lower() in ('.cbz', '.cbr', '.cbt', '.cb7'):
return file
# Secondary file types.
for file, mimetype in file_mimetypes:
if mimetype.startswith('image/'):
return file
raise NoPrimaryFile(f'Cannot find primary file for group: {files}')
def choose_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path:
"""Primary file for a stem group. Falls back to a sorted path so every caller agrees.
`_upsert_files` and `_upsert_file_groups` must pick the same path. A set's iteration
order is not stable; using it as the fallback would DELETE the existing tagged row
and INSERT a new untagged one.
"""
paths = [pathlib.Path(p) for p in files]
if not paths:
raise ValueError('Cannot find primary file without any files')
try:
return get_primary_file(paths)
except NoPrimaryFile:
return sorted(paths)[0]
def _file_names_sql(table: str) -> str:
"""SQL: the sorted names in `table`.files, as a JSON array.
Old FileGroups store each file's absolute path, new ones its name; both give the name."""
path = "json_extract(j.value, '$.path')"
return f'''(SELECT json_group_array(name) FROM (
SELECT CASE WHEN {path} LIKE '/%' THEN substr({path}, length({table}.directory) + 2) ELSE {path} END
AS name
FROM json_each({table}.files) AS j
ORDER BY name))'''
def _rank_file_groups(curs, file_group_ids: Iterable[int]) -> List[int]:
"""The ids of these FileGroups, the one most worth keeping first: modeled, then tagged, then with files, then
the newest."""
curs.execute('''
SELECT fg.id,
CASE
WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0
ELSE json_array_length(fg.files)
END AS file_count,
CASE
WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1
ELSE 0
END AS has_model,
(SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count
FROM file_group fg
WHERE fg.id IN (SELECT value FROM json_each(?))
''', (json.dumps(list(file_group_ids)),))
members = list(curs.fetchall())
members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True)
return [m['id'] for m in members]
def _count_primary_path_change(curs, file_group_id: int):
"""tags.yaml, playlists.yaml and recently_viewed.yaml save a FileGroup by its primary path; count a change of
that path for each config that holds this FileGroup."""
curs.execute('''
SELECT EXISTS (SELECT 1 FROM tag_file WHERE file_group_id = :id) AS tagged,
EXISTS (SELECT 1
FROM collection_item ci
JOIN collection c ON c.id = ci.collection_id
WHERE ci.file_group_id = :id
AND c.kind = 'playlist') AS in_playlist,
(SELECT viewed IS NOT NULL FROM file_group WHERE id = :id) AS viewed
''', {'id': file_group_id})
row = curs.fetchone()
names = [name for name, held in (('tags.yaml', row['tagged']), ('playlists.yaml', row['in_playlist']),
('recently_viewed.yaml', row['viewed'])) if held]
if names:
count_config_change_curs(curs, *names)
def _collapse_file_group(curs, loser_id: int, winner_id: int):
"""Delete the loser FileGroup after giving its tags, collection membership and viewing state to the winner.
The winner keeps its own viewing state when it has one."""
merge_file_group_relations(curs, loser_id, winner_id)
curs.execute('SELECT viewed IS NOT NULL AS viewed FROM file_group WHERE id = ?', (loser_id,))
if curs.fetchone()['viewed']:
# recently_viewed.yaml saves the loser by its primary path.
count_config_change_curs(curs, 'recently_viewed.yaml')
curs.execute('''
UPDATE file_group
SET viewed = COALESCE(file_group.viewed, loser.viewed),
progress = COALESCE(file_group.progress, loser.progress),
position = COALESCE(file_group.position, loser.position)
FROM (SELECT viewed, progress, position FROM file_group WHERE id = :loser) AS loser
WHERE file_group.id = :winner
''', {'loser': loser_id, 'winner': winner_id})
curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,))
def _give_stem_owners_their_primary_paths(curs, values: List[tuple]):
"""Give each incoming primary path to the FileGroup that already owns its (directory, stem).
`_upsert_files` then updates that FileGroup through ON CONFLICT (primary_path) instead of colliding with it on
the (directory, stem) unique index. The FileGroup keeps its id, so its tags, collection membership, model and
viewing state stay with it. If a different FileGroup owns the primary path, the two collapse into the one
`_rank_file_groups` prefers.
@param values: `_upsert_files` rows: (primary_path, directory, ..., stem)."""
incoming = json.dumps([[value[1], value[-1], value[0]] for value in values])
curs.execute('''
SELECT fg.id, json_extract(v.value, '$[2]') AS primary_path
FROM json_each(:incoming) AS v
JOIN file_group fg ON fg.directory = json_extract(v.value, '$[0]')
AND fg.stem = json_extract(v.value, '$[1]')
WHERE fg.primary_path != json_extract(v.value, '$[2]')
-- A FileGroup whose primary path is incoming is updated by its own group.
AND fg.primary_path NOT IN (SELECT json_extract(value, '$[2]') FROM json_each(:incoming))
''', {'incoming': incoming})
for stem_owner_id, primary_path in curs.fetchall():
curs.execute('SELECT id FROM file_group WHERE primary_path = ?', (primary_path,))
if path_owner := curs.fetchone():
winner_id, loser_id = _rank_file_groups(curs, [stem_owner_id, path_owner['id']])
_collapse_file_group(curs, loser_id, winner_id)
if winner_id == path_owner['id']:
# The survivor already has the primary path.
continue
_count_primary_path_change(curs, stem_owner_id)
curs.execute('UPDATE file_group SET primary_path = ? WHERE id = ?', (primary_path, stem_owner_id))
# About how many files `_upsert_files` writes per transaction. A stem's files are never split across chunks.
UPSERT_CHUNK_SIZE = 500
def _upsert_files(files: List[pathlib.Path],
progress_callback: Callable[[int, int], None] = None):
"""Insert/update all records of the provided files.
Any inserted files will be marked with `indexed=false`. Any existing files will only be updated if their
size/modification_datetime changed.
Args:
files: List of file paths to upsert
progress_callback: Optional callback(processed, total) called after each chunk is processed
It is assumed all files exist."""
total_files = len(files)
processed_count = 0
for chunk in chunks_by_stem(files, UPSERT_CHUNK_SIZE):
# Group all files by their shared full-stem. `chunks_by_stem` keeps each stem in one chunk.
grouped = group_files_by_stem(chunk)
# Convert the groups into file_groups. Extract common information into `file_group.data`.
values = dict()
# The primary path of each companion file's group.
companion_owners = dict()
for group in grouped:
# The primary file is the video/SingleFile/epub, etc.
try:
primary_path = choose_primary_file(group)
# The primary mimetype allows modelers to find its file_groups.
mimetype = get_mimetype(primary_path)
# The group uses a common modification_datetime so the group will be re-indexed when any of it's
# files are modified.
modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group))
size = sum(i.stat().st_size for i in group)
files_json = json.dumps(_paths_to_files_dict(group))
stem, suffix = split_path_stem_and_suffix(primary_path)
suffix = (suffix or '').lower() or None
values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem)
companion_owners.update((str(i), str(primary_path)) for i in group if i != primary_path)
except FileNotFoundError as e:
# A file was deleted between discovery and upsert (temp files, WAL sidecars, etc.).
refresh_logger.warning(f'File vanished during refresh, skipping group near {group[0]}', exc_info=e)
continue
values = [(str(primary_path),
str(primary_path.parent), # directory
False, # `indexed` is false to force indexing by default.
*i) for primary_path, i in values.items()]
with get_db_curs(commit=True) as curs:
# SQLite's RETURNING only exposes the post-upsert row; find the previously-indexed
# paths first so re-indexing invalidations can be counted.
primary_paths = [i[0] for i in values]
curs.execute('SELECT primary_path FROM file_group WHERE indexed = true AND primary_path IN '
'(SELECT value FROM json_each(?))', (json.dumps(primary_paths),))
previously_indexed = {i['primary_path'] for i in curs.fetchall()}
# Before INSERT so a new primary_path does not collide on (directory, stem) with the FileGroup that
# already owns that stem.
_give_stem_owners_their_primary_paths(curs, values)
placeholders, params = values_clause(values)
# Compare names, not counts: a file can replace another without changing the size or newest time.
same_files = f'{_file_names_sql("file_group")} = {_file_names_sql("EXCLUDED")}'
stmt = f'''
INSERT INTO file_group
(primary_path, directory, indexed, modification_datetime, mimetype, size, files,
suffix, stem)
VALUES {placeholders}
ON CONFLICT (primary_path) DO UPDATE
SET
directory=EXCLUDED.directory,
modification_datetime=EXCLUDED.modification_datetime,
mimetype=EXCLUDED.mimetype,
size=EXCLUDED.size,
-- Always keep suffix/stem current so grouping and suffix search stay correct.
suffix=EXCLUDED.suffix,
stem=EXCLUDED.stem,
files=(
-- Do not overwrite files unless the files have changed and need to be re-indexed.
CASE
WHEN
file_group.modification_datetime = EXCLUDED.modification_datetime
AND file_group.size = EXCLUDED.size
AND {same_files}
THEN file_group.files
ELSE EXCLUDED.files
END
),
-- Preserve TRUE `indexed` only if the files have not changed.
indexed=(
file_group.indexed = true
AND file_group.modification_datetime = EXCLUDED.modification_datetime
AND file_group.size = EXCLUDED.size
AND {same_files}
)
RETURNING id, primary_path, indexed
'''
curs.execute(stmt, params)
results = list(curs.fetchall())
# Count the files that used to be indexed, but need to be re-indexed.
invalidated_files = len([i['id'] for i in results
if i['primary_path'] in previously_indexed and not i['indexed']])
if invalidated_files:
refresh_logger.info(f'Invalidated indexes of {invalidated_files} file groups near {chunk[0]}')
else:
refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}')
# A companion file can still be the primary path of an old FileGroup (one with a stale stem). Its
# tags, collection membership and viewing state belong to its group's FileGroup.
if companion_owners:
ids_by_primary_path = {i['primary_path']: i['id'] for i in results}
curs.execute('SELECT id, primary_path FROM file_group WHERE primary_path IN '
'(SELECT value FROM json_each(?))', (json.dumps(list(companion_owners)),))
for old in curs.fetchall():
owner_id = ids_by_primary_path[companion_owners[old['primary_path']]]
_collapse_file_group(curs, old['id'], owner_id)
# Report progress after each chunk
processed_count += len(chunk)
if progress_callback:
progress_callback(processed_count, total_files)
def is_ignored_file(file: pathlib.Path) -> bool:
"""Is `file`, or the file it links to, in an ignored directory?
Refresh does not follow links, but a link can be viewed directly."""
return len(remove_files_in_ignored_directories([file, file.resolve()])) < 2
def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]:
"""Return a new list which does not contain any file paths that are in ignored directories."""
from wrolpi.db import get_db_file
ignored_directories = get_normalized_ignored_directories()
# `private` must not hide `private-notes`.
files = [i for i in files
if not any(str(i) == j or str(i).startswith(f'{j}{os.sep}') for j in ignored_directories)]
# Never index the database (or its WAL sidecars), even if the user un-ignores `config`.
db_file = str(get_db_file())
files = [i for i in files if not str(i).startswith(db_file)]
return files
def _ensure_kv_table(curs):
curs.execute('CREATE TABLE IF NOT EXISTS wrolpi_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
def get_kv_value(key: str) -> Optional[str]:
"""The value stored in wrolpi_kv for `key`, or None."""
with get_db_curs() as curs:
_ensure_kv_table(curs)
curs.execute('SELECT value FROM wrolpi_kv WHERE key = ?', (key,))
row = curs.fetchone()
return row['value'] if row else None
def set_kv_value(key: str, value: str):
with get_db_curs(commit=True) as curs:
_ensure_kv_table(curs)
curs.execute(
'INSERT INTO wrolpi_kv (key, value) VALUES (?, ?) '
'ON CONFLICT(key) DO UPDATE SET value = excluded.value',
(key, value),
)
def get_stem_algorithm_version() -> int:
"""The grouping-algorithm version last applied to file_group.stem, or 0 if never stored."""
value = get_kv_value(STEM_ALGORITHM_VERSION_KEY)
return int(value) if value else 0
def set_stem_algorithm_version(version: int):
set_kv_value(STEM_ALGORITHM_VERSION_KEY, str(version))
def ensure_file_group_stems():
"""Recompute file_group.stem when STEM_ALGORITHM_VERSION has changed.
A stored stem is a cache of split_path_stem_and_suffix. Compare joins on (directory, stem),
so an algorithm change without this backfill reports the same files as deleted+new and
drops tags, viewed timestamps, and domain rows. Must run before any compare.
After a successful backfill the kv version matches, and later refreshes skip the table scan.
Upsert, from_paths, move, and reorganize must keep stem current.
A lock serializes callers: cancelling compare_file_groups does not stop the executor
thread, and a second compare would otherwise write wrolpi_kv concurrently (SQLITE_BUSY).
"""
with _stem_backfill_lock:
stored = get_stem_algorithm_version()
if stored == STEM_ALGORITHM_VERSION:
return
with get_db_curs() as curs:
curs.execute('SELECT id, primary_path FROM file_group')
rows = list(curs.fetchall())
if rows:
updates = []
for row in rows:
stem, _ = split_path_stem_and_suffix(row['primary_path'])
updates.append((stem, row['id']))
with get_db_curs(commit=True) as curs:
for chunk in chunks(updates, 500):
curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk)
set_stem_algorithm_version(STEM_ALGORITHM_VERSION)
def backfill_null_file_group_stems():
"""Fill file_group.stem for rows that never got one. Used before creating the unique index."""
with get_db_curs() as curs:
curs.execute("SELECT id, primary_path FROM file_group WHERE stem IS NULL OR stem = ''")
rows = list(curs.fetchall())
if not rows:
return
updates = [(split_path_stem_and_suffix(row['primary_path'])[0], row['id']) for row in rows]
with get_db_curs(commit=True) as curs:
for chunk in chunks(updates, 500):
curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk)
def dedupe_file_groups_by_directory_stem() -> int:
"""Collapse FileGroups that share (directory, stem).
Prefers a row with a domain model, then tags, then non-empty files, then highest id.
Tags and collection membership on losers are moved onto the winner. Returns the
number of FileGroups deleted.
"""
with get_db_curs() as curs:
curs.execute('''
SELECT directory, stem
FROM file_group
WHERE directory IS NOT NULL
AND stem IS NOT NULL
AND stem != ''
GROUP BY directory, stem
HAVING COUNT(*) > 1
''')
keys = [(row['directory'], row['stem']) for row in curs.fetchall()]
deleted = 0
for directory, stem in keys:
with get_db_curs() as curs:
curs.execute('SELECT id FROM file_group WHERE directory = ? AND stem = ?', (directory, stem))
winner_id, *loser_ids = _rank_file_groups(curs, [row['id'] for row in curs.fetchall()])
with get_db_curs(commit=True) as curs:
for loser_id in loser_ids:
merge_file_group_relations(curs, loser_id, winner_id)
curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,))
deleted += 1
if deleted:
refresh_logger.info(f'Collapsed {deleted} duplicate FileGroups sharing (directory, stem)')
return deleted
def merge_file_group_relations(curs, loser_id: int, winner_id: int):
"""Copy the loser FileGroup's tags and collection membership onto the winner, so deleting the loser
(which cascades) does not lose them. The caller deletes the loser."""
curs.execute('''
INSERT OR IGNORE INTO tag_file (tag_id, file_group_id, created_at)
SELECT tag_id, ?, created_at
FROM tag_file
WHERE file_group_id = ?
''', (winner_id, loser_id))
curs.execute('''
UPDATE collection_item
SET file_group_id = ?
WHERE file_group_id = ?
AND collection_id NOT IN (SELECT collection_id
FROM (SELECT collection_id
FROM collection_item
WHERE file_group_id = ?))
''', (winner_id, loser_id, winner_id))
# Raw SQL: count these changes for the configs that save TagFiles and playlist items (wrolpi.config_version).
count_config_change_curs(curs, 'tags.yaml', 'playlists.yaml')
def merge_into_surviving_duplicates(curs, file_group_ids: List[int], deleting_ids: Set[int]) -> int:
"""Before `file_group_ids` are deleted, move each one's tags and collection membership onto a FileGroup
that shares its (directory, stem) and is not in `deleting_ids`. Returns how many FileGroups were merged.
A duplicate FileGroup (same directory and stem) is deleted by refresh; its relations belong to the survivor."""
params = ','.join('?' * len(file_group_ids))
curs.execute(f'''
SELECT fg.id, fg.directory, fg.stem
FROM file_group fg
WHERE fg.id IN ({params})
AND fg.stem IS NOT NULL
AND (EXISTS (SELECT 1 FROM tag_file tf WHERE tf.file_group_id = fg.id)
OR EXISTS (SELECT 1 FROM collection_item ci WHERE ci.file_group_id = fg.id))
''', file_group_ids)
merged = 0
for loser in curs.fetchall():
curs.execute('''
SELECT id
FROM file_group
WHERE directory = ?
AND stem = ?
AND id != ?
ORDER BY id DESC
''', (loser['directory'], loser['stem'], loser['id']))
survivor_id = next((row['id'] for row in curs.fetchall() if row['id'] not in deleting_ids), None)
if survivor_id is not None:
merge_file_group_relations(curs, loser['id'], survivor_id)
merged += 1
return merged
async def apply_indexers(progress_callback: Callable[[int, int], None] = None):
"""Index leftover FileGroups that no modeler claimed.
File indexing state machine
---------------------------
A FileGroup is discovered (a row exists), then modeled and/or content-indexed.
`indexed` means "a modeler or this leftover indexer has finished with this row
for the current files." It is not "a domain model exists." `model` is the
domain claim (`video` / `archive` / `doc` / `zim`).
1. Discover/upsert: new or changed groups are inserted with indexed=0.
Unchanged groups keep indexed=1.
2. Modelers run first. A modeler that exclusively owns its mimetype
(video, doc, zim) selects:
(domain_row IS NULL OR indexed != 1) AND mimetype_matches
so a FileGroup the leftover indexer already marked indexed=1 still
gets a Video/Doc/Zim row. Failures are skipped for the rest of the
run (failed_ids) and retried on the next refresh.
3. Archive shares text/html with generic HTML. It selects only
indexed != 1. HTML that is not a SingleFile stays indexed=0 so this
function can claim it. HTML that is an Archive is indexed=1 with
model='archive'.
4. This function claims every remaining indexed != 1 row. It writes
a/b/c/d_text via the Indexer registry and sets indexed=1 even on
extractor failure so poison files do not loop forever.
5. Cleanup (channel claims, empty collections, tag config) runs last.
FileGroup.do_model() is the single-file equivalent of steps 2-4, used by
uploads. Batch modelers are used by FileWorker refresh.
Args:
progress_callback: Optional callback(processed, total) called after each batch.
"""
from wrolpi.files.models import FileGroup
refresh_logger.info('Applying indexers')
# Get initial count for progress tracking
total_to_index = 0
total_indexed = 0
if progress_callback:
with get_db_session() as session:
total_to_index = session.query(FileGroup).filter(FileGroup.indexed != True).count()
progress_callback(0, total_to_index)
update_stmt = ('UPDATE file_group SET "indexed" = 1, title = :title, a_text = :a_text, b_text = :b_text,'
' c_text = :c_text, d_text = :d_text WHERE id = :id')
def _strip_surrogates(value):
return strip_surrogates(value) if isinstance(value, str) else value
while True:
# Read the next batch as plain columns and let the read transaction end. Indexers can run
# subprocesses for minutes, and SQLite requires short write transactions: a session held
# across the batch fails its commit with "database is locked" (SQLITE_BUSY_SNAPSHOT, which
# ignores busy_timeout) whenever any other connection commits a write in the meantime.
with get_db_session() as session:
rows = session.query(FileGroup.id, FileGroup.primary_path, FileGroup.mimetype, FileGroup.title) \
.filter(FileGroup.indexed != True).limit(20).all()
if not rows:
break
# Index each file on a transient FileGroup: no session, no open transaction.
processed = 0
params = []
last_path = None
for fg_id, primary_path, mimetype, title in rows:
processed += 1
file_group = FileGroup(primary_path=primary_path, mimetype=mimetype, title=title)
try:
file_group.do_index()
except Exception:
# Error has already been logged in .do_index. It is still marked as indexed
# below; we won't try to index it again.
if PYTEST:
raise
params.append(dict(id=fg_id, title=file_group.title, a_text=file_group.a_text,
b_text=file_group.b_text, c_text=file_group.c_text, d_text=file_group.d_text))
last_path = primary_path
# Sleep to catch cancel.
await asyncio.sleep(0)
# Write the batch in one short transaction (BEGIN IMMEDIATE via commit=True).
try:
with get_db_curs(commit=True) as curs:
curs.executemany(update_stmt, params)
except UnicodeEncodeError as e:
# Paths with invalid UTF-8 surrogates produce titles/texts sqlite3 cannot store.
refresh_logger.warning(f'UnicodeEncodeError during indexer update, sanitizing batch: {e}')
params = [{k: _strip_surrogates(v) for k, v in p.items()} for p in params]
try:
with get_db_curs(commit=True) as curs:
curs.executemany(update_stmt, params)
except UnicodeEncodeError as e2:
# Still failing - mark as indexed without the texts so we don't loop forever.
refresh_logger.error(f'Failed to fix UnicodeEncodeError, skipping batch: {e2}')
with get_db_curs(commit=True) as curs:
curs.execute('UPDATE file_group SET "indexed" = 1 WHERE id IN '
'(SELECT value FROM json_each(?))',
(json.dumps([p['id'] for p in params]),))
if last_path:
refresh_logger.debug(f'Indexed {processed} files near {last_path}')
# Update progress
total_indexed += processed
if progress_callback and total_to_index > 0:
progress_callback(total_indexed, total_to_index)
if processed < 20:
# Processed less than the limit, don't do the next query.
break
def upsert_directories(parent_directories, directories):
"""
Insert/update/delete directories provided. Deletes all children of `parent_directories` which are not in
`directories`.
"""
idempotency = now()
directories = list(directories) + list(parent_directories)
# Only insert directories that are children of `media_directory` and exist.
directories = get_paths_in_media_directory([i for i in directories if i.is_dir()])
if directories:
# Insert any directories that were created, update any directories which previously existed.
# Chunk the rows so the multi-row VALUES clause stays under SQLite's bound-variable limit;
# a large media directory can hold tens of thousands of directories.
values = [(str(i.absolute()), i.name, idempotency) for i in directories]
with get_db_curs(commit=True) as curs:
for chunk in chunks(values, BULK_SQL_CHUNK_SIZE):
placeholders, params = values_clause(chunk)
stmt = f'''
INSERT INTO directory (path, name, idempotency) VALUES {placeholders}
ON CONFLICT (path) DO UPDATE
SET idempotency = EXCLUDED.idempotency
'''
curs.execute(stmt, params)
with get_db_curs(commit=True) as curs:
# Delete the children of any parent directory which no longer exists.
for directory in parent_directories:
stmt = f'DELETE FROM directory WHERE path LIKE :directory AND idempotency != :idempotency'
curs.execute(stmt, dict(directory=f'{directory.absolute()}%', idempotency=idempotency))
async def search_directories_by_name(session: Session, name: str, excluded: List[str] = None, limit: int = 20) \
-> List[Directory]:
"""Find the Directories whose names contain the `name` string."""
excluded = excluded or []
directories = session.query(Directory) \
.filter(Directory.name.ilike(f'%{name}%')) \
.filter(Directory.path.notin_(excluded)) \
.order_by(asc(Directory.name)) \
.limit(limit).all()
return directories
# Pagination totals: counting a large library is a full index walk that costs seconds cold on a
# spinning disk. Pagination can tolerate a total that is a minute stale. Bypassed under pytest
# so tests see exact totals. Keyed by filters, never by offset/limit/order (except filters that
# an order implies, e.g. viewed IS NOT NULL).
SEARCH_TOTALS_CACHE_SECONDS = 60
_SEARCH_TOTALS_CACHE: Dict[str, Tuple[float, int]] = dict()
def cached_search_total(key: str, compute: Callable[[], int]) -> int:
"""Return `compute()`; cache the result briefly unless running under pytest."""
if not PYTEST:
hit = _SEARCH_TOTALS_CACHE.get(key)
if hit and time.monotonic() - hit[0] < SEARCH_TOTALS_CACHE_SECONDS:
return hit[1]
total = compute()
_SEARCH_TOTALS_CACHE[key] = (time.monotonic(), total)
return total
def search_filter_cache_key(corpus: str, **filters) -> str:
"""Stable cache key for a search total. Omits empty/false filters."""
payload = {'corpus': corpus}
for k, v in filters.items():
if v in (None, '', [], (), False):
continue
if isinstance(v, (list, tuple, set)):
v = sorted(v)
payload[k] = v
return json.dumps(payload, sort_keys=True, default=str)
def count_file_groups(session: Session, statement: str, params: dict) -> int:
"""Run a `SELECT COUNT(*) AS total ...` statement and return the integer."""
with session_curs(session) as curs:
curs.execute(statement, params)
row = curs.fetchone()
return int(row['total'])
def search_files(session: Session, search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
from_year: int = None, to_year: int = None, any_tag: bool = False, order: str = None,
url: str = None, suffix: str = None, path: str = None, deep: bool = False,
in_progress: bool = False) -> \
Tuple[List[dict], int]:
"""Search the FileGroup table.
Order the returned Files by their rank if `search_str` is provided. Return all files if
`search_str` is empty.
@param any_tag: The file must have some tag.
@param to_year: The file must be published on or before this year.
@param from_year: The file must be published on or after this year.
@param search_str: Search the ts_vector of the file. Returns all files if this is empty.
@param limit: Return only this many files.
@param offset: Offset the query.
@param mimetypes: Only return files that match these mimetypes.
@param model: Only return files that match this model.
@param tag_names: A list of tag names.
@param headline: Includes FTS search headlines if True.
@param months: A list of integers representing the index of the month of the year, starting at 1.
@param order: Used to change results from most relevant to recently viewed.
@param url: Filter by URL using case-insensitive partial match (ILIKE).
@param suffix: Only return files whose primary file has this suffix (e.g. ".bin"), case-insensitive.
@param path: Filter by primary_path using case-insensitive partial match (ILIKE).
@param deep: Search all text including captions/body (d_text); slower but more thorough.
@param in_progress: Only files the user is part way through (see wrolpi.files.progress).
"""
from wrolpi import fts
# The frontend may send explicit JSON nulls (bypassing the schema defaults); SQLite rejects
# `LIMIT NULL`/`OFFSET NULL` (Postgres treated them as no-ops).
limit = int(limit) if limit else 20
offset = int(offset) if offset else 0
params = dict(offset=offset, limit=limit, search_str=search_str)
wheres = []
selects = []
order_by = '1 ASC'
joins = []
# The fast (non-deep) path excludes d_text (captions/body) from matching.
fts_search = fts.file_group_search_join(search_str, deep=deep, headlines=bool(headline)) \
if search_str else None
if search_str and not fts_search:
# Nothing usable to search (punctuation-only or all-negated query); match nothing, like
# an empty websearch_to_tsquery did.
return [], 0
if fts_search:
joins.append(fts_search.join)
selects.append(fts_search.rank_select)
params.update(fts_search.params)
order_by = '2 DESC'
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
if model:
params['model'] = model
wheres.append('model = :model')
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
wheres, params = months_selector_to_where(wheres, params, months)
wheres, params = date_range_to_where(wheres, params, from_year, to_year)
if url:
# SQLite LIKE is case-insensitive (for ASCII), matching the old ILIKE behavior.
params['url_filter'] = f'%{url}%'
wheres.append('fg.url LIKE :url_filter')
if suffix:
# Suffix is stored lowercased with a leading dot; normalize the caller's value to match the index.
normalized = suffix if suffix.startswith('.') else f'.{suffix}'
params['suffix'] = normalized.lower()
wheres.append('fg.suffix = :suffix')
if path:
params['path_filter'] = f'%{path}%'
wheres.append('fg.primary_path LIKE :path_filter')
if in_progress:
wheres.append('fg.progress > 0 AND fg.progress < 1')
if fts_search and headline:
# b/c/d headlines come from FTS5 snippets (computed in the fts subquery); the title
# headline is computed in Python by `handle_file_group_search_results` (title is not an
# FTS column).
headline = ', fts.b_headline, fts.c_headline, fts.d_headline'
else:
headline = ''
if order == 'viewed':
order_by = 'viewed DESC NULLS LAST, 1 ASC'
elif order == '-viewed':
order_by = 'viewed ASC NULLS LAST, 1 ASC'
viewed_only = bool(order and not search_str)
if viewed_only:
# Only filter out unviewed files if search_str is not provided.
wheres.append('viewed IS NOT NULL')
where_sql = ('WHERE\n' + '\n AND '.join(wheres)) if wheres else ''
extra_selects = f', {", ".join(selects)}' if selects else ''
join = '\n'.join(joins)
stmt = f'''
SELECT fg.id{extra_selects}
{headline}
FROM file_group fg
{join}
{where_sql}
ORDER BY {order_by}
LIMIT :limit OFFSET :offset
'''
logger.debug(stmt)
# COUNT(*) OVER() in the page query forces SQLite to visit every match before LIMIT.
# Count separately (and cache) so the page can use the datetime/id index.
if not search_str and not tag_names and not mimetypes and not model and not months \
and not from_year and not to_year and not any_tag and not url and not suffix \
and not path and not viewed_only and not in_progress:
count_stmt = 'SELECT COUNT(*) AS total FROM file_group'
count_params = dict()
elif fts_search:
# Count via a direct FTS join (no ranking) rather than the snippet subquery.
fts_count = fts.file_group_search(search_str, deep=deep)
count_wheres = list(wheres)
if fts_count:
count_wheres.append(fts_count.where)
count_where_sql = ('WHERE\n' + '\n AND '.join(count_wheres)) if count_wheres else ''
count_join = fts_count.join if fts_count else ''
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {count_join} {count_where_sql}'
count_params = dict(params)
if fts_count:
count_params.update(fts_count.params)
else:
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {join} {where_sql}'
count_params = params
cache_key = search_filter_cache_key(
'files', search_str=search_str, mimetypes=mimetypes, model=model,
tag_names=tag_names, months=months, from_year=from_year, to_year=to_year,
any_tag=any_tag, url=url, suffix=suffix, path=path, deep=deep,
viewed_only=viewed_only, in_progress=in_progress,
)
total = cached_search_total(cache_key, lambda: count_file_groups(session, count_stmt, count_params))
results, total = handle_file_group_search_results(session, stmt, params, total=total)
return results, total
def handle_file_group_search_results(session: Session, statement: str, params: dict, total: int = None) \
-> Tuple[List[dict], int]:
"""
Execute the provided SQL statement and fetch the Files returned.
The statement must select `id`. Pass `total` from a separate COUNT (callers should not use
`COUNT(*) OVER()` in the page query; it prevents SQLite from applying LIMIT via an index).
WARNING: This expects specific queries to be executed and shouldn't be used for things not related to file search.
See: `search`
"""
with session_curs(session) as curs:
curs.execute(statement, params)
results = [dict(i) for i in curs.fetchall()]
if total is None:
total = results[0]['total'] if results else 0
ordered_ids = [i['id'] for i in results]
extras = [dict(
ts_rank=i.get('ts_rank'),
# Snippet columns exist only when headlines were requested; `title_headline` is
# computed below (title is not an FTS5 column).
headlines=('b_headline' in i),
title_headline=i.get('title_headline'),
b_headline=i.get('b_headline'),
c_headline=i.get('c_headline'),
d_headline=i.get('d_headline'),
) for i in results]
from modules.archive.models import Archive
from modules.videos.models import Video
# Serialized below: what Video's and Archive's `__json__` read beyond the FileGroup (see their `json_options`).
results = session.query(FileGroup, Video, Archive) \
.options(*FileGroup.json_options(), undefer(Video.ffprobe_json), joinedload(Video.channel),
joinedload(Archive.collection)) \
.filter(FileGroup.id.in_(ordered_ids)) \
.outerjoin(Video, Video.file_group_id == FileGroup.id) \
.outerjoin(Archive, Archive.file_group_id == FileGroup.id)
# Order FileGroups by their location in ordered_ids. Use dict for O(1) lookups instead of O(n) list.index().
id_to_order = {id_: idx for idx, id_ in enumerate(ordered_ids)}
file_groups = sorted(results, key=lambda i: id_to_order.get(i[0].id, 0))
results = list()
for extra, (file_group, video, archive) in zip_longest(extras, file_groups):
if video:
results.append(video.__json__())
elif archive:
results.append(archive.__json__())
else:
results.append(file_group.__json__())
# Preserve the ts_ranks, if any.
if extra:
results[-1]['ts_rank'] = extra['ts_rank']
results[-1]['title_headline'] = extra['title_headline']
results[-1]['b_headline'] = extra['b_headline']
results[-1]['c_headline'] = extra['c_headline']
results[-1]['d_headline'] = extra['d_headline']
search_str = params.get('search_str') if isinstance(params, dict) else None
# When headlines were requested, compute the title headlines in Python (the FTS5 snippet
# function only works on indexed columns; `title` is not one).
if search_str and any(extra and extra.get('headlines') for extra in extras):
from wrolpi import fts
titles = [result.get('title') or '' for result in results]
for result, (headline_text, rank) in zip(results, fts.headline_texts(titles, search_str)):
if rank > 0:
result['title_headline'] = headline_text
# When searching by text, attach a `section_hint` to any doc-modeled result so
# the UI can deep-link into the matching EPUB chapter or PDF page.
if search_str:
from modules.docs.lib import _fetch_section_hints
# `r['id']` is the FileGroup primary key; _fetch_section_hints filters
# on `file_group_id`, so this list is file-group ids (not doc.id).
fg_ids = [r['id'] for r in results if r.get('model') == 'doc']
if fg_ids:
hints = _fetch_section_hints(session, fg_ids, search_str)
for r in results:
hint = hints.get(r['id'])
if hint:
r['section_hint'] = hint
return results, total
def glob_shared_stem(path: pathlib.Path) -> List[pathlib.Path]:
"""Return all paths that share the same stem and parent path as the provided path.
Example paths: foo.mp4, foo.png, foo.info.json, foobar.txt
>>> glob_shared_stem(pathlib.Path('foo.mp4'))
['foo.mp4', 'foo.png', 'foo.info.json']
"""
path = path if isinstance(path, pathlib.Path) else pathlib.Path(path)
stem, suffix = split_path_stem_and_suffix(path)
escaped_stem = glob.escape(stem)
paths = [pathlib.Path(i) for i in path.parent.glob(f'{escaped_stem}*')
if i == path or i.name.startswith(f'{stem}.')]
return paths
def get_matching_directories(path: Union[str, Path]) -> List[pathlib.Path]:
"""
Return a list of directory strings that start with the provided path. If the path is a directory, return its
subdirectories, if the directory contains no subdirectories, return the directory.
"""
path: pathlib.Path = pathlib.Path(path) if isinstance(path, str) else path
path = path.resolve()
ignored_directories = {pathlib.Path(i).resolve() for i in get_wrolpi_config().ignored_directories}
if path.is_dir():
# The provided path is a directory, return its subdirectories, or itself if no subdirectories exist
paths = list(path.iterdir())
paths = sorted(i for i in paths if i.is_dir() and i not in ignored_directories)
if len(paths) == 0:
return [get_real_path_name(path.resolve())]
return paths
paths = path.parent.iterdir()
prefix = path.name.lower()
paths = sorted(
i for i in paths if i.is_dir() and i.name.lower().startswith(prefix) and i.resolve() not in ignored_directories
)
paths = unique_by_predicate(paths, None)
return paths
WHITESPACE = re.compile(r'[\s_\[\]()]')
def split_file_name_words(name: str) -> str:
"""Split words in a filename.
Words are assumed to be separated by underscore, dash, or space. Words with a dash are included as a group, and
individually ('self-reliance' -> ['self', 'reliance', 'self-reliance']).
>>> split_file_name_words('this self-reliance_split.txt')
['this', 'self', 'reliance', 'self-reliance', 'split', 'txt']
"""
if not name:
raise ValueError(f'Invalid filename: {name}')
try:
stem, suffix = split_path_stem_and_suffix(name)
words = []
for word1 in WHITESPACE.split(stem):
if '-' in word1[1:]:
words.extend(word1.split('-'))
words.append(word1)
# Include the suffix so the user can search without the "."
if suffix:
words.append(suffix.lstrip('.'))
words = ' '.join(i for i in words if i)
return words
except Exception as e:
logger.error(f'Failed to split filename into words: {name}', exc_info=e)
return name
def get_file_statistics(session: Session):
with session_curs(session) as curs:
curs.execute('''
SELECT
-- All items in file_group.files are real individual files.
SUM(json_array_length(files)) AS "total_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype =
'application/x-mobipocket-ebook') AS "ebook_count",
(SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files",
(SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims",
(SELECT COUNT(*) FROM tag) AS "tags_count",
SUM(size) AS "total_size",
(SELECT COUNT(*) FROM archive) AS archive_count
FROM file_group
''')
statistics = dict(curs.fetchall()[0])
statistics['total_count'] = statistics['total_count'] or 0
statistics['total_size'] = statistics['total_size'] or 0
return statistics
def group_files_by_stem(files: Iterable[pathlib.Path]) -> Generator[List[pathlib.Path], None, None]:
"""
Return lists of paths, each list contains only paths that share a common directory and stem.
Groups are ordered by (directory, stem), and each group is sorted.
>>> a = ['foo.txt', 'foo.mp4', 'bar.txt']
>>> group_files_by_stem(list(map((pathlib.Path, a))))
# Generator(['bar.txt'], ['foo.mp4', 'foo.txt'])
"""
# Sorted paths do not keep a stem's files together: `foo.tar.gz` (stem `foo.tar`) sorts between `foo.jpg` and
# `foo.txt` (stem `foo`). Group by (directory, stem) so each stem yields exactly one group.
groups = dict()
for file in files:
file = pathlib.Path(file)
key = (str(file.parent), split_path_stem_and_suffix(file)[0])
groups.setdefault(key, set()).add(file)
for key in sorted(groups):
yield sorted(groups[key])
async def _get_tag_and_file_group(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
tag_id: int):
if file_group_id:
file_group: FileGroup = session.query(FileGroup).filter_by(id=file_group_id).one_or_none()
if not file_group:
raise UnknownFile(f'Cannot find FileGroup with id {file_group_id}')
elif file_group_primary_path:
path = get_media_directory() / file_group_primary_path
file_group = FileGroup.get_by_path(session, path)
# Create the FileGroup, if possible.
if not file_group and path.is_file():
file_group = FileGroup.from_paths(session, path)
if not file_group:
raise UnknownFile(f'Cannot find FileGroup with primary_path {repr(str(file_group_primary_path))}')
else:
raise UnknownFile(f'Cannot find FileGroup without id or primary_path')
if tag_id:
tag: Tag = session.query(Tag).filter_by(id=tag_id).one_or_none()
if not tag:
raise UnknownTag(f'Cannot find Tag with id {tag_id}')
elif tag_name:
tag: Tag = Tag.get_by_name(session, tag_name)
if not tag:
raise UnknownTag(f'Cannot find Tag with name {tag_name}')
else:
raise UnknownTag('Cannot find Tag without id or name')
return file_group, tag
async def add_file_group_tag(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
tag_id: int) -> TagFile:
file_group, tag = await _get_tag_and_file_group(session, file_group_id, file_group_primary_path, tag_name, tag_id)
tag_file = file_group.add_tag(session, tag.id)
return tag_file
async def remove_file_group_tag(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
tag_id: int):
file_group, tag = await _get_tag_and_file_group(session, file_group_id, file_group_primary_path, tag_name, tag_id)
file_group.untag(session, tag.id)
def mimetypes_to_sql_wheres(wheres: List[str], params: dict, mimetypes: List[str]) -> Tuple[List[str], dict]:
if mimetypes:
local_wheres = list()
for idx, mimetype in enumerate(mimetypes):
key = f'mimetype{idx}'
if len(mimetype.split('/')) == 1:
params[key] = f'{mimetype}/%'
else:
params[key] = f'{mimetype}%'
local_wheres.append(f'mimetype LIKE :{key}')
wheres.append(f'({" OR ".join(local_wheres)})')
return wheres, params
async def search_file_suggestion_count(session: Session, search_str: str, tag_names: List[str], mimetypes: List[str],
months: List[int] = None, from_year: int = None, to_year: int = None,
any_tag: bool = False) -> dict:
"""
Return FileGroup counts of what will be returned if the search is actually performed.
Returns both the fast (a/b/c columns) and deep (all columns) counts so the UI can show the count
matching the current search mode, and offer deep search when it has more results. Both are cheap
index-bound COUNTs (no ranking).
"""
from wrolpi import fts
wheres = []
joins = list()
params = dict()
# The deep match is a superset of the fast (column-filtered) match; both are counted with the
# same statement, differing only in the MATCH expression.
deep_match = fts.translate_websearch(search_str) if search_str else None
fast_match = fts.translate_websearch(search_str, fts.ABC_COLUMNS) if search_str else None
if search_str and deep_match is None:
# Nothing usable to search; match nothing (like an empty websearch_to_tsquery).
return dict(file_groups=0, file_groups_deep=0)
if deep_match:
joins.append('JOIN file_group_fts fts ON fts.rowid = fg.id')
wheres.append('fts.file_group_fts MATCH :fts_match')
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
wheres, params = months_selector_to_where(wheres, params, months)
wheres, params = date_range_to_where(wheres, params, from_year, to_year)
joins = "\n".join(joins)
wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
stmt = f'''
SELECT COUNT(*) AS estimate
FROM file_group fg
{joins}
{wheres}
'''
logger.debug(f'search_file_suggestion_count: {stmt} {params}')
with session_curs(session) as curs:
if deep_match:
curs.execute(stmt, {**params, 'fts_match': deep_match})
deep_estimate = int(curs.fetchone()['estimate'])
curs.execute(stmt, {**params, 'fts_match': fast_match})
estimate = int(curs.fetchone()['estimate'])
else:
curs.execute(stmt, params)
estimate = deep_estimate = int(curs.fetchone()['estimate'])
return dict(file_groups=estimate, file_groups_deep=deep_estimate)
MOVE_CHUNK_SIZE = 100
def _move_file_group_files(file_group: FileGroup, new_primary_path: pathlib.Path) -> None:
"""Move all physical files of a FileGroup to a new directory.
For directory moves where filenames are preserved, this moves each file from
the old directory to the new directory keeping the same filename.
Args:
file_group: The FileGroup whose files should be moved
new_primary_path: The new path for the primary file
"""
new_directory = new_primary_path.parent
new_stem, _ = split_path_stem_and_suffix(new_primary_path, full=True)
# Move each file in the FileGroup
for file_info in file_group.files:
relative_name = file_info['path']
old_path = file_group.directory / relative_name
# Calculate new path: new_stem + original suffix
_, suffix = split_path_stem_and_suffix(relative_name)
new_path = pathlib.Path(f'{new_stem}{suffix}')
if old_path.is_file():
shutil.move(old_path, new_path)
logger.debug(f'Moved file: {old_path} -> {new_path}')
def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path, pathlib.Path]):
"""Batch UPDATE FileGroups using raw SQL after physical files have been moved.
Only updates directory and primary_path. Other fields (files, title, a_text) remain unchanged
because filenames are preserved in directory moves - only the parent directory changes.
Args:
session: Database session
chunk_plan: Mapping of old_primary_path -> new_primary_path
"""
if not chunk_plan:
return
# Build the update rows: [old_path, new_dir, new_path]
updates = json.dumps([
[str(old_path), str(new_path.parent), str(new_path)]
for old_path, new_path in chunk_plan.items()
])
# Use UPDATE FROM json_each pattern for efficient batch update. This runs on the caller's
# session (rather than a second connection) so it shares the move's transaction: a second
# connection would deadlock against the write lock the caller already holds, and its separate
# commit would survive a rollback of the rest of the move.
sql = '''
UPDATE file_group AS fg SET
directory = json_extract(v.value, '$[1]'),
primary_path = json_extract(v.value, '$[2]'),
indexed = CASE WHEN fg.data IS NULL THEN FALSE ELSE fg.indexed END
FROM json_each(:updates) AS v
WHERE fg.primary_path = json_extract(v.value, '$[0]')
'''
session.execute(sa_text(sql), dict(updates=updates))
from wrolpi.collections.config import file_groups_moved
new_paths = [str(new_path) for new_path in chunk_plan.values()]
file_groups_moved(session, session.query(FileGroup.id).filter(FileGroup.primary_path.in_(new_paths)))
def _json_serial(obj):
"""JSON serializer for objects not serializable by default json code."""
import datetime
if isinstance(obj, (datetime.datetime, datetime.date)):
return obj.isoformat()
raise TypeError(f"Type {type(obj)} not serializable")
def _bulk_update_file_groups_reorganize(updates: List[dict]):
"""Batch UPDATE FileGroups for reorganization using raw SQL.
Unlike _bulk_update_file_groups_db, this also updates the files and data JSON fields
since reorganization changes filenames, not just directories.
Args:
updates: List of dicts with keys: id, directory, primary_path, files, data
"""
if not updates:
return
with get_db_curs(commit=True) as curs:
for update in updates:
# Don't change indexed - reorganization only moves files, content unchanged
primary_path = update['primary_path']
stem, _ = split_path_stem_and_suffix(primary_path)
curs.execute('''
UPDATE file_group
SET directory = ?,
primary_path = ?,
files = ?,
data = ?,
stem = ?
WHERE id = ?
''', (
update['directory'],
primary_path,
json.dumps(update['files'], default=_json_serial),
json.dumps(update['data'], default=_json_serial) if update.get('data') else None,
stem,
update['id'],
))
# Expire SQLAlchemy's session cache so subsequent queries see the raw SQL updates.
# This is necessary because get_db_curs uses the same connection as get_db_session
# during tests, but SQLAlchemy's identity map doesn't know about raw SQL updates.
with get_db_session() as session:
session.expire_all()
from wrolpi.collections.config import file_groups_moved
file_groups_moved(session, [update['id'] for update in updates])
def delete_directory(directory: pathlib.Path, recursive: bool = False, force: bool = False):
"""Remove a directory, remove it's Directory record.
Will refuse to delete a directory if it contains Tagged Files, unless force is True.
This function is idempotent - safe to call even if the directory was already deleted."""
if directory.exists():
if recursive:
if not force:
with get_db_session() as session:
tagged = get_tagged_file_groups_by_paths(session, [directory])
if tagged:
raise FileGroupIsTagged(f'Cannot delete {tagged[0]} because it is tagged')
shutil.rmtree(directory)
else:
directory.rmdir()
# Always clean up DB record even if directory was already deleted
with get_db_session(commit=True) as session:
session.query(Directory).filter_by(path=str(directory)).delete(synchronize_session=False)
async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
"""Rename a file (and it's associated files). Preserve any tags.
If the path is a non-primary file in a FileGroup (e.g., a poster or subtitle),
the entire FileGroup will be renamed.
"""
new_path = path.with_name(new_name)
if not path.exists():
raise FileNotFoundError(f'Cannot find {path} to rename')
if new_path.exists():
raise FileConflict(f'Cannot rename {path} because {new_path} already exists')
with get_db_session(commit=True) as session:
# Find the FileGroup - it may be the primary path or a secondary file
fg: FileGroup = FileGroup.get_by_any_file_path(session, path)
if not fg:
# File wasn't yet in the DB.
fg = FileGroup.from_paths(session, path)
# If this is not the primary file, calculate the new primary path
if fg.primary_path != path:
# Extract the new stem and apply it to the primary file's suffix
new_stem, _ = split_path_stem_and_suffix(new_path, full=True)
_, primary_suffix = split_path_stem_and_suffix(fg.primary_path, full=False)
new_primary_path = pathlib.Path(f'{new_stem}{primary_suffix}')
fg.move(new_primary_path)
else:
fg.move(new_path)
from wrolpi.collections.config import file_groups_moved
file_groups_moved(session, [fg.id])
return new_path
async def rename_directory(session: Session, directory: pathlib.Path, new_name: str, send_events: bool = False,
write: Callable[[], ContextManager[Session]] = None) -> str | None:
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory.
A Collection's directory is renamed before this returns (returns None). Any other directory is renamed by a queued
file worker move, which this does not wait for; returns the move's job id.
`session` is used to read. Collections are changed in `write()` (`request.ctx.db.write` in a request), which
begins as a writer; the files are moved outside it. Without `write`, Collections are changed in `session`."""
from wrolpi.files.worker import file_worker
write = write or (lambda: contextlib.nullcontext(session))
new_directory = directory.with_name(new_name)
if new_directory.exists():
raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.')
# Check if this directory is a Collection's directory
from wrolpi.collections.models import Collection
collection: Collection = session.query(Collection).filter(Collection.directory == directory).one_or_none()
if collection:
# Use Collection's move method which handles files, config saves, and cleanup
new_directory.mkdir(parents=True, exist_ok=True)
# move_collection commits its Download changes before it moves the files, so the write lock is not held
# for the move.
with write() as session:
await collection.move_collection(new_directory, session, send_events=send_events)
return None
# Regular directory rename: the file worker moves the contents, updates nested Collections, and removes the old
# directory. This does not wait; a large rename outlasts a request, and the job reports its own progress and
# result.
return file_worker.queue_move(new_directory, list(directory.iterdir()), rename_from=directory)
async def rename(session: Session, path: pathlib.Path, new_name: str, send_events: bool = False,
write: Callable[[], ContextManager[Session]] = None) -> str | None:
"""Rename a directory or file. Preserve any tags.
Returns the file worker job id when the rename was queued (see `rename_directory`), otherwise None; the rename is
done."""
if path.is_dir():
return await rename_directory(session, path, new_name, send_events=send_events, write=write)
await rename_file(path, new_name)
return None
def get_special_directories() -> List[pathlib.Path]:
"""Return absolute directories that must never be ignored by file refresh.
These are module destinations that depend on FileGroup indexing (or are the media root).
Map is intentionally excluded: it lists PMTiles by walking the filesystem and benefits
from being ignored during refresh (very large files).
Must not create directories — this runs during refresh ignore filtering and would
otherwise mkdir videos/ (and NO CHANNEL/) as a side effect of the videos helpers.
"""
from modules.archive.lib import get_archive_directory
from modules.videos.lib import format_videos_destination
media_directory = get_media_directory()
videos_directory = media_directory / format_videos_destination()
# Avoid importing modules.zim.lib here (circular import via files.worker).
zims_directory = media_directory / get_wrolpi_config().zims_destination
return [
media_directory.resolve(),
videos_directory.resolve(),
get_archive_directory().resolve(),
(videos_directory / 'NO CHANNEL').resolve(),
zims_directory.resolve(),
]
def get_normalized_ignored_directories() -> List[str]:
"""Return absolute ignored directory paths, excluding special directories.
Special directories (media root, videos, archives, zims, …) are never treated as ignored,
even if they appear in config from older installs or manual edits.
"""
media_directory = get_media_directory()
special = {str(p) for p in get_special_directories()}
result = []
for d in get_wrolpi_config().ignored_directories or []:
path = pathlib.Path(d)
absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve())
if absolute in special:
continue
result.append(absolute)
return result
def sanitize_ignored_directories():
"""Persistently remove special directories from the ignored list if present."""
wrolpi_config = get_wrolpi_config()
ignored = list(wrolpi_config.ignored_directories or [])
if not ignored:
return
media_directory = get_media_directory()
special = {str(p) for p in get_special_directories()}
cleaned = []
removed = False
for d in ignored:
path = pathlib.Path(d)
absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve())
if absolute in special:
removed = True
logger.warning(f'Removing special directory from ignored_directories: {d}')
continue
cleaned.append(d)
if removed:
wrolpi_config.ignored_directories = cleaned
def add_ignore_directory(directory: Union[pathlib.Path, str]):
"""Add a directory to the `ignored_directories` in the WROLPi config. This directory will be ignored when
refreshing."""
media_directory = get_media_directory().resolve()
directory = pathlib.Path(directory)
if not directory.is_absolute():
directory = media_directory / directory
directory = directory.resolve()
# Drop any special directories that were previously saved as ignored.
sanitize_ignored_directories()
special_directories = set(get_special_directories())
if directory in special_directories:
raise InvalidDirectory('Refusing to ignore special directory')
if not str(directory).startswith(str(media_directory)):
raise InvalidDirectory('Cannot ignore directory not in media directory')
wrolpi_config = get_wrolpi_config()
ignored_directories = wrolpi_config.ignored_directories if wrolpi_config.ignored_directories else list()
# Accept either absolute or relative forms already present in config.
relative = str(get_relative_to_media_directory(directory))
if str(directory) in ignored_directories or relative in ignored_directories:
logger.warning(f'Directory is already ignored: {directory}')
return
ignored_directories.append(str(directory))
wrolpi_config.ignored_directories = ignored_directories
def remove_ignored_directory(directory: Union[pathlib.Path, str]):
"""Remove a directory from the `ignored_directories` in the WROLPi config."""
media_directory = get_media_directory()
path = pathlib.Path(directory)
if not path.is_absolute():
path = media_directory / path
path = path.resolve()
absolute = str(path)
ignored_directories = list(get_wrolpi_config().ignored_directories or [])
kept = []
removed = False
for d in ignored_directories:
entry = pathlib.Path(d)
entry_abs = str((media_directory / entry).resolve() if not entry.is_absolute() else entry.resolve())
if entry_abs == absolute:
removed = True
continue
kept.append(d)
if not removed:
raise UnknownDirectory('Directory is not ignored')
get_wrolpi_config().ignored_directories = kept
@dataclasses.dataclass(frozen=True)
class UpsertedFile:
"""The FileGroup `upsert_file` created or updated. Plain values: its Session is gone once it returns."""
file_group_id: int
location: str
async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> UpsertedFile:
"""Insert/update all files in the provided file's FileGroup."""
if not file or not file.is_file():
raise InvalidFile(f'Cannot upsert file that does not exist: {file}')
# Update/Insert all files in the FileGroup.
paths = glob_shared_stem(pathlib.Path(file))
# Remove any ignored files.
paths = remove_files_in_ignored_directories(paths)
if not paths:
logger.warning('upsert_file called, but all files are in ignored directories!')
raise IgnoredDirectoryError(f'all files are in ignored directories: {file}')
# Use a single session context for all database operations
with get_db_session(commit=True) as session:
# Create/update FileGroup with retry logic
file_group = None
for i in range(2):
try:
file_group = FileGroup.from_paths(session, *paths)
session.flush([file_group])
break
except IntegrityError:
# Another process inserted this FileGroup.
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
session.rollback()
if i == 1:
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
continue
if not file_group:
raise RuntimeError(f'upsert_file failed to create FileGroup! {file}')
logger.debug(f'upsert_file: {file_group}')
# Model the FileGroup (creates Archive/Video/etc. and sets indexed=True)
try:
file_group.do_model(session)
except Exception as e:
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
if PYTEST:
raise
# If user uploads a file, then remove it from the download skip list so comments can be downloaded.
# Modify the download (if any) so that the user can click on it and view the uploaded file.
if file_group.url:
if download_manager.is_skipped(file_group.url):
download_manager.remove_from_skip_list(file_group.url)
if download := Download.get_by_url(session, file_group.url):
# Mark download as completed
download.complete()
# Link the download to the location of the uploaded file.
model = file_group.get_model_record(session)
download.location = model.location if model else file_group.location
# Add tags if provided
if tag_names:
for tag_name in tag_names:
if tag_name not in file_group.tag_names:
tag = Tag.get_by_name(session, tag_name)
file_group.add_tag(session, tag.id)
# Commit all changes in one transaction
session.commit()
upserted = UpsertedFile(file_group_id=file_group.id, location=file_group.location)
# Upsert directories after FileGroup is committed
upsert_directories([], file.parents)
return upserted
def get_file_location_href(file: pathlib.Path) -> str:
"""Return the location a file can be viewed at on the UI."""
parent = str(get_relative_to_media_directory(file.parent))
preview = str(get_relative_to_media_directory(file))
if parent == '.':
# File is in the top of the media directory, App already shows top directory open.
query = urllib.parse.urlencode(dict(preview=str(preview)))
else:
query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
return f'/files?{query}'
def get_real_path_name(path: pathlib.Path) -> pathlib.Path:
"""Return the real path name of a file. This is used to get the real path of a file on macOS."""
if IS_MACOS:
for path_ in path.parent.iterdir():
if path_.name.lower() == path.name.lower():
return path_
return path
@dataclasses.dataclass
class BulkTagPreview:
"""Preview information for a bulk tagging operation."""
file_count: int = 0
shared_tag_names: List[str] = dataclasses.field(default_factory=list)
def __json__(self) -> dict:
return dict(
file_count=self.file_count,
shared_tag_names=self.shared_tag_names,
)
def get_bulk_tag_preview(session: Session, paths: List[str]) -> BulkTagPreview:
"""Get preview information for bulk tagging operation.
Returns the count of files that will be tagged and the tags that are shared by ALL files.
"""
media_directory = get_media_directory()
absolute_paths = [media_directory / p for p in paths]
# Collect all file paths (expanding directories recursively)
all_file_paths: List[pathlib.Path] = []
for path in absolute_paths:
if path.is_file():
all_file_paths.append(path)
elif path.is_dir():
for file in walk(path):
if file.is_file():
all_file_paths.append(file)
# Remove files in ignored directories
all_file_paths = remove_files_in_ignored_directories(all_file_paths)
if not all_file_paths:
return BulkTagPreview(file_count=0, shared_tag_names=[])
# Get unique files by stem (each FileGroup counted once)
unique_files = get_unique_files_by_stem(all_file_paths)
file_count = len(unique_files)
if file_count == 0:
return BulkTagPreview(file_count=0, shared_tag_names=[])
# Find tags shared by ALL files
# First, get all FileGroups and their tags
# We query by stem prefix because unique_files may contain non-primary files
# (e.g., .readability.json) but FileGroups are stored with primary_path (e.g., .html)
from sqlalchemy import or_
stems = [split_path_stem_and_suffix(f, full=True)[0] for f in unique_files]
# Chunk the OR'd LIKEs to stay within SQLite's expression-tree depth limit of 1000.
file_groups = []
for idx in range(0, len(stems), 500):
chunk = stems[idx:idx + 500]
file_groups.extend(session.query(FileGroup).filter(
or_(*[FileGroup.primary_path.like(f'{stem}%') for stem in chunk])
).all())
if not file_groups:
# No FileGroups exist yet, no shared tags
return BulkTagPreview(file_count=file_count, shared_tag_names=[])
# Get tag names for each FileGroup
tag_sets = []
for fg in file_groups:
tag_sets.append(set(fg.tag_names))
# Find intersection of all tag sets (tags shared by ALL files)
if tag_sets:
shared_tags = tag_sets[0]
for tag_set in tag_sets[1:]:
shared_tags = shared_tags.intersection(tag_set)
shared_tag_names = sorted(list(shared_tags))
else:
shared_tag_names = []
return BulkTagPreview(file_count=file_count, shared_tag_names=shared_tag_names)
@dataclasses.dataclass
class BulkTagProgress:
"""Progress information for bulk tagging operation."""
status: str = 'idle' # 'idle', 'running'
total: int = 0
completed: int = 0
add_tag_names: List[str] = dataclasses.field(default_factory=list)
remove_tag_names: List[str] = dataclasses.field(default_factory=list)
error: str = None
queued_jobs: int = 0
def __json__(self) -> dict:
return dict(
status=self.status,
total=self.total,
completed=self.completed,
add_tag_names=self.add_tag_names,
remove_tag_names=self.remove_tag_names,
error=self.error,
queued_jobs=self.queued_jobs,
)
def get_bulk_tag_progress() -> BulkTagProgress:
"""Get the current progress of bulk tagging operations."""
from wrolpi.api_utils import api_app
bulk_tag = api_app.shared_ctx.bulk_tag
return BulkTagProgress(
status=bulk_tag.get('status', 'idle'),
total=bulk_tag.get('total', 0),
completed=bulk_tag.get('completed', 0),
add_tag_names=list(bulk_tag.get('add_tag_names', [])),
remove_tag_names=list(bulk_tag.get('remove_tag_names', [])),
error=bulk_tag.get('error'),
queued_jobs=bulk_tag.get('queued_jobs', 0),
)
def queue_bulk_tag_job(paths: List[str], add_tag_names: List[str], remove_tag_names: List[str]):
"""Add a bulk tagging job to the queue."""
from wrolpi.api_utils import api_app
job = dict(
paths=paths,
add_tag_names=add_tag_names,
remove_tag_names=remove_tag_names,
)
api_app.shared_ctx.bulk_tag_queue.put(job)
# Reset progress state and update queued jobs count
try:
current_queued = api_app.shared_ctx.bulk_tag.get('queued_jobs', 0)
api_app.shared_ctx.bulk_tag.update(dict(
status='idle',
total=0,
completed=0,
error=None,
queued_jobs=current_queued + 1,
))
except Exception:
pass
BULK_TAG_CHUNK_SIZE = 100 # Files per DB session
BULK_SQL_CHUNK_SIZE = 1000 # Rows per SQL statement
def _bulk_insert_tag_files(tag_files: List[Tuple[int, int]]):
"""Bulk insert TagFiles, ignoring duplicates.
Args:
tag_files: List of (tag_id, file_group_id) tuples
"""
if not tag_files:
return
now_timestamp = now()
# Process in chunks to avoid overly large SQL statements
for i in range(0, len(tag_files), BULK_SQL_CHUNK_SIZE):
chunk = tag_files[i:i + BULK_SQL_CHUNK_SIZE]
values = [(tag_id, file_group_id, now_timestamp)
for tag_id, file_group_id in chunk]
with get_db_curs(commit=True) as curs:
placeholders, params = values_clause(values)
stmt = f'''
INSERT INTO tag_file (tag_id, file_group_id, created_at)
VALUES {placeholders}
ON CONFLICT (tag_id, file_group_id) DO NOTHING
'''
curs.execute(stmt, params)
count_config_change_curs(curs, 'tags.yaml')
logger.debug(f'Bulk inserted {len(chunk)} TagFiles')
def _bulk_delete_tag_files(tag_files: List[Tuple[int, int]]):
"""Bulk delete TagFiles.
Args:
tag_files: List of (tag_id, file_group_id) tuples
"""
if not tag_files:
return
# Process in chunks
for i in range(0, len(tag_files), BULK_SQL_CHUNK_SIZE):
chunk = tag_files[i:i + BULK_SQL_CHUNK_SIZE]
pairs = [(tag_id, file_group_id) for tag_id, file_group_id in chunk]
with get_db_curs(commit=True) as curs:
placeholders, params = values_clause(pairs)
stmt = f'''
DELETE FROM tag_file
WHERE (tag_id, file_group_id) IN (VALUES {placeholders})
'''
curs.execute(stmt, params)
logger.debug(f'Bulk deleted {curs.rowcount} TagFiles')
count_config_change_curs(curs, 'tags.yaml')
async def _process_bulk_tag_job(job: dict):
"""Process a single bulk tagging job with batched DB operations.
This function uses a three-phase approach for performance:
1. Collection: Gather all files and resolve FileGroups, collecting tag operations
2. Bulk DB operations: Execute bulk INSERT and DELETE for TagFiles
3. Switch activation: Call save_tags_config and sync_tags_directory once at the end
"""
from wrolpi.api_utils import api_app
paths = job['paths']
add_tag_names = job['add_tag_names']
remove_tag_names = job['remove_tag_names']
media_directory = get_media_directory()
absolute_paths = [media_directory / p for p in paths]
# Collect all file paths (expanding directories recursively)
all_file_paths: List[pathlib.Path] = []
for path in absolute_paths:
if path.is_file():
all_file_paths.append(path)
elif path.is_dir():
for file in walk(path):
if file.is_file():
all_file_paths.append(file)
# Remove files in ignored directories
all_file_paths = remove_files_in_ignored_directories(all_file_paths)
if not all_file_paths:
return
# Get unique files by stem
unique_files = get_unique_files_by_stem(all_file_paths)
# Update progress with total count
api_app.shared_ctx.bulk_tag.update(dict(
status='running',
total=len(unique_files),
completed=0,
add_tag_names=add_tag_names,
remove_tag_names=remove_tag_names,
error=None,
))
# Pre-resolve tag IDs (using cached lookups)
add_tag_ids = set()
remove_tag_ids = set()
with get_db_session() as session:
for tag_name in add_tag_names:
try:
add_tag_ids.add(Tag.get_id_by_name(session, tag_name))
except UnknownTag:
logger.warning(f'Unknown tag to add: {tag_name}')
for tag_name in remove_tag_names:
try:
remove_tag_ids.add(Tag.get_id_by_name(session, tag_name))
except UnknownTag:
logger.warning(f'Unknown tag to remove: {tag_name}')
if not add_tag_ids and not remove_tag_ids:
return
# Phase 1: Collect all file_group_ids and required operations
tag_files_to_insert: List[Tuple[int, int]] = [] # (tag_id, file_group_id)
tag_files_to_delete: List[Tuple[int, int]] = [] # (tag_id, file_group_id)
completed = 0
# Process files in chunks to balance memory and DB pressure
for chunk_start in range(0, len(unique_files), BULK_TAG_CHUNK_SIZE):
chunk = unique_files[chunk_start:chunk_start + BULK_TAG_CHUNK_SIZE]
with get_db_session(commit=True) as session:
for file_path in chunk:
try:
# Get or create FileGroup
file_group = FileGroup.get_by_path(session, file_path)
if not file_group:
# Create FileGroup for unindexed file
paths_for_group = glob_shared_stem(file_path)
paths_for_group = remove_files_in_ignored_directories(paths_for_group)
if paths_for_group:
try:
file_group = FileGroup.from_paths(session, *paths_for_group)
file_group.do_model(session)
session.flush([file_group])
except (NoPrimaryFile, IntegrityError) as e:
logger.warning(f'Could not create FileGroup for {file_path}: {e}')
completed += 1
api_app.shared_ctx.bulk_tag['completed'] = completed
continue
if file_group and file_group.id:
# Flush to ensure file_group.id is set
session.flush([file_group])
# Get current tag_ids for this file_group
existing_tag_ids = {tf.tag_id for tf in file_group.tag_files}
# Determine tags to add (not already present)
for tag_id in add_tag_ids:
if tag_id not in existing_tag_ids:
tag_files_to_insert.append((tag_id, file_group.id))
# Determine tags to remove (currently present)
for tag_id in remove_tag_ids:
if tag_id in existing_tag_ids:
tag_files_to_delete.append((tag_id, file_group.id))
except Exception as e:
logger.error(f'Error processing file {file_path}: {e}', exc_info=e)
completed += 1
api_app.shared_ctx.bulk_tag['completed'] = completed
# Allow cancellation between chunks
await asyncio.sleep(0)
# Phase 2: Bulk database operations
_bulk_insert_tag_files(tag_files_to_insert)
_bulk_delete_tag_files(tag_files_to_delete)
# Phase 3: Activate switches ONCE at the end
if tag_files_to_insert or tag_files_to_delete:
save_tags_config.activate_switch()
sync_tags_directory.activate_switch()
DISABLE_BULK_TAG_WORKER = bool(PYTEST)
async def bulk_tag_worker():
"""Background worker that processes bulk tagging jobs from the queue."""
import queue
from wrolpi.api_utils import api_app
if PYTEST and DISABLE_BULK_TAG_WORKER:
return
bulk_tag_ctx = api_app.shared_ctx.bulk_tag
bulk_tag_queue = api_app.shared_ctx.bulk_tag_queue
try:
# Try to get a job from the queue (non-blocking)
job = bulk_tag_queue.get_nowait()
except queue.Empty:
# No jobs in queue, stay idle
bulk_tag_ctx['status'] = 'idle'
return
# Decrement queued jobs count
try:
current_queued = bulk_tag_ctx.get('queued_jobs', 1)
bulk_tag_ctx['queued_jobs'] = max(0, current_queued - 1)
except Exception:
pass
try:
await _process_bulk_tag_job(job)
except Exception as e:
logger.error(f'Bulk tag worker error: {e}', exc_info=e)
bulk_tag_ctx['error'] = str(e)
finally:
bulk_tag_ctx['status'] = 'idle'
# Register the worker as a perpetual signal
from wrolpi.api_utils import perpetual_signal
bulk_tag_worker = perpetual_signal(sleep=1)(bulk_tag_worker)
def _detect_archive_type(path: pathlib.Path) -> str:
"""Detect archive type from file suffix. Returns 'zip', 'tar', 'rar', '7z', or raises UnsupportedArchive."""
name = path.name.lower()
if name.endswith(('.zip', '.cbz')):
return 'zip'
if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz', '.cbt')):
return 'tar'
if name.endswith(('.rar', '.cbr')):
return 'rar'
if name.endswith(('.7z', '.cb7')):
return '7z'
raise UnsupportedArchive(f'Unsupported archive format: {path.name}')
def _build_tree(entries: list) -> list:
"""Build a hierarchical tree from a flat list of archive entries.
Each entry is a dict with 'path', 'name', 'is_dir', 'size'.
Returns a nested list with 'children' on directory entries.
"""
root_children = []
# Map from directory path to its children list.
dir_map = {}
# Sort entries so parents come before children.
entries.sort(key=lambda e: e['path'])
for entry in entries:
parts = entry['path'].strip('/').split('/')
if len(parts) == 1:
# Top-level entry.
if entry['is_dir']:
entry['children'] = []
dir_map[entry['path'].rstrip('/') + '/'] = entry['children']
root_children.append(entry)
else:
# Find parent directory.
parent_path = '/'.join(parts[:-1]) + '/'
if parent_path not in dir_map:
# Create implicit parent directories.
current = ''
for part in parts[:-1]:
current += part + '/'
if current not in dir_map:
parent_entry = {
'path': current,
'name': part,
'is_dir': True,
'size': 0,
'children': [],
}
dir_map[current] = parent_entry['children']
# Attach to grandparent or root.
grandparent = '/'.join(current.rstrip('/').split('/')[:-1])
grandparent_key = grandparent + '/' if grandparent else None
if grandparent_key and grandparent_key in dir_map:
dir_map[grandparent_key].append(parent_entry)
else:
root_children.append(parent_entry)
if entry['is_dir']:
entry['children'] = []
dir_map[entry['path'].rstrip('/') + '/'] = entry['children']
parent_list = dir_map.get(parent_path)
if parent_list is not None:
parent_list.append(entry)
else:
root_children.append(entry)
return root_children
def list_archive_contents(path: pathlib.Path) -> dict:
"""List the contents of an archive file as a hierarchical tree.
Returns: {"entries": [...], "total_size": int, "total_files": int}
"""
media_directory = get_media_directory()
if not str(path).startswith(str(media_directory)):
raise InvalidFile('Path must be within the media directory')
if not path.is_file():
raise InvalidFile(f'Archive file does not exist: {path}')
archive_type = _detect_archive_type(path)
flat_entries = []
total_size = 0
total_files = 0
try:
if archive_type == 'zip':
with zipfile.ZipFile(path, 'r') as zf:
for info in zf.infolist():
is_dir = info.is_dir()
entry = {
'path': info.filename,
'name': info.filename.rstrip('/').split('/')[-1],
'is_dir': is_dir,
'size': info.file_size,
}
flat_entries.append(entry)
if not is_dir:
total_size += info.file_size
total_files += 1
elif archive_type == 'tar':
with tarfile.open(path, 'r:*') as tf:
for member in tf.getmembers():
is_dir = member.isdir()
entry = {
'path': member.name + ('/' if is_dir else ''),
'name': member.name.rstrip('/').split('/')[-1],
'is_dir': is_dir,
'size': member.size,
}
flat_entries.append(entry)
if not is_dir:
total_size += member.size
total_files += 1
elif archive_type == 'rar':
import rarfile
with rarfile.RarFile(path, 'r') as rf:
for info in rf.infolist():
is_dir = info.is_dir()
entry = {
'path': info.filename,
'name': info.filename.rstrip('/').split('/')[-1],
'is_dir': is_dir,
'size': info.file_size,
}
flat_entries.append(entry)
if not is_dir:
total_size += info.file_size
total_files += 1
elif archive_type == '7z':
import py7zr
with py7zr.SevenZipFile(path, 'r') as sz:
for entry_info in sz.list():
is_dir = entry_info.is_directory
entry = {
'path': entry_info.filename + ('/' if is_dir else ''),
'name': entry_info.filename.rstrip('/').split('/')[-1],
'is_dir': is_dir,
'size': entry_info.uncompressed if not is_dir else 0,
}
flat_entries.append(entry)
if not is_dir:
total_size += entry_info.uncompressed
total_files += 1
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
raise UnsupportedArchive(f'Cannot read archive: {e}')
tree = _build_tree(flat_entries)
return {'entries': tree, 'total_size': total_size, 'total_files': total_files}
def stream_archive_member(archive_path: pathlib.Path, member_path: str) -> Tuple[bytes, str, str]:
"""Read a single member from an archive and return its bytes, mimetype, and filename.
Security: validates member_path has no '..' components.
"""
media_directory = get_media_directory()
if not str(archive_path).startswith(str(media_directory)):
raise InvalidFile('Path must be within the media directory')
if not archive_path.is_file():
raise InvalidFile(f'Archive file does not exist: {archive_path}')
# Security: reject path traversal.
if '..' in member_path.split('/'):
raise InvalidArchiveMember('Invalid member path: contains ".."')
archive_type = _detect_archive_type(archive_path)
filename = member_path.rstrip('/').split('/')[-1]
guessed_type, _ = mimetypes.guess_type(filename)
content_type = guessed_type or 'application/octet-stream'
try:
if archive_type == 'zip':
with zipfile.ZipFile(archive_path, 'r') as zf:
try:
info = zf.getinfo(member_path)
except KeyError:
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
if info.is_dir():
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
file_bytes = zf.read(member_path)
elif archive_type == 'tar':
with tarfile.open(archive_path, 'r:*') as tf:
try:
member = tf.getmember(member_path)
except KeyError:
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
if member.isdir():
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
f = tf.extractfile(member)
if f is None:
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
file_bytes = f.read()
elif archive_type == 'rar':
import rarfile
with rarfile.RarFile(archive_path, 'r') as rf:
try:
info = rf.getinfo(member_path)
except KeyError:
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
if info.is_dir():
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
file_bytes = rf.read(member_path)
elif archive_type == '7z':
import py7zr
import tempfile
with py7zr.SevenZipFile(archive_path, 'r') as sz:
if member_path not in sz.getnames():
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
with tempfile.TemporaryDirectory() as tmpdir:
sz.extract(tmpdir, targets=[member_path])
extracted_path = pathlib.Path(tmpdir) / member_path
if not extracted_path.is_file():
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
file_bytes = extracted_path.read_bytes()
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
raise UnsupportedArchive(f'Cannot read archive: {e}')
return file_bytes, content_type, filename