wrolpi/wrolpi/files/lib.py
Roland c8201700a8
Track how far the user is through a file, kept in recently_viewed.yaml
FileGroup gains `progress` (0.0 to 1.0, 1.0 is finished) and `position` (where
to resume: a time, a page, or an EPUB CFI).  Both are returned with the file,
including by POST /api/files/file.

POST /api/files/progress records progress and is also a view.  A file 95%
through is finished and keeps no position; one barely started (under 10 s,
the first page, under 1%) is not worth resuming.  The write happens before
the response, so the next request sees it.  POST /api/files/progress/clear
starts a file over.  Both refuse files outside the media directory.

recently_viewed.yaml keeps the 1000 most recently viewed files so viewing
history and positions survive a database rebuild.  On import the newer
`viewed` wins, so a file viewed while WROLPi was starting keeps its newer
position.  The config is saved when a file is opened, when progress is final
(paused or closed), when it is cleared, and after refresh cleanup (moves,
renames, deletes); heartbeats during playback only write the database.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 15:21:38 -06:00

2731 lines
114 KiB
Python

import asyncio
import dataclasses
import datetime
import functools
import glob
import json
import mimetypes
import os
import pathlib
import re
import shutil
import subprocess
import tarfile
import threading
import time
import urllib.parse
import zipfile
from itertools import zip_longest
from pathlib import Path
from typing import Callable, List, Tuple, Union, Dict, Generator, Iterable, Set
import cachetools.func
from sqlalchemy import asc, or_, text as sa_text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, undefer, joinedload
from wrolpi.cmd import which
from wrolpi.common import get_media_directory, wrol_mode_check, logger, \
chunks, chunks_by_stem, walk, \
get_wrolpi_config, \
unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory, strip_surrogates
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
from wrolpi.db import get_db_session, get_db_curs, values_clause, session_curs
from wrolpi.downloader import download_manager, Download
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
NoPrimaryFile, InvalidDirectory, IgnoredDirectoryError, UnsupportedArchive, InvalidArchiveMember
from wrolpi.events import Events
from wrolpi.files.models import FileGroup, Directory
from wrolpi.files.progress import normalize_progress, save_recently_viewed_config
from wrolpi.lang import ISO_639_CODES, ISO_3166_CODES
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config, sync_tags_directory
from wrolpi.vars import PYTEST, IS_MACOS
try:
import magic
mime = magic.Magic(mime=True)
except ImportError:
# Magic is not installed
magic = None
mime = None
logger = logger.getChild(__name__)
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'search_files',
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'choose_primary_file', 'get_file_statistics',
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href',
'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories',
'get_normalized_ignored_directories', 'sanitize_ignored_directories',
'cached_search_total', 'search_filter_cache_key', 'count_file_groups',
'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems',
'backfill_null_file_group_stems', 'dedupe_file_groups_by_directory_stem']
def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
"""Returns all Tag names for the provided file path."""
tags = session.query(Tag) \
.join(TagFile, Tag.id == TagFile.tag_id) \
.join(FileGroup, FileGroup.id == TagFile.file_group_id) \
.filter(FileGroup.primary_path == str(file))
names = sorted([tag.name for tag in tags], key=lambda i: i.lower())
return names
def sanitize_filename_surrogates(path: pathlib.Path) -> pathlib.Path:
"""
Check if a path contains invalid UTF-8 surrogates. If so, rename the file
on disk to replace them with underscores.
Linux filesystems store filenames as bytes. Python represents invalid UTF-8
sequences as surrogates. These surrogates cannot be stored in the database.
Returns the (possibly new) path.
"""
path_str = str(path)
# Check for surrogates by trying to encode as UTF-8
try:
path_str.encode('utf-8')
return path # Path is valid
except UnicodeEncodeError:
pass
# Get raw bytes from filesystem and decode, replacing invalid sequences
raw_bytes = os.fsencode(path)
# Replace invalid UTF-8 sequences with underscore
sanitized_name = raw_bytes.decode('utf-8', errors='replace').replace('\ufffd', '_')
sanitized_path = pathlib.Path(sanitized_name)
if path.exists() and sanitized_path != path:
if sanitized_path.exists():
# Target already exists, add a suffix
stem = sanitized_path.stem
suffix = sanitized_path.suffix
counter = 1
while sanitized_path.exists():
sanitized_path = sanitized_path.with_name(f'{stem}_{counter}{suffix}')
counter += 1
logger.warning(f'Renaming file with invalid UTF-8 characters: {path!r} -> {sanitized_path}')
path.rename(sanitized_path)
return sanitized_path
return path
def _get_file_dict(session: Session, file: pathlib.Path) -> Dict:
media_directory = get_media_directory()
try:
size = file.stat().st_size
except PermissionError:
size = None
try:
mimetype = get_mimetype(file)
except PermissionError:
mimetype = None
file_group = FileGroup.get_by_path(session, file)
return dict(
path=file.relative_to(media_directory),
size=size,
mimetype=mimetype,
tags=get_file_tag_names(session, file),
progress=file_group.progress if file_group else None,
position=file_group.position if file_group else None,
)
def get_file_dict(session: Session, file: str) -> Dict:
media_directory = get_media_directory()
return _get_file_dict(session, media_directory / file)
async def set_file_viewed(session: Session, file: pathlib.Path):
"""Change FileGroup.viewed to the current datetime.
Files in ignored directories are not tracked.
"""
# Check if file is in an ignored directory
remaining = remove_files_in_ignored_directories([file])
if not remaining:
return
try:
fg = FileGroup.find_by_path(session, file)
except UnknownFile:
fg = FileGroup.from_paths(session, file)
fg.do_model(session)
fg.set_viewed()
session.commit()
def _get_or_create_file_group(session: Session, file: pathlib.Path) -> FileGroup:
if not get_paths_in_media_directory([file]):
raise InvalidFile('File must be in the media directory')
try:
return FileGroup.find_by_path(session, file)
except UnknownFile:
if not file.is_file():
raise InvalidFile(f'File does not exist: {get_relative_to_media_directory(file)}')
fg = FileGroup.from_paths(session, file)
fg.do_model(session)
return fg
def set_file_progress(session: Session, file: pathlib.Path, progress: float, position: dict = None,
final: bool = False) -> Tuple[float | None, dict | None]:
"""Record how far the User is through a file (see wrolpi.files.progress). This is also a view.
@param final: The User stopped (paused, closed); save the recently viewed config.
@return: The (progress, position) that was stored.
"""
progress, position = normalize_progress(progress, position)
# Files in ignored directories are not tracked.
if not remove_files_in_ignored_directories([file]):
return None, None
fg = _get_or_create_file_group(session, file)
# Not `set_viewed`, which saves the config; a heartbeat during playback only writes the DB.
fg.viewed = now()
fg.progress = progress
fg.position = position
session.commit()
if final:
save_recently_viewed_config.activate_switch()
return progress, position
def clear_file_progress(session: Session, file: pathlib.Path):
"""Forget how far the User is through a file, so it starts from the beginning. It remains viewed."""
fg = _get_or_create_file_group(session, file)
fg.progress = None
fg.position = None
session.commit()
save_recently_viewed_config.activate_switch()
@cachetools.func.ttl_cache(10_000, 30.0)
def _get_directory_dict(directory: pathlib.Path,
directories_cache: str, # Used to cache by requested directories.
) -> Dict:
media_directory = get_media_directory()
try:
is_empty = not next(directory.iterdir(), False)
except PermissionError:
is_empty = False
return dict(
path=f'{directory.relative_to(media_directory)}/',
is_empty=is_empty,
)
def _get_recursive_directory_dict(session: Session, directory: pathlib.Path, directories: List[pathlib.Path]) -> Dict:
directories_cache = str(sorted(directories))
d = _get_directory_dict(directory, directories_cache)
if directory in directories:
children = dict()
for path in directory.iterdir():
if not path.is_dir() and path.name in HIDDEN_FILES:
# Never show ignored files.
continue
if path.is_dir() and path in directories:
children[f'{path.name}/'] = _get_recursive_directory_dict(session, path, directories)
elif path.is_dir():
children[f'{path.name}/'] = _get_directory_dict(path, directories_cache)
else:
children[path.name] = _get_file_dict(session, path)
d['children'] = children
return d
HIDDEN_DIRECTORIES = ('lost+found',)
# macOS metadata files which should never be shown in the FileBrowser.
HIDDEN_FILES = ('.DS_Store', '._.DS_Store')
def list_directories_contents(session: Session, directories_: List[str]) -> Dict:
"""List all files down to (and within) the directories provided.
This should only be used by the frontend for it's FileBrowser."""
media_directory = get_media_directory()
# Convert all directories to pathlib.Path.
directories = {media_directory / i for i in directories_}
if invalid_directories := [i for i in directories if not i.is_dir()]:
raise FileNotFoundError(f'Invalid directories: {invalid_directories}')
# Get all unique parents for each directory, but not if they are above the media directory.
parents = {j for i in directories for j in i.parents if j not in media_directory.parents}
# Add unique parents onto the requested directories.
directories = list(directories | parents)
paths = dict()
for path in media_directory.iterdir():
if path.is_dir() and path.name in HIDDEN_DIRECTORIES:
# Never show ignored directories.
continue
if not path.is_dir() and path.name in HIDDEN_FILES:
# Never show ignored files.
continue
if path.is_dir():
paths[f'{path.name}/'] = _get_recursive_directory_dict(session, path, directories)
else:
paths[path.name] = _get_file_dict(session, path)
return paths
def get_tagged_file_groups_by_ids(session: Session, file_group_ids: List[int]) -> List[dict]:
"""Return serialized FileGroups (subset of `file_group_ids`) that have at least one TagFile.
Used by the unified delete endpoint to require force-confirmation before deleting tagged files."""
if not file_group_ids:
return []
rows = session.query(FileGroup) \
.options(*FileGroup.json_options()) \
.join(TagFile, TagFile.file_group_id == FileGroup.id) \
.filter(FileGroup.id.in_(file_group_ids)) \
.distinct() \
.all()
return [fg.__json__() for fg in rows]
def get_tagged_file_groups_by_paths(session: Session, paths: Iterable[pathlib.Path]) -> List[FileGroup]:
"""Return FileGroup objects under any of the given paths that have at least one TagFile.
For a file path, matches FileGroups whose primary_path shares the file's stem in the same
directory. For a directory path, matches FileGroups whose `directory` is the path or a
descendant. Used by the path-based delete endpoint to require force-confirmation before
deleting tagged files."""
paths = list(paths)
if not paths:
return []
tagged: List[FileGroup] = []
seen_ids = set()
for path in paths:
query = session.query(FileGroup) \
.options(*FileGroup.json_options()) \
.join(TagFile, TagFile.file_group_id == FileGroup.id)
if path.is_file():
# Could be deleting a file in a FileGroup that has been tagged.
stem, _ = split_path_stem_and_suffix(path)
query = query.filter(FileGroup.primary_path.like(f'{path.parent}/{stem}%'))
else:
# Search for any FileGroups that have been tagged under this directory.
# Use indexed directory column for efficient lookup.
path_str = str(path)
query = query.filter(or_(
FileGroup.directory == path_str,
FileGroup.directory.like(f'{path_str}/%'),
))
for file_group in query:
if path.is_dir() or any(p for p in paths if p in file_group.my_paths()):
if file_group.id not in seen_ids:
seen_ids.add(file_group.id)
tagged.append(file_group)
return tagged
def delete_file_groups(session: Session, file_group_ids: List[int], force: bool = False) -> Set[str]:
"""Delete FileGroups (and their model rows via FK CASCADE) by ID.
Returns the set of `FileGroup.model` values that were deleted, so the caller can fire any
model-specific post-delete switches (e.g., `save_channels_config` for videos).
The caller is expected to have already gated this on tag confirmation via
`get_tagged_file_groups_by_ids`. `force` propagates to FileGroup.delete so the safety net
at FileGroup.delete (raises FileGroupIsTagged if force is False and tags exist) still
protects callers that skipped the pre-check."""
files = FileGroup.get_by_ids(session, file_group_ids)
if not files:
raise UnknownFile(f'No FileGroups found for ids: {file_group_ids}')
affected_models: Set[str] = {fg.model for fg in files if fg.model}
had_tagged = any(fg.tag_names for fg in files)
# Capture domain Collection IDs for any Archive FileGroups so we can prune empty ones after
# the cascade removes the Archive rows. (Archive.delete used to do this per-archive; with
# cascade-based deletion we do it as a single post-pass.)
archive_collection_ids: Set[int] = set()
if 'archive' in affected_models:
from modules.archive.models import Archive
archive_collection_ids = {
collection_id for (collection_id,) in session.query(Archive.collection_id)
.filter(Archive.file_group_id.in_([fg.id for fg in files]))
if collection_id is not None
}
for fg in files:
fg.delete(force=force)
if archive_collection_ids:
# Flush so the FK CASCADE has removed the Archive rows before we count remaining.
session.flush()
from modules.archive.models import Archive
from wrolpi.collections.models import Collection
for collection_id in archive_collection_ids:
collection = session.query(Collection).filter_by(id=collection_id, kind='domain').one_or_none()
if collection is None:
continue
remaining = session.query(Archive).filter_by(collection_id=collection_id).count()
if remaining == 0:
session.delete(collection)
if had_tagged:
save_tags_config.activate_switch()
return affected_models
@wrol_mode_check
async def delete(*paths: Union[str, pathlib.Path], force: bool = False, post_processing: bool = True):
"""
Delete a file or directory in the media directory.
If any files are tagged and force is False, returns a list of tagged FileGroups without deleting.
If force is True, deletes even tagged files.
If post_processing is False, the synchronous DB cleanup still runs but global modelers/indexers
are skipped — use this when the caller will trigger modeling itself (e.g. the upload endpoint
follows up with `upsert_file`)."""
media_directory = get_media_directory()
paths = [media_directory / i for i in paths]
if any(i == media_directory for i in paths):
raise InvalidFile(f'Cannot delete the media directory')
for path in paths:
if not path.is_dir() and not path.is_file():
logger.warning(f'Cannot delete {path} because it is not a file or a directory.')
if len(paths) > 1:
for p1 in paths:
for p2 in paths:
if p1 == p2:
continue
if p1 != p2 and p1 in p2.parents:
raise InvalidFile(f'Cannot deleted nested paths')
# Discover any tagged FileGroups under these paths so we can either gate the deletion (when
# !force) or fire `save_tags_config` afterwards (when force=True and any were tagged).
with get_db_session() as session:
tagged_file_groups = get_tagged_file_groups_by_paths(session, paths)
if tagged_file_groups and not force:
# Serialize while session is still active.
return [i.__json__() for i in tagged_file_groups]
had_tagged = bool(tagged_file_groups)
for path in paths:
ignored_directories = get_wrolpi_config().ignored_directories
if ignored_directories and str(path) in ignored_directories:
remove_ignored_directory(path)
if path.is_dir():
delete_directory(path, recursive=True, force=force)
elif path.exists():
path.unlink()
# Refresh synchronously to clean up DB before returning.
# This ensures deleted FileGroups are removed before the caller continues
# (e.g., upload API recreating the file).
from wrolpi.files.worker import file_worker
await file_worker.refresh_sync(list(paths), post_processing=post_processing)
if had_tagged:
# `tags.yaml` references TagFiles that the refresh just removed; rewrite it.
save_tags_config.activate_switch()
def _mimetype_suffix_map(path: Path, mimetype: str):
"""Special handling for mimetypes. Python Magic may not return the correct mimetype."""
from modules.docs.models import MOBI_MIMETYPE
suffix = path.suffix.lower()
if mimetype == 'application/octet-stream':
if suffix.endswith('.mobi'):
return MOBI_MIMETYPE
if suffix.endswith('.stl'):
return 'model/stl'
if suffix.endswith('.3mf'):
return 'model/3mf'
if suffix.endswith('.mp4'):
return 'video/mp4'
if suffix.endswith('.html') or suffix.endswith('.htm'):
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
return 'text/html'
if suffix.endswith('.hgt'):
return 'application/octet-stream'
if mimetype.startswith('text/') and suffix in ('.md', '.markdown'):
# libmagic reports markdown as text/plain, or as a programming language when it sniffs code.
return 'text/markdown'
if mimetype == 'text/plain':
if suffix.endswith('.json'):
return 'application/json'
if suffix.endswith('.vtt'):
return 'text/vtt'
if suffix.endswith('.csv'):
return 'text/csv'
if suffix.endswith('.html') or suffix.endswith('.htm'):
return 'text/html'
if suffix.endswith('.stl'):
return 'model/stl'
if suffix.endswith('.scad'):
return 'application/x-openscad'
if suffix.endswith('.js'):
return 'application/javascript'
if suffix.endswith('.css'):
return 'text/css'
if suffix.endswith('.srt'):
return 'text/srt'
if suffix.endswith('.yaml') or suffix.endswith('.yml'):
return 'text/yaml'
if suffix.endswith('.azw3'):
return 'application/vnd.amazon.mobi8-ebook'
if suffix.endswith('.obj'):
return 'model/obj'
if mimetype == 'application/x-subrip':
# Fallback to old mimetype.
return 'text/srt'
if mimetype == 'application/zip' and (suffix == '.html' or suffix == '.htm'):
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
return 'text/html'
if mimetype == 'application/zip' and suffix == '.docx':
return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
if mimetype == 'application/zip' and suffix == '.cbz':
return 'application/x-cbz'
if mimetype == 'application/zip' and suffix == '.3mf':
return 'model/3mf'
if mimetype == 'application/zip' and suffix == '.odt':
return 'application/vnd.oasis.opendocument.text'
if mimetype in ('application/x-rar', 'application/x-rar-compressed') and suffix == '.cbr':
return 'application/x-cbr'
if mimetype == 'video/x-ms-asf' and suffix == '.wma':
return 'audio/x-ms-wma'
return mimetype
FILE_BIN = which('file', '/usr/bin/file')
@functools.lru_cache(maxsize=10_000)
def get_mimetype(path: Path) -> str:
"""Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command."""
if magic is None:
# This method is slow, prefer the speedier `magic` module.
cmd = (FILE_BIN, '--mime-type', str(path.absolute()))
output = subprocess.check_output(cmd)
output = output.decode()
mimetype = output.split(' ')[-1].strip()
else:
mimetype = mime.from_file(path)
mimetype = _mimetype_suffix_map(path, mimetype)
return mimetype
# Special suffixes within WROLPi.
SUFFIXES = {
'.info.json',
'.ffprobe.json',
'.live_chat.json',
'.readability.html',
'.readability.json',
'.readability.txt',
}
SUFFIXES |= {f'.{i}.srt' for i in ISO_639_CODES}
SUFFIXES |= {f'.{i}.vtt' for i in ISO_639_CODES}
EXTRA_SUFFIXES = set()
for six_ in ISO_639_CODES.keys():
EXTRA_SUFFIXES |= {f'.{six_}-{i}.srt' for i in ISO_3166_CODES}
EXTRA_SUFFIXES |= {f'.{six_}-{i}.vtt' for i in ISO_3166_CODES}
# Add auto suffixes once (these don't depend on the loop variable)
EXTRA_SUFFIXES |= {f'.{i}-auto.srt' for i in ISO_639_CODES}
EXTRA_SUFFIXES |= {f'.{i}-auto.vtt' for i in ISO_639_CODES}
# Convert to lowercase frozensets for O(1) case-insensitive lookup
_SUFFIXES_LOWER = frozenset(s.lower() for s in SUFFIXES)
_EXTRA_SUFFIXES_LOWER = frozenset(s.lower() for s in EXTRA_SUFFIXES)
_HARDCODED_SUFFIXES = frozenset(
{'.info.json', '.ffprobe.json', '.live_chat.json', '.readability.html', '.readability.json', '.readability.txt'})
# Bump when SUFFIXES / split_path_stem_and_suffix grouping rules change. A stored
# file_group.stem is a cache of this algorithm; compare joins on it, so a mismatch
# looks like delete+insert and would drop tags. ensure_file_group_stems() must run
# before any compare when this version does not match the value in wrolpi_kv.
STEM_ALGORITHM_VERSION = 1
STEM_ALGORITHM_VERSION_KEY = 'stem_algorithm_version'
# Cancelled compare_file_groups leaves ensure_file_group_stems running in a thread.
# Tests (and overlapping refreshes) share one SQLite writer; serialize the backfill.
_stem_backfill_lock = threading.Lock()
PART_PARSER = re.compile(r'(.+?)(\.f[\d]{2,3})?(\.info)?(\.\w{3,4})(\.part)', re.IGNORECASE)
def _extract_candidate_suffixes(name_lower: str) -> List[str]:
"""Extract potential suffixes from filename (last 1, 2, 3 dot-parts).
Returns suffixes in order from shortest to longest.
"""
parts = name_lower.rsplit('.', 3)
if len(parts) == 1:
return []
candidates = []
append_candidate = candidates.append
suffix = ''
for i in range(len(parts) - 1, 0, -1):
suffix = '.' + parts[i] + suffix
append_candidate(suffix)
return candidates
@functools.lru_cache(maxsize=5_000)
def _get_suffix_length(name_lower: str) -> int:
"""Get length of matching suffix using O(1) set lookups.
Returns 0 if no suffix matches.
"""
# Fast path: check hardcoded multi-part suffixes first
for suffix in _HARDCODED_SUFFIXES:
if name_lower.endswith(suffix):
return len(suffix)
candidates = _extract_candidate_suffixes(name_lower)
if not candidates:
return 0
# Check SUFFIXES (longest first for most specific match)
for suffix in reversed(candidates):
if suffix in _SUFFIXES_LOWER:
return len(suffix)
# Only check EXTRA_SUFFIXES for .srt/.vtt files
simple_suffix = candidates[0] if candidates else ''
if simple_suffix in {'.srt', '.vtt'}:
for suffix in reversed(candidates):
if suffix in _EXTRA_SUFFIXES_LOWER:
return len(suffix)
return len(simple_suffix)
@functools.lru_cache(maxsize=10_000)
def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = False) -> Tuple[str, str]:
"""Get the path's stem and suffix.
This function handles WROLPi suffixes like .info.json.
>>> split_path_stem_and_suffix('/foo/bar.txt')
('bar', '.txt')
>>> split_path_stem_and_suffix('/foo/bar.txt', full=True)
('/foo/bar', '.txt')
>>> split_path_stem_and_suffix('/foo/bar.info.json', full=True)
('/foo/bar', '.info.json')
"""
path = pathlib.Path(path) if isinstance(path, str) else path
if path.suffix == '.part':
# yt-dlp uses part files while downloading, include those in a FileGroup.
stem, info, format_num, suffix, part = PART_PARSER.match(path.name).groups()
info = info or ''
format_num = format_num or ''
stem = f'{path.parent}/{stem}' if full else stem
return stem, f'{info}{format_num}{suffix}{part}'
name = path.name
name_lower = name.lower()
# Get suffix length using O(1) set lookups instead of O(n) linear search
suffix_len = _get_suffix_length(name_lower)
if suffix_len > 0:
idx = -suffix_len
if full:
return f'{path.parent}/{name[:idx]}', name[idx:]
else:
return name[:idx], name[idx:]
# Path has no suffix.
if full:
return f'{path.parent}/{path.name}', ''
else:
return path.name, ''
def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ...] | Set[pathlib.Path]) \
-> List[pathlib.Path]:
"""Returns the first of each Path with a unique stem. Used to detect if a group of files share a stem."""
results = unique_by_predicate(files, lambda i: split_path_stem_and_suffix(i)[0])
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'get_unique_files_by_stem {files=}')
logger.trace(f'get_unique_files_by_stem {results=}')
return results
refresh_logger = logger.getChild('refresh')
def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]:
"""This generates `FileGroup.files` from a list of files.
Paths are stored as filenames only (relative to FileGroup.directory).
"""
files = list()
for file in group:
mimetype = get_mimetype(file)
stat = file.stat()
modification_datetime = str(from_timestamp(stat.st_mtime))
_, suffix = split_path_stem_and_suffix(file)
size = stat.st_size
files.append(dict(
path=file.name, # Store filename only, not full path
mimetype=mimetype,
modification_datetime=modification_datetime,
size=size,
suffix=suffix,
))
return files
def get_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path:
"""Given a list of files, return the file that we can model or index.
@raise NoPrimaryFile: If not primary file can be found."""
from modules.archive.lib import is_singlefile_file
if len(files) == 0:
raise ValueError(f'Cannot find primary file without any files')
elif len(files) == 1:
# Only one file is always the primary.
return files[0]
file_mimetypes = list()
for idx, i in enumerate(files):
try:
file_mimetypes.append((i, get_mimetype(i)))
except FileNotFoundError:
# File was deleted.
pass
if not file_mimetypes:
raise FileNotFoundError(f'Cannot find primary file. All files are no longer accessible: {files[0]}')
# EPUB has high priority.
mimetypes = {i[1] for i in file_mimetypes}
has_epub = any(i.startswith('application/epub') for i in mimetypes)
for file, mimetype in file_mimetypes:
# These are the files that can be modeled.
if is_singlefile_file(file):
return file
if mimetype.startswith('video/'):
return file
if mimetype.startswith('audio/'):
return file
if mimetype.startswith('application/epub'):
return file
if mimetype.startswith('application/pdf') and not has_epub:
return file
if mimetype.startswith('application/x-mobipocket-ebook') and not has_epub:
# Only return a MOBI if no EPUB is present.
return file
if mimetype in ('application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.oasis.opendocument.text',
'application/x-cbz', 'application/x-cbr',
'application/vnd.comicbook+zip', 'application/vnd.comicbook-rar'):
return file
if file.suffix.lower() in ('.cbz', '.cbr', '.cbt', '.cb7'):
return file
# Secondary file types.
for file, mimetype in file_mimetypes:
if mimetype.startswith('image/'):
return file
raise NoPrimaryFile(f'Cannot find primary file for group: {files}')
def choose_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path:
"""Primary file for a stem group. Falls back to a sorted path so every caller agrees.
`_upsert_files` and `_upsert_file_groups` must pick the same path. A set's iteration
order is not stable; using it as the fallback would DELETE the existing tagged row
and INSERT a new untagged one.
"""
paths = [pathlib.Path(p) for p in files]
if not paths:
raise ValueError('Cannot find primary file without any files')
try:
return get_primary_file(paths)
except NoPrimaryFile:
return sorted(paths)[0]
def _upsert_files(files: List[pathlib.Path],
progress_callback: Callable[[int, int], None] = None):
"""Insert/update all records of the provided files.
Any inserted files will be marked with `indexed=false`. Any existing files will only be updated if their
size/modification_datetime changed.
Args:
files: List of file paths to upsert
progress_callback: Optional callback(processed, total) called after each chunk is processed
It is assumed all files exist."""
total_files = len(files)
processed_count = 0
non_primary_files = set()
for chunk in chunks_by_stem(files, 500):
# Group all files by their shared full-stem. `chunk_by_stem` already sorted the files.
grouped = group_files_by_stem(chunk, pre_sorted=True)
# Convert the groups into file_groups. Extract common information into `file_group.data`.
values = dict()
for group in grouped:
# The primary file is the video/SingleFile/epub, etc.
try:
primary_path = choose_primary_file(group)
# The primary mimetype allows modelers to find its file_groups.
mimetype = get_mimetype(primary_path)
# The group uses a common modification_datetime so the group will be re-indexed when any of it's
# files are modified.
modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group))
size = sum(i.stat().st_size for i in group)
files_json = json.dumps(_paths_to_files_dict(group))
stem, suffix = split_path_stem_and_suffix(primary_path)
suffix = (suffix or '').lower() or None
values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem)
non_primary_files.update(i for i in group if i != primary_path)
except FileNotFoundError as e:
# A file was deleted between discovery and upsert (temp files, WAL sidecars, etc.).
refresh_logger.warning(f'File vanished during refresh, skipping group near {group[0]}', exc_info=e)
continue
values = [(str(primary_path),
str(primary_path.parent), # directory
False, # `indexed` is false to force indexing by default.
*i) for primary_path, i in values.items()]
with get_db_curs(commit=True) as curs:
# SQLite's RETURNING only exposes the post-upsert row; find the previously-indexed
# paths first so re-indexing invalidations can be counted.
primary_paths = [i[0] for i in values]
curs.execute('SELECT primary_path FROM file_group WHERE indexed = true AND primary_path IN '
'(SELECT value FROM json_each(?))', (json.dumps(primary_paths),))
previously_indexed = {i['primary_path'] for i in curs.fetchall()}
if non_primary_files:
# Delete before INSERT so a new primary_path that shares (directory, stem)
# with an old sidecar FileGroup does not hit the unique index.
curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))',
(json.dumps(list(map(str, non_primary_files))),))
placeholders, params = values_clause(values)
stmt = f'''
INSERT INTO file_group
(primary_path, directory, indexed, modification_datetime, mimetype, size, files,
suffix, stem)
VALUES {placeholders}
ON CONFLICT (primary_path) DO UPDATE
SET
directory=EXCLUDED.directory,
modification_datetime=EXCLUDED.modification_datetime,
mimetype=EXCLUDED.mimetype,
size=EXCLUDED.size,
-- Always keep suffix/stem current so grouping and suffix search stay correct.
suffix=EXCLUDED.suffix,
stem=EXCLUDED.stem,
files=(
-- Do not overwrite files unless the files have changed and need to be re-indexed.
CASE
WHEN
file_group.modification_datetime = EXCLUDED.modification_datetime
AND file_group.size = EXCLUDED.size
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
THEN file_group.files
ELSE EXCLUDED.files
END
),
-- Preserve TRUE `indexed` only if the files have not changed.
indexed=(
file_group.indexed = true
AND file_group.modification_datetime = EXCLUDED.modification_datetime
AND file_group.size = EXCLUDED.size
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
)
RETURNING id, primary_path, indexed
'''
curs.execute(stmt, params)
results = list(curs.fetchall())
# Count the files that used to be indexed, but need to be re-indexed.
invalidated_files = len([i['id'] for i in results
if i['primary_path'] in previously_indexed and not i['indexed']])
if invalidated_files:
refresh_logger.info(f'Invalidated indexes of {invalidated_files} file groups near {chunk[0]}')
else:
refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}')
non_primary_files.clear()
# Report progress after each chunk
processed_count += len(chunk)
if progress_callback:
progress_callback(processed_count, total_files)
def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]:
"""Return a new list which does not contain any file paths that are in ignored directories."""
from wrolpi.db import get_db_file
ignored_directories = get_normalized_ignored_directories()
files = [i for i in files if not any(str(i).startswith(j) for j in ignored_directories)]
# Never index the database (or its WAL sidecars), even if the user un-ignores `config`.
db_file = str(get_db_file())
files = [i for i in files if not str(i).startswith(db_file)]
return files
def _ensure_kv_table(curs):
curs.execute('CREATE TABLE IF NOT EXISTS wrolpi_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
def get_stem_algorithm_version() -> int:
"""The grouping-algorithm version last applied to file_group.stem, or 0 if never stored."""
with get_db_curs() as curs:
_ensure_kv_table(curs)
curs.execute('SELECT value FROM wrolpi_kv WHERE key = ?', (STEM_ALGORITHM_VERSION_KEY,))
row = curs.fetchone()
return int(row['value']) if row else 0
def set_stem_algorithm_version(version: int):
with get_db_curs(commit=True) as curs:
_ensure_kv_table(curs)
curs.execute(
'INSERT INTO wrolpi_kv (key, value) VALUES (?, ?) '
'ON CONFLICT(key) DO UPDATE SET value = excluded.value',
(STEM_ALGORITHM_VERSION_KEY, str(version)),
)
def ensure_file_group_stems():
"""Recompute file_group.stem when STEM_ALGORITHM_VERSION has changed.
A stored stem is a cache of split_path_stem_and_suffix. Compare joins on (directory, stem),
so an algorithm change without this backfill reports the same files as deleted+new and
drops tags, viewed timestamps, and domain rows. Must run before any compare.
After a successful backfill the kv version matches, and later refreshes skip the table scan.
Upsert, from_paths, move, and reorganize must keep stem current.
A lock serializes callers: cancelling compare_file_groups does not stop the executor
thread, and a second compare would otherwise write wrolpi_kv concurrently (SQLITE_BUSY).
"""
with _stem_backfill_lock:
stored = get_stem_algorithm_version()
if stored == STEM_ALGORITHM_VERSION:
return
with get_db_curs() as curs:
curs.execute('SELECT id, primary_path FROM file_group')
rows = list(curs.fetchall())
if rows:
updates = []
for row in rows:
stem, _ = split_path_stem_and_suffix(row['primary_path'])
updates.append((stem, row['id']))
with get_db_curs(commit=True) as curs:
for chunk in chunks(updates, 500):
curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk)
set_stem_algorithm_version(STEM_ALGORITHM_VERSION)
def backfill_null_file_group_stems():
"""Fill file_group.stem for rows that never got one. Used before creating the unique index."""
with get_db_curs() as curs:
curs.execute("SELECT id, primary_path FROM file_group WHERE stem IS NULL OR stem = ''")
rows = list(curs.fetchall())
if not rows:
return
updates = [(split_path_stem_and_suffix(row['primary_path'])[0], row['id']) for row in rows]
with get_db_curs(commit=True) as curs:
for chunk in chunks(updates, 500):
curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk)
def dedupe_file_groups_by_directory_stem() -> int:
"""Collapse FileGroups that share (directory, stem).
Prefers a row with a domain model, then tags, then non-empty files, then highest id.
Tags and collection membership on losers are moved onto the winner. Returns the
number of FileGroups deleted.
"""
with get_db_curs() as curs:
curs.execute('''
SELECT directory, stem
FROM file_group
WHERE directory IS NOT NULL
AND stem IS NOT NULL
AND stem != ''
GROUP BY directory, stem
HAVING COUNT(*) > 1
''')
keys = [(row['directory'], row['stem']) for row in curs.fetchall()]
deleted = 0
for directory, stem in keys:
with get_db_curs() as curs:
curs.execute('''
SELECT fg.id,
CASE
WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0
ELSE json_array_length(fg.files)
END AS file_count,
CASE
WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1
ELSE 0
END AS has_model,
(SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count
FROM file_group fg
WHERE fg.directory = ?
AND fg.stem = ?
''', (directory, stem))
members = list(curs.fetchall())
members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True)
winner_id = members[0]['id']
loser_ids = [m['id'] for m in members[1:]]
with get_db_curs(commit=True) as curs:
for loser_id in loser_ids:
curs.execute('''
INSERT OR IGNORE INTO tag_file (tag_id, file_group_id, created_at)
SELECT tag_id, ?, created_at
FROM tag_file
WHERE file_group_id = ?
''', (winner_id, loser_id))
curs.execute('''
UPDATE collection_item
SET file_group_id = ?
WHERE file_group_id = ?
AND collection_id NOT IN (SELECT collection_id
FROM (SELECT collection_id
FROM collection_item
WHERE file_group_id = ?))
''', (winner_id, loser_id, winner_id))
curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,))
deleted += 1
if deleted:
refresh_logger.info(f'Collapsed {deleted} duplicate FileGroups sharing (directory, stem)')
return deleted
async def apply_indexers(progress_callback: Callable[[int, int], None] = None):
"""Index leftover FileGroups that no modeler claimed.
File indexing state machine
---------------------------
A FileGroup is discovered (a row exists), then modeled and/or content-indexed.
`indexed` means "a modeler or this leftover indexer has finished with this row
for the current files." It is not "a domain model exists." `model` is the
domain claim (`video` / `archive` / `doc` / `zim`).
1. Discover/upsert: new or changed groups are inserted with indexed=0.
Unchanged groups keep indexed=1.
2. Modelers run first. A modeler that exclusively owns its mimetype
(video, doc, zim) selects:
(domain_row IS NULL OR indexed != 1) AND mimetype_matches
so a FileGroup the leftover indexer already marked indexed=1 still
gets a Video/Doc/Zim row. Failures are skipped for the rest of the
run (failed_ids) and retried on the next refresh.
3. Archive shares text/html with generic HTML. It selects only
indexed != 1. HTML that is not a SingleFile stays indexed=0 so this
function can claim it. HTML that is an Archive is indexed=1 with
model='archive'.
4. This function claims every remaining indexed != 1 row. It writes
a/b/c/d_text via the Indexer registry and sets indexed=1 even on
extractor failure so poison files do not loop forever.
5. Cleanup (channel claims, empty collections, tag config) runs last.
FileGroup.do_model() is the single-file equivalent of steps 2-4, used by
uploads. Batch modelers are used by FileWorker refresh.
Args:
progress_callback: Optional callback(processed, total) called after each batch.
"""
from wrolpi.files.models import FileGroup
refresh_logger.info('Applying indexers')
# Get initial count for progress tracking
total_to_index = 0
total_indexed = 0
if progress_callback:
with get_db_session() as session:
total_to_index = session.query(FileGroup).filter(FileGroup.indexed != True).count()
progress_callback(0, total_to_index)
update_stmt = ('UPDATE file_group SET "indexed" = 1, title = :title, a_text = :a_text, b_text = :b_text,'
' c_text = :c_text, d_text = :d_text WHERE id = :id')
def _strip_surrogates(value):
return strip_surrogates(value) if isinstance(value, str) else value
while True:
# Read the next batch as plain columns and let the read transaction end. Indexers can run
# subprocesses for minutes, and SQLite requires short write transactions: a session held
# across the batch fails its commit with "database is locked" (SQLITE_BUSY_SNAPSHOT, which
# ignores busy_timeout) whenever any other connection commits a write in the meantime.
with get_db_session() as session:
rows = session.query(FileGroup.id, FileGroup.primary_path, FileGroup.mimetype, FileGroup.title) \
.filter(FileGroup.indexed != True).limit(20).all()
if not rows:
break
# Index each file on a transient FileGroup: no session, no open transaction.
processed = 0
params = []
last_path = None
for fg_id, primary_path, mimetype, title in rows:
processed += 1
file_group = FileGroup(primary_path=primary_path, mimetype=mimetype, title=title)
try:
file_group.do_index()
except Exception:
# Error has already been logged in .do_index. It is still marked as indexed
# below; we won't try to index it again.
if PYTEST:
raise
params.append(dict(id=fg_id, title=file_group.title, a_text=file_group.a_text,
b_text=file_group.b_text, c_text=file_group.c_text, d_text=file_group.d_text))
last_path = primary_path
# Sleep to catch cancel.
await asyncio.sleep(0)
# Write the batch in one short transaction (BEGIN IMMEDIATE via commit=True).
try:
with get_db_curs(commit=True) as curs:
curs.executemany(update_stmt, params)
except UnicodeEncodeError as e:
# Paths with invalid UTF-8 surrogates produce titles/texts sqlite3 cannot store.
refresh_logger.warning(f'UnicodeEncodeError during indexer update, sanitizing batch: {e}')
params = [{k: _strip_surrogates(v) for k, v in p.items()} for p in params]
try:
with get_db_curs(commit=True) as curs:
curs.executemany(update_stmt, params)
except UnicodeEncodeError as e2:
# Still failing - mark as indexed without the texts so we don't loop forever.
refresh_logger.error(f'Failed to fix UnicodeEncodeError, skipping batch: {e2}')
with get_db_curs(commit=True) as curs:
curs.execute('UPDATE file_group SET "indexed" = 1 WHERE id IN '
'(SELECT value FROM json_each(?))',
(json.dumps([p['id'] for p in params]),))
if last_path:
refresh_logger.debug(f'Indexed {processed} files near {last_path}')
# Update progress
total_indexed += processed
if progress_callback and total_to_index > 0:
progress_callback(total_indexed, total_to_index)
if processed < 20:
# Processed less than the limit, don't do the next query.
break
def upsert_directories(parent_directories, directories):
"""
Insert/update/delete directories provided. Deletes all children of `parent_directories` which are not in
`directories`.
"""
idempotency = now()
directories = list(directories) + list(parent_directories)
# Only insert directories that are children of `media_directory` and exist.
directories = get_paths_in_media_directory([i for i in directories if i.is_dir()])
if directories:
# Insert any directories that were created, update any directories which previously existed.
# Chunk the rows so the multi-row VALUES clause stays under SQLite's bound-variable limit;
# a large media directory can hold tens of thousands of directories.
values = [(str(i.absolute()), i.name, idempotency) for i in directories]
with get_db_curs(commit=True) as curs:
for chunk in chunks(values, BULK_SQL_CHUNK_SIZE):
placeholders, params = values_clause(chunk)
stmt = f'''
INSERT INTO directory (path, name, idempotency) VALUES {placeholders}
ON CONFLICT (path) DO UPDATE
SET idempotency = EXCLUDED.idempotency
'''
curs.execute(stmt, params)
with get_db_curs(commit=True) as curs:
# Delete the children of any parent directory which no longer exists.
for directory in parent_directories:
stmt = f'DELETE FROM directory WHERE path LIKE :directory AND idempotency != :idempotency'
curs.execute(stmt, dict(directory=f'{directory.absolute()}%', idempotency=idempotency))
async def search_directories_by_name(session: Session, name: str, excluded: List[str] = None, limit: int = 20) \
-> List[Directory]:
"""Find the Directories whose names contain the `name` string."""
excluded = excluded or []
directories = session.query(Directory) \
.filter(Directory.name.ilike(f'%{name}%')) \
.filter(Directory.path.notin_(excluded)) \
.order_by(asc(Directory.name)) \
.limit(limit).all()
return directories
# Pagination totals: counting a large library is a full index walk that costs seconds cold on a
# spinning disk. Pagination can tolerate a total that is a minute stale. Bypassed under pytest
# so tests see exact totals. Keyed by filters, never by offset/limit/order (except filters that
# an order implies, e.g. viewed IS NOT NULL).
SEARCH_TOTALS_CACHE_SECONDS = 60
_SEARCH_TOTALS_CACHE: Dict[str, Tuple[float, int]] = dict()
def cached_search_total(key: str, compute: Callable[[], int]) -> int:
"""Return `compute()`; cache the result briefly unless running under pytest."""
if not PYTEST:
hit = _SEARCH_TOTALS_CACHE.get(key)
if hit and time.monotonic() - hit[0] < SEARCH_TOTALS_CACHE_SECONDS:
return hit[1]
total = compute()
_SEARCH_TOTALS_CACHE[key] = (time.monotonic(), total)
return total
def search_filter_cache_key(corpus: str, **filters) -> str:
"""Stable cache key for a search total. Omits empty/false filters."""
payload = {'corpus': corpus}
for k, v in filters.items():
if v in (None, '', [], (), False):
continue
if isinstance(v, (list, tuple, set)):
v = sorted(v)
payload[k] = v
return json.dumps(payload, sort_keys=True, default=str)
def count_file_groups(session: Session, statement: str, params: dict) -> int:
"""Run a `SELECT COUNT(*) AS total ...` statement and return the integer."""
with session_curs(session) as curs:
curs.execute(statement, params)
row = curs.fetchone()
return int(row['total'])
def search_files(session: Session, search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
from_year: int = None, to_year: int = None, any_tag: bool = False, order: str = None,
url: str = None, suffix: str = None, path: str = None, deep: bool = False) -> \
Tuple[List[dict], int]:
"""Search the FileGroup table.
Order the returned Files by their rank if `search_str` is provided. Return all files if
`search_str` is empty.
@param any_tag: The file must have some tag.
@param to_year: The file must be published on or before this year.
@param from_year: The file must be published on or after this year.
@param search_str: Search the ts_vector of the file. Returns all files if this is empty.
@param limit: Return only this many files.
@param offset: Offset the query.
@param mimetypes: Only return files that match these mimetypes.
@param model: Only return files that match this model.
@param tag_names: A list of tag names.
@param headline: Includes FTS search headlines if True.
@param months: A list of integers representing the index of the month of the year, starting at 1.
@param order: Used to change results from most relevant to recently viewed.
@param url: Filter by URL using case-insensitive partial match (ILIKE).
@param suffix: Only return files whose primary file has this suffix (e.g. ".bin"), case-insensitive.
@param path: Filter by primary_path using case-insensitive partial match (ILIKE).
@param deep: Search all text including captions/body (d_text); slower but more thorough.
"""
from wrolpi import fts
# The frontend may send explicit JSON nulls (bypassing the schema defaults); SQLite rejects
# `LIMIT NULL`/`OFFSET NULL` (Postgres treated them as no-ops).
limit = int(limit) if limit else 20
offset = int(offset) if offset else 0
params = dict(offset=offset, limit=limit, search_str=search_str)
wheres = []
selects = []
order_by = '1 ASC'
joins = []
# The fast (non-deep) path excludes d_text (captions/body) from matching.
fts_search = fts.file_group_search_join(search_str, deep=deep, headlines=bool(headline)) \
if search_str else None
if search_str and not fts_search:
# Nothing usable to search (punctuation-only or all-negated query); match nothing, like
# an empty websearch_to_tsquery did.
return [], 0
if fts_search:
joins.append(fts_search.join)
selects.append(fts_search.rank_select)
params.update(fts_search.params)
order_by = '2 DESC'
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
if model:
params['model'] = model
wheres.append('model = :model')
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
wheres, params = months_selector_to_where(wheres, params, months)
wheres, params = date_range_to_where(wheres, params, from_year, to_year)
if url:
# SQLite LIKE is case-insensitive (for ASCII), matching the old ILIKE behavior.
params['url_filter'] = f'%{url}%'
wheres.append('fg.url LIKE :url_filter')
if suffix:
# Suffix is stored lowercased with a leading dot; normalize the caller's value to match the index.
normalized = suffix if suffix.startswith('.') else f'.{suffix}'
params['suffix'] = normalized.lower()
wheres.append('fg.suffix = :suffix')
if path:
params['path_filter'] = f'%{path}%'
wheres.append('fg.primary_path LIKE :path_filter')
if fts_search and headline:
# b/c/d headlines come from FTS5 snippets (computed in the fts subquery); the title
# headline is computed in Python by `handle_file_group_search_results` (title is not an
# FTS column).
headline = ', fts.b_headline, fts.c_headline, fts.d_headline'
else:
headline = ''
if order == 'viewed':
order_by = 'viewed DESC NULLS LAST, 1 ASC'
elif order == '-viewed':
order_by = 'viewed ASC NULLS LAST, 1 ASC'
viewed_only = bool(order and not search_str)
if viewed_only:
# Only filter out unviewed files if search_str is not provided.
wheres.append('viewed IS NOT NULL')
where_sql = ('WHERE\n' + '\n AND '.join(wheres)) if wheres else ''
extra_selects = f', {", ".join(selects)}' if selects else ''
join = '\n'.join(joins)
stmt = f'''
SELECT fg.id{extra_selects}
{headline}
FROM file_group fg
{join}
{where_sql}
ORDER BY {order_by}
LIMIT :limit OFFSET :offset
'''
logger.debug(stmt)
# COUNT(*) OVER() in the page query forces SQLite to visit every match before LIMIT.
# Count separately (and cache) so the page can use the datetime/id index.
if not search_str and not tag_names and not mimetypes and not model and not months \
and not from_year and not to_year and not any_tag and not url and not suffix \
and not path and not viewed_only:
count_stmt = 'SELECT COUNT(*) AS total FROM file_group'
count_params = dict()
elif fts_search:
# Count via a direct FTS join (no ranking) rather than the snippet subquery.
fts_count = fts.file_group_search(search_str, deep=deep)
count_wheres = list(wheres)
if fts_count:
count_wheres.append(fts_count.where)
count_where_sql = ('WHERE\n' + '\n AND '.join(count_wheres)) if count_wheres else ''
count_join = fts_count.join if fts_count else ''
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {count_join} {count_where_sql}'
count_params = dict(params)
if fts_count:
count_params.update(fts_count.params)
else:
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {join} {where_sql}'
count_params = params
cache_key = search_filter_cache_key(
'files', search_str=search_str, mimetypes=mimetypes, model=model,
tag_names=tag_names, months=months, from_year=from_year, to_year=to_year,
any_tag=any_tag, url=url, suffix=suffix, path=path, deep=deep,
viewed_only=viewed_only,
)
total = cached_search_total(cache_key, lambda: count_file_groups(session, count_stmt, count_params))
results, total = handle_file_group_search_results(session, stmt, params, total=total)
return results, total
def handle_file_group_search_results(session: Session, statement: str, params: dict, total: int = None) \
-> Tuple[List[dict], int]:
"""
Execute the provided SQL statement and fetch the Files returned.
The statement must select `id`. Pass `total` from a separate COUNT (callers should not use
`COUNT(*) OVER()` in the page query; it prevents SQLite from applying LIMIT via an index).
WARNING: This expects specific queries to be executed and shouldn't be used for things not related to file search.
See: `search`
"""
with session_curs(session) as curs:
curs.execute(statement, params)
results = [dict(i) for i in curs.fetchall()]
if total is None:
total = results[0]['total'] if results else 0
ordered_ids = [i['id'] for i in results]
extras = [dict(
ts_rank=i.get('ts_rank'),
# Snippet columns exist only when headlines were requested; `title_headline` is
# computed below (title is not an FTS5 column).
headlines=('b_headline' in i),
title_headline=i.get('title_headline'),
b_headline=i.get('b_headline'),
c_headline=i.get('c_headline'),
d_headline=i.get('d_headline'),
) for i in results]
from modules.archive.models import Archive
from modules.videos.models import Video
# Serialized below: what Video's and Archive's `__json__` read beyond the FileGroup (see their `json_options`).
results = session.query(FileGroup, Video, Archive) \
.options(*FileGroup.json_options(), undefer(Video.ffprobe_json), joinedload(Video.channel),
joinedload(Archive.collection)) \
.filter(FileGroup.id.in_(ordered_ids)) \
.outerjoin(Video, Video.file_group_id == FileGroup.id) \
.outerjoin(Archive, Archive.file_group_id == FileGroup.id)
# Order FileGroups by their location in ordered_ids. Use dict for O(1) lookups instead of O(n) list.index().
id_to_order = {id_: idx for idx, id_ in enumerate(ordered_ids)}
file_groups = sorted(results, key=lambda i: id_to_order.get(i[0].id, 0))
results = list()
for extra, (file_group, video, archive) in zip_longest(extras, file_groups):
if video:
results.append(video.__json__())
elif archive:
results.append(archive.__json__())
else:
results.append(file_group.__json__())
# Preserve the ts_ranks, if any.
if extra:
results[-1]['ts_rank'] = extra['ts_rank']
results[-1]['title_headline'] = extra['title_headline']
results[-1]['b_headline'] = extra['b_headline']
results[-1]['c_headline'] = extra['c_headline']
results[-1]['d_headline'] = extra['d_headline']
search_str = params.get('search_str') if isinstance(params, dict) else None
# When headlines were requested, compute the title headlines in Python (the FTS5 snippet
# function only works on indexed columns; `title` is not one).
if search_str and any(extra and extra.get('headlines') for extra in extras):
from wrolpi import fts
titles = [result.get('title') or '' for result in results]
for result, (headline_text, rank) in zip(results, fts.headline_texts(titles, search_str)):
if rank > 0:
result['title_headline'] = headline_text
# When searching by text, attach a `section_hint` to any doc-modeled result so
# the UI can deep-link into the matching EPUB chapter or PDF page.
if search_str:
from modules.docs.lib import _fetch_section_hints
# `r['id']` is the FileGroup primary key; _fetch_section_hints filters
# on `file_group_id`, so this list is file-group ids (not doc.id).
fg_ids = [r['id'] for r in results if r.get('model') == 'doc']
if fg_ids:
hints = _fetch_section_hints(session, fg_ids, search_str)
for r in results:
hint = hints.get(r['id'])
if hint:
r['section_hint'] = hint
return results, total
def glob_shared_stem(path: pathlib.Path) -> List[pathlib.Path]:
"""Return all paths that share the same stem and parent path as the provided path.
Example paths: foo.mp4, foo.png, foo.info.json, foobar.txt
>>> glob_shared_stem(pathlib.Path('foo.mp4'))
['foo.mp4', 'foo.png', 'foo.info.json']
"""
path = path if isinstance(path, pathlib.Path) else pathlib.Path(path)
stem, suffix = split_path_stem_and_suffix(path)
escaped_stem = glob.escape(stem)
paths = [pathlib.Path(i) for i in path.parent.glob(f'{escaped_stem}*')
if i == path or i.name.startswith(f'{stem}.')]
return paths
def get_matching_directories(path: Union[str, Path]) -> List[pathlib.Path]:
"""
Return a list of directory strings that start with the provided path. If the path is a directory, return its
subdirectories, if the directory contains no subdirectories, return the directory.
"""
path: pathlib.Path = pathlib.Path(path) if isinstance(path, str) else path
path = path.resolve()
ignored_directories = {pathlib.Path(i).resolve() for i in get_wrolpi_config().ignored_directories}
if path.is_dir():
# The provided path is a directory, return its subdirectories, or itself if no subdirectories exist
paths = list(path.iterdir())
paths = sorted(i for i in paths if i.is_dir() and i not in ignored_directories)
if len(paths) == 0:
return [get_real_path_name(path.resolve())]
return paths
paths = path.parent.iterdir()
prefix = path.name.lower()
paths = sorted(
i for i in paths if i.is_dir() and i.name.lower().startswith(prefix) and i.resolve() not in ignored_directories
)
paths = unique_by_predicate(paths, None)
return paths
WHITESPACE = re.compile(r'[\s_\[\]()]')
def split_file_name_words(name: str) -> str:
"""Split words in a filename.
Words are assumed to be separated by underscore, dash, or space. Words with a dash are included as a group, and
individually ('self-reliance' -> ['self', 'reliance', 'self-reliance']).
>>> split_file_name_words('this self-reliance_split.txt')
['this', 'self', 'reliance', 'self-reliance', 'split', 'txt']
"""
if not name:
raise ValueError(f'Invalid filename: {name}')
try:
stem, suffix = split_path_stem_and_suffix(name)
words = []
for word1 in WHITESPACE.split(stem):
if '-' in word1[1:]:
words.extend(word1.split('-'))
words.append(word1)
# Include the suffix so the user can search without the "."
if suffix:
words.append(suffix.lstrip('.'))
words = ' '.join(i for i in words if i)
return words
except Exception as e:
logger.error(f'Failed to split filename into words: {name}', exc_info=e)
return name
def get_file_statistics(session: Session):
with session_curs(session) as curs:
curs.execute('''
SELECT
-- All items in file_group.files are real individual files.
SUM(json_array_length(files)) AS "total_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype =
'application/x-mobipocket-ebook') AS "ebook_count",
(SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files",
(SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims",
(SELECT COUNT(*) FROM tag) AS "tags_count",
SUM(size) AS "total_size",
(SELECT COUNT(*) FROM archive) AS archive_count
FROM file_group
''')
statistics = dict(curs.fetchall()[0])
statistics['total_count'] = statistics['total_count'] or 0
statistics['total_size'] = statistics['total_size'] or 0
return statistics
def group_files_by_stem(files: List[pathlib.Path], pre_sorted: bool = False) -> \
Generator[List[pathlib.Path], None, None]:
"""
Return lists of paths, each list contains only paths that share a common stem.
>>> a = ['foo.txt', 'foo.mp4', 'bar.txt']
>>> group_files_by_stem(list(map((pathlib.Path, a))))
# Generator(['foo.mp4', 'foo.txt'], ['bar.txt'])
@param files: A list of paths to be grouped. All paths must be in the same directory!
@param pre_sorted: This function requires the files to be sorted, it will sort them by default.
"""
files = sorted(files) if not pre_sorted else files.copy()
file = files.pop(0)
group = [file, ]
prev_dir, prev_stem = file.parent, split_path_stem_and_suffix(file)[0]
for file in files:
stem, _ = split_path_stem_and_suffix(file)
if stem == prev_stem and file.parent == prev_dir:
group.append(file)
continue
# Directory or stem changed, group is finished.
yield group
group = [file, ]
prev_dir, prev_stem = file.parent, stem
yield group
async def _get_tag_and_file_group(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
tag_id: int):
if file_group_id:
file_group: FileGroup = session.query(FileGroup).filter_by(id=file_group_id).one_or_none()
if not file_group:
raise UnknownFile(f'Cannot find FileGroup with id {file_group_id}')
elif file_group_primary_path:
path = get_media_directory() / file_group_primary_path
file_group = FileGroup.get_by_path(session, path)
# Create the FileGroup, if possible.
if not file_group and path.is_file():
file_group = FileGroup.from_paths(session, path)
if not file_group:
raise UnknownFile(f'Cannot find FileGroup with primary_path {repr(str(file_group_primary_path))}')
else:
raise UnknownFile(f'Cannot find FileGroup without id or primary_path')
if tag_id:
tag: Tag = session.query(Tag).filter_by(id=tag_id).one_or_none()
if not tag:
raise UnknownTag(f'Cannot find Tag with id {tag_id}')
elif tag_name:
tag: Tag = Tag.get_by_name(session, tag_name)
if not tag:
raise UnknownTag(f'Cannot find Tag with name {tag_name}')
else:
raise UnknownTag('Cannot find Tag without id or name')
return file_group, tag
async def add_file_group_tag(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
tag_id: int) -> TagFile:
file_group, tag = await _get_tag_and_file_group(session, file_group_id, file_group_primary_path, tag_name, tag_id)
tag_file = file_group.add_tag(session, tag.id)
return tag_file
async def remove_file_group_tag(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
tag_id: int):
file_group, tag = await _get_tag_and_file_group(session, file_group_id, file_group_primary_path, tag_name, tag_id)
file_group.untag(session, tag.id)
def mimetypes_to_sql_wheres(wheres: List[str], params: dict, mimetypes: List[str]) -> Tuple[List[str], dict]:
if mimetypes:
local_wheres = list()
for idx, mimetype in enumerate(mimetypes):
key = f'mimetype{idx}'
if len(mimetype.split('/')) == 1:
params[key] = f'{mimetype}/%'
else:
params[key] = f'{mimetype}%'
local_wheres.append(f'mimetype LIKE :{key}')
wheres.append(f'({" OR ".join(local_wheres)})')
return wheres, params
async def search_file_suggestion_count(session: Session, search_str: str, tag_names: List[str], mimetypes: List[str],
months: List[int] = None, from_year: int = None, to_year: int = None,
any_tag: bool = False) -> dict:
"""
Return FileGroup counts of what will be returned if the search is actually performed.
Returns both the fast (a/b/c columns) and deep (all columns) counts so the UI can show the count
matching the current search mode, and offer deep search when it has more results. Both are cheap
index-bound COUNTs (no ranking).
"""
from wrolpi import fts
wheres = []
joins = list()
params = dict()
# The deep match is a superset of the fast (column-filtered) match; both are counted with the
# same statement, differing only in the MATCH expression.
deep_match = fts.translate_websearch(search_str) if search_str else None
fast_match = fts.translate_websearch(search_str, fts.ABC_COLUMNS) if search_str else None
if search_str and deep_match is None:
# Nothing usable to search; match nothing (like an empty websearch_to_tsquery).
return dict(file_groups=0, file_groups_deep=0)
if deep_match:
joins.append('JOIN file_group_fts fts ON fts.rowid = fg.id')
wheres.append('fts.file_group_fts MATCH :fts_match')
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
wheres, params = months_selector_to_where(wheres, params, months)
wheres, params = date_range_to_where(wheres, params, from_year, to_year)
joins = "\n".join(joins)
wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
stmt = f'''
SELECT COUNT(*) AS estimate
FROM file_group fg
{joins}
{wheres}
'''
logger.debug(f'search_file_suggestion_count: {stmt} {params}')
with session_curs(session) as curs:
if deep_match:
curs.execute(stmt, {**params, 'fts_match': deep_match})
deep_estimate = int(curs.fetchone()['estimate'])
curs.execute(stmt, {**params, 'fts_match': fast_match})
estimate = int(curs.fetchone()['estimate'])
else:
curs.execute(stmt, params)
estimate = deep_estimate = int(curs.fetchone()['estimate'])
return dict(file_groups=estimate, file_groups_deep=deep_estimate)
MOVE_CHUNK_SIZE = 100
def _move_file_group_files(file_group: FileGroup, new_primary_path: pathlib.Path) -> None:
"""Move all physical files of a FileGroup to a new directory.
For directory moves where filenames are preserved, this moves each file from
the old directory to the new directory keeping the same filename.
Args:
file_group: The FileGroup whose files should be moved
new_primary_path: The new path for the primary file
"""
new_directory = new_primary_path.parent
new_stem, _ = split_path_stem_and_suffix(new_primary_path, full=True)
# Move each file in the FileGroup
for file_info in file_group.files:
relative_name = file_info['path']
old_path = file_group.directory / relative_name
# Calculate new path: new_stem + original suffix
_, suffix = split_path_stem_and_suffix(relative_name)
new_path = pathlib.Path(f'{new_stem}{suffix}')
if old_path.is_file():
shutil.move(old_path, new_path)
logger.debug(f'Moved file: {old_path} -> {new_path}')
def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path, pathlib.Path]):
"""Batch UPDATE FileGroups using raw SQL after physical files have been moved.
Only updates directory and primary_path. Other fields (files, title, a_text) remain unchanged
because filenames are preserved in directory moves - only the parent directory changes.
Args:
session: Database session
chunk_plan: Mapping of old_primary_path -> new_primary_path
"""
if not chunk_plan:
return
# Build the update rows: [old_path, new_dir, new_path]
updates = json.dumps([
[str(old_path), str(new_path.parent), str(new_path)]
for old_path, new_path in chunk_plan.items()
])
# Use UPDATE FROM json_each pattern for efficient batch update. This runs on the caller's
# session (rather than a second connection) so it shares the move's transaction: a second
# connection would deadlock against the write lock the caller already holds, and its separate
# commit would survive a rollback of the rest of the move.
sql = '''
UPDATE file_group AS fg SET
directory = json_extract(v.value, '$[1]'),
primary_path = json_extract(v.value, '$[2]'),
indexed = CASE WHEN fg.data IS NULL THEN FALSE ELSE fg.indexed END
FROM json_each(:updates) AS v
WHERE fg.primary_path = json_extract(v.value, '$[0]')
'''
session.execute(sa_text(sql), dict(updates=updates))
from wrolpi.collections.config import file_groups_moved
new_paths = [str(new_path) for new_path in chunk_plan.values()]
file_groups_moved(session, session.query(FileGroup.id).filter(FileGroup.primary_path.in_(new_paths)))
def _json_serial(obj):
"""JSON serializer for objects not serializable by default json code."""
import datetime
if isinstance(obj, (datetime.datetime, datetime.date)):
return obj.isoformat()
raise TypeError(f"Type {type(obj)} not serializable")
def _bulk_update_file_groups_reorganize(updates: List[dict]):
"""Batch UPDATE FileGroups for reorganization using raw SQL.
Unlike _bulk_update_file_groups_db, this also updates the files and data JSON fields
since reorganization changes filenames, not just directories.
Args:
updates: List of dicts with keys: id, directory, primary_path, files, data
"""
if not updates:
return
with get_db_curs(commit=True) as curs:
for update in updates:
# Don't change indexed - reorganization only moves files, content unchanged
primary_path = update['primary_path']
stem, _ = split_path_stem_and_suffix(primary_path)
curs.execute('''
UPDATE file_group
SET directory = ?,
primary_path = ?,
files = ?,
data = ?,
stem = ?
WHERE id = ?
''', (
update['directory'],
primary_path,
json.dumps(update['files'], default=_json_serial),
json.dumps(update['data'], default=_json_serial) if update.get('data') else None,
stem,
update['id'],
))
# Expire SQLAlchemy's session cache so subsequent queries see the raw SQL updates.
# This is necessary because get_db_curs uses the same connection as get_db_session
# during tests, but SQLAlchemy's identity map doesn't know about raw SQL updates.
with get_db_session() as session:
session.expire_all()
from wrolpi.collections.config import file_groups_moved
file_groups_moved(session, [update['id'] for update in updates])
def delete_directory(directory: pathlib.Path, recursive: bool = False, force: bool = False):
"""Remove a directory, remove it's Directory record.
Will refuse to delete a directory if it contains Tagged Files, unless force is True.
This function is idempotent - safe to call even if the directory was already deleted."""
if directory.exists():
if recursive:
if not force:
with get_db_session() as session:
tagged = get_tagged_file_groups_by_paths(session, [directory])
if tagged:
raise FileGroupIsTagged(f'Cannot delete {tagged[0]} because it is tagged')
shutil.rmtree(directory)
else:
directory.rmdir()
# Always clean up DB record even if directory was already deleted
with get_db_session(commit=True) as session:
session.query(Directory).filter_by(path=str(directory)).delete(synchronize_session=False)
async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
"""Rename a file (and it's associated files). Preserve any tags.
If the path is a non-primary file in a FileGroup (e.g., a poster or subtitle),
the entire FileGroup will be renamed.
"""
new_path = path.with_name(new_name)
if not path.exists():
raise FileNotFoundError(f'Cannot find {path} to rename')
if new_path.exists():
raise FileConflict(f'Cannot rename {path} because {new_path} already exists')
with get_db_session(commit=True) as session:
# Find the FileGroup - it may be the primary path or a secondary file
fg: FileGroup = FileGroup.get_by_any_file_path(session, path)
if not fg:
# File wasn't yet in the DB.
fg = FileGroup.from_paths(session, path)
# If this is not the primary file, calculate the new primary path
if fg.primary_path != path:
# Extract the new stem and apply it to the primary file's suffix
new_stem, _ = split_path_stem_and_suffix(new_path, full=True)
_, primary_suffix = split_path_stem_and_suffix(fg.primary_path, full=False)
new_primary_path = pathlib.Path(f'{new_stem}{primary_suffix}')
fg.move(new_primary_path)
else:
fg.move(new_path)
from wrolpi.collections.config import file_groups_moved
file_groups_moved(session, [fg.id])
return new_path
async def rename_directory(session: Session, directory: pathlib.Path, new_name: str, send_events: bool = False) \
-> pathlib.Path:
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory."""
from wrolpi.files.worker import file_worker
new_directory = directory.with_name(new_name)
if new_directory.exists():
raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.')
# Check if this directory is a Collection's directory
from wrolpi.collections.models import Collection
collection: Collection = session.query(Collection).filter(Collection.directory == directory).one_or_none()
if collection:
# Use Collection's move method which handles files, config saves, and cleanup
new_directory.mkdir(parents=True, exist_ok=True)
await collection.move_collection(new_directory, session, send_events=send_events)
else:
# Regular directory rename - move all paths into the new directory.
paths = list(directory.iterdir())
try:
job_id = file_worker.queue_move(new_directory, paths)
await file_worker.wait_for_job(job_id)
if send_events:
Events.send_file_move_completed(f'Directory has been renamed: {directory}')
except Exception:
if send_events:
Events.send_file_move_failed(f'Directory rename has failed: {directory}')
raise
# Update nested Collections (files already moved by file_worker above)
directory_str = str(directory)
nested_collections = session.query(Collection).filter(
Collection.directory.like(f'{directory_str}/%')
).all()
for coll in nested_collections:
relative = coll.directory.relative_to(directory)
new_coll_dir = new_directory / relative
await coll.move_collection(new_coll_dir, session, send_events=send_events, with_files=False)
# Remove the old directory (recursive=True to clean up any empty subdirectories
# that remained after files were moved out).
delete_directory(directory, recursive=True)
return new_directory
async def rename(session: Session, path: pathlib.Path, new_name: str, send_events: bool = False) -> pathlib.Path:
"""Rename a directory or file. Preserve any tags."""
if path.is_dir():
return await rename_directory(session, path, new_name, send_events=send_events)
return await rename_file(path, new_name)
def get_special_directories() -> List[pathlib.Path]:
"""Return absolute directories that must never be ignored by file refresh.
These are module destinations that depend on FileGroup indexing (or are the media root).
Map is intentionally excluded: it lists PMTiles by walking the filesystem and benefits
from being ignored during refresh (very large files).
Must not create directories — this runs during refresh ignore filtering and would
otherwise mkdir videos/ (and NO CHANNEL/) as a side effect of the videos helpers.
"""
from modules.archive.lib import get_archive_directory
from modules.videos.lib import format_videos_destination
media_directory = get_media_directory()
videos_directory = media_directory / format_videos_destination()
# Avoid importing modules.zim.lib here (circular import via files.worker).
zims_directory = media_directory / get_wrolpi_config().zims_destination
return [
media_directory.resolve(),
videos_directory.resolve(),
get_archive_directory().resolve(),
(videos_directory / 'NO CHANNEL').resolve(),
zims_directory.resolve(),
]
def get_normalized_ignored_directories() -> List[str]:
"""Return absolute ignored directory paths, excluding special directories.
Special directories (media root, videos, archives, zims, …) are never treated as ignored,
even if they appear in config from older installs or manual edits.
"""
media_directory = get_media_directory()
special = {str(p) for p in get_special_directories()}
result = []
for d in get_wrolpi_config().ignored_directories or []:
path = pathlib.Path(d)
absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve())
if absolute in special:
continue
result.append(absolute)
return result
def sanitize_ignored_directories():
"""Persistently remove special directories from the ignored list if present."""
wrolpi_config = get_wrolpi_config()
ignored = list(wrolpi_config.ignored_directories or [])
if not ignored:
return
media_directory = get_media_directory()
special = {str(p) for p in get_special_directories()}
cleaned = []
removed = False
for d in ignored:
path = pathlib.Path(d)
absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve())
if absolute in special:
removed = True
logger.warning(f'Removing special directory from ignored_directories: {d}')
continue
cleaned.append(d)
if removed:
wrolpi_config.ignored_directories = cleaned
def add_ignore_directory(directory: Union[pathlib.Path, str]):
"""Add a directory to the `ignored_directories` in the WROLPi config. This directory will be ignored when
refreshing."""
media_directory = get_media_directory().resolve()
directory = pathlib.Path(directory)
if not directory.is_absolute():
directory = media_directory / directory
directory = directory.resolve()
# Drop any special directories that were previously saved as ignored.
sanitize_ignored_directories()
special_directories = set(get_special_directories())
if directory in special_directories:
raise InvalidDirectory('Refusing to ignore special directory')
if not str(directory).startswith(str(media_directory)):
raise InvalidDirectory('Cannot ignore directory not in media directory')
wrolpi_config = get_wrolpi_config()
ignored_directories = wrolpi_config.ignored_directories if wrolpi_config.ignored_directories else list()
# Accept either absolute or relative forms already present in config.
relative = str(get_relative_to_media_directory(directory))
if str(directory) in ignored_directories or relative in ignored_directories:
logger.warning(f'Directory is already ignored: {directory}')
return
ignored_directories.append(str(directory))
wrolpi_config.ignored_directories = ignored_directories
def remove_ignored_directory(directory: Union[pathlib.Path, str]):
"""Remove a directory from the `ignored_directories` in the WROLPi config."""
media_directory = get_media_directory()
path = pathlib.Path(directory)
if not path.is_absolute():
path = media_directory / path
path = path.resolve()
absolute = str(path)
ignored_directories = list(get_wrolpi_config().ignored_directories or [])
kept = []
removed = False
for d in ignored_directories:
entry = pathlib.Path(d)
entry_abs = str((media_directory / entry).resolve() if not entry.is_absolute() else entry.resolve())
if entry_abs == absolute:
removed = True
continue
kept.append(d)
if not removed:
raise UnknownDirectory('Directory is not ignored')
get_wrolpi_config().ignored_directories = kept
async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> FileGroup:
"""Insert/update all files in the provided file's FileGroup."""
if not file or not file.is_file():
raise InvalidFile(f'Cannot upsert file that does not exist: {file}')
# Update/Insert all files in the FileGroup.
paths = glob_shared_stem(pathlib.Path(file))
# Remove any ignored files.
paths = remove_files_in_ignored_directories(paths)
if not paths:
logger.warning('upsert_file called, but all files are in ignored directories!')
raise IgnoredDirectoryError(f'all files are in ignored directories: {file}')
# Use a single session context for all database operations
with get_db_session(commit=True) as session:
# Create/update FileGroup with retry logic
file_group = None
for i in range(2):
try:
file_group = FileGroup.from_paths(session, *paths)
session.flush([file_group])
break
except IntegrityError:
# Another process inserted this FileGroup.
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
session.rollback()
if i == 1:
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
continue
if not file_group:
raise RuntimeError(f'upsert_file failed to create FileGroup! {file}')
logger.debug(f'upsert_file: {file_group}')
# Model the FileGroup (creates Archive/Video/etc. and sets indexed=True)
try:
file_group.do_model(session)
except Exception as e:
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
if PYTEST:
raise
# If user uploads a file, then remove it from the download skip list so comments can be downloaded.
# Modify the download (if any) so that the user can click on it and view the uploaded file.
if file_group.url:
if download_manager.is_skipped(file_group.url):
download_manager.remove_from_skip_list(file_group.url)
if download := Download.get_by_url(session, file_group.url):
# Mark download as completed
download.complete()
# Link the download to the location of the uploaded file.
model = file_group.get_model_record(session)
download.location = model.location if model else file_group.location
# Add tags if provided
if tag_names:
for tag_name in tag_names:
if tag_name not in file_group.tag_names:
tag = Tag.get_by_name(session, tag_name)
file_group.add_tag(session, tag.id)
# Commit all changes in one transaction
session.commit()
# Upsert directories after FileGroup is committed
upsert_directories([], file.parents)
return file_group
def get_file_location_href(file: pathlib.Path) -> str:
"""Return the location a file can be viewed at on the UI."""
parent = str(get_relative_to_media_directory(file.parent))
preview = str(get_relative_to_media_directory(file))
if parent == '.':
# File is in the top of the media directory, App already shows top directory open.
query = urllib.parse.urlencode(dict(preview=str(preview)))
else:
query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
return f'/files?{query}'
def get_real_path_name(path: pathlib.Path) -> pathlib.Path:
"""Return the real path name of a file. This is used to get the real path of a file on macOS."""
if IS_MACOS:
for path_ in path.parent.iterdir():
if path_.name.lower() == path.name.lower():
return path_
return path
@dataclasses.dataclass
class BulkTagPreview:
"""Preview information for a bulk tagging operation."""
file_count: int = 0
shared_tag_names: List[str] = dataclasses.field(default_factory=list)
def __json__(self) -> dict:
return dict(
file_count=self.file_count,
shared_tag_names=self.shared_tag_names,
)
def get_bulk_tag_preview(session: Session, paths: List[str]) -> BulkTagPreview:
"""Get preview information for bulk tagging operation.
Returns the count of files that will be tagged and the tags that are shared by ALL files.
"""
media_directory = get_media_directory()
absolute_paths = [media_directory / p for p in paths]
# Collect all file paths (expanding directories recursively)
all_file_paths: List[pathlib.Path] = []
for path in absolute_paths:
if path.is_file():
all_file_paths.append(path)
elif path.is_dir():
for file in walk(path):
if file.is_file():
all_file_paths.append(file)
# Remove files in ignored directories
all_file_paths = remove_files_in_ignored_directories(all_file_paths)
if not all_file_paths:
return BulkTagPreview(file_count=0, shared_tag_names=[])
# Get unique files by stem (each FileGroup counted once)
unique_files = get_unique_files_by_stem(all_file_paths)
file_count = len(unique_files)
if file_count == 0:
return BulkTagPreview(file_count=0, shared_tag_names=[])
# Find tags shared by ALL files
# First, get all FileGroups and their tags
# We query by stem prefix because unique_files may contain non-primary files
# (e.g., .readability.json) but FileGroups are stored with primary_path (e.g., .html)
from sqlalchemy import or_
stems = [split_path_stem_and_suffix(f, full=True)[0] for f in unique_files]
# Chunk the OR'd LIKEs to stay within SQLite's expression-tree depth limit of 1000.
file_groups = []
for idx in range(0, len(stems), 500):
chunk = stems[idx:idx + 500]
file_groups.extend(session.query(FileGroup).filter(
or_(*[FileGroup.primary_path.like(f'{stem}%') for stem in chunk])
).all())
if not file_groups:
# No FileGroups exist yet, no shared tags
return BulkTagPreview(file_count=file_count, shared_tag_names=[])
# Get tag names for each FileGroup
tag_sets = []
for fg in file_groups:
tag_sets.append(set(fg.tag_names))
# Find intersection of all tag sets (tags shared by ALL files)
if tag_sets:
shared_tags = tag_sets[0]
for tag_set in tag_sets[1:]:
shared_tags = shared_tags.intersection(tag_set)
shared_tag_names = sorted(list(shared_tags))
else:
shared_tag_names = []
return BulkTagPreview(file_count=file_count, shared_tag_names=shared_tag_names)
@dataclasses.dataclass
class BulkTagProgress:
"""Progress information for bulk tagging operation."""
status: str = 'idle' # 'idle', 'running'
total: int = 0
completed: int = 0
add_tag_names: List[str] = dataclasses.field(default_factory=list)
remove_tag_names: List[str] = dataclasses.field(default_factory=list)
error: str = None
queued_jobs: int = 0
def __json__(self) -> dict:
return dict(
status=self.status,
total=self.total,
completed=self.completed,
add_tag_names=self.add_tag_names,
remove_tag_names=self.remove_tag_names,
error=self.error,
queued_jobs=self.queued_jobs,
)
def get_bulk_tag_progress() -> BulkTagProgress:
"""Get the current progress of bulk tagging operations."""
from wrolpi.api_utils import api_app
bulk_tag = api_app.shared_ctx.bulk_tag
return BulkTagProgress(
status=bulk_tag.get('status', 'idle'),
total=bulk_tag.get('total', 0),
completed=bulk_tag.get('completed', 0),
add_tag_names=list(bulk_tag.get('add_tag_names', [])),
remove_tag_names=list(bulk_tag.get('remove_tag_names', [])),
error=bulk_tag.get('error'),
queued_jobs=bulk_tag.get('queued_jobs', 0),
)
def queue_bulk_tag_job(paths: List[str], add_tag_names: List[str], remove_tag_names: List[str]):
"""Add a bulk tagging job to the queue."""
from wrolpi.api_utils import api_app
job = dict(
paths=paths,
add_tag_names=add_tag_names,
remove_tag_names=remove_tag_names,
)
api_app.shared_ctx.bulk_tag_queue.put(job)
# Reset progress state and update queued jobs count
try:
current_queued = api_app.shared_ctx.bulk_tag.get('queued_jobs', 0)
api_app.shared_ctx.bulk_tag.update(dict(
status='idle',
total=0,
completed=0,
error=None,
queued_jobs=current_queued + 1,
))
except Exception:
pass
BULK_TAG_CHUNK_SIZE = 100 # Files per DB session
BULK_SQL_CHUNK_SIZE = 1000 # Rows per SQL statement
def _bulk_insert_tag_files(tag_files: List[Tuple[int, int]]):
"""Bulk insert TagFiles, ignoring duplicates.
Args:
tag_files: List of (tag_id, file_group_id) tuples
"""
if not tag_files:
return
now_timestamp = now()
# Process in chunks to avoid overly large SQL statements
for i in range(0, len(tag_files), BULK_SQL_CHUNK_SIZE):
chunk = tag_files[i:i + BULK_SQL_CHUNK_SIZE]
values = [(tag_id, file_group_id, now_timestamp)
for tag_id, file_group_id in chunk]
with get_db_curs(commit=True) as curs:
placeholders, params = values_clause(values)
stmt = f'''
INSERT INTO tag_file (tag_id, file_group_id, created_at)
VALUES {placeholders}
ON CONFLICT (tag_id, file_group_id) DO NOTHING
'''
curs.execute(stmt, params)
logger.debug(f'Bulk inserted {len(chunk)} TagFiles')
def _bulk_delete_tag_files(tag_files: List[Tuple[int, int]]):
"""Bulk delete TagFiles.
Args:
tag_files: List of (tag_id, file_group_id) tuples
"""
if not tag_files:
return
# Process in chunks
for i in range(0, len(tag_files), BULK_SQL_CHUNK_SIZE):
chunk = tag_files[i:i + BULK_SQL_CHUNK_SIZE]
pairs = [(tag_id, file_group_id) for tag_id, file_group_id in chunk]
with get_db_curs(commit=True) as curs:
placeholders, params = values_clause(pairs)
stmt = f'''
DELETE FROM tag_file
WHERE (tag_id, file_group_id) IN (VALUES {placeholders})
'''
curs.execute(stmt, params)
logger.debug(f'Bulk deleted {curs.rowcount} TagFiles')
async def _process_bulk_tag_job(job: dict):
"""Process a single bulk tagging job with batched DB operations.
This function uses a three-phase approach for performance:
1. Collection: Gather all files and resolve FileGroups, collecting tag operations
2. Bulk DB operations: Execute bulk INSERT and DELETE for TagFiles
3. Switch activation: Call save_tags_config and sync_tags_directory once at the end
"""
from wrolpi.api_utils import api_app
paths = job['paths']
add_tag_names = job['add_tag_names']
remove_tag_names = job['remove_tag_names']
media_directory = get_media_directory()
absolute_paths = [media_directory / p for p in paths]
# Collect all file paths (expanding directories recursively)
all_file_paths: List[pathlib.Path] = []
for path in absolute_paths:
if path.is_file():
all_file_paths.append(path)
elif path.is_dir():
for file in walk(path):
if file.is_file():
all_file_paths.append(file)
# Remove files in ignored directories
all_file_paths = remove_files_in_ignored_directories(all_file_paths)
if not all_file_paths:
return
# Get unique files by stem
unique_files = get_unique_files_by_stem(all_file_paths)
# Update progress with total count
api_app.shared_ctx.bulk_tag.update(dict(
status='running',
total=len(unique_files),
completed=0,
add_tag_names=add_tag_names,
remove_tag_names=remove_tag_names,
error=None,
))
# Pre-resolve tag IDs (using cached lookups)
add_tag_ids = set()
remove_tag_ids = set()
with get_db_session() as session:
for tag_name in add_tag_names:
try:
add_tag_ids.add(Tag.get_id_by_name(session, tag_name))
except UnknownTag:
logger.warning(f'Unknown tag to add: {tag_name}')
for tag_name in remove_tag_names:
try:
remove_tag_ids.add(Tag.get_id_by_name(session, tag_name))
except UnknownTag:
logger.warning(f'Unknown tag to remove: {tag_name}')
if not add_tag_ids and not remove_tag_ids:
return
# Phase 1: Collect all file_group_ids and required operations
tag_files_to_insert: List[Tuple[int, int]] = [] # (tag_id, file_group_id)
tag_files_to_delete: List[Tuple[int, int]] = [] # (tag_id, file_group_id)
completed = 0
# Process files in chunks to balance memory and DB pressure
for chunk_start in range(0, len(unique_files), BULK_TAG_CHUNK_SIZE):
chunk = unique_files[chunk_start:chunk_start + BULK_TAG_CHUNK_SIZE]
with get_db_session(commit=True) as session:
for file_path in chunk:
try:
# Get or create FileGroup
file_group = FileGroup.get_by_path(session, file_path)
if not file_group:
# Create FileGroup for unindexed file
paths_for_group = glob_shared_stem(file_path)
paths_for_group = remove_files_in_ignored_directories(paths_for_group)
if paths_for_group:
try:
file_group = FileGroup.from_paths(session, *paths_for_group)
file_group.do_model(session)
session.flush([file_group])
except (NoPrimaryFile, IntegrityError) as e:
logger.warning(f'Could not create FileGroup for {file_path}: {e}')
completed += 1
api_app.shared_ctx.bulk_tag['completed'] = completed
continue
if file_group and file_group.id:
# Flush to ensure file_group.id is set
session.flush([file_group])
# Get current tag_ids for this file_group
existing_tag_ids = {tf.tag_id for tf in file_group.tag_files}
# Determine tags to add (not already present)
for tag_id in add_tag_ids:
if tag_id not in existing_tag_ids:
tag_files_to_insert.append((tag_id, file_group.id))
# Determine tags to remove (currently present)
for tag_id in remove_tag_ids:
if tag_id in existing_tag_ids:
tag_files_to_delete.append((tag_id, file_group.id))
except Exception as e:
logger.error(f'Error processing file {file_path}: {e}', exc_info=e)
completed += 1
api_app.shared_ctx.bulk_tag['completed'] = completed
# Allow cancellation between chunks
await asyncio.sleep(0)
# Phase 2: Bulk database operations
_bulk_insert_tag_files(tag_files_to_insert)
_bulk_delete_tag_files(tag_files_to_delete)
# Phase 3: Activate switches ONCE at the end
if tag_files_to_insert or tag_files_to_delete:
save_tags_config.activate_switch()
sync_tags_directory.activate_switch()
DISABLE_BULK_TAG_WORKER = bool(PYTEST)
async def bulk_tag_worker():
"""Background worker that processes bulk tagging jobs from the queue."""
import queue
from wrolpi.api_utils import api_app
if PYTEST and DISABLE_BULK_TAG_WORKER:
return
bulk_tag_ctx = api_app.shared_ctx.bulk_tag
bulk_tag_queue = api_app.shared_ctx.bulk_tag_queue
try:
# Try to get a job from the queue (non-blocking)
job = bulk_tag_queue.get_nowait()
except queue.Empty:
# No jobs in queue, stay idle
bulk_tag_ctx['status'] = 'idle'
return
# Decrement queued jobs count
try:
current_queued = bulk_tag_ctx.get('queued_jobs', 1)
bulk_tag_ctx['queued_jobs'] = max(0, current_queued - 1)
except Exception:
pass
try:
await _process_bulk_tag_job(job)
except Exception as e:
logger.error(f'Bulk tag worker error: {e}', exc_info=e)
bulk_tag_ctx['error'] = str(e)
finally:
bulk_tag_ctx['status'] = 'idle'
# Register the worker as a perpetual signal
from wrolpi.api_utils import perpetual_signal
bulk_tag_worker = perpetual_signal(sleep=1)(bulk_tag_worker)
def _detect_archive_type(path: pathlib.Path) -> str:
"""Detect archive type from file suffix. Returns 'zip', 'tar', 'rar', '7z', or raises UnsupportedArchive."""
name = path.name.lower()
if name.endswith(('.zip', '.cbz')):
return 'zip'
if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz', '.cbt')):
return 'tar'
if name.endswith(('.rar', '.cbr')):
return 'rar'
if name.endswith(('.7z', '.cb7')):
return '7z'
raise UnsupportedArchive(f'Unsupported archive format: {path.name}')
def _build_tree(entries: list) -> list:
"""Build a hierarchical tree from a flat list of archive entries.
Each entry is a dict with 'path', 'name', 'is_dir', 'size'.
Returns a nested list with 'children' on directory entries.
"""
root_children = []
# Map from directory path to its children list.
dir_map = {}
# Sort entries so parents come before children.
entries.sort(key=lambda e: e['path'])
for entry in entries:
parts = entry['path'].strip('/').split('/')
if len(parts) == 1:
# Top-level entry.
if entry['is_dir']:
entry['children'] = []
dir_map[entry['path'].rstrip('/') + '/'] = entry['children']
root_children.append(entry)
else:
# Find parent directory.
parent_path = '/'.join(parts[:-1]) + '/'
if parent_path not in dir_map:
# Create implicit parent directories.
current = ''
for part in parts[:-1]:
current += part + '/'
if current not in dir_map:
parent_entry = {
'path': current,
'name': part,
'is_dir': True,
'size': 0,
'children': [],
}
dir_map[current] = parent_entry['children']
# Attach to grandparent or root.
grandparent = '/'.join(current.rstrip('/').split('/')[:-1])
grandparent_key = grandparent + '/' if grandparent else None
if grandparent_key and grandparent_key in dir_map:
dir_map[grandparent_key].append(parent_entry)
else:
root_children.append(parent_entry)
if entry['is_dir']:
entry['children'] = []
dir_map[entry['path'].rstrip('/') + '/'] = entry['children']
parent_list = dir_map.get(parent_path)
if parent_list is not None:
parent_list.append(entry)
else:
root_children.append(entry)
return root_children
def list_archive_contents(path: pathlib.Path) -> dict:
"""List the contents of an archive file as a hierarchical tree.
Returns: {"entries": [...], "total_size": int, "total_files": int}
"""
media_directory = get_media_directory()
if not str(path).startswith(str(media_directory)):
raise InvalidFile('Path must be within the media directory')
if not path.is_file():
raise InvalidFile(f'Archive file does not exist: {path}')
archive_type = _detect_archive_type(path)
flat_entries = []
total_size = 0
total_files = 0
try:
if archive_type == 'zip':
with zipfile.ZipFile(path, 'r') as zf:
for info in zf.infolist():
is_dir = info.is_dir()
entry = {
'path': info.filename,
'name': info.filename.rstrip('/').split('/')[-1],
'is_dir': is_dir,
'size': info.file_size,
}
flat_entries.append(entry)
if not is_dir:
total_size += info.file_size
total_files += 1
elif archive_type == 'tar':
with tarfile.open(path, 'r:*') as tf:
for member in tf.getmembers():
is_dir = member.isdir()
entry = {
'path': member.name + ('/' if is_dir else ''),
'name': member.name.rstrip('/').split('/')[-1],
'is_dir': is_dir,
'size': member.size,
}
flat_entries.append(entry)
if not is_dir:
total_size += member.size
total_files += 1
elif archive_type == 'rar':
import rarfile
with rarfile.RarFile(path, 'r') as rf:
for info in rf.infolist():
is_dir = info.is_dir()
entry = {
'path': info.filename,
'name': info.filename.rstrip('/').split('/')[-1],
'is_dir': is_dir,
'size': info.file_size,
}
flat_entries.append(entry)
if not is_dir:
total_size += info.file_size
total_files += 1
elif archive_type == '7z':
import py7zr
with py7zr.SevenZipFile(path, 'r') as sz:
for entry_info in sz.list():
is_dir = entry_info.is_directory
entry = {
'path': entry_info.filename + ('/' if is_dir else ''),
'name': entry_info.filename.rstrip('/').split('/')[-1],
'is_dir': is_dir,
'size': entry_info.uncompressed if not is_dir else 0,
}
flat_entries.append(entry)
if not is_dir:
total_size += entry_info.uncompressed
total_files += 1
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
raise UnsupportedArchive(f'Cannot read archive: {e}')
tree = _build_tree(flat_entries)
return {'entries': tree, 'total_size': total_size, 'total_files': total_files}
def stream_archive_member(archive_path: pathlib.Path, member_path: str) -> Tuple[bytes, str, str]:
"""Read a single member from an archive and return its bytes, mimetype, and filename.
Security: validates member_path has no '..' components.
"""
media_directory = get_media_directory()
if not str(archive_path).startswith(str(media_directory)):
raise InvalidFile('Path must be within the media directory')
if not archive_path.is_file():
raise InvalidFile(f'Archive file does not exist: {archive_path}')
# Security: reject path traversal.
if '..' in member_path.split('/'):
raise InvalidArchiveMember('Invalid member path: contains ".."')
archive_type = _detect_archive_type(archive_path)
filename = member_path.rstrip('/').split('/')[-1]
guessed_type, _ = mimetypes.guess_type(filename)
content_type = guessed_type or 'application/octet-stream'
try:
if archive_type == 'zip':
with zipfile.ZipFile(archive_path, 'r') as zf:
try:
info = zf.getinfo(member_path)
except KeyError:
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
if info.is_dir():
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
file_bytes = zf.read(member_path)
elif archive_type == 'tar':
with tarfile.open(archive_path, 'r:*') as tf:
try:
member = tf.getmember(member_path)
except KeyError:
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
if member.isdir():
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
f = tf.extractfile(member)
if f is None:
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
file_bytes = f.read()
elif archive_type == 'rar':
import rarfile
with rarfile.RarFile(archive_path, 'r') as rf:
try:
info = rf.getinfo(member_path)
except KeyError:
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
if info.is_dir():
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
file_bytes = rf.read(member_path)
elif archive_type == '7z':
import py7zr
import tempfile
with py7zr.SevenZipFile(archive_path, 'r') as sz:
if member_path not in sz.getnames():
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
with tempfile.TemporaryDirectory() as tmpdir:
sz.extract(tmpdir, targets=[member_path])
extracted_path = pathlib.Path(tmpdir) / member_path
if not extracted_path.is_file():
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
file_bytes = extracted_path.read_bytes()
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
raise UnsupportedArchive(f'Cannot read archive: {e}')
return file_bytes, content_type, filename