mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
* A directory rename is finished by the file worker, not the request
A directory rename queued a move, waited for it, then removed the old
directory and updated nested Collections itself. A rename that outlasted
Sanic's RESPONSE_TIMEOUT (60 s) was cancelled while waiting: the move still
finished, but the old directory and its emptied subdirectories were left
behind, nested Collections kept their old directories, and the browser was
told the rename failed.
The move job now finishes the rename (nested Collections, then removing the
old directory, empty directories only), and the request returns once the
move is queued. A failure is reported by the job's event. An empty
directory can be renamed (it raised before). The worker status carries
`rename_from`, so the Files page progress bar labels the rename, and the
file browser fetches again when the file worker is done.
A directory rename no longer queues a refresh of its parent: the file
worker runs the newest task first, so it scanned the directory before the
rename.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
* A queued rename returns its job, and the file browser waits for it
Review fixes:
- The cancellation test patched `wait_for_job`, which this path no longer
calls, so it cancelled nothing. It now asserts the invariant: the
request only queues the rename (it never waits, and nothing has moved
when it returns), and the file worker alone finishes it.
- The file browser inferred the end of a rename from the file worker's
polled status. A short job can start and finish between two polls, and
the first idle poll (before the job started) spent the "fetch once"
token, so the table could keep the old name. POST /api/files/rename now
returns `{"job_id": ...}` (null when the rename is already done), GET
/api/files/jobs/<job_id> reports pending/complete/failed, and the file
browser polls that job before fetching.
- useBrowseFiles ignores a fetch that settles after a newer one, so a
listing requested before a rename cannot replace the one after it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2970 lines
126 KiB
Python
2970 lines
126 KiB
Python
import asyncio
|
|
import contextlib
|
|
import dataclasses
|
|
import datetime
|
|
import functools
|
|
import glob
|
|
import hashlib
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import zipfile
|
|
from itertools import zip_longest
|
|
from pathlib import Path
|
|
from typing import Callable, ContextManager, List, Optional, Tuple, Union, Dict, Generator, Iterable, Set
|
|
|
|
import cachetools.func
|
|
from sqlalchemy import and_, asc, or_, text as sa_text
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.orm import Session, undefer, joinedload
|
|
|
|
from wrolpi.cmd import which
|
|
from wrolpi.common import get_media_directory, wrol_mode_check, logger, \
|
|
chunks, chunks_by_stem, walk, \
|
|
get_wrolpi_config, \
|
|
unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory, strip_surrogates
|
|
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
|
|
from wrolpi.config_version import count_config_change_curs
|
|
from wrolpi.db import get_db_session, get_db_curs, values_clause, session_curs
|
|
from wrolpi.downloader import download_manager, Download
|
|
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
|
|
NoPrimaryFile, InvalidDirectory, IgnoredDirectoryError, UnsupportedArchive, InvalidArchiveMember
|
|
from wrolpi.events import Events
|
|
from wrolpi.files.models import FileGroup, Directory
|
|
from wrolpi.files.progress import normalize_progress
|
|
from wrolpi.lang import ISO_639_CODES, ISO_3166_CODES
|
|
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config, sync_tags_directory
|
|
from wrolpi.vars import PYTEST, IS_MACOS
|
|
|
|
try:
|
|
import magic
|
|
|
|
mime = magic.Magic(mime=True)
|
|
except ImportError:
|
|
# Magic is not installed
|
|
magic = None
|
|
mime = None
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'search_files',
|
|
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'choose_primary_file', 'get_file_statistics',
|
|
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
|
|
'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href',
|
|
'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories',
|
|
'get_normalized_ignored_directories', 'sanitize_ignored_directories',
|
|
'cached_search_total', 'search_filter_cache_key', 'count_file_groups',
|
|
'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems', 'get_kv_value', 'set_kv_value',
|
|
'backfill_null_file_group_stems', 'dedupe_file_groups_by_directory_stem', 'merge_file_group_relations',
|
|
'merge_into_surviving_duplicates']
|
|
|
|
|
|
def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
|
|
"""Returns all Tag names for the provided file path."""
|
|
tags = session.query(Tag) \
|
|
.join(TagFile, Tag.id == TagFile.tag_id) \
|
|
.join(FileGroup, FileGroup.id == TagFile.file_group_id) \
|
|
.filter(FileGroup.primary_path == str(file))
|
|
names = sorted([tag.name for tag in tags], key=lambda i: i.lower())
|
|
return names
|
|
|
|
|
|
def sanitize_filename_surrogates(path: pathlib.Path) -> pathlib.Path:
|
|
"""
|
|
Check if a path contains invalid UTF-8 surrogates. If so, rename the file
|
|
on disk to replace them with underscores.
|
|
|
|
Linux filesystems store filenames as bytes. Python represents invalid UTF-8
|
|
sequences as surrogates. These surrogates cannot be stored in the database.
|
|
|
|
Returns the (possibly new) path.
|
|
"""
|
|
path_str = str(path)
|
|
|
|
# Check for surrogates by trying to encode as UTF-8
|
|
try:
|
|
path_str.encode('utf-8')
|
|
return path # Path is valid
|
|
except UnicodeEncodeError:
|
|
pass
|
|
|
|
# Get raw bytes from filesystem and decode, replacing invalid sequences
|
|
raw_bytes = os.fsencode(path)
|
|
# Replace invalid UTF-8 sequences with underscore
|
|
sanitized_name = raw_bytes.decode('utf-8', errors='replace').replace('\ufffd', '_')
|
|
sanitized_path = pathlib.Path(sanitized_name)
|
|
|
|
if path.exists() and sanitized_path != path:
|
|
if sanitized_path.exists():
|
|
# Target already exists, add a suffix
|
|
stem = sanitized_path.stem
|
|
suffix = sanitized_path.suffix
|
|
counter = 1
|
|
while sanitized_path.exists():
|
|
sanitized_path = sanitized_path.with_name(f'{stem}_{counter}{suffix}')
|
|
counter += 1
|
|
|
|
logger.warning(f'Renaming file with invalid UTF-8 characters: {path!r} -> {sanitized_path}')
|
|
path.rename(sanitized_path)
|
|
return sanitized_path
|
|
|
|
return path
|
|
|
|
|
|
def _get_file_dict(session: Session, file: pathlib.Path) -> Dict:
|
|
media_directory = get_media_directory()
|
|
try:
|
|
size = file.stat().st_size
|
|
except PermissionError:
|
|
size = None
|
|
try:
|
|
mimetype = get_mimetype(file)
|
|
except PermissionError:
|
|
mimetype = None
|
|
file_group = FileGroup.get_by_path(session, file)
|
|
return dict(
|
|
path=file.relative_to(media_directory),
|
|
size=size,
|
|
mimetype=mimetype,
|
|
tags=get_file_tag_names(session, file),
|
|
progress=file_group.progress if file_group else None,
|
|
position=file_group.position if file_group else None,
|
|
# When a finished file (no position) was finished; the preview compares it with its own position.
|
|
viewed=file_group.viewed if file_group else None,
|
|
)
|
|
|
|
|
|
def get_file_dict(session: Session, file: str) -> Dict:
|
|
media_directory = get_media_directory()
|
|
return _get_file_dict(session, media_directory / file)
|
|
|
|
|
|
async def set_file_viewed(file: pathlib.Path):
|
|
"""Change FileGroup.viewed to the current datetime.
|
|
|
|
Files in ignored directories are not tracked. Runs after the request that viewed the file, so it writes in a
|
|
session of its own, not the request's.
|
|
"""
|
|
if is_ignored_file(file):
|
|
return
|
|
|
|
with get_db_session(commit=True) as session:
|
|
try:
|
|
fg = FileGroup.find_by_path(session, file)
|
|
except UnknownFile:
|
|
fg = FileGroup.from_paths(session, file)
|
|
fg.do_model(session)
|
|
fg.set_viewed()
|
|
|
|
|
|
def _trackable_path(file: pathlib.Path) -> pathlib.Path | None:
|
|
"""Return `file` with any `..` collapsed, or None if it is in an ignored directory.
|
|
|
|
`..` is collapsed first so it cannot step into an ignored directory past the check.
|
|
|
|
@raise InvalidFile: The file is not in the media directory.
|
|
"""
|
|
file = pathlib.Path(os.path.normpath(file))
|
|
if not get_paths_in_media_directory([file]):
|
|
raise InvalidFile('File must be in the media directory')
|
|
if is_ignored_file(file):
|
|
return None
|
|
return file
|
|
|
|
|
|
def set_file_progress(session: Session, file: pathlib.Path, progress: float, position: dict = None) \
|
|
-> Tuple[float | None, dict | None]:
|
|
"""Record how far the User is through a file (see wrolpi.files.progress). This is also a view.
|
|
|
|
The caller owns the (write) transaction, and saves the recently viewed config after it commits.
|
|
|
|
@return: The (progress, position) that was stored, (None, None) if the file is not tracked.
|
|
"""
|
|
progress, position = normalize_progress(progress, position)
|
|
|
|
if not (file := _trackable_path(file)):
|
|
return None, None
|
|
|
|
try:
|
|
fg = FileGroup.find_by_path(session, file)
|
|
except UnknownFile:
|
|
if not file.is_file():
|
|
raise InvalidFile(f'File does not exist: {get_relative_to_media_directory(file)}')
|
|
fg = FileGroup.from_paths(session, file)
|
|
fg.do_model(session)
|
|
# Not `set_viewed`, which saves the config; a heartbeat during playback only writes the DB.
|
|
fg.viewed = now()
|
|
fg.progress = progress
|
|
fg.position = position
|
|
return progress, position
|
|
|
|
|
|
def clear_file_progress(session: Session, file: pathlib.Path) -> bool:
|
|
"""Forget how far the User is through a file, so it starts from the beginning. It remains viewed.
|
|
|
|
A file that was never tracked is left alone. The caller owns the (write) transaction.
|
|
|
|
@return: True if a FileGroup was changed.
|
|
"""
|
|
if not (file := _trackable_path(file)):
|
|
return False
|
|
if not (fg := FileGroup.get_by_path(session, file)):
|
|
return False
|
|
fg.progress = None
|
|
fg.position = None
|
|
return True
|
|
|
|
|
|
@cachetools.func.ttl_cache(10_000, 30.0)
|
|
def _get_directory_dict(directory: pathlib.Path,
|
|
directories_cache: str, # Used to cache by requested directories.
|
|
) -> Dict:
|
|
media_directory = get_media_directory()
|
|
try:
|
|
is_empty = not next(directory.iterdir(), False)
|
|
except PermissionError:
|
|
is_empty = False
|
|
return dict(
|
|
path=f'{directory.relative_to(media_directory)}/',
|
|
is_empty=is_empty,
|
|
)
|
|
|
|
|
|
def _get_recursive_directory_dict(session: Session, directory: pathlib.Path, directories: List[pathlib.Path]) -> Dict:
|
|
directories_cache = str(sorted(directories))
|
|
d = _get_directory_dict(directory, directories_cache)
|
|
|
|
if directory in directories:
|
|
children = dict()
|
|
for path in directory.iterdir():
|
|
if not path.is_dir() and path.name in HIDDEN_FILES:
|
|
# Never show ignored files.
|
|
continue
|
|
if path.is_dir() and path in directories:
|
|
children[f'{path.name}/'] = _get_recursive_directory_dict(session, path, directories)
|
|
elif path.is_dir():
|
|
children[f'{path.name}/'] = _get_directory_dict(path, directories_cache)
|
|
else:
|
|
children[path.name] = _get_file_dict(session, path)
|
|
d['children'] = children
|
|
return d
|
|
|
|
|
|
HIDDEN_DIRECTORIES = ('lost+found',)
|
|
# macOS metadata files which should never be shown in the FileBrowser.
|
|
HIDDEN_FILES = ('.DS_Store', '._.DS_Store')
|
|
|
|
|
|
def list_directories_contents(session: Session, directories_: List[str]) -> Dict:
|
|
"""List all files down to (and within) the directories provided.
|
|
|
|
This should only be used by the frontend for it's FileBrowser."""
|
|
media_directory = get_media_directory()
|
|
# Convert all directories to pathlib.Path.
|
|
directories = {media_directory / i for i in directories_}
|
|
if invalid_directories := [i for i in directories if not i.is_dir()]:
|
|
raise FileNotFoundError(f'Invalid directories: {invalid_directories}')
|
|
|
|
# Get all unique parents for each directory, but not if they are above the media directory.
|
|
parents = {j for i in directories for j in i.parents if j not in media_directory.parents}
|
|
# Add unique parents onto the requested directories.
|
|
directories = list(directories | parents)
|
|
|
|
paths = dict()
|
|
for path in media_directory.iterdir():
|
|
if path.is_dir() and path.name in HIDDEN_DIRECTORIES:
|
|
# Never show ignored directories.
|
|
continue
|
|
if not path.is_dir() and path.name in HIDDEN_FILES:
|
|
# Never show ignored files.
|
|
continue
|
|
if path.is_dir():
|
|
paths[f'{path.name}/'] = _get_recursive_directory_dict(session, path, directories)
|
|
else:
|
|
paths[path.name] = _get_file_dict(session, path)
|
|
|
|
return paths
|
|
|
|
|
|
def get_tagged_file_groups_by_ids(session: Session, file_group_ids: List[int]) -> List[dict]:
|
|
"""Return serialized FileGroups (subset of `file_group_ids`) that have at least one TagFile.
|
|
|
|
Used by the unified delete endpoint to require force-confirmation before deleting tagged files."""
|
|
if not file_group_ids:
|
|
return []
|
|
rows = session.query(FileGroup) \
|
|
.options(*FileGroup.json_options()) \
|
|
.join(TagFile, TagFile.file_group_id == FileGroup.id) \
|
|
.filter(FileGroup.id.in_(file_group_ids)) \
|
|
.distinct() \
|
|
.all()
|
|
return [fg.__json__() for fg in rows]
|
|
|
|
|
|
def directory_or_descendant(column, directory: Union[str, pathlib.Path]):
|
|
"""A filter matching `directory` and every directory beneath it.
|
|
|
|
A range is used instead of `LIKE 'directory/%'` so SQLite can use the column's index (it cannot for a
|
|
case-insensitive LIKE), and so `_` and `%` in a path are not wildcards. '0' is the character after '/'."""
|
|
directory = str(directory)
|
|
return or_(column == directory, and_(column >= f'{directory}/', column < f'{directory}0'))
|
|
|
|
|
|
def get_tagged_file_groups_by_paths(session: Session, paths: Iterable[pathlib.Path]) -> List[FileGroup]:
|
|
"""Return FileGroup objects under any of the given paths that have at least one TagFile.
|
|
|
|
For a file path, matches FileGroups whose primary_path shares the file's stem in the same
|
|
directory. For a directory path, matches FileGroups whose `directory` is the path or a
|
|
descendant. Used by the path-based delete endpoint to require force-confirmation before
|
|
deleting tagged files."""
|
|
paths = list(paths)
|
|
if not paths:
|
|
return []
|
|
tagged: List[FileGroup] = []
|
|
seen_ids = set()
|
|
for path in paths:
|
|
query = session.query(FileGroup) \
|
|
.options(*FileGroup.json_options()) \
|
|
.join(TagFile, TagFile.file_group_id == FileGroup.id)
|
|
if path.is_file():
|
|
# Could be deleting a file in a FileGroup that has been tagged.
|
|
stem, _ = split_path_stem_and_suffix(path)
|
|
query = query.filter(FileGroup.primary_path.like(f'{path.parent}/{stem}%'))
|
|
else:
|
|
# Search for any FileGroups that have been tagged under this directory.
|
|
# Use indexed directory column for efficient lookup.
|
|
query = query.filter(directory_or_descendant(FileGroup.directory, path))
|
|
for file_group in query:
|
|
if path.is_dir() or any(p for p in paths if p in file_group.my_paths()):
|
|
if file_group.id not in seen_ids:
|
|
seen_ids.add(file_group.id)
|
|
tagged.append(file_group)
|
|
return tagged
|
|
|
|
|
|
def delete_file_groups(session: Session, file_group_ids: List[int], force: bool = False) -> Set[str]:
|
|
"""Delete FileGroups (and their model rows via FK CASCADE) by ID.
|
|
|
|
Returns the set of `FileGroup.model` values that were deleted, so the caller can fire any
|
|
model-specific post-delete switches (e.g., `save_channels_config` for videos).
|
|
|
|
The caller is expected to have already gated this on tag confirmation via
|
|
`get_tagged_file_groups_by_ids`. `force` propagates to FileGroup.delete so the safety net
|
|
at FileGroup.delete (raises FileGroupIsTagged if force is False and tags exist) still
|
|
protects callers that skipped the pre-check."""
|
|
files = FileGroup.get_by_ids(session, file_group_ids)
|
|
if not files:
|
|
raise UnknownFile(f'No FileGroups found for ids: {file_group_ids}')
|
|
|
|
affected_models: Set[str] = {fg.model for fg in files if fg.model}
|
|
had_tagged = any(fg.tag_names for fg in files)
|
|
|
|
# Capture domain Collection IDs for any Archive FileGroups so we can prune empty ones after
|
|
# the cascade removes the Archive rows. (Archive.delete used to do this per-archive; with
|
|
# cascade-based deletion we do it as a single post-pass.)
|
|
archive_collection_ids: Set[int] = set()
|
|
if 'archive' in affected_models:
|
|
from modules.archive.models import Archive
|
|
archive_collection_ids = {
|
|
collection_id for (collection_id,) in session.query(Archive.collection_id)
|
|
.filter(Archive.file_group_id.in_([fg.id for fg in files]))
|
|
if collection_id is not None
|
|
}
|
|
|
|
for fg in files:
|
|
fg.delete(force=force)
|
|
|
|
if archive_collection_ids:
|
|
# Flush so the FK CASCADE has removed the Archive rows before we count remaining.
|
|
session.flush()
|
|
from modules.archive.models import Archive
|
|
from wrolpi.collections.models import Collection
|
|
for collection_id in archive_collection_ids:
|
|
collection = session.query(Collection).filter_by(id=collection_id, kind='domain').one_or_none()
|
|
if collection is None:
|
|
continue
|
|
remaining = session.query(Archive).filter_by(collection_id=collection_id).count()
|
|
if remaining == 0:
|
|
session.delete(collection)
|
|
|
|
if had_tagged:
|
|
save_tags_config.activate_switch()
|
|
|
|
return affected_models
|
|
|
|
|
|
@wrol_mode_check
|
|
async def delete(*paths: Union[str, pathlib.Path], force: bool = False, post_processing: bool = True):
|
|
"""
|
|
Delete a file or directory in the media directory.
|
|
|
|
If any files are tagged and force is False, returns a list of tagged FileGroups without deleting.
|
|
If force is True, deletes even tagged files.
|
|
|
|
The DB cleanup of the deleted FileGroups finishes before this returns; the global modelers/indexers are
|
|
queued in the FileWorker. If post_processing is False they are not queued at all — use this when the caller will trigger modeling itself (e.g. the upload endpoint
|
|
follows up with `upsert_file`)."""
|
|
media_directory = get_media_directory()
|
|
paths = [media_directory / i for i in paths]
|
|
if any(i == media_directory for i in paths):
|
|
raise InvalidFile(f'Cannot delete the media directory')
|
|
for path in paths:
|
|
if not path.is_dir() and not path.is_file():
|
|
logger.warning(f'Cannot delete {path} because it is not a file or a directory.')
|
|
if len(paths) > 1:
|
|
for p1 in paths:
|
|
for p2 in paths:
|
|
if p1 == p2:
|
|
continue
|
|
if p1 != p2 and p1 in p2.parents:
|
|
raise InvalidFile(f'Cannot deleted nested paths')
|
|
|
|
# Discover any tagged FileGroups under these paths so we can either gate the deletion (when
|
|
# !force) or fire `save_tags_config` afterwards (when force=True and any were tagged).
|
|
with get_db_session() as session:
|
|
tagged_file_groups = get_tagged_file_groups_by_paths(session, paths)
|
|
if tagged_file_groups and not force:
|
|
# Serialize while session is still active.
|
|
return [i.__json__() for i in tagged_file_groups]
|
|
had_tagged = bool(tagged_file_groups)
|
|
|
|
# Remove the Tags Directory links of the files about to be deleted, while they can still be matched to
|
|
# their sources. A deleted directory takes every file of a FileGroup inside it.
|
|
directories = [i for i in paths if i.is_dir()]
|
|
files = [i for i in paths if i.is_file()]
|
|
for file_group in tagged_file_groups:
|
|
if any(i == file_group.directory or i in file_group.directory.parents for i in directories):
|
|
file_group.unlink_tag_directory_links()
|
|
elif files:
|
|
file_group.unlink_tag_directory_links(*files)
|
|
|
|
for path in paths:
|
|
ignored_directories = get_wrolpi_config().ignored_directories
|
|
if ignored_directories and str(path) in ignored_directories:
|
|
remove_ignored_directory(path)
|
|
if path.is_dir():
|
|
delete_directory(path, recursive=True, force=force)
|
|
elif path.exists():
|
|
path.unlink()
|
|
|
|
# Refresh synchronously to clean up DB before returning.
|
|
# This ensures deleted FileGroups are removed before the caller continues
|
|
# (e.g., upload API recreating the file).
|
|
from wrolpi.files.worker import file_worker
|
|
await file_worker.refresh_sync(list(paths), post_processing=False)
|
|
if post_processing:
|
|
# The modelers and indexers cover the whole library, so the request must not wait on them. A queued
|
|
# refresh of these paths finds nothing left to change and then runs them in the FileWorker.
|
|
file_worker.queue_refresh(list(paths), send_events=False)
|
|
|
|
if had_tagged:
|
|
# `tags.yaml` references TagFiles that the refresh just removed; rewrite it.
|
|
save_tags_config.activate_switch()
|
|
|
|
|
|
def _mimetype_suffix_map(path: Path, mimetype: str):
|
|
"""Special handling for mimetypes. Python Magic may not return the correct mimetype."""
|
|
from modules.docs.models import MOBI_MIMETYPE
|
|
suffix = path.suffix.lower()
|
|
if mimetype == 'application/octet-stream':
|
|
if suffix.endswith('.mobi'):
|
|
return MOBI_MIMETYPE
|
|
if suffix.endswith('.stl'):
|
|
return 'model/stl'
|
|
if suffix.endswith('.3mf'):
|
|
return 'model/3mf'
|
|
if suffix.endswith('.mp4'):
|
|
return 'video/mp4'
|
|
if suffix.endswith('.html') or suffix.endswith('.htm'):
|
|
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
|
|
return 'text/html'
|
|
if suffix.endswith('.hgt'):
|
|
return 'application/octet-stream'
|
|
if mimetype.startswith('text/') and suffix in ('.md', '.markdown'):
|
|
# libmagic reports markdown as text/plain, or as a programming language when it sniffs code.
|
|
return 'text/markdown'
|
|
if mimetype == 'text/plain':
|
|
if suffix.endswith('.json'):
|
|
return 'application/json'
|
|
if suffix.endswith('.vtt'):
|
|
return 'text/vtt'
|
|
if suffix.endswith('.csv'):
|
|
return 'text/csv'
|
|
if suffix.endswith('.html') or suffix.endswith('.htm'):
|
|
return 'text/html'
|
|
if suffix.endswith('.stl'):
|
|
return 'model/stl'
|
|
if suffix.endswith('.scad'):
|
|
return 'application/x-openscad'
|
|
if suffix.endswith('.js'):
|
|
return 'application/javascript'
|
|
if suffix.endswith('.css'):
|
|
return 'text/css'
|
|
if suffix.endswith('.srt'):
|
|
return 'text/srt'
|
|
if suffix.endswith('.yaml') or suffix.endswith('.yml'):
|
|
return 'text/yaml'
|
|
if suffix.endswith('.azw3'):
|
|
return 'application/vnd.amazon.mobi8-ebook'
|
|
if suffix.endswith('.obj'):
|
|
return 'model/obj'
|
|
if mimetype == 'application/x-subrip':
|
|
# Fallback to old mimetype.
|
|
return 'text/srt'
|
|
if mimetype == 'application/zip' and (suffix == '.html' or suffix == '.htm'):
|
|
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
|
|
return 'text/html'
|
|
if mimetype == 'application/zip' and suffix == '.docx':
|
|
return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
|
if mimetype == 'application/zip' and suffix == '.cbz':
|
|
return 'application/x-cbz'
|
|
if mimetype == 'application/zip' and suffix == '.3mf':
|
|
return 'model/3mf'
|
|
if mimetype == 'application/zip' and suffix == '.odt':
|
|
return 'application/vnd.oasis.opendocument.text'
|
|
if mimetype in ('application/x-rar', 'application/x-rar-compressed') and suffix == '.cbr':
|
|
return 'application/x-cbr'
|
|
if mimetype == 'video/x-ms-asf' and suffix == '.wma':
|
|
return 'audio/x-ms-wma'
|
|
return mimetype
|
|
|
|
|
|
FILE_BIN = which('file', '/usr/bin/file')
|
|
|
|
|
|
@functools.lru_cache(maxsize=10_000)
|
|
def get_mimetype(path: Path) -> str:
|
|
"""Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command."""
|
|
if magic is None:
|
|
# This method is slow, prefer the speedier `magic` module.
|
|
cmd = (FILE_BIN, '--mime-type', str(path.absolute()))
|
|
output = subprocess.check_output(cmd)
|
|
output = output.decode()
|
|
mimetype = output.split(' ')[-1].strip()
|
|
else:
|
|
mimetype = mime.from_file(path)
|
|
mimetype = _mimetype_suffix_map(path, mimetype)
|
|
return mimetype
|
|
|
|
|
|
# Special suffixes within WROLPi.
|
|
SUFFIXES = {
|
|
'.info.json',
|
|
'.ffprobe.json',
|
|
'.live_chat.json',
|
|
'.readability.html',
|
|
'.readability.json',
|
|
'.readability.txt',
|
|
}
|
|
SUFFIXES |= {f'.{i}.srt' for i in ISO_639_CODES}
|
|
SUFFIXES |= {f'.{i}.vtt' for i in ISO_639_CODES}
|
|
|
|
# Convert to lowercase frozensets for O(1) case-insensitive lookup
|
|
_SUFFIXES_LOWER = frozenset(s.lower() for s in SUFFIXES)
|
|
_LANGUAGES_LOWER = frozenset(i.lower() for i in ISO_639_CODES)
|
|
_REGIONS_LOWER = frozenset(i.lower() for i in ISO_3166_CODES)
|
|
|
|
|
|
def _is_language_region_suffix(suffix_lower: str) -> bool:
|
|
"""Is this lowercase .srt/.vtt suffix a language with a region or "auto", like ".en-us.srt" or ".en-auto.vtt"?
|
|
|
|
Never build the set of every combination: 336,000 strings cost ~2s and ~65MiB in each process on a Pi 4. A
|
|
region code never contains "-"; a language code may ("en-gb")."""
|
|
language, _, region = suffix_lower[1:-4].rpartition('-')
|
|
return language in _LANGUAGES_LOWER and (region == 'auto' or region in _REGIONS_LOWER)
|
|
|
|
|
|
_HARDCODED_SUFFIXES = frozenset(
|
|
{'.info.json', '.ffprobe.json', '.live_chat.json', '.readability.html', '.readability.json', '.readability.txt'})
|
|
|
|
# Bump when SUFFIXES / split_path_stem_and_suffix grouping rules change. A stored
|
|
# file_group.stem is a cache of this algorithm; compare joins on it, so a mismatch
|
|
# looks like delete+insert and would drop tags. ensure_file_group_stems() must run
|
|
# before any compare when this version does not match the value in wrolpi_kv.
|
|
STEM_ALGORITHM_VERSION = 1
|
|
STEM_ALGORITHM_VERSION_KEY = 'stem_algorithm_version'
|
|
# Cancelled compare_file_groups leaves ensure_file_group_stems running in a thread.
|
|
# Tests (and overlapping refreshes) share one SQLite writer; serialize the backfill.
|
|
_stem_backfill_lock = threading.Lock()
|
|
|
|
PART_PARSER = re.compile(r'(.+?)(\.f[\d]{2,3})?(\.info)?(\.\w{3,4})(\.part)', re.IGNORECASE)
|
|
|
|
|
|
def _extract_candidate_suffixes(name_lower: str) -> List[str]:
|
|
"""Extract potential suffixes from filename (last 1, 2, 3 dot-parts).
|
|
|
|
Returns suffixes in order from shortest to longest.
|
|
"""
|
|
parts = name_lower.rsplit('.', 3)
|
|
if len(parts) == 1:
|
|
return []
|
|
candidates = []
|
|
append_candidate = candidates.append
|
|
suffix = ''
|
|
for i in range(len(parts) - 1, 0, -1):
|
|
suffix = '.' + parts[i] + suffix
|
|
append_candidate(suffix)
|
|
return candidates
|
|
|
|
|
|
@functools.lru_cache(maxsize=5_000)
|
|
def _get_suffix_length(name_lower: str) -> int:
|
|
"""Get length of matching suffix using O(1) set lookups.
|
|
|
|
Returns 0 if no suffix matches.
|
|
"""
|
|
# Fast path: check hardcoded multi-part suffixes first
|
|
for suffix in _HARDCODED_SUFFIXES:
|
|
if name_lower.endswith(suffix):
|
|
return len(suffix)
|
|
|
|
candidates = _extract_candidate_suffixes(name_lower)
|
|
if not candidates:
|
|
return 0
|
|
|
|
# Check SUFFIXES (longest first for most specific match)
|
|
for suffix in reversed(candidates):
|
|
if suffix in _SUFFIXES_LOWER:
|
|
return len(suffix)
|
|
|
|
# Only .srt/.vtt files have language-region suffixes
|
|
simple_suffix = candidates[0] if candidates else ''
|
|
if simple_suffix in {'.srt', '.vtt'}:
|
|
for suffix in reversed(candidates):
|
|
if _is_language_region_suffix(suffix):
|
|
return len(suffix)
|
|
|
|
return len(simple_suffix)
|
|
|
|
|
|
@functools.lru_cache(maxsize=10_000)
|
|
def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = False) -> Tuple[str, str]:
|
|
"""Get the path's stem and suffix.
|
|
|
|
This function handles WROLPi suffixes like .info.json.
|
|
|
|
>>> split_path_stem_and_suffix('/foo/bar.txt')
|
|
('bar', '.txt')
|
|
>>> split_path_stem_and_suffix('/foo/bar.txt', full=True)
|
|
('/foo/bar', '.txt')
|
|
>>> split_path_stem_and_suffix('/foo/bar.info.json', full=True)
|
|
('/foo/bar', '.info.json')
|
|
"""
|
|
path = pathlib.Path(path) if isinstance(path, str) else path
|
|
|
|
if path.suffix == '.part':
|
|
# yt-dlp uses part files while downloading, include those in a FileGroup.
|
|
stem, info, format_num, suffix, part = PART_PARSER.match(path.name).groups()
|
|
info = info or ''
|
|
format_num = format_num or ''
|
|
stem = f'{path.parent}/{stem}' if full else stem
|
|
return stem, f'{info}{format_num}{suffix}{part}'
|
|
|
|
name = path.name
|
|
name_lower = name.lower()
|
|
|
|
# Get suffix length using O(1) set lookups instead of O(n) linear search
|
|
suffix_len = _get_suffix_length(name_lower)
|
|
|
|
if suffix_len > 0:
|
|
idx = -suffix_len
|
|
if full:
|
|
return f'{path.parent}/{name[:idx]}', name[idx:]
|
|
else:
|
|
return name[:idx], name[idx:]
|
|
|
|
# Path has no suffix.
|
|
if full:
|
|
return f'{path.parent}/{path.name}', ''
|
|
else:
|
|
return path.name, ''
|
|
|
|
|
|
def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ...] | Set[pathlib.Path]) \
|
|
-> List[pathlib.Path]:
|
|
"""Returns the first of each Path with a unique stem. Used to detect if a group of files share a stem."""
|
|
results = unique_by_predicate(files, lambda i: split_path_stem_and_suffix(i)[0])
|
|
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
|
logger.trace(f'get_unique_files_by_stem {files=}')
|
|
logger.trace(f'get_unique_files_by_stem {results=}')
|
|
return results
|
|
|
|
|
|
# The metadata JSON holds the sums, so it cannot hold its own.
|
|
METADATA_JSON_SUFFIXES = ('.info.json', '.readability.json')
|
|
|
|
# A file's (modification time in nanoseconds, hex sha256), keyed by its path.
|
|
FileSums = Dict[pathlib.Path, Tuple[int, str]]
|
|
|
|
|
|
def sha256_files(paths: Iterable[pathlib.Path], known: FileSums = None) -> FileSums:
|
|
"""Hash each file, skipping the metadata JSON and files that do not exist.
|
|
|
|
A `known` sum is reused while its file's modification time is unchanged. This reads whole files; call it in a
|
|
thread."""
|
|
known = known or dict()
|
|
sums = dict()
|
|
for path in paths:
|
|
if path.name.endswith(METADATA_JSON_SUFFIXES) or not path.is_file():
|
|
continue
|
|
mtime_ns = path.stat().st_mtime_ns
|
|
if (known_sum := known.get(path)) and known_sum[0] == mtime_ns:
|
|
sums[path] = known_sum
|
|
continue
|
|
with path.open('rb') as fh:
|
|
sums[path] = (mtime_ns, hashlib.file_digest(fh, 'sha256').hexdigest())
|
|
return sums
|
|
|
|
|
|
def sha256_by_suffix(sums: FileSums) -> Dict[str, str]:
|
|
"""The hex sha256 of each file keyed by its suffix. The files of a FileGroup share a stem, so a suffix names one
|
|
file even after the group is renamed."""
|
|
return {split_path_stem_and_suffix(path)[1]: sha256 for path, (_, sha256) in sums.items()}
|
|
|
|
|
|
refresh_logger = logger.getChild('refresh')
|
|
|
|
|
|
def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]:
|
|
"""This generates `FileGroup.files` from a list of files.
|
|
|
|
Paths are stored as filenames only (relative to FileGroup.directory).
|
|
"""
|
|
files = list()
|
|
for file in group:
|
|
mimetype = get_mimetype(file)
|
|
stat = file.stat()
|
|
modification_datetime = str(from_timestamp(stat.st_mtime))
|
|
_, suffix = split_path_stem_and_suffix(file)
|
|
size = stat.st_size
|
|
files.append(dict(
|
|
path=file.name, # Store filename only, not full path
|
|
mimetype=mimetype,
|
|
modification_datetime=modification_datetime,
|
|
size=size,
|
|
suffix=suffix,
|
|
))
|
|
return files
|
|
|
|
|
|
# The stem of a yt-dlp per-format file, which is merged into the final media file. Example: The Title.f313.webm
|
|
YTDLP_FORMAT_STEM = re.compile(r'\.f\d{2,4}$')
|
|
|
|
|
|
def get_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path:
|
|
"""Given a list of files, return the file that we can model or index.
|
|
|
|
@raise NoPrimaryFile: If not primary file can be found."""
|
|
from modules.archive.lib import is_singlefile_file
|
|
|
|
if len(files) == 0:
|
|
raise ValueError(f'Cannot find primary file without any files')
|
|
elif len(files) == 1:
|
|
# Only one file is always the primary.
|
|
return files[0]
|
|
|
|
file_mimetypes = list()
|
|
for idx, i in enumerate(files):
|
|
try:
|
|
file_mimetypes.append((i, get_mimetype(i)))
|
|
except FileNotFoundError:
|
|
# File was deleted.
|
|
pass
|
|
|
|
if not file_mimetypes:
|
|
raise FileNotFoundError(f'Cannot find primary file. All files are no longer accessible: {files[0]}')
|
|
|
|
# EPUB has high priority.
|
|
mimetypes = {i[1] for i in file_mimetypes}
|
|
has_epub = any(i.startswith('application/epub') for i in mimetypes)
|
|
# A yt-dlp per-format file (video-only or audio-only) is primary only when no merged media file is present.
|
|
has_merged_media = any(m.startswith(('video/', 'audio/')) and not YTDLP_FORMAT_STEM.search(f.stem)
|
|
for f, m in file_mimetypes)
|
|
for file, mimetype in file_mimetypes:
|
|
# These are the files that can be modeled.
|
|
if is_singlefile_file(file):
|
|
return file
|
|
if mimetype.startswith(('video/', 'audio/')):
|
|
if has_merged_media and YTDLP_FORMAT_STEM.search(file.stem):
|
|
continue
|
|
return file
|
|
if mimetype.startswith('application/epub'):
|
|
return file
|
|
if mimetype.startswith('application/pdf') and not has_epub:
|
|
return file
|
|
if mimetype.startswith('application/x-mobipocket-ebook') and not has_epub:
|
|
# Only return a MOBI if no EPUB is present.
|
|
return file
|
|
if mimetype in ('application/msword',
|
|
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
'application/vnd.oasis.opendocument.text',
|
|
'application/x-cbz', 'application/x-cbr',
|
|
'application/vnd.comicbook+zip', 'application/vnd.comicbook-rar'):
|
|
return file
|
|
if file.suffix.lower() in ('.cbz', '.cbr', '.cbt', '.cb7'):
|
|
return file
|
|
|
|
# Secondary file types.
|
|
for file, mimetype in file_mimetypes:
|
|
if mimetype.startswith('image/'):
|
|
return file
|
|
|
|
raise NoPrimaryFile(f'Cannot find primary file for group: {files}')
|
|
|
|
|
|
def choose_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path:
|
|
"""Primary file for a stem group. Falls back to a sorted path so every caller agrees.
|
|
|
|
`_upsert_files` and `_upsert_file_groups` must pick the same path. A set's iteration
|
|
order is not stable; using it as the fallback would DELETE the existing tagged row
|
|
and INSERT a new untagged one.
|
|
"""
|
|
paths = [pathlib.Path(p) for p in files]
|
|
if not paths:
|
|
raise ValueError('Cannot find primary file without any files')
|
|
try:
|
|
return get_primary_file(paths)
|
|
except NoPrimaryFile:
|
|
return sorted(paths)[0]
|
|
|
|
|
|
def _file_names_sql(table: str) -> str:
|
|
"""SQL: the sorted names in `table`.files, as a JSON array.
|
|
|
|
Old FileGroups store each file's absolute path, new ones its name; both give the name."""
|
|
path = "json_extract(j.value, '$.path')"
|
|
return f'''(SELECT json_group_array(name) FROM (
|
|
SELECT CASE WHEN {path} LIKE '/%' THEN substr({path}, length({table}.directory) + 2) ELSE {path} END
|
|
AS name
|
|
FROM json_each({table}.files) AS j
|
|
ORDER BY name))'''
|
|
|
|
|
|
def _rank_file_groups(curs, file_group_ids: Iterable[int]) -> List[int]:
|
|
"""The ids of these FileGroups, the one most worth keeping first: modeled, then tagged, then with files, then
|
|
the newest."""
|
|
curs.execute('''
|
|
SELECT fg.id,
|
|
CASE
|
|
WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0
|
|
ELSE json_array_length(fg.files)
|
|
END AS file_count,
|
|
CASE
|
|
WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1
|
|
ELSE 0
|
|
END AS has_model,
|
|
(SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count
|
|
FROM file_group fg
|
|
WHERE fg.id IN (SELECT value FROM json_each(?))
|
|
''', (json.dumps(list(file_group_ids)),))
|
|
members = list(curs.fetchall())
|
|
members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True)
|
|
return [m['id'] for m in members]
|
|
|
|
|
|
def _count_primary_path_change(curs, file_group_id: int):
|
|
"""tags.yaml, playlists.yaml and recently_viewed.yaml save a FileGroup by its primary path; count a change of
|
|
that path for each config that holds this FileGroup."""
|
|
curs.execute('''
|
|
SELECT EXISTS (SELECT 1 FROM tag_file WHERE file_group_id = :id) AS tagged,
|
|
EXISTS (SELECT 1
|
|
FROM collection_item ci
|
|
JOIN collection c ON c.id = ci.collection_id
|
|
WHERE ci.file_group_id = :id
|
|
AND c.kind = 'playlist') AS in_playlist,
|
|
(SELECT viewed IS NOT NULL FROM file_group WHERE id = :id) AS viewed
|
|
''', {'id': file_group_id})
|
|
row = curs.fetchone()
|
|
names = [name for name, held in (('tags.yaml', row['tagged']), ('playlists.yaml', row['in_playlist']),
|
|
('recently_viewed.yaml', row['viewed'])) if held]
|
|
if names:
|
|
count_config_change_curs(curs, *names)
|
|
|
|
|
|
def _collapse_file_group(curs, loser_id: int, winner_id: int):
|
|
"""Delete the loser FileGroup after giving its tags, collection membership and viewing state to the winner.
|
|
|
|
The winner keeps its own viewing state when it has one."""
|
|
merge_file_group_relations(curs, loser_id, winner_id)
|
|
curs.execute('SELECT viewed IS NOT NULL AS viewed FROM file_group WHERE id = ?', (loser_id,))
|
|
if curs.fetchone()['viewed']:
|
|
# recently_viewed.yaml saves the loser by its primary path.
|
|
count_config_change_curs(curs, 'recently_viewed.yaml')
|
|
curs.execute('''
|
|
UPDATE file_group
|
|
SET viewed = COALESCE(file_group.viewed, loser.viewed),
|
|
progress = COALESCE(file_group.progress, loser.progress),
|
|
position = COALESCE(file_group.position, loser.position)
|
|
FROM (SELECT viewed, progress, position FROM file_group WHERE id = :loser) AS loser
|
|
WHERE file_group.id = :winner
|
|
''', {'loser': loser_id, 'winner': winner_id})
|
|
curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,))
|
|
|
|
|
|
def _give_stem_owners_their_primary_paths(curs, values: List[tuple]):
|
|
"""Give each incoming primary path to the FileGroup that already owns its (directory, stem).
|
|
|
|
`_upsert_files` then updates that FileGroup through ON CONFLICT (primary_path) instead of colliding with it on
|
|
the (directory, stem) unique index. The FileGroup keeps its id, so its tags, collection membership, model and
|
|
viewing state stay with it. If a different FileGroup owns the primary path, the two collapse into the one
|
|
`_rank_file_groups` prefers.
|
|
|
|
@param values: `_upsert_files` rows: (primary_path, directory, ..., stem)."""
|
|
incoming = json.dumps([[value[1], value[-1], value[0]] for value in values])
|
|
curs.execute('''
|
|
SELECT fg.id, json_extract(v.value, '$[2]') AS primary_path
|
|
FROM json_each(:incoming) AS v
|
|
JOIN file_group fg ON fg.directory = json_extract(v.value, '$[0]')
|
|
AND fg.stem = json_extract(v.value, '$[1]')
|
|
WHERE fg.primary_path != json_extract(v.value, '$[2]')
|
|
-- A FileGroup whose primary path is incoming is updated by its own group.
|
|
AND fg.primary_path NOT IN (SELECT json_extract(value, '$[2]') FROM json_each(:incoming))
|
|
''', {'incoming': incoming})
|
|
for stem_owner_id, primary_path in curs.fetchall():
|
|
curs.execute('SELECT id FROM file_group WHERE primary_path = ?', (primary_path,))
|
|
if path_owner := curs.fetchone():
|
|
winner_id, loser_id = _rank_file_groups(curs, [stem_owner_id, path_owner['id']])
|
|
_collapse_file_group(curs, loser_id, winner_id)
|
|
if winner_id == path_owner['id']:
|
|
# The survivor already has the primary path.
|
|
continue
|
|
_count_primary_path_change(curs, stem_owner_id)
|
|
curs.execute('UPDATE file_group SET primary_path = ? WHERE id = ?', (primary_path, stem_owner_id))
|
|
|
|
|
|
# About how many files `_upsert_files` writes per transaction. A stem's files are never split across chunks.
|
|
UPSERT_CHUNK_SIZE = 500
|
|
|
|
|
|
def _upsert_files(files: List[pathlib.Path],
|
|
progress_callback: Callable[[int, int], None] = None):
|
|
"""Insert/update all records of the provided files.
|
|
|
|
Any inserted files will be marked with `indexed=false`. Any existing files will only be updated if their
|
|
size/modification_datetime changed.
|
|
|
|
Args:
|
|
files: List of file paths to upsert
|
|
progress_callback: Optional callback(processed, total) called after each chunk is processed
|
|
|
|
It is assumed all files exist."""
|
|
total_files = len(files)
|
|
processed_count = 0
|
|
|
|
for chunk in chunks_by_stem(files, UPSERT_CHUNK_SIZE):
|
|
# Group all files by their shared full-stem. `chunks_by_stem` keeps each stem in one chunk.
|
|
grouped = group_files_by_stem(chunk)
|
|
# Convert the groups into file_groups. Extract common information into `file_group.data`.
|
|
values = dict()
|
|
# The primary path of each companion file's group.
|
|
companion_owners = dict()
|
|
for group in grouped:
|
|
# The primary file is the video/SingleFile/epub, etc.
|
|
try:
|
|
primary_path = choose_primary_file(group)
|
|
# The primary mimetype allows modelers to find its file_groups.
|
|
mimetype = get_mimetype(primary_path)
|
|
# The group uses a common modification_datetime so the group will be re-indexed when any of it's
|
|
# files are modified.
|
|
modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group))
|
|
size = sum(i.stat().st_size for i in group)
|
|
files_json = json.dumps(_paths_to_files_dict(group))
|
|
stem, suffix = split_path_stem_and_suffix(primary_path)
|
|
suffix = (suffix or '').lower() or None
|
|
values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem)
|
|
companion_owners.update((str(i), str(primary_path)) for i in group if i != primary_path)
|
|
except FileNotFoundError as e:
|
|
# A file was deleted between discovery and upsert (temp files, WAL sidecars, etc.).
|
|
refresh_logger.warning(f'File vanished during refresh, skipping group near {group[0]}', exc_info=e)
|
|
continue
|
|
|
|
values = [(str(primary_path),
|
|
str(primary_path.parent), # directory
|
|
False, # `indexed` is false to force indexing by default.
|
|
*i) for primary_path, i in values.items()]
|
|
with get_db_curs(commit=True) as curs:
|
|
# SQLite's RETURNING only exposes the post-upsert row; find the previously-indexed
|
|
# paths first so re-indexing invalidations can be counted.
|
|
primary_paths = [i[0] for i in values]
|
|
curs.execute('SELECT primary_path FROM file_group WHERE indexed = true AND primary_path IN '
|
|
'(SELECT value FROM json_each(?))', (json.dumps(primary_paths),))
|
|
previously_indexed = {i['primary_path'] for i in curs.fetchall()}
|
|
|
|
# Before INSERT so a new primary_path does not collide on (directory, stem) with the FileGroup that
|
|
# already owns that stem.
|
|
_give_stem_owners_their_primary_paths(curs, values)
|
|
|
|
placeholders, params = values_clause(values)
|
|
# Compare names, not counts: a file can replace another without changing the size or newest time.
|
|
same_files = f'{_file_names_sql("file_group")} = {_file_names_sql("EXCLUDED")}'
|
|
stmt = f'''
|
|
INSERT INTO file_group
|
|
(primary_path, directory, indexed, modification_datetime, mimetype, size, files,
|
|
suffix, stem)
|
|
VALUES {placeholders}
|
|
ON CONFLICT (primary_path) DO UPDATE
|
|
SET
|
|
directory=EXCLUDED.directory,
|
|
modification_datetime=EXCLUDED.modification_datetime,
|
|
mimetype=EXCLUDED.mimetype,
|
|
size=EXCLUDED.size,
|
|
-- Always keep suffix/stem current so grouping and suffix search stay correct.
|
|
suffix=EXCLUDED.suffix,
|
|
stem=EXCLUDED.stem,
|
|
files=(
|
|
-- Do not overwrite files unless the files have changed and need to be re-indexed.
|
|
CASE
|
|
WHEN
|
|
file_group.modification_datetime = EXCLUDED.modification_datetime
|
|
AND file_group.size = EXCLUDED.size
|
|
AND {same_files}
|
|
THEN file_group.files
|
|
ELSE EXCLUDED.files
|
|
END
|
|
),
|
|
-- Preserve TRUE `indexed` only if the files have not changed.
|
|
indexed=(
|
|
file_group.indexed = true
|
|
AND file_group.modification_datetime = EXCLUDED.modification_datetime
|
|
AND file_group.size = EXCLUDED.size
|
|
AND {same_files}
|
|
)
|
|
RETURNING id, primary_path, indexed
|
|
'''
|
|
curs.execute(stmt, params)
|
|
results = list(curs.fetchall())
|
|
# Count the files that used to be indexed, but need to be re-indexed.
|
|
invalidated_files = len([i['id'] for i in results
|
|
if i['primary_path'] in previously_indexed and not i['indexed']])
|
|
if invalidated_files:
|
|
refresh_logger.info(f'Invalidated indexes of {invalidated_files} file groups near {chunk[0]}')
|
|
else:
|
|
refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}')
|
|
|
|
# A companion file can still be the primary path of an old FileGroup (one with a stale stem). Its
|
|
# tags, collection membership and viewing state belong to its group's FileGroup.
|
|
if companion_owners:
|
|
ids_by_primary_path = {i['primary_path']: i['id'] for i in results}
|
|
curs.execute('SELECT id, primary_path FROM file_group WHERE primary_path IN '
|
|
'(SELECT value FROM json_each(?))', (json.dumps(list(companion_owners)),))
|
|
for old in curs.fetchall():
|
|
owner_id = ids_by_primary_path[companion_owners[old['primary_path']]]
|
|
_collapse_file_group(curs, old['id'], owner_id)
|
|
|
|
# Report progress after each chunk
|
|
processed_count += len(chunk)
|
|
if progress_callback:
|
|
progress_callback(processed_count, total_files)
|
|
|
|
|
|
def is_ignored_file(file: pathlib.Path) -> bool:
|
|
"""Is `file`, or the file it links to, in an ignored directory?
|
|
|
|
Refresh does not follow links, but a link can be viewed directly."""
|
|
return len(remove_files_in_ignored_directories([file, file.resolve()])) < 2
|
|
|
|
|
|
def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]:
|
|
"""Return a new list which does not contain any file paths that are in ignored directories."""
|
|
from wrolpi.db import get_db_file
|
|
|
|
ignored_directories = get_normalized_ignored_directories()
|
|
# `private` must not hide `private-notes`.
|
|
files = [i for i in files
|
|
if not any(str(i) == j or str(i).startswith(f'{j}{os.sep}') for j in ignored_directories)]
|
|
# Never index the database (or its WAL sidecars), even if the user un-ignores `config`.
|
|
db_file = str(get_db_file())
|
|
files = [i for i in files if not str(i).startswith(db_file)]
|
|
return files
|
|
|
|
|
|
def _ensure_kv_table(curs):
|
|
curs.execute('CREATE TABLE IF NOT EXISTS wrolpi_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
|
|
|
|
|
|
def get_kv_value(key: str) -> Optional[str]:
|
|
"""The value stored in wrolpi_kv for `key`, or None."""
|
|
with get_db_curs() as curs:
|
|
_ensure_kv_table(curs)
|
|
curs.execute('SELECT value FROM wrolpi_kv WHERE key = ?', (key,))
|
|
row = curs.fetchone()
|
|
return row['value'] if row else None
|
|
|
|
|
|
def set_kv_value(key: str, value: str):
|
|
with get_db_curs(commit=True) as curs:
|
|
_ensure_kv_table(curs)
|
|
curs.execute(
|
|
'INSERT INTO wrolpi_kv (key, value) VALUES (?, ?) '
|
|
'ON CONFLICT(key) DO UPDATE SET value = excluded.value',
|
|
(key, value),
|
|
)
|
|
|
|
|
|
def get_stem_algorithm_version() -> int:
|
|
"""The grouping-algorithm version last applied to file_group.stem, or 0 if never stored."""
|
|
value = get_kv_value(STEM_ALGORITHM_VERSION_KEY)
|
|
return int(value) if value else 0
|
|
|
|
|
|
def set_stem_algorithm_version(version: int):
|
|
set_kv_value(STEM_ALGORITHM_VERSION_KEY, str(version))
|
|
|
|
|
|
def ensure_file_group_stems():
|
|
"""Recompute file_group.stem when STEM_ALGORITHM_VERSION has changed.
|
|
|
|
A stored stem is a cache of split_path_stem_and_suffix. Compare joins on (directory, stem),
|
|
so an algorithm change without this backfill reports the same files as deleted+new and
|
|
drops tags, viewed timestamps, and domain rows. Must run before any compare.
|
|
|
|
After a successful backfill the kv version matches, and later refreshes skip the table scan.
|
|
Upsert, from_paths, move, and reorganize must keep stem current.
|
|
|
|
A lock serializes callers: cancelling compare_file_groups does not stop the executor
|
|
thread, and a second compare would otherwise write wrolpi_kv concurrently (SQLITE_BUSY).
|
|
"""
|
|
with _stem_backfill_lock:
|
|
stored = get_stem_algorithm_version()
|
|
if stored == STEM_ALGORITHM_VERSION:
|
|
return
|
|
with get_db_curs() as curs:
|
|
curs.execute('SELECT id, primary_path FROM file_group')
|
|
rows = list(curs.fetchall())
|
|
if rows:
|
|
updates = []
|
|
for row in rows:
|
|
stem, _ = split_path_stem_and_suffix(row['primary_path'])
|
|
updates.append((stem, row['id']))
|
|
with get_db_curs(commit=True) as curs:
|
|
for chunk in chunks(updates, 500):
|
|
curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk)
|
|
set_stem_algorithm_version(STEM_ALGORITHM_VERSION)
|
|
|
|
|
|
def backfill_null_file_group_stems():
|
|
"""Fill file_group.stem for rows that never got one. Used before creating the unique index."""
|
|
with get_db_curs() as curs:
|
|
curs.execute("SELECT id, primary_path FROM file_group WHERE stem IS NULL OR stem = ''")
|
|
rows = list(curs.fetchall())
|
|
if not rows:
|
|
return
|
|
updates = [(split_path_stem_and_suffix(row['primary_path'])[0], row['id']) for row in rows]
|
|
with get_db_curs(commit=True) as curs:
|
|
for chunk in chunks(updates, 500):
|
|
curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk)
|
|
|
|
|
|
def dedupe_file_groups_by_directory_stem() -> int:
|
|
"""Collapse FileGroups that share (directory, stem).
|
|
|
|
Prefers a row with a domain model, then tags, then non-empty files, then highest id.
|
|
Tags and collection membership on losers are moved onto the winner. Returns the
|
|
number of FileGroups deleted.
|
|
"""
|
|
with get_db_curs() as curs:
|
|
curs.execute('''
|
|
SELECT directory, stem
|
|
FROM file_group
|
|
WHERE directory IS NOT NULL
|
|
AND stem IS NOT NULL
|
|
AND stem != ''
|
|
GROUP BY directory, stem
|
|
HAVING COUNT(*) > 1
|
|
''')
|
|
keys = [(row['directory'], row['stem']) for row in curs.fetchall()]
|
|
|
|
deleted = 0
|
|
for directory, stem in keys:
|
|
with get_db_curs() as curs:
|
|
curs.execute('SELECT id FROM file_group WHERE directory = ? AND stem = ?', (directory, stem))
|
|
winner_id, *loser_ids = _rank_file_groups(curs, [row['id'] for row in curs.fetchall()])
|
|
with get_db_curs(commit=True) as curs:
|
|
for loser_id in loser_ids:
|
|
merge_file_group_relations(curs, loser_id, winner_id)
|
|
curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,))
|
|
deleted += 1
|
|
if deleted:
|
|
refresh_logger.info(f'Collapsed {deleted} duplicate FileGroups sharing (directory, stem)')
|
|
return deleted
|
|
|
|
|
|
def merge_file_group_relations(curs, loser_id: int, winner_id: int):
|
|
"""Copy the loser FileGroup's tags and collection membership onto the winner, so deleting the loser
|
|
(which cascades) does not lose them. The caller deletes the loser."""
|
|
curs.execute('''
|
|
INSERT OR IGNORE INTO tag_file (tag_id, file_group_id, created_at)
|
|
SELECT tag_id, ?, created_at
|
|
FROM tag_file
|
|
WHERE file_group_id = ?
|
|
''', (winner_id, loser_id))
|
|
curs.execute('''
|
|
UPDATE collection_item
|
|
SET file_group_id = ?
|
|
WHERE file_group_id = ?
|
|
AND collection_id NOT IN (SELECT collection_id
|
|
FROM (SELECT collection_id
|
|
FROM collection_item
|
|
WHERE file_group_id = ?))
|
|
''', (winner_id, loser_id, winner_id))
|
|
# Raw SQL: count these changes for the configs that save TagFiles and playlist items (wrolpi.config_version).
|
|
count_config_change_curs(curs, 'tags.yaml', 'playlists.yaml')
|
|
|
|
|
|
def merge_into_surviving_duplicates(curs, file_group_ids: List[int], deleting_ids: Set[int]) -> int:
|
|
"""Before `file_group_ids` are deleted, move each one's tags and collection membership onto a FileGroup
|
|
that shares its (directory, stem) and is not in `deleting_ids`. Returns how many FileGroups were merged.
|
|
|
|
A duplicate FileGroup (same directory and stem) is deleted by refresh; its relations belong to the survivor."""
|
|
params = ','.join('?' * len(file_group_ids))
|
|
curs.execute(f'''
|
|
SELECT fg.id, fg.directory, fg.stem
|
|
FROM file_group fg
|
|
WHERE fg.id IN ({params})
|
|
AND fg.stem IS NOT NULL
|
|
AND (EXISTS (SELECT 1 FROM tag_file tf WHERE tf.file_group_id = fg.id)
|
|
OR EXISTS (SELECT 1 FROM collection_item ci WHERE ci.file_group_id = fg.id))
|
|
''', file_group_ids)
|
|
merged = 0
|
|
for loser in curs.fetchall():
|
|
curs.execute('''
|
|
SELECT id
|
|
FROM file_group
|
|
WHERE directory = ?
|
|
AND stem = ?
|
|
AND id != ?
|
|
ORDER BY id DESC
|
|
''', (loser['directory'], loser['stem'], loser['id']))
|
|
survivor_id = next((row['id'] for row in curs.fetchall() if row['id'] not in deleting_ids), None)
|
|
if survivor_id is not None:
|
|
merge_file_group_relations(curs, loser['id'], survivor_id)
|
|
merged += 1
|
|
return merged
|
|
|
|
|
|
async def apply_indexers(progress_callback: Callable[[int, int], None] = None):
|
|
"""Index leftover FileGroups that no modeler claimed.
|
|
|
|
File indexing state machine
|
|
---------------------------
|
|
A FileGroup is discovered (a row exists), then modeled and/or content-indexed.
|
|
|
|
`indexed` means "a modeler or this leftover indexer has finished with this row
|
|
for the current files." It is not "a domain model exists." `model` is the
|
|
domain claim (`video` / `archive` / `doc` / `zim`).
|
|
|
|
1. Discover/upsert: new or changed groups are inserted with indexed=0.
|
|
Unchanged groups keep indexed=1.
|
|
2. Modelers run first. A modeler that exclusively owns its mimetype
|
|
(video, doc, zim) selects:
|
|
(domain_row IS NULL OR indexed != 1) AND mimetype_matches
|
|
so a FileGroup the leftover indexer already marked indexed=1 still
|
|
gets a Video/Doc/Zim row. Failures are skipped for the rest of the
|
|
run (failed_ids) and retried on the next refresh.
|
|
3. Archive shares text/html with generic HTML. It selects only
|
|
indexed != 1. HTML that is not a SingleFile stays indexed=0 so this
|
|
function can claim it. HTML that is an Archive is indexed=1 with
|
|
model='archive'.
|
|
4. This function claims every remaining indexed != 1 row. It writes
|
|
a/b/c/d_text via the Indexer registry and sets indexed=1 even on
|
|
extractor failure so poison files do not loop forever.
|
|
5. Cleanup (channel claims, empty collections, tag config) runs last.
|
|
|
|
FileGroup.do_model() is the single-file equivalent of steps 2-4, used by
|
|
uploads. Batch modelers are used by FileWorker refresh.
|
|
|
|
Args:
|
|
progress_callback: Optional callback(processed, total) called after each batch.
|
|
"""
|
|
from wrolpi.files.models import FileGroup
|
|
refresh_logger.info('Applying indexers')
|
|
|
|
# Get initial count for progress tracking
|
|
total_to_index = 0
|
|
total_indexed = 0
|
|
if progress_callback:
|
|
with get_db_session() as session:
|
|
total_to_index = session.query(FileGroup).filter(FileGroup.indexed != True).count()
|
|
progress_callback(0, total_to_index)
|
|
|
|
update_stmt = ('UPDATE file_group SET "indexed" = 1, title = :title, a_text = :a_text, b_text = :b_text,'
|
|
' c_text = :c_text, d_text = :d_text WHERE id = :id')
|
|
|
|
def _strip_surrogates(value):
|
|
return strip_surrogates(value) if isinstance(value, str) else value
|
|
|
|
while True:
|
|
# Read the next batch as plain columns and let the read transaction end. Indexers can run
|
|
# subprocesses for minutes, and SQLite requires short write transactions: a session held
|
|
# across the batch fails its commit with "database is locked" (SQLITE_BUSY_SNAPSHOT, which
|
|
# ignores busy_timeout) whenever any other connection commits a write in the meantime.
|
|
with get_db_session() as session:
|
|
rows = session.query(FileGroup.id, FileGroup.primary_path, FileGroup.mimetype, FileGroup.title) \
|
|
.filter(FileGroup.indexed != True).limit(20).all()
|
|
|
|
if not rows:
|
|
break
|
|
|
|
# Index each file on a transient FileGroup: no session, no open transaction.
|
|
processed = 0
|
|
params = []
|
|
last_path = None
|
|
for fg_id, primary_path, mimetype, title in rows:
|
|
processed += 1
|
|
file_group = FileGroup(primary_path=primary_path, mimetype=mimetype, title=title)
|
|
try:
|
|
file_group.do_index()
|
|
except Exception:
|
|
# Error has already been logged in .do_index. It is still marked as indexed
|
|
# below; we won't try to index it again.
|
|
if PYTEST:
|
|
raise
|
|
params.append(dict(id=fg_id, title=file_group.title, a_text=file_group.a_text,
|
|
b_text=file_group.b_text, c_text=file_group.c_text, d_text=file_group.d_text))
|
|
last_path = primary_path
|
|
|
|
# Sleep to catch cancel.
|
|
await asyncio.sleep(0)
|
|
|
|
# Write the batch in one short transaction (BEGIN IMMEDIATE via commit=True).
|
|
try:
|
|
with get_db_curs(commit=True) as curs:
|
|
curs.executemany(update_stmt, params)
|
|
except UnicodeEncodeError as e:
|
|
# Paths with invalid UTF-8 surrogates produce titles/texts sqlite3 cannot store.
|
|
refresh_logger.warning(f'UnicodeEncodeError during indexer update, sanitizing batch: {e}')
|
|
params = [{k: _strip_surrogates(v) for k, v in p.items()} for p in params]
|
|
try:
|
|
with get_db_curs(commit=True) as curs:
|
|
curs.executemany(update_stmt, params)
|
|
except UnicodeEncodeError as e2:
|
|
# Still failing - mark as indexed without the texts so we don't loop forever.
|
|
refresh_logger.error(f'Failed to fix UnicodeEncodeError, skipping batch: {e2}')
|
|
with get_db_curs(commit=True) as curs:
|
|
curs.execute('UPDATE file_group SET "indexed" = 1 WHERE id IN '
|
|
'(SELECT value FROM json_each(?))',
|
|
(json.dumps([p['id'] for p in params]),))
|
|
|
|
if last_path:
|
|
refresh_logger.debug(f'Indexed {processed} files near {last_path}')
|
|
|
|
# Update progress
|
|
total_indexed += processed
|
|
if progress_callback and total_to_index > 0:
|
|
progress_callback(total_indexed, total_to_index)
|
|
|
|
if processed < 20:
|
|
# Processed less than the limit, don't do the next query.
|
|
break
|
|
|
|
|
|
def upsert_directories(parent_directories, directories):
|
|
"""
|
|
Insert/update/delete directories provided. Deletes all children of `parent_directories` which are not in
|
|
`directories`.
|
|
"""
|
|
idempotency = now()
|
|
directories = list(directories) + list(parent_directories)
|
|
|
|
# Only insert directories that are children of `media_directory` and exist.
|
|
directories = get_paths_in_media_directory([i for i in directories if i.is_dir()])
|
|
|
|
if directories:
|
|
# Insert any directories that were created, update any directories which previously existed.
|
|
# Chunk the rows so the multi-row VALUES clause stays under SQLite's bound-variable limit;
|
|
# a large media directory can hold tens of thousands of directories.
|
|
values = [(str(i.absolute()), i.name, idempotency) for i in directories]
|
|
with get_db_curs(commit=True) as curs:
|
|
for chunk in chunks(values, BULK_SQL_CHUNK_SIZE):
|
|
placeholders, params = values_clause(chunk)
|
|
stmt = f'''
|
|
INSERT INTO directory (path, name, idempotency) VALUES {placeholders}
|
|
ON CONFLICT (path) DO UPDATE
|
|
SET idempotency = EXCLUDED.idempotency
|
|
'''
|
|
curs.execute(stmt, params)
|
|
|
|
with get_db_curs(commit=True) as curs:
|
|
# Delete the children of any parent directory which no longer exists.
|
|
for directory in parent_directories:
|
|
stmt = f'DELETE FROM directory WHERE path LIKE :directory AND idempotency != :idempotency'
|
|
curs.execute(stmt, dict(directory=f'{directory.absolute()}%', idempotency=idempotency))
|
|
|
|
|
|
async def search_directories_by_name(session: Session, name: str, excluded: List[str] = None, limit: int = 20) \
|
|
-> List[Directory]:
|
|
"""Find the Directories whose names contain the `name` string."""
|
|
excluded = excluded or []
|
|
directories = session.query(Directory) \
|
|
.filter(Directory.name.ilike(f'%{name}%')) \
|
|
.filter(Directory.path.notin_(excluded)) \
|
|
.order_by(asc(Directory.name)) \
|
|
.limit(limit).all()
|
|
return directories
|
|
|
|
|
|
# Pagination totals: counting a large library is a full index walk that costs seconds cold on a
|
|
# spinning disk. Pagination can tolerate a total that is a minute stale. Bypassed under pytest
|
|
# so tests see exact totals. Keyed by filters, never by offset/limit/order (except filters that
|
|
# an order implies, e.g. viewed IS NOT NULL).
|
|
SEARCH_TOTALS_CACHE_SECONDS = 60
|
|
_SEARCH_TOTALS_CACHE: Dict[str, Tuple[float, int]] = dict()
|
|
|
|
|
|
def cached_search_total(key: str, compute: Callable[[], int]) -> int:
|
|
"""Return `compute()`; cache the result briefly unless running under pytest."""
|
|
if not PYTEST:
|
|
hit = _SEARCH_TOTALS_CACHE.get(key)
|
|
if hit and time.monotonic() - hit[0] < SEARCH_TOTALS_CACHE_SECONDS:
|
|
return hit[1]
|
|
total = compute()
|
|
_SEARCH_TOTALS_CACHE[key] = (time.monotonic(), total)
|
|
return total
|
|
|
|
|
|
def search_filter_cache_key(corpus: str, **filters) -> str:
|
|
"""Stable cache key for a search total. Omits empty/false filters."""
|
|
payload = {'corpus': corpus}
|
|
for k, v in filters.items():
|
|
if v in (None, '', [], (), False):
|
|
continue
|
|
if isinstance(v, (list, tuple, set)):
|
|
v = sorted(v)
|
|
payload[k] = v
|
|
return json.dumps(payload, sort_keys=True, default=str)
|
|
|
|
|
|
def count_file_groups(session: Session, statement: str, params: dict) -> int:
|
|
"""Run a `SELECT COUNT(*) AS total ...` statement and return the integer."""
|
|
with session_curs(session) as curs:
|
|
curs.execute(statement, params)
|
|
row = curs.fetchone()
|
|
return int(row['total'])
|
|
|
|
|
|
def search_files(session: Session, search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
|
|
tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
|
|
from_year: int = None, to_year: int = None, any_tag: bool = False, order: str = None,
|
|
url: str = None, suffix: str = None, path: str = None, deep: bool = False,
|
|
in_progress: bool = False) -> \
|
|
Tuple[List[dict], int]:
|
|
"""Search the FileGroup table.
|
|
|
|
Order the returned Files by their rank if `search_str` is provided. Return all files if
|
|
`search_str` is empty.
|
|
|
|
@param any_tag: The file must have some tag.
|
|
@param to_year: The file must be published on or before this year.
|
|
@param from_year: The file must be published on or after this year.
|
|
@param search_str: Search the ts_vector of the file. Returns all files if this is empty.
|
|
@param limit: Return only this many files.
|
|
@param offset: Offset the query.
|
|
@param mimetypes: Only return files that match these mimetypes.
|
|
@param model: Only return files that match this model.
|
|
@param tag_names: A list of tag names.
|
|
@param headline: Includes FTS search headlines if True.
|
|
@param months: A list of integers representing the index of the month of the year, starting at 1.
|
|
@param order: Used to change results from most relevant to recently viewed.
|
|
@param url: Filter by URL using case-insensitive partial match (ILIKE).
|
|
@param suffix: Only return files whose primary file has this suffix (e.g. ".bin"), case-insensitive.
|
|
@param path: Filter by primary_path using case-insensitive partial match (ILIKE).
|
|
@param deep: Search all text including captions/body (d_text); slower but more thorough.
|
|
@param in_progress: Only files the user is part way through (see wrolpi.files.progress).
|
|
"""
|
|
from wrolpi import fts
|
|
|
|
# The frontend may send explicit JSON nulls (bypassing the schema defaults); SQLite rejects
|
|
# `LIMIT NULL`/`OFFSET NULL` (Postgres treated them as no-ops).
|
|
limit = int(limit) if limit else 20
|
|
offset = int(offset) if offset else 0
|
|
params = dict(offset=offset, limit=limit, search_str=search_str)
|
|
wheres = []
|
|
selects = []
|
|
order_by = '1 ASC'
|
|
joins = []
|
|
|
|
# The fast (non-deep) path excludes d_text (captions/body) from matching.
|
|
fts_search = fts.file_group_search_join(search_str, deep=deep, headlines=bool(headline)) \
|
|
if search_str else None
|
|
if search_str and not fts_search:
|
|
# Nothing usable to search (punctuation-only or all-negated query); match nothing, like
|
|
# an empty websearch_to_tsquery did.
|
|
return [], 0
|
|
if fts_search:
|
|
joins.append(fts_search.join)
|
|
selects.append(fts_search.rank_select)
|
|
params.update(fts_search.params)
|
|
order_by = '2 DESC'
|
|
|
|
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
|
|
if model:
|
|
params['model'] = model
|
|
wheres.append('model = :model')
|
|
|
|
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
|
|
wheres, params = months_selector_to_where(wheres, params, months)
|
|
wheres, params = date_range_to_where(wheres, params, from_year, to_year)
|
|
|
|
if url:
|
|
# SQLite LIKE is case-insensitive (for ASCII), matching the old ILIKE behavior.
|
|
params['url_filter'] = f'%{url}%'
|
|
wheres.append('fg.url LIKE :url_filter')
|
|
|
|
if suffix:
|
|
# Suffix is stored lowercased with a leading dot; normalize the caller's value to match the index.
|
|
normalized = suffix if suffix.startswith('.') else f'.{suffix}'
|
|
params['suffix'] = normalized.lower()
|
|
wheres.append('fg.suffix = :suffix')
|
|
|
|
if path:
|
|
params['path_filter'] = f'%{path}%'
|
|
wheres.append('fg.primary_path LIKE :path_filter')
|
|
|
|
if in_progress:
|
|
wheres.append('fg.progress > 0 AND fg.progress < 1')
|
|
|
|
if fts_search and headline:
|
|
# b/c/d headlines come from FTS5 snippets (computed in the fts subquery); the title
|
|
# headline is computed in Python by `handle_file_group_search_results` (title is not an
|
|
# FTS column).
|
|
headline = ', fts.b_headline, fts.c_headline, fts.d_headline'
|
|
else:
|
|
headline = ''
|
|
|
|
if order == 'viewed':
|
|
order_by = 'viewed DESC NULLS LAST, 1 ASC'
|
|
elif order == '-viewed':
|
|
order_by = 'viewed ASC NULLS LAST, 1 ASC'
|
|
|
|
viewed_only = bool(order and not search_str)
|
|
if viewed_only:
|
|
# Only filter out unviewed files if search_str is not provided.
|
|
wheres.append('viewed IS NOT NULL')
|
|
|
|
where_sql = ('WHERE\n' + '\n AND '.join(wheres)) if wheres else ''
|
|
extra_selects = f', {", ".join(selects)}' if selects else ''
|
|
join = '\n'.join(joins)
|
|
stmt = f'''
|
|
SELECT fg.id{extra_selects}
|
|
{headline}
|
|
FROM file_group fg
|
|
{join}
|
|
{where_sql}
|
|
ORDER BY {order_by}
|
|
LIMIT :limit OFFSET :offset
|
|
'''
|
|
logger.debug(stmt)
|
|
|
|
# COUNT(*) OVER() in the page query forces SQLite to visit every match before LIMIT.
|
|
# Count separately (and cache) so the page can use the datetime/id index.
|
|
if not search_str and not tag_names and not mimetypes and not model and not months \
|
|
and not from_year and not to_year and not any_tag and not url and not suffix \
|
|
and not path and not viewed_only and not in_progress:
|
|
count_stmt = 'SELECT COUNT(*) AS total FROM file_group'
|
|
count_params = dict()
|
|
elif fts_search:
|
|
# Count via a direct FTS join (no ranking) rather than the snippet subquery.
|
|
fts_count = fts.file_group_search(search_str, deep=deep)
|
|
count_wheres = list(wheres)
|
|
if fts_count:
|
|
count_wheres.append(fts_count.where)
|
|
count_where_sql = ('WHERE\n' + '\n AND '.join(count_wheres)) if count_wheres else ''
|
|
count_join = fts_count.join if fts_count else ''
|
|
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {count_join} {count_where_sql}'
|
|
count_params = dict(params)
|
|
if fts_count:
|
|
count_params.update(fts_count.params)
|
|
else:
|
|
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {join} {where_sql}'
|
|
count_params = params
|
|
|
|
cache_key = search_filter_cache_key(
|
|
'files', search_str=search_str, mimetypes=mimetypes, model=model,
|
|
tag_names=tag_names, months=months, from_year=from_year, to_year=to_year,
|
|
any_tag=any_tag, url=url, suffix=suffix, path=path, deep=deep,
|
|
viewed_only=viewed_only, in_progress=in_progress,
|
|
)
|
|
total = cached_search_total(cache_key, lambda: count_file_groups(session, count_stmt, count_params))
|
|
results, total = handle_file_group_search_results(session, stmt, params, total=total)
|
|
return results, total
|
|
|
|
|
|
def handle_file_group_search_results(session: Session, statement: str, params: dict, total: int = None) \
|
|
-> Tuple[List[dict], int]:
|
|
"""
|
|
Execute the provided SQL statement and fetch the Files returned.
|
|
|
|
The statement must select `id`. Pass `total` from a separate COUNT (callers should not use
|
|
`COUNT(*) OVER()` in the page query; it prevents SQLite from applying LIMIT via an index).
|
|
|
|
WARNING: This expects specific queries to be executed and shouldn't be used for things not related to file search.
|
|
|
|
See: `search`
|
|
"""
|
|
with session_curs(session) as curs:
|
|
curs.execute(statement, params)
|
|
results = [dict(i) for i in curs.fetchall()]
|
|
if total is None:
|
|
total = results[0]['total'] if results else 0
|
|
ordered_ids = [i['id'] for i in results]
|
|
extras = [dict(
|
|
ts_rank=i.get('ts_rank'),
|
|
# Snippet columns exist only when headlines were requested; `title_headline` is
|
|
# computed below (title is not an FTS5 column).
|
|
headlines=('b_headline' in i),
|
|
title_headline=i.get('title_headline'),
|
|
b_headline=i.get('b_headline'),
|
|
c_headline=i.get('c_headline'),
|
|
d_headline=i.get('d_headline'),
|
|
) for i in results]
|
|
|
|
from modules.archive.models import Archive
|
|
from modules.videos.models import Video
|
|
# Serialized below: what Video's and Archive's `__json__` read beyond the FileGroup (see their `json_options`).
|
|
results = session.query(FileGroup, Video, Archive) \
|
|
.options(*FileGroup.json_options(), undefer(Video.ffprobe_json), joinedload(Video.channel),
|
|
joinedload(Archive.collection)) \
|
|
.filter(FileGroup.id.in_(ordered_ids)) \
|
|
.outerjoin(Video, Video.file_group_id == FileGroup.id) \
|
|
.outerjoin(Archive, Archive.file_group_id == FileGroup.id)
|
|
# Order FileGroups by their location in ordered_ids. Use dict for O(1) lookups instead of O(n) list.index().
|
|
id_to_order = {id_: idx for idx, id_ in enumerate(ordered_ids)}
|
|
file_groups = sorted(results, key=lambda i: id_to_order.get(i[0].id, 0))
|
|
results = list()
|
|
for extra, (file_group, video, archive) in zip_longest(extras, file_groups):
|
|
if video:
|
|
results.append(video.__json__())
|
|
elif archive:
|
|
results.append(archive.__json__())
|
|
else:
|
|
results.append(file_group.__json__())
|
|
# Preserve the ts_ranks, if any.
|
|
if extra:
|
|
results[-1]['ts_rank'] = extra['ts_rank']
|
|
results[-1]['title_headline'] = extra['title_headline']
|
|
results[-1]['b_headline'] = extra['b_headline']
|
|
results[-1]['c_headline'] = extra['c_headline']
|
|
results[-1]['d_headline'] = extra['d_headline']
|
|
|
|
search_str = params.get('search_str') if isinstance(params, dict) else None
|
|
|
|
# When headlines were requested, compute the title headlines in Python (the FTS5 snippet
|
|
# function only works on indexed columns; `title` is not one).
|
|
if search_str and any(extra and extra.get('headlines') for extra in extras):
|
|
from wrolpi import fts
|
|
titles = [result.get('title') or '' for result in results]
|
|
for result, (headline_text, rank) in zip(results, fts.headline_texts(titles, search_str)):
|
|
if rank > 0:
|
|
result['title_headline'] = headline_text
|
|
|
|
# When searching by text, attach a `section_hint` to any doc-modeled result so
|
|
# the UI can deep-link into the matching EPUB chapter or PDF page.
|
|
if search_str:
|
|
from modules.docs.lib import _fetch_section_hints
|
|
# `r['id']` is the FileGroup primary key; _fetch_section_hints filters
|
|
# on `file_group_id`, so this list is file-group ids (not doc.id).
|
|
fg_ids = [r['id'] for r in results if r.get('model') == 'doc']
|
|
if fg_ids:
|
|
hints = _fetch_section_hints(session, fg_ids, search_str)
|
|
for r in results:
|
|
hint = hints.get(r['id'])
|
|
if hint:
|
|
r['section_hint'] = hint
|
|
|
|
return results, total
|
|
|
|
|
|
def glob_shared_stem(path: pathlib.Path) -> List[pathlib.Path]:
|
|
"""Return all paths that share the same stem and parent path as the provided path.
|
|
|
|
Example paths: foo.mp4, foo.png, foo.info.json, foobar.txt
|
|
>>> glob_shared_stem(pathlib.Path('foo.mp4'))
|
|
['foo.mp4', 'foo.png', 'foo.info.json']
|
|
"""
|
|
path = path if isinstance(path, pathlib.Path) else pathlib.Path(path)
|
|
|
|
stem, suffix = split_path_stem_and_suffix(path)
|
|
escaped_stem = glob.escape(stem)
|
|
paths = [pathlib.Path(i) for i in path.parent.glob(f'{escaped_stem}*')
|
|
if i == path or i.name.startswith(f'{stem}.')]
|
|
return paths
|
|
|
|
|
|
def get_matching_directories(path: Union[str, Path]) -> List[pathlib.Path]:
|
|
"""
|
|
Return a list of directory strings that start with the provided path. If the path is a directory, return its
|
|
subdirectories, if the directory contains no subdirectories, return the directory.
|
|
"""
|
|
path: pathlib.Path = pathlib.Path(path) if isinstance(path, str) else path
|
|
path = path.resolve()
|
|
|
|
ignored_directories = {pathlib.Path(i).resolve() for i in get_wrolpi_config().ignored_directories}
|
|
|
|
if path.is_dir():
|
|
# The provided path is a directory, return its subdirectories, or itself if no subdirectories exist
|
|
paths = list(path.iterdir())
|
|
paths = sorted(i for i in paths if i.is_dir() and i not in ignored_directories)
|
|
if len(paths) == 0:
|
|
return [get_real_path_name(path.resolve())]
|
|
return paths
|
|
|
|
paths = path.parent.iterdir()
|
|
prefix = path.name.lower()
|
|
paths = sorted(
|
|
i for i in paths if i.is_dir() and i.name.lower().startswith(prefix) and i.resolve() not in ignored_directories
|
|
)
|
|
paths = unique_by_predicate(paths, None)
|
|
|
|
return paths
|
|
|
|
|
|
WHITESPACE = re.compile(r'[\s_\[\]()]')
|
|
|
|
|
|
def split_file_name_words(name: str) -> str:
|
|
"""Split words in a filename.
|
|
|
|
Words are assumed to be separated by underscore, dash, or space. Words with a dash are included as a group, and
|
|
individually ('self-reliance' -> ['self', 'reliance', 'self-reliance']).
|
|
|
|
>>> split_file_name_words('this self-reliance_split.txt')
|
|
['this', 'self', 'reliance', 'self-reliance', 'split', 'txt']
|
|
"""
|
|
if not name:
|
|
raise ValueError(f'Invalid filename: {name}')
|
|
|
|
try:
|
|
stem, suffix = split_path_stem_and_suffix(name)
|
|
words = []
|
|
for word1 in WHITESPACE.split(stem):
|
|
if '-' in word1[1:]:
|
|
words.extend(word1.split('-'))
|
|
words.append(word1)
|
|
# Include the suffix so the user can search without the "."
|
|
if suffix:
|
|
words.append(suffix.lstrip('.'))
|
|
|
|
words = ' '.join(i for i in words if i)
|
|
return words
|
|
except Exception as e:
|
|
logger.error(f'Failed to split filename into words: {name}', exc_info=e)
|
|
return name
|
|
|
|
|
|
def get_file_statistics(session: Session):
|
|
with session_curs(session) as curs:
|
|
curs.execute('''
|
|
SELECT
|
|
-- All items in file_group.files are real individual files.
|
|
SUM(json_array_length(files)) AS "total_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype =
|
|
'application/x-mobipocket-ebook') AS "ebook_count",
|
|
(SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files",
|
|
(SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims",
|
|
(SELECT COUNT(*) FROM tag) AS "tags_count",
|
|
SUM(size) AS "total_size",
|
|
(SELECT COUNT(*) FROM archive) AS archive_count
|
|
FROM file_group
|
|
''')
|
|
statistics = dict(curs.fetchall()[0])
|
|
statistics['total_count'] = statistics['total_count'] or 0
|
|
statistics['total_size'] = statistics['total_size'] or 0
|
|
|
|
return statistics
|
|
|
|
|
|
def group_files_by_stem(files: Iterable[pathlib.Path]) -> Generator[List[pathlib.Path], None, None]:
|
|
"""
|
|
Return lists of paths, each list contains only paths that share a common directory and stem.
|
|
|
|
Groups are ordered by (directory, stem), and each group is sorted.
|
|
|
|
>>> a = ['foo.txt', 'foo.mp4', 'bar.txt']
|
|
>>> group_files_by_stem(list(map((pathlib.Path, a))))
|
|
# Generator(['bar.txt'], ['foo.mp4', 'foo.txt'])
|
|
"""
|
|
# Sorted paths do not keep a stem's files together: `foo.tar.gz` (stem `foo.tar`) sorts between `foo.jpg` and
|
|
# `foo.txt` (stem `foo`). Group by (directory, stem) so each stem yields exactly one group.
|
|
groups = dict()
|
|
for file in files:
|
|
file = pathlib.Path(file)
|
|
key = (str(file.parent), split_path_stem_and_suffix(file)[0])
|
|
groups.setdefault(key, set()).add(file)
|
|
for key in sorted(groups):
|
|
yield sorted(groups[key])
|
|
|
|
|
|
async def _get_tag_and_file_group(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
|
|
tag_id: int):
|
|
if file_group_id:
|
|
file_group: FileGroup = session.query(FileGroup).filter_by(id=file_group_id).one_or_none()
|
|
if not file_group:
|
|
raise UnknownFile(f'Cannot find FileGroup with id {file_group_id}')
|
|
elif file_group_primary_path:
|
|
path = get_media_directory() / file_group_primary_path
|
|
|
|
file_group = FileGroup.get_by_path(session, path)
|
|
# Create the FileGroup, if possible.
|
|
if not file_group and path.is_file():
|
|
file_group = FileGroup.from_paths(session, path)
|
|
|
|
if not file_group:
|
|
raise UnknownFile(f'Cannot find FileGroup with primary_path {repr(str(file_group_primary_path))}')
|
|
else:
|
|
raise UnknownFile(f'Cannot find FileGroup without id or primary_path')
|
|
|
|
if tag_id:
|
|
tag: Tag = session.query(Tag).filter_by(id=tag_id).one_or_none()
|
|
if not tag:
|
|
raise UnknownTag(f'Cannot find Tag with id {tag_id}')
|
|
elif tag_name:
|
|
tag: Tag = Tag.get_by_name(session, tag_name)
|
|
if not tag:
|
|
raise UnknownTag(f'Cannot find Tag with name {tag_name}')
|
|
else:
|
|
raise UnknownTag('Cannot find Tag without id or name')
|
|
|
|
return file_group, tag
|
|
|
|
|
|
async def add_file_group_tag(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
|
|
tag_id: int) -> TagFile:
|
|
file_group, tag = await _get_tag_and_file_group(session, file_group_id, file_group_primary_path, tag_name, tag_id)
|
|
tag_file = file_group.add_tag(session, tag.id)
|
|
return tag_file
|
|
|
|
|
|
async def remove_file_group_tag(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
|
|
tag_id: int):
|
|
file_group, tag = await _get_tag_and_file_group(session, file_group_id, file_group_primary_path, tag_name, tag_id)
|
|
file_group.untag(session, tag.id)
|
|
|
|
|
|
def mimetypes_to_sql_wheres(wheres: List[str], params: dict, mimetypes: List[str]) -> Tuple[List[str], dict]:
|
|
if mimetypes:
|
|
local_wheres = list()
|
|
for idx, mimetype in enumerate(mimetypes):
|
|
key = f'mimetype{idx}'
|
|
if len(mimetype.split('/')) == 1:
|
|
params[key] = f'{mimetype}/%'
|
|
else:
|
|
params[key] = f'{mimetype}%'
|
|
local_wheres.append(f'mimetype LIKE :{key}')
|
|
wheres.append(f'({" OR ".join(local_wheres)})')
|
|
return wheres, params
|
|
|
|
|
|
async def search_file_suggestion_count(session: Session, search_str: str, tag_names: List[str], mimetypes: List[str],
|
|
months: List[int] = None, from_year: int = None, to_year: int = None,
|
|
any_tag: bool = False) -> dict:
|
|
"""
|
|
Return FileGroup counts of what will be returned if the search is actually performed.
|
|
|
|
Returns both the fast (a/b/c columns) and deep (all columns) counts so the UI can show the count
|
|
matching the current search mode, and offer deep search when it has more results. Both are cheap
|
|
index-bound COUNTs (no ranking).
|
|
"""
|
|
from wrolpi import fts
|
|
|
|
wheres = []
|
|
joins = list()
|
|
params = dict()
|
|
|
|
# The deep match is a superset of the fast (column-filtered) match; both are counted with the
|
|
# same statement, differing only in the MATCH expression.
|
|
deep_match = fts.translate_websearch(search_str) if search_str else None
|
|
fast_match = fts.translate_websearch(search_str, fts.ABC_COLUMNS) if search_str else None
|
|
if search_str and deep_match is None:
|
|
# Nothing usable to search; match nothing (like an empty websearch_to_tsquery).
|
|
return dict(file_groups=0, file_groups_deep=0)
|
|
if deep_match:
|
|
joins.append('JOIN file_group_fts fts ON fts.rowid = fg.id')
|
|
wheres.append('fts.file_group_fts MATCH :fts_match')
|
|
|
|
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
|
|
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
|
|
wheres, params = months_selector_to_where(wheres, params, months)
|
|
wheres, params = date_range_to_where(wheres, params, from_year, to_year)
|
|
|
|
joins = "\n".join(joins)
|
|
wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
|
|
stmt = f'''
|
|
SELECT COUNT(*) AS estimate
|
|
FROM file_group fg
|
|
{joins}
|
|
{wheres}
|
|
'''
|
|
logger.debug(f'search_file_suggestion_count: {stmt} {params}')
|
|
|
|
with session_curs(session) as curs:
|
|
if deep_match:
|
|
curs.execute(stmt, {**params, 'fts_match': deep_match})
|
|
deep_estimate = int(curs.fetchone()['estimate'])
|
|
curs.execute(stmt, {**params, 'fts_match': fast_match})
|
|
estimate = int(curs.fetchone()['estimate'])
|
|
else:
|
|
curs.execute(stmt, params)
|
|
estimate = deep_estimate = int(curs.fetchone()['estimate'])
|
|
return dict(file_groups=estimate, file_groups_deep=deep_estimate)
|
|
|
|
|
|
MOVE_CHUNK_SIZE = 100
|
|
|
|
|
|
def _move_file_group_files(file_group: FileGroup, new_primary_path: pathlib.Path) -> None:
|
|
"""Move all physical files of a FileGroup to a new directory.
|
|
|
|
For directory moves where filenames are preserved, this moves each file from
|
|
the old directory to the new directory keeping the same filename.
|
|
|
|
Args:
|
|
file_group: The FileGroup whose files should be moved
|
|
new_primary_path: The new path for the primary file
|
|
"""
|
|
new_directory = new_primary_path.parent
|
|
new_stem, _ = split_path_stem_and_suffix(new_primary_path, full=True)
|
|
|
|
# Move each file in the FileGroup
|
|
for file_info in file_group.files:
|
|
relative_name = file_info['path']
|
|
old_path = file_group.directory / relative_name
|
|
# Calculate new path: new_stem + original suffix
|
|
_, suffix = split_path_stem_and_suffix(relative_name)
|
|
new_path = pathlib.Path(f'{new_stem}{suffix}')
|
|
|
|
if old_path.is_file():
|
|
shutil.move(old_path, new_path)
|
|
logger.debug(f'Moved file: {old_path} -> {new_path}')
|
|
|
|
|
|
def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path, pathlib.Path]):
|
|
"""Batch UPDATE FileGroups using raw SQL after physical files have been moved.
|
|
|
|
Only updates directory and primary_path. Other fields (files, title, a_text) remain unchanged
|
|
because filenames are preserved in directory moves - only the parent directory changes.
|
|
|
|
Args:
|
|
session: Database session
|
|
chunk_plan: Mapping of old_primary_path -> new_primary_path
|
|
"""
|
|
if not chunk_plan:
|
|
return
|
|
|
|
# Build the update rows: [old_path, new_dir, new_path]
|
|
updates = json.dumps([
|
|
[str(old_path), str(new_path.parent), str(new_path)]
|
|
for old_path, new_path in chunk_plan.items()
|
|
])
|
|
|
|
# Use UPDATE FROM json_each pattern for efficient batch update. This runs on the caller's
|
|
# session (rather than a second connection) so it shares the move's transaction: a second
|
|
# connection would deadlock against the write lock the caller already holds, and its separate
|
|
# commit would survive a rollback of the rest of the move.
|
|
sql = '''
|
|
UPDATE file_group AS fg SET
|
|
directory = json_extract(v.value, '$[1]'),
|
|
primary_path = json_extract(v.value, '$[2]'),
|
|
indexed = CASE WHEN fg.data IS NULL THEN FALSE ELSE fg.indexed END
|
|
FROM json_each(:updates) AS v
|
|
WHERE fg.primary_path = json_extract(v.value, '$[0]')
|
|
'''
|
|
session.execute(sa_text(sql), dict(updates=updates))
|
|
|
|
from wrolpi.collections.config import file_groups_moved
|
|
new_paths = [str(new_path) for new_path in chunk_plan.values()]
|
|
file_groups_moved(session, session.query(FileGroup.id).filter(FileGroup.primary_path.in_(new_paths)))
|
|
|
|
|
|
def _json_serial(obj):
|
|
"""JSON serializer for objects not serializable by default json code."""
|
|
import datetime
|
|
if isinstance(obj, (datetime.datetime, datetime.date)):
|
|
return obj.isoformat()
|
|
raise TypeError(f"Type {type(obj)} not serializable")
|
|
|
|
|
|
def _bulk_update_file_groups_reorganize(updates: List[dict]):
|
|
"""Batch UPDATE FileGroups for reorganization using raw SQL.
|
|
|
|
Unlike _bulk_update_file_groups_db, this also updates the files and data JSON fields
|
|
since reorganization changes filenames, not just directories.
|
|
|
|
Args:
|
|
updates: List of dicts with keys: id, directory, primary_path, files, data
|
|
"""
|
|
if not updates:
|
|
return
|
|
|
|
with get_db_curs(commit=True) as curs:
|
|
for update in updates:
|
|
# Don't change indexed - reorganization only moves files, content unchanged
|
|
primary_path = update['primary_path']
|
|
stem, _ = split_path_stem_and_suffix(primary_path)
|
|
curs.execute('''
|
|
UPDATE file_group
|
|
SET directory = ?,
|
|
primary_path = ?,
|
|
files = ?,
|
|
data = ?,
|
|
stem = ?
|
|
WHERE id = ?
|
|
''', (
|
|
update['directory'],
|
|
primary_path,
|
|
json.dumps(update['files'], default=_json_serial),
|
|
json.dumps(update['data'], default=_json_serial) if update.get('data') else None,
|
|
stem,
|
|
update['id'],
|
|
))
|
|
|
|
# Expire SQLAlchemy's session cache so subsequent queries see the raw SQL updates.
|
|
# This is necessary because get_db_curs uses the same connection as get_db_session
|
|
# during tests, but SQLAlchemy's identity map doesn't know about raw SQL updates.
|
|
with get_db_session() as session:
|
|
session.expire_all()
|
|
from wrolpi.collections.config import file_groups_moved
|
|
file_groups_moved(session, [update['id'] for update in updates])
|
|
|
|
|
|
def delete_directory(directory: pathlib.Path, recursive: bool = False, force: bool = False):
|
|
"""Remove a directory, remove it's Directory record.
|
|
|
|
Will refuse to delete a directory if it contains Tagged Files, unless force is True.
|
|
|
|
This function is idempotent - safe to call even if the directory was already deleted."""
|
|
if directory.exists():
|
|
if recursive:
|
|
if not force:
|
|
with get_db_session() as session:
|
|
tagged = get_tagged_file_groups_by_paths(session, [directory])
|
|
if tagged:
|
|
raise FileGroupIsTagged(f'Cannot delete {tagged[0]} because it is tagged')
|
|
shutil.rmtree(directory)
|
|
else:
|
|
directory.rmdir()
|
|
# Always clean up DB record even if directory was already deleted
|
|
with get_db_session(commit=True) as session:
|
|
session.query(Directory).filter_by(path=str(directory)).delete(synchronize_session=False)
|
|
|
|
|
|
async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
|
|
"""Rename a file (and it's associated files). Preserve any tags.
|
|
|
|
If the path is a non-primary file in a FileGroup (e.g., a poster or subtitle),
|
|
the entire FileGroup will be renamed.
|
|
"""
|
|
new_path = path.with_name(new_name)
|
|
if not path.exists():
|
|
raise FileNotFoundError(f'Cannot find {path} to rename')
|
|
if new_path.exists():
|
|
raise FileConflict(f'Cannot rename {path} because {new_path} already exists')
|
|
|
|
with get_db_session(commit=True) as session:
|
|
# Find the FileGroup - it may be the primary path or a secondary file
|
|
fg: FileGroup = FileGroup.get_by_any_file_path(session, path)
|
|
if not fg:
|
|
# File wasn't yet in the DB.
|
|
fg = FileGroup.from_paths(session, path)
|
|
|
|
# If this is not the primary file, calculate the new primary path
|
|
if fg.primary_path != path:
|
|
# Extract the new stem and apply it to the primary file's suffix
|
|
new_stem, _ = split_path_stem_and_suffix(new_path, full=True)
|
|
_, primary_suffix = split_path_stem_and_suffix(fg.primary_path, full=False)
|
|
new_primary_path = pathlib.Path(f'{new_stem}{primary_suffix}')
|
|
fg.move(new_primary_path)
|
|
else:
|
|
fg.move(new_path)
|
|
|
|
from wrolpi.collections.config import file_groups_moved
|
|
file_groups_moved(session, [fg.id])
|
|
|
|
return new_path
|
|
|
|
|
|
async def rename_directory(session: Session, directory: pathlib.Path, new_name: str, send_events: bool = False,
|
|
write: Callable[[], ContextManager[Session]] = None) -> str | None:
|
|
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory.
|
|
|
|
A Collection's directory is renamed before this returns (returns None). Any other directory is renamed by a queued
|
|
file worker move, which this does not wait for; returns the move's job id.
|
|
|
|
`session` is used to read. Collections are changed in `write()` (`request.ctx.db.write` in a request), which
|
|
begins as a writer; the files are moved outside it. Without `write`, Collections are changed in `session`."""
|
|
from wrolpi.files.worker import file_worker
|
|
|
|
write = write or (lambda: contextlib.nullcontext(session))
|
|
|
|
new_directory = directory.with_name(new_name)
|
|
if new_directory.exists():
|
|
raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.')
|
|
|
|
# Check if this directory is a Collection's directory
|
|
from wrolpi.collections.models import Collection
|
|
collection: Collection = session.query(Collection).filter(Collection.directory == directory).one_or_none()
|
|
|
|
if collection:
|
|
# Use Collection's move method which handles files, config saves, and cleanup
|
|
new_directory.mkdir(parents=True, exist_ok=True)
|
|
# move_collection commits its Download changes before it moves the files, so the write lock is not held
|
|
# for the move.
|
|
with write() as session:
|
|
await collection.move_collection(new_directory, session, send_events=send_events)
|
|
return None
|
|
|
|
# Regular directory rename: the file worker moves the contents, updates nested Collections, and removes the old
|
|
# directory. This does not wait; a large rename outlasts a request, and the job reports its own progress and
|
|
# result.
|
|
return file_worker.queue_move(new_directory, list(directory.iterdir()), rename_from=directory)
|
|
|
|
|
|
async def rename(session: Session, path: pathlib.Path, new_name: str, send_events: bool = False,
|
|
write: Callable[[], ContextManager[Session]] = None) -> str | None:
|
|
"""Rename a directory or file. Preserve any tags.
|
|
|
|
Returns the file worker job id when the rename was queued (see `rename_directory`), otherwise None; the rename is
|
|
done."""
|
|
if path.is_dir():
|
|
return await rename_directory(session, path, new_name, send_events=send_events, write=write)
|
|
|
|
await rename_file(path, new_name)
|
|
return None
|
|
|
|
|
|
def get_special_directories() -> List[pathlib.Path]:
|
|
"""Return absolute directories that must never be ignored by file refresh.
|
|
|
|
These are module destinations that depend on FileGroup indexing (or are the media root).
|
|
Map is intentionally excluded: it lists PMTiles by walking the filesystem and benefits
|
|
from being ignored during refresh (very large files).
|
|
|
|
Must not create directories — this runs during refresh ignore filtering and would
|
|
otherwise mkdir videos/ (and NO CHANNEL/) as a side effect of the videos helpers.
|
|
"""
|
|
from modules.archive.lib import get_archive_directory
|
|
from modules.videos.lib import format_videos_destination
|
|
|
|
media_directory = get_media_directory()
|
|
videos_directory = media_directory / format_videos_destination()
|
|
# Avoid importing modules.zim.lib here (circular import via files.worker).
|
|
zims_directory = media_directory / get_wrolpi_config().zims_destination
|
|
return [
|
|
media_directory.resolve(),
|
|
videos_directory.resolve(),
|
|
get_archive_directory().resolve(),
|
|
(videos_directory / 'NO CHANNEL').resolve(),
|
|
zims_directory.resolve(),
|
|
]
|
|
|
|
|
|
def get_normalized_ignored_directories() -> List[str]:
|
|
"""Return absolute ignored directory paths, excluding special directories.
|
|
|
|
Special directories (media root, videos, archives, zims, …) are never treated as ignored,
|
|
even if they appear in config from older installs or manual edits.
|
|
"""
|
|
media_directory = get_media_directory()
|
|
special = {str(p) for p in get_special_directories()}
|
|
result = []
|
|
for d in get_wrolpi_config().ignored_directories or []:
|
|
path = pathlib.Path(d)
|
|
absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve())
|
|
if absolute in special:
|
|
continue
|
|
result.append(absolute)
|
|
return result
|
|
|
|
|
|
def sanitize_ignored_directories():
|
|
"""Persistently remove special directories from the ignored list if present."""
|
|
wrolpi_config = get_wrolpi_config()
|
|
ignored = list(wrolpi_config.ignored_directories or [])
|
|
if not ignored:
|
|
return
|
|
|
|
media_directory = get_media_directory()
|
|
special = {str(p) for p in get_special_directories()}
|
|
cleaned = []
|
|
removed = False
|
|
for d in ignored:
|
|
path = pathlib.Path(d)
|
|
absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve())
|
|
if absolute in special:
|
|
removed = True
|
|
logger.warning(f'Removing special directory from ignored_directories: {d}')
|
|
continue
|
|
cleaned.append(d)
|
|
|
|
if removed:
|
|
wrolpi_config.ignored_directories = cleaned
|
|
|
|
|
|
def add_ignore_directory(directory: Union[pathlib.Path, str]):
|
|
"""Add a directory to the `ignored_directories` in the WROLPi config. This directory will be ignored when
|
|
refreshing."""
|
|
media_directory = get_media_directory().resolve()
|
|
|
|
directory = pathlib.Path(directory)
|
|
if not directory.is_absolute():
|
|
directory = media_directory / directory
|
|
directory = directory.resolve()
|
|
|
|
# Drop any special directories that were previously saved as ignored.
|
|
sanitize_ignored_directories()
|
|
|
|
special_directories = set(get_special_directories())
|
|
if directory in special_directories:
|
|
raise InvalidDirectory('Refusing to ignore special directory')
|
|
|
|
if not str(directory).startswith(str(media_directory)):
|
|
raise InvalidDirectory('Cannot ignore directory not in media directory')
|
|
|
|
wrolpi_config = get_wrolpi_config()
|
|
ignored_directories = wrolpi_config.ignored_directories if wrolpi_config.ignored_directories else list()
|
|
|
|
# Accept either absolute or relative forms already present in config.
|
|
relative = str(get_relative_to_media_directory(directory))
|
|
if str(directory) in ignored_directories or relative in ignored_directories:
|
|
logger.warning(f'Directory is already ignored: {directory}')
|
|
return
|
|
|
|
ignored_directories.append(str(directory))
|
|
wrolpi_config.ignored_directories = ignored_directories
|
|
|
|
|
|
def remove_ignored_directory(directory: Union[pathlib.Path, str]):
|
|
"""Remove a directory from the `ignored_directories` in the WROLPi config."""
|
|
media_directory = get_media_directory()
|
|
path = pathlib.Path(directory)
|
|
if not path.is_absolute():
|
|
path = media_directory / path
|
|
path = path.resolve()
|
|
absolute = str(path)
|
|
|
|
ignored_directories = list(get_wrolpi_config().ignored_directories or [])
|
|
kept = []
|
|
removed = False
|
|
for d in ignored_directories:
|
|
entry = pathlib.Path(d)
|
|
entry_abs = str((media_directory / entry).resolve() if not entry.is_absolute() else entry.resolve())
|
|
if entry_abs == absolute:
|
|
removed = True
|
|
continue
|
|
kept.append(d)
|
|
|
|
if not removed:
|
|
raise UnknownDirectory('Directory is not ignored')
|
|
get_wrolpi_config().ignored_directories = kept
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class UpsertedFile:
|
|
"""The FileGroup `upsert_file` created or updated. Plain values: its Session is gone once it returns."""
|
|
file_group_id: int
|
|
location: str
|
|
|
|
|
|
async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> UpsertedFile:
|
|
"""Insert/update all files in the provided file's FileGroup."""
|
|
if not file or not file.is_file():
|
|
raise InvalidFile(f'Cannot upsert file that does not exist: {file}')
|
|
|
|
# Update/Insert all files in the FileGroup.
|
|
paths = glob_shared_stem(pathlib.Path(file))
|
|
# Remove any ignored files.
|
|
paths = remove_files_in_ignored_directories(paths)
|
|
if not paths:
|
|
logger.warning('upsert_file called, but all files are in ignored directories!')
|
|
raise IgnoredDirectoryError(f'all files are in ignored directories: {file}')
|
|
|
|
# Use a single session context for all database operations
|
|
with get_db_session(commit=True) as session:
|
|
# Create/update FileGroup with retry logic
|
|
file_group = None
|
|
for i in range(2):
|
|
try:
|
|
file_group = FileGroup.from_paths(session, *paths)
|
|
session.flush([file_group])
|
|
break
|
|
except IntegrityError:
|
|
# Another process inserted this FileGroup.
|
|
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
|
|
session.rollback()
|
|
if i == 1:
|
|
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
|
|
continue
|
|
|
|
if not file_group:
|
|
raise RuntimeError(f'upsert_file failed to create FileGroup! {file}')
|
|
|
|
logger.debug(f'upsert_file: {file_group}')
|
|
|
|
# Model the FileGroup (creates Archive/Video/etc. and sets indexed=True)
|
|
try:
|
|
file_group.do_model(session)
|
|
except Exception as e:
|
|
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
|
|
if PYTEST:
|
|
raise
|
|
|
|
# If user uploads a file, then remove it from the download skip list so comments can be downloaded.
|
|
# Modify the download (if any) so that the user can click on it and view the uploaded file.
|
|
if file_group.url:
|
|
if download_manager.is_skipped(file_group.url):
|
|
download_manager.remove_from_skip_list(file_group.url)
|
|
if download := Download.get_by_url(session, file_group.url):
|
|
# Mark download as completed
|
|
download.complete()
|
|
# Link the download to the location of the uploaded file.
|
|
model = file_group.get_model_record(session)
|
|
download.location = model.location if model else file_group.location
|
|
|
|
# Add tags if provided
|
|
if tag_names:
|
|
for tag_name in tag_names:
|
|
if tag_name not in file_group.tag_names:
|
|
tag = Tag.get_by_name(session, tag_name)
|
|
file_group.add_tag(session, tag.id)
|
|
|
|
# Commit all changes in one transaction
|
|
session.commit()
|
|
upserted = UpsertedFile(file_group_id=file_group.id, location=file_group.location)
|
|
|
|
# Upsert directories after FileGroup is committed
|
|
upsert_directories([], file.parents)
|
|
|
|
return upserted
|
|
|
|
|
|
def get_file_location_href(file: pathlib.Path) -> str:
|
|
"""Return the location a file can be viewed at on the UI."""
|
|
parent = str(get_relative_to_media_directory(file.parent))
|
|
preview = str(get_relative_to_media_directory(file))
|
|
if parent == '.':
|
|
# File is in the top of the media directory, App already shows top directory open.
|
|
query = urllib.parse.urlencode(dict(preview=str(preview)))
|
|
else:
|
|
query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
|
|
return f'/files?{query}'
|
|
|
|
|
|
def get_real_path_name(path: pathlib.Path) -> pathlib.Path:
|
|
"""Return the real path name of a file. This is used to get the real path of a file on macOS."""
|
|
if IS_MACOS:
|
|
for path_ in path.parent.iterdir():
|
|
if path_.name.lower() == path.name.lower():
|
|
return path_
|
|
return path
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class BulkTagPreview:
|
|
"""Preview information for a bulk tagging operation."""
|
|
file_count: int = 0
|
|
shared_tag_names: List[str] = dataclasses.field(default_factory=list)
|
|
|
|
def __json__(self) -> dict:
|
|
return dict(
|
|
file_count=self.file_count,
|
|
shared_tag_names=self.shared_tag_names,
|
|
)
|
|
|
|
|
|
def get_bulk_tag_preview(session: Session, paths: List[str]) -> BulkTagPreview:
|
|
"""Get preview information for bulk tagging operation.
|
|
|
|
Returns the count of files that will be tagged and the tags that are shared by ALL files.
|
|
"""
|
|
media_directory = get_media_directory()
|
|
absolute_paths = [media_directory / p for p in paths]
|
|
|
|
# Collect all file paths (expanding directories recursively)
|
|
all_file_paths: List[pathlib.Path] = []
|
|
for path in absolute_paths:
|
|
if path.is_file():
|
|
all_file_paths.append(path)
|
|
elif path.is_dir():
|
|
for file in walk(path):
|
|
if file.is_file():
|
|
all_file_paths.append(file)
|
|
|
|
# Remove files in ignored directories
|
|
all_file_paths = remove_files_in_ignored_directories(all_file_paths)
|
|
|
|
if not all_file_paths:
|
|
return BulkTagPreview(file_count=0, shared_tag_names=[])
|
|
|
|
# Get unique files by stem (each FileGroup counted once)
|
|
unique_files = get_unique_files_by_stem(all_file_paths)
|
|
file_count = len(unique_files)
|
|
|
|
if file_count == 0:
|
|
return BulkTagPreview(file_count=0, shared_tag_names=[])
|
|
|
|
# Find tags shared by ALL files
|
|
# First, get all FileGroups and their tags
|
|
# We query by stem prefix because unique_files may contain non-primary files
|
|
# (e.g., .readability.json) but FileGroups are stored with primary_path (e.g., .html)
|
|
from sqlalchemy import or_
|
|
stems = [split_path_stem_and_suffix(f, full=True)[0] for f in unique_files]
|
|
# Chunk the OR'd LIKEs to stay within SQLite's expression-tree depth limit of 1000.
|
|
file_groups = []
|
|
for idx in range(0, len(stems), 500):
|
|
chunk = stems[idx:idx + 500]
|
|
file_groups.extend(session.query(FileGroup).filter(
|
|
or_(*[FileGroup.primary_path.like(f'{stem}%') for stem in chunk])
|
|
).all())
|
|
|
|
if not file_groups:
|
|
# No FileGroups exist yet, no shared tags
|
|
return BulkTagPreview(file_count=file_count, shared_tag_names=[])
|
|
|
|
# Get tag names for each FileGroup
|
|
tag_sets = []
|
|
for fg in file_groups:
|
|
tag_sets.append(set(fg.tag_names))
|
|
|
|
# Find intersection of all tag sets (tags shared by ALL files)
|
|
if tag_sets:
|
|
shared_tags = tag_sets[0]
|
|
for tag_set in tag_sets[1:]:
|
|
shared_tags = shared_tags.intersection(tag_set)
|
|
shared_tag_names = sorted(list(shared_tags))
|
|
else:
|
|
shared_tag_names = []
|
|
|
|
return BulkTagPreview(file_count=file_count, shared_tag_names=shared_tag_names)
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class BulkTagProgress:
|
|
"""Progress information for bulk tagging operation."""
|
|
status: str = 'idle' # 'idle', 'running'
|
|
total: int = 0
|
|
completed: int = 0
|
|
add_tag_names: List[str] = dataclasses.field(default_factory=list)
|
|
remove_tag_names: List[str] = dataclasses.field(default_factory=list)
|
|
error: str = None
|
|
queued_jobs: int = 0
|
|
|
|
def __json__(self) -> dict:
|
|
return dict(
|
|
status=self.status,
|
|
total=self.total,
|
|
completed=self.completed,
|
|
add_tag_names=self.add_tag_names,
|
|
remove_tag_names=self.remove_tag_names,
|
|
error=self.error,
|
|
queued_jobs=self.queued_jobs,
|
|
)
|
|
|
|
|
|
def get_bulk_tag_progress() -> BulkTagProgress:
|
|
"""Get the current progress of bulk tagging operations."""
|
|
from wrolpi.api_utils import api_app
|
|
|
|
bulk_tag = api_app.shared_ctx.bulk_tag
|
|
return BulkTagProgress(
|
|
status=bulk_tag.get('status', 'idle'),
|
|
total=bulk_tag.get('total', 0),
|
|
completed=bulk_tag.get('completed', 0),
|
|
add_tag_names=list(bulk_tag.get('add_tag_names', [])),
|
|
remove_tag_names=list(bulk_tag.get('remove_tag_names', [])),
|
|
error=bulk_tag.get('error'),
|
|
queued_jobs=bulk_tag.get('queued_jobs', 0),
|
|
)
|
|
|
|
|
|
def queue_bulk_tag_job(paths: List[str], add_tag_names: List[str], remove_tag_names: List[str]):
|
|
"""Add a bulk tagging job to the queue."""
|
|
from wrolpi.api_utils import api_app
|
|
|
|
job = dict(
|
|
paths=paths,
|
|
add_tag_names=add_tag_names,
|
|
remove_tag_names=remove_tag_names,
|
|
)
|
|
api_app.shared_ctx.bulk_tag_queue.put(job)
|
|
|
|
# Reset progress state and update queued jobs count
|
|
try:
|
|
current_queued = api_app.shared_ctx.bulk_tag.get('queued_jobs', 0)
|
|
api_app.shared_ctx.bulk_tag.update(dict(
|
|
status='idle',
|
|
total=0,
|
|
completed=0,
|
|
error=None,
|
|
queued_jobs=current_queued + 1,
|
|
))
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
BULK_TAG_CHUNK_SIZE = 100 # Files per DB session
|
|
BULK_SQL_CHUNK_SIZE = 1000 # Rows per SQL statement
|
|
|
|
|
|
def _bulk_insert_tag_files(tag_files: List[Tuple[int, int]]):
|
|
"""Bulk insert TagFiles, ignoring duplicates.
|
|
|
|
Args:
|
|
tag_files: List of (tag_id, file_group_id) tuples
|
|
"""
|
|
if not tag_files:
|
|
return
|
|
|
|
now_timestamp = now()
|
|
|
|
# Process in chunks to avoid overly large SQL statements
|
|
for i in range(0, len(tag_files), BULK_SQL_CHUNK_SIZE):
|
|
chunk = tag_files[i:i + BULK_SQL_CHUNK_SIZE]
|
|
values = [(tag_id, file_group_id, now_timestamp)
|
|
for tag_id, file_group_id in chunk]
|
|
|
|
with get_db_curs(commit=True) as curs:
|
|
placeholders, params = values_clause(values)
|
|
stmt = f'''
|
|
INSERT INTO tag_file (tag_id, file_group_id, created_at)
|
|
VALUES {placeholders}
|
|
ON CONFLICT (tag_id, file_group_id) DO NOTHING
|
|
'''
|
|
curs.execute(stmt, params)
|
|
count_config_change_curs(curs, 'tags.yaml')
|
|
logger.debug(f'Bulk inserted {len(chunk)} TagFiles')
|
|
|
|
|
|
def _bulk_delete_tag_files(tag_files: List[Tuple[int, int]]):
|
|
"""Bulk delete TagFiles.
|
|
|
|
Args:
|
|
tag_files: List of (tag_id, file_group_id) tuples
|
|
"""
|
|
if not tag_files:
|
|
return
|
|
|
|
# Process in chunks
|
|
for i in range(0, len(tag_files), BULK_SQL_CHUNK_SIZE):
|
|
chunk = tag_files[i:i + BULK_SQL_CHUNK_SIZE]
|
|
pairs = [(tag_id, file_group_id) for tag_id, file_group_id in chunk]
|
|
|
|
with get_db_curs(commit=True) as curs:
|
|
placeholders, params = values_clause(pairs)
|
|
stmt = f'''
|
|
DELETE FROM tag_file
|
|
WHERE (tag_id, file_group_id) IN (VALUES {placeholders})
|
|
'''
|
|
curs.execute(stmt, params)
|
|
logger.debug(f'Bulk deleted {curs.rowcount} TagFiles')
|
|
count_config_change_curs(curs, 'tags.yaml')
|
|
|
|
|
|
async def _process_bulk_tag_job(job: dict):
|
|
"""Process a single bulk tagging job with batched DB operations.
|
|
|
|
This function uses a three-phase approach for performance:
|
|
1. Collection: Gather all files and resolve FileGroups, collecting tag operations
|
|
2. Bulk DB operations: Execute bulk INSERT and DELETE for TagFiles
|
|
3. Switch activation: Call save_tags_config and sync_tags_directory once at the end
|
|
"""
|
|
from wrolpi.api_utils import api_app
|
|
|
|
paths = job['paths']
|
|
add_tag_names = job['add_tag_names']
|
|
remove_tag_names = job['remove_tag_names']
|
|
|
|
media_directory = get_media_directory()
|
|
absolute_paths = [media_directory / p for p in paths]
|
|
|
|
# Collect all file paths (expanding directories recursively)
|
|
all_file_paths: List[pathlib.Path] = []
|
|
for path in absolute_paths:
|
|
if path.is_file():
|
|
all_file_paths.append(path)
|
|
elif path.is_dir():
|
|
for file in walk(path):
|
|
if file.is_file():
|
|
all_file_paths.append(file)
|
|
|
|
# Remove files in ignored directories
|
|
all_file_paths = remove_files_in_ignored_directories(all_file_paths)
|
|
|
|
if not all_file_paths:
|
|
return
|
|
|
|
# Get unique files by stem
|
|
unique_files = get_unique_files_by_stem(all_file_paths)
|
|
|
|
# Update progress with total count
|
|
api_app.shared_ctx.bulk_tag.update(dict(
|
|
status='running',
|
|
total=len(unique_files),
|
|
completed=0,
|
|
add_tag_names=add_tag_names,
|
|
remove_tag_names=remove_tag_names,
|
|
error=None,
|
|
))
|
|
|
|
# Pre-resolve tag IDs (using cached lookups)
|
|
add_tag_ids = set()
|
|
remove_tag_ids = set()
|
|
with get_db_session() as session:
|
|
for tag_name in add_tag_names:
|
|
try:
|
|
add_tag_ids.add(Tag.get_id_by_name(session, tag_name))
|
|
except UnknownTag:
|
|
logger.warning(f'Unknown tag to add: {tag_name}')
|
|
for tag_name in remove_tag_names:
|
|
try:
|
|
remove_tag_ids.add(Tag.get_id_by_name(session, tag_name))
|
|
except UnknownTag:
|
|
logger.warning(f'Unknown tag to remove: {tag_name}')
|
|
|
|
if not add_tag_ids and not remove_tag_ids:
|
|
return
|
|
|
|
# Phase 1: Collect all file_group_ids and required operations
|
|
tag_files_to_insert: List[Tuple[int, int]] = [] # (tag_id, file_group_id)
|
|
tag_files_to_delete: List[Tuple[int, int]] = [] # (tag_id, file_group_id)
|
|
|
|
completed = 0
|
|
|
|
# Process files in chunks to balance memory and DB pressure
|
|
for chunk_start in range(0, len(unique_files), BULK_TAG_CHUNK_SIZE):
|
|
chunk = unique_files[chunk_start:chunk_start + BULK_TAG_CHUNK_SIZE]
|
|
|
|
with get_db_session(commit=True) as session:
|
|
for file_path in chunk:
|
|
try:
|
|
# Get or create FileGroup
|
|
file_group = FileGroup.get_by_path(session, file_path)
|
|
if not file_group:
|
|
# Create FileGroup for unindexed file
|
|
paths_for_group = glob_shared_stem(file_path)
|
|
paths_for_group = remove_files_in_ignored_directories(paths_for_group)
|
|
if paths_for_group:
|
|
try:
|
|
file_group = FileGroup.from_paths(session, *paths_for_group)
|
|
file_group.do_model(session)
|
|
session.flush([file_group])
|
|
except (NoPrimaryFile, IntegrityError) as e:
|
|
logger.warning(f'Could not create FileGroup for {file_path}: {e}')
|
|
completed += 1
|
|
api_app.shared_ctx.bulk_tag['completed'] = completed
|
|
continue
|
|
|
|
if file_group and file_group.id:
|
|
# Flush to ensure file_group.id is set
|
|
session.flush([file_group])
|
|
|
|
# Get current tag_ids for this file_group
|
|
existing_tag_ids = {tf.tag_id for tf in file_group.tag_files}
|
|
|
|
# Determine tags to add (not already present)
|
|
for tag_id in add_tag_ids:
|
|
if tag_id not in existing_tag_ids:
|
|
tag_files_to_insert.append((tag_id, file_group.id))
|
|
|
|
# Determine tags to remove (currently present)
|
|
for tag_id in remove_tag_ids:
|
|
if tag_id in existing_tag_ids:
|
|
tag_files_to_delete.append((tag_id, file_group.id))
|
|
|
|
except Exception as e:
|
|
logger.error(f'Error processing file {file_path}: {e}', exc_info=e)
|
|
|
|
completed += 1
|
|
api_app.shared_ctx.bulk_tag['completed'] = completed
|
|
|
|
# Allow cancellation between chunks
|
|
await asyncio.sleep(0)
|
|
|
|
# Phase 2: Bulk database operations
|
|
_bulk_insert_tag_files(tag_files_to_insert)
|
|
_bulk_delete_tag_files(tag_files_to_delete)
|
|
|
|
# Phase 3: Activate switches ONCE at the end
|
|
if tag_files_to_insert or tag_files_to_delete:
|
|
save_tags_config.activate_switch()
|
|
sync_tags_directory.activate_switch()
|
|
|
|
|
|
DISABLE_BULK_TAG_WORKER = bool(PYTEST)
|
|
|
|
|
|
async def bulk_tag_worker():
|
|
"""Background worker that processes bulk tagging jobs from the queue."""
|
|
import queue
|
|
from wrolpi.api_utils import api_app
|
|
|
|
if PYTEST and DISABLE_BULK_TAG_WORKER:
|
|
return
|
|
|
|
bulk_tag_ctx = api_app.shared_ctx.bulk_tag
|
|
bulk_tag_queue = api_app.shared_ctx.bulk_tag_queue
|
|
|
|
try:
|
|
# Try to get a job from the queue (non-blocking)
|
|
job = bulk_tag_queue.get_nowait()
|
|
except queue.Empty:
|
|
# No jobs in queue, stay idle
|
|
bulk_tag_ctx['status'] = 'idle'
|
|
return
|
|
|
|
# Decrement queued jobs count
|
|
try:
|
|
current_queued = bulk_tag_ctx.get('queued_jobs', 1)
|
|
bulk_tag_ctx['queued_jobs'] = max(0, current_queued - 1)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
await _process_bulk_tag_job(job)
|
|
except Exception as e:
|
|
logger.error(f'Bulk tag worker error: {e}', exc_info=e)
|
|
bulk_tag_ctx['error'] = str(e)
|
|
finally:
|
|
bulk_tag_ctx['status'] = 'idle'
|
|
|
|
|
|
# Register the worker as a perpetual signal
|
|
from wrolpi.api_utils import perpetual_signal
|
|
|
|
bulk_tag_worker = perpetual_signal(sleep=1)(bulk_tag_worker)
|
|
|
|
|
|
def _detect_archive_type(path: pathlib.Path) -> str:
|
|
"""Detect archive type from file suffix. Returns 'zip', 'tar', 'rar', '7z', or raises UnsupportedArchive."""
|
|
name = path.name.lower()
|
|
if name.endswith(('.zip', '.cbz')):
|
|
return 'zip'
|
|
if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz', '.cbt')):
|
|
return 'tar'
|
|
if name.endswith(('.rar', '.cbr')):
|
|
return 'rar'
|
|
if name.endswith(('.7z', '.cb7')):
|
|
return '7z'
|
|
raise UnsupportedArchive(f'Unsupported archive format: {path.name}')
|
|
|
|
|
|
def _build_tree(entries: list) -> list:
|
|
"""Build a hierarchical tree from a flat list of archive entries.
|
|
|
|
Each entry is a dict with 'path', 'name', 'is_dir', 'size'.
|
|
Returns a nested list with 'children' on directory entries.
|
|
"""
|
|
root_children = []
|
|
# Map from directory path to its children list.
|
|
dir_map = {}
|
|
|
|
# Sort entries so parents come before children.
|
|
entries.sort(key=lambda e: e['path'])
|
|
|
|
for entry in entries:
|
|
parts = entry['path'].strip('/').split('/')
|
|
if len(parts) == 1:
|
|
# Top-level entry.
|
|
if entry['is_dir']:
|
|
entry['children'] = []
|
|
dir_map[entry['path'].rstrip('/') + '/'] = entry['children']
|
|
root_children.append(entry)
|
|
else:
|
|
# Find parent directory.
|
|
parent_path = '/'.join(parts[:-1]) + '/'
|
|
if parent_path not in dir_map:
|
|
# Create implicit parent directories.
|
|
current = ''
|
|
for part in parts[:-1]:
|
|
current += part + '/'
|
|
if current not in dir_map:
|
|
parent_entry = {
|
|
'path': current,
|
|
'name': part,
|
|
'is_dir': True,
|
|
'size': 0,
|
|
'children': [],
|
|
}
|
|
dir_map[current] = parent_entry['children']
|
|
# Attach to grandparent or root.
|
|
grandparent = '/'.join(current.rstrip('/').split('/')[:-1])
|
|
grandparent_key = grandparent + '/' if grandparent else None
|
|
if grandparent_key and grandparent_key in dir_map:
|
|
dir_map[grandparent_key].append(parent_entry)
|
|
else:
|
|
root_children.append(parent_entry)
|
|
|
|
if entry['is_dir']:
|
|
entry['children'] = []
|
|
dir_map[entry['path'].rstrip('/') + '/'] = entry['children']
|
|
|
|
parent_list = dir_map.get(parent_path)
|
|
if parent_list is not None:
|
|
parent_list.append(entry)
|
|
else:
|
|
root_children.append(entry)
|
|
|
|
return root_children
|
|
|
|
|
|
def list_archive_contents(path: pathlib.Path) -> dict:
|
|
"""List the contents of an archive file as a hierarchical tree.
|
|
|
|
Returns: {"entries": [...], "total_size": int, "total_files": int}
|
|
"""
|
|
media_directory = get_media_directory()
|
|
if not str(path).startswith(str(media_directory)):
|
|
raise InvalidFile('Path must be within the media directory')
|
|
if not path.is_file():
|
|
raise InvalidFile(f'Archive file does not exist: {path}')
|
|
|
|
archive_type = _detect_archive_type(path)
|
|
flat_entries = []
|
|
total_size = 0
|
|
total_files = 0
|
|
|
|
try:
|
|
if archive_type == 'zip':
|
|
with zipfile.ZipFile(path, 'r') as zf:
|
|
for info in zf.infolist():
|
|
is_dir = info.is_dir()
|
|
entry = {
|
|
'path': info.filename,
|
|
'name': info.filename.rstrip('/').split('/')[-1],
|
|
'is_dir': is_dir,
|
|
'size': info.file_size,
|
|
}
|
|
flat_entries.append(entry)
|
|
if not is_dir:
|
|
total_size += info.file_size
|
|
total_files += 1
|
|
elif archive_type == 'tar':
|
|
with tarfile.open(path, 'r:*') as tf:
|
|
for member in tf.getmembers():
|
|
is_dir = member.isdir()
|
|
entry = {
|
|
'path': member.name + ('/' if is_dir else ''),
|
|
'name': member.name.rstrip('/').split('/')[-1],
|
|
'is_dir': is_dir,
|
|
'size': member.size,
|
|
}
|
|
flat_entries.append(entry)
|
|
if not is_dir:
|
|
total_size += member.size
|
|
total_files += 1
|
|
elif archive_type == 'rar':
|
|
import rarfile
|
|
with rarfile.RarFile(path, 'r') as rf:
|
|
for info in rf.infolist():
|
|
is_dir = info.is_dir()
|
|
entry = {
|
|
'path': info.filename,
|
|
'name': info.filename.rstrip('/').split('/')[-1],
|
|
'is_dir': is_dir,
|
|
'size': info.file_size,
|
|
}
|
|
flat_entries.append(entry)
|
|
if not is_dir:
|
|
total_size += info.file_size
|
|
total_files += 1
|
|
elif archive_type == '7z':
|
|
import py7zr
|
|
with py7zr.SevenZipFile(path, 'r') as sz:
|
|
for entry_info in sz.list():
|
|
is_dir = entry_info.is_directory
|
|
entry = {
|
|
'path': entry_info.filename + ('/' if is_dir else ''),
|
|
'name': entry_info.filename.rstrip('/').split('/')[-1],
|
|
'is_dir': is_dir,
|
|
'size': entry_info.uncompressed if not is_dir else 0,
|
|
}
|
|
flat_entries.append(entry)
|
|
if not is_dir:
|
|
total_size += entry_info.uncompressed
|
|
total_files += 1
|
|
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
|
|
raise UnsupportedArchive(f'Cannot read archive: {e}')
|
|
|
|
tree = _build_tree(flat_entries)
|
|
return {'entries': tree, 'total_size': total_size, 'total_files': total_files}
|
|
|
|
|
|
def stream_archive_member(archive_path: pathlib.Path, member_path: str) -> Tuple[bytes, str, str]:
|
|
"""Read a single member from an archive and return its bytes, mimetype, and filename.
|
|
|
|
Security: validates member_path has no '..' components.
|
|
"""
|
|
media_directory = get_media_directory()
|
|
if not str(archive_path).startswith(str(media_directory)):
|
|
raise InvalidFile('Path must be within the media directory')
|
|
if not archive_path.is_file():
|
|
raise InvalidFile(f'Archive file does not exist: {archive_path}')
|
|
|
|
# Security: reject path traversal.
|
|
if '..' in member_path.split('/'):
|
|
raise InvalidArchiveMember('Invalid member path: contains ".."')
|
|
|
|
archive_type = _detect_archive_type(archive_path)
|
|
filename = member_path.rstrip('/').split('/')[-1]
|
|
guessed_type, _ = mimetypes.guess_type(filename)
|
|
content_type = guessed_type or 'application/octet-stream'
|
|
|
|
try:
|
|
if archive_type == 'zip':
|
|
with zipfile.ZipFile(archive_path, 'r') as zf:
|
|
try:
|
|
info = zf.getinfo(member_path)
|
|
except KeyError:
|
|
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
|
|
if info.is_dir():
|
|
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
|
|
file_bytes = zf.read(member_path)
|
|
elif archive_type == 'tar':
|
|
with tarfile.open(archive_path, 'r:*') as tf:
|
|
try:
|
|
member = tf.getmember(member_path)
|
|
except KeyError:
|
|
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
|
|
if member.isdir():
|
|
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
|
|
f = tf.extractfile(member)
|
|
if f is None:
|
|
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
|
|
file_bytes = f.read()
|
|
elif archive_type == 'rar':
|
|
import rarfile
|
|
with rarfile.RarFile(archive_path, 'r') as rf:
|
|
try:
|
|
info = rf.getinfo(member_path)
|
|
except KeyError:
|
|
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
|
|
if info.is_dir():
|
|
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
|
|
file_bytes = rf.read(member_path)
|
|
elif archive_type == '7z':
|
|
import py7zr
|
|
import tempfile
|
|
with py7zr.SevenZipFile(archive_path, 'r') as sz:
|
|
if member_path not in sz.getnames():
|
|
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
sz.extract(tmpdir, targets=[member_path])
|
|
extracted_path = pathlib.Path(tmpdir) / member_path
|
|
if not extracted_path.is_file():
|
|
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
|
|
file_bytes = extracted_path.read_bytes()
|
|
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
|
|
raise UnsupportedArchive(f'Cannot read archive: {e}')
|
|
|
|
return file_bytes, content_type, filename
|