mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
FileGroup gains `progress` (0.0 to 1.0, 1.0 is finished) and `position` (where to resume: a time, a page, or an EPUB CFI). Both are returned with the file, including by POST /api/files/file. POST /api/files/progress records progress and is also a view. A file 95% through is finished and keeps no position; one barely started (under 10 s, the first page, under 1%) is not worth resuming. The write happens before the response, so the next request sees it. POST /api/files/progress/clear starts a file over. Both refuse files outside the media directory. recently_viewed.yaml keeps the 1000 most recently viewed files so viewing history and positions survive a database rebuild. On import the newer `viewed` wins, so a file viewed while WROLPi was starting keeps its newer position. The config is saved when a file is opened, when progress is final (paused or closed), when it is cleared, and after refresh cleanup (moves, renames, deletes); heartbeats during playback only write the database. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2731 lines
114 KiB
Python
2731 lines
114 KiB
Python
import asyncio
|
|
import dataclasses
|
|
import datetime
|
|
import functools
|
|
import glob
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import zipfile
|
|
from itertools import zip_longest
|
|
from pathlib import Path
|
|
from typing import Callable, List, Tuple, Union, Dict, Generator, Iterable, Set
|
|
|
|
import cachetools.func
|
|
from sqlalchemy import asc, or_, text as sa_text
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.orm import Session, undefer, joinedload
|
|
|
|
from wrolpi.cmd import which
|
|
from wrolpi.common import get_media_directory, wrol_mode_check, logger, \
|
|
chunks, chunks_by_stem, walk, \
|
|
get_wrolpi_config, \
|
|
unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory, strip_surrogates
|
|
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
|
|
from wrolpi.db import get_db_session, get_db_curs, values_clause, session_curs
|
|
from wrolpi.downloader import download_manager, Download
|
|
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
|
|
NoPrimaryFile, InvalidDirectory, IgnoredDirectoryError, UnsupportedArchive, InvalidArchiveMember
|
|
from wrolpi.events import Events
|
|
from wrolpi.files.models import FileGroup, Directory
|
|
from wrolpi.files.progress import normalize_progress, save_recently_viewed_config
|
|
from wrolpi.lang import ISO_639_CODES, ISO_3166_CODES
|
|
from wrolpi.tags import TagFile, Tag, tag_append_sub_select_where, save_tags_config, sync_tags_directory
|
|
from wrolpi.vars import PYTEST, IS_MACOS
|
|
|
|
try:
|
|
import magic
|
|
|
|
mime = magic.Magic(mime=True)
|
|
except ImportError:
|
|
# Magic is not installed
|
|
magic = None
|
|
mime = None
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'search_files',
|
|
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'choose_primary_file', 'get_file_statistics',
|
|
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
|
|
'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href',
|
|
'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories',
|
|
'get_normalized_ignored_directories', 'sanitize_ignored_directories',
|
|
'cached_search_total', 'search_filter_cache_key', 'count_file_groups',
|
|
'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems',
|
|
'backfill_null_file_group_stems', 'dedupe_file_groups_by_directory_stem']
|
|
|
|
|
|
def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
|
|
"""Returns all Tag names for the provided file path."""
|
|
tags = session.query(Tag) \
|
|
.join(TagFile, Tag.id == TagFile.tag_id) \
|
|
.join(FileGroup, FileGroup.id == TagFile.file_group_id) \
|
|
.filter(FileGroup.primary_path == str(file))
|
|
names = sorted([tag.name for tag in tags], key=lambda i: i.lower())
|
|
return names
|
|
|
|
|
|
def sanitize_filename_surrogates(path: pathlib.Path) -> pathlib.Path:
|
|
"""
|
|
Check if a path contains invalid UTF-8 surrogates. If so, rename the file
|
|
on disk to replace them with underscores.
|
|
|
|
Linux filesystems store filenames as bytes. Python represents invalid UTF-8
|
|
sequences as surrogates. These surrogates cannot be stored in the database.
|
|
|
|
Returns the (possibly new) path.
|
|
"""
|
|
path_str = str(path)
|
|
|
|
# Check for surrogates by trying to encode as UTF-8
|
|
try:
|
|
path_str.encode('utf-8')
|
|
return path # Path is valid
|
|
except UnicodeEncodeError:
|
|
pass
|
|
|
|
# Get raw bytes from filesystem and decode, replacing invalid sequences
|
|
raw_bytes = os.fsencode(path)
|
|
# Replace invalid UTF-8 sequences with underscore
|
|
sanitized_name = raw_bytes.decode('utf-8', errors='replace').replace('\ufffd', '_')
|
|
sanitized_path = pathlib.Path(sanitized_name)
|
|
|
|
if path.exists() and sanitized_path != path:
|
|
if sanitized_path.exists():
|
|
# Target already exists, add a suffix
|
|
stem = sanitized_path.stem
|
|
suffix = sanitized_path.suffix
|
|
counter = 1
|
|
while sanitized_path.exists():
|
|
sanitized_path = sanitized_path.with_name(f'{stem}_{counter}{suffix}')
|
|
counter += 1
|
|
|
|
logger.warning(f'Renaming file with invalid UTF-8 characters: {path!r} -> {sanitized_path}')
|
|
path.rename(sanitized_path)
|
|
return sanitized_path
|
|
|
|
return path
|
|
|
|
|
|
def _get_file_dict(session: Session, file: pathlib.Path) -> Dict:
|
|
media_directory = get_media_directory()
|
|
try:
|
|
size = file.stat().st_size
|
|
except PermissionError:
|
|
size = None
|
|
try:
|
|
mimetype = get_mimetype(file)
|
|
except PermissionError:
|
|
mimetype = None
|
|
file_group = FileGroup.get_by_path(session, file)
|
|
return dict(
|
|
path=file.relative_to(media_directory),
|
|
size=size,
|
|
mimetype=mimetype,
|
|
tags=get_file_tag_names(session, file),
|
|
progress=file_group.progress if file_group else None,
|
|
position=file_group.position if file_group else None,
|
|
)
|
|
|
|
|
|
def get_file_dict(session: Session, file: str) -> Dict:
|
|
media_directory = get_media_directory()
|
|
return _get_file_dict(session, media_directory / file)
|
|
|
|
|
|
async def set_file_viewed(session: Session, file: pathlib.Path):
|
|
"""Change FileGroup.viewed to the current datetime.
|
|
|
|
Files in ignored directories are not tracked.
|
|
"""
|
|
# Check if file is in an ignored directory
|
|
remaining = remove_files_in_ignored_directories([file])
|
|
if not remaining:
|
|
return
|
|
|
|
try:
|
|
fg = FileGroup.find_by_path(session, file)
|
|
except UnknownFile:
|
|
fg = FileGroup.from_paths(session, file)
|
|
fg.do_model(session)
|
|
fg.set_viewed()
|
|
session.commit()
|
|
|
|
|
|
def _get_or_create_file_group(session: Session, file: pathlib.Path) -> FileGroup:
|
|
if not get_paths_in_media_directory([file]):
|
|
raise InvalidFile('File must be in the media directory')
|
|
try:
|
|
return FileGroup.find_by_path(session, file)
|
|
except UnknownFile:
|
|
if not file.is_file():
|
|
raise InvalidFile(f'File does not exist: {get_relative_to_media_directory(file)}')
|
|
fg = FileGroup.from_paths(session, file)
|
|
fg.do_model(session)
|
|
return fg
|
|
|
|
|
|
def set_file_progress(session: Session, file: pathlib.Path, progress: float, position: dict = None,
|
|
final: bool = False) -> Tuple[float | None, dict | None]:
|
|
"""Record how far the User is through a file (see wrolpi.files.progress). This is also a view.
|
|
|
|
@param final: The User stopped (paused, closed); save the recently viewed config.
|
|
@return: The (progress, position) that was stored.
|
|
"""
|
|
progress, position = normalize_progress(progress, position)
|
|
|
|
# Files in ignored directories are not tracked.
|
|
if not remove_files_in_ignored_directories([file]):
|
|
return None, None
|
|
|
|
fg = _get_or_create_file_group(session, file)
|
|
# Not `set_viewed`, which saves the config; a heartbeat during playback only writes the DB.
|
|
fg.viewed = now()
|
|
fg.progress = progress
|
|
fg.position = position
|
|
session.commit()
|
|
|
|
if final:
|
|
save_recently_viewed_config.activate_switch()
|
|
return progress, position
|
|
|
|
|
|
def clear_file_progress(session: Session, file: pathlib.Path):
|
|
"""Forget how far the User is through a file, so it starts from the beginning. It remains viewed."""
|
|
fg = _get_or_create_file_group(session, file)
|
|
fg.progress = None
|
|
fg.position = None
|
|
session.commit()
|
|
save_recently_viewed_config.activate_switch()
|
|
|
|
|
|
@cachetools.func.ttl_cache(10_000, 30.0)
|
|
def _get_directory_dict(directory: pathlib.Path,
|
|
directories_cache: str, # Used to cache by requested directories.
|
|
) -> Dict:
|
|
media_directory = get_media_directory()
|
|
try:
|
|
is_empty = not next(directory.iterdir(), False)
|
|
except PermissionError:
|
|
is_empty = False
|
|
return dict(
|
|
path=f'{directory.relative_to(media_directory)}/',
|
|
is_empty=is_empty,
|
|
)
|
|
|
|
|
|
def _get_recursive_directory_dict(session: Session, directory: pathlib.Path, directories: List[pathlib.Path]) -> Dict:
|
|
directories_cache = str(sorted(directories))
|
|
d = _get_directory_dict(directory, directories_cache)
|
|
|
|
if directory in directories:
|
|
children = dict()
|
|
for path in directory.iterdir():
|
|
if not path.is_dir() and path.name in HIDDEN_FILES:
|
|
# Never show ignored files.
|
|
continue
|
|
if path.is_dir() and path in directories:
|
|
children[f'{path.name}/'] = _get_recursive_directory_dict(session, path, directories)
|
|
elif path.is_dir():
|
|
children[f'{path.name}/'] = _get_directory_dict(path, directories_cache)
|
|
else:
|
|
children[path.name] = _get_file_dict(session, path)
|
|
d['children'] = children
|
|
return d
|
|
|
|
|
|
HIDDEN_DIRECTORIES = ('lost+found',)
|
|
# macOS metadata files which should never be shown in the FileBrowser.
|
|
HIDDEN_FILES = ('.DS_Store', '._.DS_Store')
|
|
|
|
|
|
def list_directories_contents(session: Session, directories_: List[str]) -> Dict:
|
|
"""List all files down to (and within) the directories provided.
|
|
|
|
This should only be used by the frontend for it's FileBrowser."""
|
|
media_directory = get_media_directory()
|
|
# Convert all directories to pathlib.Path.
|
|
directories = {media_directory / i for i in directories_}
|
|
if invalid_directories := [i for i in directories if not i.is_dir()]:
|
|
raise FileNotFoundError(f'Invalid directories: {invalid_directories}')
|
|
|
|
# Get all unique parents for each directory, but not if they are above the media directory.
|
|
parents = {j for i in directories for j in i.parents if j not in media_directory.parents}
|
|
# Add unique parents onto the requested directories.
|
|
directories = list(directories | parents)
|
|
|
|
paths = dict()
|
|
for path in media_directory.iterdir():
|
|
if path.is_dir() and path.name in HIDDEN_DIRECTORIES:
|
|
# Never show ignored directories.
|
|
continue
|
|
if not path.is_dir() and path.name in HIDDEN_FILES:
|
|
# Never show ignored files.
|
|
continue
|
|
if path.is_dir():
|
|
paths[f'{path.name}/'] = _get_recursive_directory_dict(session, path, directories)
|
|
else:
|
|
paths[path.name] = _get_file_dict(session, path)
|
|
|
|
return paths
|
|
|
|
|
|
def get_tagged_file_groups_by_ids(session: Session, file_group_ids: List[int]) -> List[dict]:
|
|
"""Return serialized FileGroups (subset of `file_group_ids`) that have at least one TagFile.
|
|
|
|
Used by the unified delete endpoint to require force-confirmation before deleting tagged files."""
|
|
if not file_group_ids:
|
|
return []
|
|
rows = session.query(FileGroup) \
|
|
.options(*FileGroup.json_options()) \
|
|
.join(TagFile, TagFile.file_group_id == FileGroup.id) \
|
|
.filter(FileGroup.id.in_(file_group_ids)) \
|
|
.distinct() \
|
|
.all()
|
|
return [fg.__json__() for fg in rows]
|
|
|
|
|
|
def get_tagged_file_groups_by_paths(session: Session, paths: Iterable[pathlib.Path]) -> List[FileGroup]:
|
|
"""Return FileGroup objects under any of the given paths that have at least one TagFile.
|
|
|
|
For a file path, matches FileGroups whose primary_path shares the file's stem in the same
|
|
directory. For a directory path, matches FileGroups whose `directory` is the path or a
|
|
descendant. Used by the path-based delete endpoint to require force-confirmation before
|
|
deleting tagged files."""
|
|
paths = list(paths)
|
|
if not paths:
|
|
return []
|
|
tagged: List[FileGroup] = []
|
|
seen_ids = set()
|
|
for path in paths:
|
|
query = session.query(FileGroup) \
|
|
.options(*FileGroup.json_options()) \
|
|
.join(TagFile, TagFile.file_group_id == FileGroup.id)
|
|
if path.is_file():
|
|
# Could be deleting a file in a FileGroup that has been tagged.
|
|
stem, _ = split_path_stem_and_suffix(path)
|
|
query = query.filter(FileGroup.primary_path.like(f'{path.parent}/{stem}%'))
|
|
else:
|
|
# Search for any FileGroups that have been tagged under this directory.
|
|
# Use indexed directory column for efficient lookup.
|
|
path_str = str(path)
|
|
query = query.filter(or_(
|
|
FileGroup.directory == path_str,
|
|
FileGroup.directory.like(f'{path_str}/%'),
|
|
))
|
|
for file_group in query:
|
|
if path.is_dir() or any(p for p in paths if p in file_group.my_paths()):
|
|
if file_group.id not in seen_ids:
|
|
seen_ids.add(file_group.id)
|
|
tagged.append(file_group)
|
|
return tagged
|
|
|
|
|
|
def delete_file_groups(session: Session, file_group_ids: List[int], force: bool = False) -> Set[str]:
|
|
"""Delete FileGroups (and their model rows via FK CASCADE) by ID.
|
|
|
|
Returns the set of `FileGroup.model` values that were deleted, so the caller can fire any
|
|
model-specific post-delete switches (e.g., `save_channels_config` for videos).
|
|
|
|
The caller is expected to have already gated this on tag confirmation via
|
|
`get_tagged_file_groups_by_ids`. `force` propagates to FileGroup.delete so the safety net
|
|
at FileGroup.delete (raises FileGroupIsTagged if force is False and tags exist) still
|
|
protects callers that skipped the pre-check."""
|
|
files = FileGroup.get_by_ids(session, file_group_ids)
|
|
if not files:
|
|
raise UnknownFile(f'No FileGroups found for ids: {file_group_ids}')
|
|
|
|
affected_models: Set[str] = {fg.model for fg in files if fg.model}
|
|
had_tagged = any(fg.tag_names for fg in files)
|
|
|
|
# Capture domain Collection IDs for any Archive FileGroups so we can prune empty ones after
|
|
# the cascade removes the Archive rows. (Archive.delete used to do this per-archive; with
|
|
# cascade-based deletion we do it as a single post-pass.)
|
|
archive_collection_ids: Set[int] = set()
|
|
if 'archive' in affected_models:
|
|
from modules.archive.models import Archive
|
|
archive_collection_ids = {
|
|
collection_id for (collection_id,) in session.query(Archive.collection_id)
|
|
.filter(Archive.file_group_id.in_([fg.id for fg in files]))
|
|
if collection_id is not None
|
|
}
|
|
|
|
for fg in files:
|
|
fg.delete(force=force)
|
|
|
|
if archive_collection_ids:
|
|
# Flush so the FK CASCADE has removed the Archive rows before we count remaining.
|
|
session.flush()
|
|
from modules.archive.models import Archive
|
|
from wrolpi.collections.models import Collection
|
|
for collection_id in archive_collection_ids:
|
|
collection = session.query(Collection).filter_by(id=collection_id, kind='domain').one_or_none()
|
|
if collection is None:
|
|
continue
|
|
remaining = session.query(Archive).filter_by(collection_id=collection_id).count()
|
|
if remaining == 0:
|
|
session.delete(collection)
|
|
|
|
if had_tagged:
|
|
save_tags_config.activate_switch()
|
|
|
|
return affected_models
|
|
|
|
|
|
@wrol_mode_check
|
|
async def delete(*paths: Union[str, pathlib.Path], force: bool = False, post_processing: bool = True):
|
|
"""
|
|
Delete a file or directory in the media directory.
|
|
|
|
If any files are tagged and force is False, returns a list of tagged FileGroups without deleting.
|
|
If force is True, deletes even tagged files.
|
|
|
|
If post_processing is False, the synchronous DB cleanup still runs but global modelers/indexers
|
|
are skipped — use this when the caller will trigger modeling itself (e.g. the upload endpoint
|
|
follows up with `upsert_file`)."""
|
|
media_directory = get_media_directory()
|
|
paths = [media_directory / i for i in paths]
|
|
if any(i == media_directory for i in paths):
|
|
raise InvalidFile(f'Cannot delete the media directory')
|
|
for path in paths:
|
|
if not path.is_dir() and not path.is_file():
|
|
logger.warning(f'Cannot delete {path} because it is not a file or a directory.')
|
|
if len(paths) > 1:
|
|
for p1 in paths:
|
|
for p2 in paths:
|
|
if p1 == p2:
|
|
continue
|
|
if p1 != p2 and p1 in p2.parents:
|
|
raise InvalidFile(f'Cannot deleted nested paths')
|
|
|
|
# Discover any tagged FileGroups under these paths so we can either gate the deletion (when
|
|
# !force) or fire `save_tags_config` afterwards (when force=True and any were tagged).
|
|
with get_db_session() as session:
|
|
tagged_file_groups = get_tagged_file_groups_by_paths(session, paths)
|
|
if tagged_file_groups and not force:
|
|
# Serialize while session is still active.
|
|
return [i.__json__() for i in tagged_file_groups]
|
|
had_tagged = bool(tagged_file_groups)
|
|
|
|
for path in paths:
|
|
ignored_directories = get_wrolpi_config().ignored_directories
|
|
if ignored_directories and str(path) in ignored_directories:
|
|
remove_ignored_directory(path)
|
|
if path.is_dir():
|
|
delete_directory(path, recursive=True, force=force)
|
|
elif path.exists():
|
|
path.unlink()
|
|
|
|
# Refresh synchronously to clean up DB before returning.
|
|
# This ensures deleted FileGroups are removed before the caller continues
|
|
# (e.g., upload API recreating the file).
|
|
from wrolpi.files.worker import file_worker
|
|
await file_worker.refresh_sync(list(paths), post_processing=post_processing)
|
|
|
|
if had_tagged:
|
|
# `tags.yaml` references TagFiles that the refresh just removed; rewrite it.
|
|
save_tags_config.activate_switch()
|
|
|
|
|
|
def _mimetype_suffix_map(path: Path, mimetype: str):
|
|
"""Special handling for mimetypes. Python Magic may not return the correct mimetype."""
|
|
from modules.docs.models import MOBI_MIMETYPE
|
|
suffix = path.suffix.lower()
|
|
if mimetype == 'application/octet-stream':
|
|
if suffix.endswith('.mobi'):
|
|
return MOBI_MIMETYPE
|
|
if suffix.endswith('.stl'):
|
|
return 'model/stl'
|
|
if suffix.endswith('.3mf'):
|
|
return 'model/3mf'
|
|
if suffix.endswith('.mp4'):
|
|
return 'video/mp4'
|
|
if suffix.endswith('.html') or suffix.endswith('.htm'):
|
|
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
|
|
return 'text/html'
|
|
if suffix.endswith('.hgt'):
|
|
return 'application/octet-stream'
|
|
if mimetype.startswith('text/') and suffix in ('.md', '.markdown'):
|
|
# libmagic reports markdown as text/plain, or as a programming language when it sniffs code.
|
|
return 'text/markdown'
|
|
if mimetype == 'text/plain':
|
|
if suffix.endswith('.json'):
|
|
return 'application/json'
|
|
if suffix.endswith('.vtt'):
|
|
return 'text/vtt'
|
|
if suffix.endswith('.csv'):
|
|
return 'text/csv'
|
|
if suffix.endswith('.html') or suffix.endswith('.htm'):
|
|
return 'text/html'
|
|
if suffix.endswith('.stl'):
|
|
return 'model/stl'
|
|
if suffix.endswith('.scad'):
|
|
return 'application/x-openscad'
|
|
if suffix.endswith('.js'):
|
|
return 'application/javascript'
|
|
if suffix.endswith('.css'):
|
|
return 'text/css'
|
|
if suffix.endswith('.srt'):
|
|
return 'text/srt'
|
|
if suffix.endswith('.yaml') or suffix.endswith('.yml'):
|
|
return 'text/yaml'
|
|
if suffix.endswith('.azw3'):
|
|
return 'application/vnd.amazon.mobi8-ebook'
|
|
if suffix.endswith('.obj'):
|
|
return 'model/obj'
|
|
if mimetype == 'application/x-subrip':
|
|
# Fallback to old mimetype.
|
|
return 'text/srt'
|
|
if mimetype == 'application/zip' and (suffix == '.html' or suffix == '.htm'):
|
|
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
|
|
return 'text/html'
|
|
if mimetype == 'application/zip' and suffix == '.docx':
|
|
return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
|
if mimetype == 'application/zip' and suffix == '.cbz':
|
|
return 'application/x-cbz'
|
|
if mimetype == 'application/zip' and suffix == '.3mf':
|
|
return 'model/3mf'
|
|
if mimetype == 'application/zip' and suffix == '.odt':
|
|
return 'application/vnd.oasis.opendocument.text'
|
|
if mimetype in ('application/x-rar', 'application/x-rar-compressed') and suffix == '.cbr':
|
|
return 'application/x-cbr'
|
|
if mimetype == 'video/x-ms-asf' and suffix == '.wma':
|
|
return 'audio/x-ms-wma'
|
|
return mimetype
|
|
|
|
|
|
FILE_BIN = which('file', '/usr/bin/file')
|
|
|
|
|
|
@functools.lru_cache(maxsize=10_000)
|
|
def get_mimetype(path: Path) -> str:
|
|
"""Get the mimetype of a file, prefer using `magic`, fallback to builtin `file` command."""
|
|
if magic is None:
|
|
# This method is slow, prefer the speedier `magic` module.
|
|
cmd = (FILE_BIN, '--mime-type', str(path.absolute()))
|
|
output = subprocess.check_output(cmd)
|
|
output = output.decode()
|
|
mimetype = output.split(' ')[-1].strip()
|
|
else:
|
|
mimetype = mime.from_file(path)
|
|
mimetype = _mimetype_suffix_map(path, mimetype)
|
|
return mimetype
|
|
|
|
|
|
# Special suffixes within WROLPi.
|
|
SUFFIXES = {
|
|
'.info.json',
|
|
'.ffprobe.json',
|
|
'.live_chat.json',
|
|
'.readability.html',
|
|
'.readability.json',
|
|
'.readability.txt',
|
|
}
|
|
SUFFIXES |= {f'.{i}.srt' for i in ISO_639_CODES}
|
|
SUFFIXES |= {f'.{i}.vtt' for i in ISO_639_CODES}
|
|
EXTRA_SUFFIXES = set()
|
|
for six_ in ISO_639_CODES.keys():
|
|
EXTRA_SUFFIXES |= {f'.{six_}-{i}.srt' for i in ISO_3166_CODES}
|
|
EXTRA_SUFFIXES |= {f'.{six_}-{i}.vtt' for i in ISO_3166_CODES}
|
|
# Add auto suffixes once (these don't depend on the loop variable)
|
|
EXTRA_SUFFIXES |= {f'.{i}-auto.srt' for i in ISO_639_CODES}
|
|
EXTRA_SUFFIXES |= {f'.{i}-auto.vtt' for i in ISO_639_CODES}
|
|
|
|
# Convert to lowercase frozensets for O(1) case-insensitive lookup
|
|
_SUFFIXES_LOWER = frozenset(s.lower() for s in SUFFIXES)
|
|
_EXTRA_SUFFIXES_LOWER = frozenset(s.lower() for s in EXTRA_SUFFIXES)
|
|
_HARDCODED_SUFFIXES = frozenset(
|
|
{'.info.json', '.ffprobe.json', '.live_chat.json', '.readability.html', '.readability.json', '.readability.txt'})
|
|
|
|
# Bump when SUFFIXES / split_path_stem_and_suffix grouping rules change. A stored
|
|
# file_group.stem is a cache of this algorithm; compare joins on it, so a mismatch
|
|
# looks like delete+insert and would drop tags. ensure_file_group_stems() must run
|
|
# before any compare when this version does not match the value in wrolpi_kv.
|
|
STEM_ALGORITHM_VERSION = 1
|
|
STEM_ALGORITHM_VERSION_KEY = 'stem_algorithm_version'
|
|
# Cancelled compare_file_groups leaves ensure_file_group_stems running in a thread.
|
|
# Tests (and overlapping refreshes) share one SQLite writer; serialize the backfill.
|
|
_stem_backfill_lock = threading.Lock()
|
|
|
|
PART_PARSER = re.compile(r'(.+?)(\.f[\d]{2,3})?(\.info)?(\.\w{3,4})(\.part)', re.IGNORECASE)
|
|
|
|
|
|
def _extract_candidate_suffixes(name_lower: str) -> List[str]:
|
|
"""Extract potential suffixes from filename (last 1, 2, 3 dot-parts).
|
|
|
|
Returns suffixes in order from shortest to longest.
|
|
"""
|
|
parts = name_lower.rsplit('.', 3)
|
|
if len(parts) == 1:
|
|
return []
|
|
candidates = []
|
|
append_candidate = candidates.append
|
|
suffix = ''
|
|
for i in range(len(parts) - 1, 0, -1):
|
|
suffix = '.' + parts[i] + suffix
|
|
append_candidate(suffix)
|
|
return candidates
|
|
|
|
|
|
@functools.lru_cache(maxsize=5_000)
|
|
def _get_suffix_length(name_lower: str) -> int:
|
|
"""Get length of matching suffix using O(1) set lookups.
|
|
|
|
Returns 0 if no suffix matches.
|
|
"""
|
|
# Fast path: check hardcoded multi-part suffixes first
|
|
for suffix in _HARDCODED_SUFFIXES:
|
|
if name_lower.endswith(suffix):
|
|
return len(suffix)
|
|
|
|
candidates = _extract_candidate_suffixes(name_lower)
|
|
if not candidates:
|
|
return 0
|
|
|
|
# Check SUFFIXES (longest first for most specific match)
|
|
for suffix in reversed(candidates):
|
|
if suffix in _SUFFIXES_LOWER:
|
|
return len(suffix)
|
|
|
|
# Only check EXTRA_SUFFIXES for .srt/.vtt files
|
|
simple_suffix = candidates[0] if candidates else ''
|
|
if simple_suffix in {'.srt', '.vtt'}:
|
|
for suffix in reversed(candidates):
|
|
if suffix in _EXTRA_SUFFIXES_LOWER:
|
|
return len(suffix)
|
|
|
|
return len(simple_suffix)
|
|
|
|
|
|
@functools.lru_cache(maxsize=10_000)
|
|
def split_path_stem_and_suffix(path: Union[pathlib.Path, str], full: bool = False) -> Tuple[str, str]:
|
|
"""Get the path's stem and suffix.
|
|
|
|
This function handles WROLPi suffixes like .info.json.
|
|
|
|
>>> split_path_stem_and_suffix('/foo/bar.txt')
|
|
('bar', '.txt')
|
|
>>> split_path_stem_and_suffix('/foo/bar.txt', full=True)
|
|
('/foo/bar', '.txt')
|
|
>>> split_path_stem_and_suffix('/foo/bar.info.json', full=True)
|
|
('/foo/bar', '.info.json')
|
|
"""
|
|
path = pathlib.Path(path) if isinstance(path, str) else path
|
|
|
|
if path.suffix == '.part':
|
|
# yt-dlp uses part files while downloading, include those in a FileGroup.
|
|
stem, info, format_num, suffix, part = PART_PARSER.match(path.name).groups()
|
|
info = info or ''
|
|
format_num = format_num or ''
|
|
stem = f'{path.parent}/{stem}' if full else stem
|
|
return stem, f'{info}{format_num}{suffix}{part}'
|
|
|
|
name = path.name
|
|
name_lower = name.lower()
|
|
|
|
# Get suffix length using O(1) set lookups instead of O(n) linear search
|
|
suffix_len = _get_suffix_length(name_lower)
|
|
|
|
if suffix_len > 0:
|
|
idx = -suffix_len
|
|
if full:
|
|
return f'{path.parent}/{name[:idx]}', name[idx:]
|
|
else:
|
|
return name[:idx], name[idx:]
|
|
|
|
# Path has no suffix.
|
|
if full:
|
|
return f'{path.parent}/{path.name}', ''
|
|
else:
|
|
return path.name, ''
|
|
|
|
|
|
def get_unique_files_by_stem(files: List[pathlib.Path] | Tuple[pathlib.Path, ...] | Set[pathlib.Path]) \
|
|
-> List[pathlib.Path]:
|
|
"""Returns the first of each Path with a unique stem. Used to detect if a group of files share a stem."""
|
|
results = unique_by_predicate(files, lambda i: split_path_stem_and_suffix(i)[0])
|
|
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
|
logger.trace(f'get_unique_files_by_stem {files=}')
|
|
logger.trace(f'get_unique_files_by_stem {results=}')
|
|
return results
|
|
|
|
|
|
refresh_logger = logger.getChild('refresh')
|
|
|
|
|
|
def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]:
|
|
"""This generates `FileGroup.files` from a list of files.
|
|
|
|
Paths are stored as filenames only (relative to FileGroup.directory).
|
|
"""
|
|
files = list()
|
|
for file in group:
|
|
mimetype = get_mimetype(file)
|
|
stat = file.stat()
|
|
modification_datetime = str(from_timestamp(stat.st_mtime))
|
|
_, suffix = split_path_stem_and_suffix(file)
|
|
size = stat.st_size
|
|
files.append(dict(
|
|
path=file.name, # Store filename only, not full path
|
|
mimetype=mimetype,
|
|
modification_datetime=modification_datetime,
|
|
size=size,
|
|
suffix=suffix,
|
|
))
|
|
return files
|
|
|
|
|
|
def get_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path:
|
|
"""Given a list of files, return the file that we can model or index.
|
|
|
|
@raise NoPrimaryFile: If not primary file can be found."""
|
|
from modules.archive.lib import is_singlefile_file
|
|
|
|
if len(files) == 0:
|
|
raise ValueError(f'Cannot find primary file without any files')
|
|
elif len(files) == 1:
|
|
# Only one file is always the primary.
|
|
return files[0]
|
|
|
|
file_mimetypes = list()
|
|
for idx, i in enumerate(files):
|
|
try:
|
|
file_mimetypes.append((i, get_mimetype(i)))
|
|
except FileNotFoundError:
|
|
# File was deleted.
|
|
pass
|
|
|
|
if not file_mimetypes:
|
|
raise FileNotFoundError(f'Cannot find primary file. All files are no longer accessible: {files[0]}')
|
|
|
|
# EPUB has high priority.
|
|
mimetypes = {i[1] for i in file_mimetypes}
|
|
has_epub = any(i.startswith('application/epub') for i in mimetypes)
|
|
for file, mimetype in file_mimetypes:
|
|
# These are the files that can be modeled.
|
|
if is_singlefile_file(file):
|
|
return file
|
|
if mimetype.startswith('video/'):
|
|
return file
|
|
if mimetype.startswith('audio/'):
|
|
return file
|
|
if mimetype.startswith('application/epub'):
|
|
return file
|
|
if mimetype.startswith('application/pdf') and not has_epub:
|
|
return file
|
|
if mimetype.startswith('application/x-mobipocket-ebook') and not has_epub:
|
|
# Only return a MOBI if no EPUB is present.
|
|
return file
|
|
if mimetype in ('application/msword',
|
|
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
'application/vnd.oasis.opendocument.text',
|
|
'application/x-cbz', 'application/x-cbr',
|
|
'application/vnd.comicbook+zip', 'application/vnd.comicbook-rar'):
|
|
return file
|
|
if file.suffix.lower() in ('.cbz', '.cbr', '.cbt', '.cb7'):
|
|
return file
|
|
|
|
# Secondary file types.
|
|
for file, mimetype in file_mimetypes:
|
|
if mimetype.startswith('image/'):
|
|
return file
|
|
|
|
raise NoPrimaryFile(f'Cannot find primary file for group: {files}')
|
|
|
|
|
|
def choose_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path:
|
|
"""Primary file for a stem group. Falls back to a sorted path so every caller agrees.
|
|
|
|
`_upsert_files` and `_upsert_file_groups` must pick the same path. A set's iteration
|
|
order is not stable; using it as the fallback would DELETE the existing tagged row
|
|
and INSERT a new untagged one.
|
|
"""
|
|
paths = [pathlib.Path(p) for p in files]
|
|
if not paths:
|
|
raise ValueError('Cannot find primary file without any files')
|
|
try:
|
|
return get_primary_file(paths)
|
|
except NoPrimaryFile:
|
|
return sorted(paths)[0]
|
|
|
|
|
|
def _upsert_files(files: List[pathlib.Path],
|
|
progress_callback: Callable[[int, int], None] = None):
|
|
"""Insert/update all records of the provided files.
|
|
|
|
Any inserted files will be marked with `indexed=false`. Any existing files will only be updated if their
|
|
size/modification_datetime changed.
|
|
|
|
Args:
|
|
files: List of file paths to upsert
|
|
progress_callback: Optional callback(processed, total) called after each chunk is processed
|
|
|
|
It is assumed all files exist."""
|
|
total_files = len(files)
|
|
processed_count = 0
|
|
|
|
non_primary_files = set()
|
|
for chunk in chunks_by_stem(files, 500):
|
|
# Group all files by their shared full-stem. `chunk_by_stem` already sorted the files.
|
|
grouped = group_files_by_stem(chunk, pre_sorted=True)
|
|
# Convert the groups into file_groups. Extract common information into `file_group.data`.
|
|
values = dict()
|
|
for group in grouped:
|
|
# The primary file is the video/SingleFile/epub, etc.
|
|
try:
|
|
primary_path = choose_primary_file(group)
|
|
# The primary mimetype allows modelers to find its file_groups.
|
|
mimetype = get_mimetype(primary_path)
|
|
# The group uses a common modification_datetime so the group will be re-indexed when any of it's
|
|
# files are modified.
|
|
modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group))
|
|
size = sum(i.stat().st_size for i in group)
|
|
files_json = json.dumps(_paths_to_files_dict(group))
|
|
stem, suffix = split_path_stem_and_suffix(primary_path)
|
|
suffix = (suffix or '').lower() or None
|
|
values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem)
|
|
non_primary_files.update(i for i in group if i != primary_path)
|
|
except FileNotFoundError as e:
|
|
# A file was deleted between discovery and upsert (temp files, WAL sidecars, etc.).
|
|
refresh_logger.warning(f'File vanished during refresh, skipping group near {group[0]}', exc_info=e)
|
|
continue
|
|
|
|
values = [(str(primary_path),
|
|
str(primary_path.parent), # directory
|
|
False, # `indexed` is false to force indexing by default.
|
|
*i) for primary_path, i in values.items()]
|
|
with get_db_curs(commit=True) as curs:
|
|
# SQLite's RETURNING only exposes the post-upsert row; find the previously-indexed
|
|
# paths first so re-indexing invalidations can be counted.
|
|
primary_paths = [i[0] for i in values]
|
|
curs.execute('SELECT primary_path FROM file_group WHERE indexed = true AND primary_path IN '
|
|
'(SELECT value FROM json_each(?))', (json.dumps(primary_paths),))
|
|
previously_indexed = {i['primary_path'] for i in curs.fetchall()}
|
|
|
|
if non_primary_files:
|
|
# Delete before INSERT so a new primary_path that shares (directory, stem)
|
|
# with an old sidecar FileGroup does not hit the unique index.
|
|
curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))',
|
|
(json.dumps(list(map(str, non_primary_files))),))
|
|
|
|
placeholders, params = values_clause(values)
|
|
stmt = f'''
|
|
INSERT INTO file_group
|
|
(primary_path, directory, indexed, modification_datetime, mimetype, size, files,
|
|
suffix, stem)
|
|
VALUES {placeholders}
|
|
ON CONFLICT (primary_path) DO UPDATE
|
|
SET
|
|
directory=EXCLUDED.directory,
|
|
modification_datetime=EXCLUDED.modification_datetime,
|
|
mimetype=EXCLUDED.mimetype,
|
|
size=EXCLUDED.size,
|
|
-- Always keep suffix/stem current so grouping and suffix search stay correct.
|
|
suffix=EXCLUDED.suffix,
|
|
stem=EXCLUDED.stem,
|
|
files=(
|
|
-- Do not overwrite files unless the files have changed and need to be re-indexed.
|
|
CASE
|
|
WHEN
|
|
file_group.modification_datetime = EXCLUDED.modification_datetime
|
|
AND file_group.size = EXCLUDED.size
|
|
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
|
|
THEN file_group.files
|
|
ELSE EXCLUDED.files
|
|
END
|
|
),
|
|
-- Preserve TRUE `indexed` only if the files have not changed.
|
|
indexed=(
|
|
file_group.indexed = true
|
|
AND file_group.modification_datetime = EXCLUDED.modification_datetime
|
|
AND file_group.size = EXCLUDED.size
|
|
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
|
|
)
|
|
RETURNING id, primary_path, indexed
|
|
'''
|
|
curs.execute(stmt, params)
|
|
results = list(curs.fetchall())
|
|
# Count the files that used to be indexed, but need to be re-indexed.
|
|
invalidated_files = len([i['id'] for i in results
|
|
if i['primary_path'] in previously_indexed and not i['indexed']])
|
|
if invalidated_files:
|
|
refresh_logger.info(f'Invalidated indexes of {invalidated_files} file groups near {chunk[0]}')
|
|
else:
|
|
refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}')
|
|
|
|
non_primary_files.clear()
|
|
|
|
# Report progress after each chunk
|
|
processed_count += len(chunk)
|
|
if progress_callback:
|
|
progress_callback(processed_count, total_files)
|
|
|
|
|
|
def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]:
|
|
"""Return a new list which does not contain any file paths that are in ignored directories."""
|
|
from wrolpi.db import get_db_file
|
|
|
|
ignored_directories = get_normalized_ignored_directories()
|
|
files = [i for i in files if not any(str(i).startswith(j) for j in ignored_directories)]
|
|
# Never index the database (or its WAL sidecars), even if the user un-ignores `config`.
|
|
db_file = str(get_db_file())
|
|
files = [i for i in files if not str(i).startswith(db_file)]
|
|
return files
|
|
|
|
|
|
def _ensure_kv_table(curs):
|
|
curs.execute('CREATE TABLE IF NOT EXISTS wrolpi_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
|
|
|
|
|
|
def get_stem_algorithm_version() -> int:
|
|
"""The grouping-algorithm version last applied to file_group.stem, or 0 if never stored."""
|
|
with get_db_curs() as curs:
|
|
_ensure_kv_table(curs)
|
|
curs.execute('SELECT value FROM wrolpi_kv WHERE key = ?', (STEM_ALGORITHM_VERSION_KEY,))
|
|
row = curs.fetchone()
|
|
return int(row['value']) if row else 0
|
|
|
|
|
|
def set_stem_algorithm_version(version: int):
|
|
with get_db_curs(commit=True) as curs:
|
|
_ensure_kv_table(curs)
|
|
curs.execute(
|
|
'INSERT INTO wrolpi_kv (key, value) VALUES (?, ?) '
|
|
'ON CONFLICT(key) DO UPDATE SET value = excluded.value',
|
|
(STEM_ALGORITHM_VERSION_KEY, str(version)),
|
|
)
|
|
|
|
|
|
def ensure_file_group_stems():
|
|
"""Recompute file_group.stem when STEM_ALGORITHM_VERSION has changed.
|
|
|
|
A stored stem is a cache of split_path_stem_and_suffix. Compare joins on (directory, stem),
|
|
so an algorithm change without this backfill reports the same files as deleted+new and
|
|
drops tags, viewed timestamps, and domain rows. Must run before any compare.
|
|
|
|
After a successful backfill the kv version matches, and later refreshes skip the table scan.
|
|
Upsert, from_paths, move, and reorganize must keep stem current.
|
|
|
|
A lock serializes callers: cancelling compare_file_groups does not stop the executor
|
|
thread, and a second compare would otherwise write wrolpi_kv concurrently (SQLITE_BUSY).
|
|
"""
|
|
with _stem_backfill_lock:
|
|
stored = get_stem_algorithm_version()
|
|
if stored == STEM_ALGORITHM_VERSION:
|
|
return
|
|
with get_db_curs() as curs:
|
|
curs.execute('SELECT id, primary_path FROM file_group')
|
|
rows = list(curs.fetchall())
|
|
if rows:
|
|
updates = []
|
|
for row in rows:
|
|
stem, _ = split_path_stem_and_suffix(row['primary_path'])
|
|
updates.append((stem, row['id']))
|
|
with get_db_curs(commit=True) as curs:
|
|
for chunk in chunks(updates, 500):
|
|
curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk)
|
|
set_stem_algorithm_version(STEM_ALGORITHM_VERSION)
|
|
|
|
|
|
def backfill_null_file_group_stems():
|
|
"""Fill file_group.stem for rows that never got one. Used before creating the unique index."""
|
|
with get_db_curs() as curs:
|
|
curs.execute("SELECT id, primary_path FROM file_group WHERE stem IS NULL OR stem = ''")
|
|
rows = list(curs.fetchall())
|
|
if not rows:
|
|
return
|
|
updates = [(split_path_stem_and_suffix(row['primary_path'])[0], row['id']) for row in rows]
|
|
with get_db_curs(commit=True) as curs:
|
|
for chunk in chunks(updates, 500):
|
|
curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk)
|
|
|
|
|
|
def dedupe_file_groups_by_directory_stem() -> int:
|
|
"""Collapse FileGroups that share (directory, stem).
|
|
|
|
Prefers a row with a domain model, then tags, then non-empty files, then highest id.
|
|
Tags and collection membership on losers are moved onto the winner. Returns the
|
|
number of FileGroups deleted.
|
|
"""
|
|
with get_db_curs() as curs:
|
|
curs.execute('''
|
|
SELECT directory, stem
|
|
FROM file_group
|
|
WHERE directory IS NOT NULL
|
|
AND stem IS NOT NULL
|
|
AND stem != ''
|
|
GROUP BY directory, stem
|
|
HAVING COUNT(*) > 1
|
|
''')
|
|
keys = [(row['directory'], row['stem']) for row in curs.fetchall()]
|
|
|
|
deleted = 0
|
|
for directory, stem in keys:
|
|
with get_db_curs() as curs:
|
|
curs.execute('''
|
|
SELECT fg.id,
|
|
CASE
|
|
WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0
|
|
ELSE json_array_length(fg.files)
|
|
END AS file_count,
|
|
CASE
|
|
WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1
|
|
ELSE 0
|
|
END AS has_model,
|
|
(SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count
|
|
FROM file_group fg
|
|
WHERE fg.directory = ?
|
|
AND fg.stem = ?
|
|
''', (directory, stem))
|
|
members = list(curs.fetchall())
|
|
members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True)
|
|
winner_id = members[0]['id']
|
|
loser_ids = [m['id'] for m in members[1:]]
|
|
with get_db_curs(commit=True) as curs:
|
|
for loser_id in loser_ids:
|
|
curs.execute('''
|
|
INSERT OR IGNORE INTO tag_file (tag_id, file_group_id, created_at)
|
|
SELECT tag_id, ?, created_at
|
|
FROM tag_file
|
|
WHERE file_group_id = ?
|
|
''', (winner_id, loser_id))
|
|
curs.execute('''
|
|
UPDATE collection_item
|
|
SET file_group_id = ?
|
|
WHERE file_group_id = ?
|
|
AND collection_id NOT IN (SELECT collection_id
|
|
FROM (SELECT collection_id
|
|
FROM collection_item
|
|
WHERE file_group_id = ?))
|
|
''', (winner_id, loser_id, winner_id))
|
|
curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,))
|
|
deleted += 1
|
|
if deleted:
|
|
refresh_logger.info(f'Collapsed {deleted} duplicate FileGroups sharing (directory, stem)')
|
|
return deleted
|
|
|
|
|
|
async def apply_indexers(progress_callback: Callable[[int, int], None] = None):
|
|
"""Index leftover FileGroups that no modeler claimed.
|
|
|
|
File indexing state machine
|
|
---------------------------
|
|
A FileGroup is discovered (a row exists), then modeled and/or content-indexed.
|
|
|
|
`indexed` means "a modeler or this leftover indexer has finished with this row
|
|
for the current files." It is not "a domain model exists." `model` is the
|
|
domain claim (`video` / `archive` / `doc` / `zim`).
|
|
|
|
1. Discover/upsert: new or changed groups are inserted with indexed=0.
|
|
Unchanged groups keep indexed=1.
|
|
2. Modelers run first. A modeler that exclusively owns its mimetype
|
|
(video, doc, zim) selects:
|
|
(domain_row IS NULL OR indexed != 1) AND mimetype_matches
|
|
so a FileGroup the leftover indexer already marked indexed=1 still
|
|
gets a Video/Doc/Zim row. Failures are skipped for the rest of the
|
|
run (failed_ids) and retried on the next refresh.
|
|
3. Archive shares text/html with generic HTML. It selects only
|
|
indexed != 1. HTML that is not a SingleFile stays indexed=0 so this
|
|
function can claim it. HTML that is an Archive is indexed=1 with
|
|
model='archive'.
|
|
4. This function claims every remaining indexed != 1 row. It writes
|
|
a/b/c/d_text via the Indexer registry and sets indexed=1 even on
|
|
extractor failure so poison files do not loop forever.
|
|
5. Cleanup (channel claims, empty collections, tag config) runs last.
|
|
|
|
FileGroup.do_model() is the single-file equivalent of steps 2-4, used by
|
|
uploads. Batch modelers are used by FileWorker refresh.
|
|
|
|
Args:
|
|
progress_callback: Optional callback(processed, total) called after each batch.
|
|
"""
|
|
from wrolpi.files.models import FileGroup
|
|
refresh_logger.info('Applying indexers')
|
|
|
|
# Get initial count for progress tracking
|
|
total_to_index = 0
|
|
total_indexed = 0
|
|
if progress_callback:
|
|
with get_db_session() as session:
|
|
total_to_index = session.query(FileGroup).filter(FileGroup.indexed != True).count()
|
|
progress_callback(0, total_to_index)
|
|
|
|
update_stmt = ('UPDATE file_group SET "indexed" = 1, title = :title, a_text = :a_text, b_text = :b_text,'
|
|
' c_text = :c_text, d_text = :d_text WHERE id = :id')
|
|
|
|
def _strip_surrogates(value):
|
|
return strip_surrogates(value) if isinstance(value, str) else value
|
|
|
|
while True:
|
|
# Read the next batch as plain columns and let the read transaction end. Indexers can run
|
|
# subprocesses for minutes, and SQLite requires short write transactions: a session held
|
|
# across the batch fails its commit with "database is locked" (SQLITE_BUSY_SNAPSHOT, which
|
|
# ignores busy_timeout) whenever any other connection commits a write in the meantime.
|
|
with get_db_session() as session:
|
|
rows = session.query(FileGroup.id, FileGroup.primary_path, FileGroup.mimetype, FileGroup.title) \
|
|
.filter(FileGroup.indexed != True).limit(20).all()
|
|
|
|
if not rows:
|
|
break
|
|
|
|
# Index each file on a transient FileGroup: no session, no open transaction.
|
|
processed = 0
|
|
params = []
|
|
last_path = None
|
|
for fg_id, primary_path, mimetype, title in rows:
|
|
processed += 1
|
|
file_group = FileGroup(primary_path=primary_path, mimetype=mimetype, title=title)
|
|
try:
|
|
file_group.do_index()
|
|
except Exception:
|
|
# Error has already been logged in .do_index. It is still marked as indexed
|
|
# below; we won't try to index it again.
|
|
if PYTEST:
|
|
raise
|
|
params.append(dict(id=fg_id, title=file_group.title, a_text=file_group.a_text,
|
|
b_text=file_group.b_text, c_text=file_group.c_text, d_text=file_group.d_text))
|
|
last_path = primary_path
|
|
|
|
# Sleep to catch cancel.
|
|
await asyncio.sleep(0)
|
|
|
|
# Write the batch in one short transaction (BEGIN IMMEDIATE via commit=True).
|
|
try:
|
|
with get_db_curs(commit=True) as curs:
|
|
curs.executemany(update_stmt, params)
|
|
except UnicodeEncodeError as e:
|
|
# Paths with invalid UTF-8 surrogates produce titles/texts sqlite3 cannot store.
|
|
refresh_logger.warning(f'UnicodeEncodeError during indexer update, sanitizing batch: {e}')
|
|
params = [{k: _strip_surrogates(v) for k, v in p.items()} for p in params]
|
|
try:
|
|
with get_db_curs(commit=True) as curs:
|
|
curs.executemany(update_stmt, params)
|
|
except UnicodeEncodeError as e2:
|
|
# Still failing - mark as indexed without the texts so we don't loop forever.
|
|
refresh_logger.error(f'Failed to fix UnicodeEncodeError, skipping batch: {e2}')
|
|
with get_db_curs(commit=True) as curs:
|
|
curs.execute('UPDATE file_group SET "indexed" = 1 WHERE id IN '
|
|
'(SELECT value FROM json_each(?))',
|
|
(json.dumps([p['id'] for p in params]),))
|
|
|
|
if last_path:
|
|
refresh_logger.debug(f'Indexed {processed} files near {last_path}')
|
|
|
|
# Update progress
|
|
total_indexed += processed
|
|
if progress_callback and total_to_index > 0:
|
|
progress_callback(total_indexed, total_to_index)
|
|
|
|
if processed < 20:
|
|
# Processed less than the limit, don't do the next query.
|
|
break
|
|
|
|
|
|
def upsert_directories(parent_directories, directories):
|
|
"""
|
|
Insert/update/delete directories provided. Deletes all children of `parent_directories` which are not in
|
|
`directories`.
|
|
"""
|
|
idempotency = now()
|
|
directories = list(directories) + list(parent_directories)
|
|
|
|
# Only insert directories that are children of `media_directory` and exist.
|
|
directories = get_paths_in_media_directory([i for i in directories if i.is_dir()])
|
|
|
|
if directories:
|
|
# Insert any directories that were created, update any directories which previously existed.
|
|
# Chunk the rows so the multi-row VALUES clause stays under SQLite's bound-variable limit;
|
|
# a large media directory can hold tens of thousands of directories.
|
|
values = [(str(i.absolute()), i.name, idempotency) for i in directories]
|
|
with get_db_curs(commit=True) as curs:
|
|
for chunk in chunks(values, BULK_SQL_CHUNK_SIZE):
|
|
placeholders, params = values_clause(chunk)
|
|
stmt = f'''
|
|
INSERT INTO directory (path, name, idempotency) VALUES {placeholders}
|
|
ON CONFLICT (path) DO UPDATE
|
|
SET idempotency = EXCLUDED.idempotency
|
|
'''
|
|
curs.execute(stmt, params)
|
|
|
|
with get_db_curs(commit=True) as curs:
|
|
# Delete the children of any parent directory which no longer exists.
|
|
for directory in parent_directories:
|
|
stmt = f'DELETE FROM directory WHERE path LIKE :directory AND idempotency != :idempotency'
|
|
curs.execute(stmt, dict(directory=f'{directory.absolute()}%', idempotency=idempotency))
|
|
|
|
|
|
async def search_directories_by_name(session: Session, name: str, excluded: List[str] = None, limit: int = 20) \
|
|
-> List[Directory]:
|
|
"""Find the Directories whose names contain the `name` string."""
|
|
excluded = excluded or []
|
|
directories = session.query(Directory) \
|
|
.filter(Directory.name.ilike(f'%{name}%')) \
|
|
.filter(Directory.path.notin_(excluded)) \
|
|
.order_by(asc(Directory.name)) \
|
|
.limit(limit).all()
|
|
return directories
|
|
|
|
|
|
# Pagination totals: counting a large library is a full index walk that costs seconds cold on a
|
|
# spinning disk. Pagination can tolerate a total that is a minute stale. Bypassed under pytest
|
|
# so tests see exact totals. Keyed by filters, never by offset/limit/order (except filters that
|
|
# an order implies, e.g. viewed IS NOT NULL).
|
|
SEARCH_TOTALS_CACHE_SECONDS = 60
|
|
_SEARCH_TOTALS_CACHE: Dict[str, Tuple[float, int]] = dict()
|
|
|
|
|
|
def cached_search_total(key: str, compute: Callable[[], int]) -> int:
|
|
"""Return `compute()`; cache the result briefly unless running under pytest."""
|
|
if not PYTEST:
|
|
hit = _SEARCH_TOTALS_CACHE.get(key)
|
|
if hit and time.monotonic() - hit[0] < SEARCH_TOTALS_CACHE_SECONDS:
|
|
return hit[1]
|
|
total = compute()
|
|
_SEARCH_TOTALS_CACHE[key] = (time.monotonic(), total)
|
|
return total
|
|
|
|
|
|
def search_filter_cache_key(corpus: str, **filters) -> str:
|
|
"""Stable cache key for a search total. Omits empty/false filters."""
|
|
payload = {'corpus': corpus}
|
|
for k, v in filters.items():
|
|
if v in (None, '', [], (), False):
|
|
continue
|
|
if isinstance(v, (list, tuple, set)):
|
|
v = sorted(v)
|
|
payload[k] = v
|
|
return json.dumps(payload, sort_keys=True, default=str)
|
|
|
|
|
|
def count_file_groups(session: Session, statement: str, params: dict) -> int:
|
|
"""Run a `SELECT COUNT(*) AS total ...` statement and return the integer."""
|
|
with session_curs(session) as curs:
|
|
curs.execute(statement, params)
|
|
row = curs.fetchone()
|
|
return int(row['total'])
|
|
|
|
|
|
def search_files(session: Session, search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
|
|
tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
|
|
from_year: int = None, to_year: int = None, any_tag: bool = False, order: str = None,
|
|
url: str = None, suffix: str = None, path: str = None, deep: bool = False) -> \
|
|
Tuple[List[dict], int]:
|
|
"""Search the FileGroup table.
|
|
|
|
Order the returned Files by their rank if `search_str` is provided. Return all files if
|
|
`search_str` is empty.
|
|
|
|
@param any_tag: The file must have some tag.
|
|
@param to_year: The file must be published on or before this year.
|
|
@param from_year: The file must be published on or after this year.
|
|
@param search_str: Search the ts_vector of the file. Returns all files if this is empty.
|
|
@param limit: Return only this many files.
|
|
@param offset: Offset the query.
|
|
@param mimetypes: Only return files that match these mimetypes.
|
|
@param model: Only return files that match this model.
|
|
@param tag_names: A list of tag names.
|
|
@param headline: Includes FTS search headlines if True.
|
|
@param months: A list of integers representing the index of the month of the year, starting at 1.
|
|
@param order: Used to change results from most relevant to recently viewed.
|
|
@param url: Filter by URL using case-insensitive partial match (ILIKE).
|
|
@param suffix: Only return files whose primary file has this suffix (e.g. ".bin"), case-insensitive.
|
|
@param path: Filter by primary_path using case-insensitive partial match (ILIKE).
|
|
@param deep: Search all text including captions/body (d_text); slower but more thorough.
|
|
"""
|
|
from wrolpi import fts
|
|
|
|
# The frontend may send explicit JSON nulls (bypassing the schema defaults); SQLite rejects
|
|
# `LIMIT NULL`/`OFFSET NULL` (Postgres treated them as no-ops).
|
|
limit = int(limit) if limit else 20
|
|
offset = int(offset) if offset else 0
|
|
params = dict(offset=offset, limit=limit, search_str=search_str)
|
|
wheres = []
|
|
selects = []
|
|
order_by = '1 ASC'
|
|
joins = []
|
|
|
|
# The fast (non-deep) path excludes d_text (captions/body) from matching.
|
|
fts_search = fts.file_group_search_join(search_str, deep=deep, headlines=bool(headline)) \
|
|
if search_str else None
|
|
if search_str and not fts_search:
|
|
# Nothing usable to search (punctuation-only or all-negated query); match nothing, like
|
|
# an empty websearch_to_tsquery did.
|
|
return [], 0
|
|
if fts_search:
|
|
joins.append(fts_search.join)
|
|
selects.append(fts_search.rank_select)
|
|
params.update(fts_search.params)
|
|
order_by = '2 DESC'
|
|
|
|
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
|
|
if model:
|
|
params['model'] = model
|
|
wheres.append('model = :model')
|
|
|
|
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
|
|
wheres, params = months_selector_to_where(wheres, params, months)
|
|
wheres, params = date_range_to_where(wheres, params, from_year, to_year)
|
|
|
|
if url:
|
|
# SQLite LIKE is case-insensitive (for ASCII), matching the old ILIKE behavior.
|
|
params['url_filter'] = f'%{url}%'
|
|
wheres.append('fg.url LIKE :url_filter')
|
|
|
|
if suffix:
|
|
# Suffix is stored lowercased with a leading dot; normalize the caller's value to match the index.
|
|
normalized = suffix if suffix.startswith('.') else f'.{suffix}'
|
|
params['suffix'] = normalized.lower()
|
|
wheres.append('fg.suffix = :suffix')
|
|
|
|
if path:
|
|
params['path_filter'] = f'%{path}%'
|
|
wheres.append('fg.primary_path LIKE :path_filter')
|
|
|
|
if fts_search and headline:
|
|
# b/c/d headlines come from FTS5 snippets (computed in the fts subquery); the title
|
|
# headline is computed in Python by `handle_file_group_search_results` (title is not an
|
|
# FTS column).
|
|
headline = ', fts.b_headline, fts.c_headline, fts.d_headline'
|
|
else:
|
|
headline = ''
|
|
|
|
if order == 'viewed':
|
|
order_by = 'viewed DESC NULLS LAST, 1 ASC'
|
|
elif order == '-viewed':
|
|
order_by = 'viewed ASC NULLS LAST, 1 ASC'
|
|
|
|
viewed_only = bool(order and not search_str)
|
|
if viewed_only:
|
|
# Only filter out unviewed files if search_str is not provided.
|
|
wheres.append('viewed IS NOT NULL')
|
|
|
|
where_sql = ('WHERE\n' + '\n AND '.join(wheres)) if wheres else ''
|
|
extra_selects = f', {", ".join(selects)}' if selects else ''
|
|
join = '\n'.join(joins)
|
|
stmt = f'''
|
|
SELECT fg.id{extra_selects}
|
|
{headline}
|
|
FROM file_group fg
|
|
{join}
|
|
{where_sql}
|
|
ORDER BY {order_by}
|
|
LIMIT :limit OFFSET :offset
|
|
'''
|
|
logger.debug(stmt)
|
|
|
|
# COUNT(*) OVER() in the page query forces SQLite to visit every match before LIMIT.
|
|
# Count separately (and cache) so the page can use the datetime/id index.
|
|
if not search_str and not tag_names and not mimetypes and not model and not months \
|
|
and not from_year and not to_year and not any_tag and not url and not suffix \
|
|
and not path and not viewed_only:
|
|
count_stmt = 'SELECT COUNT(*) AS total FROM file_group'
|
|
count_params = dict()
|
|
elif fts_search:
|
|
# Count via a direct FTS join (no ranking) rather than the snippet subquery.
|
|
fts_count = fts.file_group_search(search_str, deep=deep)
|
|
count_wheres = list(wheres)
|
|
if fts_count:
|
|
count_wheres.append(fts_count.where)
|
|
count_where_sql = ('WHERE\n' + '\n AND '.join(count_wheres)) if count_wheres else ''
|
|
count_join = fts_count.join if fts_count else ''
|
|
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {count_join} {count_where_sql}'
|
|
count_params = dict(params)
|
|
if fts_count:
|
|
count_params.update(fts_count.params)
|
|
else:
|
|
count_stmt = f'SELECT COUNT(*) AS total FROM file_group fg {join} {where_sql}'
|
|
count_params = params
|
|
|
|
cache_key = search_filter_cache_key(
|
|
'files', search_str=search_str, mimetypes=mimetypes, model=model,
|
|
tag_names=tag_names, months=months, from_year=from_year, to_year=to_year,
|
|
any_tag=any_tag, url=url, suffix=suffix, path=path, deep=deep,
|
|
viewed_only=viewed_only,
|
|
)
|
|
total = cached_search_total(cache_key, lambda: count_file_groups(session, count_stmt, count_params))
|
|
results, total = handle_file_group_search_results(session, stmt, params, total=total)
|
|
return results, total
|
|
|
|
|
|
def handle_file_group_search_results(session: Session, statement: str, params: dict, total: int = None) \
|
|
-> Tuple[List[dict], int]:
|
|
"""
|
|
Execute the provided SQL statement and fetch the Files returned.
|
|
|
|
The statement must select `id`. Pass `total` from a separate COUNT (callers should not use
|
|
`COUNT(*) OVER()` in the page query; it prevents SQLite from applying LIMIT via an index).
|
|
|
|
WARNING: This expects specific queries to be executed and shouldn't be used for things not related to file search.
|
|
|
|
See: `search`
|
|
"""
|
|
with session_curs(session) as curs:
|
|
curs.execute(statement, params)
|
|
results = [dict(i) for i in curs.fetchall()]
|
|
if total is None:
|
|
total = results[0]['total'] if results else 0
|
|
ordered_ids = [i['id'] for i in results]
|
|
extras = [dict(
|
|
ts_rank=i.get('ts_rank'),
|
|
# Snippet columns exist only when headlines were requested; `title_headline` is
|
|
# computed below (title is not an FTS5 column).
|
|
headlines=('b_headline' in i),
|
|
title_headline=i.get('title_headline'),
|
|
b_headline=i.get('b_headline'),
|
|
c_headline=i.get('c_headline'),
|
|
d_headline=i.get('d_headline'),
|
|
) for i in results]
|
|
|
|
from modules.archive.models import Archive
|
|
from modules.videos.models import Video
|
|
# Serialized below: what Video's and Archive's `__json__` read beyond the FileGroup (see their `json_options`).
|
|
results = session.query(FileGroup, Video, Archive) \
|
|
.options(*FileGroup.json_options(), undefer(Video.ffprobe_json), joinedload(Video.channel),
|
|
joinedload(Archive.collection)) \
|
|
.filter(FileGroup.id.in_(ordered_ids)) \
|
|
.outerjoin(Video, Video.file_group_id == FileGroup.id) \
|
|
.outerjoin(Archive, Archive.file_group_id == FileGroup.id)
|
|
# Order FileGroups by their location in ordered_ids. Use dict for O(1) lookups instead of O(n) list.index().
|
|
id_to_order = {id_: idx for idx, id_ in enumerate(ordered_ids)}
|
|
file_groups = sorted(results, key=lambda i: id_to_order.get(i[0].id, 0))
|
|
results = list()
|
|
for extra, (file_group, video, archive) in zip_longest(extras, file_groups):
|
|
if video:
|
|
results.append(video.__json__())
|
|
elif archive:
|
|
results.append(archive.__json__())
|
|
else:
|
|
results.append(file_group.__json__())
|
|
# Preserve the ts_ranks, if any.
|
|
if extra:
|
|
results[-1]['ts_rank'] = extra['ts_rank']
|
|
results[-1]['title_headline'] = extra['title_headline']
|
|
results[-1]['b_headline'] = extra['b_headline']
|
|
results[-1]['c_headline'] = extra['c_headline']
|
|
results[-1]['d_headline'] = extra['d_headline']
|
|
|
|
search_str = params.get('search_str') if isinstance(params, dict) else None
|
|
|
|
# When headlines were requested, compute the title headlines in Python (the FTS5 snippet
|
|
# function only works on indexed columns; `title` is not one).
|
|
if search_str and any(extra and extra.get('headlines') for extra in extras):
|
|
from wrolpi import fts
|
|
titles = [result.get('title') or '' for result in results]
|
|
for result, (headline_text, rank) in zip(results, fts.headline_texts(titles, search_str)):
|
|
if rank > 0:
|
|
result['title_headline'] = headline_text
|
|
|
|
# When searching by text, attach a `section_hint` to any doc-modeled result so
|
|
# the UI can deep-link into the matching EPUB chapter or PDF page.
|
|
if search_str:
|
|
from modules.docs.lib import _fetch_section_hints
|
|
# `r['id']` is the FileGroup primary key; _fetch_section_hints filters
|
|
# on `file_group_id`, so this list is file-group ids (not doc.id).
|
|
fg_ids = [r['id'] for r in results if r.get('model') == 'doc']
|
|
if fg_ids:
|
|
hints = _fetch_section_hints(session, fg_ids, search_str)
|
|
for r in results:
|
|
hint = hints.get(r['id'])
|
|
if hint:
|
|
r['section_hint'] = hint
|
|
|
|
return results, total
|
|
|
|
|
|
def glob_shared_stem(path: pathlib.Path) -> List[pathlib.Path]:
|
|
"""Return all paths that share the same stem and parent path as the provided path.
|
|
|
|
Example paths: foo.mp4, foo.png, foo.info.json, foobar.txt
|
|
>>> glob_shared_stem(pathlib.Path('foo.mp4'))
|
|
['foo.mp4', 'foo.png', 'foo.info.json']
|
|
"""
|
|
path = path if isinstance(path, pathlib.Path) else pathlib.Path(path)
|
|
|
|
stem, suffix = split_path_stem_and_suffix(path)
|
|
escaped_stem = glob.escape(stem)
|
|
paths = [pathlib.Path(i) for i in path.parent.glob(f'{escaped_stem}*')
|
|
if i == path or i.name.startswith(f'{stem}.')]
|
|
return paths
|
|
|
|
|
|
def get_matching_directories(path: Union[str, Path]) -> List[pathlib.Path]:
|
|
"""
|
|
Return a list of directory strings that start with the provided path. If the path is a directory, return its
|
|
subdirectories, if the directory contains no subdirectories, return the directory.
|
|
"""
|
|
path: pathlib.Path = pathlib.Path(path) if isinstance(path, str) else path
|
|
path = path.resolve()
|
|
|
|
ignored_directories = {pathlib.Path(i).resolve() for i in get_wrolpi_config().ignored_directories}
|
|
|
|
if path.is_dir():
|
|
# The provided path is a directory, return its subdirectories, or itself if no subdirectories exist
|
|
paths = list(path.iterdir())
|
|
paths = sorted(i for i in paths if i.is_dir() and i not in ignored_directories)
|
|
if len(paths) == 0:
|
|
return [get_real_path_name(path.resolve())]
|
|
return paths
|
|
|
|
paths = path.parent.iterdir()
|
|
prefix = path.name.lower()
|
|
paths = sorted(
|
|
i for i in paths if i.is_dir() and i.name.lower().startswith(prefix) and i.resolve() not in ignored_directories
|
|
)
|
|
paths = unique_by_predicate(paths, None)
|
|
|
|
return paths
|
|
|
|
|
|
WHITESPACE = re.compile(r'[\s_\[\]()]')
|
|
|
|
|
|
def split_file_name_words(name: str) -> str:
|
|
"""Split words in a filename.
|
|
|
|
Words are assumed to be separated by underscore, dash, or space. Words with a dash are included as a group, and
|
|
individually ('self-reliance' -> ['self', 'reliance', 'self-reliance']).
|
|
|
|
>>> split_file_name_words('this self-reliance_split.txt')
|
|
['this', 'self', 'reliance', 'self-reliance', 'split', 'txt']
|
|
"""
|
|
if not name:
|
|
raise ValueError(f'Invalid filename: {name}')
|
|
|
|
try:
|
|
stem, suffix = split_path_stem_and_suffix(name)
|
|
words = []
|
|
for word1 in WHITESPACE.split(stem):
|
|
if '-' in word1[1:]:
|
|
words.extend(word1.split('-'))
|
|
words.append(word1)
|
|
# Include the suffix so the user can search without the "."
|
|
if suffix:
|
|
words.append(suffix.lstrip('.'))
|
|
|
|
words = ' '.join(i for i in words if i)
|
|
return words
|
|
except Exception as e:
|
|
logger.error(f'Failed to split filename into words: {name}', exc_info=e)
|
|
return name
|
|
|
|
|
|
def get_file_statistics(session: Session):
|
|
with session_curs(session) as curs:
|
|
curs.execute('''
|
|
SELECT
|
|
-- All items in file_group.files are real individual files.
|
|
SUM(json_array_length(files)) AS "total_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count",
|
|
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype =
|
|
'application/x-mobipocket-ebook') AS "ebook_count",
|
|
(SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files",
|
|
(SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims",
|
|
(SELECT COUNT(*) FROM tag) AS "tags_count",
|
|
SUM(size) AS "total_size",
|
|
(SELECT COUNT(*) FROM archive) AS archive_count
|
|
FROM file_group
|
|
''')
|
|
statistics = dict(curs.fetchall()[0])
|
|
statistics['total_count'] = statistics['total_count'] or 0
|
|
statistics['total_size'] = statistics['total_size'] or 0
|
|
|
|
return statistics
|
|
|
|
|
|
def group_files_by_stem(files: List[pathlib.Path], pre_sorted: bool = False) -> \
|
|
Generator[List[pathlib.Path], None, None]:
|
|
"""
|
|
Return lists of paths, each list contains only paths that share a common stem.
|
|
|
|
>>> a = ['foo.txt', 'foo.mp4', 'bar.txt']
|
|
>>> group_files_by_stem(list(map((pathlib.Path, a))))
|
|
# Generator(['foo.mp4', 'foo.txt'], ['bar.txt'])
|
|
|
|
@param files: A list of paths to be grouped. All paths must be in the same directory!
|
|
@param pre_sorted: This function requires the files to be sorted, it will sort them by default.
|
|
"""
|
|
files = sorted(files) if not pre_sorted else files.copy()
|
|
file = files.pop(0)
|
|
group = [file, ]
|
|
prev_dir, prev_stem = file.parent, split_path_stem_and_suffix(file)[0]
|
|
for file in files:
|
|
stem, _ = split_path_stem_and_suffix(file)
|
|
if stem == prev_stem and file.parent == prev_dir:
|
|
group.append(file)
|
|
continue
|
|
# Directory or stem changed, group is finished.
|
|
yield group
|
|
group = [file, ]
|
|
prev_dir, prev_stem = file.parent, stem
|
|
yield group
|
|
|
|
|
|
async def _get_tag_and_file_group(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
|
|
tag_id: int):
|
|
if file_group_id:
|
|
file_group: FileGroup = session.query(FileGroup).filter_by(id=file_group_id).one_or_none()
|
|
if not file_group:
|
|
raise UnknownFile(f'Cannot find FileGroup with id {file_group_id}')
|
|
elif file_group_primary_path:
|
|
path = get_media_directory() / file_group_primary_path
|
|
|
|
file_group = FileGroup.get_by_path(session, path)
|
|
# Create the FileGroup, if possible.
|
|
if not file_group and path.is_file():
|
|
file_group = FileGroup.from_paths(session, path)
|
|
|
|
if not file_group:
|
|
raise UnknownFile(f'Cannot find FileGroup with primary_path {repr(str(file_group_primary_path))}')
|
|
else:
|
|
raise UnknownFile(f'Cannot find FileGroup without id or primary_path')
|
|
|
|
if tag_id:
|
|
tag: Tag = session.query(Tag).filter_by(id=tag_id).one_or_none()
|
|
if not tag:
|
|
raise UnknownTag(f'Cannot find Tag with id {tag_id}')
|
|
elif tag_name:
|
|
tag: Tag = Tag.get_by_name(session, tag_name)
|
|
if not tag:
|
|
raise UnknownTag(f'Cannot find Tag with name {tag_name}')
|
|
else:
|
|
raise UnknownTag('Cannot find Tag without id or name')
|
|
|
|
return file_group, tag
|
|
|
|
|
|
async def add_file_group_tag(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
|
|
tag_id: int) -> TagFile:
|
|
file_group, tag = await _get_tag_and_file_group(session, file_group_id, file_group_primary_path, tag_name, tag_id)
|
|
tag_file = file_group.add_tag(session, tag.id)
|
|
return tag_file
|
|
|
|
|
|
async def remove_file_group_tag(session: Session, file_group_id: int, file_group_primary_path: str, tag_name: str,
|
|
tag_id: int):
|
|
file_group, tag = await _get_tag_and_file_group(session, file_group_id, file_group_primary_path, tag_name, tag_id)
|
|
file_group.untag(session, tag.id)
|
|
|
|
|
|
def mimetypes_to_sql_wheres(wheres: List[str], params: dict, mimetypes: List[str]) -> Tuple[List[str], dict]:
|
|
if mimetypes:
|
|
local_wheres = list()
|
|
for idx, mimetype in enumerate(mimetypes):
|
|
key = f'mimetype{idx}'
|
|
if len(mimetype.split('/')) == 1:
|
|
params[key] = f'{mimetype}/%'
|
|
else:
|
|
params[key] = f'{mimetype}%'
|
|
local_wheres.append(f'mimetype LIKE :{key}')
|
|
wheres.append(f'({" OR ".join(local_wheres)})')
|
|
return wheres, params
|
|
|
|
|
|
async def search_file_suggestion_count(session: Session, search_str: str, tag_names: List[str], mimetypes: List[str],
|
|
months: List[int] = None, from_year: int = None, to_year: int = None,
|
|
any_tag: bool = False) -> dict:
|
|
"""
|
|
Return FileGroup counts of what will be returned if the search is actually performed.
|
|
|
|
Returns both the fast (a/b/c columns) and deep (all columns) counts so the UI can show the count
|
|
matching the current search mode, and offer deep search when it has more results. Both are cheap
|
|
index-bound COUNTs (no ranking).
|
|
"""
|
|
from wrolpi import fts
|
|
|
|
wheres = []
|
|
joins = list()
|
|
params = dict()
|
|
|
|
# The deep match is a superset of the fast (column-filtered) match; both are counted with the
|
|
# same statement, differing only in the MATCH expression.
|
|
deep_match = fts.translate_websearch(search_str) if search_str else None
|
|
fast_match = fts.translate_websearch(search_str, fts.ABC_COLUMNS) if search_str else None
|
|
if search_str and deep_match is None:
|
|
# Nothing usable to search; match nothing (like an empty websearch_to_tsquery).
|
|
return dict(file_groups=0, file_groups_deep=0)
|
|
if deep_match:
|
|
joins.append('JOIN file_group_fts fts ON fts.rowid = fg.id')
|
|
wheres.append('fts.file_group_fts MATCH :fts_match')
|
|
|
|
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
|
|
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
|
|
wheres, params = months_selector_to_where(wheres, params, months)
|
|
wheres, params = date_range_to_where(wheres, params, from_year, to_year)
|
|
|
|
joins = "\n".join(joins)
|
|
wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
|
|
stmt = f'''
|
|
SELECT COUNT(*) AS estimate
|
|
FROM file_group fg
|
|
{joins}
|
|
{wheres}
|
|
'''
|
|
logger.debug(f'search_file_suggestion_count: {stmt} {params}')
|
|
|
|
with session_curs(session) as curs:
|
|
if deep_match:
|
|
curs.execute(stmt, {**params, 'fts_match': deep_match})
|
|
deep_estimate = int(curs.fetchone()['estimate'])
|
|
curs.execute(stmt, {**params, 'fts_match': fast_match})
|
|
estimate = int(curs.fetchone()['estimate'])
|
|
else:
|
|
curs.execute(stmt, params)
|
|
estimate = deep_estimate = int(curs.fetchone()['estimate'])
|
|
return dict(file_groups=estimate, file_groups_deep=deep_estimate)
|
|
|
|
|
|
MOVE_CHUNK_SIZE = 100
|
|
|
|
|
|
def _move_file_group_files(file_group: FileGroup, new_primary_path: pathlib.Path) -> None:
|
|
"""Move all physical files of a FileGroup to a new directory.
|
|
|
|
For directory moves where filenames are preserved, this moves each file from
|
|
the old directory to the new directory keeping the same filename.
|
|
|
|
Args:
|
|
file_group: The FileGroup whose files should be moved
|
|
new_primary_path: The new path for the primary file
|
|
"""
|
|
new_directory = new_primary_path.parent
|
|
new_stem, _ = split_path_stem_and_suffix(new_primary_path, full=True)
|
|
|
|
# Move each file in the FileGroup
|
|
for file_info in file_group.files:
|
|
relative_name = file_info['path']
|
|
old_path = file_group.directory / relative_name
|
|
# Calculate new path: new_stem + original suffix
|
|
_, suffix = split_path_stem_and_suffix(relative_name)
|
|
new_path = pathlib.Path(f'{new_stem}{suffix}')
|
|
|
|
if old_path.is_file():
|
|
shutil.move(old_path, new_path)
|
|
logger.debug(f'Moved file: {old_path} -> {new_path}')
|
|
|
|
|
|
def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path, pathlib.Path]):
|
|
"""Batch UPDATE FileGroups using raw SQL after physical files have been moved.
|
|
|
|
Only updates directory and primary_path. Other fields (files, title, a_text) remain unchanged
|
|
because filenames are preserved in directory moves - only the parent directory changes.
|
|
|
|
Args:
|
|
session: Database session
|
|
chunk_plan: Mapping of old_primary_path -> new_primary_path
|
|
"""
|
|
if not chunk_plan:
|
|
return
|
|
|
|
# Build the update rows: [old_path, new_dir, new_path]
|
|
updates = json.dumps([
|
|
[str(old_path), str(new_path.parent), str(new_path)]
|
|
for old_path, new_path in chunk_plan.items()
|
|
])
|
|
|
|
# Use UPDATE FROM json_each pattern for efficient batch update. This runs on the caller's
|
|
# session (rather than a second connection) so it shares the move's transaction: a second
|
|
# connection would deadlock against the write lock the caller already holds, and its separate
|
|
# commit would survive a rollback of the rest of the move.
|
|
sql = '''
|
|
UPDATE file_group AS fg SET
|
|
directory = json_extract(v.value, '$[1]'),
|
|
primary_path = json_extract(v.value, '$[2]'),
|
|
indexed = CASE WHEN fg.data IS NULL THEN FALSE ELSE fg.indexed END
|
|
FROM json_each(:updates) AS v
|
|
WHERE fg.primary_path = json_extract(v.value, '$[0]')
|
|
'''
|
|
session.execute(sa_text(sql), dict(updates=updates))
|
|
|
|
from wrolpi.collections.config import file_groups_moved
|
|
new_paths = [str(new_path) for new_path in chunk_plan.values()]
|
|
file_groups_moved(session, session.query(FileGroup.id).filter(FileGroup.primary_path.in_(new_paths)))
|
|
|
|
|
|
def _json_serial(obj):
|
|
"""JSON serializer for objects not serializable by default json code."""
|
|
import datetime
|
|
if isinstance(obj, (datetime.datetime, datetime.date)):
|
|
return obj.isoformat()
|
|
raise TypeError(f"Type {type(obj)} not serializable")
|
|
|
|
|
|
def _bulk_update_file_groups_reorganize(updates: List[dict]):
|
|
"""Batch UPDATE FileGroups for reorganization using raw SQL.
|
|
|
|
Unlike _bulk_update_file_groups_db, this also updates the files and data JSON fields
|
|
since reorganization changes filenames, not just directories.
|
|
|
|
Args:
|
|
updates: List of dicts with keys: id, directory, primary_path, files, data
|
|
"""
|
|
if not updates:
|
|
return
|
|
|
|
with get_db_curs(commit=True) as curs:
|
|
for update in updates:
|
|
# Don't change indexed - reorganization only moves files, content unchanged
|
|
primary_path = update['primary_path']
|
|
stem, _ = split_path_stem_and_suffix(primary_path)
|
|
curs.execute('''
|
|
UPDATE file_group
|
|
SET directory = ?,
|
|
primary_path = ?,
|
|
files = ?,
|
|
data = ?,
|
|
stem = ?
|
|
WHERE id = ?
|
|
''', (
|
|
update['directory'],
|
|
primary_path,
|
|
json.dumps(update['files'], default=_json_serial),
|
|
json.dumps(update['data'], default=_json_serial) if update.get('data') else None,
|
|
stem,
|
|
update['id'],
|
|
))
|
|
|
|
# Expire SQLAlchemy's session cache so subsequent queries see the raw SQL updates.
|
|
# This is necessary because get_db_curs uses the same connection as get_db_session
|
|
# during tests, but SQLAlchemy's identity map doesn't know about raw SQL updates.
|
|
with get_db_session() as session:
|
|
session.expire_all()
|
|
from wrolpi.collections.config import file_groups_moved
|
|
file_groups_moved(session, [update['id'] for update in updates])
|
|
|
|
|
|
def delete_directory(directory: pathlib.Path, recursive: bool = False, force: bool = False):
|
|
"""Remove a directory, remove it's Directory record.
|
|
|
|
Will refuse to delete a directory if it contains Tagged Files, unless force is True.
|
|
|
|
This function is idempotent - safe to call even if the directory was already deleted."""
|
|
if directory.exists():
|
|
if recursive:
|
|
if not force:
|
|
with get_db_session() as session:
|
|
tagged = get_tagged_file_groups_by_paths(session, [directory])
|
|
if tagged:
|
|
raise FileGroupIsTagged(f'Cannot delete {tagged[0]} because it is tagged')
|
|
shutil.rmtree(directory)
|
|
else:
|
|
directory.rmdir()
|
|
# Always clean up DB record even if directory was already deleted
|
|
with get_db_session(commit=True) as session:
|
|
session.query(Directory).filter_by(path=str(directory)).delete(synchronize_session=False)
|
|
|
|
|
|
async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
|
|
"""Rename a file (and it's associated files). Preserve any tags.
|
|
|
|
If the path is a non-primary file in a FileGroup (e.g., a poster or subtitle),
|
|
the entire FileGroup will be renamed.
|
|
"""
|
|
new_path = path.with_name(new_name)
|
|
if not path.exists():
|
|
raise FileNotFoundError(f'Cannot find {path} to rename')
|
|
if new_path.exists():
|
|
raise FileConflict(f'Cannot rename {path} because {new_path} already exists')
|
|
|
|
with get_db_session(commit=True) as session:
|
|
# Find the FileGroup - it may be the primary path or a secondary file
|
|
fg: FileGroup = FileGroup.get_by_any_file_path(session, path)
|
|
if not fg:
|
|
# File wasn't yet in the DB.
|
|
fg = FileGroup.from_paths(session, path)
|
|
|
|
# If this is not the primary file, calculate the new primary path
|
|
if fg.primary_path != path:
|
|
# Extract the new stem and apply it to the primary file's suffix
|
|
new_stem, _ = split_path_stem_and_suffix(new_path, full=True)
|
|
_, primary_suffix = split_path_stem_and_suffix(fg.primary_path, full=False)
|
|
new_primary_path = pathlib.Path(f'{new_stem}{primary_suffix}')
|
|
fg.move(new_primary_path)
|
|
else:
|
|
fg.move(new_path)
|
|
|
|
from wrolpi.collections.config import file_groups_moved
|
|
file_groups_moved(session, [fg.id])
|
|
|
|
return new_path
|
|
|
|
|
|
async def rename_directory(session: Session, directory: pathlib.Path, new_name: str, send_events: bool = False) \
|
|
-> pathlib.Path:
|
|
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory."""
|
|
from wrolpi.files.worker import file_worker
|
|
|
|
new_directory = directory.with_name(new_name)
|
|
if new_directory.exists():
|
|
raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.')
|
|
|
|
# Check if this directory is a Collection's directory
|
|
from wrolpi.collections.models import Collection
|
|
collection: Collection = session.query(Collection).filter(Collection.directory == directory).one_or_none()
|
|
|
|
if collection:
|
|
# Use Collection's move method which handles files, config saves, and cleanup
|
|
new_directory.mkdir(parents=True, exist_ok=True)
|
|
await collection.move_collection(new_directory, session, send_events=send_events)
|
|
else:
|
|
# Regular directory rename - move all paths into the new directory.
|
|
paths = list(directory.iterdir())
|
|
try:
|
|
job_id = file_worker.queue_move(new_directory, paths)
|
|
await file_worker.wait_for_job(job_id)
|
|
if send_events:
|
|
Events.send_file_move_completed(f'Directory has been renamed: {directory}')
|
|
except Exception:
|
|
if send_events:
|
|
Events.send_file_move_failed(f'Directory rename has failed: {directory}')
|
|
raise
|
|
|
|
# Update nested Collections (files already moved by file_worker above)
|
|
directory_str = str(directory)
|
|
nested_collections = session.query(Collection).filter(
|
|
Collection.directory.like(f'{directory_str}/%')
|
|
).all()
|
|
for coll in nested_collections:
|
|
relative = coll.directory.relative_to(directory)
|
|
new_coll_dir = new_directory / relative
|
|
await coll.move_collection(new_coll_dir, session, send_events=send_events, with_files=False)
|
|
|
|
# Remove the old directory (recursive=True to clean up any empty subdirectories
|
|
# that remained after files were moved out).
|
|
delete_directory(directory, recursive=True)
|
|
|
|
return new_directory
|
|
|
|
|
|
async def rename(session: Session, path: pathlib.Path, new_name: str, send_events: bool = False) -> pathlib.Path:
|
|
"""Rename a directory or file. Preserve any tags."""
|
|
if path.is_dir():
|
|
return await rename_directory(session, path, new_name, send_events=send_events)
|
|
|
|
return await rename_file(path, new_name)
|
|
|
|
|
|
def get_special_directories() -> List[pathlib.Path]:
|
|
"""Return absolute directories that must never be ignored by file refresh.
|
|
|
|
These are module destinations that depend on FileGroup indexing (or are the media root).
|
|
Map is intentionally excluded: it lists PMTiles by walking the filesystem and benefits
|
|
from being ignored during refresh (very large files).
|
|
|
|
Must not create directories — this runs during refresh ignore filtering and would
|
|
otherwise mkdir videos/ (and NO CHANNEL/) as a side effect of the videos helpers.
|
|
"""
|
|
from modules.archive.lib import get_archive_directory
|
|
from modules.videos.lib import format_videos_destination
|
|
|
|
media_directory = get_media_directory()
|
|
videos_directory = media_directory / format_videos_destination()
|
|
# Avoid importing modules.zim.lib here (circular import via files.worker).
|
|
zims_directory = media_directory / get_wrolpi_config().zims_destination
|
|
return [
|
|
media_directory.resolve(),
|
|
videos_directory.resolve(),
|
|
get_archive_directory().resolve(),
|
|
(videos_directory / 'NO CHANNEL').resolve(),
|
|
zims_directory.resolve(),
|
|
]
|
|
|
|
|
|
def get_normalized_ignored_directories() -> List[str]:
|
|
"""Return absolute ignored directory paths, excluding special directories.
|
|
|
|
Special directories (media root, videos, archives, zims, …) are never treated as ignored,
|
|
even if they appear in config from older installs or manual edits.
|
|
"""
|
|
media_directory = get_media_directory()
|
|
special = {str(p) for p in get_special_directories()}
|
|
result = []
|
|
for d in get_wrolpi_config().ignored_directories or []:
|
|
path = pathlib.Path(d)
|
|
absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve())
|
|
if absolute in special:
|
|
continue
|
|
result.append(absolute)
|
|
return result
|
|
|
|
|
|
def sanitize_ignored_directories():
|
|
"""Persistently remove special directories from the ignored list if present."""
|
|
wrolpi_config = get_wrolpi_config()
|
|
ignored = list(wrolpi_config.ignored_directories or [])
|
|
if not ignored:
|
|
return
|
|
|
|
media_directory = get_media_directory()
|
|
special = {str(p) for p in get_special_directories()}
|
|
cleaned = []
|
|
removed = False
|
|
for d in ignored:
|
|
path = pathlib.Path(d)
|
|
absolute = str((media_directory / path).resolve() if not path.is_absolute() else path.resolve())
|
|
if absolute in special:
|
|
removed = True
|
|
logger.warning(f'Removing special directory from ignored_directories: {d}')
|
|
continue
|
|
cleaned.append(d)
|
|
|
|
if removed:
|
|
wrolpi_config.ignored_directories = cleaned
|
|
|
|
|
|
def add_ignore_directory(directory: Union[pathlib.Path, str]):
|
|
"""Add a directory to the `ignored_directories` in the WROLPi config. This directory will be ignored when
|
|
refreshing."""
|
|
media_directory = get_media_directory().resolve()
|
|
|
|
directory = pathlib.Path(directory)
|
|
if not directory.is_absolute():
|
|
directory = media_directory / directory
|
|
directory = directory.resolve()
|
|
|
|
# Drop any special directories that were previously saved as ignored.
|
|
sanitize_ignored_directories()
|
|
|
|
special_directories = set(get_special_directories())
|
|
if directory in special_directories:
|
|
raise InvalidDirectory('Refusing to ignore special directory')
|
|
|
|
if not str(directory).startswith(str(media_directory)):
|
|
raise InvalidDirectory('Cannot ignore directory not in media directory')
|
|
|
|
wrolpi_config = get_wrolpi_config()
|
|
ignored_directories = wrolpi_config.ignored_directories if wrolpi_config.ignored_directories else list()
|
|
|
|
# Accept either absolute or relative forms already present in config.
|
|
relative = str(get_relative_to_media_directory(directory))
|
|
if str(directory) in ignored_directories or relative in ignored_directories:
|
|
logger.warning(f'Directory is already ignored: {directory}')
|
|
return
|
|
|
|
ignored_directories.append(str(directory))
|
|
wrolpi_config.ignored_directories = ignored_directories
|
|
|
|
|
|
def remove_ignored_directory(directory: Union[pathlib.Path, str]):
|
|
"""Remove a directory from the `ignored_directories` in the WROLPi config."""
|
|
media_directory = get_media_directory()
|
|
path = pathlib.Path(directory)
|
|
if not path.is_absolute():
|
|
path = media_directory / path
|
|
path = path.resolve()
|
|
absolute = str(path)
|
|
|
|
ignored_directories = list(get_wrolpi_config().ignored_directories or [])
|
|
kept = []
|
|
removed = False
|
|
for d in ignored_directories:
|
|
entry = pathlib.Path(d)
|
|
entry_abs = str((media_directory / entry).resolve() if not entry.is_absolute() else entry.resolve())
|
|
if entry_abs == absolute:
|
|
removed = True
|
|
continue
|
|
kept.append(d)
|
|
|
|
if not removed:
|
|
raise UnknownDirectory('Directory is not ignored')
|
|
get_wrolpi_config().ignored_directories = kept
|
|
|
|
|
|
async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> FileGroup:
|
|
"""Insert/update all files in the provided file's FileGroup."""
|
|
if not file or not file.is_file():
|
|
raise InvalidFile(f'Cannot upsert file that does not exist: {file}')
|
|
|
|
# Update/Insert all files in the FileGroup.
|
|
paths = glob_shared_stem(pathlib.Path(file))
|
|
# Remove any ignored files.
|
|
paths = remove_files_in_ignored_directories(paths)
|
|
if not paths:
|
|
logger.warning('upsert_file called, but all files are in ignored directories!')
|
|
raise IgnoredDirectoryError(f'all files are in ignored directories: {file}')
|
|
|
|
# Use a single session context for all database operations
|
|
with get_db_session(commit=True) as session:
|
|
# Create/update FileGroup with retry logic
|
|
file_group = None
|
|
for i in range(2):
|
|
try:
|
|
file_group = FileGroup.from_paths(session, *paths)
|
|
session.flush([file_group])
|
|
break
|
|
except IntegrityError:
|
|
# Another process inserted this FileGroup.
|
|
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
|
|
session.rollback()
|
|
if i == 1:
|
|
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
|
|
continue
|
|
|
|
if not file_group:
|
|
raise RuntimeError(f'upsert_file failed to create FileGroup! {file}')
|
|
|
|
logger.debug(f'upsert_file: {file_group}')
|
|
|
|
# Model the FileGroup (creates Archive/Video/etc. and sets indexed=True)
|
|
try:
|
|
file_group.do_model(session)
|
|
except Exception as e:
|
|
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
|
|
if PYTEST:
|
|
raise
|
|
|
|
# If user uploads a file, then remove it from the download skip list so comments can be downloaded.
|
|
# Modify the download (if any) so that the user can click on it and view the uploaded file.
|
|
if file_group.url:
|
|
if download_manager.is_skipped(file_group.url):
|
|
download_manager.remove_from_skip_list(file_group.url)
|
|
if download := Download.get_by_url(session, file_group.url):
|
|
# Mark download as completed
|
|
download.complete()
|
|
# Link the download to the location of the uploaded file.
|
|
model = file_group.get_model_record(session)
|
|
download.location = model.location if model else file_group.location
|
|
|
|
# Add tags if provided
|
|
if tag_names:
|
|
for tag_name in tag_names:
|
|
if tag_name not in file_group.tag_names:
|
|
tag = Tag.get_by_name(session, tag_name)
|
|
file_group.add_tag(session, tag.id)
|
|
|
|
# Commit all changes in one transaction
|
|
session.commit()
|
|
|
|
# Upsert directories after FileGroup is committed
|
|
upsert_directories([], file.parents)
|
|
|
|
return file_group
|
|
|
|
|
|
def get_file_location_href(file: pathlib.Path) -> str:
|
|
"""Return the location a file can be viewed at on the UI."""
|
|
parent = str(get_relative_to_media_directory(file.parent))
|
|
preview = str(get_relative_to_media_directory(file))
|
|
if parent == '.':
|
|
# File is in the top of the media directory, App already shows top directory open.
|
|
query = urllib.parse.urlencode(dict(preview=str(preview)))
|
|
else:
|
|
query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
|
|
return f'/files?{query}'
|
|
|
|
|
|
def get_real_path_name(path: pathlib.Path) -> pathlib.Path:
|
|
"""Return the real path name of a file. This is used to get the real path of a file on macOS."""
|
|
if IS_MACOS:
|
|
for path_ in path.parent.iterdir():
|
|
if path_.name.lower() == path.name.lower():
|
|
return path_
|
|
return path
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class BulkTagPreview:
|
|
"""Preview information for a bulk tagging operation."""
|
|
file_count: int = 0
|
|
shared_tag_names: List[str] = dataclasses.field(default_factory=list)
|
|
|
|
def __json__(self) -> dict:
|
|
return dict(
|
|
file_count=self.file_count,
|
|
shared_tag_names=self.shared_tag_names,
|
|
)
|
|
|
|
|
|
def get_bulk_tag_preview(session: Session, paths: List[str]) -> BulkTagPreview:
|
|
"""Get preview information for bulk tagging operation.
|
|
|
|
Returns the count of files that will be tagged and the tags that are shared by ALL files.
|
|
"""
|
|
media_directory = get_media_directory()
|
|
absolute_paths = [media_directory / p for p in paths]
|
|
|
|
# Collect all file paths (expanding directories recursively)
|
|
all_file_paths: List[pathlib.Path] = []
|
|
for path in absolute_paths:
|
|
if path.is_file():
|
|
all_file_paths.append(path)
|
|
elif path.is_dir():
|
|
for file in walk(path):
|
|
if file.is_file():
|
|
all_file_paths.append(file)
|
|
|
|
# Remove files in ignored directories
|
|
all_file_paths = remove_files_in_ignored_directories(all_file_paths)
|
|
|
|
if not all_file_paths:
|
|
return BulkTagPreview(file_count=0, shared_tag_names=[])
|
|
|
|
# Get unique files by stem (each FileGroup counted once)
|
|
unique_files = get_unique_files_by_stem(all_file_paths)
|
|
file_count = len(unique_files)
|
|
|
|
if file_count == 0:
|
|
return BulkTagPreview(file_count=0, shared_tag_names=[])
|
|
|
|
# Find tags shared by ALL files
|
|
# First, get all FileGroups and their tags
|
|
# We query by stem prefix because unique_files may contain non-primary files
|
|
# (e.g., .readability.json) but FileGroups are stored with primary_path (e.g., .html)
|
|
from sqlalchemy import or_
|
|
stems = [split_path_stem_and_suffix(f, full=True)[0] for f in unique_files]
|
|
# Chunk the OR'd LIKEs to stay within SQLite's expression-tree depth limit of 1000.
|
|
file_groups = []
|
|
for idx in range(0, len(stems), 500):
|
|
chunk = stems[idx:idx + 500]
|
|
file_groups.extend(session.query(FileGroup).filter(
|
|
or_(*[FileGroup.primary_path.like(f'{stem}%') for stem in chunk])
|
|
).all())
|
|
|
|
if not file_groups:
|
|
# No FileGroups exist yet, no shared tags
|
|
return BulkTagPreview(file_count=file_count, shared_tag_names=[])
|
|
|
|
# Get tag names for each FileGroup
|
|
tag_sets = []
|
|
for fg in file_groups:
|
|
tag_sets.append(set(fg.tag_names))
|
|
|
|
# Find intersection of all tag sets (tags shared by ALL files)
|
|
if tag_sets:
|
|
shared_tags = tag_sets[0]
|
|
for tag_set in tag_sets[1:]:
|
|
shared_tags = shared_tags.intersection(tag_set)
|
|
shared_tag_names = sorted(list(shared_tags))
|
|
else:
|
|
shared_tag_names = []
|
|
|
|
return BulkTagPreview(file_count=file_count, shared_tag_names=shared_tag_names)
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class BulkTagProgress:
|
|
"""Progress information for bulk tagging operation."""
|
|
status: str = 'idle' # 'idle', 'running'
|
|
total: int = 0
|
|
completed: int = 0
|
|
add_tag_names: List[str] = dataclasses.field(default_factory=list)
|
|
remove_tag_names: List[str] = dataclasses.field(default_factory=list)
|
|
error: str = None
|
|
queued_jobs: int = 0
|
|
|
|
def __json__(self) -> dict:
|
|
return dict(
|
|
status=self.status,
|
|
total=self.total,
|
|
completed=self.completed,
|
|
add_tag_names=self.add_tag_names,
|
|
remove_tag_names=self.remove_tag_names,
|
|
error=self.error,
|
|
queued_jobs=self.queued_jobs,
|
|
)
|
|
|
|
|
|
def get_bulk_tag_progress() -> BulkTagProgress:
|
|
"""Get the current progress of bulk tagging operations."""
|
|
from wrolpi.api_utils import api_app
|
|
|
|
bulk_tag = api_app.shared_ctx.bulk_tag
|
|
return BulkTagProgress(
|
|
status=bulk_tag.get('status', 'idle'),
|
|
total=bulk_tag.get('total', 0),
|
|
completed=bulk_tag.get('completed', 0),
|
|
add_tag_names=list(bulk_tag.get('add_tag_names', [])),
|
|
remove_tag_names=list(bulk_tag.get('remove_tag_names', [])),
|
|
error=bulk_tag.get('error'),
|
|
queued_jobs=bulk_tag.get('queued_jobs', 0),
|
|
)
|
|
|
|
|
|
def queue_bulk_tag_job(paths: List[str], add_tag_names: List[str], remove_tag_names: List[str]):
|
|
"""Add a bulk tagging job to the queue."""
|
|
from wrolpi.api_utils import api_app
|
|
|
|
job = dict(
|
|
paths=paths,
|
|
add_tag_names=add_tag_names,
|
|
remove_tag_names=remove_tag_names,
|
|
)
|
|
api_app.shared_ctx.bulk_tag_queue.put(job)
|
|
|
|
# Reset progress state and update queued jobs count
|
|
try:
|
|
current_queued = api_app.shared_ctx.bulk_tag.get('queued_jobs', 0)
|
|
api_app.shared_ctx.bulk_tag.update(dict(
|
|
status='idle',
|
|
total=0,
|
|
completed=0,
|
|
error=None,
|
|
queued_jobs=current_queued + 1,
|
|
))
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
BULK_TAG_CHUNK_SIZE = 100 # Files per DB session
|
|
BULK_SQL_CHUNK_SIZE = 1000 # Rows per SQL statement
|
|
|
|
|
|
def _bulk_insert_tag_files(tag_files: List[Tuple[int, int]]):
|
|
"""Bulk insert TagFiles, ignoring duplicates.
|
|
|
|
Args:
|
|
tag_files: List of (tag_id, file_group_id) tuples
|
|
"""
|
|
if not tag_files:
|
|
return
|
|
|
|
now_timestamp = now()
|
|
|
|
# Process in chunks to avoid overly large SQL statements
|
|
for i in range(0, len(tag_files), BULK_SQL_CHUNK_SIZE):
|
|
chunk = tag_files[i:i + BULK_SQL_CHUNK_SIZE]
|
|
values = [(tag_id, file_group_id, now_timestamp)
|
|
for tag_id, file_group_id in chunk]
|
|
|
|
with get_db_curs(commit=True) as curs:
|
|
placeholders, params = values_clause(values)
|
|
stmt = f'''
|
|
INSERT INTO tag_file (tag_id, file_group_id, created_at)
|
|
VALUES {placeholders}
|
|
ON CONFLICT (tag_id, file_group_id) DO NOTHING
|
|
'''
|
|
curs.execute(stmt, params)
|
|
logger.debug(f'Bulk inserted {len(chunk)} TagFiles')
|
|
|
|
|
|
def _bulk_delete_tag_files(tag_files: List[Tuple[int, int]]):
|
|
"""Bulk delete TagFiles.
|
|
|
|
Args:
|
|
tag_files: List of (tag_id, file_group_id) tuples
|
|
"""
|
|
if not tag_files:
|
|
return
|
|
|
|
# Process in chunks
|
|
for i in range(0, len(tag_files), BULK_SQL_CHUNK_SIZE):
|
|
chunk = tag_files[i:i + BULK_SQL_CHUNK_SIZE]
|
|
pairs = [(tag_id, file_group_id) for tag_id, file_group_id in chunk]
|
|
|
|
with get_db_curs(commit=True) as curs:
|
|
placeholders, params = values_clause(pairs)
|
|
stmt = f'''
|
|
DELETE FROM tag_file
|
|
WHERE (tag_id, file_group_id) IN (VALUES {placeholders})
|
|
'''
|
|
curs.execute(stmt, params)
|
|
logger.debug(f'Bulk deleted {curs.rowcount} TagFiles')
|
|
|
|
|
|
async def _process_bulk_tag_job(job: dict):
|
|
"""Process a single bulk tagging job with batched DB operations.
|
|
|
|
This function uses a three-phase approach for performance:
|
|
1. Collection: Gather all files and resolve FileGroups, collecting tag operations
|
|
2. Bulk DB operations: Execute bulk INSERT and DELETE for TagFiles
|
|
3. Switch activation: Call save_tags_config and sync_tags_directory once at the end
|
|
"""
|
|
from wrolpi.api_utils import api_app
|
|
|
|
paths = job['paths']
|
|
add_tag_names = job['add_tag_names']
|
|
remove_tag_names = job['remove_tag_names']
|
|
|
|
media_directory = get_media_directory()
|
|
absolute_paths = [media_directory / p for p in paths]
|
|
|
|
# Collect all file paths (expanding directories recursively)
|
|
all_file_paths: List[pathlib.Path] = []
|
|
for path in absolute_paths:
|
|
if path.is_file():
|
|
all_file_paths.append(path)
|
|
elif path.is_dir():
|
|
for file in walk(path):
|
|
if file.is_file():
|
|
all_file_paths.append(file)
|
|
|
|
# Remove files in ignored directories
|
|
all_file_paths = remove_files_in_ignored_directories(all_file_paths)
|
|
|
|
if not all_file_paths:
|
|
return
|
|
|
|
# Get unique files by stem
|
|
unique_files = get_unique_files_by_stem(all_file_paths)
|
|
|
|
# Update progress with total count
|
|
api_app.shared_ctx.bulk_tag.update(dict(
|
|
status='running',
|
|
total=len(unique_files),
|
|
completed=0,
|
|
add_tag_names=add_tag_names,
|
|
remove_tag_names=remove_tag_names,
|
|
error=None,
|
|
))
|
|
|
|
# Pre-resolve tag IDs (using cached lookups)
|
|
add_tag_ids = set()
|
|
remove_tag_ids = set()
|
|
with get_db_session() as session:
|
|
for tag_name in add_tag_names:
|
|
try:
|
|
add_tag_ids.add(Tag.get_id_by_name(session, tag_name))
|
|
except UnknownTag:
|
|
logger.warning(f'Unknown tag to add: {tag_name}')
|
|
for tag_name in remove_tag_names:
|
|
try:
|
|
remove_tag_ids.add(Tag.get_id_by_name(session, tag_name))
|
|
except UnknownTag:
|
|
logger.warning(f'Unknown tag to remove: {tag_name}')
|
|
|
|
if not add_tag_ids and not remove_tag_ids:
|
|
return
|
|
|
|
# Phase 1: Collect all file_group_ids and required operations
|
|
tag_files_to_insert: List[Tuple[int, int]] = [] # (tag_id, file_group_id)
|
|
tag_files_to_delete: List[Tuple[int, int]] = [] # (tag_id, file_group_id)
|
|
|
|
completed = 0
|
|
|
|
# Process files in chunks to balance memory and DB pressure
|
|
for chunk_start in range(0, len(unique_files), BULK_TAG_CHUNK_SIZE):
|
|
chunk = unique_files[chunk_start:chunk_start + BULK_TAG_CHUNK_SIZE]
|
|
|
|
with get_db_session(commit=True) as session:
|
|
for file_path in chunk:
|
|
try:
|
|
# Get or create FileGroup
|
|
file_group = FileGroup.get_by_path(session, file_path)
|
|
if not file_group:
|
|
# Create FileGroup for unindexed file
|
|
paths_for_group = glob_shared_stem(file_path)
|
|
paths_for_group = remove_files_in_ignored_directories(paths_for_group)
|
|
if paths_for_group:
|
|
try:
|
|
file_group = FileGroup.from_paths(session, *paths_for_group)
|
|
file_group.do_model(session)
|
|
session.flush([file_group])
|
|
except (NoPrimaryFile, IntegrityError) as e:
|
|
logger.warning(f'Could not create FileGroup for {file_path}: {e}')
|
|
completed += 1
|
|
api_app.shared_ctx.bulk_tag['completed'] = completed
|
|
continue
|
|
|
|
if file_group and file_group.id:
|
|
# Flush to ensure file_group.id is set
|
|
session.flush([file_group])
|
|
|
|
# Get current tag_ids for this file_group
|
|
existing_tag_ids = {tf.tag_id for tf in file_group.tag_files}
|
|
|
|
# Determine tags to add (not already present)
|
|
for tag_id in add_tag_ids:
|
|
if tag_id not in existing_tag_ids:
|
|
tag_files_to_insert.append((tag_id, file_group.id))
|
|
|
|
# Determine tags to remove (currently present)
|
|
for tag_id in remove_tag_ids:
|
|
if tag_id in existing_tag_ids:
|
|
tag_files_to_delete.append((tag_id, file_group.id))
|
|
|
|
except Exception as e:
|
|
logger.error(f'Error processing file {file_path}: {e}', exc_info=e)
|
|
|
|
completed += 1
|
|
api_app.shared_ctx.bulk_tag['completed'] = completed
|
|
|
|
# Allow cancellation between chunks
|
|
await asyncio.sleep(0)
|
|
|
|
# Phase 2: Bulk database operations
|
|
_bulk_insert_tag_files(tag_files_to_insert)
|
|
_bulk_delete_tag_files(tag_files_to_delete)
|
|
|
|
# Phase 3: Activate switches ONCE at the end
|
|
if tag_files_to_insert or tag_files_to_delete:
|
|
save_tags_config.activate_switch()
|
|
sync_tags_directory.activate_switch()
|
|
|
|
|
|
DISABLE_BULK_TAG_WORKER = bool(PYTEST)
|
|
|
|
|
|
async def bulk_tag_worker():
|
|
"""Background worker that processes bulk tagging jobs from the queue."""
|
|
import queue
|
|
from wrolpi.api_utils import api_app
|
|
|
|
if PYTEST and DISABLE_BULK_TAG_WORKER:
|
|
return
|
|
|
|
bulk_tag_ctx = api_app.shared_ctx.bulk_tag
|
|
bulk_tag_queue = api_app.shared_ctx.bulk_tag_queue
|
|
|
|
try:
|
|
# Try to get a job from the queue (non-blocking)
|
|
job = bulk_tag_queue.get_nowait()
|
|
except queue.Empty:
|
|
# No jobs in queue, stay idle
|
|
bulk_tag_ctx['status'] = 'idle'
|
|
return
|
|
|
|
# Decrement queued jobs count
|
|
try:
|
|
current_queued = bulk_tag_ctx.get('queued_jobs', 1)
|
|
bulk_tag_ctx['queued_jobs'] = max(0, current_queued - 1)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
await _process_bulk_tag_job(job)
|
|
except Exception as e:
|
|
logger.error(f'Bulk tag worker error: {e}', exc_info=e)
|
|
bulk_tag_ctx['error'] = str(e)
|
|
finally:
|
|
bulk_tag_ctx['status'] = 'idle'
|
|
|
|
|
|
# Register the worker as a perpetual signal
|
|
from wrolpi.api_utils import perpetual_signal
|
|
|
|
bulk_tag_worker = perpetual_signal(sleep=1)(bulk_tag_worker)
|
|
|
|
|
|
def _detect_archive_type(path: pathlib.Path) -> str:
|
|
"""Detect archive type from file suffix. Returns 'zip', 'tar', 'rar', '7z', or raises UnsupportedArchive."""
|
|
name = path.name.lower()
|
|
if name.endswith(('.zip', '.cbz')):
|
|
return 'zip'
|
|
if name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tar.xz', '.cbt')):
|
|
return 'tar'
|
|
if name.endswith(('.rar', '.cbr')):
|
|
return 'rar'
|
|
if name.endswith(('.7z', '.cb7')):
|
|
return '7z'
|
|
raise UnsupportedArchive(f'Unsupported archive format: {path.name}')
|
|
|
|
|
|
def _build_tree(entries: list) -> list:
|
|
"""Build a hierarchical tree from a flat list of archive entries.
|
|
|
|
Each entry is a dict with 'path', 'name', 'is_dir', 'size'.
|
|
Returns a nested list with 'children' on directory entries.
|
|
"""
|
|
root_children = []
|
|
# Map from directory path to its children list.
|
|
dir_map = {}
|
|
|
|
# Sort entries so parents come before children.
|
|
entries.sort(key=lambda e: e['path'])
|
|
|
|
for entry in entries:
|
|
parts = entry['path'].strip('/').split('/')
|
|
if len(parts) == 1:
|
|
# Top-level entry.
|
|
if entry['is_dir']:
|
|
entry['children'] = []
|
|
dir_map[entry['path'].rstrip('/') + '/'] = entry['children']
|
|
root_children.append(entry)
|
|
else:
|
|
# Find parent directory.
|
|
parent_path = '/'.join(parts[:-1]) + '/'
|
|
if parent_path not in dir_map:
|
|
# Create implicit parent directories.
|
|
current = ''
|
|
for part in parts[:-1]:
|
|
current += part + '/'
|
|
if current not in dir_map:
|
|
parent_entry = {
|
|
'path': current,
|
|
'name': part,
|
|
'is_dir': True,
|
|
'size': 0,
|
|
'children': [],
|
|
}
|
|
dir_map[current] = parent_entry['children']
|
|
# Attach to grandparent or root.
|
|
grandparent = '/'.join(current.rstrip('/').split('/')[:-1])
|
|
grandparent_key = grandparent + '/' if grandparent else None
|
|
if grandparent_key and grandparent_key in dir_map:
|
|
dir_map[grandparent_key].append(parent_entry)
|
|
else:
|
|
root_children.append(parent_entry)
|
|
|
|
if entry['is_dir']:
|
|
entry['children'] = []
|
|
dir_map[entry['path'].rstrip('/') + '/'] = entry['children']
|
|
|
|
parent_list = dir_map.get(parent_path)
|
|
if parent_list is not None:
|
|
parent_list.append(entry)
|
|
else:
|
|
root_children.append(entry)
|
|
|
|
return root_children
|
|
|
|
|
|
def list_archive_contents(path: pathlib.Path) -> dict:
|
|
"""List the contents of an archive file as a hierarchical tree.
|
|
|
|
Returns: {"entries": [...], "total_size": int, "total_files": int}
|
|
"""
|
|
media_directory = get_media_directory()
|
|
if not str(path).startswith(str(media_directory)):
|
|
raise InvalidFile('Path must be within the media directory')
|
|
if not path.is_file():
|
|
raise InvalidFile(f'Archive file does not exist: {path}')
|
|
|
|
archive_type = _detect_archive_type(path)
|
|
flat_entries = []
|
|
total_size = 0
|
|
total_files = 0
|
|
|
|
try:
|
|
if archive_type == 'zip':
|
|
with zipfile.ZipFile(path, 'r') as zf:
|
|
for info in zf.infolist():
|
|
is_dir = info.is_dir()
|
|
entry = {
|
|
'path': info.filename,
|
|
'name': info.filename.rstrip('/').split('/')[-1],
|
|
'is_dir': is_dir,
|
|
'size': info.file_size,
|
|
}
|
|
flat_entries.append(entry)
|
|
if not is_dir:
|
|
total_size += info.file_size
|
|
total_files += 1
|
|
elif archive_type == 'tar':
|
|
with tarfile.open(path, 'r:*') as tf:
|
|
for member in tf.getmembers():
|
|
is_dir = member.isdir()
|
|
entry = {
|
|
'path': member.name + ('/' if is_dir else ''),
|
|
'name': member.name.rstrip('/').split('/')[-1],
|
|
'is_dir': is_dir,
|
|
'size': member.size,
|
|
}
|
|
flat_entries.append(entry)
|
|
if not is_dir:
|
|
total_size += member.size
|
|
total_files += 1
|
|
elif archive_type == 'rar':
|
|
import rarfile
|
|
with rarfile.RarFile(path, 'r') as rf:
|
|
for info in rf.infolist():
|
|
is_dir = info.is_dir()
|
|
entry = {
|
|
'path': info.filename,
|
|
'name': info.filename.rstrip('/').split('/')[-1],
|
|
'is_dir': is_dir,
|
|
'size': info.file_size,
|
|
}
|
|
flat_entries.append(entry)
|
|
if not is_dir:
|
|
total_size += info.file_size
|
|
total_files += 1
|
|
elif archive_type == '7z':
|
|
import py7zr
|
|
with py7zr.SevenZipFile(path, 'r') as sz:
|
|
for entry_info in sz.list():
|
|
is_dir = entry_info.is_directory
|
|
entry = {
|
|
'path': entry_info.filename + ('/' if is_dir else ''),
|
|
'name': entry_info.filename.rstrip('/').split('/')[-1],
|
|
'is_dir': is_dir,
|
|
'size': entry_info.uncompressed if not is_dir else 0,
|
|
}
|
|
flat_entries.append(entry)
|
|
if not is_dir:
|
|
total_size += entry_info.uncompressed
|
|
total_files += 1
|
|
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
|
|
raise UnsupportedArchive(f'Cannot read archive: {e}')
|
|
|
|
tree = _build_tree(flat_entries)
|
|
return {'entries': tree, 'total_size': total_size, 'total_files': total_files}
|
|
|
|
|
|
def stream_archive_member(archive_path: pathlib.Path, member_path: str) -> Tuple[bytes, str, str]:
|
|
"""Read a single member from an archive and return its bytes, mimetype, and filename.
|
|
|
|
Security: validates member_path has no '..' components.
|
|
"""
|
|
media_directory = get_media_directory()
|
|
if not str(archive_path).startswith(str(media_directory)):
|
|
raise InvalidFile('Path must be within the media directory')
|
|
if not archive_path.is_file():
|
|
raise InvalidFile(f'Archive file does not exist: {archive_path}')
|
|
|
|
# Security: reject path traversal.
|
|
if '..' in member_path.split('/'):
|
|
raise InvalidArchiveMember('Invalid member path: contains ".."')
|
|
|
|
archive_type = _detect_archive_type(archive_path)
|
|
filename = member_path.rstrip('/').split('/')[-1]
|
|
guessed_type, _ = mimetypes.guess_type(filename)
|
|
content_type = guessed_type or 'application/octet-stream'
|
|
|
|
try:
|
|
if archive_type == 'zip':
|
|
with zipfile.ZipFile(archive_path, 'r') as zf:
|
|
try:
|
|
info = zf.getinfo(member_path)
|
|
except KeyError:
|
|
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
|
|
if info.is_dir():
|
|
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
|
|
file_bytes = zf.read(member_path)
|
|
elif archive_type == 'tar':
|
|
with tarfile.open(archive_path, 'r:*') as tf:
|
|
try:
|
|
member = tf.getmember(member_path)
|
|
except KeyError:
|
|
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
|
|
if member.isdir():
|
|
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
|
|
f = tf.extractfile(member)
|
|
if f is None:
|
|
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
|
|
file_bytes = f.read()
|
|
elif archive_type == 'rar':
|
|
import rarfile
|
|
with rarfile.RarFile(archive_path, 'r') as rf:
|
|
try:
|
|
info = rf.getinfo(member_path)
|
|
except KeyError:
|
|
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
|
|
if info.is_dir():
|
|
raise InvalidArchiveMember(f'Cannot download a directory: {member_path}')
|
|
file_bytes = rf.read(member_path)
|
|
elif archive_type == '7z':
|
|
import py7zr
|
|
import tempfile
|
|
with py7zr.SevenZipFile(archive_path, 'r') as sz:
|
|
if member_path not in sz.getnames():
|
|
raise InvalidArchiveMember(f'Member not found in archive: {member_path}')
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
sz.extract(tmpdir, targets=[member_path])
|
|
extracted_path = pathlib.Path(tmpdir) / member_path
|
|
if not extracted_path.is_file():
|
|
raise InvalidArchiveMember(f'Cannot extract member: {member_path}')
|
|
file_bytes = extracted_path.read_bytes()
|
|
except (zipfile.BadZipFile, tarfile.TarError, EOFError, OSError) as e:
|
|
raise UnsupportedArchive(f'Cannot read archive: {e}')
|
|
|
|
return file_bytes, content_type, filename
|