wrolpi/wrolpi/files/worker.py

2468 lines
105 KiB
Python

"""
File comparison worker for comparing database FileGroups to filesystem state.
This module provides memory-efficient comparison of potentially millions of files
on resource-constrained devices like Raspberry Pi.
"""
import asyncio
import os
import pathlib
import shlex
import shutil
import stat as stat_module
from collections import OrderedDict
from dataclasses import dataclass, field
from enum import Enum, auto
from multiprocessing.queues import Queue
from pathlib import Path
from typing import AsyncGenerator, Callable, Set, Dict, List, Tuple
from sqlalchemy import text, or_
from wrolpi import flags
from wrolpi.common import apply_modelers, apply_refresh_cleanup
from wrolpi.common import get_media_directory, get_wrolpi_config, logger, walk, chunks, unique_by_predicate
from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import NoPrimaryFile
from wrolpi.events import Events
from wrolpi.files.lib import (
split_path_stem_and_suffix, _upsert_files, get_unique_files_by_stem, glob_shared_stem,
group_files_by_stem, get_primary_file, delete_directory, apply_indexers,
_move_file_group_files, _bulk_update_file_groups_db, MOVE_CHUNK_SIZE,
_bulk_update_file_groups_reorganize,
)
logger = logger.getChild(__name__)
# Update status every N items to avoid excessive overhead
PROGRESS_UPDATE_INTERVAL = 100
__all__ = [
'FileGroupDiff',
'FileComparisonResult',
'compare_file_groups',
'count_files',
'file_worker',
]
@dataclass
class FileGroupDiff:
"""Represents changes needed for a FileGroup."""
directory: Path | str
stem: str
db_files: Set[str] = field(default_factory=set) # Filenames in DB
fs_files: Set[str] = field(default_factory=set) # Filenames on filesystem
file_group_id: int | None = None
@property
def added_files(self) -> Set[str]:
"""Files on disk but not in DB."""
return self.fs_files - self.db_files
@property
def removed_files(self) -> Set[str]:
"""Files in DB but not on disk."""
return self.db_files - self.fs_files
@property
def is_new(self) -> bool:
"""FileGroup doesn't exist in DB."""
return self.file_group_id is None and bool(self.fs_files)
@property
def is_deleted(self) -> bool:
"""All files deleted from disk."""
return not self.fs_files and bool(self.db_files)
@property
def needs_update(self) -> bool:
"""FileGroup exists but files changed (not completely deleted)."""
return (self.file_group_id is not None
and bool(self.added_files or self.removed_files)
and bool(self.fs_files)) # Not deleted - still has files on disk
@property
def is_unchanged(self) -> bool:
"""FileGroup matches filesystem exactly."""
return self.db_files == self.fs_files and bool(self.db_files)
@dataclass
class FileComparisonResult:
"""Result of comparing DB to filesystem at FileGroup level."""
unchanged: list # FileGroupDiff where is_unchanged
new: list # FileGroupDiff where is_new
deleted: list # FileGroupDiff where is_deleted
modified: list # FileGroupDiff where needs_update
def _diff_to_paths(diff: FileGroupDiff) -> list[pathlib.Path]:
"""Convert a FileGroupDiff to full file paths."""
directory = pathlib.Path(diff.directory)
return [directory / filename for filename in diff.fs_files]
def _get_normalized_ignored_directories() -> list[str]:
"""Get ignored directories as absolute paths.
Matches the normalization logic in remove_files_in_ignored_directories().
"""
ignored = list(map(str, get_wrolpi_config().ignored_directories))
media_dir = get_media_directory()
result = []
for d in ignored:
p = pathlib.Path(d)
if not p.is_absolute():
result.append(str(media_dir / d))
else:
result.append(d)
return result
async def count_files(directories: list[Path]) -> int:
"""Lightning-fast file count using find | wc -l.
Uses the same find arguments as _stream_filesystem_paths for consistency.
Excludes directories listed in get_wrolpi_config().ignored_directories.
Pipes to wc -l to avoid loading all paths into Python memory.
"""
if not directories:
return 0
# Build find arguments with ignored directory exclusions
find_args = [shlex.quote(str(d)) for d in directories]
find_args.extend(['-type', 'f', '-not', '-path', "'*/.*'"])
# Add exclusions for each ignored directory
for ignored in _get_normalized_ignored_directories():
find_args.extend(['-not', '-path', shlex.quote(f'{ignored}/*')])
cmd = f"find {' '.join(find_args)} | wc -l"
proc = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
)
try:
stdout, _ = await proc.communicate()
return int(stdout.strip())
finally:
if proc.returncode is None:
proc.kill()
await proc.wait()
async def count_files_with_progress(
directories: list[Path],
callback: Callable[[int], None] = None,
batch_size: int = 1000,
) -> int:
"""Count files with progress updates.
Streams find output and calls callback with running count every batch_size files.
Excludes hidden files and directories listed in ignored_directories config.
"""
if not directories:
return 0
find_args = [str(d) for d in directories]
find_args.extend(['-type', 'f', '-not', '-path', '*/.*'])
for ignored in _get_normalized_ignored_directories():
find_args.extend(['-not', '-path', f'{ignored}/*'])
proc = await asyncio.create_subprocess_exec(
'find', *find_args,
stdout=asyncio.subprocess.PIPE,
)
try:
count = 0
async for line in proc.stdout:
count += 1
if callback and count % batch_size == 0:
callback(count)
await asyncio.sleep(0) # Yield for cancellation
if callback:
callback(count) # Final count
await proc.wait()
return count
finally:
if proc.returncode is None:
proc.kill()
await proc.wait()
async def find_directories(directories: list[Path]) -> set[Path]:
"""Find all subdirectories using find command.
Excludes hidden directories and ignored directories from config.
"""
if not directories:
return set()
find_args = [str(d) for d in directories]
find_args.extend(['-type', 'd', '-not', '-path', '*/.*'])
for ignored in _get_normalized_ignored_directories():
find_args.extend(['-not', '-path', f'{ignored}/*'])
proc = await asyncio.create_subprocess_exec(
'find', *find_args,
stdout=asyncio.subprocess.PIPE,
)
try:
found = set()
count = 0
async for line in proc.stdout:
found.add(Path(line.decode().strip()))
count += 1
if count % 100 == 0:
await asyncio.sleep(0) # Yield for cancellation
await proc.wait()
return found
finally:
if proc.returncode is None:
proc.kill()
await proc.wait()
async def _stream_filesystem_paths(root: Path) -> AsyncGenerator[str, None]:
"""Stream file paths using find command. Memory efficient and cancellable.
Excludes hidden files and ignored directories from config.
"""
# Build find arguments with ignored directory exclusions
find_args = [str(root), '-type', 'f', '-not', '-path', '*/.*']
for ignored in _get_normalized_ignored_directories():
find_args.extend(['-not', '-path', f'{ignored}/*'])
proc = await asyncio.create_subprocess_exec(
'find', *find_args,
stdout=asyncio.subprocess.PIPE,
)
try:
async for line in proc.stdout:
path = line.decode().rstrip('\n')
if path:
yield path
await proc.wait()
finally:
# Kill subprocess if still running (handles both normal exit and cancellation)
if proc.returncode is None:
proc.kill()
await proc.wait()
def _insert_fs_batch(session, batch: list):
"""Bulk insert filesystem files using unnest for speed."""
if not batch:
return
session.execute(
text("""
INSERT INTO fs_files (directory, filename, stem)
SELECT *
FROM unnest(
CAST(:dirs AS text[]),
CAST(:files AS text[]),
CAST(:stems AS text[])
)
ON CONFLICT DO NOTHING
"""),
{
"dirs": [b[0] for b in batch],
"files": [b[1] for b in batch],
"stems": [b[2] for b in batch],
}
)
async def compare_file_groups(
root: Path = None,
batch_size: int = 10000,
progress_callback: Callable[[int], None] = None,
) -> FileComparisonResult:
"""
Compare FileGroups in DB to filesystem. Memory efficient for Raspberry Pi.
Uses a temporary PostgreSQL table to handle millions of files without
loading everything into Python memory.
Returns which FileGroups are new, deleted, modified, or unchanged.
This function is async and cancellable - when cancelled, it will terminate
the subprocess scanning the filesystem.
"""
root = root or get_media_directory()
with get_db_session() as session:
# Create temp table for filesystem files with computed stems
session.execute(text("""
CREATE TEMP TABLE IF NOT EXISTS fs_files
(
directory TEXT NOT NULL,
filename TEXT NOT NULL,
stem TEXT NOT NULL,
PRIMARY KEY (directory, filename)
) ON COMMIT DROP
"""))
session.execute(text("TRUNCATE fs_files"))
# Stream filesystem files into temp table with stems
batch = []
file_count = 0
path_gen = _stream_filesystem_paths(root)
try:
async for path in path_gen:
p = Path(path)
directory = str(p.parent)
filename = p.name
stem, _ = split_path_stem_and_suffix(p)
batch.append((directory, filename, stem))
if len(batch) >= batch_size:
_insert_fs_batch(session, batch)
file_count += len(batch)
batch = []
if progress_callback:
progress_callback(file_count)
await asyncio.sleep(0) # Cancellation yield point
finally:
await path_gen.aclose() # Ensure subprocess is killed on cancellation
if batch:
_insert_fs_batch(session, batch)
file_count += len(batch)
logger.info(f'Scanned {file_count} files from filesystem')
# Index for fast grouping
session.execute(text("CREATE INDEX IF NOT EXISTS fs_files_stem_idx ON fs_files(directory, stem)"))
session.execute(text("ANALYZE fs_files"))
# Get filesystem files grouped by (directory, stem)
fs_groups: Dict[Tuple[str, str], Set[str]] = {}
result = session.execute(text("""
SELECT directory, stem, array_agg(filename) as files
FROM fs_files
GROUP BY directory, stem
"""))
for row in result:
fs_groups[(row.directory, row.stem)] = set(row.files)
logger.info(f'Found {len(fs_groups)} file groups on filesystem')
# Get all DB FileGroups within root directory (both with files and empty)
# Include modification_datetime for mtime comparison
db_groups: Dict[Tuple[str, str], Tuple[int, Set[str], float]] = {} # key -> (id, files, mtime_timestamp)
root_str = str(root)
# Initialize result lists before the loop so empty FileGroups can be processed inline
unchanged = []
new = []
deleted = []
modified = []
empty_count = 0
result = session.execute(text("""
SELECT fg.id,
fg.directory,
fg.primary_path,
fg.files IS NULL OR fg.files = '[]'::jsonb as is_empty,
EXTRACT(EPOCH FROM fg.modification_datetime) as mtime_epoch,
CASE
WHEN fg.files IS NOT NULL AND jsonb_typeof(fg.files) = 'array' AND
fg.files != '[]'::jsonb
THEN (SELECT array_agg(f ->> 'path')
FROM jsonb_array_elements(fg.files) as f)
ELSE NULL
END as files
FROM file_group fg
WHERE fg.directory IS NOT NULL
AND (fg.directory = :root OR fg.directory LIKE :root_pattern)
"""), {"root": root_str, "root_pattern": f"{root_str}/%"})
for row in result:
if row.is_empty:
# Process empty FileGroup immediately (no intermediate list)
empty_count += 1
stem, _ = split_path_stem_and_suffix(Path(row.primary_path))
key = (row.directory, stem)
fs_files = fs_groups.pop(key, set()) # Remove from fs_groups to avoid double-processing
diff = FileGroupDiff(
directory=Path(row.directory),
stem=stem,
db_files=set(),
fs_files=fs_files,
file_group_id=row.id,
)
if fs_files:
# Files exist on disk but not in DB - update the FileGroup
modified.append(diff)
else:
# No files on disk and no files in DB - delete the FileGroup
deleted.append(diff)
elif row.files:
# Compute stem from first file (all files in group share stem)
first_file = row.files[0]
stem, _ = split_path_stem_and_suffix(Path(row.directory) / first_file)
db_groups[(row.directory, stem)] = (row.id, set(row.files), row.mtime_epoch or 0)
logger.info(f'Found {len(db_groups)} file groups in database')
if empty_count:
logger.debug(f'Found {empty_count} FileGroups with empty files arrays')
# Recompute all_keys after removing empty FileGroup keys from fs_groups
all_keys = set(fs_groups.keys()) | set(db_groups.keys())
for key in all_keys:
directory, stem = key
fs_files = fs_groups.get(key, set())
db_id, db_files, db_mtime = db_groups.get(key, (None, set(), 0))
diff = FileGroupDiff(
directory=Path(directory),
stem=stem,
db_files=db_files,
fs_files=fs_files,
file_group_id=db_id,
)
if diff.is_new:
new.append(diff)
elif diff.is_deleted:
deleted.append(diff)
elif diff.needs_update:
modified.append(diff)
elif diff.is_unchanged:
# Files match by name - check if content changed via mtime
# Get max mtime from filesystem files
try:
dir_path = Path(directory)
fs_mtime = max(
(dir_path / filename).stat().st_mtime
for filename in fs_files
)
# If filesystem mtime is newer than DB mtime, content changed
if fs_mtime > float(db_mtime) + 0.001: # small tolerance for float comparison
modified.append(diff)
else:
unchanged.append(diff)
except (OSError, ValueError):
# If we can't stat files, treat as unchanged
unchanged.append(diff)
logger.info(
f'Comparison complete: {len(unchanged)} unchanged, {len(new)} new, '
f'{len(deleted)} deleted, {len(modified)} modified'
)
return FileComparisonResult(
unchanged=unchanged,
new=new,
deleted=deleted,
modified=modified,
)
async def build_move_plan_bulk(
sources: list[pathlib.Path],
destination: pathlib.Path,
progress_callback: Callable[[int, int], None] = None,
) -> Tuple[Dict[pathlib.Path, pathlib.Path], list[pathlib.Path]]:
"""
Build move plan using bulk SQL operations instead of per-source queries.
Uses a temp table pattern similar to compare_file_groups for O(1) query complexity.
Args:
sources: List of file/directory paths to move
destination: Target directory
progress_callback: Optional callback(processed, total) called during file collection
Returns:
Tuple of (plan dict mapping old_primary_path -> new_primary_path,
list of old_directories for cleanup)
"""
from wrolpi.files.models import FileGroup
plan: Dict[pathlib.Path, pathlib.Path] = dict()
old_directories: list[pathlib.Path] = list(unique_by_predicate(
i if i.is_dir() else i.parent for i in sources
))
# Deduplicate sources by stem
sources = list(get_unique_files_by_stem(sources))
logger.info(f'build_move_plan_bulk: processing {len(sources)} sources')
# Partition sources into files, directories, and deleted (pre-moved)
# Optimized: single stat() call per path instead of up to 3 (is_file, is_dir, exists)
file_sources: list[pathlib.Path] = []
dir_sources: list[pathlib.Path] = []
deleted_sources: list[pathlib.Path] = []
total_sources = len(sources)
for i, source in enumerate(sources, 1):
try:
st = source.stat()
if stat_module.S_ISREG(st.st_mode):
file_sources.append(source)
elif stat_module.S_ISDIR(st.st_mode):
dir_sources.append(source)
# Other types (symlinks, etc.) are silently skipped
except FileNotFoundError:
deleted_sources.append(source)
# Report progress during source partitioning (first 50% of planning)
if progress_callback and i % 100 == 0:
progress_callback(i, total_sources * 2) # *2 because file collection is second half
with get_db_session() as session:
# Create temp table for source paths
session.execute(text("""
CREATE TEMP TABLE IF NOT EXISTS move_sources
(
source_path TEXT PRIMARY KEY,
source_type TEXT NOT NULL,
source_directory TEXT
) ON COMMIT DROP
"""))
session.execute(text("TRUNCATE move_sources"))
# Collect all paths to insert: file paths + expanded directory contents
all_file_paths: list[pathlib.Path] = []
files_collected = 0
# Use total_sources as the denominator for the second half of progress
total_items = len(file_sources) + len(dir_sources)
# For files, collect the file and its shared-stem siblings
for i, source in enumerate(file_sources, 1):
files = glob_shared_stem(source)
all_file_paths.extend(files)
files_collected += 1
# Report progress (second half: 50-100%)
if progress_callback and files_collected % 50 == 0:
progress = total_sources + (files_collected * total_sources // max(total_items, 1))
progress_callback(progress, total_sources * 2)
# For directories, walk and collect all files
dir_file_mapping: Dict[pathlib.Path, pathlib.Path] = {} # file -> source_dir
for i, source_dir in enumerate(dir_sources, 1):
for f in walk(source_dir):
if f.is_file():
all_file_paths.append(f)
dir_file_mapping[f] = source_dir
files_collected += 1
# Report progress during directory walking (every 10 directories)
if progress_callback and files_collected % 10 == 0:
progress = total_sources + (files_collected * total_sources // max(total_items, 1))
progress_callback(progress, total_sources * 2)
# Final progress update before deduplication
if progress_callback:
progress_callback(total_sources * 2, total_sources * 2)
# Deduplicate
all_file_paths = list(set(all_file_paths))
# Bulk insert file paths into temp table
if all_file_paths:
paths_data = [(str(p), 'file', str(p.parent)) for p in all_file_paths]
session.execute(
text("""
INSERT INTO move_sources (source_path, source_type, source_directory)
SELECT *
FROM unnest(
CAST(:paths AS text[]),
CAST(:types AS text[]),
CAST(:dirs AS text[])
)
ON CONFLICT DO NOTHING
"""),
{
"paths": [p[0] for p in paths_data],
"types": [p[1] for p in paths_data],
"dirs": [p[2] for p in paths_data],
}
)
# Also insert deleted sources (for pre-moved files lookup)
if deleted_sources:
deleted_data = [(str(p), 'deleted', str(p.parent)) for p in deleted_sources]
session.execute(
text("""
INSERT INTO move_sources (source_path, source_type, source_directory)
SELECT *
FROM unnest(
CAST(:paths AS text[]),
CAST(:types AS text[]),
CAST(:dirs AS text[])
)
ON CONFLICT DO NOTHING
"""),
{
"paths": [p[0] for p in deleted_data],
"types": [p[1] for p in deleted_data],
"dirs": [p[2] for p in deleted_data],
}
)
# Single bulk query to get all FileGroups matching our sources
result = session.execute(text("""
SELECT fg.id, fg.primary_path, fg.directory, ms.source_path, ms.source_type
FROM file_group fg
JOIN move_sources ms ON fg.primary_path = ms.source_path
"""))
# Build lookup of found FileGroups
fg_by_path: Dict[str, Tuple[int, str, str]] = {} # path -> (id, primary_path, directory)
for row in result:
fg_by_path[row.source_path] = (row.id, row.primary_path, row.directory)
logger.info(f'build_move_plan_bulk: found {len(fg_by_path)} FileGroups for {len(all_file_paths)} files')
# Build plan for file sources (direct files, not from directories)
file_source_set = set()
for source in file_sources:
files = glob_shared_stem(source)
file_source_set.update(str(f) for f in files)
# Process files from direct file sources
for path_str, (fg_id, primary_path, directory) in fg_by_path.items():
path = pathlib.Path(path_str)
primary_path = pathlib.Path(primary_path)
# Is this from a file source or a directory source?
if path_str in file_source_set:
# Direct file source -> destination / filename
new_path = destination / primary_path.name
elif path in dir_file_mapping:
# From directory source -> destination / source_dir.name / relative_path
source_dir = dir_file_mapping[path]
new_path = destination / source_dir.name / primary_path.relative_to(source_dir)
else:
# Deleted source (pre-moved) -> destination / filename
new_path = destination / primary_path.name
# Check for conflicts (destination exists)
if new_path.exists():
if not primary_path.exists():
# Pre-moved - OK to proceed
plan[primary_path] = new_path
else:
raise FileExistsError(f'Cannot move: {new_path} already exists')
else:
plan[primary_path] = new_path
# Handle files not in DB - create FileGroups
found_paths = set(fg_by_path.keys())
missing_db_files = [p for p in all_file_paths if str(p) not in found_paths and p.exists()]
if missing_db_files:
logger.info(f'build_move_plan_bulk: creating FileGroups for {len(missing_db_files)} files not in DB')
for paths in group_files_by_stem(missing_db_files):
try:
get_primary_file(paths)
fg = FileGroup.from_paths(session, *paths)
session.flush()
# Determine destination path
if any(str(p) in file_source_set for p in paths):
new_path = destination / fg.primary_path.name
else:
# Find which directory source this came from
source_dir = None
for p in paths:
if p in dir_file_mapping:
source_dir = dir_file_mapping[p]
break
if source_dir:
new_path = destination / source_dir.name / fg.primary_path.relative_to(source_dir)
else:
new_path = destination / fg.primary_path.name
if new_path.exists() and fg.primary_path.exists():
raise FileExistsError(f'Cannot move: {new_path} already exists')
plan[fg.primary_path] = new_path
except NoPrimaryFile:
for file in paths:
fg = FileGroup.from_paths(session, file)
session.flush()
if any(str(p) in file_source_set for p in paths):
new_path = destination / fg.primary_path.name
elif file in dir_file_mapping:
source_dir = dir_file_mapping[file]
new_path = destination / source_dir.name / fg.primary_path.relative_to(source_dir)
else:
new_path = destination / fg.primary_path.name
if new_path.exists() and fg.primary_path.exists():
raise FileExistsError(f'Cannot move: {new_path} already exists')
plan[fg.primary_path] = new_path
# Add directories to plan for cleanup
for source_dir in dir_sources:
for directory in (i for i in walk(source_dir) if i.is_dir()):
new_directory = destination / source_dir.name / directory.relative_to(source_dir)
plan[directory] = new_directory
# Commit to drop the temporary table
session.commit()
# Sort plan by depth (deepest first) for safe move ordering
plan = OrderedDict(
sorted(plan.items(), key=lambda i: (len(i[0].parents), i[0].name), reverse=True)
)
logger.info(f'build_move_plan_bulk: plan has {len(plan)} items')
return plan, old_directories
class FileTaskType(str, Enum):
count = auto() # Simply count the files.
refresh = auto() # Update the DB to match the files that exist on disk.
move = auto() # Move files and directories to a new location, file names are not changed.
rename = auto() # Rename files, requires deeper updating of FileGroup.files and FileGroup.data.
tag = auto() # Add TagFile records for files.
reorganize = auto() # Reorganize files with arbitrary source->dest mappings for collection reorganization.
batch_reorganize = auto() # Batch reorganize multiple collections sequentially.
@dataclass
class FileTask:
task_type: FileTaskType
paths: list[pathlib.Path]
destination: pathlib.Path = None # For move tasks
count: int = None
found_directories: set[pathlib.Path] = None # Directories found during count phase
unchanged_files: bool = None
new_files: list[pathlib.Path] = None
deleted_files: bool = None
modified_files: bool = None
prev_task_type: FileTaskType = None
next_task_type: FileTaskType = None
job_id: str = None # Unique ID for tracking completion
expand_stems: bool = True # Whether to expand files to their FileGroup stem-mates
# For reorganize tasks: list of (source_path, dest_path) tuples
move_mappings: list[tuple[pathlib.Path, pathlib.Path]] = None
collection_id: int = None # Collection being reorganized
collection_kind: str = None # 'channel' or 'domain' for reorganize tasks
# For batch_reorganize tasks
collection_ids: list[int] = None # List of collection IDs to process
batch_kind: str = None # 'channel' or 'domain'
# For deferred file_format update (only updated after successful completion)
pending_file_format: str = None
class FileWorker:
def __init__(self):
# Use last-in, first out so a task that inserts another task is handled next.
self.private_queue = asyncio.LifoQueue()
@property
def public_queue(self) -> Queue:
from wrolpi.api_utils import api_app
return api_app.shared_ctx.file_worker_public_queue
@property
def status(self) -> dict:
from wrolpi.api_utils import api_app
return api_app.shared_ctx.file_worker_status
def update_status(self, **kwargs):
"""Update file worker status in shared context."""
from wrolpi.api_utils import api_app
for key, value in kwargs.items():
api_app.shared_ctx.file_worker_status[key] = value
def reset_status(self):
"""Reset status to idle."""
self.update_status(
status='idle',
task_type=None,
paths=[],
destination=None,
error=None,
operation_total=0,
operation_processed=0,
operation_percent=0,
)
@property
def _jobs(self) -> dict:
"""Access the shared job tracking dict."""
from wrolpi.api_utils import api_app
return api_app.shared_ctx.file_worker_jobs
def _set_job_status(self, job_id: str, status: str):
"""Set the status of a tracked job."""
self._jobs[job_id] = status
def _complete_job(self, job_id: str):
"""Mark a job as complete."""
if job_id:
self._set_job_status(job_id, 'complete')
def get_job_status(self, job_id: str) -> str | None:
"""Get the status of a tracked job."""
job_status = self._jobs.get(job_id)
if not job_status:
raise RuntimeError(f'Job {job_id} does not exist')
return job_status
async def wait_for_job(self, job_id: str, timeout: float = 300):
"""Wait for a job to complete.
Transfers and processes the queue while waiting to ensure jobs are actually executed.
This is especially important in tests and synchronous contexts.
Args:
job_id: The job ID to wait for
timeout: Maximum time to wait in seconds (default 5 minutes)
Raises:
TimeoutError: If job doesn't complete within timeout
"""
import time
start = time.time()
iteration = 0
while time.time() - start < timeout:
iteration += 1
status = self.get_job_status(job_id)
if status == 'complete':
return
# Transfer from public to private queue, then process
self.transfer_queue()
await self.process_queue()
await asyncio.sleep(0.01)
logger.error(f'wait_for_job: TIMEOUT job_id={job_id} after {iteration} iterations')
raise TimeoutError(f'Job {job_id} did not complete in time')
def _get_relative_path(self, path: pathlib.Path) -> str:
"""Get relative path from media directory for display in event messages."""
media_directory = get_media_directory()
try:
return str(path.relative_to(media_directory))
except ValueError:
return str(path)
def queue_refresh(self, paths: list[pathlib.Path | str], expand_stems: bool = True) -> str:
"""Queue a refresh task for background processing.
Args:
paths: List of file or directory paths to refresh
expand_stems: Whether to expand files to their FileGroup stem-mates.
Set to False when API users explicitly select specific files.
Returns:
A unique job_id that can be used with wait_for_job() to track completion.
"""
import uuid
job_id = f'refresh-{uuid.uuid4().hex[:8]}'
task = FileTask(
FileTaskType.count,
paths,
next_task_type=FileTaskType.refresh,
job_id=job_id,
expand_stems=expand_stems,
)
self._set_job_status(job_id, 'pending')
self.public_queue.put_nowait(task)
return job_id
def queue_move(self, destination: pathlib.Path, sources: list[pathlib.Path]) -> str:
"""Queue a move task for background processing.
Args:
destination: Target directory (absolute path)
sources: List of files/directories to move (absolute paths)
Returns:
Job ID for tracking completion
"""
import uuid
job_id = f'move-{uuid.uuid4().hex[:8]}'
task = FileTask(FileTaskType.move, sources, destination=destination, job_id=job_id)
self._set_job_status(job_id, 'pending')
self.public_queue.put_nowait(task)
return job_id
def queue_reorganize(
self,
move_mappings: list[tuple[pathlib.Path, pathlib.Path]],
collection_id: int = None,
collection_kind: str = None,
pending_file_format: str = None,
) -> str:
"""Queue a reorganize task for collection file reorganization.
Unlike queue_move which moves all files to a single destination,
queue_reorganize handles arbitrary source->destination mappings
where each file can move to a different location.
Args:
move_mappings: List of (source_path, dest_path) tuples
collection_id: Optional collection ID being reorganized (for status tracking)
collection_kind: 'channel' or 'domain' (for status tracking / navbar navigation)
pending_file_format: File format to set on collection after successful completion
(deferred update enables retry on failure)
Returns:
Job ID for tracking completion
"""
import uuid
job_id = f'reorganize-{uuid.uuid4().hex[:8]}'
# Paths list is empty for reorganize tasks - we use move_mappings instead
task = FileTask(
FileTaskType.reorganize,
paths=[],
move_mappings=move_mappings,
collection_id=collection_id,
collection_kind=collection_kind,
job_id=job_id,
pending_file_format=pending_file_format,
)
self._set_job_status(job_id, 'pending')
self.public_queue.put_nowait(task)
return job_id
def queue_batch_reorganize(
self,
collection_ids: list[int],
kind: str,
) -> str:
"""Queue a batch reorganize task for multiple collections.
Processes collections sequentially, stopping on first failure.
Tracks overall progress and per-collection progress.
Args:
collection_ids: List of collection IDs to reorganize
kind: 'channel' or 'domain'
Returns:
Batch job ID for tracking completion
"""
import uuid
job_id = f'batch-reorganize-{uuid.uuid4().hex[:8]}'
task = FileTask(
FileTaskType.batch_reorganize,
paths=[],
collection_ids=collection_ids,
batch_kind=kind,
job_id=job_id,
)
self._set_job_status(job_id, 'pending')
self.public_queue.put_nowait(task)
return job_id
async def refresh_sync(self, paths: list[pathlib.Path]):
"""Synchronously refresh specific files. Use sparingly - prefer queue_refresh."""
if not paths:
return
result = await self._refresh_files_directly(paths)
self._cleanup_modified_models(result.modified)
await self._upsert_file_groups(result.new + result.modified)
await self._delete_file_groups(result.deleted)
await self._apply_post_processing(is_global_refresh=False)
def transfer_queue(self):
"""Transfer items from the public queue to the private queue."""
while self.public_queue.qsize() > 0:
item = self.public_queue.get_nowait()
self.private_queue.put_nowait(item)
async def process_queue(self):
try:
task: FileTask = self.private_queue.get_nowait()
except asyncio.QueueEmpty:
# No tasks - clear flag if set and both queues are empty
if flags.file_worker_busy.is_set() and self.public_queue.qsize() == 0:
flags.file_worker_busy.clear()
await asyncio.sleep(0)
return
# Set flag before processing any task (if not already set)
if not flags.file_worker_busy.is_set():
flags.file_worker_busy.set()
try:
match task.task_type:
case FileTaskType.count:
await self.handle_count(task)
case FileTaskType.refresh:
await self.handle_refresh(task)
case FileTaskType.move:
await self.handle_move(task)
case FileTaskType.reorganize:
await self.handle_reorganize(task)
case FileTaskType.batch_reorganize:
await self.handle_batch_reorganize(task)
finally:
# Clear flag if both queues are now empty
if self.private_queue.qsize() == 0 and self.public_queue.qsize() == 0:
flags.file_worker_busy.clear()
async def handle_count(self, task: FileTask):
"""Count files in the task's paths and chain to next task if specified."""
self.update_status(
status='counting',
task_type=task.next_task_type.name if task.next_task_type else 'count',
paths=[str(p) for p in task.paths],
operation_total=0, # Unknown until complete
operation_processed=0,
operation_percent=0,
)
with flags.file_worker_counting:
directories = [pathlib.Path(p) for p in task.paths if pathlib.Path(p).is_dir()]
files = [pathlib.Path(p) for p in task.paths if pathlib.Path(p).is_file()]
def update_count(count: int):
self.update_status(operation_processed=count)
# Count files in directories with progress updates + individual files
dir_count = await count_files_with_progress(directories, callback=update_count) if directories else 0
file_count = len(files)
total_count = dir_count + file_count
# Find all directories for tracking in the database
found_directories = await find_directories(directories) if directories else set()
self.update_status(operation_total=total_count, operation_processed=total_count, operation_percent=100)
logger.info(
f'Counted {total_count} files in {len(task.paths)} paths, found {len(found_directories)} directories')
if task.next_task_type == FileTaskType.refresh:
next_task = FileTask(
FileTaskType.refresh,
task.paths,
count=total_count,
found_directories=found_directories,
prev_task_type=FileTaskType.count,
job_id=task.job_id,
expand_stems=task.expand_stems,
)
self.private_queue.put_nowait(next_task)
else:
self.reset_status()
async def handle_refresh(self, task: FileTask):
from wrolpi.errors import UnknownDirectory
media_directory = get_media_directory()
# Check that media directory exists
if not media_directory.is_dir():
raise UnknownDirectory('Refusing to refresh because media directory does not exist.')
# Separate files from directories
all_paths = [pathlib.Path(p) for p in task.paths]
file_paths = [p for p in all_paths if p.is_file()]
dir_paths = [p for p in all_paths if p.is_dir()]
# Deleted paths might be files or directories - handle as files for FileGroup cleanup
deleted_paths = [p for p in all_paths if not p.exists()]
file_paths.extend(deleted_paths)
# For global refreshes, check that there are non-ignored files
is_global_refresh = len(dir_paths) == 1 and dir_paths[0] == media_directory and not file_paths
if is_global_refresh:
ignored_directories = set(_get_normalized_ignored_directories())
has_non_ignored_files = False
for item in media_directory.iterdir():
if item.name.startswith('.'):
continue
if str(item) in ignored_directories:
continue
has_non_ignored_files = True
break
if not has_non_ignored_files:
raise UnknownDirectory(
'Refusing to refresh because media directory contains only ignored files or is empty.'
)
# Directories require a count first; files do not
if dir_paths and task.count is None:
# Missing count for directories, send it back through count first.
count_task = FileTask(
FileTaskType.count,
task.paths,
next_task_type=FileTaskType.refresh,
)
self.private_queue.put_nowait(count_task)
return
# Send appropriate start event based on refresh scope
if is_global_refresh:
logger.info(f'Starting global refresh with {task.count} files')
Events.send_global_refresh_started()
elif dir_paths and len(dir_paths) == 1 and not file_paths:
relative_path = self._get_relative_path(dir_paths[0])
Events.send_directory_refresh(f'Refreshing: {relative_path}')
elif file_paths:
Events.send_files_refreshed(f'Refreshing {len(file_paths)} files')
else:
Events.send_files_refreshed(f'Refreshing {len(task.paths)} paths')
# Process files and directories within discovery flag context
# This covers comparing, upserting, and deleting phases
file_result = None
dir_result = None
with flags.file_worker_discovery:
# Process files directly (fast path)
# expand_stems controls whether to expand files to their FileGroup stem-mates.
# API callers set expand_stems=False when users explicitly select specific files.
# Deleted paths (handled above) will still be processed individually.
if file_paths:
self.update_status(
status='comparing',
operation_total=len(file_paths),
operation_processed=0,
operation_percent=0,
)
logger.info(f'Refreshing {len(file_paths)} files directly')
file_result = await self._refresh_files_directly(file_paths, expand_stems=task.expand_stems)
# Process file results
self._cleanup_modified_models(file_result.modified)
await self._upsert_file_groups(file_result.new + file_result.modified)
await self._delete_file_groups(file_result.deleted)
# Process directories with full scan (existing path)
if dir_paths:
self.update_status(
status='comparing',
operation_total=task.count,
operation_processed=0,
operation_percent=0,
)
logger.info(f'Comparing {task.count} files in {len(dir_paths)} directories')
def on_compare_progress(count: int):
percent = int((count / task.count) * 100) if task.count > 0 else 0
self.update_status(operation_processed=count, operation_percent=percent)
root = dir_paths[0] if len(dir_paths) == 1 else None
dir_result = await compare_file_groups(root=root, progress_callback=on_compare_progress)
if is_global_refresh:
Events.send_global_refresh_discovery_completed()
logger.info(
f'Refresh comparison: {len(dir_result.new)} new, {len(dir_result.modified)} modified, '
f'{len(dir_result.deleted)} deleted, {len(dir_result.unchanged)} unchanged'
)
# Process directory results
self._cleanup_modified_models(dir_result.modified)
await self._upsert_file_groups(dir_result.new + dir_result.modified)
await self._delete_file_groups(dir_result.deleted)
# Run indexers, modelers, and cleanup once for all changes
await self._apply_post_processing(is_global_refresh=is_global_refresh)
# Track directories in the database (use directories found during count phase)
if dir_paths:
from wrolpi.files.lib import upsert_directories
parent_directories = set(dir_paths)
found_directories = task.found_directories or set()
found_directories = found_directories - parent_directories
upsert_directories(parent_directories, found_directories)
# Clean up directory entries for deleted paths
if deleted_paths:
from wrolpi.db import get_db_curs
with get_db_curs(commit=True) as curs:
for path in deleted_paths:
curs.execute('DELETE FROM directory WHERE path = %s', (str(path.absolute()),))
# Send appropriate completion event based on refresh scope
if is_global_refresh:
Events.send_global_after_refresh_completed()
Events.send_refresh_completed()
elif dir_paths and len(dir_paths) == 1 and not file_paths:
relative_path = self._get_relative_path(dir_paths[0])
Events.send_directory_refresh(f'Refreshed: {relative_path}')
elif file_paths:
Events.send_files_refreshed(f'Refreshed {len(file_paths)} files')
else:
Events.send_files_refreshed(f'Refreshed {len(task.paths)} paths')
# Set refresh_complete flag only when refreshing the entire media directory
if is_global_refresh:
flags.refresh_complete.set()
# Mark job as complete if tracking
self._complete_job(task.job_id)
self.reset_status()
async def _upsert_file_groups(self, diffs: list[FileGroupDiff]):
"""Insert new FileGroups or update modified ones."""
if not diffs:
return
total = len(diffs)
self.update_status(
status='upserting',
operation_total=total,
operation_processed=0,
operation_percent=0,
)
# For modified diffs where file_group_id exists, we need to handle primary_path changes.
# If the primary file changed (e.g., video deleted, leaving only poster), the ON CONFLICT
# won't match, so we need to update the existing FileGroup's primary_path first.
from wrolpi.files.lib import NoPrimaryFile
modified_ids_to_update = []
for diff in diffs:
if diff.file_group_id and diff.fs_files:
# This is a modified FileGroup - may need to update primary_path
new_paths = [diff.directory / f for f in diff.fs_files]
try:
new_primary = get_primary_file(new_paths)
except NoPrimaryFile:
# No clear primary file, just use the first one
new_primary = new_paths[0] if new_paths else None
if new_primary:
modified_ids_to_update.append((diff.file_group_id, str(new_primary)))
# Pre-update primary_paths for modified FileGroups so ON CONFLICT will match
# Only update if the primary_path is actually changing to avoid unique constraint violations
# Optimized: batch all checks and updates to reduce N queries to 2
if modified_ids_to_update:
with get_db_curs(commit=True) as curs:
# Build lookup of id -> new_primary_path
id_to_path = {fg_id: new_path for fg_id, new_path in modified_ids_to_update}
all_new_paths = list(id_to_path.values())
all_ids = list(id_to_path.keys())
# Batch check: find all primary_paths that already exist on OTHER FileGroups
curs.execute(
'''SELECT primary_path, id
FROM file_group
WHERE primary_path = ANY (%s)''',
(all_new_paths,)
)
existing_path_to_id = {row[0]: row[1] for row in curs.fetchall()}
# Filter out updates where the path is already used by a different FileGroup
valid_updates = []
for fg_id, new_path in modified_ids_to_update:
existing_id = existing_path_to_id.get(new_path)
if existing_id is None or existing_id == fg_id:
# Path is free OR already belongs to this FileGroup (no change needed)
valid_updates.append((fg_id, new_path))
else:
logger.warning(
f'Skipping primary_path update for fg_id={fg_id}: {new_path} already exists')
# Bulk UPDATE using VALUES clause for all valid updates
if valid_updates:
curs.execute(
'''UPDATE file_group AS fg
SET primary_path = v.new_path
FROM (SELECT unnest(%s::integer[]) AS id, unnest(%s::text[]) AS new_path) AS v
WHERE fg.id = v.id
AND fg.primary_path != v.new_path''',
([u[0] for u in valid_updates], [u[1] for u in valid_updates])
)
# Collect all file paths from the diffs (fast operation, no progress tracking needed)
all_paths = []
for diff in diffs:
all_paths.extend(_diff_to_paths(diff))
# Update total to reflect actual files being processed
total_files = len(all_paths)
self.update_status(
operation_total=total_files,
operation_processed=0,
operation_percent=0,
)
# Create progress callback for the actual DB upsert operations
def on_upsert_progress(processed: int, total: int):
percent = int((processed / total) * 100) if total > 0 else 0
self.update_status(operation_processed=processed, operation_percent=percent)
# Use existing _upsert_files function which handles grouping and primary file detection
idempotency = now()
await asyncio.to_thread(_upsert_files, all_paths, idempotency, on_upsert_progress)
logger.info(f'Upserted {len(diffs)} FileGroups ({len(all_paths)} files)')
async def _delete_file_groups(self, diffs: list[FileGroupDiff]):
"""Delete FileGroups where all files have been removed from disk.
Handles auto-removing tags, Download cleanup, and FileGroup deletion.
Physical files are already gone - this only cleans up DB records.
Processes in batches for progress reporting.
"""
if not diffs:
return
file_group_ids = [diff.file_group_id for diff in diffs if diff.file_group_id]
if not file_group_ids:
return
total = len(file_group_ids)
self.update_status(
status='deleting',
operation_total=total,
operation_processed=0,
operation_percent=0,
)
from wrolpi.downloader import Download, download_manager
from wrolpi.files.models import FileGroup
from wrolpi.tags import TagFile, save_tags_config, sync_tags_directory
# Process in batches for progress reporting
batch_size = PROGRESS_UPDATE_INTERVAL
deleted_count = 0
had_tags = False
all_urls = []
for batch_start in range(0, total, batch_size):
batch_ids = file_group_ids[batch_start:batch_start + batch_size]
with get_db_session(commit=True) as session:
# Check if any have tags
if session.query(TagFile).filter(
TagFile.file_group_id.in_(batch_ids)
).count() > 0:
had_tags = True
# Get URLs for skip list
file_groups = session.query(FileGroup).filter(
FileGroup.id.in_(batch_ids)
).all()
for fg in file_groups:
if fg.url:
all_urls.append(fg.url)
if Download.get_by_url(session, fg.url):
session.query(Download).filter(
Download.url == fg.url
).delete(synchronize_session=False)
# Delete FileGroups (TagFiles cascade-deleted automatically)
batch_deleted = session.query(FileGroup).filter(
FileGroup.id.in_(batch_ids)
).delete(synchronize_session=False)
deleted_count += batch_deleted
# Update progress
processed = min(batch_start + batch_size, total)
percent = int((processed / total) * 100)
self.update_status(operation_processed=processed, operation_percent=percent)
# Add URLs to skip list
for url in all_urls:
download_manager.add_to_skip_list(url)
# Trigger tag config saves if needed
if had_tags:
save_tags_config.activate_switch()
sync_tags_directory.activate_switch()
logger.info(f'Deleted {deleted_count} FileGroups ({len(all_urls)} URLs added to skip list)')
def _cleanup_modified_models(self, diffs: list[FileGroupDiff]):
"""Delete Videos/Archives/EBooks for modified FileGroups where primary file was removed.
Optimized: uses a single UNION ALL query to fetch all model types at once,
reducing 3 SELECT queries to 1.
"""
if not diffs:
return
# Get file_group_ids from modified diffs that have removed files
file_group_ids = [d.file_group_id for d in diffs if d.file_group_id and d.removed_files]
if not file_group_ids:
return
# Build mapping of file_group_id -> removed files
removed_by_id = {d.file_group_id: d.removed_files for d in diffs if d.file_group_id}
with get_db_curs(commit=True) as curs:
# Single UNION ALL query to fetch all models at once (reduces 3 queries to 1)
curs.execute('''
SELECT 'video' AS model_type, v.id AS model_id, fg.id AS fg_id, fg.primary_path
FROM video v
JOIN file_group fg ON v.file_group_id = fg.id
WHERE fg.id = ANY (%s)
UNION ALL
SELECT 'archive', a.id, fg.id, fg.primary_path
FROM archive a
JOIN file_group fg ON a.file_group_id = fg.id
WHERE fg.id = ANY (%s)
UNION ALL
SELECT 'ebook', e.id, fg.id, fg.primary_path
FROM ebook e
JOIN file_group fg ON e.file_group_id = fg.id
WHERE fg.id = ANY (%s)
''', (file_group_ids, file_group_ids, file_group_ids))
# Group results by model type
video_ids_to_delete = []
archive_ids_to_delete = []
ebook_ids_to_delete = []
for model_type, model_id, fg_id, primary_path in curs.fetchall():
basename = pathlib.Path(primary_path).name
if basename in removed_by_id.get(fg_id, set()):
if model_type == 'video':
video_ids_to_delete.append(model_id)
elif model_type == 'archive':
archive_ids_to_delete.append(model_id)
elif model_type == 'ebook':
ebook_ids_to_delete.append(model_id)
# Batch delete each model type
if video_ids_to_delete:
logger.info(f'Deleting {len(video_ids_to_delete)} Videos whose primary file was removed')
curs.execute('''
UPDATE file_group
SET model = NULL
WHERE id IN (SELECT file_group_id FROM video WHERE id = ANY (%s))
''', (video_ids_to_delete,))
curs.execute('DELETE FROM video WHERE id = ANY(%s)', (video_ids_to_delete,))
if archive_ids_to_delete:
logger.info(f'Deleting {len(archive_ids_to_delete)} Archives whose primary file was removed')
curs.execute('''
UPDATE file_group
SET model = NULL
WHERE id IN (SELECT file_group_id FROM archive WHERE id = ANY (%s))
''', (archive_ids_to_delete,))
curs.execute('DELETE FROM archive WHERE id = ANY(%s)', (archive_ids_to_delete,))
if ebook_ids_to_delete:
logger.info(f'Deleting {len(ebook_ids_to_delete)} EBooks whose primary file was removed')
curs.execute('''
UPDATE file_group
SET model = NULL
WHERE id IN (SELECT file_group_id FROM ebook WHERE id = ANY (%s))
''', (ebook_ids_to_delete,))
curs.execute('DELETE FROM ebook WHERE id = ANY(%s)', (ebook_ids_to_delete,))
def _validate_move_paths(
self,
sources: List[pathlib.Path],
destination: pathlib.Path,
media_directory: pathlib.Path,
) -> bool:
"""Validate that all source and destination paths are within the media directory.
Returns True if valid, False otherwise.
"""
for source in sources:
if not str(source).startswith(str(media_directory)):
logger.error(f'{source} is not within the media directory')
self.reset_status()
return False
if not str(destination).startswith(str(media_directory)):
logger.error(f'{destination} is not within the media directory')
self.reset_status()
return False
return True
async def _execute_move_chunks(
self,
plan: Dict[pathlib.Path, pathlib.Path],
session,
created_directories: Set[pathlib.Path],
revert_plan: Dict[pathlib.Path, pathlib.Path],
) -> Set[pathlib.Path]:
"""Execute the move plan in chunks, updating progress.
Returns the set of new directories created.
"""
from wrolpi.files.models import FileGroup, Directory
total_items = len(plan)
processed = 0
new_directories: Set[pathlib.Path] = set()
# Query existing directories once
existing_directories = {
pathlib.Path(d[0]) for d in session.query(Directory.path).all()
}
inserted_directories: Set[pathlib.Path] = set()
for chunk in chunks(list(plan.items()), MOVE_CHUNK_SIZE):
chunk_plan = {}
# Include files that exist OR don't exist (pre-moved)
# but exclude directories
old_files = [old for old, new in chunk if old.is_file() or not old.is_dir()]
old_dirs = [old for old, new in chunk if old.is_dir()]
# Get FileGroups for files in this chunk
file_groups = session.query(FileGroup).filter(
FileGroup.primary_path.in_([str(f) for f in old_files])
).all()
# Build lookup
fg_by_path = {fg.primary_path: fg for fg in file_groups}
# Move each FileGroup's files
for old_file in old_files:
fg = fg_by_path.get(old_file)
if fg:
new_path = plan[old_file]
parent = new_path.parent
parent.mkdir(parents=True, exist_ok=True)
if parent not in new_directories:
new_directories.add(parent)
created_directories.add(parent)
# Only move physical files if source exists
# (files may have been pre-moved by user)
if old_file.exists():
_move_file_group_files(fg, new_path)
# Track for rollback (new -> old)
revert_plan[new_path] = old_file
chunk_plan[old_file] = new_path
# Delete old directories
for old_dir in old_dirs:
if old_dir.is_dir():
try:
delete_directory(old_dir)
except OSError:
pass # Directory not empty yet
# Insert new Directory records
missing_dirs = new_directories - existing_directories - inserted_directories
if missing_dirs:
session.add_all([
Directory(path=str(d), name=d.name) for d in missing_dirs
])
inserted_directories.update(missing_dirs)
# Bulk update FileGroups
_bulk_update_file_groups_db(session, chunk_plan)
session.flush()
# Update progress
processed += len(chunk)
percent = int((processed / total_items) * 100) if total_items > 0 else 100
self.update_status(
operation_processed=processed,
operation_percent=percent,
)
await asyncio.sleep(0) # Yield for cancellation
return new_directories
def _cleanup_old_directories(
self,
sources: List[pathlib.Path],
old_directories: List[pathlib.Path],
) -> None:
"""Clean up old directories after a move operation.
Deletes empty subdirectories within the source tree, but preserves
the root source directories themselves (even if empty).
Optimized: uses single stat() calls and caches directory status to reduce
redundant filesystem operations on slow RPi storage.
"""
from wrolpi.files.lib import delete_directory
# Identify directory sources that were moved (not file parents)
# Only these directories should have their subdirectories cleaned up
# Optimized: single stat() call per source instead of up to 2 (is_dir + potential exists)
moved_directories = set()
for source in sources:
try:
st = source.stat()
if stat_module.S_ISDIR(st.st_mode):
moved_directories.add(source)
except FileNotFoundError:
pass # Source doesn't exist anymore
# Collect subdirectories within moved directories for cleanup
# These are nested subdirs that may be empty after files were moved out
dirs_to_check = set()
for root in moved_directories:
try:
# os.walk efficiently yields directories without redundant stat calls
for dirpath, dirnames, _ in os.walk(root):
for dirname in dirnames:
dirs_to_check.add(pathlib.Path(dirpath) / dirname)
except OSError:
pass # Root was deleted during move
# Sort deepest first to delete children before parents
dirs_to_check = sorted(dirs_to_check, key=lambda p: len(p.parts), reverse=True)
# Batch collect directories that no longer exist for DB cleanup
deleted_dirs_for_db = []
for directory in dirs_to_check:
if directory in moved_directories:
continue # Never delete the moved directory roots themselves
try:
# Try to delete if empty - rmdir fails on non-empty dirs
# delete_directory handles the DB cleanup too
delete_directory(directory)
except FileNotFoundError:
# Directory was already deleted, collect for batch DB cleanup
deleted_dirs_for_db.append(str(directory))
except OSError:
pass # Directory not empty, leave it
# Batch cleanup DB records for directories that were already deleted
if deleted_dirs_for_db:
with get_db_curs(commit=True) as curs:
curs.execute('DELETE FROM directory WHERE path = ANY(%s)', (deleted_dirs_for_db,))
async def _refresh_files_directly(
self,
file_paths: list[pathlib.Path],
expand_stems: bool = True,
) -> FileComparisonResult:
"""Efficiently refresh specific files without scanning entire directories.
Args:
file_paths: List of file paths to refresh
expand_stems: If True, expands each file to its full FileGroup (via glob_shared_stem)
so that refreshing a non-primary file refreshes the whole group.
If False, only refreshes the exact files specified.
Returns a FileComparisonResult with diffs for the affected FileGroups.
"""
from wrolpi.files.models import FileGroup
# Track deleted file paths separately for special handling
deleted_paths: set[pathlib.Path] = set()
# Collect files to process
all_fs_files: set[pathlib.Path] = set()
for file_path in file_paths:
if file_path.is_file():
if expand_stems:
# Expand to all files sharing the same stem (full FileGroup)
related = glob_shared_stem(file_path)
all_fs_files.update(related)
else:
# Only include the specific file
all_fs_files.add(file_path)
elif not file_path.exists():
# File was deleted - still try to find related files or process deletion
deleted_paths.add(file_path)
if expand_stems:
related = glob_shared_stem(file_path)
if related:
all_fs_files.update(related)
else:
# No related files exist, add the original path for deletion processing
all_fs_files.add(file_path)
else:
all_fs_files.add(file_path)
if not all_fs_files:
return FileComparisonResult(unchanged=[], new=[], deleted=[], modified=[])
# Group files by (directory, stem)
fs_groups: Dict[Tuple[str, str], Set[str]] = {}
for file_path in all_fs_files:
directory = str(file_path.parent)
stem, _ = split_path_stem_and_suffix(file_path)
key = (directory, stem)
if key not in fs_groups:
fs_groups[key] = set()
if file_path.exists():
fs_groups[key].add(file_path.name)
# Query existing FileGroups that might match our files
directories = list(set(str(f.parent) for f in all_fs_files))
db_groups: Dict[Tuple[str, str], Tuple[int, Set[str], float]] = {}
with get_db_session() as session:
# Query FileGroups in our directories
query = session.query(FileGroup).filter(FileGroup.directory.in_(directories))
file_groups = query.all()
for fg in file_groups:
if not fg.files:
# Empty FileGroup - compute stem from primary_path
stem, _ = split_path_stem_and_suffix(fg.primary_path)
key = (fg.directory, stem)
if key in fs_groups:
mtime = fg.modification_datetime.timestamp() if fg.modification_datetime else 0
db_groups[key] = (fg.id, set(), mtime)
else:
# Compute stem from first file
first_file = fg.files[0]['path']
stem, _ = split_path_stem_and_suffix(pathlib.Path(fg.directory) / first_file)
key = (fg.directory, stem)
# Include this FileGroup if its stem matches our targets
if key in fs_groups:
fg_filenames = {pathlib.Path(f['path']).name for f in fg.files}
mtime = fg.modification_datetime.timestamp() if fg.modification_datetime else 0
db_groups[key] = (fg.id, fg_filenames, mtime)
# Also query for FileGroups by primary_path for deleted files
# This handles cases where the file was deleted and no related files exist
if deleted_paths:
deleted_path_strs = [str(p) for p in deleted_paths]
deleted_fgs = session.query(FileGroup).filter(
FileGroup.primary_path.in_(deleted_path_strs)
).all()
for fg in deleted_fgs:
stem, _ = split_path_stem_and_suffix(fg.primary_path)
key = (fg.directory, stem)
if key not in db_groups:
fg_filenames = {pathlib.Path(f['path']).name for f in fg.files} if fg.files else set()
mtime = fg.modification_datetime.timestamp() if fg.modification_datetime else 0
db_groups[key] = (fg.id, fg_filenames, mtime)
# Ensure the key is in fs_groups (with empty set for deleted)
if key not in fs_groups:
fs_groups[key] = set()
# Also query for FileGroups contained in deleted directories
# This handles cases where a directory was deleted with files inside
for deleted_path in deleted_paths:
deleted_path_str = str(deleted_path)
dir_deleted_fgs = session.query(FileGroup).filter(
or_(
FileGroup.directory == deleted_path_str,
FileGroup.directory.like(f'{deleted_path_str}/%')
)
).all()
for fg in dir_deleted_fgs:
stem, _ = split_path_stem_and_suffix(fg.primary_path)
key = (fg.directory, stem)
if key not in db_groups:
fg_filenames = {pathlib.Path(f['path']).name for f in fg.files} if fg.files else set()
mtime = fg.modification_datetime.timestamp() if fg.modification_datetime else 0
db_groups[key] = (fg.id, fg_filenames, mtime)
# Ensure the key is in fs_groups (with empty set for deleted)
if key not in fs_groups:
fs_groups[key] = set()
# Build diffs by comparing fs_groups with db_groups
unchanged = []
new = []
deleted = []
modified = []
all_keys = set(fs_groups.keys()) | set(db_groups.keys())
for key in all_keys:
directory, stem = key
fs_files = fs_groups.get(key, set())
db_id, db_files, db_mtime = db_groups.get(key, (None, set(), 0))
diff = FileGroupDiff(
directory=Path(directory),
stem=stem,
db_files=db_files,
fs_files=fs_files,
file_group_id=db_id,
)
if diff.is_new:
new.append(diff)
elif diff.is_deleted:
deleted.append(diff)
elif diff.needs_update:
modified.append(diff)
elif diff.is_unchanged:
# Check mtime for content changes
try:
dir_path = Path(directory)
fs_mtime = max(
(dir_path / filename).stat().st_mtime
for filename in fs_files
)
if fs_mtime > db_mtime + 1:
modified.append(diff)
else:
unchanged.append(diff)
except (OSError, ValueError):
unchanged.append(diff)
logger.info(
f'Direct file refresh: {len(unchanged)} unchanged, {len(new)} new, '
f'{len(deleted)} deleted, {len(modified)} modified'
)
return FileComparisonResult(
unchanged=unchanged,
new=new,
deleted=deleted,
modified=modified,
)
async def _apply_post_processing(self, is_global_refresh: bool = False) -> None:
"""Run modelers, indexers, and cleanup after file operations.
Args:
is_global_refresh: If True, send global_* events. Otherwise, skip them.
"""
from wrolpi.tags import save_tags_config
def on_modeling_progress(processed: int, total: int):
percent = int((processed / total) * 100) if total > 0 else 0
self.update_status(operation_total=total, operation_processed=processed, operation_percent=percent)
self.update_status(status='modeling', operation_total=0, operation_processed=0, operation_percent=0)
with flags.file_worker_modeling:
await apply_modelers(progress_callback=on_modeling_progress)
if is_global_refresh:
Events.send_global_refresh_modeling_completed()
def on_indexing_progress(processed: int, total: int):
percent = int((processed / total) * 100) if total > 0 else 0
self.update_status(operation_total=total, operation_processed=processed, operation_percent=percent)
self.update_status(status='indexing', operation_total=0, operation_processed=0, operation_percent=0)
with flags.file_worker_indexing:
await apply_indexers(progress_callback=on_indexing_progress)
if is_global_refresh:
Events.send_global_refresh_indexing_completed()
self.update_status(status='cleanup', operation_total=0, operation_processed=0, operation_percent=0)
with flags.file_worker_cleanup:
await apply_refresh_cleanup()
save_tags_config.activate_switch()
def _revert_move(
self,
revert_plan: Dict[pathlib.Path, pathlib.Path],
created_directories: Set[pathlib.Path],
destination: pathlib.Path,
destination_existed: bool,
) -> None:
"""Revert a failed move operation by moving files back to original locations."""
# Rollback: move files back to original locations
# Use shutil.move directly (not _move_file_group_files) to avoid any
# issues with the function that may have caused the original failure
if revert_plan:
for new_path, old_path in revert_plan.items():
if new_path.exists():
try:
old_path.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(new_path), str(old_path))
logger.debug(f'Reverted: {new_path} -> {old_path}')
except Exception as revert_error:
logger.error(f'Failed to revert {new_path}: {revert_error}')
# Clean up created directories (deepest first)
dirs_to_clean = sorted(created_directories, key=lambda p: len(p.parents), reverse=True)
for directory in dirs_to_clean:
if directory.is_dir():
try:
# Only delete if empty
if not any(directory.iterdir()):
delete_directory(directory)
except (OSError, StopIteration):
pass
# Clean up destination if we created it
if not destination_existed and destination.is_dir():
try:
# Walk destination and delete empty dirs (deepest first)
for path in sorted(walk(destination), key=lambda p: len(p.parents), reverse=True):
if path.is_dir() and not any(path.iterdir()):
delete_directory(path)
# Finally delete destination if empty
if not any(destination.iterdir()):
delete_directory(destination)
except (OSError, StopIteration):
pass
async def handle_move(self, task: FileTask):
"""Handle move task with progress tracking.
Moves files/directories to the destination using bulk operations.
Updates status throughout for frontend progress display.
"""
from wrolpi.events import Events
destination = task.destination
sources = task.paths
if not destination or not sources:
logger.error('Invalid move task: missing destination or sources')
self.reset_status()
return
media_directory = get_media_directory()
if not self._validate_move_paths(sources, destination, media_directory):
return
# Estimate total for planning phase based on number of sources
planning_total = len(sources) * 2 # Approximation for progress tracking
self.update_status(
status='planning',
task_type='move',
paths=[str(p) for p in sources],
destination=str(destination),
operation_total=planning_total,
operation_processed=0,
operation_percent=0,
error=None,
)
# Track if destination existed before we create it (for cleanup on failure)
destination_existed = destination.is_dir()
# Track moved files for rollback (new_path -> old_path)
revert_plan: Dict[pathlib.Path, pathlib.Path] = {}
# Track directories created during move (for cleanup on failure)
created_directories: Set[pathlib.Path] = set()
plan = {}
# Progress callback for planning phase
def on_planning_progress(processed: int, total: int):
percent = int((processed / total) * 100) if total > 0 else 0
self.update_status(operation_processed=processed, operation_total=total, operation_percent=percent)
try:
destination.mkdir(parents=True, exist_ok=True)
# Build the move plan using bulk SQL operations
plan, old_directories = await build_move_plan_bulk(sources, destination, on_planning_progress)
# Update status with total
self.update_status(
status='moving',
operation_total=len(plan),
)
# Execute the plan in chunks
with flags.file_worker_discovery:
with get_db_session(commit=True) as session:
await self._execute_move_chunks(
plan, session, created_directories, revert_plan
)
self._cleanup_old_directories(sources, old_directories)
await self._apply_post_processing()
logger.info(f'Move completed: {len(plan)} items moved to {destination}')
Events.send_file_move_completed(f'Moved {len(sources)} items to {destination}')
self._complete_job(task.job_id)
except asyncio.CancelledError:
logger.warning('Move task was cancelled')
raise
except Exception as e:
logger.error(f'Move failed: {e}, reverting {len(revert_plan)} items', exc_info=e)
self.update_status(status='reverting', error=str(e))
self._revert_move(revert_plan, created_directories, destination, destination_existed)
Events.send_file_move_failed(f'Move to {destination} failed: {e}')
finally:
self.reset_status()
async def handle_reorganize(self, task: FileTask):
"""Handle reorganize task for collection file reorganization.
Unlike handle_move which moves all files to a single destination,
handle_reorganize handles arbitrary source->destination mappings
where each file (or FileGroup) can move to a different location.
This is used when a user changes their file_name_format configuration
and wants to reorganize existing files to match the new format.
Optimized for bulk operations:
- Single query to pre-fetch all FileGroups
- Uses FileGroup.files instead of filesystem globs
- Dict lookup for O(n) path matching
- Batched database updates
"""
from wrolpi.events import Events
from wrolpi.files.models import FileGroup
move_mappings = task.move_mappings
if not move_mappings:
logger.info('Reorganize task has no move mappings, nothing to do')
self._complete_job(task.job_id)
self.reset_status()
return
total_moves = len(move_mappings)
logger.info(f'Starting reorganize task with {total_moves} file moves')
# Transition job from 'pending' to 'running' so progress can be tracked
self._set_job_status(task.job_id, 'running')
self.update_status(
status='reorganizing',
task_type='reorganize',
paths=[],
destination=None,
operation_total=total_moves,
operation_processed=0,
operation_percent=0,
error=None,
collection_id=task.collection_id,
collection_kind=task.collection_kind,
)
# Track successful moves for rollback
completed_moves: list[tuple[pathlib.Path, pathlib.Path]] = []
# Track created directories for cleanup
created_directories: set[pathlib.Path] = set()
# Track source directories to clean up after moves
source_directories: set[pathlib.Path] = set()
try:
with flags.file_worker_discovery:
# Phase 1: Bulk pre-fetch all FileGroups
source_paths = [str(s) for s, d in move_mappings]
with get_db_session() as session:
file_groups = session.query(FileGroup).filter(
FileGroup.primary_path.in_(source_paths)
).all()
# Build lookup dict and detach from session
# Note: fg.primary_path is a pathlib.Path (from MediaPathType), but
# source_paths are strings, so we use str() for the dictionary keys
fg_by_path = {}
for fg in file_groups:
fg_by_path[str(fg.primary_path)] = {
'id': fg.id,
'directory': pathlib.Path(fg.directory),
'files': list(fg.files), # Copy the list
'data': dict(fg.data) if fg.data else None,
}
processed = 0
pending_updates = []
for source_path, dest_path in move_mappings:
source_path = pathlib.Path(source_path)
dest_path = pathlib.Path(dest_path)
source_exists = source_path.exists()
dest_exists = dest_path.exists()
# Look up the pre-fetched FileGroup
fg_info = fg_by_path.get(str(source_path))
if not fg_info:
logger.debug(f'FileGroup not found by primary_path: {source_path}')
processed += 1
continue
if source_exists and dest_exists:
# Conflict: both exist - skip with warning
logger.warning(f'Reorganize: both source and destination exist: {source_path} -> {dest_path}')
processed += 1
continue
if not source_exists and not dest_exists:
# Neither exists - skip with warning
logger.warning(f'Reorganize: neither source nor destination exist: {source_path}')
processed += 1
continue
if not source_exists and dest_exists:
# Recovery case: file already moved, just update DB to match destination
logger.info(f'Reorganize recovery: updating DB for already-moved file: {dest_path}')
dest_stem, _ = split_path_stem_and_suffix(dest_path)
# Find existing files at destination that match the FileGroup's file extensions
moved_files = []
for file_info in fg_info['files']:
_, suffix = split_path_stem_and_suffix(pathlib.Path(file_info['path']))
dest_file = dest_path.parent / f'{dest_stem}{suffix}'
if dest_file.exists():
# Map original source path to destination for DB update
src_file = fg_info['directory'] / file_info['path']
moved_files.append((src_file, dest_file))
if moved_files:
# Build DB update using existing destination files
moved_lookup = {src: dst for src, dst in moved_files}
new_files = []
for file_info in fg_info['files']:
old_file_path = pathlib.Path(file_info['path'])
if not old_file_path.is_absolute():
old_file_path = fg_info['directory'] / old_file_path
if old_file_path in moved_lookup:
file_info = dict(file_info)
file_info['path'] = moved_lookup[old_file_path].name
new_files.append(file_info)
new_data = fg_info['data']
if new_data:
new_data = dict(new_data)
filename_lookup = {src.name: dst.name for src, dst in moved_files}
for key, value in new_data.items():
if isinstance(value, str) and value in filename_lookup:
new_data[key] = filename_lookup[value]
elif isinstance(value, list):
new_data[key] = [filename_lookup.get(v, v) for v in value]
pending_updates.append({
'id': fg_info['id'],
'directory': str(dest_path.parent),
'primary_path': str(dest_path),
'files': new_files,
'data': new_data,
})
completed_moves.append((source_path, dest_path))
processed += 1
if processed % PROGRESS_UPDATE_INTERVAL == 0:
percent = int((processed / total_moves) * 100)
self.update_status(operation_processed=processed, operation_percent=percent)
continue
# Normal case: source exists, dest doesn't - move files
# Track source directory for cleanup
source_directories.add(source_path.parent)
# Create destination directory if needed
if not dest_path.parent.exists():
dest_path.parent.mkdir(parents=True, exist_ok=True)
created_directories.add(dest_path.parent)
# Phase 2: Use FileGroup.files instead of glob_shared_stem()
source_stem_files = []
for file_info in fg_info['files']:
file_path = pathlib.Path(file_info['path'])
if not file_path.is_absolute():
file_path = fg_info['directory'] / file_path
source_stem_files.append(file_path)
# Move all files in the FileGroup
dest_stem, _ = split_path_stem_and_suffix(dest_path)
moved_files = []
for src_file in source_stem_files:
if not src_file.exists():
continue
# Compute destination for this file based on its suffix
_, src_suffix = split_path_stem_and_suffix(src_file)
dest_file = dest_path.parent / f'{dest_stem}{src_suffix}'
try:
shutil.move(str(src_file), str(dest_file))
moved_files.append((src_file, dest_file))
logger.debug(f'Reorganize: moved {src_file} -> {dest_file}')
except Exception as e:
logger.error(f'Failed to move {src_file} -> {dest_file}: {e}')
raise
# Prepare FileGroup update
if moved_files:
# Phase 3: Use dict lookup instead of nested loop
moved_lookup = {src: dst for src, dst in moved_files}
# Update files list with new paths
new_files = []
for file_info in fg_info['files']:
old_file_path = pathlib.Path(file_info['path'])
if not old_file_path.is_absolute():
old_file_path = fg_info['directory'] / old_file_path
# Find the corresponding new path using dict lookup
if old_file_path in moved_lookup:
file_info = dict(file_info)
file_info['path'] = moved_lookup[old_file_path].name
new_files.append(file_info)
# Update data dict if it has path references
# Data fields store filenames only (no '/'), so use filename-to-filename lookup
new_data = fg_info['data']
if new_data:
new_data = dict(new_data)
# Build filename-to-filename lookup (not full paths)
filename_lookup = {src.name: dst.name for src, dst in moved_files}
for key, value in new_data.items():
if isinstance(value, str) and value in filename_lookup:
new_data[key] = filename_lookup[value]
elif isinstance(value, list):
# Handle list fields like caption_paths
new_data[key] = [filename_lookup.get(v, v) for v in value]
# Phase 4: Collect update for batch processing
pending_updates.append({
'id': fg_info['id'],
'directory': str(dest_path.parent),
'primary_path': str(dest_path),
'files': new_files,
'data': new_data,
})
completed_moves.append((source_path, dest_path))
# Flush every MOVE_CHUNK_SIZE items
if len(pending_updates) >= MOVE_CHUNK_SIZE:
_bulk_update_file_groups_reorganize(pending_updates)
pending_updates = []
processed += 1
# Update progress
if processed % PROGRESS_UPDATE_INTERVAL == 0 or processed == total_moves:
percent = int((processed / total_moves) * 100)
logger.info(f'Reorganize progress: {processed}/{total_moves} files ({percent}%)')
self.update_status(
operation_processed=processed,
operation_percent=percent,
)
# Final flush of pending updates
if pending_updates:
_bulk_update_file_groups_reorganize(pending_updates)
# Clean up empty source directories
for directory in sorted(source_directories, key=lambda p: len(p.parents), reverse=True):
try:
if directory.is_dir() and not any(directory.iterdir()):
delete_directory(directory)
logger.debug(f'Cleaned up empty directory: {directory}')
except (OSError, StopIteration):
pass
# Skip post-processing - reorganization only moves files, nothing to re-index
# Update file_format after successful completion (deferred update for resumability)
if task.collection_id and task.pending_file_format:
from wrolpi.collections.models import Collection
with get_db_session(commit=True) as session:
collection = session.query(Collection).filter_by(id=task.collection_id).one_or_none()
if collection:
collection.file_format = task.pending_file_format
logger.info(f'Updated file_format for collection {collection.name}')
# Trigger config save
if collection.kind == 'domain':
from modules.archive.lib import save_domains_config
save_domains_config.activate_switch()
elif collection.kind == 'channel':
from modules.videos.lib import save_channels_config
save_channels_config.activate_switch()
logger.info(f'Reorganize completed: {len(completed_moves)} files moved')
Events.send_file_move_completed(f'Reorganized {len(completed_moves)} files')
self._complete_job(task.job_id)
except asyncio.CancelledError:
logger.warning('Reorganize task was cancelled')
raise
except Exception as e:
logger.error(f'Reorganize failed: {e}', exc_info=e)
self.update_status(status='error', error=str(e))
# Note: We don't attempt automatic rollback for reorganize tasks
# as the partial state may be complex. User should refresh files.
Events.send_file_move_failed(f'Reorganize failed: {e}')
finally:
self.reset_status()
async def handle_batch_reorganize(self, task: FileTask):
"""Handle batch reorganization of multiple collections.
Processes collections sequentially:
1. For each collection, execute reorganization
2. Track overall progress and per-collection progress
3. Stop on first failure, report which collection failed
Progress tracking:
- overall_percent = (completed_collections + current_collection_%) / total_collections
- batch_status dict tracks completed list, current collection, and failure info
"""
from wrolpi.collections.reorganize import execute_reorganization
from wrolpi.collections.models import Collection
from wrolpi.events import Events
collection_ids = task.collection_ids or []
kind = task.batch_kind
if not collection_ids:
logger.info('Batch reorganize task has no collections, nothing to do')
self._complete_job(task.job_id)
self.reset_status()
return
total_collections = len(collection_ids)
logger.info(f'Starting batch reorganization of {total_collections} {kind} collections')
# Transition job from 'pending' to 'running'
self._set_job_status(task.job_id, 'running')
# Initialize batch status tracking
batch_status = {
'batch_job_id': task.job_id, # Include job ID so frontend can resume polling
'total_collections': total_collections,
'completed': [],
'current_collection': None,
'failed_collection': None,
'error': None,
'overall_percent': 0,
}
self.update_status(
status='batch_reorganizing',
task_type='batch_reorganize',
paths=[],
destination=None,
operation_total=total_collections,
operation_processed=0,
operation_percent=0,
error=None,
batch_status=batch_status,
collection_kind=kind,
)
try:
for idx, collection_id in enumerate(collection_ids):
# Fetch collection info
with get_db_session() as session:
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
if not collection:
logger.warning(f'Batch reorganize: collection {collection_id} not found, skipping')
continue
collection_name = collection.name
# Update current collection in status
batch_status['current_collection'] = {
'id': collection_id,
'name': collection_name,
'status': 'running',
'total': 0,
'completed': 0,
'percent': 0,
}
self.update_status(batch_status=batch_status)
logger.info(f'Batch reorganize: processing {collection_name} ({idx + 1}/{total_collections})')
try:
# Execute reorganization for this collection
# This queues a sub-job that we need to wait for
job_id = execute_reorganization(collection_id, session)
if job_id:
# Wait for the reorganization job to complete
# Poll for completion while updating batch status
await self._wait_for_sub_job_with_progress(
job_id, batch_status, collection_id, collection_name, idx, total_collections
)
# Mark this collection as completed
batch_status['completed'].append({
'id': collection_id,
'name': collection_name,
'status': 'complete',
})
# Update overall progress
completed_count = len(batch_status['completed'])
batch_status['overall_percent'] = int((completed_count / total_collections) * 100)
batch_status['current_collection'] = None
self.update_status(
operation_processed=completed_count,
operation_percent=batch_status['overall_percent'],
batch_status=batch_status,
)
logger.info(f'Batch reorganize: completed {collection_name}')
except Exception as e:
# Collection failed - stop batch processing
logger.error(f'Batch reorganize: {collection_name} failed: {e}')
# Get the actual channel ID for the UI link (collection_id != channel_id)
channel_id = None
if kind == 'channel':
from modules.videos.models import Channel
channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none()
channel_id = channel.id if channel else None
batch_status['failed_collection'] = {
'id': collection_id,
'name': collection_name,
'channel_id': channel_id,
}
batch_status['error'] = str(e)
batch_status['current_collection'] = None
self.update_status(
status='error',
error=str(e),
batch_status=batch_status,
)
self._set_job_status(task.job_id, 'failed')
Events.send_file_move_failed(
f'Batch reorganization failed on {collection_name}: {e}'
)
return
# All collections completed successfully
logger.info(f'Batch reorganization completed: {len(batch_status["completed"])} collections processed')
Events.send_file_move_completed(
f'Batch reorganization completed: {len(batch_status["completed"])} {kind}s reorganized'
)
self._complete_job(task.job_id)
except asyncio.CancelledError:
logger.warning('Batch reorganize task was cancelled')
raise
except Exception as e:
logger.error(f'Batch reorganize failed: {e}', exc_info=e)
batch_status['error'] = str(e)
self.update_status(status='error', error=str(e), batch_status=batch_status)
self._set_job_status(task.job_id, 'failed')
Events.send_file_move_failed(f'Batch reorganization failed: {e}')
finally:
self.reset_status()
async def _wait_for_sub_job_with_progress(
self,
job_id: str,
batch_status: dict,
collection_id: int,
collection_name: str,
collection_idx: int,
total_collections: int,
):
"""Wait for a sub-job to complete while updating batch progress.
This is used by handle_batch_reorganize to wait for each collection's
reorganization job while keeping the batch status updated.
"""
import time
start = time.time()
timeout = 3600 # 1 hour max per collection
while time.time() - start < timeout:
status = self.get_job_status(job_id)
if status == 'complete':
return
# Transfer and process queue FIRST to ensure status is updated
self.transfer_queue()
await self.process_queue()
# Update current collection progress based on worker status (now fresh)
worker_status = self.status
current = batch_status.get('current_collection', {})
if current:
current['total'] = worker_status.get('operation_total', 0)
current['completed'] = worker_status.get('operation_processed', 0)
current['percent'] = worker_status.get('operation_percent', 0)
batch_status['current_collection'] = current
# Calculate overall percent including current progress
completed_count = len(batch_status.get('completed', []))
current_progress = current['percent'] / 100
batch_status['overall_percent'] = int(
((completed_count + current_progress) / total_collections) * 100
)
self.update_status(batch_status=batch_status)
await asyncio.sleep(0.05)
raise TimeoutError(f'Collection {collection_name} reorganization timed out')
# All processes create a FileWorker, but only one receives the signals and actually does work.
file_worker = FileWorker()