Moving collections in background task.

This commit is contained in:
Roland 2025-12-21 12:13:39 -07:00
parent 78251b067e
commit 457406a7f7
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
3 changed files with 49 additions and 11 deletions

View file

@ -11,8 +11,9 @@ from sqlalchemy import func
from sqlalchemy.orm import Session
from wrolpi import flags
from wrolpi.common import logger, get_relative_to_media_directory
from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
from wrolpi.db import get_db_session
import pathlib
from wrolpi.errors import RefreshConflict
from wrolpi.errors import ValidationError
from wrolpi.events import Events
@ -309,6 +310,25 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None:
Events.send_directory_refresh(f'Refreshing: {relative_dir}')
async def _background_move_collection(collection_id: int, target_directory: pathlib.Path):
"""Helper to run collection move in background with its own database session.
This allows the move to continue even if the original HTTP request is cancelled
(e.g., user closes browser tab).
"""
try:
with get_db_session(commit=True) as session:
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
if not collection:
logger.error(f'_background_move_collection: collection {collection_id} not found')
Events.send_file_move_failed(f'Collection move failed: collection not found')
return
await collection.move_collection(target_directory, session, send_events=True)
except Exception as e:
logger.error(f'_background_move_collection: failed for collection_id={collection_id}', exc_info=e)
Events.send_file_move_failed(f'Collection move failed: {e}')
async def tag_collection(
session: Session,
collection_id: int,
@ -357,6 +377,10 @@ async def tag_collection(
target_directory != old_directory
)
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, '
f'{old_directory} -> {target_directory}')
if need_to_move and flags.refreshing.is_set():
raise RefreshConflict('Refusing to move collection while file refresh is in progress')
@ -372,10 +396,12 @@ async def tag_collection(
from modules.videos.lib import save_channels_config
save_channels_config.activate_switch()
# Move files if directory changed
# Move files if directory changed - run in background so closing tab won't cancel it
if need_to_move:
target_directory.mkdir(parents=True, exist_ok=True)
await collection.move_collection(target_directory, session, send_events=True)
# Commit tag changes before starting background move
session.commit()
background_task(_background_move_collection(collection.id, target_directory))
# Return info about the un-tagging operation
relative_dir = get_relative_to_media_directory(target_directory) if target_directory else None
@ -413,6 +439,10 @@ async def tag_collection(
target_directory != old_directory
)
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, '
f'{old_directory} -> {target_directory}')
if need_to_move and flags.refreshing.is_set():
raise RefreshConflict('Refusing to move collection while file refresh is in progress')
@ -435,10 +465,12 @@ async def tag_collection(
from modules.videos.lib import save_channels_config
save_channels_config.activate_switch()
# Move files if directory changed
# Move files if directory changed - run in background so closing tab won't cancel it
if need_to_move:
target_directory.mkdir(parents=True, exist_ok=True)
await collection.move_collection(target_directory, session, send_events=True)
# Commit tag changes before starting background move
session.commit()
background_task(_background_move_collection(collection.id, target_directory))
# Return info about the tagging operation
relative_dir = get_relative_to_media_directory(target_directory) if target_directory else None

View file

@ -7,7 +7,7 @@ from sqlalchemy.orm.collections import InstrumentedList
from wrolpi import flags
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \
unique_by_predicate
unique_by_predicate, TRACE_LEVEL
from wrolpi.downloader import Download, save_downloads_config
from wrolpi.errors import ValidationError
from wrolpi.events import Events
@ -861,6 +861,8 @@ class Collection(ModelHelper, Base):
downloads = list(self.downloads)
downloads.extend(Download.get_all_by_destination(session, from_directory))
downloads = unique_by_predicate(downloads, lambda i: i.id)
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'move_collection: updating {len(downloads)} download destinations')
for download in downloads:
download.destination = to_directory
session.flush(downloads)
@ -884,7 +886,10 @@ class Collection(ModelHelper, Base):
if send_events:
Events.send_file_move_completed(f'Collection {repr(self.name)} was moved (directory missing)')
else:
await move_files(session, directory, *list(old_directory.iterdir()))
files_to_move = list(old_directory.iterdir())
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'move_collection: moving {len(files_to_move)} items from {old_directory}')
await move_files(session, directory, *files_to_move)
if send_events:
Events.send_file_move_completed(f'Collection {repr(self.name)} was moved')
except Exception as e:

View file

@ -1336,12 +1336,14 @@ async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.P
# Sort plan by the deepest files first.
plan: OrderedDict = OrderedDict(sorted(plan.items(), key=lambda i: (len(i[0].parents), i[0].name), reverse=True))
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'_move: plan created with {len(plan)} items to {destination}')
# Revert plan is built out as files are moved.
revert_plan = OrderedDict()
new_directories = set()
def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path]):
def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path], is_revert: bool = False):
"""Apply the move plan to all the FileGroups.
@warning: Cannot be cancelled!"""
@ -1382,10 +1384,10 @@ async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.P
delete_directory(source)
logger.info(f'Move execution completed')
except Exception as e:
logger.error(f'Move failed', exc_info=e)
logger.error(f'_move failed, reverting {len(revert_plan)} items', exc_info=e)
new_directories = set()
# Move files back. Get copy because do_plan will change the revert plan.
do_plan(revert_plan.copy())
do_plan(revert_plan.copy(), is_revert=True)
# Delete the directories that were created.
directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
for path in directories:
@ -1435,7 +1437,6 @@ async def move(session: Session, destination: pathlib.Path, *sources: pathlib.Pa
Returns an OrderedDict, the key is the file's old location, the value is where the file was moved.
"""
with flags.refreshing:
from wrolpi.api_utils import api_app
api_app.shared_ctx.refresh['counted_files'] = 0