From 457406a7f76662dc29f8e6a04b11ff574274c8e2 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 21 Dec 2025 12:13:39 -0700 Subject: [PATCH] Moving collections in background task. --- wrolpi/collections/lib.py | 42 +++++++++++++++++++++++++++++++----- wrolpi/collections/models.py | 9 ++++++-- wrolpi/files/lib.py | 9 ++++---- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/wrolpi/collections/lib.py b/wrolpi/collections/lib.py index ace3dc4a..27f2824b 100644 --- a/wrolpi/collections/lib.py +++ b/wrolpi/collections/lib.py @@ -11,8 +11,9 @@ from sqlalchemy import func from sqlalchemy.orm import Session from wrolpi import flags -from wrolpi.common import logger, get_relative_to_media_directory +from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task from wrolpi.db import get_db_session +import pathlib from wrolpi.errors import RefreshConflict from wrolpi.errors import ValidationError from wrolpi.events import Events @@ -309,6 +310,25 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None: Events.send_directory_refresh(f'Refreshing: {relative_dir}') +async def _background_move_collection(collection_id: int, target_directory: pathlib.Path): + """Helper to run collection move in background with its own database session. + + This allows the move to continue even if the original HTTP request is cancelled + (e.g., user closes browser tab). + """ + try: + with get_db_session(commit=True) as session: + collection = session.query(Collection).filter_by(id=collection_id).one_or_none() + if not collection: + logger.error(f'_background_move_collection: collection {collection_id} not found') + Events.send_file_move_failed(f'Collection move failed: collection not found') + return + await collection.move_collection(target_directory, session, send_events=True) + except Exception as e: + logger.error(f'_background_move_collection: failed for collection_id={collection_id}', exc_info=e) + Events.send_file_move_failed(f'Collection move failed: {e}') + + async def tag_collection( session: Session, collection_id: int, @@ -357,6 +377,10 @@ async def tag_collection( target_directory != old_directory ) + if __debug__ and logger.isEnabledFor(TRACE_LEVEL): + logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, ' + f'{old_directory} -> {target_directory}') + if need_to_move and flags.refreshing.is_set(): raise RefreshConflict('Refusing to move collection while file refresh is in progress') @@ -372,10 +396,12 @@ async def tag_collection( from modules.videos.lib import save_channels_config save_channels_config.activate_switch() - # Move files if directory changed + # Move files if directory changed - run in background so closing tab won't cancel it if need_to_move: target_directory.mkdir(parents=True, exist_ok=True) - await collection.move_collection(target_directory, session, send_events=True) + # Commit tag changes before starting background move + session.commit() + background_task(_background_move_collection(collection.id, target_directory)) # Return info about the un-tagging operation relative_dir = get_relative_to_media_directory(target_directory) if target_directory else None @@ -413,6 +439,10 @@ async def tag_collection( target_directory != old_directory ) + if __debug__ and logger.isEnabledFor(TRACE_LEVEL): + logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, ' + f'{old_directory} -> {target_directory}') + if need_to_move and flags.refreshing.is_set(): raise RefreshConflict('Refusing to move collection while file refresh is in progress') @@ -435,10 +465,12 @@ async def tag_collection( from modules.videos.lib import save_channels_config save_channels_config.activate_switch() - # Move files if directory changed + # Move files if directory changed - run in background so closing tab won't cancel it if need_to_move: target_directory.mkdir(parents=True, exist_ok=True) - await collection.move_collection(target_directory, session, send_events=True) + # Commit tag changes before starting background move + session.commit() + background_task(_background_move_collection(collection.id, target_directory)) # Return info about the tagging operation relative_dir = get_relative_to_media_directory(target_directory) if target_directory else None diff --git a/wrolpi/collections/models.py b/wrolpi/collections/models.py index d26c38e7..d5eed96e 100644 --- a/wrolpi/collections/models.py +++ b/wrolpi/collections/models.py @@ -7,7 +7,7 @@ from sqlalchemy.orm.collections import InstrumentedList from wrolpi import flags from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \ - unique_by_predicate + unique_by_predicate, TRACE_LEVEL from wrolpi.downloader import Download, save_downloads_config from wrolpi.errors import ValidationError from wrolpi.events import Events @@ -861,6 +861,8 @@ class Collection(ModelHelper, Base): downloads = list(self.downloads) downloads.extend(Download.get_all_by_destination(session, from_directory)) downloads = unique_by_predicate(downloads, lambda i: i.id) + if __debug__ and logger.isEnabledFor(TRACE_LEVEL): + logger.trace(f'move_collection: updating {len(downloads)} download destinations') for download in downloads: download.destination = to_directory session.flush(downloads) @@ -884,7 +886,10 @@ class Collection(ModelHelper, Base): if send_events: Events.send_file_move_completed(f'Collection {repr(self.name)} was moved (directory missing)') else: - await move_files(session, directory, *list(old_directory.iterdir())) + files_to_move = list(old_directory.iterdir()) + if __debug__ and logger.isEnabledFor(TRACE_LEVEL): + logger.trace(f'move_collection: moving {len(files_to_move)} items from {old_directory}') + await move_files(session, directory, *files_to_move) if send_events: Events.send_file_move_completed(f'Collection {repr(self.name)} was moved') except Exception as e: diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index f1782bf5..8e284b10 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -1336,12 +1336,14 @@ async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.P # Sort plan by the deepest files first. plan: OrderedDict = OrderedDict(sorted(plan.items(), key=lambda i: (len(i[0].parents), i[0].name), reverse=True)) + if __debug__ and logger.isEnabledFor(TRACE_LEVEL): + logger.trace(f'_move: plan created with {len(plan)} items to {destination}') # Revert plan is built out as files are moved. revert_plan = OrderedDict() new_directories = set() - def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path]): + def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path], is_revert: bool = False): """Apply the move plan to all the FileGroups. @warning: Cannot be cancelled!""" @@ -1382,10 +1384,10 @@ async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.P delete_directory(source) logger.info(f'Move execution completed') except Exception as e: - logger.error(f'Move failed', exc_info=e) + logger.error(f'_move failed, reverting {len(revert_plan)} items', exc_info=e) new_directories = set() # Move files back. Get copy because do_plan will change the revert plan. - do_plan(revert_plan.copy()) + do_plan(revert_plan.copy(), is_revert=True) # Delete the directories that were created. directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True) for path in directories: @@ -1435,7 +1437,6 @@ async def move(session: Session, destination: pathlib.Path, *sources: pathlib.Pa Returns an OrderedDict, the key is the file's old location, the value is where the file was moved. """ - with flags.refreshing: from wrolpi.api_utils import api_app api_app.shared_ctx.refresh['counted_files'] = 0