mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 23:16:17 -05:00
Moving collections in background task.
This commit is contained in:
parent
78251b067e
commit
457406a7f7
3 changed files with 49 additions and 11 deletions
|
|
@ -11,8 +11,9 @@ from sqlalchemy import func
|
|||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi import flags
|
||||
from wrolpi.common import logger, get_relative_to_media_directory
|
||||
from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
|
||||
from wrolpi.db import get_db_session
|
||||
import pathlib
|
||||
from wrolpi.errors import RefreshConflict
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.events import Events
|
||||
|
|
@ -309,6 +310,25 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None:
|
|||
Events.send_directory_refresh(f'Refreshing: {relative_dir}')
|
||||
|
||||
|
||||
async def _background_move_collection(collection_id: int, target_directory: pathlib.Path):
|
||||
"""Helper to run collection move in background with its own database session.
|
||||
|
||||
This allows the move to continue even if the original HTTP request is cancelled
|
||||
(e.g., user closes browser tab).
|
||||
"""
|
||||
try:
|
||||
with get_db_session(commit=True) as session:
|
||||
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
|
||||
if not collection:
|
||||
logger.error(f'_background_move_collection: collection {collection_id} not found')
|
||||
Events.send_file_move_failed(f'Collection move failed: collection not found')
|
||||
return
|
||||
await collection.move_collection(target_directory, session, send_events=True)
|
||||
except Exception as e:
|
||||
logger.error(f'_background_move_collection: failed for collection_id={collection_id}', exc_info=e)
|
||||
Events.send_file_move_failed(f'Collection move failed: {e}')
|
||||
|
||||
|
||||
async def tag_collection(
|
||||
session: Session,
|
||||
collection_id: int,
|
||||
|
|
@ -357,6 +377,10 @@ async def tag_collection(
|
|||
target_directory != old_directory
|
||||
)
|
||||
|
||||
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, '
|
||||
f'{old_directory} -> {target_directory}')
|
||||
|
||||
if need_to_move and flags.refreshing.is_set():
|
||||
raise RefreshConflict('Refusing to move collection while file refresh is in progress')
|
||||
|
||||
|
|
@ -372,10 +396,12 @@ async def tag_collection(
|
|||
from modules.videos.lib import save_channels_config
|
||||
save_channels_config.activate_switch()
|
||||
|
||||
# Move files if directory changed
|
||||
# Move files if directory changed - run in background so closing tab won't cancel it
|
||||
if need_to_move:
|
||||
target_directory.mkdir(parents=True, exist_ok=True)
|
||||
await collection.move_collection(target_directory, session, send_events=True)
|
||||
# Commit tag changes before starting background move
|
||||
session.commit()
|
||||
background_task(_background_move_collection(collection.id, target_directory))
|
||||
|
||||
# Return info about the un-tagging operation
|
||||
relative_dir = get_relative_to_media_directory(target_directory) if target_directory else None
|
||||
|
|
@ -413,6 +439,10 @@ async def tag_collection(
|
|||
target_directory != old_directory
|
||||
)
|
||||
|
||||
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, '
|
||||
f'{old_directory} -> {target_directory}')
|
||||
|
||||
if need_to_move and flags.refreshing.is_set():
|
||||
raise RefreshConflict('Refusing to move collection while file refresh is in progress')
|
||||
|
||||
|
|
@ -435,10 +465,12 @@ async def tag_collection(
|
|||
from modules.videos.lib import save_channels_config
|
||||
save_channels_config.activate_switch()
|
||||
|
||||
# Move files if directory changed
|
||||
# Move files if directory changed - run in background so closing tab won't cancel it
|
||||
if need_to_move:
|
||||
target_directory.mkdir(parents=True, exist_ok=True)
|
||||
await collection.move_collection(target_directory, session, send_events=True)
|
||||
# Commit tag changes before starting background move
|
||||
session.commit()
|
||||
background_task(_background_move_collection(collection.id, target_directory))
|
||||
|
||||
# Return info about the tagging operation
|
||||
relative_dir = get_relative_to_media_directory(target_directory) if target_directory else None
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from sqlalchemy.orm.collections import InstrumentedList
|
|||
|
||||
from wrolpi import flags
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \
|
||||
unique_by_predicate
|
||||
unique_by_predicate, TRACE_LEVEL
|
||||
from wrolpi.downloader import Download, save_downloads_config
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.events import Events
|
||||
|
|
@ -861,6 +861,8 @@ class Collection(ModelHelper, Base):
|
|||
downloads = list(self.downloads)
|
||||
downloads.extend(Download.get_all_by_destination(session, from_directory))
|
||||
downloads = unique_by_predicate(downloads, lambda i: i.id)
|
||||
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(f'move_collection: updating {len(downloads)} download destinations')
|
||||
for download in downloads:
|
||||
download.destination = to_directory
|
||||
session.flush(downloads)
|
||||
|
|
@ -884,7 +886,10 @@ class Collection(ModelHelper, Base):
|
|||
if send_events:
|
||||
Events.send_file_move_completed(f'Collection {repr(self.name)} was moved (directory missing)')
|
||||
else:
|
||||
await move_files(session, directory, *list(old_directory.iterdir()))
|
||||
files_to_move = list(old_directory.iterdir())
|
||||
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(f'move_collection: moving {len(files_to_move)} items from {old_directory}')
|
||||
await move_files(session, directory, *files_to_move)
|
||||
if send_events:
|
||||
Events.send_file_move_completed(f'Collection {repr(self.name)} was moved')
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -1336,12 +1336,14 @@ async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.P
|
|||
|
||||
# Sort plan by the deepest files first.
|
||||
plan: OrderedDict = OrderedDict(sorted(plan.items(), key=lambda i: (len(i[0].parents), i[0].name), reverse=True))
|
||||
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(f'_move: plan created with {len(plan)} items to {destination}')
|
||||
|
||||
# Revert plan is built out as files are moved.
|
||||
revert_plan = OrderedDict()
|
||||
new_directories = set()
|
||||
|
||||
def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path]):
|
||||
def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path], is_revert: bool = False):
|
||||
"""Apply the move plan to all the FileGroups.
|
||||
|
||||
@warning: Cannot be cancelled!"""
|
||||
|
|
@ -1382,10 +1384,10 @@ async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.P
|
|||
delete_directory(source)
|
||||
logger.info(f'Move execution completed')
|
||||
except Exception as e:
|
||||
logger.error(f'Move failed', exc_info=e)
|
||||
logger.error(f'_move failed, reverting {len(revert_plan)} items', exc_info=e)
|
||||
new_directories = set()
|
||||
# Move files back. Get copy because do_plan will change the revert plan.
|
||||
do_plan(revert_plan.copy())
|
||||
do_plan(revert_plan.copy(), is_revert=True)
|
||||
# Delete the directories that were created.
|
||||
directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
|
||||
for path in directories:
|
||||
|
|
@ -1435,7 +1437,6 @@ async def move(session: Session, destination: pathlib.Path, *sources: pathlib.Pa
|
|||
|
||||
Returns an OrderedDict, the key is the file's old location, the value is where the file was moved.
|
||||
"""
|
||||
|
||||
with flags.refreshing:
|
||||
from wrolpi.api_utils import api_app
|
||||
api_app.shared_ctx.refresh['counted_files'] = 0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue