diff --git a/wrolpi/files/move.py b/wrolpi/files/move.py index 2e38deb5..13f17bc4 100644 --- a/wrolpi/files/move.py +++ b/wrolpi/files/move.py @@ -10,6 +10,7 @@ import os import pathlib import shutil import stat as stat_module +import threading from collections import OrderedDict from pathlib import Path from typing import Callable, Dict, List, Set, Tuple @@ -26,9 +27,13 @@ from wrolpi.files.lib import ( _move_file_group_files, _bulk_update_file_groups_db, MOVE_CHUNK_SIZE, get_normalized_ignored_directories, remove_files_in_ignored_directories, ) +from wrolpi.vars import PYTEST logger = logger.getChild(__name__) +# How often a move waits for its thread to leave the transaction. +MOVE_THREAD_POLL_SECONDS = 0.05 + async def build_move_plan_bulk( sources: list[pathlib.Path], @@ -163,11 +168,13 @@ async def build_move_plan_bulk( [{"path": str(p), "type": 'deleted', "dir": str(p.parent)} for p in deleted_sources], ) - # Single bulk query to get all FileGroups matching our sources + # Single bulk query to get all FileGroups matching our sources. CROSS JOIN makes SQLite read + # move_sources first and look each path up in file_group's primary_path index; the temp table has no + # statistics, so a plain JOIN scans all of file_group (gigabytes on a large library). result = session.execute(text(""" SELECT fg.id, fg.primary_path, fg.directory, ms.source_path, ms.source_type - FROM file_group fg - JOIN move_sources ms ON fg.primary_path = ms.source_path + FROM move_sources ms + CROSS JOIN file_group fg ON fg.primary_path = ms.source_path """)) # Build lookup of found FileGroups @@ -293,7 +300,74 @@ class FileMoveMixin: return False return True - async def _execute_move_chunks( + def _execute_move_plan( + self, + plan: Dict[pathlib.Path, pathlib.Path], + created_directories: Set[pathlib.Path], + revert_plan: Dict[pathlib.Path, pathlib.Path], + ) -> None: + """Move the files and update the DB in one transaction, which commits when this returns. + + This reads FileGroups/Directories then updates them, so it must begin as a writer; a deferred + transaction's lock upgrade fails instantly when anything else is writing (see `build_move_plan_bulk`).""" + with flags.file_worker_discovery: + with get_db_session(commit=True) as session: + self._execute_move_chunks(plan, session, created_directories, revert_plan) + + async def _execute_move_plan_off_loop( + self, + plan: Dict[pathlib.Path, pathlib.Path], + created_directories: Set[pathlib.Path], + revert_plan: Dict[pathlib.Path, pathlib.Path], + ) -> None: + """Run `_execute_move_plan` in a thread, so the event loop never waits while the move holds the write lock. + + The process that moves files also downloads. If the move's transaction yielded to the event loop, a + download's write would wait out the whole busy_timeout synchronously -- the event loop cannot resume the + move that holds the lock -- and then fail. In a thread, the move keeps going and the other writer gets the + lock when it commits. + + Tests share one connection, which is not safe to use from two threads, so tests run it inline.""" + if PYTEST: + self._execute_move_plan(plan, created_directories, revert_plan) + return + + # A thread cannot be interrupted, and the caller reverts `revert_plan` when this raises. So this returns or + # raises only after the thread has left its transaction, however often this task is cancelled meanwhile. + # (Shutdown cancels it more than once: the perpetual loops are cancelled, then asyncio.run cancels every + # pending task, including any asyncio future of the thread, which then reads "done" while the thread runs.) + # The thread's own Event is the only signal that its transaction is over. + finished = threading.Event() + outcome = dict() + + def run(): + try: + self._execute_move_plan(plan, created_directories, revert_plan) + except BaseException as e: + outcome['error'] = e + finally: + finished.set() + + asyncio.get_running_loop().run_in_executor(None, run) + cancelled = False + while not finished.is_set(): + try: + await asyncio.sleep(MOVE_THREAD_POLL_SECONDS) + except asyncio.CancelledError: + cancelled = True + asyncio.current_task().uncancel() + + if error := outcome.get('error'): + # It rolled back; the caller reverts the files it moved. + if cancelled: + raise asyncio.CancelledError() from error + raise error + if cancelled: + # It committed; the files and the DB agree, so there is nothing to revert. + revert_plan.clear() + raise asyncio.CancelledError() + + def _execute_move_chunks( self, plan: Dict[pathlib.Path, pathlib.Path], session, @@ -378,7 +452,6 @@ class FileMoveMixin: operation_processed=processed, operation_percent=percent, ) - await asyncio.sleep(0) # Yield for cancellation return new_directories @@ -588,14 +661,7 @@ class FileMoveMixin: operation_total=len(plan), ) - # Execute the plan in chunks. This reads FileGroups/Directories then updates them, so - # it must begin as a writer; a deferred transaction's lock upgrade fails instantly when - # anything else is writing (see `build_move_plan_bulk`). - with flags.file_worker_discovery: - with get_db_session(commit=True) as session: - await self._execute_move_chunks( - plan, session, created_directories, revert_plan - ) + await self._execute_move_plan_off_loop(plan, created_directories, revert_plan) # The move committed; the files and the DB agree, so a later failure must not move the files back. revert_plan.clear() diff --git a/wrolpi/files/test/test_worker_move.py b/wrolpi/files/test/test_worker_move.py index d1231953..4c390bac 100644 --- a/wrolpi/files/test/test_worker_move.py +++ b/wrolpi/files/test/test_worker_move.py @@ -1,11 +1,17 @@ """Tests for the FileWorker move task.""" +import asyncio import contextlib +import threading +import time import pytest +from sqlalchemy import event, text from wrolpi.conftest import production_like_sessions, write_lock_held_briefly +from wrolpi.db import get_db_session +from wrolpi.files import move as move_module from wrolpi.files import worker as worker_module -from wrolpi.files.lib import _move_file_group_files +from wrolpi.files.lib import _move_file_group_files, MOVE_CHUNK_SIZE from wrolpi.files.models import FileGroup, Directory from wrolpi.files.worker import file_worker, FileTask, FileTaskType, FileWorkerJobFailed, build_move_plan_bulk @@ -719,3 +725,144 @@ async def test_file_worker_move_survives_concurrent_writer( session = maker() fg = session.query(FileGroup).one() assert fg.primary_path == dest / 'file1.txt', 'FileGroup was not updated to the new path' + + +@pytest.mark.asyncio +async def test_file_worker_move_does_not_stall_writers_on_the_event_loop( + async_client, test_session, test_directory, make_files_structure, monkeypatch +): + """A move's write transaction must not need the event loop to finish. + + The process that moves files also downloads. A download's write that waits on the move's lock blocks the event + loop; if the move's transaction yielded to the event loop (between chunks), it could not resume, and the write + would wait out the whole busy_timeout and then fail with "database is locked".""" + files = make_files_structure([f'source/{i}.txt' for i in range(MOVE_CHUNK_SIZE + 50)]) + for file in files: + FileGroup.from_paths(test_session, file) + test_session.commit() + dest = test_directory / 'destination' + dest.mkdir() + + file_worker.private_queue.put_nowait(FileTask(FileTaskType.move, files, destination=dest)) + # Production runs the move in a thread; the fresh connections from `production_like_sessions` allow that here. + monkeypatch.setattr(move_module, 'PYTEST', False) + + move_done = asyncio.Event() + errors = [] + + async def writer(): + # Writes alongside the move, as a download does. + while not move_done.is_set(): + try: + with get_db_session(commit=True): + pass + except Exception as e: + errors.append(e) + return + await asyncio.sleep(0.01) + + with production_like_sessions(test_session) as maker: + writer_task = asyncio.create_task(writer()) + start = time.monotonic() + try: + await file_worker.process_queue() + finally: + move_done.set() + await writer_task + elapsed = time.monotonic() - start + + assert not errors, errors + assert elapsed < 15, f'the move and the writer took {elapsed:.0f}s; the event loop waited on the move\'s lock' + assert sorted(i.name for i in dest.iterdir()) == sorted(i.name for i in files) + session = maker() + assert {fg.directory for fg in session.query(FileGroup)} == {dest} + + +@pytest.mark.asyncio +async def test_build_move_plan_does_not_scan_every_file_group(async_client, test_session, test_directory, + make_files_structure): + """Planning looks up the moved files by primary_path, rather than reading all of file_group (gigabytes on a large + library, which made planning a 109 file move take over a minute).""" + files = make_files_structure(['source/one.txt', 'source/two.txt']) + for file in files: + FileGroup.from_paths(test_session, file) + for i in range(50): + test_session.add(FileGroup(primary_path=test_directory / f'other/{i}.txt', directory=test_directory / 'other')) + test_session.commit() + # A real library has statistics (PRAGMA optimize), with which SQLite chose to scan file_group for a plain JOIN. + test_session.execute(text('ANALYZE')) + + plans = [] + + def explain_lookup(conn, cursor, statement, parameters, context, executemany): + if 'SELECT fg.id' in statement and 'move_sources' in statement: + plans.extend(row[-1] for row in cursor.connection.execute(f'EXPLAIN QUERY PLAN {statement}')) + + engine = test_session.get_bind() + event.listen(engine, 'after_cursor_execute', explain_lookup) + try: + plan, _ = await build_move_plan_bulk(files, test_directory / 'destination') + finally: + event.remove(engine, 'after_cursor_execute', explain_lookup) + + assert len(plan) == 2 + assert plans, 'the FileGroup lookup was not run' + assert not [i for i in plans if i.startswith('SCAN fg')], plans + + +def _cancel_like_shutdown(mover, revert_plan, thread_finished): + """Run a move in a fresh event loop, and stop it the way the perpetual process shuts down: `run_loops` cancels + the file worker loop, stops waiting for it, and `asyncio.run` then cancels every pending task again. + + Returns what the move's caller (`handle_move`) would see when the CancelledError reaches it.""" + seen = dict() + + async def handle_move(): + try: + await mover._execute_move_plan_off_loop({}, set(), revert_plan) + except asyncio.CancelledError: + seen['thread_finished'] = thread_finished.is_set() + seen['revert_plan'] = dict(revert_plan) + raise + + async def serve(): + task = asyncio.create_task(handle_move()) + await asyncio.sleep(0.05) + task.cancel() + await asyncio.wait([task], timeout=0.05) + + asyncio.run(serve()) + return seen + + +def test_move_thread_is_not_reverted_after_it_commits(monkeypatch): + """However often shutdown cancels a move, the caller sees the cancellation only after the move's thread has left + its transaction; when it committed, there is nothing to revert.""" + monkeypatch.setattr(move_module, 'PYTEST', False) + mover = move_module.FileMoveMixin() + thread_finished = threading.Event() + + def commit_slowly(plan, created_directories, revert_plan): + time.sleep(0.5) + thread_finished.set() + + mover._execute_move_plan = commit_slowly + seen = _cancel_like_shutdown(mover, {'new.txt': 'old.txt'}, thread_finished) + assert seen == dict(thread_finished=True, revert_plan={}) + + +def test_move_thread_that_rolls_back_is_reverted_after_it_finishes(monkeypatch): + """A cancelled move whose thread rolled back keeps its revert plan, so the caller moves the files back, but only + after the thread is done moving them.""" + monkeypatch.setattr(move_module, 'PYTEST', False) + mover = move_module.FileMoveMixin() + thread_finished = threading.Event() + + def fail_slowly(plan, created_directories, revert_plan): + time.sleep(0.5) + thread_finished.set() + raise RuntimeError('the transaction rolled back') + + mover._execute_move_plan = fail_slowly + seen = _cancel_like_shutdown(mover, {'new.txt': 'old.txt'}, thread_finished) + assert seen == dict(thread_finished=True, revert_plan={'new.txt': 'old.txt'})