A move neither scans every FileGroup nor stalls the downloader's writes (#951)

* A move neither scans every FileGroup nor stalls the downloader's writes

Two stalls made a 109 file move take three minutes on a large library:

- Planning joined the move_sources temp table to file_group.  The temp
  table has no statistics, so with a real library's statistics SQLite
  scanned all of file_group (233k rows, 2.4 GB) instead of looking up 109
  paths in the primary_path index.  CROSS JOIN fixes the join order.

- The move's write transaction yielded to the event loop between chunks.
  The process that moves files also downloads; a download's write then
  waited on the move's lock synchronously, so the event loop could not
  resume the move: each write waited out the 30 s busy_timeout and failed
  ("Unable to renew downloads!").  The transaction now runs in a thread
  (inline in tests, which share one connection), so another writer waits
  only until the move commits.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* A cancelled move waits for its thread however often it is cancelled

Review fix: the move waited for its thread on the first cancellation by
awaiting the thread's future again, unshielded.  Shutdown cancels twice
(run_loops cancels the file worker loop and stops waiting after 15 s, then
asyncio.run cancels every pending task, including the thread's future), so
the second cancellation raised while the thread was still in its
transaction, and handle_move reverted the files the thread then committed.

The move now waits on a threading.Event the thread sets when it leaves its
transaction, absorbing every cancellation meanwhile.  A cancelled future
reads "done" while its thread still runs, so it cannot be that signal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-11-07 14:47:03 -07:00 • committed by Roland
parent aff5f1eb1f
commit 6c24d7d762
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
2 changed files with 227 additions and 14 deletions

View file

@ -10,6 +10,7 @@ import os
import pathlib
import shutil
import stat as stat_module
import threading
from collections import OrderedDict
from pathlib import Path
from typing import Callable, Dict, List, Set, Tuple
@ -26,9 +27,13 @@ from wrolpi.files.lib import (
_move_file_group_files, _bulk_update_file_groups_db, MOVE_CHUNK_SIZE,
get_normalized_ignored_directories, remove_files_in_ignored_directories,
)
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
# How often a move waits for its thread to leave the transaction.
MOVE_THREAD_POLL_SECONDS = 0.05
async def build_move_plan_bulk(
sources: list[pathlib.Path],
@ -163,11 +168,13 @@ async def build_move_plan_bulk(
[{"path": str(p), "type": 'deleted', "dir": str(p.parent)} for p in deleted_sources],
)
# Single bulk query to get all FileGroups matching our sources
# Single bulk query to get all FileGroups matching our sources. CROSS JOIN makes SQLite read
# move_sources first and look each path up in file_group's primary_path index; the temp table has no
# statistics, so a plain JOIN scans all of file_group (gigabytes on a large library).
result = session.execute(text("""
SELECT fg.id, fg.primary_path, fg.directory, ms.source_path, ms.source_type
FROM file_group fg
JOIN move_sources ms ON fg.primary_path = ms.source_path
FROM move_sources ms
CROSS JOIN file_group fg ON fg.primary_path = ms.source_path
"""))
# Build lookup of found FileGroups
@ -293,7 +300,74 @@ class FileMoveMixin:
return False
return True
async def _execute_move_chunks(
def _execute_move_plan(
self,
plan: Dict[pathlib.Path, pathlib.Path],
created_directories: Set[pathlib.Path],
revert_plan: Dict[pathlib.Path, pathlib.Path],
) -> None:
"""Move the files and update the DB in one transaction, which commits when this returns.
This reads FileGroups/Directories then updates them, so it must begin as a writer; a deferred
transaction's lock upgrade fails instantly when anything else is writing (see `build_move_plan_bulk`)."""
with flags.file_worker_discovery:
with get_db_session(commit=True) as session:
self._execute_move_chunks(plan, session, created_directories, revert_plan)
async def _execute_move_plan_off_loop(
self,
plan: Dict[pathlib.Path, pathlib.Path],
created_directories: Set[pathlib.Path],
revert_plan: Dict[pathlib.Path, pathlib.Path],
) -> None:
"""Run `_execute_move_plan` in a thread, so the event loop never waits while the move holds the write lock.
The process that moves files also downloads. If the move's transaction yielded to the event loop, a
download's write would wait out the whole busy_timeout synchronously -- the event loop cannot resume the
move that holds the lock -- and then fail. In a thread, the move keeps going and the other writer gets the
lock when it commits.
Tests share one connection, which is not safe to use from two threads, so tests run it inline."""
if PYTEST:
self._execute_move_plan(plan, created_directories, revert_plan)
return
# A thread cannot be interrupted, and the caller reverts `revert_plan` when this raises. So this returns or
# raises only after the thread has left its transaction, however often this task is cancelled meanwhile.
# (Shutdown cancels it more than once: the perpetual loops are cancelled, then asyncio.run cancels every
# pending task, including any asyncio future of the thread, which then reads "done" while the thread runs.)
# The thread's own Event is the only signal that its transaction is over.
finished = threading.Event()
outcome = dict()
def run():
try:
self._execute_move_plan(plan, created_directories, revert_plan)
except BaseException as e:
outcome['error'] = e
finally:
finished.set()
asyncio.get_running_loop().run_in_executor(None, run)
cancelled = False
while not finished.is_set():
try:
await asyncio.sleep(MOVE_THREAD_POLL_SECONDS)
except asyncio.CancelledError:
cancelled = True
asyncio.current_task().uncancel()
if error := outcome.get('error'):
# It rolled back; the caller reverts the files it moved.
if cancelled:
raise asyncio.CancelledError() from error
raise error
if cancelled:
# It committed; the files and the DB agree, so there is nothing to revert.
revert_plan.clear()
raise asyncio.CancelledError()
def _execute_move_chunks(
self,
plan: Dict[pathlib.Path, pathlib.Path],
session,
@ -378,7 +452,6 @@ class FileMoveMixin:
operation_processed=processed,
operation_percent=percent,
)
await asyncio.sleep(0) # Yield for cancellation
return new_directories
@ -588,14 +661,7 @@ class FileMoveMixin:
operation_total=len(plan),
)
# Execute the plan in chunks. This reads FileGroups/Directories then updates them, so
# it must begin as a writer; a deferred transaction's lock upgrade fails instantly when
# anything else is writing (see `build_move_plan_bulk`).
with flags.file_worker_discovery:
with get_db_session(commit=True) as session:
await self._execute_move_chunks(
plan, session, created_directories, revert_plan
)
await self._execute_move_plan_off_loop(plan, created_directories, revert_plan)
# The move committed; the files and the DB agree, so a later failure must not move the files back.
revert_plan.clear()

View file

@ -1,11 +1,17 @@
"""Tests for the FileWorker move task."""
import asyncio
import contextlib
import threading
import time
import pytest
from sqlalchemy import event, text
from wrolpi.conftest import production_like_sessions, write_lock_held_briefly
from wrolpi.db import get_db_session
from wrolpi.files import move as move_module
from wrolpi.files import worker as worker_module
from wrolpi.files.lib import _move_file_group_files
from wrolpi.files.lib import _move_file_group_files, MOVE_CHUNK_SIZE
from wrolpi.files.models import FileGroup, Directory
from wrolpi.files.worker import file_worker, FileTask, FileTaskType, FileWorkerJobFailed, build_move_plan_bulk
@ -719,3 +725,144 @@ async def test_file_worker_move_survives_concurrent_writer(
session = maker()
fg = session.query(FileGroup).one()
assert fg.primary_path == dest / 'file1.txt', 'FileGroup was not updated to the new path'
@pytest.mark.asyncio
async def test_file_worker_move_does_not_stall_writers_on_the_event_loop(
async_client, test_session, test_directory, make_files_structure, monkeypatch
):
"""A move's write transaction must not need the event loop to finish.
The process that moves files also downloads. A download's write that waits on the move's lock blocks the event
loop; if the move's transaction yielded to the event loop (between chunks), it could not resume, and the write
would wait out the whole busy_timeout and then fail with "database is locked"."""
files = make_files_structure([f'source/{i}.txt' for i in range(MOVE_CHUNK_SIZE + 50)])
for file in files:
FileGroup.from_paths(test_session, file)
test_session.commit()
dest = test_directory / 'destination'
dest.mkdir()
file_worker.private_queue.put_nowait(FileTask(FileTaskType.move, files, destination=dest))
# Production runs the move in a thread; the fresh connections from `production_like_sessions` allow that here.
monkeypatch.setattr(move_module, 'PYTEST', False)
move_done = asyncio.Event()
errors = []
async def writer():
# Writes alongside the move, as a download does.
while not move_done.is_set():
try:
with get_db_session(commit=True):
pass
except Exception as e:
errors.append(e)
return
await asyncio.sleep(0.01)
with production_like_sessions(test_session) as maker:
writer_task = asyncio.create_task(writer())
start = time.monotonic()
try:
await file_worker.process_queue()
finally:
move_done.set()
await writer_task
elapsed = time.monotonic() - start
assert not errors, errors
assert elapsed < 15, f'the move and the writer took {elapsed:.0f}s; the event loop waited on the move\'s lock'
assert sorted(i.name for i in dest.iterdir()) == sorted(i.name for i in files)
session = maker()
assert {fg.directory for fg in session.query(FileGroup)} == {dest}
@pytest.mark.asyncio
async def test_build_move_plan_does_not_scan_every_file_group(async_client, test_session, test_directory,
make_files_structure):
"""Planning looks up the moved files by primary_path, rather than reading all of file_group (gigabytes on a large
library, which made planning a 109 file move take over a minute)."""
files = make_files_structure(['source/one.txt', 'source/two.txt'])
for file in files:
FileGroup.from_paths(test_session, file)
for i in range(50):
test_session.add(FileGroup(primary_path=test_directory / f'other/{i}.txt', directory=test_directory / 'other'))
test_session.commit()
# A real library has statistics (PRAGMA optimize), with which SQLite chose to scan file_group for a plain JOIN.
test_session.execute(text('ANALYZE'))
plans = []
def explain_lookup(conn, cursor, statement, parameters, context, executemany):
if 'SELECT fg.id' in statement and 'move_sources' in statement:
plans.extend(row[-1] for row in cursor.connection.execute(f'EXPLAIN QUERY PLAN {statement}'))
engine = test_session.get_bind()
event.listen(engine, 'after_cursor_execute', explain_lookup)
try:
plan, _ = await build_move_plan_bulk(files, test_directory / 'destination')
finally:
event.remove(engine, 'after_cursor_execute', explain_lookup)
assert len(plan) == 2
assert plans, 'the FileGroup lookup was not run'
assert not [i for i in plans if i.startswith('SCAN fg')], plans
def _cancel_like_shutdown(mover, revert_plan, thread_finished):
"""Run a move in a fresh event loop, and stop it the way the perpetual process shuts down: `run_loops` cancels
the file worker loop, stops waiting for it, and `asyncio.run` then cancels every pending task again.
Returns what the move's caller (`handle_move`) would see when the CancelledError reaches it."""
seen = dict()
async def handle_move():
try:
await mover._execute_move_plan_off_loop({}, set(), revert_plan)
except asyncio.CancelledError:
seen['thread_finished'] = thread_finished.is_set()
seen['revert_plan'] = dict(revert_plan)
raise
async def serve():
task = asyncio.create_task(handle_move())
await asyncio.sleep(0.05)
task.cancel()
await asyncio.wait([task], timeout=0.05)
asyncio.run(serve())
return seen
def test_move_thread_is_not_reverted_after_it_commits(monkeypatch):
"""However often shutdown cancels a move, the caller sees the cancellation only after the move's thread has left
its transaction; when it committed, there is nothing to revert."""
monkeypatch.setattr(move_module, 'PYTEST', False)
mover = move_module.FileMoveMixin()
thread_finished = threading.Event()
def commit_slowly(plan, created_directories, revert_plan):
time.sleep(0.5)
thread_finished.set()
mover._execute_move_plan = commit_slowly
seen = _cancel_like_shutdown(mover, {'new.txt': 'old.txt'}, thread_finished)
assert seen == dict(thread_finished=True, revert_plan={})
def test_move_thread_that_rolls_back_is_reverted_after_it_finishes(monkeypatch):
"""A cancelled move whose thread rolled back keeps its revert plan, so the caller moves the files back, but only
after the thread is done moving them."""
monkeypatch.setattr(move_module, 'PYTEST', False)
mover = move_module.FileMoveMixin()
thread_finished = threading.Event()
def fail_slowly(plan, created_directories, revert_plan):
time.sleep(0.5)
thread_finished.set()
raise RuntimeError('the transaction rolled back')
mover._execute_move_plan = fail_slowly
seen = _cancel_like_shutdown(mover, {'new.txt': 'old.txt'}, thread_finished)
assert seen == dict(thread_finished=True, revert_plan={'new.txt': 'old.txt'})