Skipping post-processing until the end when files are being uploaded, processing each time is very slow.

This commit is contained in:
Roland 2026-05-05 14:09:24 -06:00
parent ba63170e55
commit 5eefc2098f
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
4 changed files with 42 additions and 6 deletions

View file

@ -472,7 +472,8 @@ async def post_upload(request: Request):
if chunk_num == 0:
try:
# Delete any conflicting FileGroups if the user is overwriting.
await lib.delete(output)
# Skip post-processing, that will be done after upload.
await lib.delete(output, post_processing=False)
except InvalidFile:
# No conflicting files, good.
pass

View file

@ -317,12 +317,16 @@ def delete_file_groups(session: Session, file_group_ids: List[int], force: bool
@wrol_mode_check
async def delete(*paths: Union[str, pathlib.Path], force: bool = False):
async def delete(*paths: Union[str, pathlib.Path], force: bool = False, post_processing: bool = True):
"""
Delete a file or directory in the media directory.
If any files are tagged and force is False, returns a list of tagged FileGroups without deleting.
If force is True, deletes even tagged files."""
If force is True, deletes even tagged files.
If post_processing is False, the synchronous DB cleanup still runs but global modelers/indexers
are skipped — use this when the caller will trigger modeling itself (e.g. the upload endpoint
follows up with `upsert_file`)."""
media_directory = get_media_directory()
paths = [media_directory / i for i in paths]
if any(i == media_directory for i in paths):
@ -360,7 +364,7 @@ async def delete(*paths: Union[str, pathlib.Path], force: bool = False):
# This ensures deleted FileGroups are removed before the caller continues
# (e.g., upload API recreating the file).
from wrolpi.files.worker import file_worker
await file_worker.refresh_sync(list(paths))
await file_worker.refresh_sync(list(paths), post_processing=post_processing)
if had_tagged:
# `tags.yaml` references TagFiles that the refresh just removed; rewrite it.

View file

@ -1495,3 +1495,29 @@ async def test_move_planning_tracks_progress(async_client, test_session, test_di
# Check that at least one update has a non-zero total
has_total = any(u.get('operation_total', 0) > 0 for u in planning_statuses)
assert has_total
@pytest.mark.asyncio
async def test_refresh_sync_skips_post_processing_when_unchanged(async_client, test_session, test_directory):
"""`refresh_sync` must not invoke global modelers/indexers when nothing changed.
Regression: every chunk-0 of an upload calls `lib.delete(target)` on a path that does not
yet exist; that path used to trigger `_apply_post_processing`, which runs the entire backlog
of modelers + indexers and was timing out the upload request (Sanic 503 Response Timeout)."""
missing_path = test_directory / 'does-not-exist.txt'
called = False
original = file_worker._apply_post_processing
async def tracking(*args, **kwargs):
nonlocal called
called = True
return await original(*args, **kwargs)
file_worker._apply_post_processing = tracking
try:
await file_worker.refresh_sync([missing_path])
finally:
file_worker._apply_post_processing = original
assert called is False, '_apply_post_processing should be skipped when no files changed'

View file

@ -975,7 +975,7 @@ class FileWorker:
self.public_queue.put_nowait(task)
return job_id
async def refresh_sync(self, paths: list[pathlib.Path]):
async def refresh_sync(self, paths: list[pathlib.Path], post_processing: bool = True):
"""Synchronously refresh specific files. Use sparingly - prefer queue_refresh."""
if not paths:
return
@ -983,7 +983,12 @@ class FileWorker:
self._cleanup_modified_models(result.modified)
await self._upsert_file_groups(result.new + result.modified)
await self._delete_file_groups(result.deleted)
await self._apply_post_processing(is_global_refresh=False)
if post_processing and (result.new or result.modified or result.deleted):
# Skip global modelers/indexers when the caller opts out (e.g. `lib.delete` from the
# upload path defers that to the post-upload `upsert_file`) or when nothing changed.
# Without this guard a synchronous caller would block on the entire backlog of
# unindexed files and could exceed Sanic's RESPONSE_TIMEOUT.
await self._apply_post_processing(is_global_refresh=False)
def transfer_queue(self):
"""Transfer items from the public queue to the private queue."""