From 5eefc2098fc82d27026c4a3beae46eb33bba88af Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 5 May 2026 14:09:24 -0600 Subject: [PATCH] Skipping post-processing until the end when files are being uploaded, processing each time is very slow. --- wrolpi/files/api.py | 3 ++- wrolpi/files/lib.py | 10 +++++++--- wrolpi/files/test/test_worker.py | 26 ++++++++++++++++++++++++++ wrolpi/files/worker.py | 9 +++++++-- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index e70c8592..29bde213 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -472,7 +472,8 @@ async def post_upload(request: Request): if chunk_num == 0: try: # Delete any conflicting FileGroups if the user is overwriting. - await lib.delete(output) + # Skip post-processing, that will be done after upload. + await lib.delete(output, post_processing=False) except InvalidFile: # No conflicting files, good. pass diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index afad6c99..bacba670 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -317,12 +317,16 @@ def delete_file_groups(session: Session, file_group_ids: List[int], force: bool @wrol_mode_check -async def delete(*paths: Union[str, pathlib.Path], force: bool = False): +async def delete(*paths: Union[str, pathlib.Path], force: bool = False, post_processing: bool = True): """ Delete a file or directory in the media directory. If any files are tagged and force is False, returns a list of tagged FileGroups without deleting. - If force is True, deletes even tagged files.""" + If force is True, deletes even tagged files. + + If post_processing is False, the synchronous DB cleanup still runs but global modelers/indexers + are skipped — use this when the caller will trigger modeling itself (e.g. the upload endpoint + follows up with `upsert_file`).""" media_directory = get_media_directory() paths = [media_directory / i for i in paths] if any(i == media_directory for i in paths): @@ -360,7 +364,7 @@ async def delete(*paths: Union[str, pathlib.Path], force: bool = False): # This ensures deleted FileGroups are removed before the caller continues # (e.g., upload API recreating the file). from wrolpi.files.worker import file_worker - await file_worker.refresh_sync(list(paths)) + await file_worker.refresh_sync(list(paths), post_processing=post_processing) if had_tagged: # `tags.yaml` references TagFiles that the refresh just removed; rewrite it. diff --git a/wrolpi/files/test/test_worker.py b/wrolpi/files/test/test_worker.py index 3ec1f776..66aaf542 100644 --- a/wrolpi/files/test/test_worker.py +++ b/wrolpi/files/test/test_worker.py @@ -1495,3 +1495,29 @@ async def test_move_planning_tracks_progress(async_client, test_session, test_di # Check that at least one update has a non-zero total has_total = any(u.get('operation_total', 0) > 0 for u in planning_statuses) assert has_total + + +@pytest.mark.asyncio +async def test_refresh_sync_skips_post_processing_when_unchanged(async_client, test_session, test_directory): + """`refresh_sync` must not invoke global modelers/indexers when nothing changed. + + Regression: every chunk-0 of an upload calls `lib.delete(target)` on a path that does not + yet exist; that path used to trigger `_apply_post_processing`, which runs the entire backlog + of modelers + indexers and was timing out the upload request (Sanic 503 Response Timeout).""" + missing_path = test_directory / 'does-not-exist.txt' + + called = False + original = file_worker._apply_post_processing + + async def tracking(*args, **kwargs): + nonlocal called + called = True + return await original(*args, **kwargs) + + file_worker._apply_post_processing = tracking + try: + await file_worker.refresh_sync([missing_path]) + finally: + file_worker._apply_post_processing = original + + assert called is False, '_apply_post_processing should be skipped when no files changed' diff --git a/wrolpi/files/worker.py b/wrolpi/files/worker.py index 4642dda2..8ceccb48 100644 --- a/wrolpi/files/worker.py +++ b/wrolpi/files/worker.py @@ -975,7 +975,7 @@ class FileWorker: self.public_queue.put_nowait(task) return job_id - async def refresh_sync(self, paths: list[pathlib.Path]): + async def refresh_sync(self, paths: list[pathlib.Path], post_processing: bool = True): """Synchronously refresh specific files. Use sparingly - prefer queue_refresh.""" if not paths: return @@ -983,7 +983,12 @@ class FileWorker: self._cleanup_modified_models(result.modified) await self._upsert_file_groups(result.new + result.modified) await self._delete_file_groups(result.deleted) - await self._apply_post_processing(is_global_refresh=False) + if post_processing and (result.new or result.modified or result.deleted): + # Skip global modelers/indexers when the caller opts out (e.g. `lib.delete` from the + # upload path defers that to the post-upload `upsert_file`) or when nothing changed. + # Without this guard a synchronous caller would block on the entire backlog of + # unindexed files and could exceed Sanic's RESPONSE_TIMEOUT. + await self._apply_post_processing(is_global_refresh=False) def transfer_queue(self): """Transfer items from the public queue to the private queue."""