mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Skipping post-processing until the end when files are being uploaded, processing each time is very slow.
This commit is contained in:
parent
ba63170e55
commit
5eefc2098f
4 changed files with 42 additions and 6 deletions
|
|
@ -472,7 +472,8 @@ async def post_upload(request: Request):
|
|||
if chunk_num == 0:
|
||||
try:
|
||||
# Delete any conflicting FileGroups if the user is overwriting.
|
||||
await lib.delete(output)
|
||||
# Skip post-processing, that will be done after upload.
|
||||
await lib.delete(output, post_processing=False)
|
||||
except InvalidFile:
|
||||
# No conflicting files, good.
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -317,12 +317,16 @@ def delete_file_groups(session: Session, file_group_ids: List[int], force: bool
|
|||
|
||||
|
||||
@wrol_mode_check
|
||||
async def delete(*paths: Union[str, pathlib.Path], force: bool = False):
|
||||
async def delete(*paths: Union[str, pathlib.Path], force: bool = False, post_processing: bool = True):
|
||||
"""
|
||||
Delete a file or directory in the media directory.
|
||||
|
||||
If any files are tagged and force is False, returns a list of tagged FileGroups without deleting.
|
||||
If force is True, deletes even tagged files."""
|
||||
If force is True, deletes even tagged files.
|
||||
|
||||
If post_processing is False, the synchronous DB cleanup still runs but global modelers/indexers
|
||||
are skipped — use this when the caller will trigger modeling itself (e.g. the upload endpoint
|
||||
follows up with `upsert_file`)."""
|
||||
media_directory = get_media_directory()
|
||||
paths = [media_directory / i for i in paths]
|
||||
if any(i == media_directory for i in paths):
|
||||
|
|
@ -360,7 +364,7 @@ async def delete(*paths: Union[str, pathlib.Path], force: bool = False):
|
|||
# This ensures deleted FileGroups are removed before the caller continues
|
||||
# (e.g., upload API recreating the file).
|
||||
from wrolpi.files.worker import file_worker
|
||||
await file_worker.refresh_sync(list(paths))
|
||||
await file_worker.refresh_sync(list(paths), post_processing=post_processing)
|
||||
|
||||
if had_tagged:
|
||||
# `tags.yaml` references TagFiles that the refresh just removed; rewrite it.
|
||||
|
|
|
|||
|
|
@ -1495,3 +1495,29 @@ async def test_move_planning_tracks_progress(async_client, test_session, test_di
|
|||
# Check that at least one update has a non-zero total
|
||||
has_total = any(u.get('operation_total', 0) > 0 for u in planning_statuses)
|
||||
assert has_total
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_sync_skips_post_processing_when_unchanged(async_client, test_session, test_directory):
|
||||
"""`refresh_sync` must not invoke global modelers/indexers when nothing changed.
|
||||
|
||||
Regression: every chunk-0 of an upload calls `lib.delete(target)` on a path that does not
|
||||
yet exist; that path used to trigger `_apply_post_processing`, which runs the entire backlog
|
||||
of modelers + indexers and was timing out the upload request (Sanic 503 Response Timeout)."""
|
||||
missing_path = test_directory / 'does-not-exist.txt'
|
||||
|
||||
called = False
|
||||
original = file_worker._apply_post_processing
|
||||
|
||||
async def tracking(*args, **kwargs):
|
||||
nonlocal called
|
||||
called = True
|
||||
return await original(*args, **kwargs)
|
||||
|
||||
file_worker._apply_post_processing = tracking
|
||||
try:
|
||||
await file_worker.refresh_sync([missing_path])
|
||||
finally:
|
||||
file_worker._apply_post_processing = original
|
||||
|
||||
assert called is False, '_apply_post_processing should be skipped when no files changed'
|
||||
|
|
|
|||
|
|
@ -975,7 +975,7 @@ class FileWorker:
|
|||
self.public_queue.put_nowait(task)
|
||||
return job_id
|
||||
|
||||
async def refresh_sync(self, paths: list[pathlib.Path]):
|
||||
async def refresh_sync(self, paths: list[pathlib.Path], post_processing: bool = True):
|
||||
"""Synchronously refresh specific files. Use sparingly - prefer queue_refresh."""
|
||||
if not paths:
|
||||
return
|
||||
|
|
@ -983,7 +983,12 @@ class FileWorker:
|
|||
self._cleanup_modified_models(result.modified)
|
||||
await self._upsert_file_groups(result.new + result.modified)
|
||||
await self._delete_file_groups(result.deleted)
|
||||
await self._apply_post_processing(is_global_refresh=False)
|
||||
if post_processing and (result.new or result.modified or result.deleted):
|
||||
# Skip global modelers/indexers when the caller opts out (e.g. `lib.delete` from the
|
||||
# upload path defers that to the post-upload `upsert_file`) or when nothing changed.
|
||||
# Without this guard a synchronous caller would block on the entire backlog of
|
||||
# unindexed files and could exceed Sanic's RESPONSE_TIMEOUT.
|
||||
await self._apply_post_processing(is_global_refresh=False)
|
||||
|
||||
def transfer_queue(self):
|
||||
"""Transfer items from the public queue to the private queue."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue