mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Fix Downloads stuck "pending": persist finalize_download mutations (#473)
* Persist finalize_download mutations; isolate child-download enqueue A channel/RSS Download could get stuck permanently "pending". Root cause: the phase-split dispatch in signal_download_download passes the Download loaded in an already-closed session to finalize_download, so it is DETACHED. ChannelDownloader.finalize_download sets `download.sub_downloader = 'video'` (and info_json/collection_id) on that detached object, so the write never reaches the DB. The status block then re-queries the Download fresh, reads sub_downloader=None, and calls create_downloads(downloader_name=None) -> `InvalidDownload: Cannot find downloader with name None`. That exception was raised inside the status-commit transaction (outside the inner try), so it (a) rolled back the parent's complete(), reverting it to the `pending` set by started(), and (b) was swallowed by the outer bare `except Exception`, which only logs. Result: the parent never failed, never completed — it sat `pending` forever, re-crashing the same way on every restart. Fixes: - finalize_download is now given a Download re-loaded in (attached to) its own session, so its mutations persist. - Child-download creation moves to its own transaction after the parent's status is committed. A failure there is logged and cannot roll back or strand the parent. The existing test harness reuses one shared DB session for every get_db_session call, so detached-object and rollback bugs never reproduced under it (which is why this shipped). Add `production_like_sessions` to conftest.py — a context manager handing out a fresh Session per call, like production — and two regression tests that fail without the fix. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * Re-attach Download in prepare_download too (consistency) prepare_download receives the same detached Download as finalize_download did, so VideoDownloader.prepare_download's writes to the row (destination/tag_names) are not reliably persisted. Re-load the Download attached to the prepare session, mirroring the finalize fix, so the row's metadata is accurate. Unlike sub_downloader (which the status block re-reads and feeds to create_downloads), nothing downstream re-reads these values, so this is a consistency/cosmetic improvement, not a functional bug — and it cannot be made to fail in a test (the loaded Download's session outlives the worker), so no dedicated test is added. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * Log (not swallow) session-close failures in production_like_sessions Addresses Greptile review: surface a failed session.close() at debug level instead of silently passing. engine.dispose() still does the authoritative cleanup, so the close failure remains non-fatal. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
57d8030f09
commit
4fe58d8675
3 changed files with 156 additions and 11 deletions
|
|
@ -2,6 +2,7 @@
|
|||
Fixtures for Pytest tests.
|
||||
"""
|
||||
import asyncio
|
||||
import contextlib
|
||||
import copy
|
||||
import http.server
|
||||
import json
|
||||
|
|
@ -30,6 +31,7 @@ from PIL import Image
|
|||
from sanic_testing.testing import SanicASGITestClient
|
||||
from sqlalchemy.engine import Engine, create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
# Import root api so blueprints are attached.
|
||||
import wrolpi.root_api # noqa
|
||||
|
|
@ -402,6 +404,42 @@ def make_test_ctx(*, is_cancelled=None, can_download=None,
|
|||
)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def production_like_sessions(test_session) -> Generator[sessionmaker, Any, None]:
|
||||
"""Patch get_db_session to hand out a *fresh* Session per call, like production.
|
||||
|
||||
The `test_session` fixture reuses one shared Session for every `get_db_session` call, so an object loaded in
|
||||
one `with` block stays attached for the next and exceptions never roll back. That convenience masks
|
||||
detached-object and transaction-rollback bugs that only bite production's per-call sessions. Use this to
|
||||
exercise that behavior: the yielded sessionmaker creates independent Sessions so a test can verify committed
|
||||
DB state the way a separate worker would see it.
|
||||
|
||||
A dedicated NullPool engine is used and disposed on exit so its connections are released before the test DB
|
||||
is dropped during fixture teardown.
|
||||
"""
|
||||
engine = create_engine(test_session.get_bind().url, poolclass=NullPool)
|
||||
maker = sessionmaker(bind=engine)
|
||||
sessions = []
|
||||
|
||||
def factory():
|
||||
session = maker()
|
||||
sessions.append(session)
|
||||
return engine, session
|
||||
|
||||
try:
|
||||
with mock.patch('wrolpi.db._get_db_session', factory):
|
||||
yield maker
|
||||
finally:
|
||||
for session in sessions:
|
||||
try:
|
||||
session.close()
|
||||
except Exception as e:
|
||||
# Best-effort: engine.dispose() below releases the connections regardless, so a failed
|
||||
# close() is inconsequential. Log at debug so it surfaces under -vvv without noising CI.
|
||||
logger.debug(f'production_like_sessions: failed to close a session: {e}')
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_downloader(test_download_manager) -> Downloader:
|
||||
class TestDownloader(Downloader, ABC):
|
||||
|
|
|
|||
|
|
@ -1563,8 +1563,12 @@ async def signal_download_download(download_id: int, download_url: str):
|
|||
# ExecutedX dataclass; subclasses also use that protocol to skip finalize
|
||||
# for early-exit cases (e.g. metadata-only video downloads).
|
||||
ctx = DownloadContext.production(download_manager, download_id)
|
||||
with get_db_session(commit=True) as s:
|
||||
prepared = downloader.prepare_download(s, download)
|
||||
with get_db_session(commit=True) as session:
|
||||
# Re-load the Download attached to this session so prepare_download's mutations
|
||||
# (e.g. VideoDownloader resolving destination/tag_names onto the row) persist; the
|
||||
# `download` loaded above is detached and writes to it would be silently lost.
|
||||
download = Download.find_by_id(session, download_id)
|
||||
prepared = downloader.prepare_download(session, download)
|
||||
# Subclasses with unavoidable async prep work (e.g. VideoDownloader's
|
||||
# extract_info / channel resolution) may declare prepare_download as
|
||||
# `async def`; await the coroutine here.
|
||||
|
|
@ -1577,8 +1581,12 @@ async def signal_download_download(download_id: int, download_url: str):
|
|||
# cancel_wrapper or execute_download short-circuited with a result.
|
||||
result = executed
|
||||
else:
|
||||
with get_db_session(commit=True) as s:
|
||||
result = downloader.finalize_download(s, download, executed)
|
||||
with get_db_session(commit=True) as session:
|
||||
# Re-load the Download attached to this session so finalize_download's mutations
|
||||
# (sub_downloader, info_json, collection_id) are persisted. The `download` loaded above
|
||||
# is detached (its session has closed), so writes to it would be silently lost.
|
||||
download = Download.find_by_id(session, download_id)
|
||||
result = downloader.finalize_download(session, download, executed)
|
||||
# finalize_download may also be async (same rationale as prepare).
|
||||
if asyncio.iscoroutine(result):
|
||||
result = await result
|
||||
|
|
@ -1648,13 +1656,23 @@ async def signal_download_download(download_id: int, download_url: str):
|
|||
else:
|
||||
download.next_download = download_manager.calculate_next_download(session, download)
|
||||
|
||||
urls = download.filter_excluded(result.downloads) if result.downloads else None
|
||||
if urls:
|
||||
worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}')
|
||||
child_settings = dict(result.settings) if result.settings else {}
|
||||
child_settings['parent_download_url'] = download.url
|
||||
download_manager.create_downloads(session, urls, downloader_name=download.sub_downloader,
|
||||
settings=child_settings, reset_attempts=True)
|
||||
# Enqueue any child downloads in a separate transaction. The parent's status is already committed,
|
||||
# so a failure here (e.g. an unresolvable sub_downloader) cannot roll it back or leave it silently
|
||||
# "pending" — the error is logged and the parent stays finalized.
|
||||
if result.downloads:
|
||||
try:
|
||||
with get_db_session(commit=True) as session:
|
||||
download = Download.find_by_id(session, download_id)
|
||||
urls = download.filter_excluded(result.downloads)
|
||||
if urls:
|
||||
worker_logger.info(f'Adding {len(urls)} downloads from result of {download.url}')
|
||||
child_settings = dict(result.settings) if result.settings else {}
|
||||
child_settings['parent_download_url'] = download.url
|
||||
download_manager.create_downloads(session, urls,
|
||||
downloader_name=download.sub_downloader,
|
||||
settings=child_settings, reset_attempts=True)
|
||||
except Exception as e:
|
||||
worker_logger.error(f'Failed to enqueue child downloads from {url}', exc_info=e)
|
||||
|
||||
# Remove this domain from the running list.
|
||||
download_manager._delete_processing_domain(download_domain)
|
||||
|
|
|
|||
|
|
@ -151,6 +151,95 @@ async def test_cancel_wrapper_outer_cancellation(test_session, test_download_man
|
|||
assert inner_cancelled.is_set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_download_persists_mutations(test_session, test_download_manager, test_downloader):
|
||||
"""finalize_download's mutations to the Download (e.g. sub_downloader) must persist.
|
||||
|
||||
Reproduces the perpetually-pending bug: the phase-split dispatch passed a *detached* Download to
|
||||
finalize_download, so `download.sub_downloader = ...` was silently lost. The status block then re-read
|
||||
sub_downloader=None and called create_downloads(downloader_name=None) -> InvalidDownload, which was
|
||||
swallowed, rolling back the parent's complete() and leaving it stuck `pending`."""
|
||||
from wrolpi.downloader import Downloader, DownloadResult, signal_download_download
|
||||
from wrolpi.conftest import production_like_sessions
|
||||
|
||||
class FinalizeChannelLikeDownloader(Downloader, ABC):
|
||||
"""Mimics ChannelDownloader: execute returns a non-DownloadResult so finalize runs, and finalize sets
|
||||
sub_downloader and returns child URLs to enqueue."""
|
||||
name = 'finalize_channel_like'
|
||||
|
||||
def prepare_download(self, session, download):
|
||||
return object() # sentinel -> phase-split (finalize) path
|
||||
|
||||
async def execute_download(self, prepared, ctx, download=None):
|
||||
await asyncio.sleep(0)
|
||||
return object() # not a DownloadResult -> finalize_download runs
|
||||
|
||||
def finalize_download(self, session, download, executed):
|
||||
download.sub_downloader = test_downloader.name
|
||||
return DownloadResult(success=True, downloads=['https://example.com/child-video'])
|
||||
|
||||
test_download_manager.register_downloader(FinalizeChannelLikeDownloader())
|
||||
parent = test_download_manager.create_download(test_session, 'https://example.com/channel/videos',
|
||||
'finalize_channel_like')
|
||||
parent_id = parent.id
|
||||
test_session.commit()
|
||||
|
||||
with production_like_sessions(test_session) as maker:
|
||||
await signal_download_download(parent_id, 'https://example.com/channel/videos')
|
||||
verify = maker()
|
||||
try:
|
||||
parent = verify.query(Download).filter_by(id=parent_id).one()
|
||||
assert parent.status == 'complete', f'parent should complete, got {parent.status}'
|
||||
assert parent.sub_downloader == test_downloader.name, 'finalize_download mutation must persist'
|
||||
child = verify.query(Download).filter_by(url='https://example.com/child-video').one()
|
||||
assert child.downloader == test_downloader.name, 'child enqueued with the persisted sub_downloader'
|
||||
finally:
|
||||
verify.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_child_download_failure_does_not_leave_parent_pending(test_session, test_download_manager,
|
||||
test_downloader):
|
||||
"""A failure while enqueuing child downloads must not roll back / strand the successful parent.
|
||||
|
||||
The parent download succeeded (it fetched the listing); enqueuing children is a separate concern. If
|
||||
create_downloads raises (here: an unresolvable sub_downloader), the parent must still be marked complete and
|
||||
the error surfaced, not silently left `pending`."""
|
||||
from wrolpi.downloader import Downloader, DownloadResult, signal_download_download
|
||||
from wrolpi.conftest import production_like_sessions
|
||||
|
||||
class BadChildDownloader(Downloader, ABC):
|
||||
name = 'bad_child'
|
||||
|
||||
def prepare_download(self, session, download):
|
||||
return object()
|
||||
|
||||
async def execute_download(self, prepared, ctx, download=None):
|
||||
await asyncio.sleep(0)
|
||||
return object()
|
||||
|
||||
def finalize_download(self, session, download, executed):
|
||||
# Success, but no sub_downloader set -> create_downloads(downloader_name=None) will raise.
|
||||
return DownloadResult(success=True, downloads=['https://example.com/orphan-child'])
|
||||
|
||||
test_download_manager.register_downloader(BadChildDownloader())
|
||||
parent = test_download_manager.create_download(test_session, 'https://example.com/bad/videos', 'bad_child')
|
||||
parent_id = parent.id
|
||||
test_session.commit()
|
||||
|
||||
with production_like_sessions(test_session) as maker:
|
||||
await signal_download_download(parent_id, 'https://example.com/bad/videos')
|
||||
verify = maker()
|
||||
try:
|
||||
parent = verify.query(Download).filter_by(id=parent_id).one()
|
||||
assert parent.status == 'complete', \
|
||||
f'parent must complete despite child-enqueue failure, got {parent.status}'
|
||||
# The child could not be created, but the parent did not get stuck.
|
||||
assert verify.query(Download).filter_by(url='https://example.com/orphan-child').count() == 0
|
||||
finally:
|
||||
verify.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_old_once_downloads(test_session, test_download_manager, test_downloader):
|
||||
"""Once-downloads over a month old should be deleted."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue