Fix Downloads stuck "pending": persist finalize_download mutations (#473)

* Persist finalize_download mutations; isolate child-download enqueue

A channel/RSS Download could get stuck permanently "pending".  Root cause:
the phase-split dispatch in signal_download_download passes the Download
loaded in an already-closed session to finalize_download, so it is
DETACHED.  ChannelDownloader.finalize_download sets
`download.sub_downloader = 'video'` (and info_json/collection_id) on that
detached object, so the write never reaches the DB.  The status block then
re-queries the Download fresh, reads sub_downloader=None, and calls
create_downloads(downloader_name=None) ->
`InvalidDownload: Cannot find downloader with name None`.

That exception was raised inside the status-commit transaction (outside the
inner try), so it (a) rolled back the parent's complete(), reverting it to
the `pending` set by started(), and (b) was swallowed by the outer bare
`except Exception`, which only logs.  Result: the parent never failed, never
completed — it sat `pending` forever, re-crashing the same way on every
restart.

Fixes:
- finalize_download is now given a Download re-loaded in (attached to) its
  own session, so its mutations persist.
- Child-download creation moves to its own transaction after the parent's
  status is committed.  A failure there is logged and cannot roll back or
  strand the parent.

The existing test harness reuses one shared DB session for every
get_db_session call, so detached-object and rollback bugs never reproduced
under it (which is why this shipped).  Add `production_like_sessions` to
conftest.py — a context manager handing out a fresh Session per call, like
production — and two regression tests that fail without the fix.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* Re-attach Download in prepare_download too (consistency)

prepare_download receives the same detached Download as finalize_download
did, so VideoDownloader.prepare_download's writes to the row
(destination/tag_names) are not reliably persisted.  Re-load the Download
attached to the prepare session, mirroring the finalize fix, so the row's
metadata is accurate.

Unlike sub_downloader (which the status block re-reads and feeds to
create_downloads), nothing downstream re-reads these values, so this is a
consistency/cosmetic improvement, not a functional bug — and it cannot be
made to fail in a test (the loaded Download's session outlives the worker),
so no dedicated test is added.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* Log (not swallow) session-close failures in production_like_sessions

Addresses Greptile review: surface a failed session.close() at debug level
instead of silently passing.  engine.dispose() still does the authoritative
cleanup, so the close failure remains non-fatal.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-06-04 17:18:30 -06:00 • committed by Roland
parent 57d8030f09
commit 4fe58d8675
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
3 changed files with 156 additions and 11 deletions

View file

@ -2,6 +2,7 @@
Fixtures for Pytest tests.
"""
import asyncio
import contextlib
import copy
import http.server
import json
@ -30,6 +31,7 @@ from PIL import Image
from sanic_testing.testing import SanicASGITestClient
from sqlalchemy.engine import Engine, create_engine
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import NullPool
# Import root api so blueprints are attached.
import wrolpi.root_api # noqa
@ -402,6 +404,42 @@ def make_test_ctx(*, is_cancelled=None, can_download=None,
)
@contextlib.contextmanager
def production_like_sessions(test_session) -> Generator[sessionmaker, Any, None]:
"""Patch get_db_session to hand out a *fresh* Session per call, like production.
The `test_session` fixture reuses one shared Session for every `get_db_session` call, so an object loaded in
one `with` block stays attached for the next and exceptions never roll back. That convenience masks
detached-object and transaction-rollback bugs that only bite production's per-call sessions. Use this to
exercise that behavior: the yielded sessionmaker creates independent Sessions so a test can verify committed
DB state the way a separate worker would see it.
A dedicated NullPool engine is used and disposed on exit so its connections are released before the test DB
is dropped during fixture teardown.
"""
engine = create_engine(test_session.get_bind().url, poolclass=NullPool)
maker = sessionmaker(bind=engine)
sessions = []
def factory():
session = maker()
sessions.append(session)
return engine, session
try:
with mock.patch('wrolpi.db._get_db_session', factory):
yield maker
finally:
for session in sessions:
try:
session.close()
except Exception as e:
# Best-effort: engine.dispose() below releases the connections regardless, so a failed
# close() is inconsequential. Log at debug so it surfaces under -vvv without noising CI.
logger.debug(f'production_like_sessions: failed to close a session: {e}')
engine.dispose()
@pytest.fixture
def test_downloader(test_download_manager) -> Downloader:
class TestDownloader(Downloader, ABC):

View file

@ -1563,8 +1563,12 @@ async def signal_download_download(download_id: int, download_url: str):
# ExecutedX dataclass; subclasses also use that protocol to skip finalize
# for early-exit cases (e.g. metadata-only video downloads).
ctx = DownloadContext.production(download_manager, download_id)
with get_db_session(commit=True) as s:
prepared = downloader.prepare_download(s, download)
with get_db_session(commit=True) as session:
# Re-load the Download attached to this session so prepare_download's mutations
# (e.g. VideoDownloader resolving destination/tag_names onto the row) persist; the
# `download` loaded above is detached and writes to it would be silently lost.
download = Download.find_by_id(session, download_id)
prepared = downloader.prepare_download(session, download)
# Subclasses with unavoidable async prep work (e.g. VideoDownloader's
# extract_info / channel resolution) may declare prepare_download as
# `async def`; await the coroutine here.
@ -1577,8 +1581,12 @@ async def signal_download_download(download_id: int, download_url: str):
# cancel_wrapper or execute_download short-circuited with a result.
result = executed
else:
with get_db_session(commit=True) as s:
result = downloader.finalize_download(s, download, executed)
with get_db_session(commit=True) as session:
# Re-load the Download attached to this session so finalize_download's mutations
# (sub_downloader, info_json, collection_id) are persisted. The `download` loaded above
# is detached (its session has closed), so writes to it would be silently lost.
download = Download.find_by_id(session, download_id)
result = downloader.finalize_download(session, download, executed)
# finalize_download may also be async (same rationale as prepare).
if asyncio.iscoroutine(result):
result = await result
@ -1648,13 +1656,23 @@ async def signal_download_download(download_id: int, download_url: str):
else:
download.next_download = download_manager.calculate_next_download(session, download)
urls = download.filter_excluded(result.downloads) if result.downloads else None
if urls:
worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}')
child_settings = dict(result.settings) if result.settings else {}
child_settings['parent_download_url'] = download.url
download_manager.create_downloads(session, urls, downloader_name=download.sub_downloader,
settings=child_settings, reset_attempts=True)
# Enqueue any child downloads in a separate transaction. The parent's status is already committed,
# so a failure here (e.g. an unresolvable sub_downloader) cannot roll it back or leave it silently
# "pending" — the error is logged and the parent stays finalized.
if result.downloads:
try:
with get_db_session(commit=True) as session:
download = Download.find_by_id(session, download_id)
urls = download.filter_excluded(result.downloads)
if urls:
worker_logger.info(f'Adding {len(urls)} downloads from result of {download.url}')
child_settings = dict(result.settings) if result.settings else {}
child_settings['parent_download_url'] = download.url
download_manager.create_downloads(session, urls,
downloader_name=download.sub_downloader,
settings=child_settings, reset_attempts=True)
except Exception as e:
worker_logger.error(f'Failed to enqueue child downloads from {url}', exc_info=e)
# Remove this domain from the running list.
download_manager._delete_processing_domain(download_domain)

View file

@ -151,6 +151,95 @@ async def test_cancel_wrapper_outer_cancellation(test_session, test_download_man
assert inner_cancelled.is_set()
@pytest.mark.asyncio
async def test_finalize_download_persists_mutations(test_session, test_download_manager, test_downloader):
"""finalize_download's mutations to the Download (e.g. sub_downloader) must persist.
Reproduces the perpetually-pending bug: the phase-split dispatch passed a *detached* Download to
finalize_download, so `download.sub_downloader = ...` was silently lost. The status block then re-read
sub_downloader=None and called create_downloads(downloader_name=None) -> InvalidDownload, which was
swallowed, rolling back the parent's complete() and leaving it stuck `pending`."""
from wrolpi.downloader import Downloader, DownloadResult, signal_download_download
from wrolpi.conftest import production_like_sessions
class FinalizeChannelLikeDownloader(Downloader, ABC):
"""Mimics ChannelDownloader: execute returns a non-DownloadResult so finalize runs, and finalize sets
sub_downloader and returns child URLs to enqueue."""
name = 'finalize_channel_like'
def prepare_download(self, session, download):
return object() # sentinel -> phase-split (finalize) path
async def execute_download(self, prepared, ctx, download=None):
await asyncio.sleep(0)
return object() # not a DownloadResult -> finalize_download runs
def finalize_download(self, session, download, executed):
download.sub_downloader = test_downloader.name
return DownloadResult(success=True, downloads=['https://example.com/child-video'])
test_download_manager.register_downloader(FinalizeChannelLikeDownloader())
parent = test_download_manager.create_download(test_session, 'https://example.com/channel/videos',
'finalize_channel_like')
parent_id = parent.id
test_session.commit()
with production_like_sessions(test_session) as maker:
await signal_download_download(parent_id, 'https://example.com/channel/videos')
verify = maker()
try:
parent = verify.query(Download).filter_by(id=parent_id).one()
assert parent.status == 'complete', f'parent should complete, got {parent.status}'
assert parent.sub_downloader == test_downloader.name, 'finalize_download mutation must persist'
child = verify.query(Download).filter_by(url='https://example.com/child-video').one()
assert child.downloader == test_downloader.name, 'child enqueued with the persisted sub_downloader'
finally:
verify.close()
@pytest.mark.asyncio
async def test_child_download_failure_does_not_leave_parent_pending(test_session, test_download_manager,
test_downloader):
"""A failure while enqueuing child downloads must not roll back / strand the successful parent.
The parent download succeeded (it fetched the listing); enqueuing children is a separate concern. If
create_downloads raises (here: an unresolvable sub_downloader), the parent must still be marked complete and
the error surfaced, not silently left `pending`."""
from wrolpi.downloader import Downloader, DownloadResult, signal_download_download
from wrolpi.conftest import production_like_sessions
class BadChildDownloader(Downloader, ABC):
name = 'bad_child'
def prepare_download(self, session, download):
return object()
async def execute_download(self, prepared, ctx, download=None):
await asyncio.sleep(0)
return object()
def finalize_download(self, session, download, executed):
# Success, but no sub_downloader set -> create_downloads(downloader_name=None) will raise.
return DownloadResult(success=True, downloads=['https://example.com/orphan-child'])
test_download_manager.register_downloader(BadChildDownloader())
parent = test_download_manager.create_download(test_session, 'https://example.com/bad/videos', 'bad_child')
parent_id = parent.id
test_session.commit()
with production_like_sessions(test_session) as maker:
await signal_download_download(parent_id, 'https://example.com/bad/videos')
verify = maker()
try:
parent = verify.query(Download).filter_by(id=parent_id).one()
assert parent.status == 'complete', \
f'parent must complete despite child-enqueue failure, got {parent.status}'
# The child could not be created, but the parent did not get stuck.
assert verify.query(Download).filter_by(url='https://example.com/orphan-child').count() == 0
finally:
verify.close()
@pytest.mark.asyncio
async def test_delete_old_once_downloads(test_session, test_download_manager, test_downloader):
"""Once-downloads over a month old should be deleted."""