diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 33bcc7f1..f5d460ad 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -2,6 +2,7 @@ Fixtures for Pytest tests. """ import asyncio +import contextlib import copy import http.server import json @@ -30,6 +31,7 @@ from PIL import Image from sanic_testing.testing import SanicASGITestClient from sqlalchemy.engine import Engine, create_engine from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import NullPool # Import root api so blueprints are attached. import wrolpi.root_api # noqa @@ -402,6 +404,42 @@ def make_test_ctx(*, is_cancelled=None, can_download=None, ) +@contextlib.contextmanager +def production_like_sessions(test_session) -> Generator[sessionmaker, Any, None]: + """Patch get_db_session to hand out a *fresh* Session per call, like production. + + The `test_session` fixture reuses one shared Session for every `get_db_session` call, so an object loaded in + one `with` block stays attached for the next and exceptions never roll back. That convenience masks + detached-object and transaction-rollback bugs that only bite production's per-call sessions. Use this to + exercise that behavior: the yielded sessionmaker creates independent Sessions so a test can verify committed + DB state the way a separate worker would see it. + + A dedicated NullPool engine is used and disposed on exit so its connections are released before the test DB + is dropped during fixture teardown. + """ + engine = create_engine(test_session.get_bind().url, poolclass=NullPool) + maker = sessionmaker(bind=engine) + sessions = [] + + def factory(): + session = maker() + sessions.append(session) + return engine, session + + try: + with mock.patch('wrolpi.db._get_db_session', factory): + yield maker + finally: + for session in sessions: + try: + session.close() + except Exception as e: + # Best-effort: engine.dispose() below releases the connections regardless, so a failed + # close() is inconsequential. Log at debug so it surfaces under -vvv without noising CI. + logger.debug(f'production_like_sessions: failed to close a session: {e}') + engine.dispose() + + @pytest.fixture def test_downloader(test_download_manager) -> Downloader: class TestDownloader(Downloader, ABC): diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 2d0cd2a2..c68fd577 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1563,8 +1563,12 @@ async def signal_download_download(download_id: int, download_url: str): # ExecutedX dataclass; subclasses also use that protocol to skip finalize # for early-exit cases (e.g. metadata-only video downloads). ctx = DownloadContext.production(download_manager, download_id) - with get_db_session(commit=True) as s: - prepared = downloader.prepare_download(s, download) + with get_db_session(commit=True) as session: + # Re-load the Download attached to this session so prepare_download's mutations + # (e.g. VideoDownloader resolving destination/tag_names onto the row) persist; the + # `download` loaded above is detached and writes to it would be silently lost. + download = Download.find_by_id(session, download_id) + prepared = downloader.prepare_download(session, download) # Subclasses with unavoidable async prep work (e.g. VideoDownloader's # extract_info / channel resolution) may declare prepare_download as # `async def`; await the coroutine here. @@ -1577,8 +1581,12 @@ async def signal_download_download(download_id: int, download_url: str): # cancel_wrapper or execute_download short-circuited with a result. result = executed else: - with get_db_session(commit=True) as s: - result = downloader.finalize_download(s, download, executed) + with get_db_session(commit=True) as session: + # Re-load the Download attached to this session so finalize_download's mutations + # (sub_downloader, info_json, collection_id) are persisted. The `download` loaded above + # is detached (its session has closed), so writes to it would be silently lost. + download = Download.find_by_id(session, download_id) + result = downloader.finalize_download(session, download, executed) # finalize_download may also be async (same rationale as prepare). if asyncio.iscoroutine(result): result = await result @@ -1648,13 +1656,23 @@ async def signal_download_download(download_id: int, download_url: str): else: download.next_download = download_manager.calculate_next_download(session, download) - urls = download.filter_excluded(result.downloads) if result.downloads else None - if urls: - worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}') - child_settings = dict(result.settings) if result.settings else {} - child_settings['parent_download_url'] = download.url - download_manager.create_downloads(session, urls, downloader_name=download.sub_downloader, - settings=child_settings, reset_attempts=True) + # Enqueue any child downloads in a separate transaction. The parent's status is already committed, + # so a failure here (e.g. an unresolvable sub_downloader) cannot roll it back or leave it silently + # "pending" — the error is logged and the parent stays finalized. + if result.downloads: + try: + with get_db_session(commit=True) as session: + download = Download.find_by_id(session, download_id) + urls = download.filter_excluded(result.downloads) + if urls: + worker_logger.info(f'Adding {len(urls)} downloads from result of {download.url}') + child_settings = dict(result.settings) if result.settings else {} + child_settings['parent_download_url'] = download.url + download_manager.create_downloads(session, urls, + downloader_name=download.sub_downloader, + settings=child_settings, reset_attempts=True) + except Exception as e: + worker_logger.error(f'Failed to enqueue child downloads from {url}', exc_info=e) # Remove this domain from the running list. download_manager._delete_processing_domain(download_domain) diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index d4f38ca6..d41803a2 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -151,6 +151,95 @@ async def test_cancel_wrapper_outer_cancellation(test_session, test_download_man assert inner_cancelled.is_set() +@pytest.mark.asyncio +async def test_finalize_download_persists_mutations(test_session, test_download_manager, test_downloader): + """finalize_download's mutations to the Download (e.g. sub_downloader) must persist. + + Reproduces the perpetually-pending bug: the phase-split dispatch passed a *detached* Download to + finalize_download, so `download.sub_downloader = ...` was silently lost. The status block then re-read + sub_downloader=None and called create_downloads(downloader_name=None) -> InvalidDownload, which was + swallowed, rolling back the parent's complete() and leaving it stuck `pending`.""" + from wrolpi.downloader import Downloader, DownloadResult, signal_download_download + from wrolpi.conftest import production_like_sessions + + class FinalizeChannelLikeDownloader(Downloader, ABC): + """Mimics ChannelDownloader: execute returns a non-DownloadResult so finalize runs, and finalize sets + sub_downloader and returns child URLs to enqueue.""" + name = 'finalize_channel_like' + + def prepare_download(self, session, download): + return object() # sentinel -> phase-split (finalize) path + + async def execute_download(self, prepared, ctx, download=None): + await asyncio.sleep(0) + return object() # not a DownloadResult -> finalize_download runs + + def finalize_download(self, session, download, executed): + download.sub_downloader = test_downloader.name + return DownloadResult(success=True, downloads=['https://example.com/child-video']) + + test_download_manager.register_downloader(FinalizeChannelLikeDownloader()) + parent = test_download_manager.create_download(test_session, 'https://example.com/channel/videos', + 'finalize_channel_like') + parent_id = parent.id + test_session.commit() + + with production_like_sessions(test_session) as maker: + await signal_download_download(parent_id, 'https://example.com/channel/videos') + verify = maker() + try: + parent = verify.query(Download).filter_by(id=parent_id).one() + assert parent.status == 'complete', f'parent should complete, got {parent.status}' + assert parent.sub_downloader == test_downloader.name, 'finalize_download mutation must persist' + child = verify.query(Download).filter_by(url='https://example.com/child-video').one() + assert child.downloader == test_downloader.name, 'child enqueued with the persisted sub_downloader' + finally: + verify.close() + + +@pytest.mark.asyncio +async def test_child_download_failure_does_not_leave_parent_pending(test_session, test_download_manager, + test_downloader): + """A failure while enqueuing child downloads must not roll back / strand the successful parent. + + The parent download succeeded (it fetched the listing); enqueuing children is a separate concern. If + create_downloads raises (here: an unresolvable sub_downloader), the parent must still be marked complete and + the error surfaced, not silently left `pending`.""" + from wrolpi.downloader import Downloader, DownloadResult, signal_download_download + from wrolpi.conftest import production_like_sessions + + class BadChildDownloader(Downloader, ABC): + name = 'bad_child' + + def prepare_download(self, session, download): + return object() + + async def execute_download(self, prepared, ctx, download=None): + await asyncio.sleep(0) + return object() + + def finalize_download(self, session, download, executed): + # Success, but no sub_downloader set -> create_downloads(downloader_name=None) will raise. + return DownloadResult(success=True, downloads=['https://example.com/orphan-child']) + + test_download_manager.register_downloader(BadChildDownloader()) + parent = test_download_manager.create_download(test_session, 'https://example.com/bad/videos', 'bad_child') + parent_id = parent.id + test_session.commit() + + with production_like_sessions(test_session) as maker: + await signal_download_download(parent_id, 'https://example.com/bad/videos') + verify = maker() + try: + parent = verify.query(Download).filter_by(id=parent_id).one() + assert parent.status == 'complete', \ + f'parent must complete despite child-enqueue failure, got {parent.status}' + # The child could not be created, but the parent did not get stuck. + assert verify.query(Download).filter_by(url='https://example.com/orphan-child').count() == 0 + finally: + verify.close() + + @pytest.mark.asyncio async def test_delete_old_once_downloads(test_session, test_download_manager, test_downloader): """Once-downloads over a month old should be deleted."""