From 9b48d6f25efa6fff8628b7b72e2af2d706ab0728 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:21:06 -0600 Subject: [PATCH] downloads: a deferred once-download that nothing ever retried (#666) * downloads: a deferred once-download that nothing ever retried Deferring a download only schedules it: `defer()` sets the status and `calculate_next_download` sets `next_download`. Acting on that schedule was `renew_recurring_downloads`, which filters on `frequency != None` -- so only recurring downloads were ever marked "new" again. A once-download has no frequency. One transient error (an HTTP 403 from the host, a dropped connection) deferred it with a `next_download` that nothing read, and it stayed deferred until someone retried it by hand. `delete_old_once_downloads` did not clear it either -- that requires a `last_successful_download`, which a download that never succeeded does not have. So every transient failure added one more permanently-stuck row to the downloads list. Add `renew_deferred_once_downloads`, called from `do_downloads` alongside the recurring renewal. `attempts` is preserved, so `calculate_next_download`'s 3^attempts backoff keeps growing across retries: a URL that never succeeds backs off toward never instead of retrying at a fixed interval. Co-Authored-By: Claude Opus 5 (1M context) * review: comments that narrated the bug instead of the contract The new docstrings restated why the bug existed and how the fix was reasoned about. That history belongs in the PR, not in code that has to stay true. Keep the invariant that a reader needs: attempts are not reset, so the backoff grows. In the test, the assertions already document the contract. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- wrolpi/downloader.py | 36 +++++++++++++++ wrolpi/test/test_downloader.py | 81 ++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index a5da35c6..3b4cc286 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1253,6 +1253,11 @@ class DownloadManager: except Exception as e: self.log_error(f'Unable to renew downloads!', exc_info=e) + try: + self.renew_deferred_once_downloads() + except Exception as e: + self.log_error(f'Unable to renew deferred once downloads!', exc_info=e) + try: self.delete_old_once_downloads() except Exception as e: @@ -1382,6 +1387,37 @@ class DownloadManager: if renewed_count: self.log_debug(f'Renewed {renewed_count} recurring downloads') + def renew_deferred_once_downloads(self): + """Mark any deferred once-downloads whose backoff has elapsed as "new" so they are retried. + + `renew()` is called without `reset_attempts` so `calculate_next_download`'s backoff keeps growing.""" + now_ = now() + + # Decide what is due in a read session so the usual cycle (nothing due) never takes the write + # lock. See `renew_recurring_downloads` for why the read and write are split. The status filter + # runs in SQL because deferred once-downloads are few; the datetime comparison runs in Python to + # match how the rest of this class compares `next_download`. + with get_db_session() as session: + due_ids = [i.id for i in session.query(Download).filter( + Download.frequency == None, # noqa + Download.status == DownloadStatus.deferred, + ) if i.next_download and i.next_download < now_] + + if not due_ids: + return + + with get_db_session(commit=True) as session: + renewed_count = 0 + for download in session.query(Download).filter(Download.id.in_(due_ids)): + # A row's status may have moved on since the read above (the dispatcher claims + # downloads constantly), so re-check it here. + if download.is_deferred: + download.renew() + renewed_count += 1 + + if renewed_count: + self.log_debug(f'Renewed {renewed_count} deferred once downloads') + @staticmethod def get_downloads(session: Session) -> List[Download]: downloads = list(session.query(Download).all()) diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index d7a9e0ec..a751e1f9 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -617,6 +617,87 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now assert download.next_download == datetime(2020, 1, 1, 5, 0, 0, 997200, tzinfo=pytz.UTC) +@pytest.mark.asyncio +async def test_deferred_once_download_is_retried(test_session, test_download_manager, fake_now, test_downloader, + await_switches): + """A once-download that was deferred is retried once its backoff has elapsed.""" + fake_now(datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)) + test_downloader.set_test_failure() + + test_download_manager.create_download(test_session, 'https://example.com/once', test_downloader.name) + await test_download_manager.wait_for_all_downloads() + + # The download failed, so it is deferred for 3 hours (3^attempts). + download = test_session.query(Download).one() + assert download.frequency is None, 'This must be a once-download' + assert download.is_deferred, download.status_code + assert download.attempts == 1 + assert download.next_download == datetime(2020, 1, 1, 3, 0, 0, tzinfo=pytz.UTC) + + # An hour before it is due, it is left alone. + fake_now(datetime(2020, 1, 1, 2, 0, 0, tzinfo=pytz.UTC)) + test_download_manager.renew_deferred_once_downloads() + download = test_session.query(Download).one() + assert download.is_deferred, download.status_code + + # Once due, it is renewed and downloaded again. This time it succeeds. + fake_now(datetime(2020, 1, 1, 3, 0, 1, tzinfo=pytz.UTC)) + test_download_manager.renew_deferred_once_downloads() + download = test_session.query(Download).one() + assert download.is_new, download.status_code + assert download.attempts == 1, 'Attempts are preserved so the backoff keeps growing' + + test_downloader.set_test_success() + await test_download_manager.wait_for_all_downloads() + download = test_session.query(Download).one() + assert download.is_complete, download.status_code + assert download.attempts == 2 + + +@pytest.mark.asyncio +async def test_deferred_once_download_backoff_grows(test_session, test_download_manager, fake_now, test_downloader, + await_switches): + """Repeated failures of a once-download back off exponentially rather than retrying in a loop.""" + fake_now(datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)) + test_downloader.set_test_failure() + + test_download_manager.create_download(test_session, 'https://example.com/once', test_downloader.name) + await test_download_manager.wait_for_all_downloads() + + download = test_session.query(Download).one() + # First failure: 3^1 hours. + assert download.next_download == datetime(2020, 1, 1, 3, 0, 0, tzinfo=pytz.UTC) + + fake_now(datetime(2020, 1, 1, 3, 0, 1, tzinfo=pytz.UTC)) + test_download_manager.renew_deferred_once_downloads() + await test_download_manager.wait_for_all_downloads() + + download = test_session.query(Download).one() + assert download.is_deferred, download.status_code + assert download.attempts == 2 + # Second failure: 3^2 hours. + assert download.next_download == datetime(2020, 1, 1, 12, 0, 1, tzinfo=pytz.UTC) + + +@pytest.mark.asyncio +async def test_failed_once_download_is_not_renewed(test_session, test_download_manager, fake_now, test_downloader, + await_switches): + """A once-download that failed unrecoverably is never renewed, no matter how overdue it is.""" + fake_now(datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)) + test_downloader.set_test_unrecoverable_exception() + + test_download_manager.create_download(test_session, 'https://example.com/once', test_downloader.name) + await test_download_manager.wait_for_all_downloads() + + download = test_session.query(Download).one() + assert download.is_failed, download.status_code + + fake_now(datetime(2020, 2, 1, 0, 0, 0, tzinfo=pytz.UTC)) + test_download_manager.renew_deferred_once_downloads() + download = test_session.query(Download).one() + assert download.is_failed, download.status_code + + @pytest.mark.asyncio async def test_max_attempts(test_session, test_download_manager, test_downloader, await_switches): """A Download will only be attempted so many times, it will eventually be deleted."""