downloads: a deferred once-download that nothing ever retried (#666)
Some checks are pending
CodeQL / Analyze (push) Waiting to run
Controller Tests / controller-tests (push) Waiting to run
Jest Tests / jest-tests (push) Waiting to run

* downloads: a deferred once-download that nothing ever retried

Deferring a download only schedules it: `defer()` sets the status and
`calculate_next_download` sets `next_download`.  Acting on that schedule was
`renew_recurring_downloads`, which filters on `frequency != None` -- so only
recurring downloads were ever marked "new" again.

A once-download has no frequency.  One transient error (an HTTP 403 from the
host, a dropped connection) deferred it with a `next_download` that nothing
read, and it stayed deferred until someone retried it by hand.
`delete_old_once_downloads` did not clear it either -- that requires a
`last_successful_download`, which a download that never succeeded does not
have.  So every transient failure added one more permanently-stuck row to the
downloads list.

Add `renew_deferred_once_downloads`, called from `do_downloads` alongside the
recurring renewal.  `attempts` is preserved, so `calculate_next_download`'s
3^attempts backoff keeps growing across retries: a URL that never succeeds
backs off toward never instead of retrying at a fixed interval.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* review: comments that narrated the bug instead of the contract

The new docstrings restated why the bug existed and how the fix was reasoned
about.  That history belongs in the PR, not in code that has to stay true.
Keep the invariant that a reader needs: attempts are not reset, so the backoff
grows.  In the test, the assertions already document the contract.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-08-17 15:21:06 -06:00 • committed by Roland
parent b46faa005a
commit 9b48d6f25e
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
2 changed files with 117 additions and 0 deletions

View file

@ -1253,6 +1253,11 @@ class DownloadManager:
except Exception as e:
self.log_error(f'Unable to renew downloads!', exc_info=e)
try:
self.renew_deferred_once_downloads()
except Exception as e:
self.log_error(f'Unable to renew deferred once downloads!', exc_info=e)
try:
self.delete_old_once_downloads()
except Exception as e:
@ -1382,6 +1387,37 @@ class DownloadManager:
if renewed_count:
self.log_debug(f'Renewed {renewed_count} recurring downloads')
def renew_deferred_once_downloads(self):
"""Mark any deferred once-downloads whose backoff has elapsed as "new" so they are retried.
`renew()` is called without `reset_attempts` so `calculate_next_download`'s backoff keeps growing."""
now_ = now()
# Decide what is due in a read session so the usual cycle (nothing due) never takes the write
# lock. See `renew_recurring_downloads` for why the read and write are split. The status filter
# runs in SQL because deferred once-downloads are few; the datetime comparison runs in Python to
# match how the rest of this class compares `next_download`.
with get_db_session() as session:
due_ids = [i.id for i in session.query(Download).filter(
Download.frequency == None, # noqa
Download.status == DownloadStatus.deferred,
) if i.next_download and i.next_download < now_]
if not due_ids:
return
with get_db_session(commit=True) as session:
renewed_count = 0
for download in session.query(Download).filter(Download.id.in_(due_ids)):
# A row's status may have moved on since the read above (the dispatcher claims
# downloads constantly), so re-check it here.
if download.is_deferred:
download.renew()
renewed_count += 1
if renewed_count:
self.log_debug(f'Renewed {renewed_count} deferred once downloads')
@staticmethod
def get_downloads(session: Session) -> List[Download]:
downloads = list(session.query(Download).all())

View file

@ -617,6 +617,87 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
assert download.next_download == datetime(2020, 1, 1, 5, 0, 0, 997200, tzinfo=pytz.UTC)
@pytest.mark.asyncio
async def test_deferred_once_download_is_retried(test_session, test_download_manager, fake_now, test_downloader,
await_switches):
"""A once-download that was deferred is retried once its backoff has elapsed."""
fake_now(datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC))
test_downloader.set_test_failure()
test_download_manager.create_download(test_session, 'https://example.com/once', test_downloader.name)
await test_download_manager.wait_for_all_downloads()
# The download failed, so it is deferred for 3 hours (3^attempts).
download = test_session.query(Download).one()
assert download.frequency is None, 'This must be a once-download'
assert download.is_deferred, download.status_code
assert download.attempts == 1
assert download.next_download == datetime(2020, 1, 1, 3, 0, 0, tzinfo=pytz.UTC)
# An hour before it is due, it is left alone.
fake_now(datetime(2020, 1, 1, 2, 0, 0, tzinfo=pytz.UTC))
test_download_manager.renew_deferred_once_downloads()
download = test_session.query(Download).one()
assert download.is_deferred, download.status_code
# Once due, it is renewed and downloaded again. This time it succeeds.
fake_now(datetime(2020, 1, 1, 3, 0, 1, tzinfo=pytz.UTC))
test_download_manager.renew_deferred_once_downloads()
download = test_session.query(Download).one()
assert download.is_new, download.status_code
assert download.attempts == 1, 'Attempts are preserved so the backoff keeps growing'
test_downloader.set_test_success()
await test_download_manager.wait_for_all_downloads()
download = test_session.query(Download).one()
assert download.is_complete, download.status_code
assert download.attempts == 2
@pytest.mark.asyncio
async def test_deferred_once_download_backoff_grows(test_session, test_download_manager, fake_now, test_downloader,
await_switches):
"""Repeated failures of a once-download back off exponentially rather than retrying in a loop."""
fake_now(datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC))
test_downloader.set_test_failure()
test_download_manager.create_download(test_session, 'https://example.com/once', test_downloader.name)
await test_download_manager.wait_for_all_downloads()
download = test_session.query(Download).one()
# First failure: 3^1 hours.
assert download.next_download == datetime(2020, 1, 1, 3, 0, 0, tzinfo=pytz.UTC)
fake_now(datetime(2020, 1, 1, 3, 0, 1, tzinfo=pytz.UTC))
test_download_manager.renew_deferred_once_downloads()
await test_download_manager.wait_for_all_downloads()
download = test_session.query(Download).one()
assert download.is_deferred, download.status_code
assert download.attempts == 2
# Second failure: 3^2 hours.
assert download.next_download == datetime(2020, 1, 1, 12, 0, 1, tzinfo=pytz.UTC)
@pytest.mark.asyncio
async def test_failed_once_download_is_not_renewed(test_session, test_download_manager, fake_now, test_downloader,
await_switches):
"""A once-download that failed unrecoverably is never renewed, no matter how overdue it is."""
fake_now(datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC))
test_downloader.set_test_unrecoverable_exception()
test_download_manager.create_download(test_session, 'https://example.com/once', test_downloader.name)
await test_download_manager.wait_for_all_downloads()
download = test_session.query(Download).one()
assert download.is_failed, download.status_code
fake_now(datetime(2020, 2, 1, 0, 0, 0, tzinfo=pytz.UTC))
test_download_manager.renew_deferred_once_downloads()
download = test_session.query(Download).one()
assert download.is_failed, download.status_code
@pytest.mark.asyncio
async def test_max_attempts(test_session, test_download_manager, test_downloader, await_switches):
"""A Download will only be attempted so many times, it will eventually be deleted."""