diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index a5da35c6..3b4cc286 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1253,6 +1253,11 @@ class DownloadManager: except Exception as e: self.log_error(f'Unable to renew downloads!', exc_info=e) + try: + self.renew_deferred_once_downloads() + except Exception as e: + self.log_error(f'Unable to renew deferred once downloads!', exc_info=e) + try: self.delete_old_once_downloads() except Exception as e: @@ -1382,6 +1387,37 @@ class DownloadManager: if renewed_count: self.log_debug(f'Renewed {renewed_count} recurring downloads') + def renew_deferred_once_downloads(self): + """Mark any deferred once-downloads whose backoff has elapsed as "new" so they are retried. + + `renew()` is called without `reset_attempts` so `calculate_next_download`'s backoff keeps growing.""" + now_ = now() + + # Decide what is due in a read session so the usual cycle (nothing due) never takes the write + # lock. See `renew_recurring_downloads` for why the read and write are split. The status filter + # runs in SQL because deferred once-downloads are few; the datetime comparison runs in Python to + # match how the rest of this class compares `next_download`. + with get_db_session() as session: + due_ids = [i.id for i in session.query(Download).filter( + Download.frequency == None, # noqa + Download.status == DownloadStatus.deferred, + ) if i.next_download and i.next_download < now_] + + if not due_ids: + return + + with get_db_session(commit=True) as session: + renewed_count = 0 + for download in session.query(Download).filter(Download.id.in_(due_ids)): + # A row's status may have moved on since the read above (the dispatcher claims + # downloads constantly), so re-check it here. + if download.is_deferred: + download.renew() + renewed_count += 1 + + if renewed_count: + self.log_debug(f'Renewed {renewed_count} deferred once downloads') + @staticmethod def get_downloads(session: Session) -> List[Download]: downloads = list(session.query(Download).all()) diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index d7a9e0ec..a751e1f9 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -617,6 +617,87 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now assert download.next_download == datetime(2020, 1, 1, 5, 0, 0, 997200, tzinfo=pytz.UTC) +@pytest.mark.asyncio +async def test_deferred_once_download_is_retried(test_session, test_download_manager, fake_now, test_downloader, + await_switches): + """A once-download that was deferred is retried once its backoff has elapsed.""" + fake_now(datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)) + test_downloader.set_test_failure() + + test_download_manager.create_download(test_session, 'https://example.com/once', test_downloader.name) + await test_download_manager.wait_for_all_downloads() + + # The download failed, so it is deferred for 3 hours (3^attempts). + download = test_session.query(Download).one() + assert download.frequency is None, 'This must be a once-download' + assert download.is_deferred, download.status_code + assert download.attempts == 1 + assert download.next_download == datetime(2020, 1, 1, 3, 0, 0, tzinfo=pytz.UTC) + + # An hour before it is due, it is left alone. + fake_now(datetime(2020, 1, 1, 2, 0, 0, tzinfo=pytz.UTC)) + test_download_manager.renew_deferred_once_downloads() + download = test_session.query(Download).one() + assert download.is_deferred, download.status_code + + # Once due, it is renewed and downloaded again. This time it succeeds. + fake_now(datetime(2020, 1, 1, 3, 0, 1, tzinfo=pytz.UTC)) + test_download_manager.renew_deferred_once_downloads() + download = test_session.query(Download).one() + assert download.is_new, download.status_code + assert download.attempts == 1, 'Attempts are preserved so the backoff keeps growing' + + test_downloader.set_test_success() + await test_download_manager.wait_for_all_downloads() + download = test_session.query(Download).one() + assert download.is_complete, download.status_code + assert download.attempts == 2 + + +@pytest.mark.asyncio +async def test_deferred_once_download_backoff_grows(test_session, test_download_manager, fake_now, test_downloader, + await_switches): + """Repeated failures of a once-download back off exponentially rather than retrying in a loop.""" + fake_now(datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)) + test_downloader.set_test_failure() + + test_download_manager.create_download(test_session, 'https://example.com/once', test_downloader.name) + await test_download_manager.wait_for_all_downloads() + + download = test_session.query(Download).one() + # First failure: 3^1 hours. + assert download.next_download == datetime(2020, 1, 1, 3, 0, 0, tzinfo=pytz.UTC) + + fake_now(datetime(2020, 1, 1, 3, 0, 1, tzinfo=pytz.UTC)) + test_download_manager.renew_deferred_once_downloads() + await test_download_manager.wait_for_all_downloads() + + download = test_session.query(Download).one() + assert download.is_deferred, download.status_code + assert download.attempts == 2 + # Second failure: 3^2 hours. + assert download.next_download == datetime(2020, 1, 1, 12, 0, 1, tzinfo=pytz.UTC) + + +@pytest.mark.asyncio +async def test_failed_once_download_is_not_renewed(test_session, test_download_manager, fake_now, test_downloader, + await_switches): + """A once-download that failed unrecoverably is never renewed, no matter how overdue it is.""" + fake_now(datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)) + test_downloader.set_test_unrecoverable_exception() + + test_download_manager.create_download(test_session, 'https://example.com/once', test_downloader.name) + await test_download_manager.wait_for_all_downloads() + + download = test_session.query(Download).one() + assert download.is_failed, download.status_code + + fake_now(datetime(2020, 2, 1, 0, 0, 0, tzinfo=pytz.UTC)) + test_download_manager.renew_deferred_once_downloads() + download = test_session.query(Download).one() + assert download.is_failed, download.status_code + + @pytest.mark.asyncio async def test_max_attempts(test_session, test_download_manager, test_downloader, await_switches): """A Download will only be attempted so many times, it will eventually be deleted."""