diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index cd99de5f..13b65aca 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1326,23 +1326,30 @@ class DownloadManager: raise TimeoutError('Downloads never finished!') @staticmethod - def retry_downloads(session: Session, reset_attempts: bool = False): + def retry_downloads(session: Session, reset_attempts: bool = False, include_failed: bool = True): """Set any incomplete Downloads to `new` so they will be retried. The caller commits. @param reset_attempts: Will set `download.attempts` to 0 if True. + @param include_failed: Also renew failed Downloads (the user's Retry); startup leaves them failed. """ values = dict(status=DownloadStatus.new, attempts=0) if reset_attempts else dict(status=DownloadStatus.new) + statuses = [DownloadStatus.pending, DownloadStatus.deferred] + if include_failed: + statuses.append(DownloadStatus.failed) retried = session.query(Download) \ - .filter(Download.status.in_((DownloadStatus.pending, DownloadStatus.deferred, DownloadStatus.failed))) \ + .filter(Download.status.in_(statuses)) \ .update(values, synchronize_session=False) if retried: # A bulk update is not flushed: count it. `status` is saved only by download_manager.yaml. count_config_change(session, DownloadManagerConfig.file_name) def retry_downloads_on_startup(self): - """Startup is not a request: retry incomplete Downloads in a short write transaction of its own.""" + """Startup is not a request: retry incomplete Downloads in a short write transaction of its own. + + A failed once-download gave up (`UnrecoverableDownloadError`, or its downloader's attempt limit) and stays + failed: renewed, it only failed again at every boot, with a traceback in the log each time.""" with get_db_session(commit=True) as session: - self.retry_downloads(session) + self.retry_downloads(session, include_failed=False) def get_new_downloads(self, session: Session) -> Generator[Download, None, None]: """ diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 9abd3c12..17f24239 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -1298,9 +1298,19 @@ async def test_retry_downloads_includes_failed(test_session, test_download_manag @pytest.mark.asyncio async def test_retry_downloads_on_startup(test_session, test_download_manager, test_downloader): - """Startup is not a request; it retries incomplete Downloads in a write transaction it commits itself.""" + """Startup is not a request; it retries incomplete Downloads in a write transaction it commits itself. + + A failed once-download gave up (its downloader raised UnrecoverableDownloadError, or it reached its attempt + limit). Renewing it at startup only had it fail again, with a traceback in the log, at every boot; only the + user's Retry brings it back.""" failed = test_download_manager.create_download(test_session, 'https://example.com/failed', test_downloader.name) failed.fail() + failed.attempts = 15 + deferred = test_download_manager.create_download(test_session, 'https://example.com/later', test_downloader.name) + deferred.defer() + pending = test_download_manager.create_download(test_session, 'https://example.com/pending', + test_downloader.name) + pending.started() complete = test_download_manager.create_download(test_session, 'https://example.com/done', test_downloader.name) complete.complete() test_session.commit() @@ -1309,7 +1319,9 @@ async def test_retry_downloads_on_startup(test_session, test_download_manager, t test_download_manager.retry_downloads_on_startup() test_session.expire_all() - assert failed.status == 'new' + assert (failed.status, failed.attempts) == ('failed', 15) + assert deferred.status == 'new' + assert pending.status == 'new' assert complete.status == 'complete'