From 6a94ffd0b2c56175bd0b1d68c252b204d265cea0 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 17 Jun 2022 13:52:15 -0600 Subject: [PATCH] A user can start a download even if the URL is in the skip list --- wrolpi/conftest.py | 13 +++++++++++-- wrolpi/downloader.py | 8 ++++++-- wrolpi/test/test_downloader.py | 13 ++++++++++--- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index c7973b78..47cea0e1 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -3,7 +3,7 @@ Fixtures for Pytest tests. """ import pathlib import tempfile -from typing import Tuple +from typing import Tuple, Set from unittest import mock from uuid import uuid1 @@ -15,7 +15,7 @@ from sqlalchemy.orm import Session, sessionmaker from wrolpi.common import set_test_media_directory, Base, set_test_config from wrolpi.dates import set_test_now from wrolpi.db import postgres_engine, get_db_args -from wrolpi.downloader import DownloadManager, DownloadResult, set_test_download_manager_config +from wrolpi.downloader import DownloadManager, DownloadResult, set_test_download_manager_config, Download from wrolpi.root_api import BLUEPRINTS, api_app @@ -143,3 +143,12 @@ def successful_download(): @pytest.fixture def failed_download(): return DownloadResult(error='pytest.fixture failed_download error', success=False) + + +@pytest.fixture +def assert_download_urls(test_session): + def asserter(urls: Set[str]): + downloads = test_session.query(Download).all() + assert {i.url for i in downloads} == set(urls) + + return asserter diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index b8afb366..2696cdc2 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -328,8 +328,12 @@ class DownloadManager: for url in urls: if not url: raise ValueError('Download must have a URL') - if url in DOWNLOAD_MANAGER_CONFIG.skip_urls: - logger.warning(f'Skipping {url} because it is in download_manager.yaml skip list.') + if url in DOWNLOAD_MANAGER_CONFIG.skip_urls and reset_attempts: + # User manually entered this download, remove it from the skip list. + DOWNLOAD_MANAGER_CONFIG.skip_urls = [i for i in DOWNLOAD_MANAGER_CONFIG.skip_urls if i != url] + DOWNLOAD_MANAGER_CONFIG.save() + elif url in DOWNLOAD_MANAGER_CONFIG.skip_urls: + logger.warning(f'Skipping {url} because it is in the download_manager.yaml skip list.') continue info_json = None if not downloader: diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 3aa9b4a3..ce210bba 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -346,7 +346,7 @@ def test_max_attempts(test_session, test_download_manager): assert download.status == 'failed' -def test_skip_urls(test_session, test_download_manager): +def test_skip_urls(test_session, test_download_manager, test_client, assert_download_urls): """The DownloadManager will not create downloads for URLs in it's skip list.""" _, session = get_db_context() from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG @@ -362,8 +362,15 @@ def test_skip_urls(test_session, test_download_manager): 'https://example.com/skipme', 'https://example.com/2', ], downloader_name=HTTPDownloader.name) - downloads = test_download_manager.get_downloads(test_session) - assert {i.url for i in downloads} == {'https://example.com/1', 'https://example.com/2'} + assert_download_urls({'https://example.com/1', 'https://example.com/2'}) + assert DOWNLOAD_MANAGER_CONFIG.skip_urls == ['https://example.com/skipme'] + + test_download_manager.delete_completed() + + # The user can start a download even if the URL is in the skip list. + test_download_manager.create_download('https://example.com/skipme', session=test_session, reset_attempts=True) + assert_download_urls({'https://example.com/skipme'}) + assert DOWNLOAD_MANAGER_CONFIG.skip_urls == [] def test_process_runner_timeout(test_directory):