A user can start a download even if the URL is in the skip list

This commit is contained in:
Roland 2022-06-17 13:52:15 -06:00
parent 089faa2b76
commit 6a94ffd0b2
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
3 changed files with 27 additions and 7 deletions

View file

@ -3,7 +3,7 @@ Fixtures for Pytest tests.
"""
import pathlib
import tempfile
from typing import Tuple
from typing import Tuple, Set
from unittest import mock
from uuid import uuid1
@ -15,7 +15,7 @@ from sqlalchemy.orm import Session, sessionmaker
from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.dates import set_test_now
from wrolpi.db import postgres_engine, get_db_args
from wrolpi.downloader import DownloadManager, DownloadResult, set_test_download_manager_config
from wrolpi.downloader import DownloadManager, DownloadResult, set_test_download_manager_config, Download
from wrolpi.root_api import BLUEPRINTS, api_app
@ -143,3 +143,12 @@ def successful_download():
@pytest.fixture
def failed_download():
return DownloadResult(error='pytest.fixture failed_download error', success=False)
@pytest.fixture
def assert_download_urls(test_session):
def asserter(urls: Set[str]):
downloads = test_session.query(Download).all()
assert {i.url for i in downloads} == set(urls)
return asserter

View file

@ -328,8 +328,12 @@ class DownloadManager:
for url in urls:
if not url:
raise ValueError('Download must have a URL')
if url in DOWNLOAD_MANAGER_CONFIG.skip_urls:
logger.warning(f'Skipping {url} because it is in download_manager.yaml skip list.')
if url in DOWNLOAD_MANAGER_CONFIG.skip_urls and reset_attempts:
# User manually entered this download, remove it from the skip list.
DOWNLOAD_MANAGER_CONFIG.skip_urls = [i for i in DOWNLOAD_MANAGER_CONFIG.skip_urls if i != url]
DOWNLOAD_MANAGER_CONFIG.save()
elif url in DOWNLOAD_MANAGER_CONFIG.skip_urls:
logger.warning(f'Skipping {url} because it is in the download_manager.yaml skip list.')
continue
info_json = None
if not downloader:

View file

@ -346,7 +346,7 @@ def test_max_attempts(test_session, test_download_manager):
assert download.status == 'failed'
def test_skip_urls(test_session, test_download_manager):
def test_skip_urls(test_session, test_download_manager, test_client, assert_download_urls):
"""The DownloadManager will not create downloads for URLs in it's skip list."""
_, session = get_db_context()
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
@ -362,8 +362,15 @@ def test_skip_urls(test_session, test_download_manager):
'https://example.com/skipme',
'https://example.com/2',
], downloader_name=HTTPDownloader.name)
downloads = test_download_manager.get_downloads(test_session)
assert {i.url for i in downloads} == {'https://example.com/1', 'https://example.com/2'}
assert_download_urls({'https://example.com/1', 'https://example.com/2'})
assert DOWNLOAD_MANAGER_CONFIG.skip_urls == ['https://example.com/skipme']
test_download_manager.delete_completed()
# The user can start a download even if the URL is in the skip list.
test_download_manager.create_download('https://example.com/skipme', session=test_session, reset_attempts=True)
assert_download_urls({'https://example.com/skipme'})
assert DOWNLOAD_MANAGER_CONFIG.skip_urls == []
def test_process_runner_timeout(test_directory):