mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
Requiring downloader name to be specified.
This commit is contained in:
parent
5692ddb20d
commit
55cf76a884
15 changed files with 215 additions and 320 deletions
|
|
@ -23,6 +23,10 @@ function handleEvents(events) {
|
|||
eventToast('Refresh completed', 'All files have been refreshed.');
|
||||
}
|
||||
|
||||
if (event === 'downloads_disabled') {
|
||||
eventToast('Downloads Disabled', message, 'info');
|
||||
}
|
||||
|
||||
if (subject) {
|
||||
newestEvents[subject] = dt;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -412,6 +412,16 @@ export async function getArchive(archiveId) {
|
|||
}
|
||||
|
||||
export async function postDownload(urls, downloader, frequency, sub_downloader, excludedURLs, destination) {
|
||||
if (!downloader) {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Failed to submit download',
|
||||
description: 'downloader is required, but was not provided',
|
||||
time: 5000,
|
||||
})
|
||||
throw new Error('downloader is required, but was not provided');
|
||||
}
|
||||
|
||||
let body = {
|
||||
urls: urls,
|
||||
downloader: downloader,
|
||||
|
|
|
|||
|
|
@ -208,7 +208,7 @@ class StoppableRow extends React.Component {
|
|||
|
||||
handleStart = async (e) => {
|
||||
e.preventDefault();
|
||||
let downloader = this.props.downloader || null;
|
||||
let downloader = this.props.downloader;
|
||||
await postDownload(`${this.props.url}`, downloader);
|
||||
this.closeStart();
|
||||
await this.props.fetchDownloads();
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
|
||||
|
||||
# Archive downloader is the last downloader which should be used.
|
||||
archive_downloader = ArchiveDownloader(priority=100)
|
||||
archive_downloader = ArchiveDownloader()
|
||||
|
||||
|
||||
def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
|
||||
|
|
|
|||
|
|
@ -423,7 +423,8 @@ def test_change_channel_url(test_client, test_session, test_download_manager, do
|
|||
test_session.commit()
|
||||
|
||||
download = test_session.query(Download).one()
|
||||
assert download.url == download_channel.url == 'https://example.com/new-url', "Channel's download URL was not changed."
|
||||
assert download.url == download_channel.url == 'https://example.com/new-url', \
|
||||
"Channel's download URL was not changed."
|
||||
|
||||
download_channel.update({'download_frequency': None})
|
||||
assert not list(test_session.query(Download).all())
|
||||
|
|
@ -456,13 +457,15 @@ def test_search_videos_channel(test_client, test_session, video_factory):
|
|||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['file_groups']) == 1
|
||||
assert response.json['totals']['file_groups'] == 1
|
||||
assert_dict_contains(response.json['file_groups'][0], dict(primary_path='vid2.mp4', video=dict(channel_id=channel1.id)))
|
||||
assert_dict_contains(response.json['file_groups'][0],
|
||||
dict(primary_path='vid2.mp4', video=dict(channel_id=channel1.id)))
|
||||
|
||||
d = dict(channel_id=channel2.id)
|
||||
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['file_groups']) == 1
|
||||
assert_dict_contains(response.json['file_groups'][0], dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
|
||||
assert_dict_contains(response.json['file_groups'][0],
|
||||
dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
|
||||
|
||||
|
||||
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory, assert_video_search):
|
||||
|
|
|
|||
|
|
@ -64,7 +64,8 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch
|
|||
url='https://example.com/channel1',
|
||||
download_frequency=DownloadFrequency.weekly,
|
||||
)
|
||||
download = Download(url=channel.url, downloader='video_channel', frequency=channel.download_frequency)
|
||||
download = Download(url=channel.url, downloader='video_channel', frequency=channel.download_frequency,
|
||||
sub_downloader='video')
|
||||
test_session.add_all([channel, download])
|
||||
test_session.commit()
|
||||
return channel
|
||||
|
|
@ -140,7 +141,7 @@ def video_factory(test_session, test_directory):
|
|||
def video_download_manager(test_download_manager) -> DownloadManager:
|
||||
"""Get a DownloadManager ready to download Videos and Channels."""
|
||||
channel_downloader = ChannelDownloader()
|
||||
video_downloader = VideoDownloader(priority=40)
|
||||
video_downloader = VideoDownloader()
|
||||
|
||||
test_download_manager.register_downloader(channel_downloader)
|
||||
test_download_manager.register_downloader(video_downloader)
|
||||
|
|
|
|||
|
|
@ -393,7 +393,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
|
||||
channel_downloader = ChannelDownloader()
|
||||
# Videos may match the ChannelDownloader, give it a higher priority.
|
||||
video_downloader = VideoDownloader(40)
|
||||
video_downloader = VideoDownloader()
|
||||
|
||||
|
||||
def get_or_create_channel(source_id: str = None, url: str = None, name: str = None) -> Channel:
|
||||
|
|
|
|||
|
|
@ -10,11 +10,9 @@ from modules.videos.channel.lib import download_channel
|
|||
from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \
|
||||
get_or_create_channel, channel_downloader, video_downloader
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import Download, DownloadResult
|
||||
from wrolpi.errors import InvalidDownload
|
||||
from wrolpi.test.common import skip_circleci
|
||||
from wrolpi.test.test_downloader import HTTPDownloader
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
|
|
@ -133,21 +131,6 @@ def test_channel_valid_url():
|
|||
assert channel_downloader.valid_url('https://www.youtube.com/playlist?list=PLCdlMQeP-TbG12nkBCt0E96yr3EfwcH4E')[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
'url,expected', [
|
||||
('https://www.youtube.com/c/LearningSelfReliance/videos', channel_downloader),
|
||||
('https://www.youtube.com/c/LearningSelfReliance', channel_downloader),
|
||||
('https://www.youtube.com/watch?v=31jPEBiAC3c', video_downloader),
|
||||
]
|
||||
)
|
||||
async def test_get_downloader(test_session, test_download_manager, url, expected):
|
||||
"""The correct Downloader is gotten."""
|
||||
test_download_manager.register_downloader(channel_downloader)
|
||||
test_download_manager.register_downloader(video_downloader)
|
||||
assert test_download_manager.get_downloader(url)[0] == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_no_channel(test_session, video_download_manager, video_factory, test_directory,
|
||||
mock_video_extract_info, mock_video_process_runner):
|
||||
|
|
@ -157,6 +140,8 @@ async def test_download_no_channel(test_session, video_download_manager, video_f
|
|||
video_path = channel_dir / 'video.mp4'
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
|
||||
|
||||
channel_downloader, video_downloader = video_download_manager.instances
|
||||
|
||||
# Video has no channel
|
||||
info_json = copy(example_video_json)
|
||||
del info_json['channel']
|
||||
|
|
@ -167,7 +152,7 @@ async def test_download_no_channel(test_session, video_download_manager, video_f
|
|||
mock_video_extract_info.return_value = info_json
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
|
||||
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
||||
video_download_manager.create_download(url)
|
||||
video_download_manager.create_download(url, video_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
|
||||
mock_video_process_runner.assert_called_once()
|
||||
|
|
@ -186,11 +171,13 @@ async def test_download_channel(test_session, simple_channel, video_download_man
|
|||
If a Channel has `match_regex` only those videos with matching titles will be downloaded."""
|
||||
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
|
||||
|
||||
channel_downloader, video_downloader = video_download_manager.instances
|
||||
|
||||
mock_video_prepare_filename.return_value = video_file
|
||||
mock_video_extract_info.return_value = example_channel_json
|
||||
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
|
||||
mock_get_channel.return_value = simple_channel
|
||||
video_download_manager.create_download(url)
|
||||
video_download_manager.create_download(url, channel_downloader.name, sub_downloader_name=video_downloader.name)
|
||||
# Download channel.
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
# Download videos.
|
||||
|
|
@ -212,7 +199,8 @@ async def test_download_channel(test_session, simple_channel, video_download_man
|
|||
test_session.commit()
|
||||
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
|
||||
mock_get_channel.return_value = simple_channel
|
||||
video_download_manager.recurring_download(url, 100)
|
||||
video_download_manager.recurring_download(url, 100, channel_downloader.name,
|
||||
sub_downloader_name=video_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
downloads = video_download_manager.get_once_downloads(test_session)
|
||||
assert [i.url for i in downloads] == ['https://youtube.com/watch?v=video_2_url']
|
||||
|
|
@ -260,31 +248,13 @@ def test_get_or_create_channel(test_session):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_info_save(test_session, video_download_manager, mock_video_extract_info,
|
||||
mock_video_prepare_filename,
|
||||
mock_video_process_runner):
|
||||
"""
|
||||
An info dict from Downloader.valid_url is saved to the Downloader and can be used by the Downloader later.
|
||||
"""
|
||||
mock_video_extract_info.return_value = {'some': 'info'}
|
||||
mock_video_prepare_filename.return_value = 'some path'
|
||||
video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80')
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
assert mock_video_extract_info.call_count == 2 # Only called twice. The "info" gathering call was skipped.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_selected_downloader(test_session, video_download_manager, successful_download):
|
||||
async def test_selected_downloader(test_session, video_download_manager, test_downloader, successful_download):
|
||||
"""
|
||||
A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
|
||||
"""
|
||||
# The test HTTP Downloader will claim to be able to handle any HTTP URL; will pretend to have downloaded anything.
|
||||
http_downloader = HTTPDownloader()
|
||||
mock_do_download = http_downloader.do_download = mock.MagicMock()
|
||||
# The test Downloader will pretend to have downloaded anything.
|
||||
mock_do_download = test_downloader.do_download = mock.MagicMock()
|
||||
mock_do_download.return_value = successful_download
|
||||
mock_http_valid_url = http_downloader.valid_url = mock.MagicMock()
|
||||
mock_http_valid_url.return_value = (True, None)
|
||||
video_download_manager.register_downloader(http_downloader)
|
||||
|
||||
def check_attempts(attempts):
|
||||
assert test_session.query(Download).one().attempts == attempts
|
||||
|
|
@ -292,42 +262,30 @@ async def test_selected_downloader(test_session, video_download_manager, success
|
|||
# DownloadManager will attempt to choose the Downloader.
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
mock_extract_info.side_effect = UnsupportedError('not this one')
|
||||
video_download_manager.create_download('https://example.com')
|
||||
video_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
mock_http_valid_url.assert_called_once()
|
||||
mock_extract_info.assert_called_once()
|
||||
mock_do_download.assert_called_once()
|
||||
|
||||
check_attempts(1)
|
||||
|
||||
mock_http_valid_url.reset_mock()
|
||||
mock_do_download.reset_mock()
|
||||
|
||||
# DownloadManager will only use the HTTPDownloader.
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
video_download_manager.create_download('https://example.com', downloader='http')
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
|
||||
mock_extract_info.assert_not_called()
|
||||
mock_http_valid_url.assert_not_called()
|
||||
# DownloadManager called do_download.
|
||||
mock_do_download.assert_called_once()
|
||||
# DownloadManager will only use the TestDownloader.
|
||||
video_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
# DownloadManager called do_download.
|
||||
mock_do_download.assert_called_once()
|
||||
|
||||
check_attempts(2)
|
||||
|
||||
mock_http_valid_url.reset_mock()
|
||||
mock_do_download.reset_mock()
|
||||
|
||||
# DownloadManager will only use the HTTPDownloader, even if the download fails.
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
mock_do_download.side_effect = Exception('oh no!')
|
||||
video_download_manager.create_download('https://example.com', downloader='http')
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
|
||||
mock_extract_info.assert_not_called()
|
||||
mock_http_valid_url.assert_not_called()
|
||||
# DownloadManager called do_download.
|
||||
mock_do_download.assert_called_once()
|
||||
# DownloadManager will only use the TestDownloader, even if the download fails.
|
||||
mock_do_download.side_effect = Exception('oh no!')
|
||||
video_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
# DownloadManager called do_download.
|
||||
mock_do_download.assert_called_once()
|
||||
|
||||
check_attempts(3)
|
||||
|
||||
|
|
@ -346,7 +304,7 @@ def test_bad_downloader(test_session, video_download_manager):
|
|||
Attempting to use an unknown downloader should raise an error.
|
||||
"""
|
||||
with pytest.raises(InvalidDownload):
|
||||
video_download_manager.create_download('https://example.com', downloader='bad downloader')
|
||||
video_download_manager.create_download('https://example.com', downloader_name='bad downloader')
|
||||
|
||||
|
||||
def test_channel_download_no_download(test_session, video_download_manager, simple_channel):
|
||||
|
|
@ -361,24 +319,6 @@ def test_channel_download_no_download(test_session, video_download_manager, simp
|
|||
download_channel(simple_channel.id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_download_url(test_session, test_download_manager, mock_video_extract_info):
|
||||
"""An invalid url should not be attempted again."""
|
||||
_, session = get_db_context()
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \
|
||||
mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader:
|
||||
video_downloader = VideoDownloader()
|
||||
test_download_manager.register_downloader(video_downloader)
|
||||
mock_get_downloader.return_value = (video_downloader, None)
|
||||
mock_valid_url.return_value = (True, {}) # url is valid, but is not a video URL.
|
||||
mock_video_extract_info.side_effect = UnsupportedError('oops')
|
||||
test_download_manager.create_download('invalid url', downloader=VideoDownloader.name)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
||||
download = session.query(Download).one()
|
||||
assert download.status == 'failed'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_download(test_session, test_directory, simple_channel, video_download_manager,
|
||||
mock_video_process_runner, image_file):
|
||||
|
|
@ -399,7 +339,7 @@ async def test_video_download(test_session, test_directory, simple_channel, vide
|
|||
mock_extract_info.return_value = example_video_json
|
||||
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
||||
|
||||
video_download_manager.create_download(url)
|
||||
video_download_manager.create_download(url, video_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
|
||||
mock_video_process_runner.assert_called_once()
|
||||
|
|
@ -427,7 +367,7 @@ async def test_download_result(test_session, test_directory, video_download_mana
|
|||
mock_video_extract_info.return_value = example_video_json
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
|
||||
mock_prepare_filename.return_value = [video_file, {'id': 'foo'}]
|
||||
video_download_manager.create_download('https://example.com')
|
||||
video_download_manager.create_download('https://example.com', video_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
|
||||
download: Download = test_session.query(Download).one()
|
||||
|
|
@ -451,7 +391,7 @@ async def test_download_destination(test_session, test_directory, video_download
|
|||
# This result will be ignored during the test, we just need some value so the download does not fail.
|
||||
mock_video_prepare_filename.return_value = str(test_directory / 'test video.mp4')
|
||||
|
||||
video_download_manager.create_download('https://example.com/1', downloader=VideoDownloader.name)
|
||||
video_download_manager.create_download('https://example.com/1', downloader_name=VideoDownloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
# Output directory matches the channel directory.
|
||||
assert mock_video_prepare_filename.call_args_list[0].kwargs['ydl'].params['outtmpl']['default'] \
|
||||
|
|
@ -460,7 +400,8 @@ async def test_download_destination(test_session, test_directory, video_download
|
|||
mock_video_prepare_filename.reset_mock()
|
||||
|
||||
settings = dict(destination=f'{test_directory}/custom')
|
||||
video_download_manager.create_download('https://example.com/2', downloader=VideoDownloader.name, settings=settings)
|
||||
video_download_manager.create_download('https://example.com/2', downloader_name=VideoDownloader.name,
|
||||
settings=settings)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
# Output directory matches the custom directory specified.
|
||||
assert mock_video_prepare_filename.call_args_list[0].kwargs['ydl'].params['outtmpl']['default'] \
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import multiprocessing
|
|||
import pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
from abc import ABC
|
||||
from itertools import zip_longest
|
||||
from typing import List, Callable, Union, Dict
|
||||
from typing import Tuple, Set
|
||||
|
|
@ -29,7 +30,7 @@ from wrolpi.common import iterify, log_level_context
|
|||
from wrolpi.common import set_test_media_directory, Base, set_test_config
|
||||
from wrolpi.dates import set_test_now
|
||||
from wrolpi.db import postgres_engine, get_db_args
|
||||
from wrolpi.downloader import DownloadManager, DownloadResult, set_test_download_manager_config, Download
|
||||
from wrolpi.downloader import DownloadManager, DownloadResult, set_test_download_manager_config, Download, Downloader
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.root_api import BLUEPRINTS, api_app
|
||||
from wrolpi.tags import Tag
|
||||
|
|
@ -179,13 +180,44 @@ def failed_download():
|
|||
|
||||
@pytest.fixture
|
||||
def assert_download_urls(test_session):
|
||||
def asserter(urls: Set[str]):
|
||||
def asserter(expected_urls: Set[str]):
|
||||
downloads = test_session.query(Download).all()
|
||||
assert {i.url for i in downloads} == set(urls)
|
||||
urls = {i.url for i in downloads}
|
||||
if urls != set(expected_urls):
|
||||
raise AssertionError(f'Download URLs do not match: {urls} != {expected_urls}')
|
||||
|
||||
return asserter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_downloads(test_session):
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
def asserter(expected: List[Dict]):
|
||||
downloads = test_session.query(Download).order_by(Download.url)
|
||||
for download_, expected_ in zip_longest(downloads, expected):
|
||||
assert_dict_contains(download_.__json__(), expected_)
|
||||
if (count := test_session.query(Download).count()) != len(expected):
|
||||
raise AssertionError(f'Download count does not match: {count} != {len(expected)}')
|
||||
|
||||
return asserter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_downloader(test_download_manager):
|
||||
class TestDownloader(Downloader, ABC):
|
||||
"""A testing Downloader"""
|
||||
name = 'test_downloader'
|
||||
|
||||
def __repr__(self):
|
||||
return '<TESTING Downloader>'
|
||||
|
||||
test_downloader = TestDownloader()
|
||||
test_download_manager.register_downloader(test_downloader)
|
||||
|
||||
return test_downloader
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def video_file(test_directory) -> pathlib.Path:
|
||||
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
|
||||
|
|
|
|||
|
|
@ -130,15 +130,14 @@ class Download(ModelHelper, Base): # noqa
|
|||
if self.downloader:
|
||||
return self.manager.get_downloader_by_name(self.downloader)
|
||||
|
||||
downloader, _ = self.manager.get_downloader(self.url)
|
||||
return downloader
|
||||
raise UnrecoverableDownloadError(f'Cannot find downloader for {repr(str(self.url))}')
|
||||
|
||||
@property
|
||||
def domain(self):
|
||||
return urlparse(self.url).netloc
|
||||
|
||||
@property
|
||||
def manager(self):
|
||||
def manager(self) -> 'DownloadManager':
|
||||
if self._manager:
|
||||
return self._manager
|
||||
raise ValueError('No manager has been set!')
|
||||
|
|
@ -163,19 +162,11 @@ class Downloader:
|
|||
listable: bool = True
|
||||
timeout: int = None
|
||||
|
||||
def __init__(self, priority: int = 50, name: str = None, timeout: int = None):
|
||||
"""
|
||||
Lower `priority` means the downloader will be checked first. Valid priority: 0-100
|
||||
|
||||
Downloaders of equal priority will be used at random.
|
||||
"""
|
||||
def __init__(self, name: str = None, timeout: int = None):
|
||||
if not name and not self.name:
|
||||
raise NotImplementedError('Downloader must have a name!')
|
||||
if not 0 <= priority <= 100:
|
||||
raise ValueError(f'Priority of {priority} of {self} is invalid!')
|
||||
|
||||
self.name: str = self.name or name
|
||||
self.priority: int = priority
|
||||
self.timeout: int = timeout or self.timeout
|
||||
self._kill = multiprocessing.Event()
|
||||
|
||||
|
|
@ -288,7 +279,6 @@ class DownloadManager:
|
|||
Only one download from each domain will be downloaded at a time. If we only have one domain (example.com) in the
|
||||
URLs to be downloaded, then only one worker will be busy.
|
||||
"""
|
||||
priority_sorter = partial(sorted, key=attrgetter('priority'))
|
||||
manager = multiprocessing.Event()
|
||||
|
||||
def __init__(self):
|
||||
|
|
@ -318,8 +308,7 @@ class DownloadManager:
|
|||
|
||||
instance.manager = self
|
||||
|
||||
i = (*self.instances, instance)
|
||||
self.instances = tuple(self.priority_sorter(i))
|
||||
self.instances = (*self.instances, instance)
|
||||
self._instances[instance.name] = instance
|
||||
|
||||
async def download_worker(self, num: int):
|
||||
|
|
@ -404,10 +393,8 @@ class DownloadManager:
|
|||
if result.downloads:
|
||||
worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}')
|
||||
urls = download.filter_excluded(result.downloads)
|
||||
downloads = self.create_downloads(urls, session,
|
||||
downloader=download.sub_downloader,
|
||||
settings=result.settings,
|
||||
)
|
||||
self.create_downloads(urls, session, downloader_name=download.sub_downloader,
|
||||
settings=result.settings)
|
||||
|
||||
if try_again is False and not download.frequency:
|
||||
# Only once-downloads can fail.
|
||||
|
|
@ -511,16 +498,11 @@ class DownloadManager:
|
|||
|
||||
background_task(_perpetual_download())
|
||||
|
||||
def get_downloader(self, url: str) -> Tuple[Downloader, Dict]:
|
||||
for i in self.instances:
|
||||
valid, info = i.valid_url(url)
|
||||
if valid:
|
||||
return i, info
|
||||
raise InvalidDownload(f'Invalid URL {url=}')
|
||||
|
||||
def get_downloader_by_name(self, name: str) -> Optional[Downloader]:
|
||||
"""Attempt to find a registered Downloader by its name. Returns None if it cannot be found."""
|
||||
return self._instances.get(name)
|
||||
if downloader := self._instances.get(name):
|
||||
return downloader
|
||||
raise InvalidDownload(f'Cannot find downloader with name {name}')
|
||||
|
||||
def get_or_create_download(self, url: str, session: Session) -> Download:
|
||||
"""Get a Download by its URL, if it cannot be found create one."""
|
||||
|
|
@ -539,18 +521,16 @@ class DownloadManager:
|
|||
return download
|
||||
|
||||
@optional_session
|
||||
def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None,
|
||||
reset_attempts: bool = False, sub_downloader: str = None, settings: dict = None) \
|
||||
def create_downloads(self, urls: List[str], downloader_name: str, session: Session = None,
|
||||
reset_attempts: bool = False, sub_downloader_name: str = None, settings: dict = None) \
|
||||
-> List[Download]:
|
||||
"""Schedule all URLs for download. If one cannot be downloaded, none will be added."""
|
||||
if not all(urls):
|
||||
raise ValueError('Download must have a URL')
|
||||
|
||||
downloads = []
|
||||
shared_downloader = self.get_downloader_by_name(downloader) if downloader else None
|
||||
if downloader and not shared_downloader:
|
||||
# Could not look up the downloader.
|
||||
raise InvalidDownload(f'Unknown downloader {downloader}')
|
||||
# Throws an error if no downloader is found.
|
||||
self.get_downloader_by_name(downloader_name)
|
||||
|
||||
with session.transaction:
|
||||
for url in urls:
|
||||
|
|
@ -560,20 +540,12 @@ class DownloadManager:
|
|||
elif url in DOWNLOAD_MANAGER_CONFIG.skip_urls:
|
||||
self.log_warning(f'Skipping {url} because it is in the download_manager.yaml skip list.')
|
||||
continue
|
||||
info_json = None
|
||||
if not shared_downloader:
|
||||
# User has requested automatic downloader selection, try and find it.
|
||||
local_downloader, info_json = self.get_downloader(url)
|
||||
else:
|
||||
# One Downloader provided for all URLs, use it.
|
||||
local_downloader = shared_downloader
|
||||
|
||||
download = self.get_or_create_download(url, session)
|
||||
# Download may have failed, try again.
|
||||
download.renew(reset_attempts=reset_attempts)
|
||||
download.downloader = local_downloader.name
|
||||
download.info_json = info_json or download.info_json
|
||||
download.sub_downloader = sub_downloader
|
||||
download.downloader = downloader_name
|
||||
download.sub_downloader = sub_downloader_name
|
||||
download.settings = settings
|
||||
downloads.append(download)
|
||||
|
||||
|
|
@ -590,24 +562,27 @@ class DownloadManager:
|
|||
return downloads
|
||||
|
||||
@optional_session
|
||||
def create_download(self, url: str, session: Session = None, downloader: str = None, reset_attempts: bool = False,
|
||||
sub_downloader: str = None, settings: Dict = None) -> Download:
|
||||
def create_download(self, url: str, downloader_name: str, session: Session = None, reset_attempts: bool = False,
|
||||
sub_downloader_name: str = None, settings: Dict = None) -> Download:
|
||||
"""Schedule a URL for download. If the URL failed previously, it may be retried."""
|
||||
downloads = self.create_downloads([url], session=session, downloader=downloader, settings=settings,
|
||||
reset_attempts=reset_attempts, sub_downloader=sub_downloader)
|
||||
downloads = self.create_downloads([url], session=session, downloader_name=downloader_name,
|
||||
reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name,
|
||||
settings=settings)
|
||||
return downloads[0]
|
||||
|
||||
def recurring_download(self, url: str, frequency: int, downloader: str = None,
|
||||
sub_downloader: str = None, reset_attempts: bool = False,
|
||||
@optional_session
|
||||
def recurring_download(self, url: str, frequency: int, downloader_name: str, session: Session = None,
|
||||
sub_downloader_name: str = None, reset_attempts: bool = False,
|
||||
settings: Dict = None) -> Download:
|
||||
"""Schedule a recurring download."""
|
||||
if not frequency:
|
||||
if not frequency or not isinstance(frequency, int):
|
||||
raise ValueError('Recurring download must have a frequency!')
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
download, = self.create_downloads([url, ], session=session, downloader=downloader, settings=settings,
|
||||
sub_downloader=sub_downloader, reset_attempts=reset_attempts)
|
||||
download.frequency = frequency
|
||||
download, = self.create_downloads([url, ], session=session, downloader_name=downloader_name,
|
||||
reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name,
|
||||
settings=settings)
|
||||
download.frequency = frequency
|
||||
session.commit()
|
||||
|
||||
return download
|
||||
|
||||
|
|
@ -1205,4 +1180,4 @@ class RSSDownloader(Downloader, ABC):
|
|||
return False
|
||||
|
||||
|
||||
rss_downloader = RSSDownloader(30)
|
||||
rss_downloader = RSSDownloader()
|
||||
|
|
|
|||
|
|
@ -45,6 +45,10 @@ class Events:
|
|||
def send_ready(message: str = None):
|
||||
send_event('ready', message)
|
||||
|
||||
@staticmethod
|
||||
def send_downloads_disabled(message: str = None):
|
||||
send_event('downloads_disabled', message, subject='downloads')
|
||||
|
||||
|
||||
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
|
||||
log = f'{event=}'
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media
|
|||
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics
|
||||
from wrolpi.dates import now, strptime
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.errors import WROLModeEnabled, API_ERRORS, APIError, ValidationError, HotspotError
|
||||
from wrolpi.events import get_events
|
||||
from wrolpi.errors import WROLModeEnabled, API_ERRORS, APIError, ValidationError, HotspotError, InvalidDownload
|
||||
from wrolpi.events import get_events, Events
|
||||
from wrolpi.files.lib import get_file_statistics
|
||||
from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
|
||||
truthy_arg
|
||||
|
|
@ -205,25 +205,30 @@ def valid_regex(_: Request, body: schema.RegexRequest):
|
|||
|
||||
|
||||
@api_bp.post('/download')
|
||||
@openapi.description('Download the many URLs that are provided.')
|
||||
@openapi.description('Download all the URLs that are provided.')
|
||||
@validate(schema.DownloadRequest)
|
||||
@wrol_mode_check
|
||||
async def post_download(_: Request, body: schema.DownloadRequest):
|
||||
# URLs are provided in a textarea, lets split all lines.
|
||||
urls = [i.strip() for i in str(body.urls).strip().splitlines()]
|
||||
downloader = download_manager.get_downloader_by_name(body.downloader)
|
||||
if not downloader:
|
||||
raise InvalidDownload(f'Cannot find downloader with name {body.downloader}')
|
||||
|
||||
excluded_urls = [i.strip() for i in body.excluded_urls.split(',')] if body.excluded_urls else None
|
||||
downloader = body.downloader
|
||||
if not downloader or downloader in ('auto', 'None', 'null'):
|
||||
downloader = None
|
||||
destination = str(get_media_directory() / body.destination) if body.destination else None
|
||||
settings = dict(excluded_urls=excluded_urls, destination=destination)
|
||||
if body.frequency:
|
||||
download_manager.recurring_download(urls[0], body.frequency, downloader=downloader,
|
||||
sub_downloader=body.sub_downloader, reset_attempts=True,
|
||||
download_manager.recurring_download(urls[0], body.frequency, downloader_name=body.downloader,
|
||||
sub_downloader_name=body.sub_downloader, reset_attempts=True,
|
||||
settings=settings)
|
||||
else:
|
||||
download_manager.create_downloads(urls, downloader=downloader, sub_downloader=body.sub_downloader,
|
||||
reset_attempts=True, settings=settings)
|
||||
download_manager.create_downloads(urls, downloader_name=body.downloader, reset_attempts=True,
|
||||
sub_downloader_name=body.sub_downloader, settings=settings)
|
||||
if download_manager.disabled:
|
||||
# Downloads are disabled, warn the user.
|
||||
Events.send_downloads_disabled('Download created. But, downloads are disabled.')
|
||||
|
||||
return response.empty()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ class EchoResponse:
|
|||
@dataclass
|
||||
class DownloadRequest:
|
||||
urls: str
|
||||
downloader: Optional[str] = None
|
||||
downloader: str
|
||||
frequency: Optional[int] = None
|
||||
sub_downloader: Optional[str] = None
|
||||
excluded_urls: Optional[str] = None
|
||||
|
|
|
|||
|
|
@ -18,74 +18,30 @@ from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeE
|
|||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
||||
class PermissiveDownloader(Downloader):
|
||||
"""A testing Downloader which always says it's valid."""
|
||||
name = 'permissive'
|
||||
|
||||
def __repr__(self):
|
||||
return '<TESTING Permissive Downloader>'
|
||||
|
||||
def valid_url(self, url: str):
|
||||
return True, None
|
||||
|
||||
|
||||
class HTTPDownloader(Downloader):
|
||||
"""A testing Downloader which says its valid when a URL starts with http/https"""
|
||||
name = 'http'
|
||||
|
||||
def __repr__(self):
|
||||
return '<TESTING HTTP Downloader>'
|
||||
|
||||
def valid_url(self, url: str):
|
||||
return url.startswith('https://') or url.startswith('http://'), None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
def test_register_downloader(test_session, test_download_manager):
|
||||
"""Downloaders can be registered and have specific priorities."""
|
||||
assert test_download_manager.instances == tuple()
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
assert test_download_manager.instances == (http_downloader,)
|
||||
|
||||
# PermissiveDownloader is first priority.
|
||||
permissive_downloader = PermissiveDownloader(priority=0)
|
||||
test_download_manager.register_downloader(permissive_downloader)
|
||||
assert test_download_manager.instances == (permissive_downloader, http_downloader)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
with pytest.raises(ValueError):
|
||||
test_download_manager.register_downloader(permissive_downloader)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_old_once_downloads(test_session, test_download_manager):
|
||||
async def test_delete_old_once_downloads(test_session, test_download_manager, test_downloader):
|
||||
"""Once-downloads over a month old should be deleted."""
|
||||
with mock.patch('wrolpi.downloader.now') as mock_now:
|
||||
mock_now.return_value = datetime(2020, 6, 5, 0, 0, tzinfo=pytz.UTC)
|
||||
permissive_downloader = PermissiveDownloader(priority=0)
|
||||
test_download_manager.register_downloader(permissive_downloader)
|
||||
|
||||
# Recurring downloads should not be deleted.
|
||||
d1 = test_download_manager.create_download('https://example.com/1')
|
||||
d2 = test_download_manager.create_download('https://example.com/2')
|
||||
d1 = test_download_manager.create_download('https://example.com/1', test_downloader.name)
|
||||
d2 = test_download_manager.create_download('https://example.com/2', test_downloader.name)
|
||||
d1.frequency = 1
|
||||
d2.frequency = 1
|
||||
d2.started()
|
||||
# Should be deleted.
|
||||
d3 = test_download_manager.create_download('https://example.com/3')
|
||||
d4 = test_download_manager.create_download('https://example.com/4')
|
||||
d3 = test_download_manager.create_download('https://example.com/3', test_downloader.name)
|
||||
d4 = test_download_manager.create_download('https://example.com/4', test_downloader.name)
|
||||
d3.complete()
|
||||
d4.complete()
|
||||
d3.last_successful_download = datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
|
||||
d4.last_successful_download = datetime(2020, 5, 1, 0, 0, 0, tzinfo=pytz.UTC)
|
||||
# Not a month old.
|
||||
d5 = test_download_manager.create_download('https://example.com/5')
|
||||
d5 = test_download_manager.create_download('https://example.com/5', test_downloader.name)
|
||||
d5.last_successful_download = datetime(2020, 6, 1, 0, 0, 0, tzinfo=pytz.UTC)
|
||||
# An old, but pending download should not be deleted.
|
||||
d6 = test_download_manager.create_download('https://example.com/6')
|
||||
d6 = test_download_manager.create_download('https://example.com/6', test_downloader.name)
|
||||
d6.last_successful_download = datetime(2020, 4, 1, 0, 0, 0, tzinfo=pytz.UTC)
|
||||
d6.started()
|
||||
|
||||
|
|
@ -104,15 +60,13 @@ async def test_delete_old_once_downloads(test_session, test_download_manager):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_download(test_session, test_download_manager):
|
||||
async def test_create_download(test_session, test_download_manager, test_downloader):
|
||||
assert len(test_download_manager.get_downloads(test_session)) == 0
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download = MagicMock()
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
test_downloader.do_download = MagicMock()
|
||||
test_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
|
||||
test_download_manager.create_download('https://example.com')
|
||||
test_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
download = test_download_manager.get_download(test_session, 'https://example.com')
|
||||
assert download.url == 'https://example.com'
|
||||
|
||||
|
|
@ -122,22 +76,11 @@ async def test_ensure_download(test_session, test_download_manager):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_downloads(test_session, test_download_manager):
|
||||
async def test_create_downloads(test_session, test_download_manager, test_downloader, assert_download_urls):
|
||||
"""Multiple downloads can be scheduled using DownloadManager.create_downloads."""
|
||||
http_downloader = HTTPDownloader(priority=0)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
# Both URLs are valid and are scheduled.
|
||||
test_download_manager.create_downloads(['https://example.com/1', 'https://example.com/2'])
|
||||
downloads = test_download_manager.get_downloads(test_session)
|
||||
assert {i.url for i in downloads} == {'https://example.com/1', 'https://example.com/2'}
|
||||
|
||||
# One URL is bad, neither should be scheduled.
|
||||
with pytest.raises(InvalidDownload):
|
||||
test_download_manager.create_downloads(['https://example.com/3', 'bad url should fail'])
|
||||
|
||||
downloads = test_download_manager.get_downloads(test_session)
|
||||
assert {i.url for i in downloads} == {'https://example.com/1', 'https://example.com/2'}
|
||||
# Both URLs are scheduled.
|
||||
test_download_manager.create_downloads(['https://example.com/1', 'https://example.com/2'], test_downloader.name)
|
||||
assert_download_urls({'https://example.com/1', 'https://example.com/2'})
|
||||
|
||||
|
||||
def test_downloader_must_have_name():
|
||||
|
|
@ -164,21 +107,15 @@ async def test_download_wrol_mode(test_session, test_download_manager):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_get_downloader(test_session, test_download_manager):
|
||||
async def test_download_get_downloader(test_session, test_download_manager, test_downloader):
|
||||
"""
|
||||
A Download can find it's Downloader.
|
||||
"""
|
||||
permissive_downloader = PermissiveDownloader()
|
||||
http_downloader = HTTPDownloader()
|
||||
test_download_manager.register_downloader(permissive_downloader)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
download1 = test_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
assert download1.get_downloader() == test_downloader
|
||||
|
||||
download1 = test_download_manager.create_download('https://example.com')
|
||||
assert download1.get_downloader() == permissive_downloader
|
||||
|
||||
download2 = test_download_manager.create_download('https://example.com')
|
||||
download2.downloader = 'bad downloader'
|
||||
assert download2.get_downloader() is None
|
||||
with pytest.raises(InvalidDownload):
|
||||
download2 = test_download_manager.create_download('https://example.com', 'bad downloader')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -208,22 +145,20 @@ def test_calculate_next_download(test_session, test_download_manager, fake_now):
|
|||
d3 = Download(url='https://example.com/3', frequency=DownloadFrequency.weekly)
|
||||
test_session.add_all([d1, d2, d3])
|
||||
test_session.commit()
|
||||
# Downloads are spread out over the next week.
|
||||
# Downloads are ead out over the next week.
|
||||
assert test_download_manager.calculate_next_download(d1) == datetime(2000, 1, 8, tzinfo=pytz.UTC)
|
||||
assert test_download_manager.calculate_next_download(d2) == datetime(2000, 1, 11, 12, tzinfo=pytz.UTC)
|
||||
assert test_download_manager.calculate_next_download(d3) == datetime(2000, 1, 9, 18, tzinfo=pytz.UTC)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recurring_downloads(test_session, test_download_manager, fake_now):
|
||||
async def test_recurring_downloads(test_session, test_download_manager, fake_now, test_downloader):
|
||||
"""A recurring Download should be downloaded forever."""
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download = MagicMock()
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
test_downloader.do_download = MagicMock()
|
||||
test_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
|
||||
# Download every hour.
|
||||
test_download_manager.recurring_download('https://example.com', Seconds.hour)
|
||||
test_download_manager.recurring_download('https://example.com', Seconds.hour, test_downloader.name)
|
||||
|
||||
# One download is scheduled.
|
||||
downloads = test_download_manager.get_new_downloads(test_session)
|
||||
|
|
@ -233,7 +168,7 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
|
|||
|
||||
# Download is processed and successful, no longer "new".
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
test_downloader.do_download.assert_called_once()
|
||||
assert list(test_download_manager.get_new_downloads(test_session)) == []
|
||||
downloads = list(test_download_manager.get_recurring_downloads(test_session))
|
||||
assert len(downloads) == 1
|
||||
|
|
@ -258,10 +193,10 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
|
|||
assert download.status == 'new'
|
||||
|
||||
# Try the download, but it fails.
|
||||
http_downloader.do_download.reset_mock()
|
||||
http_downloader.do_download.return_value = DownloadResult(success=False)
|
||||
test_downloader.do_download.reset_mock()
|
||||
test_downloader.do_download.return_value = DownloadResult(success=False)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
test_downloader.do_download.assert_called_once()
|
||||
download = test_session.query(Download).one()
|
||||
# Download is deferred, last successful download remains the same.
|
||||
assert download.status == 'deferred'
|
||||
|
|
@ -271,12 +206,12 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
|
|||
assert download.next_download == expected
|
||||
|
||||
# Try the download again, it finally succeeds.
|
||||
http_downloader.do_download.reset_mock()
|
||||
test_downloader.do_download.reset_mock()
|
||||
now_ = fake_now(datetime(2020, 1, 1, 4, 0, 1, tzinfo=pytz.UTC))
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_download_manager.renew_recurring_downloads(test_session)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
test_downloader.do_download.assert_called_once()
|
||||
download = test_session.query(Download).one()
|
||||
assert download.status == 'complete'
|
||||
assert download.last_successful_download == now_
|
||||
|
|
@ -285,28 +220,26 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_attempts(test_session, test_download_manager):
|
||||
async def test_max_attempts(test_session, test_download_manager, test_downloader):
|
||||
"""A Download will only be attempted so many times, it will eventually be deleted."""
|
||||
_, session = get_db_context()
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download = MagicMock()
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
test_downloader.do_download = MagicMock()
|
||||
test_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
|
||||
test_download_manager.create_download('https://example.com')
|
||||
test_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
download = session.query(Download).one()
|
||||
assert download.attempts == 1
|
||||
|
||||
test_download_manager.create_download('https://example.com')
|
||||
test_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
download = session.query(Download).one()
|
||||
assert download.attempts == 2
|
||||
|
||||
# There are no further attempts.
|
||||
http_downloader.do_download.side_effect = UnrecoverableDownloadError()
|
||||
test_download_manager.create_download('https://example.com')
|
||||
test_downloader.do_download.side_effect = UnrecoverableDownloadError()
|
||||
test_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
download = session.query(Download).one()
|
||||
assert download.attempts == 3
|
||||
|
|
@ -314,22 +247,20 @@ async def test_max_attempts(test_session, test_download_manager):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skip_urls(test_session, test_download_manager, test_client, assert_download_urls):
|
||||
async def test_skip_urls(test_session, test_download_manager, test_client, assert_download_urls, test_downloader):
|
||||
"""The DownloadManager will not create downloads for URLs in it's skip list."""
|
||||
_, session = get_db_context()
|
||||
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
|
||||
DOWNLOAD_MANAGER_CONFIG.skip_urls = ['https://example.com/skipme']
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download = MagicMock()
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
test_downloader.do_download = MagicMock()
|
||||
test_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
|
||||
test_download_manager.create_downloads([
|
||||
'https://example.com/1',
|
||||
'https://example.com/skipme',
|
||||
'https://example.com/2',
|
||||
], downloader=HTTPDownloader.name)
|
||||
], downloader_name=test_downloader.name)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
assert_download_urls({'https://example.com/1', 'https://example.com/2'})
|
||||
assert DOWNLOAD_MANAGER_CONFIG.skip_urls == ['https://example.com/skipme']
|
||||
|
|
@ -337,7 +268,7 @@ async def test_skip_urls(test_session, test_download_manager, test_client, asser
|
|||
test_download_manager.delete_completed()
|
||||
|
||||
# The user can start a download even if the URL is in the skip list.
|
||||
test_download_manager.create_download('https://example.com/skipme', reset_attempts=True)
|
||||
test_download_manager.create_download('https://example.com/skipme', test_downloader.name, reset_attempts=True)
|
||||
assert_download_urls({'https://example.com/skipme'})
|
||||
assert DOWNLOAD_MANAGER_CONFIG.skip_urls == []
|
||||
|
||||
|
|
@ -346,7 +277,7 @@ async def test_skip_urls(test_session, test_download_manager, test_client, asser
|
|||
async def test_process_runner_timeout(test_directory):
|
||||
"""A Downloader can cancel it's download using a timeout."""
|
||||
# Default timeout of 3 seconds.
|
||||
downloader = Downloader(0, 'downloader', timeout=3)
|
||||
downloader = Downloader('downloader', timeout=3)
|
||||
|
||||
# Default timeout is obeyed.
|
||||
start = datetime.now()
|
||||
|
|
@ -369,14 +300,11 @@ async def test_process_runner_timeout(test_directory):
|
|||
assert 2 < elapsed.total_seconds() < 4
|
||||
|
||||
|
||||
def test_crud_download(test_client, test_session, test_download_manager):
|
||||
def test_crud_download(test_client, test_session, test_download_manager, test_downloader):
|
||||
"""Test the ways that Downloads can be created."""
|
||||
http_downloader = HTTPDownloader()
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
body = dict(
|
||||
urls='https://example.com',
|
||||
downloader=http_downloader.name,
|
||||
downloader=test_downloader.name,
|
||||
frequency=DownloadFrequency.weekly,
|
||||
excluded_urls='example.org,something',
|
||||
)
|
||||
|
|
@ -386,7 +314,7 @@ def test_crud_download(test_client, test_session, test_download_manager):
|
|||
download: Download = test_session.query(Download).one()
|
||||
assert download.id
|
||||
assert download.url == 'https://example.com'
|
||||
assert download.downloader == 'http'
|
||||
assert download.downloader == test_downloader.name
|
||||
assert download.frequency == DownloadFrequency.weekly
|
||||
assert download.settings['excluded_urls'] == ['example.org', 'something']
|
||||
|
||||
|
|
@ -409,33 +337,23 @@ def test_crud_download(test_client, test_session, test_download_manager):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_downloads_config(test_session, test_client, test_download_manager, test_download_manager_config):
|
||||
async def test_downloads_config(test_session, test_client, test_download_manager, test_download_manager_config,
|
||||
test_downloader, assert_downloads):
|
||||
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
|
||||
|
||||
def assert_downloads(expected_):
|
||||
assert test_session.query(Download).count() == len(expected_), 'Download count does not match'
|
||||
downloads = test_session.query(Download).order_by(Download.url)
|
||||
for download_, expected_ in zip_longest(downloads, expected_):
|
||||
assert_dict_contains(download_.__json__(), expected_)
|
||||
|
||||
# Can import with an empty config.
|
||||
await import_downloads_config()
|
||||
assert_downloads([])
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download = MagicMock()
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
test_downloader.do_download = MagicMock()
|
||||
test_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
|
||||
download1, download2 = test_download_manager.create_downloads([
|
||||
'https://example.com/1',
|
||||
'https://example.com/2',
|
||||
], downloader=HTTPDownloader.name)
|
||||
test_download_manager.recurring_download(
|
||||
'https://example.com/3',
|
||||
frequency=DownloadFrequency.weekly,
|
||||
downloader=HTTPDownloader.name,
|
||||
)
|
||||
], downloader_name=test_downloader.name)
|
||||
test_download_manager.recurring_download('https://example.com/3', frequency=DownloadFrequency.weekly,
|
||||
downloader_name=test_downloader.name)
|
||||
download2.next_download = datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC)
|
||||
# Completed once-downloads should be ignored.
|
||||
download1.last_successful_download = datetime(2000, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
|
||||
|
|
@ -452,13 +370,13 @@ async def test_downloads_config(test_session, test_client, test_download_manager
|
|||
url='https://example.com/2',
|
||||
frequency=None,
|
||||
next_download=datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC),
|
||||
downloader='http',
|
||||
downloader='test_downloader',
|
||||
sub_downloader=None,
|
||||
),
|
||||
dict(
|
||||
url='https://example.com/3',
|
||||
frequency=DownloadFrequency.weekly,
|
||||
downloader='http',
|
||||
downloader='test_downloader',
|
||||
sub_downloader=None,
|
||||
),
|
||||
]
|
||||
|
|
@ -476,13 +394,13 @@ async def test_downloads_config(test_session, test_client, test_download_manager
|
|||
url='https://example.com/2',
|
||||
frequency=None,
|
||||
next_download=datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC),
|
||||
downloader='http',
|
||||
downloader='test_downloader',
|
||||
sub_downloader=None,
|
||||
),
|
||||
dict(
|
||||
url='https://example.com/3',
|
||||
frequency=DownloadFrequency.weekly,
|
||||
downloader='http',
|
||||
downloader='test_downloader',
|
||||
sub_downloader=None,
|
||||
),
|
||||
]
|
||||
|
|
@ -494,12 +412,10 @@ async def test_downloads_config(test_session, test_client, test_download_manager
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_excluded_urls(test_client, test_session, test_download_manager):
|
||||
async def test_download_excluded_urls(test_client, test_session, test_download_manager, test_downloader):
|
||||
"""Test that URLs that have been excluded are ignored by the download worker."""
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.already_downloaded = lambda *i, **kw: []
|
||||
http_downloader.do_download = lambda *i, **kw: DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
test_downloader.already_downloaded = lambda *i, **kw: []
|
||||
test_downloader.do_download = lambda *i, **kw: DownloadResult(success=True)
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
|
||||
|
|
@ -514,9 +430,9 @@ async def test_download_excluded_urls(test_client, test_session, test_download_m
|
|||
]
|
||||
)
|
||||
settings = dict(excluded_urls=['example.org', 'example.gov'])
|
||||
feed_download: Download = test_download_manager.create_download('https://example.com/feed',
|
||||
test_session, sub_downloader='http',
|
||||
downloader=RSSDownloader.name,
|
||||
feed_download: Download = test_download_manager.create_download('https://example.com/feed', rss_downloader.name,
|
||||
test_session,
|
||||
sub_downloader_name=test_downloader.name,
|
||||
settings=settings)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
||||
|
|
|
|||
|
|
@ -5,11 +5,12 @@ import mock
|
|||
import pytest
|
||||
|
||||
from wrolpi.db import optional_session
|
||||
from wrolpi.downloader import Download, DownloadResult, RSSDownloader
|
||||
from wrolpi.test.test_downloader import HTTPDownloader
|
||||
from wrolpi.downloader import Download, DownloadResult, RSSDownloader, Downloader
|
||||
|
||||
|
||||
class RSSHTTPDownloader(HTTPDownloader):
|
||||
class RSSHTTPDownloader(Downloader):
|
||||
name = 'rss_http'
|
||||
|
||||
def do_download(self, download: Download) -> DownloadResult:
|
||||
return DownloadResult(success=True)
|
||||
|
||||
|
|
@ -44,7 +45,8 @@ async def test_rss_download(test_session, test_download_manager):
|
|||
dict(link='https://example.com/c'),
|
||||
]
|
||||
)
|
||||
test_download_manager.create_download('https://example.com/feed', sub_downloader='http')
|
||||
test_download_manager.create_download('https://example.com/feed', rss_downloader.name,
|
||||
sub_downloader_name='rss_http')
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
||||
# Feed download is complete.
|
||||
|
|
@ -63,7 +65,8 @@ async def test_rss_download(test_session, test_download_manager):
|
|||
dict(link='https://example.com/d'),
|
||||
]
|
||||
)
|
||||
test_download_manager.create_download('https://example.com/feed', sub_downloader='http')
|
||||
test_download_manager.create_download('https://example.com/feed', rss_downloader.name,
|
||||
sub_downloader_name='rss_http')
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
||||
# Only the new URLs are Archived.
|
||||
|
|
@ -84,7 +87,8 @@ async def test_rss_no_entries(test_session, test_download_manager):
|
|||
|
||||
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
|
||||
mock_parse_feed.return_value = dict(bozo=0, ) # missing `entries`
|
||||
test_download_manager.create_download('https://example.com/feed', test_session, sub_downloader='http')
|
||||
test_download_manager.create_download('https://example.com/feed', rss_downloader.name, test_session,
|
||||
sub_downloader_name='rss_http')
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
||||
(download,) = test_download_manager.get_downloads(test_session)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue