Requiring downloader name to be specified.

This commit is contained in:
Roland 2023-04-01 15:34:46 -06:00
parent 5692ddb20d
commit 55cf76a884
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
15 changed files with 215 additions and 320 deletions

View file

@ -23,6 +23,10 @@ function handleEvents(events) {
eventToast('Refresh completed', 'All files have been refreshed.');
}
if (event === 'downloads_disabled') {
eventToast('Downloads Disabled', message, 'info');
}
if (subject) {
newestEvents[subject] = dt;
}

View file

@ -412,6 +412,16 @@ export async function getArchive(archiveId) {
}
export async function postDownload(urls, downloader, frequency, sub_downloader, excludedURLs, destination) {
if (!downloader) {
toast({
type: 'error',
title: 'Failed to submit download',
description: 'downloader is required, but was not provided',
time: 5000,
})
throw new Error('downloader is required, but was not provided');
}
let body = {
urls: urls,
downloader: downloader,

View file

@ -208,7 +208,7 @@ class StoppableRow extends React.Component {
handleStart = async (e) => {
e.preventDefault();
let downloader = this.props.downloader || null;
let downloader = this.props.downloader;
await postDownload(`${this.props.url}`, downloader);
this.closeStart();
await this.props.fetchDownloads();

View file

@ -53,7 +53,7 @@ class ArchiveDownloader(Downloader, ABC):
# Archive downloader is the last downloader which should be used.
archive_downloader = ArchiveDownloader(priority=100)
archive_downloader = ArchiveDownloader()
def model_archive(file_group: FileGroup, session: Session = None) -> Archive:

View file

@ -423,7 +423,8 @@ def test_change_channel_url(test_client, test_session, test_download_manager, do
test_session.commit()
download = test_session.query(Download).one()
assert download.url == download_channel.url == 'https://example.com/new-url', "Channel's download URL was not changed."
assert download.url == download_channel.url == 'https://example.com/new-url', \
"Channel's download URL was not changed."
download_channel.update({'download_frequency': None})
assert not list(test_session.query(Download).all())
@ -456,13 +457,15 @@ def test_search_videos_channel(test_client, test_session, video_factory):
assert response.status_code == HTTPStatus.OK
assert len(response.json['file_groups']) == 1
assert response.json['totals']['file_groups'] == 1
assert_dict_contains(response.json['file_groups'][0], dict(primary_path='vid2.mp4', video=dict(channel_id=channel1.id)))
assert_dict_contains(response.json['file_groups'][0],
dict(primary_path='vid2.mp4', video=dict(channel_id=channel1.id)))
d = dict(channel_id=channel2.id)
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['file_groups']) == 1
assert_dict_contains(response.json['file_groups'][0], dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
assert_dict_contains(response.json['file_groups'][0],
dict(primary_path='vid1.mp4', video=dict(channel_id=channel2.id)))
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory, assert_video_search):

View file

@ -64,7 +64,8 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch
url='https://example.com/channel1',
download_frequency=DownloadFrequency.weekly,
)
download = Download(url=channel.url, downloader='video_channel', frequency=channel.download_frequency)
download = Download(url=channel.url, downloader='video_channel', frequency=channel.download_frequency,
sub_downloader='video')
test_session.add_all([channel, download])
test_session.commit()
return channel
@ -140,7 +141,7 @@ def video_factory(test_session, test_directory):
def video_download_manager(test_download_manager) -> DownloadManager:
"""Get a DownloadManager ready to download Videos and Channels."""
channel_downloader = ChannelDownloader()
video_downloader = VideoDownloader(priority=40)
video_downloader = VideoDownloader()
test_download_manager.register_downloader(channel_downloader)
test_download_manager.register_downloader(video_downloader)

View file

@ -393,7 +393,7 @@ class VideoDownloader(Downloader, ABC):
channel_downloader = ChannelDownloader()
# Videos may match the ChannelDownloader, give it a higher priority.
video_downloader = VideoDownloader(40)
video_downloader = VideoDownloader()
def get_or_create_channel(source_id: str = None, url: str = None, name: str = None) -> Channel:

View file

@ -10,11 +10,9 @@ from modules.videos.channel.lib import download_channel
from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \
get_or_create_channel, channel_downloader, video_downloader
from modules.videos.models import Channel, Video
from wrolpi.db import get_db_context
from wrolpi.downloader import Download, DownloadResult
from wrolpi.errors import InvalidDownload
from wrolpi.test.common import skip_circleci
from wrolpi.test.test_downloader import HTTPDownloader
from wrolpi.vars import PROJECT_DIR
@ -133,21 +131,6 @@ def test_channel_valid_url():
assert channel_downloader.valid_url('https://www.youtube.com/playlist?list=PLCdlMQeP-TbG12nkBCt0E96yr3EfwcH4E')[0]
@pytest.mark.asyncio
@pytest.mark.parametrize(
'url,expected', [
('https://www.youtube.com/c/LearningSelfReliance/videos', channel_downloader),
('https://www.youtube.com/c/LearningSelfReliance', channel_downloader),
('https://www.youtube.com/watch?v=31jPEBiAC3c', video_downloader),
]
)
async def test_get_downloader(test_session, test_download_manager, url, expected):
"""The correct Downloader is gotten."""
test_download_manager.register_downloader(channel_downloader)
test_download_manager.register_downloader(video_downloader)
assert test_download_manager.get_downloader(url)[0] == expected
@pytest.mark.asyncio
async def test_download_no_channel(test_session, video_download_manager, video_factory, test_directory,
mock_video_extract_info, mock_video_process_runner):
@ -157,6 +140,8 @@ async def test_download_no_channel(test_session, video_download_manager, video_f
video_path = channel_dir / 'video.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
channel_downloader, video_downloader = video_download_manager.instances
# Video has no channel
info_json = copy(example_video_json)
del info_json['channel']
@ -167,7 +152,7 @@ async def test_download_no_channel(test_session, video_download_manager, video_f
mock_video_extract_info.return_value = info_json
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
video_download_manager.create_download(url)
video_download_manager.create_download(url, video_downloader.name)
await video_download_manager.wait_for_all_downloads()
mock_video_process_runner.assert_called_once()
@ -186,11 +171,13 @@ async def test_download_channel(test_session, simple_channel, video_download_man
If a Channel has `match_regex` only those videos with matching titles will be downloaded."""
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
channel_downloader, video_downloader = video_download_manager.instances
mock_video_prepare_filename.return_value = video_file
mock_video_extract_info.return_value = example_channel_json
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
mock_get_channel.return_value = simple_channel
video_download_manager.create_download(url)
video_download_manager.create_download(url, channel_downloader.name, sub_downloader_name=video_downloader.name)
# Download channel.
await video_download_manager.wait_for_all_downloads()
# Download videos.
@ -212,7 +199,8 @@ async def test_download_channel(test_session, simple_channel, video_download_man
test_session.commit()
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
mock_get_channel.return_value = simple_channel
video_download_manager.recurring_download(url, 100)
video_download_manager.recurring_download(url, 100, channel_downloader.name,
sub_downloader_name=video_downloader.name)
await video_download_manager.wait_for_all_downloads()
downloads = video_download_manager.get_once_downloads(test_session)
assert [i.url for i in downloads] == ['https://youtube.com/watch?v=video_2_url']
@ -260,31 +248,13 @@ def test_get_or_create_channel(test_session):
@pytest.mark.asyncio
async def test_download_info_save(test_session, video_download_manager, mock_video_extract_info,
mock_video_prepare_filename,
mock_video_process_runner):
"""
An info dict from Downloader.valid_url is saved to the Downloader and can be used by the Downloader later.
"""
mock_video_extract_info.return_value = {'some': 'info'}
mock_video_prepare_filename.return_value = 'some path'
video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80')
await video_download_manager.wait_for_all_downloads()
assert mock_video_extract_info.call_count == 2 # Only called twice. The "info" gathering call was skipped.
@pytest.mark.asyncio
async def test_selected_downloader(test_session, video_download_manager, successful_download):
async def test_selected_downloader(test_session, video_download_manager, test_downloader, successful_download):
"""
A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
"""
# The test HTTP Downloader will claim to be able to handle any HTTP URL; will pretend to have downloaded anything.
http_downloader = HTTPDownloader()
mock_do_download = http_downloader.do_download = mock.MagicMock()
# The test Downloader will pretend to have downloaded anything.
mock_do_download = test_downloader.do_download = mock.MagicMock()
mock_do_download.return_value = successful_download
mock_http_valid_url = http_downloader.valid_url = mock.MagicMock()
mock_http_valid_url.return_value = (True, None)
video_download_manager.register_downloader(http_downloader)
def check_attempts(attempts):
assert test_session.query(Download).one().attempts == attempts
@ -292,42 +262,30 @@ async def test_selected_downloader(test_session, video_download_manager, success
# DownloadManager will attempt to choose the Downloader.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_extract_info.side_effect = UnsupportedError('not this one')
video_download_manager.create_download('https://example.com')
video_download_manager.create_download('https://example.com', test_downloader.name)
await video_download_manager.wait_for_all_downloads()
mock_http_valid_url.assert_called_once()
mock_extract_info.assert_called_once()
mock_do_download.assert_called_once()
check_attempts(1)
mock_http_valid_url.reset_mock()
mock_do_download.reset_mock()
# DownloadManager will only use the HTTPDownloader.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
video_download_manager.create_download('https://example.com', downloader='http')
await video_download_manager.wait_for_all_downloads()
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
mock_extract_info.assert_not_called()
mock_http_valid_url.assert_not_called()
# DownloadManager called do_download.
mock_do_download.assert_called_once()
# DownloadManager will only use the TestDownloader.
video_download_manager.create_download('https://example.com', test_downloader.name)
await video_download_manager.wait_for_all_downloads()
# DownloadManager called do_download.
mock_do_download.assert_called_once()
check_attempts(2)
mock_http_valid_url.reset_mock()
mock_do_download.reset_mock()
# DownloadManager will only use the HTTPDownloader, even if the download fails.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_do_download.side_effect = Exception('oh no!')
video_download_manager.create_download('https://example.com', downloader='http')
await video_download_manager.wait_for_all_downloads()
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
mock_extract_info.assert_not_called()
mock_http_valid_url.assert_not_called()
# DownloadManager called do_download.
mock_do_download.assert_called_once()
# DownloadManager will only use the TestDownloader, even if the download fails.
mock_do_download.side_effect = Exception('oh no!')
video_download_manager.create_download('https://example.com', test_downloader.name)
await video_download_manager.wait_for_all_downloads()
# DownloadManager called do_download.
mock_do_download.assert_called_once()
check_attempts(3)
@ -346,7 +304,7 @@ def test_bad_downloader(test_session, video_download_manager):
Attempting to use an unknown downloader should raise an error.
"""
with pytest.raises(InvalidDownload):
video_download_manager.create_download('https://example.com', downloader='bad downloader')
video_download_manager.create_download('https://example.com', downloader_name='bad downloader')
def test_channel_download_no_download(test_session, video_download_manager, simple_channel):
@ -361,24 +319,6 @@ def test_channel_download_no_download(test_session, video_download_manager, simp
download_channel(simple_channel.id)
@pytest.mark.asyncio
async def test_invalid_download_url(test_session, test_download_manager, mock_video_extract_info):
"""An invalid url should not be attempted again."""
_, session = get_db_context()
with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \
mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader:
video_downloader = VideoDownloader()
test_download_manager.register_downloader(video_downloader)
mock_get_downloader.return_value = (video_downloader, None)
mock_valid_url.return_value = (True, {}) # url is valid, but is not a video URL.
mock_video_extract_info.side_effect = UnsupportedError('oops')
test_download_manager.create_download('invalid url', downloader=VideoDownloader.name)
await test_download_manager.wait_for_all_downloads()
download = session.query(Download).one()
assert download.status == 'failed'
@pytest.mark.asyncio
async def test_video_download(test_session, test_directory, simple_channel, video_download_manager,
mock_video_process_runner, image_file):
@ -399,7 +339,7 @@ async def test_video_download(test_session, test_directory, simple_channel, vide
mock_extract_info.return_value = example_video_json
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
video_download_manager.create_download(url)
video_download_manager.create_download(url, video_downloader.name)
await video_download_manager.wait_for_all_downloads()
mock_video_process_runner.assert_called_once()
@ -427,7 +367,7 @@ async def test_download_result(test_session, test_directory, video_download_mana
mock_video_extract_info.return_value = example_video_json
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
mock_prepare_filename.return_value = [video_file, {'id': 'foo'}]
video_download_manager.create_download('https://example.com')
video_download_manager.create_download('https://example.com', video_downloader.name)
await video_download_manager.wait_for_all_downloads()
download: Download = test_session.query(Download).one()
@ -451,7 +391,7 @@ async def test_download_destination(test_session, test_directory, video_download
# This result will be ignored during the test, we just need some value so the download does not fail.
mock_video_prepare_filename.return_value = str(test_directory / 'test video.mp4')
video_download_manager.create_download('https://example.com/1', downloader=VideoDownloader.name)
video_download_manager.create_download('https://example.com/1', downloader_name=VideoDownloader.name)
await video_download_manager.wait_for_all_downloads()
# Output directory matches the channel directory.
assert mock_video_prepare_filename.call_args_list[0].kwargs['ydl'].params['outtmpl']['default'] \
@ -460,7 +400,8 @@ async def test_download_destination(test_session, test_directory, video_download
mock_video_prepare_filename.reset_mock()
settings = dict(destination=f'{test_directory}/custom')
video_download_manager.create_download('https://example.com/2', downloader=VideoDownloader.name, settings=settings)
video_download_manager.create_download('https://example.com/2', downloader_name=VideoDownloader.name,
settings=settings)
await video_download_manager.wait_for_all_downloads()
# Output directory matches the custom directory specified.
assert mock_video_prepare_filename.call_args_list[0].kwargs['ydl'].params['outtmpl']['default'] \

View file

@ -9,6 +9,7 @@ import multiprocessing
import pathlib
import shutil
import tempfile
from abc import ABC
from itertools import zip_longest
from typing import List, Callable, Union, Dict
from typing import Tuple, Set
@ -29,7 +30,7 @@ from wrolpi.common import iterify, log_level_context
from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.dates import set_test_now
from wrolpi.db import postgres_engine, get_db_args
from wrolpi.downloader import DownloadManager, DownloadResult, set_test_download_manager_config, Download
from wrolpi.downloader import DownloadManager, DownloadResult, set_test_download_manager_config, Download, Downloader
from wrolpi.files import lib as files_lib
from wrolpi.root_api import BLUEPRINTS, api_app
from wrolpi.tags import Tag
@ -179,13 +180,44 @@ def failed_download():
@pytest.fixture
def assert_download_urls(test_session):
def asserter(urls: Set[str]):
def asserter(expected_urls: Set[str]):
downloads = test_session.query(Download).all()
assert {i.url for i in downloads} == set(urls)
urls = {i.url for i in downloads}
if urls != set(expected_urls):
raise AssertionError(f'Download URLs do not match: {urls} != {expected_urls}')
return asserter
@pytest.fixture
def assert_downloads(test_session):
from wrolpi.test.common import assert_dict_contains
def asserter(expected: List[Dict]):
downloads = test_session.query(Download).order_by(Download.url)
for download_, expected_ in zip_longest(downloads, expected):
assert_dict_contains(download_.__json__(), expected_)
if (count := test_session.query(Download).count()) != len(expected):
raise AssertionError(f'Download count does not match: {count} != {len(expected)}')
return asserter
@pytest.fixture
def test_downloader(test_download_manager):
class TestDownloader(Downloader, ABC):
"""A testing Downloader"""
name = 'test_downloader'
def __repr__(self):
return '<TESTING Downloader>'
test_downloader = TestDownloader()
test_download_manager.register_downloader(test_downloader)
return test_downloader
@pytest.fixture
def video_file(test_directory) -> pathlib.Path:
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""

View file

@ -130,15 +130,14 @@ class Download(ModelHelper, Base): # noqa
if self.downloader:
return self.manager.get_downloader_by_name(self.downloader)
downloader, _ = self.manager.get_downloader(self.url)
return downloader
raise UnrecoverableDownloadError(f'Cannot find downloader for {repr(str(self.url))}')
@property
def domain(self):
return urlparse(self.url).netloc
@property
def manager(self):
def manager(self) -> 'DownloadManager':
if self._manager:
return self._manager
raise ValueError('No manager has been set!')
@ -163,19 +162,11 @@ class Downloader:
listable: bool = True
timeout: int = None
def __init__(self, priority: int = 50, name: str = None, timeout: int = None):
"""
Lower `priority` means the downloader will be checked first. Valid priority: 0-100
Downloaders of equal priority will be used at random.
"""
def __init__(self, name: str = None, timeout: int = None):
if not name and not self.name:
raise NotImplementedError('Downloader must have a name!')
if not 0 <= priority <= 100:
raise ValueError(f'Priority of {priority} of {self} is invalid!')
self.name: str = self.name or name
self.priority: int = priority
self.timeout: int = timeout or self.timeout
self._kill = multiprocessing.Event()
@ -288,7 +279,6 @@ class DownloadManager:
Only one download from each domain will be downloaded at a time. If we only have one domain (example.com) in the
URLs to be downloaded, then only one worker will be busy.
"""
priority_sorter = partial(sorted, key=attrgetter('priority'))
manager = multiprocessing.Event()
def __init__(self):
@ -318,8 +308,7 @@ class DownloadManager:
instance.manager = self
i = (*self.instances, instance)
self.instances = tuple(self.priority_sorter(i))
self.instances = (*self.instances, instance)
self._instances[instance.name] = instance
async def download_worker(self, num: int):
@ -404,10 +393,8 @@ class DownloadManager:
if result.downloads:
worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}')
urls = download.filter_excluded(result.downloads)
downloads = self.create_downloads(urls, session,
downloader=download.sub_downloader,
settings=result.settings,
)
self.create_downloads(urls, session, downloader_name=download.sub_downloader,
settings=result.settings)
if try_again is False and not download.frequency:
# Only once-downloads can fail.
@ -511,16 +498,11 @@ class DownloadManager:
background_task(_perpetual_download())
def get_downloader(self, url: str) -> Tuple[Downloader, Dict]:
for i in self.instances:
valid, info = i.valid_url(url)
if valid:
return i, info
raise InvalidDownload(f'Invalid URL {url=}')
def get_downloader_by_name(self, name: str) -> Optional[Downloader]:
"""Attempt to find a registered Downloader by its name. Returns None if it cannot be found."""
return self._instances.get(name)
if downloader := self._instances.get(name):
return downloader
raise InvalidDownload(f'Cannot find downloader with name {name}')
def get_or_create_download(self, url: str, session: Session) -> Download:
"""Get a Download by its URL, if it cannot be found create one."""
@ -539,18 +521,16 @@ class DownloadManager:
return download
@optional_session
def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None,
reset_attempts: bool = False, sub_downloader: str = None, settings: dict = None) \
def create_downloads(self, urls: List[str], downloader_name: str, session: Session = None,
reset_attempts: bool = False, sub_downloader_name: str = None, settings: dict = None) \
-> List[Download]:
"""Schedule all URLs for download. If one cannot be downloaded, none will be added."""
if not all(urls):
raise ValueError('Download must have a URL')
downloads = []
shared_downloader = self.get_downloader_by_name(downloader) if downloader else None
if downloader and not shared_downloader:
# Could not look up the downloader.
raise InvalidDownload(f'Unknown downloader {downloader}')
# Throws an error if no downloader is found.
self.get_downloader_by_name(downloader_name)
with session.transaction:
for url in urls:
@ -560,20 +540,12 @@ class DownloadManager:
elif url in DOWNLOAD_MANAGER_CONFIG.skip_urls:
self.log_warning(f'Skipping {url} because it is in the download_manager.yaml skip list.')
continue
info_json = None
if not shared_downloader:
# User has requested automatic downloader selection, try and find it.
local_downloader, info_json = self.get_downloader(url)
else:
# One Downloader provided for all URLs, use it.
local_downloader = shared_downloader
download = self.get_or_create_download(url, session)
# Download may have failed, try again.
download.renew(reset_attempts=reset_attempts)
download.downloader = local_downloader.name
download.info_json = info_json or download.info_json
download.sub_downloader = sub_downloader
download.downloader = downloader_name
download.sub_downloader = sub_downloader_name
download.settings = settings
downloads.append(download)
@ -590,24 +562,27 @@ class DownloadManager:
return downloads
@optional_session
def create_download(self, url: str, session: Session = None, downloader: str = None, reset_attempts: bool = False,
sub_downloader: str = None, settings: Dict = None) -> Download:
def create_download(self, url: str, downloader_name: str, session: Session = None, reset_attempts: bool = False,
sub_downloader_name: str = None, settings: Dict = None) -> Download:
"""Schedule a URL for download. If the URL failed previously, it may be retried."""
downloads = self.create_downloads([url], session=session, downloader=downloader, settings=settings,
reset_attempts=reset_attempts, sub_downloader=sub_downloader)
downloads = self.create_downloads([url], session=session, downloader_name=downloader_name,
reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name,
settings=settings)
return downloads[0]
def recurring_download(self, url: str, frequency: int, downloader: str = None,
sub_downloader: str = None, reset_attempts: bool = False,
@optional_session
def recurring_download(self, url: str, frequency: int, downloader_name: str, session: Session = None,
sub_downloader_name: str = None, reset_attempts: bool = False,
settings: Dict = None) -> Download:
"""Schedule a recurring download."""
if not frequency:
if not frequency or not isinstance(frequency, int):
raise ValueError('Recurring download must have a frequency!')
with get_db_session(commit=True) as session:
download, = self.create_downloads([url, ], session=session, downloader=downloader, settings=settings,
sub_downloader=sub_downloader, reset_attempts=reset_attempts)
download.frequency = frequency
download, = self.create_downloads([url, ], session=session, downloader_name=downloader_name,
reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name,
settings=settings)
download.frequency = frequency
session.commit()
return download
@ -1205,4 +1180,4 @@ class RSSDownloader(Downloader, ABC):
return False
rss_downloader = RSSDownloader(30)
rss_downloader = RSSDownloader()

View file

@ -45,6 +45,10 @@ class Events:
def send_ready(message: str = None):
send_event('ready', message)
@staticmethod
def send_downloads_disabled(message: str = None):
send_event('downloads_disabled', message, subject='downloads')
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
log = f'{event=}'

View file

@ -21,8 +21,8 @@ from wrolpi.common import logger, get_config, wrol_mode_enabled, Base, get_media
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics
from wrolpi.dates import now, strptime
from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, API_ERRORS, APIError, ValidationError, HotspotError
from wrolpi.events import get_events
from wrolpi.errors import WROLModeEnabled, API_ERRORS, APIError, ValidationError, HotspotError, InvalidDownload
from wrolpi.events import get_events, Events
from wrolpi.files.lib import get_file_statistics
from wrolpi.vars import API_HOST, API_PORT, DOCKERIZED, API_DEBUG, API_ACCESS_LOG, API_WORKERS, API_AUTO_RELOAD, \
truthy_arg
@ -205,25 +205,30 @@ def valid_regex(_: Request, body: schema.RegexRequest):
@api_bp.post('/download')
@openapi.description('Download the many URLs that are provided.')
@openapi.description('Download all the URLs that are provided.')
@validate(schema.DownloadRequest)
@wrol_mode_check
async def post_download(_: Request, body: schema.DownloadRequest):
# URLs are provided in a textarea, lets split all lines.
urls = [i.strip() for i in str(body.urls).strip().splitlines()]
downloader = download_manager.get_downloader_by_name(body.downloader)
if not downloader:
raise InvalidDownload(f'Cannot find downloader with name {body.downloader}')
excluded_urls = [i.strip() for i in body.excluded_urls.split(',')] if body.excluded_urls else None
downloader = body.downloader
if not downloader or downloader in ('auto', 'None', 'null'):
downloader = None
destination = str(get_media_directory() / body.destination) if body.destination else None
settings = dict(excluded_urls=excluded_urls, destination=destination)
if body.frequency:
download_manager.recurring_download(urls[0], body.frequency, downloader=downloader,
sub_downloader=body.sub_downloader, reset_attempts=True,
download_manager.recurring_download(urls[0], body.frequency, downloader_name=body.downloader,
sub_downloader_name=body.sub_downloader, reset_attempts=True,
settings=settings)
else:
download_manager.create_downloads(urls, downloader=downloader, sub_downloader=body.sub_downloader,
reset_attempts=True, settings=settings)
download_manager.create_downloads(urls, downloader_name=body.downloader, reset_attempts=True,
sub_downloader_name=body.sub_downloader, settings=settings)
if download_manager.disabled:
# Downloads are disabled, warn the user.
Events.send_downloads_disabled('Download created. But, downloads are disabled.')
return response.empty()

View file

@ -49,7 +49,7 @@ class EchoResponse:
@dataclass
class DownloadRequest:
urls: str
downloader: Optional[str] = None
downloader: str
frequency: Optional[int] = None
sub_downloader: Optional[str] = None
excluded_urls: Optional[str] = None

View file

@ -18,74 +18,30 @@ from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeE
from wrolpi.test.common import assert_dict_contains
class PermissiveDownloader(Downloader):
"""A testing Downloader which always says it's valid."""
name = 'permissive'
def __repr__(self):
return '<TESTING Permissive Downloader>'
def valid_url(self, url: str):
return True, None
class HTTPDownloader(Downloader):
"""A testing Downloader which says its valid when a URL starts with http/https"""
name = 'http'
def __repr__(self):
return '<TESTING HTTP Downloader>'
def valid_url(self, url: str):
return url.startswith('https://') or url.startswith('http://'), None
@pytest.mark.asyncio
def test_register_downloader(test_session, test_download_manager):
"""Downloaders can be registered and have specific priorities."""
assert test_download_manager.instances == tuple()
http_downloader = HTTPDownloader()
test_download_manager.register_downloader(http_downloader)
assert test_download_manager.instances == (http_downloader,)
# PermissiveDownloader is first priority.
permissive_downloader = PermissiveDownloader(priority=0)
test_download_manager.register_downloader(permissive_downloader)
assert test_download_manager.instances == (permissive_downloader, http_downloader)
with pytest.raises(ValueError):
test_download_manager.register_downloader(http_downloader)
with pytest.raises(ValueError):
test_download_manager.register_downloader(permissive_downloader)
@pytest.mark.asyncio
async def test_delete_old_once_downloads(test_session, test_download_manager):
async def test_delete_old_once_downloads(test_session, test_download_manager, test_downloader):
"""Once-downloads over a month old should be deleted."""
with mock.patch('wrolpi.downloader.now') as mock_now:
mock_now.return_value = datetime(2020, 6, 5, 0, 0, tzinfo=pytz.UTC)
permissive_downloader = PermissiveDownloader(priority=0)
test_download_manager.register_downloader(permissive_downloader)
# Recurring downloads should not be deleted.
d1 = test_download_manager.create_download('https://example.com/1')
d2 = test_download_manager.create_download('https://example.com/2')
d1 = test_download_manager.create_download('https://example.com/1', test_downloader.name)
d2 = test_download_manager.create_download('https://example.com/2', test_downloader.name)
d1.frequency = 1
d2.frequency = 1
d2.started()
# Should be deleted.
d3 = test_download_manager.create_download('https://example.com/3')
d4 = test_download_manager.create_download('https://example.com/4')
d3 = test_download_manager.create_download('https://example.com/3', test_downloader.name)
d4 = test_download_manager.create_download('https://example.com/4', test_downloader.name)
d3.complete()
d4.complete()
d3.last_successful_download = datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
d4.last_successful_download = datetime(2020, 5, 1, 0, 0, 0, tzinfo=pytz.UTC)
# Not a month old.
d5 = test_download_manager.create_download('https://example.com/5')
d5 = test_download_manager.create_download('https://example.com/5', test_downloader.name)
d5.last_successful_download = datetime(2020, 6, 1, 0, 0, 0, tzinfo=pytz.UTC)
# An old, but pending download should not be deleted.
d6 = test_download_manager.create_download('https://example.com/6')
d6 = test_download_manager.create_download('https://example.com/6', test_downloader.name)
d6.last_successful_download = datetime(2020, 4, 1, 0, 0, 0, tzinfo=pytz.UTC)
d6.started()
@ -104,15 +60,13 @@ async def test_delete_old_once_downloads(test_session, test_download_manager):
@pytest.mark.asyncio
async def test_ensure_download(test_session, test_download_manager):
async def test_create_download(test_session, test_download_manager, test_downloader):
assert len(test_download_manager.get_downloads(test_session)) == 0
http_downloader = HTTPDownloader()
http_downloader.do_download = MagicMock()
http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
test_downloader.do_download = MagicMock()
test_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.create_download('https://example.com')
test_download_manager.create_download('https://example.com', test_downloader.name)
download = test_download_manager.get_download(test_session, 'https://example.com')
assert download.url == 'https://example.com'
@ -122,22 +76,11 @@ async def test_ensure_download(test_session, test_download_manager):
@pytest.mark.asyncio
async def test_create_downloads(test_session, test_download_manager):
async def test_create_downloads(test_session, test_download_manager, test_downloader, assert_download_urls):
"""Multiple downloads can be scheduled using DownloadManager.create_downloads."""
http_downloader = HTTPDownloader(priority=0)
test_download_manager.register_downloader(http_downloader)
# Both URLs are valid and are scheduled.
test_download_manager.create_downloads(['https://example.com/1', 'https://example.com/2'])
downloads = test_download_manager.get_downloads(test_session)
assert {i.url for i in downloads} == {'https://example.com/1', 'https://example.com/2'}
# One URL is bad, neither should be scheduled.
with pytest.raises(InvalidDownload):
test_download_manager.create_downloads(['https://example.com/3', 'bad url should fail'])
downloads = test_download_manager.get_downloads(test_session)
assert {i.url for i in downloads} == {'https://example.com/1', 'https://example.com/2'}
# Both URLs are scheduled.
test_download_manager.create_downloads(['https://example.com/1', 'https://example.com/2'], test_downloader.name)
assert_download_urls({'https://example.com/1', 'https://example.com/2'})
def test_downloader_must_have_name():
@ -164,21 +107,15 @@ async def test_download_wrol_mode(test_session, test_download_manager):
@pytest.mark.asyncio
async def test_download_get_downloader(test_session, test_download_manager):
async def test_download_get_downloader(test_session, test_download_manager, test_downloader):
"""
A Download can find it's Downloader.
"""
permissive_downloader = PermissiveDownloader()
http_downloader = HTTPDownloader()
test_download_manager.register_downloader(permissive_downloader)
test_download_manager.register_downloader(http_downloader)
download1 = test_download_manager.create_download('https://example.com', test_downloader.name)
assert download1.get_downloader() == test_downloader
download1 = test_download_manager.create_download('https://example.com')
assert download1.get_downloader() == permissive_downloader
download2 = test_download_manager.create_download('https://example.com')
download2.downloader = 'bad downloader'
assert download2.get_downloader() is None
with pytest.raises(InvalidDownload):
download2 = test_download_manager.create_download('https://example.com', 'bad downloader')
@pytest.mark.asyncio
@ -208,22 +145,20 @@ def test_calculate_next_download(test_session, test_download_manager, fake_now):
d3 = Download(url='https://example.com/3', frequency=DownloadFrequency.weekly)
test_session.add_all([d1, d2, d3])
test_session.commit()
# Downloads are spread out over the next week.
# Downloads are ead out over the next week.
assert test_download_manager.calculate_next_download(d1) == datetime(2000, 1, 8, tzinfo=pytz.UTC)
assert test_download_manager.calculate_next_download(d2) == datetime(2000, 1, 11, 12, tzinfo=pytz.UTC)
assert test_download_manager.calculate_next_download(d3) == datetime(2000, 1, 9, 18, tzinfo=pytz.UTC)
@pytest.mark.asyncio
async def test_recurring_downloads(test_session, test_download_manager, fake_now):
async def test_recurring_downloads(test_session, test_download_manager, fake_now, test_downloader):
"""A recurring Download should be downloaded forever."""
http_downloader = HTTPDownloader()
http_downloader.do_download = MagicMock()
http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
test_downloader.do_download = MagicMock()
test_downloader.do_download.return_value = DownloadResult(success=True)
# Download every hour.
test_download_manager.recurring_download('https://example.com', Seconds.hour)
test_download_manager.recurring_download('https://example.com', Seconds.hour, test_downloader.name)
# One download is scheduled.
downloads = test_download_manager.get_new_downloads(test_session)
@ -233,7 +168,7 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
# Download is processed and successful, no longer "new".
await test_download_manager.wait_for_all_downloads()
http_downloader.do_download.assert_called_once()
test_downloader.do_download.assert_called_once()
assert list(test_download_manager.get_new_downloads(test_session)) == []
downloads = list(test_download_manager.get_recurring_downloads(test_session))
assert len(downloads) == 1
@ -258,10 +193,10 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
assert download.status == 'new'
# Try the download, but it fails.
http_downloader.do_download.reset_mock()
http_downloader.do_download.return_value = DownloadResult(success=False)
test_downloader.do_download.reset_mock()
test_downloader.do_download.return_value = DownloadResult(success=False)
await test_download_manager.wait_for_all_downloads()
http_downloader.do_download.assert_called_once()
test_downloader.do_download.assert_called_once()
download = test_session.query(Download).one()
# Download is deferred, last successful download remains the same.
assert download.status == 'deferred'
@ -271,12 +206,12 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
assert download.next_download == expected
# Try the download again, it finally succeeds.
http_downloader.do_download.reset_mock()
test_downloader.do_download.reset_mock()
now_ = fake_now(datetime(2020, 1, 1, 4, 0, 1, tzinfo=pytz.UTC))
http_downloader.do_download.return_value = DownloadResult(success=True)
test_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.renew_recurring_downloads(test_session)
await test_download_manager.wait_for_all_downloads()
http_downloader.do_download.assert_called_once()
test_downloader.do_download.assert_called_once()
download = test_session.query(Download).one()
assert download.status == 'complete'
assert download.last_successful_download == now_
@ -285,28 +220,26 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
@pytest.mark.asyncio
async def test_max_attempts(test_session, test_download_manager):
async def test_max_attempts(test_session, test_download_manager, test_downloader):
"""A Download will only be attempted so many times, it will eventually be deleted."""
_, session = get_db_context()
http_downloader = HTTPDownloader()
http_downloader.do_download = MagicMock()
http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
test_downloader.do_download = MagicMock()
test_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.create_download('https://example.com')
test_download_manager.create_download('https://example.com', test_downloader.name)
await test_download_manager.wait_for_all_downloads()
download = session.query(Download).one()
assert download.attempts == 1
test_download_manager.create_download('https://example.com')
test_download_manager.create_download('https://example.com', test_downloader.name)
await test_download_manager.wait_for_all_downloads()
download = session.query(Download).one()
assert download.attempts == 2
# There are no further attempts.
http_downloader.do_download.side_effect = UnrecoverableDownloadError()
test_download_manager.create_download('https://example.com')
test_downloader.do_download.side_effect = UnrecoverableDownloadError()
test_download_manager.create_download('https://example.com', test_downloader.name)
await test_download_manager.wait_for_all_downloads()
download = session.query(Download).one()
assert download.attempts == 3
@ -314,22 +247,20 @@ async def test_max_attempts(test_session, test_download_manager):
@pytest.mark.asyncio
async def test_skip_urls(test_session, test_download_manager, test_client, assert_download_urls):
async def test_skip_urls(test_session, test_download_manager, test_client, assert_download_urls, test_downloader):
"""The DownloadManager will not create downloads for URLs in it's skip list."""
_, session = get_db_context()
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
DOWNLOAD_MANAGER_CONFIG.skip_urls = ['https://example.com/skipme']
http_downloader = HTTPDownloader()
http_downloader.do_download = MagicMock()
http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
test_downloader.do_download = MagicMock()
test_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.create_downloads([
'https://example.com/1',
'https://example.com/skipme',
'https://example.com/2',
], downloader=HTTPDownloader.name)
], downloader_name=test_downloader.name)
await test_download_manager.wait_for_all_downloads()
assert_download_urls({'https://example.com/1', 'https://example.com/2'})
assert DOWNLOAD_MANAGER_CONFIG.skip_urls == ['https://example.com/skipme']
@ -337,7 +268,7 @@ async def test_skip_urls(test_session, test_download_manager, test_client, asser
test_download_manager.delete_completed()
# The user can start a download even if the URL is in the skip list.
test_download_manager.create_download('https://example.com/skipme', reset_attempts=True)
test_download_manager.create_download('https://example.com/skipme', test_downloader.name, reset_attempts=True)
assert_download_urls({'https://example.com/skipme'})
assert DOWNLOAD_MANAGER_CONFIG.skip_urls == []
@ -346,7 +277,7 @@ async def test_skip_urls(test_session, test_download_manager, test_client, asser
async def test_process_runner_timeout(test_directory):
"""A Downloader can cancel it's download using a timeout."""
# Default timeout of 3 seconds.
downloader = Downloader(0, 'downloader', timeout=3)
downloader = Downloader('downloader', timeout=3)
# Default timeout is obeyed.
start = datetime.now()
@ -369,14 +300,11 @@ async def test_process_runner_timeout(test_directory):
assert 2 < elapsed.total_seconds() < 4
def test_crud_download(test_client, test_session, test_download_manager):
def test_crud_download(test_client, test_session, test_download_manager, test_downloader):
"""Test the ways that Downloads can be created."""
http_downloader = HTTPDownloader()
test_download_manager.register_downloader(http_downloader)
body = dict(
urls='https://example.com',
downloader=http_downloader.name,
downloader=test_downloader.name,
frequency=DownloadFrequency.weekly,
excluded_urls='example.org,something',
)
@ -386,7 +314,7 @@ def test_crud_download(test_client, test_session, test_download_manager):
download: Download = test_session.query(Download).one()
assert download.id
assert download.url == 'https://example.com'
assert download.downloader == 'http'
assert download.downloader == test_downloader.name
assert download.frequency == DownloadFrequency.weekly
assert download.settings['excluded_urls'] == ['example.org', 'something']
@ -409,33 +337,23 @@ def test_crud_download(test_client, test_session, test_download_manager):
@pytest.mark.asyncio
async def test_downloads_config(test_session, test_client, test_download_manager, test_download_manager_config):
async def test_downloads_config(test_session, test_client, test_download_manager, test_download_manager_config,
test_downloader, assert_downloads):
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
def assert_downloads(expected_):
assert test_session.query(Download).count() == len(expected_), 'Download count does not match'
downloads = test_session.query(Download).order_by(Download.url)
for download_, expected_ in zip_longest(downloads, expected_):
assert_dict_contains(download_.__json__(), expected_)
# Can import with an empty config.
await import_downloads_config()
assert_downloads([])
http_downloader = HTTPDownloader()
http_downloader.do_download = MagicMock()
http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
test_downloader.do_download = MagicMock()
test_downloader.do_download.return_value = DownloadResult(success=True)
download1, download2 = test_download_manager.create_downloads([
'https://example.com/1',
'https://example.com/2',
], downloader=HTTPDownloader.name)
test_download_manager.recurring_download(
'https://example.com/3',
frequency=DownloadFrequency.weekly,
downloader=HTTPDownloader.name,
)
], downloader_name=test_downloader.name)
test_download_manager.recurring_download('https://example.com/3', frequency=DownloadFrequency.weekly,
downloader_name=test_downloader.name)
download2.next_download = datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC)
# Completed once-downloads should be ignored.
download1.last_successful_download = datetime(2000, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
@ -452,13 +370,13 @@ async def test_downloads_config(test_session, test_client, test_download_manager
url='https://example.com/2',
frequency=None,
next_download=datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC),
downloader='http',
downloader='test_downloader',
sub_downloader=None,
),
dict(
url='https://example.com/3',
frequency=DownloadFrequency.weekly,
downloader='http',
downloader='test_downloader',
sub_downloader=None,
),
]
@ -476,13 +394,13 @@ async def test_downloads_config(test_session, test_client, test_download_manager
url='https://example.com/2',
frequency=None,
next_download=datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC),
downloader='http',
downloader='test_downloader',
sub_downloader=None,
),
dict(
url='https://example.com/3',
frequency=DownloadFrequency.weekly,
downloader='http',
downloader='test_downloader',
sub_downloader=None,
),
]
@ -494,12 +412,10 @@ async def test_downloads_config(test_session, test_client, test_download_manager
@pytest.mark.asyncio
async def test_download_excluded_urls(test_client, test_session, test_download_manager):
async def test_download_excluded_urls(test_client, test_session, test_download_manager, test_downloader):
"""Test that URLs that have been excluded are ignored by the download worker."""
http_downloader = HTTPDownloader()
http_downloader.already_downloaded = lambda *i, **kw: []
http_downloader.do_download = lambda *i, **kw: DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
test_downloader.already_downloaded = lambda *i, **kw: []
test_downloader.do_download = lambda *i, **kw: DownloadResult(success=True)
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
@ -514,9 +430,9 @@ async def test_download_excluded_urls(test_client, test_session, test_download_m
]
)
settings = dict(excluded_urls=['example.org', 'example.gov'])
feed_download: Download = test_download_manager.create_download('https://example.com/feed',
test_session, sub_downloader='http',
downloader=RSSDownloader.name,
feed_download: Download = test_download_manager.create_download('https://example.com/feed', rss_downloader.name,
test_session,
sub_downloader_name=test_downloader.name,
settings=settings)
await test_download_manager.wait_for_all_downloads()

View file

@ -5,11 +5,12 @@ import mock
import pytest
from wrolpi.db import optional_session
from wrolpi.downloader import Download, DownloadResult, RSSDownloader
from wrolpi.test.test_downloader import HTTPDownloader
from wrolpi.downloader import Download, DownloadResult, RSSDownloader, Downloader
class RSSHTTPDownloader(HTTPDownloader):
class RSSHTTPDownloader(Downloader):
name = 'rss_http'
def do_download(self, download: Download) -> DownloadResult:
return DownloadResult(success=True)
@ -44,7 +45,8 @@ async def test_rss_download(test_session, test_download_manager):
dict(link='https://example.com/c'),
]
)
test_download_manager.create_download('https://example.com/feed', sub_downloader='http')
test_download_manager.create_download('https://example.com/feed', rss_downloader.name,
sub_downloader_name='rss_http')
await test_download_manager.wait_for_all_downloads()
# Feed download is complete.
@ -63,7 +65,8 @@ async def test_rss_download(test_session, test_download_manager):
dict(link='https://example.com/d'),
]
)
test_download_manager.create_download('https://example.com/feed', sub_downloader='http')
test_download_manager.create_download('https://example.com/feed', rss_downloader.name,
sub_downloader_name='rss_http')
await test_download_manager.wait_for_all_downloads()
# Only the new URLs are Archived.
@ -84,7 +87,8 @@ async def test_rss_no_entries(test_session, test_download_manager):
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
mock_parse_feed.return_value = dict(bozo=0, ) # missing `entries`
test_download_manager.create_download('https://example.com/feed', test_session, sub_downloader='http')
test_download_manager.create_download('https://example.com/feed', rss_downloader.name, test_session,
sub_downloader_name='rss_http')
await test_download_manager.wait_for_all_downloads()
(download,) = test_download_manager.get_downloads(test_session)