mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
Raising error when Downloader.do_download is not programmed correctly. Fixing any tests that were not using do_download correctly.
This commit is contained in:
parent
a510960c8c
commit
351351d90c
6 changed files with 31 additions and 54 deletions
|
|
@ -794,8 +794,8 @@ export function mimetypeIconName(mimetype, lowerPath = '') {
|
|||
return 'closed captioning';
|
||||
} else if (mimetype.startsWith('application/x-openscad') || mimetype.startsWith('model/stl')) {
|
||||
return 'cube';
|
||||
} else if (mimetype.startsWith('application/x-dosexec') || mimetype.startsWith('application/x-msi')) {
|
||||
return 'microsoft';
|
||||
} else if (mimetype.startsWith('application/x-dosexec') || mimetype.startsWith('application/x-msi') || mimetype.startsWith('application/vnd.microsoft.portable-executable')) {
|
||||
return 'windows';
|
||||
} else if (mimetype.startsWith('audio/')) {
|
||||
return 'file audio';
|
||||
} else if (mimetype.startsWith('application/vnd.openxmlformats-officedocument.wordprocessingml.document')) {
|
||||
|
|
@ -813,6 +813,8 @@ export function mimetypeIconName(mimetype, lowerPath = '') {
|
|||
return 'apple';
|
||||
} else if (lowerPath.endsWith('.azw3')) {
|
||||
return 'book';
|
||||
} else if (lowerPath.endsWith('.exe')) {
|
||||
return 'windows';
|
||||
}
|
||||
} else if (mimetype.startsWith('application/vnd.openxmlformats-officedocument.spreadsheetml.') || mimetype.startsWith('application/vnd.ms-excel') || mimetype.startsWith('application/vnd.oasis.opendocument.spreadsheet')) {
|
||||
return 'file excel'
|
||||
|
|
|
|||
|
|
@ -212,49 +212,6 @@ def test_get_or_create_channel(test_session):
|
|||
assert get_or_create_channel(source_id='quux') == channel
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_selected_downloader(test_session, video_download_manager, test_downloader, successful_download):
|
||||
"""
|
||||
A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
|
||||
"""
|
||||
# The test Downloader will pretend to have downloaded anything.
|
||||
mock_do_download = test_downloader.do_download = mock.MagicMock()
|
||||
mock_do_download.return_value = successful_download
|
||||
|
||||
def check_attempts(attempts):
|
||||
assert test_session.query(Download).one().attempts == attempts
|
||||
|
||||
# DownloadManager will attempt to choose the Downloader.
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
mock_extract_info.side_effect = UnsupportedError('not this one')
|
||||
video_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
mock_do_download.assert_called_once()
|
||||
|
||||
check_attempts(1)
|
||||
|
||||
mock_do_download.reset_mock()
|
||||
|
||||
# DownloadManager will only use the TestDownloader.
|
||||
video_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
# DownloadManager called do_download.
|
||||
mock_do_download.assert_called_once()
|
||||
|
||||
check_attempts(2)
|
||||
|
||||
mock_do_download.reset_mock()
|
||||
|
||||
# DownloadManager will only use the TestDownloader, even if the download fails.
|
||||
mock_do_download.side_effect = Exception('oh no!')
|
||||
video_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
# DownloadManager called do_download.
|
||||
mock_do_download.assert_called_once()
|
||||
|
||||
check_attempts(3)
|
||||
|
||||
|
||||
def test_channel_downloader_hidden(video_download_manager):
|
||||
"""
|
||||
ChannelDownloader should not be presented to the User.
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from wrolpi.dates import set_test_now
|
|||
from wrolpi.db import postgres_engine, get_db_args
|
||||
from wrolpi.downloader import DownloadManager, DownloadResult, Download, Downloader, \
|
||||
downloads_manager_config_context
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.files.models import Directory, FileGroup
|
||||
from wrolpi.root_api import BLUEPRINTS, api_app
|
||||
from wrolpi.tags import Tag
|
||||
|
|
@ -256,7 +257,21 @@ def test_downloader(test_download_manager):
|
|||
|
||||
self.do_download.side_effect = _
|
||||
|
||||
def set_test_exception(self, exception: Exception = Exception('Test downloader exception')):
|
||||
async def _(*a, **kwargs):
|
||||
raise exception
|
||||
|
||||
self.do_download.side_effect = _
|
||||
|
||||
def set_test_unrecoverable_exception(self):
|
||||
async def _(*a, **kwargs):
|
||||
raise UnrecoverableDownloadError()
|
||||
|
||||
self.do_download.side_effect = _
|
||||
|
||||
test_downloader = TestDownloader()
|
||||
# Default to successful download.
|
||||
test_downloader.set_test_success()
|
||||
test_download_manager.register_downloader(test_downloader)
|
||||
|
||||
return test_downloader
|
||||
|
|
@ -379,8 +394,9 @@ def make_files_structure(test_directory) -> Callable:
|
|||
A fixture which creates test files passed to it. If a list is provided, empty files will be created at those
|
||||
locations. If a dict is provided, files will be created containing the value of the dict item.
|
||||
"""
|
||||
|
||||
def create_files(paths: Union[List, Dict], file_groups: bool = False, session: Session = None) -> List[
|
||||
pathlib.Path]:
|
||||
pathlib.Path]:
|
||||
files = []
|
||||
|
||||
@iterify(list)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import inspect
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
|
|
@ -388,6 +389,8 @@ class DownloadManager:
|
|||
try:
|
||||
# Create download coroutine; wrap it, so it can be canceled
|
||||
coro = downloader.do_download(download)
|
||||
if not inspect.iscoroutine(coro):
|
||||
raise RuntimeError(f'Coroutine expected from {downloader} do_download method.')
|
||||
result = await downloader.cancel_wrapper(coro, download)
|
||||
except UnrecoverableDownloadError as e:
|
||||
# Download failed and should not be retried.
|
||||
|
|
|
|||
|
|
@ -64,8 +64,7 @@ async def test_delete_old_once_downloads(test_session, test_download_manager, te
|
|||
async def test_create_download(test_session, test_download_manager, test_downloader):
|
||||
assert len(test_download_manager.get_downloads(test_session)) == 0
|
||||
|
||||
test_downloader.do_download = MagicMock()
|
||||
test_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_downloader.set_test_success()
|
||||
|
||||
test_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
download = test_download_manager.get_download(test_session, 'https://example.com')
|
||||
|
|
@ -246,8 +245,7 @@ async def test_max_attempts(test_session, test_download_manager, test_downloader
|
|||
"""A Download will only be attempted so many times, it will eventually be deleted."""
|
||||
_, session = get_db_context()
|
||||
|
||||
test_downloader.do_download = MagicMock()
|
||||
test_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_downloader.set_test_success()
|
||||
|
||||
test_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
|
@ -260,7 +258,7 @@ async def test_max_attempts(test_session, test_download_manager, test_downloader
|
|||
assert download.attempts == 2
|
||||
|
||||
# There are no further attempts.
|
||||
test_downloader.do_download.side_effect = UnrecoverableDownloadError()
|
||||
test_downloader.set_test_unrecoverable_exception()
|
||||
test_download_manager.create_download('https://example.com', test_downloader.name)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
download = session.query(Download).one()
|
||||
|
|
@ -364,8 +362,7 @@ async def test_downloads_config(test_session, test_client, test_download_manager
|
|||
await import_downloads_config()
|
||||
assert_downloads([])
|
||||
|
||||
test_downloader.do_download = MagicMock()
|
||||
test_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_downloader.set_test_success()
|
||||
|
||||
download1, download2 = test_download_manager.create_downloads([
|
||||
'https://example.com/1',
|
||||
|
|
@ -450,7 +447,7 @@ async def test_downloads_config(test_session, test_client, test_download_manager
|
|||
async def test_download_excluded_urls(test_client, test_session, test_download_manager, test_downloader):
|
||||
"""Test that URLs that have been excluded are ignored by the download worker."""
|
||||
test_downloader.already_downloaded = lambda *i, **kw: []
|
||||
test_downloader.do_download = lambda *i, **kw: DownloadResult(success=True)
|
||||
test_downloader.set_test_success()
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
|
||||
|
|
|
|||
|
|
@ -412,6 +412,8 @@ async def test_restart_download(test_session, test_async_client, test_download_m
|
|||
test_session.commit()
|
||||
assert test_session.query(Download).one().status == 'failed'
|
||||
|
||||
test_downloader.set_test_failure()
|
||||
|
||||
# Download is now "new" again.
|
||||
request, response = await test_async_client.post(f'/api/download/{download.id}/restart')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue