Raising error when Downloader.do_download is not programmed correctly. Fixing any tests that were not using do_download correctly.

This commit is contained in:
Roland 2023-09-20 12:17:07 -06:00
parent a510960c8c
commit 351351d90c
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
6 changed files with 31 additions and 54 deletions

View file

@ -794,8 +794,8 @@ export function mimetypeIconName(mimetype, lowerPath = '') {
return 'closed captioning';
} else if (mimetype.startsWith('application/x-openscad') || mimetype.startsWith('model/stl')) {
return 'cube';
} else if (mimetype.startsWith('application/x-dosexec') || mimetype.startsWith('application/x-msi')) {
return 'microsoft';
} else if (mimetype.startsWith('application/x-dosexec') || mimetype.startsWith('application/x-msi') || mimetype.startsWith('application/vnd.microsoft.portable-executable')) {
return 'windows';
} else if (mimetype.startsWith('audio/')) {
return 'file audio';
} else if (mimetype.startsWith('application/vnd.openxmlformats-officedocument.wordprocessingml.document')) {
@ -813,6 +813,8 @@ export function mimetypeIconName(mimetype, lowerPath = '') {
return 'apple';
} else if (lowerPath.endsWith('.azw3')) {
return 'book';
} else if (lowerPath.endsWith('.exe')) {
return 'windows';
}
} else if (mimetype.startsWith('application/vnd.openxmlformats-officedocument.spreadsheetml.') || mimetype.startsWith('application/vnd.ms-excel') || mimetype.startsWith('application/vnd.oasis.opendocument.spreadsheet')) {
return 'file excel'

View file

@ -212,49 +212,6 @@ def test_get_or_create_channel(test_session):
assert get_or_create_channel(source_id='quux') == channel
@pytest.mark.asyncio
async def test_selected_downloader(test_session, video_download_manager, test_downloader, successful_download):
"""
A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
"""
# The test Downloader will pretend to have downloaded anything.
mock_do_download = test_downloader.do_download = mock.MagicMock()
mock_do_download.return_value = successful_download
def check_attempts(attempts):
assert test_session.query(Download).one().attempts == attempts
# DownloadManager will attempt to choose the Downloader.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_extract_info.side_effect = UnsupportedError('not this one')
video_download_manager.create_download('https://example.com', test_downloader.name)
await video_download_manager.wait_for_all_downloads()
mock_do_download.assert_called_once()
check_attempts(1)
mock_do_download.reset_mock()
# DownloadManager will only use the TestDownloader.
video_download_manager.create_download('https://example.com', test_downloader.name)
await video_download_manager.wait_for_all_downloads()
# DownloadManager called do_download.
mock_do_download.assert_called_once()
check_attempts(2)
mock_do_download.reset_mock()
# DownloadManager will only use the TestDownloader, even if the download fails.
mock_do_download.side_effect = Exception('oh no!')
video_download_manager.create_download('https://example.com', test_downloader.name)
await video_download_manager.wait_for_all_downloads()
# DownloadManager called do_download.
mock_do_download.assert_called_once()
check_attempts(3)
def test_channel_downloader_hidden(video_download_manager):
"""
ChannelDownloader should not be presented to the User.

View file

@ -35,6 +35,7 @@ from wrolpi.dates import set_test_now
from wrolpi.db import postgres_engine, get_db_args
from wrolpi.downloader import DownloadManager, DownloadResult, Download, Downloader, \
downloads_manager_config_context
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.models import Directory, FileGroup
from wrolpi.root_api import BLUEPRINTS, api_app
from wrolpi.tags import Tag
@ -256,7 +257,21 @@ def test_downloader(test_download_manager):
self.do_download.side_effect = _
def set_test_exception(self, exception: Exception = Exception('Test downloader exception')):
async def _(*a, **kwargs):
raise exception
self.do_download.side_effect = _
def set_test_unrecoverable_exception(self):
async def _(*a, **kwargs):
raise UnrecoverableDownloadError()
self.do_download.side_effect = _
test_downloader = TestDownloader()
# Default to successful download.
test_downloader.set_test_success()
test_download_manager.register_downloader(test_downloader)
return test_downloader
@ -379,8 +394,9 @@ def make_files_structure(test_directory) -> Callable:
A fixture which creates test files passed to it. If a list is provided, empty files will be created at those
locations. If a dict is provided, files will be created containing the value of the dict item.
"""
def create_files(paths: Union[List, Dict], file_groups: bool = False, session: Session = None) -> List[
pathlib.Path]:
pathlib.Path]:
files = []
@iterify(list)

View file

@ -1,5 +1,6 @@
import asyncio
import contextlib
import inspect
import logging
import multiprocessing
import os
@ -388,6 +389,8 @@ class DownloadManager:
try:
# Create download coroutine; wrap it, so it can be canceled
coro = downloader.do_download(download)
if not inspect.iscoroutine(coro):
raise RuntimeError(f'Coroutine expected from {downloader} do_download method.')
result = await downloader.cancel_wrapper(coro, download)
except UnrecoverableDownloadError as e:
# Download failed and should not be retried.

View file

@ -64,8 +64,7 @@ async def test_delete_old_once_downloads(test_session, test_download_manager, te
async def test_create_download(test_session, test_download_manager, test_downloader):
assert len(test_download_manager.get_downloads(test_session)) == 0
test_downloader.do_download = MagicMock()
test_downloader.do_download.return_value = DownloadResult(success=True)
test_downloader.set_test_success()
test_download_manager.create_download('https://example.com', test_downloader.name)
download = test_download_manager.get_download(test_session, 'https://example.com')
@ -246,8 +245,7 @@ async def test_max_attempts(test_session, test_download_manager, test_downloader
"""A Download will only be attempted so many times, it will eventually be deleted."""
_, session = get_db_context()
test_downloader.do_download = MagicMock()
test_downloader.do_download.return_value = DownloadResult(success=True)
test_downloader.set_test_success()
test_download_manager.create_download('https://example.com', test_downloader.name)
await test_download_manager.wait_for_all_downloads()
@ -260,7 +258,7 @@ async def test_max_attempts(test_session, test_download_manager, test_downloader
assert download.attempts == 2
# There are no further attempts.
test_downloader.do_download.side_effect = UnrecoverableDownloadError()
test_downloader.set_test_unrecoverable_exception()
test_download_manager.create_download('https://example.com', test_downloader.name)
await test_download_manager.wait_for_all_downloads()
download = session.query(Download).one()
@ -364,8 +362,7 @@ async def test_downloads_config(test_session, test_client, test_download_manager
await import_downloads_config()
assert_downloads([])
test_downloader.do_download = MagicMock()
test_downloader.do_download.return_value = DownloadResult(success=True)
test_downloader.set_test_success()
download1, download2 = test_download_manager.create_downloads([
'https://example.com/1',
@ -450,7 +447,7 @@ async def test_downloads_config(test_session, test_client, test_download_manager
async def test_download_excluded_urls(test_client, test_session, test_download_manager, test_downloader):
"""Test that URLs that have been excluded are ignored by the download worker."""
test_downloader.already_downloaded = lambda *i, **kw: []
test_downloader.do_download = lambda *i, **kw: DownloadResult(success=True)
test_downloader.set_test_success()
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)

View file

@ -412,6 +412,8 @@ async def test_restart_download(test_session, test_async_client, test_download_m
test_session.commit()
assert test_session.query(Download).one().status == 'failed'
test_downloader.set_test_failure()
# Download is now "new" again.
request, response = await test_async_client.post(f'/api/download/{download.id}/restart')
assert response.status_code == HTTPStatus.NO_CONTENT