From 351351d90cb36a45f753528a2737be304caed179 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 20 Sep 2023 12:17:07 -0600 Subject: [PATCH] Raising error when Downloader.do_download is not programmed correctly. Fixing any tests that were not using do_download correctly. --- app/src/components/Common.js | 6 ++-- modules/videos/test/test_downloader.py | 43 -------------------------- wrolpi/conftest.py | 18 ++++++++++- wrolpi/downloader.py | 3 ++ wrolpi/test/test_downloader.py | 13 +++----- wrolpi/test/test_root_api.py | 2 ++ 6 files changed, 31 insertions(+), 54 deletions(-) diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 3a9cd2ce..019e113d 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -794,8 +794,8 @@ export function mimetypeIconName(mimetype, lowerPath = '') { return 'closed captioning'; } else if (mimetype.startsWith('application/x-openscad') || mimetype.startsWith('model/stl')) { return 'cube'; - } else if (mimetype.startsWith('application/x-dosexec') || mimetype.startsWith('application/x-msi')) { - return 'microsoft'; + } else if (mimetype.startsWith('application/x-dosexec') || mimetype.startsWith('application/x-msi') || mimetype.startsWith('application/vnd.microsoft.portable-executable')) { + return 'windows'; } else if (mimetype.startsWith('audio/')) { return 'file audio'; } else if (mimetype.startsWith('application/vnd.openxmlformats-officedocument.wordprocessingml.document')) { @@ -813,6 +813,8 @@ export function mimetypeIconName(mimetype, lowerPath = '') { return 'apple'; } else if (lowerPath.endsWith('.azw3')) { return 'book'; + } else if (lowerPath.endsWith('.exe')) { + return 'windows'; } } else if (mimetype.startsWith('application/vnd.openxmlformats-officedocument.spreadsheetml.') || mimetype.startsWith('application/vnd.ms-excel') || mimetype.startsWith('application/vnd.oasis.opendocument.spreadsheet')) { return 'file excel' diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 81c01a4a..f8f34485 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -212,49 +212,6 @@ def test_get_or_create_channel(test_session): assert get_or_create_channel(source_id='quux') == channel -@pytest.mark.asyncio -async def test_selected_downloader(test_session, video_download_manager, test_downloader, successful_download): - """ - A user can specify which Downloader to use, no other should ever be used (even if the Download fails). - """ - # The test Downloader will pretend to have downloaded anything. - mock_do_download = test_downloader.do_download = mock.MagicMock() - mock_do_download.return_value = successful_download - - def check_attempts(attempts): - assert test_session.query(Download).one().attempts == attempts - - # DownloadManager will attempt to choose the Downloader. - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: - mock_extract_info.side_effect = UnsupportedError('not this one') - video_download_manager.create_download('https://example.com', test_downloader.name) - await video_download_manager.wait_for_all_downloads() - mock_do_download.assert_called_once() - - check_attempts(1) - - mock_do_download.reset_mock() - - # DownloadManager will only use the TestDownloader. - video_download_manager.create_download('https://example.com', test_downloader.name) - await video_download_manager.wait_for_all_downloads() - # DownloadManager called do_download. - mock_do_download.assert_called_once() - - check_attempts(2) - - mock_do_download.reset_mock() - - # DownloadManager will only use the TestDownloader, even if the download fails. - mock_do_download.side_effect = Exception('oh no!') - video_download_manager.create_download('https://example.com', test_downloader.name) - await video_download_manager.wait_for_all_downloads() - # DownloadManager called do_download. - mock_do_download.assert_called_once() - - check_attempts(3) - - def test_channel_downloader_hidden(video_download_manager): """ ChannelDownloader should not be presented to the User. diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 07b20446..fd06e65b 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -35,6 +35,7 @@ from wrolpi.dates import set_test_now from wrolpi.db import postgres_engine, get_db_args from wrolpi.downloader import DownloadManager, DownloadResult, Download, Downloader, \ downloads_manager_config_context +from wrolpi.errors import UnrecoverableDownloadError from wrolpi.files.models import Directory, FileGroup from wrolpi.root_api import BLUEPRINTS, api_app from wrolpi.tags import Tag @@ -256,7 +257,21 @@ def test_downloader(test_download_manager): self.do_download.side_effect = _ + def set_test_exception(self, exception: Exception = Exception('Test downloader exception')): + async def _(*a, **kwargs): + raise exception + + self.do_download.side_effect = _ + + def set_test_unrecoverable_exception(self): + async def _(*a, **kwargs): + raise UnrecoverableDownloadError() + + self.do_download.side_effect = _ + test_downloader = TestDownloader() + # Default to successful download. + test_downloader.set_test_success() test_download_manager.register_downloader(test_downloader) return test_downloader @@ -379,8 +394,9 @@ def make_files_structure(test_directory) -> Callable: A fixture which creates test files passed to it. If a list is provided, empty files will be created at those locations. If a dict is provided, files will be created containing the value of the dict item. """ + def create_files(paths: Union[List, Dict], file_groups: bool = False, session: Session = None) -> List[ - pathlib.Path]: + pathlib.Path]: files = [] @iterify(list) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index d17cf4f4..36f10a67 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1,5 +1,6 @@ import asyncio import contextlib +import inspect import logging import multiprocessing import os @@ -388,6 +389,8 @@ class DownloadManager: try: # Create download coroutine; wrap it, so it can be canceled coro = downloader.do_download(download) + if not inspect.iscoroutine(coro): + raise RuntimeError(f'Coroutine expected from {downloader} do_download method.') result = await downloader.cancel_wrapper(coro, download) except UnrecoverableDownloadError as e: # Download failed and should not be retried. diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 236eab61..9b9c5b72 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -64,8 +64,7 @@ async def test_delete_old_once_downloads(test_session, test_download_manager, te async def test_create_download(test_session, test_download_manager, test_downloader): assert len(test_download_manager.get_downloads(test_session)) == 0 - test_downloader.do_download = MagicMock() - test_downloader.do_download.return_value = DownloadResult(success=True) + test_downloader.set_test_success() test_download_manager.create_download('https://example.com', test_downloader.name) download = test_download_manager.get_download(test_session, 'https://example.com') @@ -246,8 +245,7 @@ async def test_max_attempts(test_session, test_download_manager, test_downloader """A Download will only be attempted so many times, it will eventually be deleted.""" _, session = get_db_context() - test_downloader.do_download = MagicMock() - test_downloader.do_download.return_value = DownloadResult(success=True) + test_downloader.set_test_success() test_download_manager.create_download('https://example.com', test_downloader.name) await test_download_manager.wait_for_all_downloads() @@ -260,7 +258,7 @@ async def test_max_attempts(test_session, test_download_manager, test_downloader assert download.attempts == 2 # There are no further attempts. - test_downloader.do_download.side_effect = UnrecoverableDownloadError() + test_downloader.set_test_unrecoverable_exception() test_download_manager.create_download('https://example.com', test_downloader.name) await test_download_manager.wait_for_all_downloads() download = session.query(Download).one() @@ -364,8 +362,7 @@ async def test_downloads_config(test_session, test_client, test_download_manager await import_downloads_config() assert_downloads([]) - test_downloader.do_download = MagicMock() - test_downloader.do_download.return_value = DownloadResult(success=True) + test_downloader.set_test_success() download1, download2 = test_download_manager.create_downloads([ 'https://example.com/1', @@ -450,7 +447,7 @@ async def test_downloads_config(test_session, test_client, test_download_manager async def test_download_excluded_urls(test_client, test_session, test_download_manager, test_downloader): """Test that URLs that have been excluded are ignored by the download worker.""" test_downloader.already_downloaded = lambda *i, **kw: [] - test_downloader.do_download = lambda *i, **kw: DownloadResult(success=True) + test_downloader.set_test_success() rss_downloader = RSSDownloader() test_download_manager.register_downloader(rss_downloader) diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 2417b582..e357435a 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -412,6 +412,8 @@ async def test_restart_download(test_session, test_async_client, test_download_m test_session.commit() assert test_session.query(Download).one().status == 'failed' + test_downloader.set_test_failure() + # Download is now "new" again. request, response = await test_async_client.post(f'/api/download/{download.id}/restart') assert response.status_code == HTTPStatus.NO_CONTENT