From 399cb0f107ea9ca37a3d34e7c945ea597202f58d Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 21 Mar 2025 17:14:34 -0600 Subject: [PATCH] Testing that downloads have absolute destinations. Handling 'undefined', 'null', etc, dates. --- modules/videos/downloader.py | 6 ++++-- modules/videos/test/test_common.py | 6 +++++- wrolpi/common.py | 2 +- wrolpi/dates.py | 3 +++ wrolpi/test/test_dates.py | 6 ++++++ wrolpi/test/test_root_api.py | 8 ++++++-- 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 2bd8b453..18bc07cb 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -19,7 +19,7 @@ from yt_dlp.extractor import YoutubeTabIE # noqa from wrolpi.cmd import YT_DLP_BIN from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \ - trim_file_name, cached_multiprocessing_result + trim_file_name, cached_multiprocessing_result, get_absolute_media_path from wrolpi.dates import now from wrolpi.db import get_db_session from wrolpi.db import optional_session @@ -182,7 +182,8 @@ class ChannelDownloader(Downloader, ABC): if channel: settings.update(dict(channel_id=channel.id, channel_url=download.url)) if download.destination: - settings['destination'] = str(download.destination) # Need str for JSON conversion + destination = get_absolute_media_path(download.destination) + settings['destination'] = str(destination) # Need str for JSON conversion if download.tag_names: settings['tag_names'] = download.tag_names if video_resolutions := download_settings.get('video_resolutions'): @@ -370,6 +371,7 @@ class VideoDownloader(Downloader, ABC): # Make output directory. (Maybe string from settings) out_dir = out_dir if isinstance(out_dir, pathlib.Path) else pathlib.Path(out_dir) + out_dir = get_absolute_media_path(out_dir) out_dir.mkdir(exist_ok=True, parents=True) if tag_names and logger.isEnabledFor(logging.DEBUG): diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 93b72086..eb0f1f6c 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -20,7 +20,11 @@ from ..lib import save_channels_config, get_channels_config, import_channels_con def test_get_absolute_media_path(test_directory): path = get_absolute_media_path('videos') - assert str(path).endswith('videos') + assert str(path) == f'{test_directory}/videos' + + # An absolute path is not changed. + path = get_absolute_media_path(path) + assert str(path) == f'{test_directory}/videos' def test_extract_video_duration(test_directory, corrupted_video_file): diff --git a/wrolpi/common.py b/wrolpi/common.py index 10152d7b..35a6f9ea 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -390,7 +390,7 @@ def set_test_media_directory(path): def get_media_directory() -> Path: """Get the media directory. - This will typically be /opt/wrolpi, or something else from the .env. + This will typically be /media/wrolpi, or something else from the .env. During testing, this function returns TEST_MEDIA_DIRECTORY. """ diff --git a/wrolpi/dates.py b/wrolpi/dates.py index 72e95403..cf7add7a 100644 --- a/wrolpi/dates.py +++ b/wrolpi/dates.py @@ -46,6 +46,9 @@ def strpdate(dt: str) -> datetime: """ Attempt to parse a datetime string. Tries to find a Timezone, if possible. DB requires timezone. """ + if not dt or dt in ('undefined', 'null'): + InvalidDatetime(f'Unable to parse empty datetime string: {dt}') + try: if dt.count('/') == 2: # No timezone info. diff --git a/wrolpi/test/test_dates.py b/wrolpi/test/test_dates.py index b0b1c962..6d2dd968 100644 --- a/wrolpi/test/test_dates.py +++ b/wrolpi/test/test_dates.py @@ -145,3 +145,9 @@ def test_invalid_strpdate(): with pytest.raises(InvalidDatetime): dates.strpdate('27/04/2024 18:52:55') + with pytest.raises(InvalidDatetime): + dates.strpdate('undefined') + with pytest.raises(InvalidDatetime): + dates.strpdate('null') + with pytest.raises(InvalidDatetime): + dates.strpdate(None) diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 7422a32a..6fc4f926 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -419,7 +419,7 @@ async def test_get_status(async_client, test_session): @pytest.mark.asyncio -async def test_download_crud(test_session, async_client, test_download_manager_config, tag_factory): +async def test_download_crud(test_session, async_client, test_download_manager_config, tag_factory, test_directory): """Test creating once-downloads and recurring downloads.""" tag1, tag2 = await tag_factory(), await tag_factory() @@ -432,6 +432,7 @@ async def test_download_crud(test_session, async_client, test_download_manager_c urls=['https://example.com/1', ], frequency=1_000, downloader='archive', + destination='archives/example.com', tag_names=None, # tag_names can be a None. ) request, response = await async_client.post('/api/download', content=json.dumps(body)) @@ -452,6 +453,8 @@ async def test_download_crud(test_session, async_client, test_download_manager_c download1, download2, download3 = test_session.query(Download).order_by(Download.url).all() assert not download1.settings + # destination is absolute + assert download1.destination == test_directory / 'archives/example.com' expected_download = dict(download1.__json__()).copy() # Update a download. @@ -461,7 +464,8 @@ async def test_download_crud(test_session, async_client, test_download_manager_c frequency=1_000, tag_names=[tag1.name, tag2.name], ) - expected_download.update({'url': 'https://example.com/1', 'tag_names': [tag1.name, tag2.name]}) + expected_download.update( + {'url': 'https://example.com/1', 'tag_names': [tag1.name, tag2.name], 'destination': None}) request, response = await async_client.put(f'/api/download/{download1.id}', json=body) assert response.status_code == HTTPStatus.NO_CONTENT test_session.flush([download2])