From f94fc0fc5ad0dfc96b25b6d7317a782b3d8493eb Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 11 Mar 2026 15:04:02 -0600 Subject: [PATCH] Handling when CDN provides FileDownloader with a UUID instead of the file name to download. --- wrolpi/common.py | 6 ++ wrolpi/downloader.py | 4 ++ wrolpi/files/test/test_downloader.py | 98 ++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) diff --git a/wrolpi/common.py b/wrolpi/common.py index a26705ff..8824c549 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -1732,6 +1732,12 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo: parsed = urlparse(url) info.name = parsed.path.split('/')[-1] + # If the resolved name has no extension but the original URL does, prefer the URL's filename. + # This prevents UUID-like names from CDN redirects. + url_name = urlparse(url).path.split('/')[-1] + if info.name and '.' not in info.name and '.' in url_name: + info.name = url_name + return info diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index a9c07615..efdba3e7 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -419,6 +419,10 @@ class Downloader: cmd = (*cmd, '-M', meta4_path, ) + else: + # Force aria2c to use our predicted filename. This avoids mismatches when + # the URL redirects to a CDN path that doesn't match the original filename. + cmd = (*cmd, '-o', output_path.name) result = await self.process_runner(download, cmd, destination) error = result.stderr.decode() diff --git a/wrolpi/files/test/test_downloader.py b/wrolpi/files/test/test_downloader.py index 80bc3618..645ef149 100644 --- a/wrolpi/files/test/test_downloader.py +++ b/wrolpi/files/test/test_downloader.py @@ -1,7 +1,9 @@ import hashlib from http import HTTPStatus +from unittest import mock from wrolpi import common +from wrolpi.common import DownloadFileInfo from wrolpi.downloader import Download from wrolpi.files.downloader import FileDownloader from wrolpi.test.common import skip_circleci @@ -88,3 +90,99 @@ async def test_file_downloader_meta4(test_session, make_files_structure, test_di assert (test_directory / f'downloads/{video_file.name}').read_bytes() == video_bytes # The meta4 was saved to a temporary file, but not saved. assert not (test_directory / 'downloads/foo.txt.meta4').exists() + + +async def test_get_download_info_uuid_fallback(): + """get_download_info falls back to the original URL's filename when the resolved name has no extension.""" + + class FakeResponse: + headers = { + 'Content-Type': 'application/octet-stream', + } + status = 200 + + from contextlib import asynccontextmanager + + @asynccontextmanager + async def mock_head(url, timeout): + # Simulate a CDN redirect where Location is a UUID path. + resp = FakeResponse() + resp.headers = {**resp.headers, 'Location': 'https://cdn.example.com/6f21bc74-aaa8-47f6-ae98-2aebf359df94'} + yield resp + + with mock.patch('wrolpi.common.aiohttp_head', mock_head): + info = await common.get_download_info('https://github.com/user/repo/releases/download/v1.0/somefile.tar.gz') + + # The UUID name should be replaced with the original URL's filename. + assert info.name == 'somefile.tar.gz' + + +async def test_file_downloader_aria2c_uses_output_flag(test_directory): + """The aria2c command includes -o with the predicted filename when not using metalink.""" + file_downloader = FileDownloader() + + destination = test_directory / 'downloads' + destination.mkdir() + + captured_cmd = None + + async def fake_process_runner(download, cmd, dest): + nonlocal captured_cmd + captured_cmd = cmd + # Create the expected output file so the downloader doesn't raise. + (destination / 'somefile.tar.gz').touch() + + class FakeResult: + return_code = 0 + stderr = b'' + + return FakeResult() + + fake_info = DownloadFileInfo(name='somefile.tar.gz', size=1024, status=200) + + with mock.patch('wrolpi.downloader.get_download_info', mock.AsyncMock(return_value=fake_info)), \ + mock.patch.object(file_downloader, 'process_runner', fake_process_runner): + download = mock.MagicMock() + url = 'https://github.com/user/repo/releases/download/v1.0/somefile.tar.gz' + await file_downloader.download_file(download, url, destination) + + # The command should include -o with the expected filename. + assert '-o' in captured_cmd + o_index = captured_cmd.index('-o') + assert captured_cmd[o_index + 1] == 'somefile.tar.gz' + + +async def test_file_downloader_meta4_no_output_flag(test_directory): + """The aria2c command should NOT include -o when using a metalink file.""" + file_downloader = FileDownloader() + + destination = test_directory / 'downloads' + destination.mkdir() + + captured_cmd = None + + async def fake_process_runner(download, cmd, dest): + nonlocal captured_cmd + captured_cmd = cmd + # Create the expected output file. + (destination / 'somefile.tar.gz').touch() + + class FakeResult: + return_code = 0 + stderr = b'' + + return FakeResult() + + fake_info = DownloadFileInfo(name='somefile.tar.gz', size=1024, status=200) + + with mock.patch('wrolpi.downloader.get_download_info', mock.AsyncMock(return_value=fake_info)), \ + mock.patch.object(file_downloader, 'get_meta4_contents', mock.AsyncMock(return_value=b'')), \ + mock.patch.object(file_downloader, 'process_runner', fake_process_runner): + download = mock.MagicMock() + url = 'https://example.com/somefile.tar.gz' + await file_downloader.download_file(download, url, destination) + + # When metalink is used, -o should NOT be in the command. + assert '-o' not in captured_cmd + # But -M should be present. + assert '-M' in captured_cmd