diff --git a/modules/map/downloader.py b/modules/map/downloader.py index b612de17..616f1e39 100644 --- a/modules/map/downloader.py +++ b/modules/map/downloader.py @@ -1,9 +1,12 @@ import pathlib +import re import tempfile from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, List, Optional +from urllib.parse import urlparse from xml.etree import ElementTree as ET +from xml.sax.saxutils import escape, quoteattr from sqlalchemy.orm import Session @@ -28,12 +31,18 @@ __all__ = [ logger = logger.getChild(__name__) METALINK_NS = {'ml': 'urn:ietf:params:xml:ns:metalink'} +# aria2c's exit code when a downloaded file does not match the meta4's hash. +ARIA2C_CHECKSUM_ERROR = 32 +# Map files are downloaded here, and moved into the map directory only once aria2c has checked them against the signed +# hash. A failed download must never leave a file where `prepare_download` would take it as already downloaded. +PARTIAL_DIRECTORY = '.partial' -async def verify_meta4_signature(meta4_contents: bytes) -> bool: - """Parse meta4 XML bytes, extract the hash and its GPG signature, and verify. +async def verified_meta4_sha256(meta4_contents: bytes) -> Optional[str]: + """Parse meta4 XML bytes, extract the sha-256 hash and its GPG signature, and verify. - Returns True if the signature is valid, False if invalid or missing.""" + Returns the hash if its signature is valid, None if invalid or missing. Only this hash is signed; nothing else in + the meta4 is trusted.""" try: root = ET.fromstring(meta4_contents) file_el = root.find('ml:file', METALINK_NS) @@ -48,14 +57,18 @@ async def verify_meta4_signature(meta4_contents: bytes) -> bool: logger.warning('meta4 missing element') return False if sig_el is None or sig_el.text is None: - logger.warning('meta4 missing element — skipping verification') + logger.warning('meta4 missing element') return False sha256_hash = hash_el.text.strip() signature = sig_el.text.strip() except ET.ParseError as e: logger.warning(f'Failed to parse meta4 XML: {e}') - return False + return None + + if not re.fullmatch(r'[0-9a-f]{64}', sha256_hash): + logger.warning('meta4 sha-256 hash is malformed') + return None # Write hash and signature to temp files for GPG verification. with tempfile.TemporaryDirectory() as tmpdir: @@ -64,7 +77,40 @@ async def verify_meta4_signature(meta4_contents: bytes) -> bool: hash_path.write_text(sha256_hash) sig_path.write_text(signature) - return await verify_gpg_signature(hash_path, sig_path) + if await verify_gpg_signature(hash_path, sig_path): + return sha256_hash + return None + + +def meta4_file_name(url: str) -> Optional[str]: + """The name aria2c saves `url` as, or None if the URL's file name is not a plain one.""" + name = PurePosixPath(urlparse(url).path).name + return name if re.fullmatch(r'\w[\w.-]*', name) else None + + +def make_meta4(url: str, name: str, sha256: str) -> bytes: + """A meta4 for aria2c holding only `url`, the `name` to save it as, and its signed `sha256` hash. + + The downloaded meta4 is never given to aria2c: it would also obey any other hash (e.g. a sha-512 of a tampered + file), pieces, files, or URLs in it, none of which are signed.""" + return ( + '\n' + '\n' + f' \n' + f' {sha256}\n' + f' {escape(url)}\n' + ' \n' + '\n' + ).encode() + + +def remove_rejected_partial(error: Exception, partial_path: pathlib.Path): + """Delete a partial download that aria2c rejected for not matching the signed hash; a retry would only reject it + again. Other failures keep it, so the next attempt resumes.""" + if f'return code {ARIA2C_CHECKSUM_ERROR}' in str(error): + logger.warning(f'Deleting download that did not match its signed hash: {partial_path}') + partial_path.unlink(missing_ok=True) + partial_path.with_name(f'{partial_path.name}.aria2').unlink(missing_ok=True) # --------------------------------------------------------------------------- @@ -312,34 +358,48 @@ class MapExtractDownloader(Downloader): # `download` with a URL. url = download.url - # Fetch meta4 once, GPG-verify the hash signature before downloading. + # Fetch meta4 once, GPG-verify the hash signature before downloading. A map file is never downloaded + # without a signed hash. meta4_contents = await self.get_meta4_contents(url) - if meta4_contents: - if not await verify_meta4_signature(meta4_contents): - return ExecutedMapExtract( - region=prepared.region, output_path=prepared.output_path, - map_directory=prepared.map_directory, - error=f'meta4 hash signature verification failed for {prepared.region}', - ) - logger.info(f'meta4 signature verified for {prepared.region}') - else: - logger.warning(f'No meta4 available for {prepared.region} — downloading without hash verification') + if not meta4_contents: + return ExecutedMapExtract( + region=prepared.region, output_path=prepared.output_path, + map_directory=prepared.map_directory, + error=f'No signed hash (meta4) available for {prepared.region}, not downloading', + ) + if not (sha256 := await verified_meta4_sha256(meta4_contents)): + return ExecutedMapExtract( + region=prepared.region, output_path=prepared.output_path, + map_directory=prepared.map_directory, + error=f'meta4 hash signature verification failed for {prepared.region}', + ) + logger.info(f'meta4 signature verified for {prepared.region}') + if not (name := meta4_file_name(url)): + return ExecutedMapExtract( + region=prepared.region, output_path=prepared.output_path, + map_directory=prepared.map_directory, + error=f'No usable file name in {url}', + ) - # Download via aria2c, passing the pre-verified meta4 for hash checking. + # Download via aria2c, which refuses a file that does not match the signed hash. logger.warning(f'Downloading {prepared.region} from {url}') - downloaded_path = None + partial_directory = prepared.map_directory / PARTIAL_DIRECTORY + partial_directory.mkdir(parents=True, exist_ok=True) try: downloaded_path = await self.download_file( - download, url, prepared.map_directory, - check_for_meta4=False, concurrent=1, meta4_xml=meta4_contents, + download, url, partial_directory, + check_for_meta4=False, concurrent=1, meta4_xml=make_meta4(url, name, sha256), ctx=ctx, ) + # Only the file aria2c saved under the meta4's name was checked against the signed hash. The path + # download_file reports comes from the server's response, so it may name some other file. + if downloaded_path != partial_directory / name: + raise RuntimeError(f'Downloaded {downloaded_path.name!r}, expected {name!r}') downloaded_path.rename(prepared.tmp_path) except Exception as e: logger.error(f'Download failed for {prepared.region}: {e}') prepared.tmp_path.unlink(missing_ok=True) - if downloaded_path: - downloaded_path.unlink(missing_ok=True) + remove_rejected_partial(e, partial_directory / name) return ExecutedMapExtract( region=prepared.region, output_path=prepared.output_path, map_directory=prepared.map_directory, @@ -475,32 +535,44 @@ class MapSearchIndexDownloader(Downloader): download = download if download is not None else Download(url=prepared.pmtiles_name) url = download.url - # Fetch and verify meta4 (GPG-signed hash). + # Fetch and verify meta4 (GPG-signed hash). A search index is never downloaded without a signed hash. meta4_contents = await self.get_meta4_contents(url) - if meta4_contents: - if not await verify_meta4_signature(meta4_contents): - return ExecutedMapSearchIndex( - region=prepared.region, - error=f'meta4 signature verification failed for {prepared.region} search index', - ) - logger.info(f'Search index meta4 verified for {prepared.region}') + if not meta4_contents: + return ExecutedMapSearchIndex( + region=prepared.region, + error=f'No signed hash (meta4) available for {prepared.region} search index, not downloading', + ) + if not (sha256 := await verified_meta4_sha256(meta4_contents)): + return ExecutedMapSearchIndex( + region=prepared.region, + error=f'meta4 signature verification failed for {prepared.region} search index', + ) + logger.info(f'Search index meta4 verified for {prepared.region}') + if not (name := meta4_file_name(url)): + return ExecutedMapSearchIndex( + region=prepared.region, + error=f'No usable file name in {url}', + ) - # Download via aria2c with meta4 hash verification. + # Download via aria2c, which refuses a file that does not match the signed hash. logger.warning(f'Downloading search index for {prepared.region} from {url}') - downloaded_path = None + partial_directory = prepared.map_directory / PARTIAL_DIRECTORY + partial_directory.mkdir(parents=True, exist_ok=True) try: downloaded_path = await self.download_file( - download, url, prepared.map_directory, - check_for_meta4=False, concurrent=1, meta4_xml=meta4_contents, + download, url, partial_directory, + check_for_meta4=False, concurrent=1, meta4_xml=make_meta4(url, name, sha256), ctx=ctx, ) + # Only the file aria2c saved under the meta4's name was checked against the signed hash. The path + # download_file reports comes from the server's response, so it may name some other file. + if downloaded_path != partial_directory / name: + raise RuntimeError(f'Downloaded {downloaded_path.name!r}, expected {name!r}') # Rename to match the local PMTiles filename. downloaded_path.rename(prepared.search_db_path) except Exception as e: logger.error(f'Search index download failed for {prepared.region}: {e}') - if downloaded_path: - downloaded_path.unlink(missing_ok=True) - prepared.search_db_path.unlink(missing_ok=True) + remove_rejected_partial(e, partial_directory / name) return ExecutedMapSearchIndex( region=prepared.region, error=f'Download failed: {str(e)[:500]}', diff --git a/modules/map/test/test_downloader.py b/modules/map/test/test_downloader.py index f7904e3c..cc4a69cd 100644 --- a/modules/map/test/test_downloader.py +++ b/modules/map/test/test_downloader.py @@ -1,12 +1,14 @@ """Phase-split tests for MapCatalogDownloader / MapExtractDownloader / MapSearchIndexDownloader. These exercise prepare_download and finalize_download in isolation — no async_client, -no test_download_manager, no real subprocess. Execute paths that drive download_file + -GPG verification + filesystem rename are integration territory and are deliberately -omitted; the parent download_file ctx threading is already covered by the FileDownloader -migration tests. +no test_download_manager, no real subprocess. Execute is tested only for refusing a download +without a signed hash; the paths that drive download_file + GPG verification + filesystem +rename are integration territory and are deliberately omitted; the parent download_file ctx +threading is already covered by the FileDownloader migration tests. """ +import hashlib from unittest import mock +from xml.etree import ElementTree import pytest @@ -23,6 +25,7 @@ from modules.map.downloader import ( map_search_index_downloader, ) from wrolpi.downloader import Download +from wrolpi.test.common import skip_ci # --------------------------------------------------------------------------- @@ -320,3 +323,201 @@ def test_extract_finalize_builds_search_index(test_session, test_directory, has_ assert result.success is True assert rebuild.called is not has_search_download + + +# --------------------------------------------------------------------------- +# Signed hash required before downloading +# --------------------------------------------------------------------------- + + +def _prepared_extract(map_dir): + return map_extract_downloader.prepare_download(None, Download( + url='https://x/us-2024-01.pmtiles', downloader='map_extract', + settings={'region': 'us', 'version': '2024-01'}, + )) + + +def _prepared_search_index(map_dir): + (map_dir / 'us-2024-01.pmtiles').write_bytes(b'pretend pmtiles') + return map_search_index_downloader.prepare_download(None, Download( + url='https://x/us-2024-01.search.db', downloader='map_search_index', + settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'}, + )) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('downloader,prepare', [ + (map_extract_downloader, _prepared_extract), + (map_search_index_downloader, _prepared_search_index), +], ids=['extract', 'search_index']) +@pytest.mark.parametrize('has_meta4,signature_valid', [ + (False, True), # The CDN has no .meta4 (or it could not be fetched). + (True, False), # The .meta4's hash is not signed by WROLPi. +], ids=['no_meta4', 'bad_signature']) +async def test_map_download_requires_signed_hash(test_directory, monkeypatch, downloader, prepare, has_meta4, + signature_valid): + """A map file is never downloaded unless its .meta4 carries a hash signed by WROLPi; aria2c then refuses any + file that does not match that hash.""" + map_dir = test_directory / 'maps' + map_dir.mkdir() + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + prepared = prepare(map_dir) + download = Download(url=f'https://x/{prepared.region}.file') + meta4 = INJECTED_META4.format(url=download.url).encode() if has_meta4 else None + + with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \ + mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=signature_valid)), \ + mock.patch.object(downloader, 'download_file', mock.AsyncMock()) as download_file: + executed = await downloader.execute_download(prepared, make_test_ctx(), download) + + assert executed.error and 'us' in executed.error + download_file.assert_not_called() + + +GENUINE_SHA256 = hashlib.sha256(b'genuine contents').hexdigest() +TAMPERED_SHA512 = hashlib.sha512(b'tampered contents').hexdigest() +# Only the sha-256 hash of a meta4 is signed; everything else in it is whatever the server sent. +INJECTED_META4 = f''' + + + {GENUINE_SHA256} + pretend signature + {TAMPERED_SHA512} + {TAMPERED_SHA512} + {{url}} + https://elsewhere.example/us-2024-01.pmtiles + + + https://elsewhere.example/extra.pmtiles + + +''' + + +@pytest.mark.asyncio +@pytest.mark.parametrize('downloader,prepare', [ + (map_extract_downloader, _prepared_extract), + (map_search_index_downloader, _prepared_search_index), +], ids=['extract', 'search_index']) +async def test_map_download_passes_only_the_signed_hash(test_directory, monkeypatch, downloader, prepare): + """aria2c gets a meta4 holding only the signed sha-256 hash and the download URL, never the downloaded meta4, + whose other hashes, pieces, files, and URLs are not signed.""" + map_dir = test_directory / 'maps' + map_dir.mkdir() + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + prepared = prepare(map_dir) + url = 'https://x/us-2024-01.pmtiles' + meta4 = INJECTED_META4.format(url=url).encode() + + with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \ + mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \ + mock.patch.object(downloader, 'download_file', mock.AsyncMock(side_effect=RuntimeError('stop'))) as df: + await downloader.execute_download(prepared, make_test_ctx(), Download(url=url)) + + passed = ElementTree.fromstring(df.call_args.kwargs['meta4_xml']) + ns = {'ml': 'urn:ietf:params:xml:ns:metalink'} + files = passed.findall('ml:file', ns) + assert len(files) == 1 + assert [(i.get('type'), i.text) for i in files[0].iter('{urn:ietf:params:xml:ns:metalink}hash')] \ + == [('sha-256', GENUINE_SHA256)] + assert [i.text for i in files[0].findall('ml:url', ns)] == [url] + assert files[0].find('ml:pieces', ns) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize('sha256', ['', 'not-hex', GENUINE_SHA256 + '00', f'{GENUINE_SHA256[:-1]}<']) +async def test_map_download_refuses_a_malformed_signed_hash(test_directory, monkeypatch, sha256): + map_dir = test_directory / 'maps' + map_dir.mkdir() + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + prepared = _prepared_extract(map_dir) + meta4 = INJECTED_META4.format(url='https://x/us-2024-01.pmtiles').replace(GENUINE_SHA256, sha256, 1) + + with mock.patch.object(map_extract_downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4.encode())), \ + mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \ + mock.patch.object(map_extract_downloader, 'download_file', mock.AsyncMock()) as download_file: + executed = await map_extract_downloader.execute_download(prepared, make_test_ctx(), Download(url='https://x/a')) + + assert executed.error + download_file.assert_not_called() + + +@skip_ci +@pytest.mark.asyncio +async def test_map_download_refuses_tampered_file_with_injected_hash(test_directory, monkeypatch, + simple_file_server): + """Real aria2c: a server that adds a sha-512 of its tampered file to a genuinely signed meta4 does not get the + tampered file accepted, the rejected file is not left where the next attempt would take it as downloaded, and a + retry downloads the genuine file.""" + map_dir = test_directory / 'maps' + map_dir.mkdir() + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + served = test_directory / 'us-2024-01.pmtiles' + served.write_bytes(b'tampered contents') + host, port = simple_file_server.server_address + url = f'http://{host}:{port}/us-2024-01.pmtiles' # noqa + + meta4 = INJECTED_META4.format(url=url).encode() + with mock.patch.object(map_extract_downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \ + mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)): + prepared = _prepared_extract(map_dir) + executed = await map_extract_downloader.execute_download(prepared, make_test_ctx(), Download(url=url)) + assert executed.error + assert list(map_dir.glob('*.pmtiles')) == [] + # The next attempt does not skip the download. + assert not _prepared_extract(map_dir).already_done + + # The server now serves the genuine file. + served.write_bytes(b'genuine contents') + prepared = _prepared_extract(map_dir) + executed = await map_extract_downloader.execute_download(prepared, make_test_ctx(), Download(url=url)) + assert not executed.error, executed.error + assert prepared.output_path.read_bytes() == b'genuine contents' + + +@pytest.mark.asyncio +@pytest.mark.parametrize('downloader,prepare', [ + (map_extract_downloader, _prepared_extract), + (map_search_index_downloader, _prepared_search_index), +], ids=['extract', 'search_index']) +async def test_map_download_moves_only_the_checked_file(test_directory, monkeypatch, downloader, prepare): + """Only the file aria2c checked (named in the meta4 WROLPi wrote) is moved into place, never the path + download_file reports, which comes from the server's HEAD response (Content-Disposition, redirects).""" + map_dir = test_directory / 'maps' + map_dir.mkdir() + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + prepared = prepare(map_dir) + url = 'https://x/us-2024-01.pmtiles' + # An unchecked leftover of an interrupted download, which the server names in its HEAD response. + leftover = map_dir / '.partial/other.pmtiles' + leftover.parent.mkdir() + leftover.write_bytes(b'tampered contents') + + meta4 = INJECTED_META4.format(url=url).encode() + with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \ + mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \ + mock.patch.object(downloader, 'download_file', mock.AsyncMock(return_value=leftover)): + executed = await downloader.execute_download(prepared, make_test_ctx(), Download(url=url)) + + assert executed.error + assert leftover.read_bytes() == b'tampered contents' + assert {i.name for i in map_dir.iterdir()} <= {'.partial', 'us-2024-01.pmtiles'} + assert not any(i.read_bytes() == b'tampered contents' for i in map_dir.glob('*.*') if i.is_file()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('url', ['https://x/', 'https://x/..', 'https://x/.hidden', 'https://x/a%2F..']) +async def test_map_download_refuses_unusable_file_name(test_directory, monkeypatch, url): + map_dir = test_directory / 'maps' + map_dir.mkdir() + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + prepared = _prepared_extract(map_dir) + + meta4 = INJECTED_META4.format(url=url).encode() + with mock.patch.object(map_extract_downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \ + mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \ + mock.patch.object(map_extract_downloader, 'download_file', mock.AsyncMock()) as download_file: + executed = await map_extract_downloader.execute_download(prepared, make_test_ctx(), Download(url=url)) + + assert executed.error + download_file.assert_not_called() diff --git a/wrolpi/files/test/test_downloader.py b/wrolpi/files/test/test_downloader.py index d6ce3e9d..0beb123d 100644 --- a/wrolpi/files/test/test_downloader.py +++ b/wrolpi/files/test/test_downloader.py @@ -7,7 +7,7 @@ import pytest from wrolpi import common from wrolpi.common import DownloadFileInfo from wrolpi.downloader import Download -from wrolpi.errors import UnrecoverableDownloadError +from wrolpi.errors import DownloadError, UnrecoverableDownloadError from wrolpi.files.downloader import FileDownloader, PreparedFile, file_downloader from wrolpi.test.common import skip_ci @@ -95,6 +95,33 @@ async def test_file_downloader_meta4(test_session, make_files_structure, test_di assert not (test_directory / 'downloads/foo.txt.meta4').exists() + +@skip_ci +@pytest.mark.parametrize('meta4_name', ['served.bin', 'other.bin'], ids=['same_name', 'renamed']) +async def test_download_file_meta4_hash_mismatch(test_directory, simple_file_server, meta4_name): + """aria2c refuses a file that does not match the meta4's hash, even when the meta4's (unsigned) name differs from + the URL's. Map downloads rely on this: only the hash in a meta4 is signed.""" + host, port = simple_file_server.server_address + url = f'http://{host}:{port}' # noqa + served = test_directory / 'served.bin' + served.write_bytes(b'tampered contents') + genuine = test_directory / 'genuine.bin' + genuine.write_bytes(b'genuine contents') + destination = test_directory / 'downloads' + destination.mkdir() + + meta4 = META4_XML.format(url=url, name=meta4_name, size=served.stat().st_size, + sha256=hashlib.sha256(b'genuine contents').hexdigest()) + if meta4_name == 'other.bin': + # The renamed meta4 points aria2c at a genuine file, while the URL serves a tampered one. + meta4 = meta4.replace(f'{url}/other.bin', f'{url}/genuine.bin') + + from wrolpi.conftest import make_test_ctx + with pytest.raises(DownloadError): + await FileDownloader().download_file(mock.MagicMock(), f'{url}/served.bin', destination, + meta4_xml=meta4.encode(), ctx=make_test_ctx()) + + async def test_get_download_info_uuid_fallback(): """get_download_info falls back to the original URL's filename when the resolved name has no extension."""