wrolpi/modules/map/test/test_downloader.py
lrnselfreliance 21006eae09
Map files are never downloaded without a signed hash (#930)
* Map files are never downloaded without a signed hash

A region, terrain, or search index download whose .meta4 is missing or unfetchable now fails (and retries later)
instead of downloading with TLS as its only check.  The search index downloader had no fallback branch at all, so it
downloaded with no hash whenever the .meta4 was missing.

A real aria2c test pins what this relies on: aria2c refuses a file that does not match the signed hash, even when
the meta4's unsigned name differs from the URL.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* aria2c checks map files only against the signed hash, and a rejected file is never kept

Only the sha-256 hash in a meta4 is signed, but aria2c was given the whole downloaded meta4.  A sha-512 of a tampered
file added beside the signed sha-256 made aria2c accept the tampered file.  aria2c now gets a meta4 built from the
signed hash and the download URL alone.

A rejected download was left in the map directory under its final name, where the next attempt took it as already
downloaded.  Map files now download into map/.partial and move into place only once aria2c accepts them; a file that
fails the hash check is deleted so a retry starts over, other failures keep it so the download resumes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Only the map file aria2c checked is moved into place

download_file reports the path named by the server's HEAD response (Content-Disposition, redirects), while aria2c
saves and checks the file under the name in the meta4 WROLPi writes.  When they differed, an unchecked file (such as
an interrupted download's leftover in map/.partial) could be moved into place.  The meta4 name now comes from the
URL, must be a plain file name, and any other reported path fails the download.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-01 13:03:16 -07:00

523 lines
23 KiB
Python

"""Phase-split tests for MapCatalogDownloader / MapExtractDownloader / MapSearchIndexDownloader.
These exercise prepare_download and finalize_download in isolation — no async_client,
no test_download_manager, no real subprocess. Execute is tested only for refusing a download
without a signed hash; the paths that drive download_file + GPG verification + filesystem
rename are integration territory and are deliberately omitted; the parent download_file ctx
threading is already covered by the FileDownloader migration tests.
"""
import hashlib
from unittest import mock
from xml.etree import ElementTree
import pytest
from modules.archive.conftest import make_test_ctx
from modules.map.downloader import (
ExecutedMapCatalog,
ExecutedMapExtract,
ExecutedMapSearchIndex,
PreparedMapCatalog,
PreparedMapExtract,
PreparedMapSearchIndex,
map_catalog_downloader,
map_extract_downloader,
map_search_index_downloader,
)
from wrolpi.downloader import Download
from wrolpi.test.common import skip_ci
# ---------------------------------------------------------------------------
# MapCatalogDownloader
# ---------------------------------------------------------------------------
def test_catalog_prepare_extracts_subscribed_regions(test_session):
"""prepare_download surfaces the user's region subscriptions."""
download = Download(
url='https://maps.example.com/manifest.json',
downloader='map_catalog',
settings={'regions': [{'region': 'us'}, {'region': 'eu'}]},
)
prepared = map_catalog_downloader.prepare_download(test_session, download)
assert isinstance(prepared, PreparedMapCatalog)
assert prepared.url == download.url
assert prepared.subscribed_regions == [{'region': 'us'}, {'region': 'eu'}]
def test_catalog_prepare_handles_no_regions_in_settings(test_session):
"""No 'regions' key → empty list (handled in finalize as success no-op)."""
download = Download(
url='https://maps.example.com/manifest.json',
downloader='map_catalog',
settings={},
)
prepared = map_catalog_downloader.prepare_download(test_session, download)
assert prepared.subscribed_regions == []
@pytest.mark.asyncio
async def test_catalog_execute_handles_fetch_failure(test_session):
"""A fetch_manifest exception ends up on executed.error rather than raising."""
async def fake_fetch(url):
raise RuntimeError('CDN down')
prepared = PreparedMapCatalog(url='https://x', subscribed_regions=[{'region': 'us'}])
with mock.patch('modules.map.lib.fetch_manifest', fake_fetch):
executed = await map_catalog_downloader.execute_download(prepared, make_test_ctx())
assert executed.manifest is None
assert 'CDN down' in (executed.error or '')
def test_catalog_finalize_no_subscribed_regions_returns_success(test_session):
"""User has no subscriptions → success no-op (skips DB writes)."""
download = Download(url='https://x', downloader='map_catalog')
executed = ExecutedMapCatalog(
manifest={'regions': {'us': 'https://x/us.pmtiles'}, 'version': '2024-01'},
subscribed_regions=[],
)
result = map_catalog_downloader.finalize_download(test_session, download, executed)
assert result.success is True
def test_catalog_finalize_no_manifest_regions_returns_failure(test_session):
"""Manifest came back without a regions block → failure."""
download = Download(url='https://x', downloader='map_catalog')
executed = ExecutedMapCatalog(
manifest={'regions': {}, 'version': '2024-01'},
subscribed_regions=[{'region': 'us'}],
)
result = map_catalog_downloader.finalize_download(test_session, download, executed)
assert result.success is False
assert 'does not contain regions' in (result.error or '')
def test_catalog_finalize_propagates_executed_error(test_session):
"""An error raised during execute_download flows into the DownloadResult."""
download = Download(url='https://x', downloader='map_catalog')
executed = ExecutedMapCatalog(
manifest=None, subscribed_regions=[{'region': 'us'}],
error='Failed to fetch manifest: timeout',
)
result = map_catalog_downloader.finalize_download(test_session, download, executed)
assert result.success is False
assert 'timeout' in (result.error or '')
# ---------------------------------------------------------------------------
# MapExtractDownloader
# ---------------------------------------------------------------------------
def test_extract_prepare_missing_region_sets_error(test_session, test_directory, monkeypatch):
"""A download without settings.region fails fast in prepare."""
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: test_directory / 'maps')
download = Download(
url='https://x/foo.pmtiles',
downloader='map_extract',
settings={},
)
prepared = map_extract_downloader.prepare_download(test_session, download)
assert prepared.error == 'No region in download settings'
def test_extract_prepare_already_done(test_session, test_directory, monkeypatch):
"""If the versioned output already exists, prepare flags already_done so execute
becomes a no-op."""
map_dir = test_directory / 'maps'
map_dir.mkdir(parents=True, exist_ok=True)
(map_dir / 'us-2024-01.pmtiles').write_bytes(b'pretend pmtiles')
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
download = Download(
url='https://x/us.pmtiles',
downloader='map_extract',
settings={'region': 'us', 'version': '2024-01'},
)
prepared = map_extract_downloader.prepare_download(test_session, download)
assert prepared.already_done is True
assert prepared.error is None
assert prepared.output_name == 'us-2024-01.pmtiles'
def test_extract_prepare_cleans_leftover_tmp(test_session, test_directory, monkeypatch):
"""A leftover .tmp file from a prior crashed run is removed in prepare."""
map_dir = test_directory / 'maps'
map_dir.mkdir(parents=True, exist_ok=True)
leftover = map_dir / 'us-2024-01.pmtiles.tmp'
leftover.write_bytes(b'partial')
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
download = Download(
url='https://x/us.pmtiles',
downloader='map_extract',
settings={'region': 'us', 'version': '2024-01'},
)
prepared = map_extract_downloader.prepare_download(test_session, download)
assert prepared.already_done is False
assert not leftover.exists()
assert prepared.tmp_path == leftover # path is set, just not on disk anymore
def test_extract_finalize_skipped(test_session, test_directory):
"""skipped → success without touching DB."""
download = Download(url='https://x', downloader='map_extract')
executed = ExecutedMapExtract(
region='us', output_path=test_directory / 'us.pmtiles',
map_directory=test_directory, skipped=True,
)
result = map_extract_downloader.finalize_download(test_session, download, executed)
assert result.success is True
assert result.location == '/map'
def test_extract_finalize_propagates_error(test_session, test_directory):
"""An execute-phase error becomes a failure DownloadResult."""
download = Download(url='https://x', downloader='map_extract')
executed = ExecutedMapExtract(
region='us', output_path=test_directory / 'us.pmtiles',
map_directory=test_directory,
error='Download failed: timeout',
)
result = map_extract_downloader.finalize_download(test_session, download, executed)
assert result.success is False
assert 'timeout' in (result.error or '')
# ---------------------------------------------------------------------------
# MapSearchIndexDownloader
# ---------------------------------------------------------------------------
def test_search_index_prepare_missing_settings_sets_error(test_session, test_directory, monkeypatch):
"""No region or pmtiles_name in settings → error."""
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: test_directory / 'maps')
download = Download(
url='https://x/us.search.db',
downloader='map_search_index',
settings={'region': 'us'}, # missing pmtiles_name
)
prepared = map_search_index_downloader.prepare_download(test_session, download)
assert prepared.error == 'Missing region or pmtiles_name in settings'
def test_search_index_prepare_pmtiles_missing_sets_error(test_session, test_directory, monkeypatch):
"""If the companion PMTiles file isn't on disk yet, prepare fails so the manager
can defer + retry."""
map_dir = test_directory / 'maps'
map_dir.mkdir(parents=True, exist_ok=True)
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
download = Download(
url='https://x/us.search.db',
downloader='map_search_index',
settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'},
)
prepared = map_search_index_downloader.prepare_download(test_session, download)
assert prepared.error and 'PMTiles file not found' in prepared.error
def test_search_index_prepare_already_done(test_session, test_directory, monkeypatch):
"""Both pmtiles and search.db exist → already_done short-circuit."""
map_dir = test_directory / 'maps'
map_dir.mkdir(parents=True, exist_ok=True)
(map_dir / 'us-2024-01.pmtiles').write_bytes(b'tiles')
(map_dir / 'us-2024-01.search.db').write_bytes(b'db')
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
download = Download(
url='https://x/us.search.db',
downloader='map_search_index',
settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'},
)
prepared = map_search_index_downloader.prepare_download(test_session, download)
assert prepared.already_done is True
assert prepared.error is None
def test_search_index_finalize_skipped(test_session):
"""skipped → success."""
download = Download(url='https://x', downloader='map_search_index')
executed = ExecutedMapSearchIndex(region='us', skipped=True)
result = map_search_index_downloader.finalize_download(test_session, download, executed)
assert result.success is True
def test_search_index_finalize_error(test_session):
"""Error path returns failure with the error message."""
download = Download(url='https://x', downloader='map_search_index')
executed = ExecutedMapSearchIndex(region='us', error='Download failed: 500')
result = map_search_index_downloader.finalize_download(test_session, download, executed)
assert result.success is False
assert '500' in (result.error or '')
def test_catalog_finalize_replaces_stale_extract_download(test_session, test_directory, test_download_manager,
monkeypatch):
"""A new manifest version replaces the region's old extract Download (found by its settings' region)."""
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: test_directory / 'maps')
test_session.add(Download(url='https://x/us-2024-01.pmtiles', downloader='map_extract',
settings={'region': 'us', 'version': '2024-01'}))
test_session.commit()
download = Download(url='https://x/manifest.json', downloader='map_catalog')
executed = ExecutedMapCatalog(
manifest={'regions': {'us': 'https://x/us-2024-02.pmtiles'}, 'version': '2024-02'},
subscribed_regions=[{'region': 'us'}],
)
result = map_catalog_downloader.finalize_download(test_session, download, executed)
assert result.success is True, result.error
downloads = test_session.query(Download).filter_by(downloader='map_extract').all()
assert [(i.url, i.settings['version']) for i in downloads] == [('https://x/us-2024-02.pmtiles', '2024-02')]
@pytest.mark.parametrize('has_search_download', [True, False])
def test_extract_finalize_builds_search_index(test_session, test_directory, has_search_download):
"""The search index is built locally unless a search index Download exists for the region."""
if has_search_download:
test_session.add(Download(url='https://x/us.search.db', downloader='map_search_index',
settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'}))
test_session.commit()
download = Download(url='https://x/us.pmtiles', downloader='map_extract')
executed = ExecutedMapExtract(
region='us', output_path=test_directory / 'us-2024-01.pmtiles', map_directory=test_directory,
)
with mock.patch('modules.map.search.rebuild_search_index') as rebuild:
result = map_extract_downloader.finalize_download(test_session, download, executed)
assert result.success is True
assert rebuild.called is not has_search_download
# ---------------------------------------------------------------------------
# Signed hash required before downloading
# ---------------------------------------------------------------------------
def _prepared_extract(map_dir):
return map_extract_downloader.prepare_download(None, Download(
url='https://x/us-2024-01.pmtiles', downloader='map_extract',
settings={'region': 'us', 'version': '2024-01'},
))
def _prepared_search_index(map_dir):
(map_dir / 'us-2024-01.pmtiles').write_bytes(b'pretend pmtiles')
return map_search_index_downloader.prepare_download(None, Download(
url='https://x/us-2024-01.search.db', downloader='map_search_index',
settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'},
))
@pytest.mark.asyncio
@pytest.mark.parametrize('downloader,prepare', [
(map_extract_downloader, _prepared_extract),
(map_search_index_downloader, _prepared_search_index),
], ids=['extract', 'search_index'])
@pytest.mark.parametrize('has_meta4,signature_valid', [
(False, True), # The CDN has no .meta4 (or it could not be fetched).
(True, False), # The .meta4's hash is not signed by WROLPi.
], ids=['no_meta4', 'bad_signature'])
async def test_map_download_requires_signed_hash(test_directory, monkeypatch, downloader, prepare, has_meta4,
signature_valid):
"""A map file is never downloaded unless its .meta4 carries a hash signed by WROLPi; aria2c then refuses any
file that does not match that hash."""
map_dir = test_directory / 'maps'
map_dir.mkdir()
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
prepared = prepare(map_dir)
download = Download(url=f'https://x/{prepared.region}.file')
meta4 = INJECTED_META4.format(url=download.url).encode() if has_meta4 else None
with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=signature_valid)), \
mock.patch.object(downloader, 'download_file', mock.AsyncMock()) as download_file:
executed = await downloader.execute_download(prepared, make_test_ctx(), download)
assert executed.error and 'us' in executed.error
download_file.assert_not_called()
GENUINE_SHA256 = hashlib.sha256(b'genuine contents').hexdigest()
TAMPERED_SHA512 = hashlib.sha512(b'tampered contents').hexdigest()
# Only the sha-256 hash of a meta4 is signed; everything else in it is whatever the server sent.
INJECTED_META4 = f'''<?xml version="1.0" encoding="UTF-8"?>
<metalink xmlns="urn:ietf:params:xml:ns:metalink">
<file name="us-2024-01.pmtiles">
<hash type="sha-256">{GENUINE_SHA256}</hash>
<signature mediatype="application/pgp-signature">pretend signature</signature>
<hash type="sha-512">{TAMPERED_SHA512}</hash>
<pieces length="1" type="sha-512"><hash>{TAMPERED_SHA512}</hash></pieces>
<url>{{url}}</url>
<url>https://elsewhere.example/us-2024-01.pmtiles</url>
</file>
<file name="extra.pmtiles">
<url>https://elsewhere.example/extra.pmtiles</url>
</file>
</metalink>
'''
@pytest.mark.asyncio
@pytest.mark.parametrize('downloader,prepare', [
(map_extract_downloader, _prepared_extract),
(map_search_index_downloader, _prepared_search_index),
], ids=['extract', 'search_index'])
async def test_map_download_passes_only_the_signed_hash(test_directory, monkeypatch, downloader, prepare):
"""aria2c gets a meta4 holding only the signed sha-256 hash and the download URL, never the downloaded meta4,
whose other hashes, pieces, files, and URLs are not signed."""
map_dir = test_directory / 'maps'
map_dir.mkdir()
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
prepared = prepare(map_dir)
url = 'https://x/us-2024-01.pmtiles'
meta4 = INJECTED_META4.format(url=url).encode()
with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \
mock.patch.object(downloader, 'download_file', mock.AsyncMock(side_effect=RuntimeError('stop'))) as df:
await downloader.execute_download(prepared, make_test_ctx(), Download(url=url))
passed = ElementTree.fromstring(df.call_args.kwargs['meta4_xml'])
ns = {'ml': 'urn:ietf:params:xml:ns:metalink'}
files = passed.findall('ml:file', ns)
assert len(files) == 1
assert [(i.get('type'), i.text) for i in files[0].iter('{urn:ietf:params:xml:ns:metalink}hash')] \
== [('sha-256', GENUINE_SHA256)]
assert [i.text for i in files[0].findall('ml:url', ns)] == [url]
assert files[0].find('ml:pieces', ns) is None
@pytest.mark.asyncio
@pytest.mark.parametrize('sha256', ['', 'not-hex', GENUINE_SHA256 + '00', f'{GENUINE_SHA256[:-1]}<'])
async def test_map_download_refuses_a_malformed_signed_hash(test_directory, monkeypatch, sha256):
map_dir = test_directory / 'maps'
map_dir.mkdir()
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
prepared = _prepared_extract(map_dir)
meta4 = INJECTED_META4.format(url='https://x/us-2024-01.pmtiles').replace(GENUINE_SHA256, sha256, 1)
with mock.patch.object(map_extract_downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4.encode())), \
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \
mock.patch.object(map_extract_downloader, 'download_file', mock.AsyncMock()) as download_file:
executed = await map_extract_downloader.execute_download(prepared, make_test_ctx(), Download(url='https://x/a'))
assert executed.error
download_file.assert_not_called()
@skip_ci
@pytest.mark.asyncio
async def test_map_download_refuses_tampered_file_with_injected_hash(test_directory, monkeypatch,
simple_file_server):
"""Real aria2c: a server that adds a sha-512 of its tampered file to a genuinely signed meta4 does not get the
tampered file accepted, the rejected file is not left where the next attempt would take it as downloaded, and a
retry downloads the genuine file."""
map_dir = test_directory / 'maps'
map_dir.mkdir()
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
served = test_directory / 'us-2024-01.pmtiles'
served.write_bytes(b'tampered contents')
host, port = simple_file_server.server_address
url = f'http://{host}:{port}/us-2024-01.pmtiles' # noqa
meta4 = INJECTED_META4.format(url=url).encode()
with mock.patch.object(map_extract_downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)):
prepared = _prepared_extract(map_dir)
executed = await map_extract_downloader.execute_download(prepared, make_test_ctx(), Download(url=url))
assert executed.error
assert list(map_dir.glob('*.pmtiles')) == []
# The next attempt does not skip the download.
assert not _prepared_extract(map_dir).already_done
# The server now serves the genuine file.
served.write_bytes(b'genuine contents')
prepared = _prepared_extract(map_dir)
executed = await map_extract_downloader.execute_download(prepared, make_test_ctx(), Download(url=url))
assert not executed.error, executed.error
assert prepared.output_path.read_bytes() == b'genuine contents'
@pytest.mark.asyncio
@pytest.mark.parametrize('downloader,prepare', [
(map_extract_downloader, _prepared_extract),
(map_search_index_downloader, _prepared_search_index),
], ids=['extract', 'search_index'])
async def test_map_download_moves_only_the_checked_file(test_directory, monkeypatch, downloader, prepare):
"""Only the file aria2c checked (named in the meta4 WROLPi wrote) is moved into place, never the path
download_file reports, which comes from the server's HEAD response (Content-Disposition, redirects)."""
map_dir = test_directory / 'maps'
map_dir.mkdir()
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
prepared = prepare(map_dir)
url = 'https://x/us-2024-01.pmtiles'
# An unchecked leftover of an interrupted download, which the server names in its HEAD response.
leftover = map_dir / '.partial/other.pmtiles'
leftover.parent.mkdir()
leftover.write_bytes(b'tampered contents')
meta4 = INJECTED_META4.format(url=url).encode()
with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \
mock.patch.object(downloader, 'download_file', mock.AsyncMock(return_value=leftover)):
executed = await downloader.execute_download(prepared, make_test_ctx(), Download(url=url))
assert executed.error
assert leftover.read_bytes() == b'tampered contents'
assert {i.name for i in map_dir.iterdir()} <= {'.partial', 'us-2024-01.pmtiles'}
assert not any(i.read_bytes() == b'tampered contents' for i in map_dir.glob('*.*') if i.is_file())
@pytest.mark.asyncio
@pytest.mark.parametrize('url', ['https://x/', 'https://x/..', 'https://x/.hidden', 'https://x/a%2F..'])
async def test_map_download_refuses_unusable_file_name(test_directory, monkeypatch, url):
map_dir = test_directory / 'maps'
map_dir.mkdir()
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
prepared = _prepared_extract(map_dir)
meta4 = INJECTED_META4.format(url=url).encode()
with mock.patch.object(map_extract_downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \
mock.patch.object(map_extract_downloader, 'download_file', mock.AsyncMock()) as download_file:
executed = await map_extract_downloader.execute_download(prepared, make_test_ctx(), Download(url=url))
assert executed.error
download_file.assert_not_called()