Map downloads are checked against the hash in the signed manifest (#934)

* Map downloads are checked against the hash in the signed manifest

A meta4 signs only a bare hash, so any genuinely signed meta4 (an older version of a region, another region) passed
its own check.  The manifest now lists each file's sha256; it is signed as a whole, so the hash is tied to the URL.
The catalog copies each hash into its download's settings, and aria2c checks the file against it.  Downloads from a
manifest without hashes still require the file's signed meta4.

build_map_regions.py writes the hashes, and hash_map_manifest.py adds them to the published manifest from the
verified meta4s without rebuilding any maps.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* hash_map_manifest.py hashes each file's bytes instead of trusting its meta4's name

Only the bare hash in a meta4 is signed, so an older version's meta4 with its name rewritten (or terrain's, whose name
never changes) passed the name and duplicate-hash checks.  Each file is now hashed from a local copy of the published
files (--files) or downloaded from the CDN, and must match its signed meta4.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-11-01 20:38:47 -07:00 • committed by Roland
parent ee6c384dc6
commit e6b7b9e5dd
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
5 changed files with 341 additions and 55 deletions

View file

@ -31,6 +31,7 @@ __all__ = [
logger = logger.getChild(__name__)
METALINK_NS = {'ml': 'urn:ietf:params:xml:ns:metalink'}
SHA256_PATTERN = re.compile(r'[0-9a-f]{64}')
# aria2c's exit code when a downloaded file does not match the meta4's hash.
ARIA2C_CHECKSUM_ERROR = 32
# Map files are downloaded here, and moved into the map directory only once aria2c has checked them against the signed
@ -66,7 +67,7 @@ async def verified_meta4_sha256(meta4_contents: bytes) -> Optional[str]:
logger.warning(f'Failed to parse meta4 XML: {e}')
return None
if not re.fullmatch(r'[0-9a-f]{64}', sha256_hash):
if not SHA256_PATTERN.fullmatch(sha256_hash):
logger.warning('meta4 sha-256 hash is malformed')
return None
@ -82,6 +83,27 @@ async def verified_meta4_sha256(meta4_contents: bytes) -> Optional[str]:
return None
async def get_signed_sha256(downloader: Downloader, url: str, manifest_sha256: Optional[str],
label: str) -> tuple[Optional[str], Optional[str]]:
"""The signed sha-256 hash a map file must match, or an error explaining why there is none.
The hash from the signed manifest is used when it has one: a meta4 signs only a bare hash, so any genuinely signed
meta4 (an older version, another region) would pass its own check. Downloads from a manifest without hashes use
the file's signed meta4."""
if manifest_sha256:
if SHA256_PATTERN.fullmatch(manifest_sha256):
return manifest_sha256, None
return None, f'Malformed hash in the manifest for {label}'
meta4_contents = await downloader.get_meta4_contents(url)
if not meta4_contents:
return None, f'No signed hash (meta4) available for {label}, not downloading'
if not (sha256 := await verified_meta4_sha256(meta4_contents)):
return None, f'meta4 hash signature verification failed for {label}'
logger.info(f'meta4 signature verified for {label}')
return sha256, None
def meta4_file_name(url: str) -> Optional[str]:
"""The name aria2c saves `url` as, or None if the URL's file name is not a plain one."""
name = PurePosixPath(urlparse(url).path).name
@ -175,6 +197,7 @@ class MapCatalogDownloader(Downloader):
version = manifest.get('version', '')
manifest_regions = manifest.get('regions', {})
search_indexes = manifest.get('search_indexes', {})
hashes = manifest.get('sha256', {})
if not manifest_regions:
return DownloadResult(success=False, error='Manifest does not contain regions')
@ -221,6 +244,7 @@ class MapCatalogDownloader(Downloader):
'region': region,
'version': version,
'search_index_url': search_indexes.get(region, ''),
'sha256': hashes.get(region_url),
}
child.renew(reset_attempts=True)
created += 1
@ -239,7 +263,8 @@ class MapCatalogDownloader(Downloader):
session, search_url, reset_attempts=True, override_skip=True)
search_child.downloader = 'map_search_index'
search_child.location = '/map/manage'
search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name}
search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name,
'sha256': hashes.get(search_url)}
search_child.renew(reset_attempts=True)
logger.info(f'Created search index download for {region}')
@ -273,6 +298,7 @@ class PreparedMapExtract:
output_name: str
output_path: pathlib.Path
tmp_path: pathlib.Path
sha256: Optional[str] = None # From the signed manifest.
already_done: bool = False
error: Optional[str] = None
@ -297,9 +323,7 @@ class MapExtractDownloader(Downloader):
Settings contain: region, version — set by MapCatalogDownloader.
Fetches the meta4 file once, GPG-verifies the hash signature, then passes
the same meta4 bytes to aria2c for download hash verification. This ensures
the hash that was GPG-verified is the same hash aria2c checks against.
aria2c checks the file against a signed hash (see `get_signed_sha256`), given to it in a meta4 that WROLPi writes.
"""
name = 'map_extract'
@ -329,6 +353,7 @@ class MapExtractDownloader(Downloader):
prepared = PreparedMapExtract(
region=region, version=version, map_directory=map_directory,
output_name=output_name, output_path=output_path, tmp_path=tmp_path,
sha256=settings.get('sha256'),
)
# Skip if versioned file already exists.
@ -358,22 +383,13 @@ class MapExtractDownloader(Downloader):
# `download` with a URL.
url = download.url
# Fetch meta4 once, GPG-verify the hash signature before downloading. A map file is never downloaded
# without a signed hash.
meta4_contents = await self.get_meta4_contents(url)
if not meta4_contents:
# A map file is never downloaded without a signed hash.
sha256, error = await get_signed_sha256(self, url, prepared.sha256, prepared.region)
if error:
return ExecutedMapExtract(
region=prepared.region, output_path=prepared.output_path,
map_directory=prepared.map_directory,
error=f'No signed hash (meta4) available for {prepared.region}, not downloading',
map_directory=prepared.map_directory, error=error,
)
if not (sha256 := await verified_meta4_sha256(meta4_contents)):
return ExecutedMapExtract(
region=prepared.region, output_path=prepared.output_path,
map_directory=prepared.map_directory,
error=f'meta4 hash signature verification failed for {prepared.region}',
)
logger.info(f'meta4 signature verified for {prepared.region}')
if not (name := meta4_file_name(url)):
return ExecutedMapExtract(
region=prepared.region, output_path=prepared.output_path,
@ -476,6 +492,7 @@ class PreparedMapSearchIndex:
map_directory: pathlib.Path
pmtiles_path: pathlib.Path
search_db_path: pathlib.Path
sha256: Optional[str] = None # From the signed manifest.
already_done: bool = False
error: Optional[str] = None
@ -509,6 +526,7 @@ class MapSearchIndexDownloader(Downloader):
pmtiles_path=map_directory / pmtiles_name if pmtiles_name else map_directory,
search_db_path=(map_directory / pmtiles_name).with_suffix('.search.db')
if pmtiles_name else map_directory,
sha256=settings.get('sha256'),
)
if not region or not pmtiles_name:
@ -535,19 +553,10 @@ class MapSearchIndexDownloader(Downloader):
download = download if download is not None else Download(url=prepared.pmtiles_name)
url = download.url
# Fetch and verify meta4 (GPG-signed hash). A search index is never downloaded without a signed hash.
meta4_contents = await self.get_meta4_contents(url)
if not meta4_contents:
return ExecutedMapSearchIndex(
region=prepared.region,
error=f'No signed hash (meta4) available for {prepared.region} search index, not downloading',
)
if not (sha256 := await verified_meta4_sha256(meta4_contents)):
return ExecutedMapSearchIndex(
region=prepared.region,
error=f'meta4 signature verification failed for {prepared.region} search index',
)
logger.info(f'Search index meta4 verified for {prepared.region}')
# A search index is never downloaded without a signed hash.
sha256, error = await get_signed_sha256(self, url, prepared.sha256, f'{prepared.region} search index')
if error:
return ExecutedMapSearchIndex(region=prepared.region, error=error)
if not (name := meta4_file_name(url)):
return ExecutedMapSearchIndex(
region=prepared.region,

View file

@ -330,18 +330,18 @@ def test_extract_finalize_builds_search_index(test_session, test_directory, has_
# ---------------------------------------------------------------------------
def _prepared_extract(map_dir):
def _prepared_extract(map_dir, sha256: str = None):
return map_extract_downloader.prepare_download(None, Download(
url='https://x/us-2024-01.pmtiles', downloader='map_extract',
settings={'region': 'us', 'version': '2024-01'},
settings={'region': 'us', 'version': '2024-01', 'sha256': sha256},
))
def _prepared_search_index(map_dir):
def _prepared_search_index(map_dir, sha256: str = None):
(map_dir / 'us-2024-01.pmtiles').write_bytes(b'pretend pmtiles')
return map_search_index_downloader.prepare_download(None, Download(
url='https://x/us-2024-01.search.db', downloader='map_search_index',
settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'},
settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles', 'sha256': sha256},
))
@ -521,3 +521,75 @@ async def test_map_download_refuses_unusable_file_name(test_directory, monkeypat
assert executed.error
download_file.assert_not_called()
# ---------------------------------------------------------------------------
# Hashes from the signed manifest
# ---------------------------------------------------------------------------
MANIFEST_SHA256 = hashlib.sha256(b'manifest contents').hexdigest()
def test_catalog_finalize_copies_manifest_hashes(test_session, test_directory, test_download_manager, monkeypatch):
"""Each child download gets its file's hash from the signed manifest."""
map_dir = test_directory / 'maps'
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
pmtiles_url, search_url = 'https://x/us-2024-02.pmtiles', 'https://x/us-2024-02.search.db'
executed = ExecutedMapCatalog(
manifest={'regions': {'us': pmtiles_url}, 'search_indexes': {'us': search_url}, 'version': '2024-02',
'sha256': {pmtiles_url: MANIFEST_SHA256, search_url: GENUINE_SHA256}},
subscribed_regions=[{'region': 'us'}],
)
download = Download(url='https://x/manifest.json', downloader='map_catalog')
result = map_catalog_downloader.finalize_download(test_session, download, executed)
assert result.success is True, result.error
hashes = {i.downloader: i.settings.get('sha256') for i in test_session.query(Download)}
assert hashes == {'map_extract': MANIFEST_SHA256, 'map_search_index': GENUINE_SHA256}
@pytest.mark.asyncio
@pytest.mark.parametrize('downloader,prepare', [
(map_extract_downloader, _prepared_extract),
(map_search_index_downloader, _prepared_search_index),
], ids=['extract', 'search_index'])
async def test_map_download_uses_the_manifest_hash(test_directory, monkeypatch, downloader, prepare):
"""A hash from the signed manifest is the one aria2c checks. A meta4 signs only a bare hash, so any genuine
meta4 (an older version, another region) would pass its own signature check."""
map_dir = test_directory / 'maps'
map_dir.mkdir()
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
prepared = prepare(map_dir, sha256=MANIFEST_SHA256)
url = 'https://x/us-2024-01.pmtiles'
# A genuinely signed meta4, but of some other file.
meta4 = INJECTED_META4.format(url=url).encode()
with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)) as get_meta4, \
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \
mock.patch.object(downloader, 'download_file', mock.AsyncMock(side_effect=RuntimeError('stop'))) as df:
await downloader.execute_download(prepared, make_test_ctx(), Download(url=url))
get_meta4.assert_not_called()
passed = ElementTree.fromstring(df.call_args.kwargs['meta4_xml'])
assert [i.text for i in passed.iter('{urn:ietf:params:xml:ns:metalink}hash')] == [MANIFEST_SHA256]
@pytest.mark.asyncio
@pytest.mark.parametrize('downloader,prepare', [
(map_extract_downloader, _prepared_extract),
(map_search_index_downloader, _prepared_search_index),
], ids=['extract', 'search_index'])
async def test_map_download_refuses_a_malformed_manifest_hash(test_directory, monkeypatch, downloader, prepare):
map_dir = test_directory / 'maps'
map_dir.mkdir()
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
prepared = prepare(map_dir, sha256='not-a-hash')
meta4 = INJECTED_META4.format(url='https://x/a.pmtiles').encode()
with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \
mock.patch.object(downloader, 'download_file', mock.AsyncMock()) as download_file:
executed = await downloader.execute_download(prepared, make_test_ctx(), Download(url='https://x/a.pmtiles'))
assert executed.error
download_file.assert_not_called()

View file

@ -0,0 +1,108 @@
import hashlib
from unittest import mock
import pytest
from wrolpi.scripts.hash_map_manifest import manifest_hashes
META4 = '''<?xml version="1.0" encoding="UTF-8"?>
<metalink xmlns="urn:ietf:params:xml:ns:metalink">
<file name="{name}">
<hash type="sha-256">{sha256}</hash>
<signature mediatype="application/pgp-signature">pretend signature</signature>
</file>
</metalink>
'''
FILES = {'us-1.pmtiles': b'pmtiles', 'us-1.search.db': b'search', 'terrain-z0-8.pmtiles': b'terrain'}
def _sha256(contents: bytes) -> str:
return hashlib.sha256(contents).hexdigest()
@pytest.fixture
def served(test_directory, simple_file_server):
"""The published files, served over HTTP, and their manifest."""
host, port = simple_file_server.server_address
base = f'http://{host}:{port}' # noqa
for name, contents in FILES.items():
(test_directory / name).write_bytes(contents)
manifest = {
'regions': {'us': f'{base}/us-1.pmtiles', 'terrain-z0-8': f'{base}/terrain-z0-8.pmtiles'},
'search_indexes': {'us': f'{base}/us-1.search.db'},
}
# Each file's genuine, signed meta4.
meta4s = {f'{base}/{name}': META4.format(name=name, sha256=_sha256(contents)) for name, contents in FILES.items()}
return base, manifest, meta4s
async def _hashes(manifest: dict, meta4s: dict, files_directory=None, signature_valid: bool = True):
async def get_meta4_contents(url):
return meta4s[url].encode() if url in meta4s else None
with mock.patch('wrolpi.scripts.hash_map_manifest.Downloader.get_meta4_contents', get_meta4_contents), \
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=signature_valid)):
return await manifest_hashes(manifest, files_directory)
@pytest.mark.asyncio
@pytest.mark.parametrize('local', [False, True], ids=['downloaded', 'local'])
async def test_manifest_hashes(served, tmp_path, local):
"""Each hash is of the file's bytes (a local copy of the published files, or downloaded), and matches its signed
meta4."""
base, manifest, meta4s = served
files_directory = None
if local:
files_directory = tmp_path / 'published'
files_directory.mkdir()
for name, contents in FILES.items():
(files_directory / name).write_bytes(contents)
# The local copy is read; the server's file is not.
manifest['regions']['us'] = 'http://127.0.0.1:9/us-1.pmtiles'
meta4s[manifest['regions']['us']] = meta4s.pop(f'{base}/us-1.pmtiles')
assert await _hashes(manifest, meta4s, files_directory) == {
manifest['regions']['us']: _sha256(b'pmtiles'),
f'{base}/terrain-z0-8.pmtiles': _sha256(b'terrain'),
f'{base}/us-1.search.db': _sha256(b'search'),
}
@pytest.mark.asyncio
async def test_manifest_hashes_refuses_a_missing_meta4(served):
base, manifest, meta4s = served
del meta4s[f'{base}/us-1.search.db']
with pytest.raises(RuntimeError):
await _hashes(manifest, meta4s)
@pytest.mark.asyncio
async def test_manifest_hashes_refuses_an_unsigned_meta4(served):
_, manifest, meta4s = served
with pytest.raises(RuntimeError):
await _hashes(manifest, meta4s, signature_valid=False)
@pytest.mark.asyncio
@pytest.mark.parametrize('name,old_meta4_name', [
# An older version's signed meta4, its unsigned name rewritten to the current file's.
('us-1.search.db', 'us-1.search.db'),
# Terrain is unversioned, so an older signed meta4 already has the current name.
('terrain-z0-8.pmtiles', 'terrain-z0-8.pmtiles'),
])
async def test_manifest_hashes_refuses_an_older_signed_meta4(served, name, old_meta4_name):
"""Only a bare hash is signed in a meta4, so only the file's bytes show which file a signed hash belongs to."""
base, manifest, meta4s = served
meta4s[f'{base}/{name}'] = META4.format(name=old_meta4_name, sha256=_sha256(b'older version'))
with pytest.raises(RuntimeError):
await _hashes(manifest, meta4s)
@pytest.mark.asyncio
async def test_manifest_hashes_refuses_a_local_copy_that_differs(served, tmp_path):
_, manifest, meta4s = served
files_directory = tmp_path / 'published'
files_directory.mkdir()
(files_directory / 'us-1.pmtiles').write_bytes(b'something else')
with pytest.raises(RuntimeError):
await _hashes(manifest, meta4s, files_directory)

View file

@ -10,7 +10,7 @@ Example:
This script:
1. Reads region definitions from modules/map/catalog.py
2. Runs `pmtiles extract` for each bbox region
3. Generates manifest.json with direct download URLs for each region
3. Generates manifest.json with direct download URLs and the sha256 of each file, then signs it
4. Prints a summary of all extracted files and sizes
Requirements:
@ -87,8 +87,8 @@ def generate_meta4(name: str, url: str, sha256: str, size: int, signature: str)
)
def write_meta4(file_path: str, cdn_url: str):
"""Compute hash, sign it, and write a signed .meta4 file alongside the given file."""
def write_meta4(file_path: str, cdn_url: str) -> str:
"""Compute hash, sign it, and write a signed .meta4 file alongside the given file. Returns the hash."""
name = os.path.basename(file_path)
size = os.path.getsize(file_path)
sha256 = sha256_file(file_path)
@ -99,6 +99,24 @@ def write_meta4(file_path: str, cdn_url: str):
with open(meta4_path, 'w') as f:
f.write(meta4_xml)
print(f' Written {os.path.basename(meta4_path)}')
return sha256
def sign_manifest(manifest_path: str):
"""Write the manifest's detached GPG signature beside it, or exit."""
sig_path = f'{manifest_path}.sig'
print(f'Signing manifest...')
result = subprocess.run(
['gpg', '-u', 'roland@learningselfreliance.com', '--detach-sign', '--armor', '-o', sig_path, manifest_path],
capture_output=True,
)
if result.returncode == 0:
print(f'Signature written to: {sig_path}')
else:
print(f'ERROR: GPG signing failed: {result.stderr.decode().strip()}')
print(f'Sign manually and re-run, or fix GPG key access:'
f'\n gpg -u roland@learningselfreliance.com --detach-sign --armor -o {sig_path} {manifest_path}')
sys.exit(1)
def extract_region(planet_path: str, output_path: str, bbox: str, threads: int = 4) -> bool:
@ -269,6 +287,8 @@ def main():
# Extract each bbox region.
manifest_regions = {}
search_index_urls = {}
# WROLPi checks each download against its hash here; a meta4's signed hash is not tied to any one file.
hashes = {}
results = []
failed = []
@ -298,7 +318,7 @@ def main():
size = os.path.getsize(output_path)
cdn_url = f'{cdn_base}/{output_name}'
write_meta4(output_path, cdn_url)
hashes[cdn_url] = write_meta4(output_path, cdn_url)
results.append((r['name'], region, size))
manifest_regions[region] = cdn_url
@ -308,7 +328,7 @@ def main():
enrich_search_db_with_regions(search_db_path)
search_db_name = os.path.basename(str(search_db_path))
search_cdn_url = f'{cdn_base}/{search_db_name}'
write_meta4(str(search_db_path), search_cdn_url)
hashes[search_cdn_url] = write_meta4(str(search_db_path), search_cdn_url)
search_index_urls[region] = search_cdn_url
# Add terrain to manifest if the file exists in output_dir.
@ -317,7 +337,7 @@ def main():
terrain_path = os.path.join(output_dir, terrain_name)
if os.path.isfile(terrain_path):
cdn_url = f'{cdn_base}/{terrain_name}'
write_meta4(terrain_path, cdn_url)
hashes[cdn_url] = write_meta4(terrain_path, cdn_url)
manifest_regions[terrain_region['region']] = cdn_url
size = os.path.getsize(terrain_path)
results.append((terrain_region['name'], terrain_region['region'], size))
@ -336,26 +356,14 @@ def main():
# Add search index URLs.
if search_index_urls:
manifest['search_indexes'] = search_index_urls
manifest['sha256'] = hashes
manifest_path = os.path.join(output_dir, 'manifest.json')
with open(manifest_path, 'w') as f:
json.dump(manifest, f, indent=2)
print(f'\nManifest written to: {manifest_path}')
# Sign manifest with GPG.
sig_path = f'{manifest_path}.sig'
print(f'Signing manifest...')
result = subprocess.run(
['gpg', '-u', 'roland@learningselfreliance.com', '--detach-sign', '--armor', '-o', sig_path, manifest_path],
capture_output=True,
)
if result.returncode == 0:
print(f'Signature written to: {sig_path}')
else:
print(f'ERROR: GPG signing failed: {result.stderr.decode().strip()}')
print(f'Sign manually and re-run, or fix GPG key access:'
f'\n gpg -u roland@learningselfreliance.com --detach-sign --armor -o {sig_path} {manifest_path}')
sys.exit(1)
sign_manifest(manifest_path)
# Print summary.
print(f'\n{"="*70}')

View file

@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""Add each file's sha256 to the published map manifest, without rebuilding any maps.
Usage:
python wrolpi/scripts/hash_map_manifest.py <output_dir> [--files DIR] [--manifest-url URL]
Each file is hashed from its bytes: the copy in --files (e.g. the build_map_regions.py output_dir) when it has one,
otherwise downloaded from the CDN. Every hash must match the file's signed .meta4, and the published manifest's
signature is verified, then the manifest is written to output_dir with a `sha256` of each URL, and signed. Upload
manifest.json and manifest.json.sig to the CDN.
A .meta4 alone cannot show which file its hash belongs to: only the bare hash is signed, so an older version's .meta4
(with its unsigned name rewritten) passes its signature check.
Manifests from build_map_regions.py already include the hashes.
"""
import argparse
import asyncio
import hashlib
import json
import os
import pathlib
import sys
import requests
# Add project root to path so we can import the map module.
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
from modules.map.catalog import MANIFEST_URL
from modules.map.downloader import meta4_file_name, verified_meta4_sha256
from modules.map.lib import fetch_manifest
from wrolpi.downloader import Downloader
from wrolpi.scripts.build_map_regions import sign_manifest
def file_sha256(url: str, files_directory: pathlib.Path = None) -> str:
"""sha256 of the file's local copy in `files_directory`, if it has one, otherwise of the file downloaded from
`url`."""
h = hashlib.sha256()
if files_directory and (path := files_directory / meta4_file_name(url)).is_file():
with path.open('rb') as f:
for chunk in iter(lambda: f.read(8 * 1024 * 1024), b''):
h.update(chunk)
else:
with requests.get(url, stream=True, timeout=60) as response:
response.raise_for_status()
for chunk in response.iter_content(8 * 1024 * 1024):
h.update(chunk)
return h.hexdigest()
async def manifest_hashes(manifest: dict, files_directory: pathlib.Path = None) -> dict:
"""{url: sha256} of every file in the manifest, hashed from the file's bytes, each matching its signed .meta4."""
urls = [*manifest['regions'].values(), *manifest.get('search_indexes', {}).values()]
hashes = {}
for url in urls:
meta4 = await Downloader.get_meta4_contents(url)
if not meta4 or not (signed_sha256 := await verified_meta4_sha256(meta4)):
raise RuntimeError(f'No validly signed meta4 for {url}')
if (sha256 := file_sha256(url, files_directory)) != signed_sha256:
raise RuntimeError(f'{url} has sha256 {sha256}, its signed meta4 says {signed_sha256}')
hashes[url] = sha256
print(f'{sha256} {url}', flush=True)
return hashes
def main():
parser = argparse.ArgumentParser(description='Add each file\'s sha256 to the published map manifest.')
parser.add_argument('output_dir', help='Directory to write manifest.json and manifest.json.sig to')
parser.add_argument('--files', type=pathlib.Path,
help='Directory of the published files; any not in it are downloaded from the CDN')
parser.add_argument('--manifest-url', default=MANIFEST_URL, help=f'(default: {MANIFEST_URL})')
args = parser.parse_args()
manifest = asyncio.run(fetch_manifest(args.manifest_url))
manifest['sha256'] = asyncio.run(manifest_hashes(manifest, args.files))
os.makedirs(args.output_dir, exist_ok=True)
manifest_path = os.path.join(os.path.abspath(args.output_dir), 'manifest.json')
with open(manifest_path, 'w') as f:
json.dump(manifest, f, indent=2)
print(f'\nManifest written to: {manifest_path}')
sign_manifest(manifest_path)
print('Upload manifest.json and manifest.json.sig to the CDN.')
if __name__ == '__main__':
main()