diff --git a/modules/map/downloader.py b/modules/map/downloader.py index 616f1e39..3c2d9c4d 100644 --- a/modules/map/downloader.py +++ b/modules/map/downloader.py @@ -31,6 +31,7 @@ __all__ = [ logger = logger.getChild(__name__) METALINK_NS = {'ml': 'urn:ietf:params:xml:ns:metalink'} +SHA256_PATTERN = re.compile(r'[0-9a-f]{64}') # aria2c's exit code when a downloaded file does not match the meta4's hash. ARIA2C_CHECKSUM_ERROR = 32 # Map files are downloaded here, and moved into the map directory only once aria2c has checked them against the signed @@ -66,7 +67,7 @@ async def verified_meta4_sha256(meta4_contents: bytes) -> Optional[str]: logger.warning(f'Failed to parse meta4 XML: {e}') return None - if not re.fullmatch(r'[0-9a-f]{64}', sha256_hash): + if not SHA256_PATTERN.fullmatch(sha256_hash): logger.warning('meta4 sha-256 hash is malformed') return None @@ -82,6 +83,27 @@ async def verified_meta4_sha256(meta4_contents: bytes) -> Optional[str]: return None +async def get_signed_sha256(downloader: Downloader, url: str, manifest_sha256: Optional[str], + label: str) -> tuple[Optional[str], Optional[str]]: + """The signed sha-256 hash a map file must match, or an error explaining why there is none. + + The hash from the signed manifest is used when it has one: a meta4 signs only a bare hash, so any genuinely signed + meta4 (an older version, another region) would pass its own check. Downloads from a manifest without hashes use + the file's signed meta4.""" + if manifest_sha256: + if SHA256_PATTERN.fullmatch(manifest_sha256): + return manifest_sha256, None + return None, f'Malformed hash in the manifest for {label}' + + meta4_contents = await downloader.get_meta4_contents(url) + if not meta4_contents: + return None, f'No signed hash (meta4) available for {label}, not downloading' + if not (sha256 := await verified_meta4_sha256(meta4_contents)): + return None, f'meta4 hash signature verification failed for {label}' + logger.info(f'meta4 signature verified for {label}') + return sha256, None + + def meta4_file_name(url: str) -> Optional[str]: """The name aria2c saves `url` as, or None if the URL's file name is not a plain one.""" name = PurePosixPath(urlparse(url).path).name @@ -175,6 +197,7 @@ class MapCatalogDownloader(Downloader): version = manifest.get('version', '') manifest_regions = manifest.get('regions', {}) search_indexes = manifest.get('search_indexes', {}) + hashes = manifest.get('sha256', {}) if not manifest_regions: return DownloadResult(success=False, error='Manifest does not contain regions') @@ -221,6 +244,7 @@ class MapCatalogDownloader(Downloader): 'region': region, 'version': version, 'search_index_url': search_indexes.get(region, ''), + 'sha256': hashes.get(region_url), } child.renew(reset_attempts=True) created += 1 @@ -239,7 +263,8 @@ class MapCatalogDownloader(Downloader): session, search_url, reset_attempts=True, override_skip=True) search_child.downloader = 'map_search_index' search_child.location = '/map/manage' - search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name} + search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name, + 'sha256': hashes.get(search_url)} search_child.renew(reset_attempts=True) logger.info(f'Created search index download for {region}') @@ -273,6 +298,7 @@ class PreparedMapExtract: output_name: str output_path: pathlib.Path tmp_path: pathlib.Path + sha256: Optional[str] = None # From the signed manifest. already_done: bool = False error: Optional[str] = None @@ -297,9 +323,7 @@ class MapExtractDownloader(Downloader): Settings contain: region, version — set by MapCatalogDownloader. - Fetches the meta4 file once, GPG-verifies the hash signature, then passes - the same meta4 bytes to aria2c for download hash verification. This ensures - the hash that was GPG-verified is the same hash aria2c checks against. + aria2c checks the file against a signed hash (see `get_signed_sha256`), given to it in a meta4 that WROLPi writes. """ name = 'map_extract' @@ -329,6 +353,7 @@ class MapExtractDownloader(Downloader): prepared = PreparedMapExtract( region=region, version=version, map_directory=map_directory, output_name=output_name, output_path=output_path, tmp_path=tmp_path, + sha256=settings.get('sha256'), ) # Skip if versioned file already exists. @@ -358,22 +383,13 @@ class MapExtractDownloader(Downloader): # `download` with a URL. url = download.url - # Fetch meta4 once, GPG-verify the hash signature before downloading. A map file is never downloaded - # without a signed hash. - meta4_contents = await self.get_meta4_contents(url) - if not meta4_contents: + # A map file is never downloaded without a signed hash. + sha256, error = await get_signed_sha256(self, url, prepared.sha256, prepared.region) + if error: return ExecutedMapExtract( region=prepared.region, output_path=prepared.output_path, - map_directory=prepared.map_directory, - error=f'No signed hash (meta4) available for {prepared.region}, not downloading', + map_directory=prepared.map_directory, error=error, ) - if not (sha256 := await verified_meta4_sha256(meta4_contents)): - return ExecutedMapExtract( - region=prepared.region, output_path=prepared.output_path, - map_directory=prepared.map_directory, - error=f'meta4 hash signature verification failed for {prepared.region}', - ) - logger.info(f'meta4 signature verified for {prepared.region}') if not (name := meta4_file_name(url)): return ExecutedMapExtract( region=prepared.region, output_path=prepared.output_path, @@ -476,6 +492,7 @@ class PreparedMapSearchIndex: map_directory: pathlib.Path pmtiles_path: pathlib.Path search_db_path: pathlib.Path + sha256: Optional[str] = None # From the signed manifest. already_done: bool = False error: Optional[str] = None @@ -509,6 +526,7 @@ class MapSearchIndexDownloader(Downloader): pmtiles_path=map_directory / pmtiles_name if pmtiles_name else map_directory, search_db_path=(map_directory / pmtiles_name).with_suffix('.search.db') if pmtiles_name else map_directory, + sha256=settings.get('sha256'), ) if not region or not pmtiles_name: @@ -535,19 +553,10 @@ class MapSearchIndexDownloader(Downloader): download = download if download is not None else Download(url=prepared.pmtiles_name) url = download.url - # Fetch and verify meta4 (GPG-signed hash). A search index is never downloaded without a signed hash. - meta4_contents = await self.get_meta4_contents(url) - if not meta4_contents: - return ExecutedMapSearchIndex( - region=prepared.region, - error=f'No signed hash (meta4) available for {prepared.region} search index, not downloading', - ) - if not (sha256 := await verified_meta4_sha256(meta4_contents)): - return ExecutedMapSearchIndex( - region=prepared.region, - error=f'meta4 signature verification failed for {prepared.region} search index', - ) - logger.info(f'Search index meta4 verified for {prepared.region}') + # A search index is never downloaded without a signed hash. + sha256, error = await get_signed_sha256(self, url, prepared.sha256, f'{prepared.region} search index') + if error: + return ExecutedMapSearchIndex(region=prepared.region, error=error) if not (name := meta4_file_name(url)): return ExecutedMapSearchIndex( region=prepared.region, diff --git a/modules/map/test/test_downloader.py b/modules/map/test/test_downloader.py index cc4a69cd..52d90f47 100644 --- a/modules/map/test/test_downloader.py +++ b/modules/map/test/test_downloader.py @@ -330,18 +330,18 @@ def test_extract_finalize_builds_search_index(test_session, test_directory, has_ # --------------------------------------------------------------------------- -def _prepared_extract(map_dir): +def _prepared_extract(map_dir, sha256: str = None): return map_extract_downloader.prepare_download(None, Download( url='https://x/us-2024-01.pmtiles', downloader='map_extract', - settings={'region': 'us', 'version': '2024-01'}, + settings={'region': 'us', 'version': '2024-01', 'sha256': sha256}, )) -def _prepared_search_index(map_dir): +def _prepared_search_index(map_dir, sha256: str = None): (map_dir / 'us-2024-01.pmtiles').write_bytes(b'pretend pmtiles') return map_search_index_downloader.prepare_download(None, Download( url='https://x/us-2024-01.search.db', downloader='map_search_index', - settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'}, + settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles', 'sha256': sha256}, )) @@ -521,3 +521,75 @@ async def test_map_download_refuses_unusable_file_name(test_directory, monkeypat assert executed.error download_file.assert_not_called() + + +# --------------------------------------------------------------------------- +# Hashes from the signed manifest +# --------------------------------------------------------------------------- + +MANIFEST_SHA256 = hashlib.sha256(b'manifest contents').hexdigest() + + +def test_catalog_finalize_copies_manifest_hashes(test_session, test_directory, test_download_manager, monkeypatch): + """Each child download gets its file's hash from the signed manifest.""" + map_dir = test_directory / 'maps' + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + pmtiles_url, search_url = 'https://x/us-2024-02.pmtiles', 'https://x/us-2024-02.search.db' + executed = ExecutedMapCatalog( + manifest={'regions': {'us': pmtiles_url}, 'search_indexes': {'us': search_url}, 'version': '2024-02', + 'sha256': {pmtiles_url: MANIFEST_SHA256, search_url: GENUINE_SHA256}}, + subscribed_regions=[{'region': 'us'}], + ) + download = Download(url='https://x/manifest.json', downloader='map_catalog') + result = map_catalog_downloader.finalize_download(test_session, download, executed) + assert result.success is True, result.error + + hashes = {i.downloader: i.settings.get('sha256') for i in test_session.query(Download)} + assert hashes == {'map_extract': MANIFEST_SHA256, 'map_search_index': GENUINE_SHA256} + + +@pytest.mark.asyncio +@pytest.mark.parametrize('downloader,prepare', [ + (map_extract_downloader, _prepared_extract), + (map_search_index_downloader, _prepared_search_index), +], ids=['extract', 'search_index']) +async def test_map_download_uses_the_manifest_hash(test_directory, monkeypatch, downloader, prepare): + """A hash from the signed manifest is the one aria2c checks. A meta4 signs only a bare hash, so any genuine + meta4 (an older version, another region) would pass its own signature check.""" + map_dir = test_directory / 'maps' + map_dir.mkdir() + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + prepared = prepare(map_dir, sha256=MANIFEST_SHA256) + url = 'https://x/us-2024-01.pmtiles' + # A genuinely signed meta4, but of some other file. + meta4 = INJECTED_META4.format(url=url).encode() + + with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)) as get_meta4, \ + mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \ + mock.patch.object(downloader, 'download_file', mock.AsyncMock(side_effect=RuntimeError('stop'))) as df: + await downloader.execute_download(prepared, make_test_ctx(), Download(url=url)) + + get_meta4.assert_not_called() + passed = ElementTree.fromstring(df.call_args.kwargs['meta4_xml']) + assert [i.text for i in passed.iter('{urn:ietf:params:xml:ns:metalink}hash')] == [MANIFEST_SHA256] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('downloader,prepare', [ + (map_extract_downloader, _prepared_extract), + (map_search_index_downloader, _prepared_search_index), +], ids=['extract', 'search_index']) +async def test_map_download_refuses_a_malformed_manifest_hash(test_directory, monkeypatch, downloader, prepare): + map_dir = test_directory / 'maps' + map_dir.mkdir() + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + prepared = prepare(map_dir, sha256='not-a-hash') + meta4 = INJECTED_META4.format(url='https://x/a.pmtiles').encode() + + with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \ + mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \ + mock.patch.object(downloader, 'download_file', mock.AsyncMock()) as download_file: + executed = await downloader.execute_download(prepared, make_test_ctx(), Download(url='https://x/a.pmtiles')) + + assert executed.error + download_file.assert_not_called() diff --git a/modules/map/test/test_hash_map_manifest.py b/modules/map/test/test_hash_map_manifest.py new file mode 100644 index 00000000..9131e7f2 --- /dev/null +++ b/modules/map/test/test_hash_map_manifest.py @@ -0,0 +1,108 @@ +import hashlib +from unittest import mock + +import pytest + +from wrolpi.scripts.hash_map_manifest import manifest_hashes + +META4 = ''' + + + {sha256} + pretend signature + + +''' +FILES = {'us-1.pmtiles': b'pmtiles', 'us-1.search.db': b'search', 'terrain-z0-8.pmtiles': b'terrain'} + + +def _sha256(contents: bytes) -> str: + return hashlib.sha256(contents).hexdigest() + + +@pytest.fixture +def served(test_directory, simple_file_server): + """The published files, served over HTTP, and their manifest.""" + host, port = simple_file_server.server_address + base = f'http://{host}:{port}' # noqa + for name, contents in FILES.items(): + (test_directory / name).write_bytes(contents) + manifest = { + 'regions': {'us': f'{base}/us-1.pmtiles', 'terrain-z0-8': f'{base}/terrain-z0-8.pmtiles'}, + 'search_indexes': {'us': f'{base}/us-1.search.db'}, + } + # Each file's genuine, signed meta4. + meta4s = {f'{base}/{name}': META4.format(name=name, sha256=_sha256(contents)) for name, contents in FILES.items()} + return base, manifest, meta4s + + +async def _hashes(manifest: dict, meta4s: dict, files_directory=None, signature_valid: bool = True): + async def get_meta4_contents(url): + return meta4s[url].encode() if url in meta4s else None + + with mock.patch('wrolpi.scripts.hash_map_manifest.Downloader.get_meta4_contents', get_meta4_contents), \ + mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=signature_valid)): + return await manifest_hashes(manifest, files_directory) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('local', [False, True], ids=['downloaded', 'local']) +async def test_manifest_hashes(served, tmp_path, local): + """Each hash is of the file's bytes (a local copy of the published files, or downloaded), and matches its signed + meta4.""" + base, manifest, meta4s = served + files_directory = None + if local: + files_directory = tmp_path / 'published' + files_directory.mkdir() + for name, contents in FILES.items(): + (files_directory / name).write_bytes(contents) + # The local copy is read; the server's file is not. + manifest['regions']['us'] = 'http://127.0.0.1:9/us-1.pmtiles' + meta4s[manifest['regions']['us']] = meta4s.pop(f'{base}/us-1.pmtiles') + + assert await _hashes(manifest, meta4s, files_directory) == { + manifest['regions']['us']: _sha256(b'pmtiles'), + f'{base}/terrain-z0-8.pmtiles': _sha256(b'terrain'), + f'{base}/us-1.search.db': _sha256(b'search'), + } + + +@pytest.mark.asyncio +async def test_manifest_hashes_refuses_a_missing_meta4(served): + base, manifest, meta4s = served + del meta4s[f'{base}/us-1.search.db'] + with pytest.raises(RuntimeError): + await _hashes(manifest, meta4s) + + +@pytest.mark.asyncio +async def test_manifest_hashes_refuses_an_unsigned_meta4(served): + _, manifest, meta4s = served + with pytest.raises(RuntimeError): + await _hashes(manifest, meta4s, signature_valid=False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('name,old_meta4_name', [ + # An older version's signed meta4, its unsigned name rewritten to the current file's. + ('us-1.search.db', 'us-1.search.db'), + # Terrain is unversioned, so an older signed meta4 already has the current name. + ('terrain-z0-8.pmtiles', 'terrain-z0-8.pmtiles'), +]) +async def test_manifest_hashes_refuses_an_older_signed_meta4(served, name, old_meta4_name): + """Only a bare hash is signed in a meta4, so only the file's bytes show which file a signed hash belongs to.""" + base, manifest, meta4s = served + meta4s[f'{base}/{name}'] = META4.format(name=old_meta4_name, sha256=_sha256(b'older version')) + with pytest.raises(RuntimeError): + await _hashes(manifest, meta4s) + + +@pytest.mark.asyncio +async def test_manifest_hashes_refuses_a_local_copy_that_differs(served, tmp_path): + _, manifest, meta4s = served + files_directory = tmp_path / 'published' + files_directory.mkdir() + (files_directory / 'us-1.pmtiles').write_bytes(b'something else') + with pytest.raises(RuntimeError): + await _hashes(manifest, meta4s, files_directory) diff --git a/wrolpi/scripts/build_map_regions.py b/wrolpi/scripts/build_map_regions.py index 17bc6c2e..74bbbf44 100755 --- a/wrolpi/scripts/build_map_regions.py +++ b/wrolpi/scripts/build_map_regions.py @@ -10,7 +10,7 @@ Example: This script: 1. Reads region definitions from modules/map/catalog.py 2. Runs `pmtiles extract` for each bbox region - 3. Generates manifest.json with direct download URLs for each region + 3. Generates manifest.json with direct download URLs and the sha256 of each file, then signs it 4. Prints a summary of all extracted files and sizes Requirements: @@ -87,8 +87,8 @@ def generate_meta4(name: str, url: str, sha256: str, size: int, signature: str) ) -def write_meta4(file_path: str, cdn_url: str): - """Compute hash, sign it, and write a signed .meta4 file alongside the given file.""" +def write_meta4(file_path: str, cdn_url: str) -> str: + """Compute hash, sign it, and write a signed .meta4 file alongside the given file. Returns the hash.""" name = os.path.basename(file_path) size = os.path.getsize(file_path) sha256 = sha256_file(file_path) @@ -99,6 +99,24 @@ def write_meta4(file_path: str, cdn_url: str): with open(meta4_path, 'w') as f: f.write(meta4_xml) print(f' Written {os.path.basename(meta4_path)}') + return sha256 + + +def sign_manifest(manifest_path: str): + """Write the manifest's detached GPG signature beside it, or exit.""" + sig_path = f'{manifest_path}.sig' + print(f'Signing manifest...') + result = subprocess.run( + ['gpg', '-u', 'roland@learningselfreliance.com', '--detach-sign', '--armor', '-o', sig_path, manifest_path], + capture_output=True, + ) + if result.returncode == 0: + print(f'Signature written to: {sig_path}') + else: + print(f'ERROR: GPG signing failed: {result.stderr.decode().strip()}') + print(f'Sign manually and re-run, or fix GPG key access:' + f'\n gpg -u roland@learningselfreliance.com --detach-sign --armor -o {sig_path} {manifest_path}') + sys.exit(1) def extract_region(planet_path: str, output_path: str, bbox: str, threads: int = 4) -> bool: @@ -269,6 +287,8 @@ def main(): # Extract each bbox region. manifest_regions = {} search_index_urls = {} + # WROLPi checks each download against its hash here; a meta4's signed hash is not tied to any one file. + hashes = {} results = [] failed = [] @@ -298,7 +318,7 @@ def main(): size = os.path.getsize(output_path) cdn_url = f'{cdn_base}/{output_name}' - write_meta4(output_path, cdn_url) + hashes[cdn_url] = write_meta4(output_path, cdn_url) results.append((r['name'], region, size)) manifest_regions[region] = cdn_url @@ -308,7 +328,7 @@ def main(): enrich_search_db_with_regions(search_db_path) search_db_name = os.path.basename(str(search_db_path)) search_cdn_url = f'{cdn_base}/{search_db_name}' - write_meta4(str(search_db_path), search_cdn_url) + hashes[search_cdn_url] = write_meta4(str(search_db_path), search_cdn_url) search_index_urls[region] = search_cdn_url # Add terrain to manifest if the file exists in output_dir. @@ -317,7 +337,7 @@ def main(): terrain_path = os.path.join(output_dir, terrain_name) if os.path.isfile(terrain_path): cdn_url = f'{cdn_base}/{terrain_name}' - write_meta4(terrain_path, cdn_url) + hashes[cdn_url] = write_meta4(terrain_path, cdn_url) manifest_regions[terrain_region['region']] = cdn_url size = os.path.getsize(terrain_path) results.append((terrain_region['name'], terrain_region['region'], size)) @@ -336,26 +356,14 @@ def main(): # Add search index URLs. if search_index_urls: manifest['search_indexes'] = search_index_urls + manifest['sha256'] = hashes manifest_path = os.path.join(output_dir, 'manifest.json') with open(manifest_path, 'w') as f: json.dump(manifest, f, indent=2) print(f'\nManifest written to: {manifest_path}') - # Sign manifest with GPG. - sig_path = f'{manifest_path}.sig' - print(f'Signing manifest...') - result = subprocess.run( - ['gpg', '-u', 'roland@learningselfreliance.com', '--detach-sign', '--armor', '-o', sig_path, manifest_path], - capture_output=True, - ) - if result.returncode == 0: - print(f'Signature written to: {sig_path}') - else: - print(f'ERROR: GPG signing failed: {result.stderr.decode().strip()}') - print(f'Sign manually and re-run, or fix GPG key access:' - f'\n gpg -u roland@learningselfreliance.com --detach-sign --armor -o {sig_path} {manifest_path}') - sys.exit(1) + sign_manifest(manifest_path) # Print summary. print(f'\n{"="*70}') diff --git a/wrolpi/scripts/hash_map_manifest.py b/wrolpi/scripts/hash_map_manifest.py new file mode 100755 index 00000000..c92bcede --- /dev/null +++ b/wrolpi/scripts/hash_map_manifest.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Add each file's sha256 to the published map manifest, without rebuilding any maps. + +Usage: + python wrolpi/scripts/hash_map_manifest.py [--files DIR] [--manifest-url URL] + +Each file is hashed from its bytes: the copy in --files (e.g. the build_map_regions.py output_dir) when it has one, +otherwise downloaded from the CDN. Every hash must match the file's signed .meta4, and the published manifest's +signature is verified, then the manifest is written to output_dir with a `sha256` of each URL, and signed. Upload +manifest.json and manifest.json.sig to the CDN. + +A .meta4 alone cannot show which file its hash belongs to: only the bare hash is signed, so an older version's .meta4 +(with its unsigned name rewritten) passes its signature check. + +Manifests from build_map_regions.py already include the hashes. +""" +import argparse +import asyncio +import hashlib +import json +import os +import pathlib +import sys + +import requests + +# Add project root to path so we can import the map module. +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + +from modules.map.catalog import MANIFEST_URL +from modules.map.downloader import meta4_file_name, verified_meta4_sha256 +from modules.map.lib import fetch_manifest +from wrolpi.downloader import Downloader +from wrolpi.scripts.build_map_regions import sign_manifest + + +def file_sha256(url: str, files_directory: pathlib.Path = None) -> str: + """sha256 of the file's local copy in `files_directory`, if it has one, otherwise of the file downloaded from + `url`.""" + h = hashlib.sha256() + if files_directory and (path := files_directory / meta4_file_name(url)).is_file(): + with path.open('rb') as f: + for chunk in iter(lambda: f.read(8 * 1024 * 1024), b''): + h.update(chunk) + else: + with requests.get(url, stream=True, timeout=60) as response: + response.raise_for_status() + for chunk in response.iter_content(8 * 1024 * 1024): + h.update(chunk) + return h.hexdigest() + + +async def manifest_hashes(manifest: dict, files_directory: pathlib.Path = None) -> dict: + """{url: sha256} of every file in the manifest, hashed from the file's bytes, each matching its signed .meta4.""" + urls = [*manifest['regions'].values(), *manifest.get('search_indexes', {}).values()] + hashes = {} + for url in urls: + meta4 = await Downloader.get_meta4_contents(url) + if not meta4 or not (signed_sha256 := await verified_meta4_sha256(meta4)): + raise RuntimeError(f'No validly signed meta4 for {url}') + if (sha256 := file_sha256(url, files_directory)) != signed_sha256: + raise RuntimeError(f'{url} has sha256 {sha256}, its signed meta4 says {signed_sha256}') + hashes[url] = sha256 + print(f'{sha256} {url}', flush=True) + return hashes + + +def main(): + parser = argparse.ArgumentParser(description='Add each file\'s sha256 to the published map manifest.') + parser.add_argument('output_dir', help='Directory to write manifest.json and manifest.json.sig to') + parser.add_argument('--files', type=pathlib.Path, + help='Directory of the published files; any not in it are downloaded from the CDN') + parser.add_argument('--manifest-url', default=MANIFEST_URL, help=f'(default: {MANIFEST_URL})') + args = parser.parse_args() + + manifest = asyncio.run(fetch_manifest(args.manifest_url)) + manifest['sha256'] = asyncio.run(manifest_hashes(manifest, args.files)) + + os.makedirs(args.output_dir, exist_ok=True) + manifest_path = os.path.join(os.path.abspath(args.output_dir), 'manifest.json') + with open(manifest_path, 'w') as f: + json.dump(manifest, f, indent=2) + print(f'\nManifest written to: {manifest_path}') + sign_manifest(manifest_path) + print('Upload manifest.json and manifest.json.sig to the CDN.') + + +if __name__ == '__main__': + main()