diff --git a/modules/map/downloader.py b/modules/map/downloader.py
index 616f1e39..3c2d9c4d 100644
--- a/modules/map/downloader.py
+++ b/modules/map/downloader.py
@@ -31,6 +31,7 @@ __all__ = [
logger = logger.getChild(__name__)
METALINK_NS = {'ml': 'urn:ietf:params:xml:ns:metalink'}
+SHA256_PATTERN = re.compile(r'[0-9a-f]{64}')
# aria2c's exit code when a downloaded file does not match the meta4's hash.
ARIA2C_CHECKSUM_ERROR = 32
# Map files are downloaded here, and moved into the map directory only once aria2c has checked them against the signed
@@ -66,7 +67,7 @@ async def verified_meta4_sha256(meta4_contents: bytes) -> Optional[str]:
logger.warning(f'Failed to parse meta4 XML: {e}')
return None
- if not re.fullmatch(r'[0-9a-f]{64}', sha256_hash):
+ if not SHA256_PATTERN.fullmatch(sha256_hash):
logger.warning('meta4 sha-256 hash is malformed')
return None
@@ -82,6 +83,27 @@ async def verified_meta4_sha256(meta4_contents: bytes) -> Optional[str]:
return None
+async def get_signed_sha256(downloader: Downloader, url: str, manifest_sha256: Optional[str],
+ label: str) -> tuple[Optional[str], Optional[str]]:
+ """The signed sha-256 hash a map file must match, or an error explaining why there is none.
+
+ The hash from the signed manifest is used when it has one: a meta4 signs only a bare hash, so any genuinely signed
+ meta4 (an older version, another region) would pass its own check. Downloads from a manifest without hashes use
+ the file's signed meta4."""
+ if manifest_sha256:
+ if SHA256_PATTERN.fullmatch(manifest_sha256):
+ return manifest_sha256, None
+ return None, f'Malformed hash in the manifest for {label}'
+
+ meta4_contents = await downloader.get_meta4_contents(url)
+ if not meta4_contents:
+ return None, f'No signed hash (meta4) available for {label}, not downloading'
+ if not (sha256 := await verified_meta4_sha256(meta4_contents)):
+ return None, f'meta4 hash signature verification failed for {label}'
+ logger.info(f'meta4 signature verified for {label}')
+ return sha256, None
+
+
def meta4_file_name(url: str) -> Optional[str]:
"""The name aria2c saves `url` as, or None if the URL's file name is not a plain one."""
name = PurePosixPath(urlparse(url).path).name
@@ -175,6 +197,7 @@ class MapCatalogDownloader(Downloader):
version = manifest.get('version', '')
manifest_regions = manifest.get('regions', {})
search_indexes = manifest.get('search_indexes', {})
+ hashes = manifest.get('sha256', {})
if not manifest_regions:
return DownloadResult(success=False, error='Manifest does not contain regions')
@@ -221,6 +244,7 @@ class MapCatalogDownloader(Downloader):
'region': region,
'version': version,
'search_index_url': search_indexes.get(region, ''),
+ 'sha256': hashes.get(region_url),
}
child.renew(reset_attempts=True)
created += 1
@@ -239,7 +263,8 @@ class MapCatalogDownloader(Downloader):
session, search_url, reset_attempts=True, override_skip=True)
search_child.downloader = 'map_search_index'
search_child.location = '/map/manage'
- search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name}
+ search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name,
+ 'sha256': hashes.get(search_url)}
search_child.renew(reset_attempts=True)
logger.info(f'Created search index download for {region}')
@@ -273,6 +298,7 @@ class PreparedMapExtract:
output_name: str
output_path: pathlib.Path
tmp_path: pathlib.Path
+ sha256: Optional[str] = None # From the signed manifest.
already_done: bool = False
error: Optional[str] = None
@@ -297,9 +323,7 @@ class MapExtractDownloader(Downloader):
Settings contain: region, version — set by MapCatalogDownloader.
- Fetches the meta4 file once, GPG-verifies the hash signature, then passes
- the same meta4 bytes to aria2c for download hash verification. This ensures
- the hash that was GPG-verified is the same hash aria2c checks against.
+ aria2c checks the file against a signed hash (see `get_signed_sha256`), given to it in a meta4 that WROLPi writes.
"""
name = 'map_extract'
@@ -329,6 +353,7 @@ class MapExtractDownloader(Downloader):
prepared = PreparedMapExtract(
region=region, version=version, map_directory=map_directory,
output_name=output_name, output_path=output_path, tmp_path=tmp_path,
+ sha256=settings.get('sha256'),
)
# Skip if versioned file already exists.
@@ -358,22 +383,13 @@ class MapExtractDownloader(Downloader):
# `download` with a URL.
url = download.url
- # Fetch meta4 once, GPG-verify the hash signature before downloading. A map file is never downloaded
- # without a signed hash.
- meta4_contents = await self.get_meta4_contents(url)
- if not meta4_contents:
+ # A map file is never downloaded without a signed hash.
+ sha256, error = await get_signed_sha256(self, url, prepared.sha256, prepared.region)
+ if error:
return ExecutedMapExtract(
region=prepared.region, output_path=prepared.output_path,
- map_directory=prepared.map_directory,
- error=f'No signed hash (meta4) available for {prepared.region}, not downloading',
+ map_directory=prepared.map_directory, error=error,
)
- if not (sha256 := await verified_meta4_sha256(meta4_contents)):
- return ExecutedMapExtract(
- region=prepared.region, output_path=prepared.output_path,
- map_directory=prepared.map_directory,
- error=f'meta4 hash signature verification failed for {prepared.region}',
- )
- logger.info(f'meta4 signature verified for {prepared.region}')
if not (name := meta4_file_name(url)):
return ExecutedMapExtract(
region=prepared.region, output_path=prepared.output_path,
@@ -476,6 +492,7 @@ class PreparedMapSearchIndex:
map_directory: pathlib.Path
pmtiles_path: pathlib.Path
search_db_path: pathlib.Path
+ sha256: Optional[str] = None # From the signed manifest.
already_done: bool = False
error: Optional[str] = None
@@ -509,6 +526,7 @@ class MapSearchIndexDownloader(Downloader):
pmtiles_path=map_directory / pmtiles_name if pmtiles_name else map_directory,
search_db_path=(map_directory / pmtiles_name).with_suffix('.search.db')
if pmtiles_name else map_directory,
+ sha256=settings.get('sha256'),
)
if not region or not pmtiles_name:
@@ -535,19 +553,10 @@ class MapSearchIndexDownloader(Downloader):
download = download if download is not None else Download(url=prepared.pmtiles_name)
url = download.url
- # Fetch and verify meta4 (GPG-signed hash). A search index is never downloaded without a signed hash.
- meta4_contents = await self.get_meta4_contents(url)
- if not meta4_contents:
- return ExecutedMapSearchIndex(
- region=prepared.region,
- error=f'No signed hash (meta4) available for {prepared.region} search index, not downloading',
- )
- if not (sha256 := await verified_meta4_sha256(meta4_contents)):
- return ExecutedMapSearchIndex(
- region=prepared.region,
- error=f'meta4 signature verification failed for {prepared.region} search index',
- )
- logger.info(f'Search index meta4 verified for {prepared.region}')
+ # A search index is never downloaded without a signed hash.
+ sha256, error = await get_signed_sha256(self, url, prepared.sha256, f'{prepared.region} search index')
+ if error:
+ return ExecutedMapSearchIndex(region=prepared.region, error=error)
if not (name := meta4_file_name(url)):
return ExecutedMapSearchIndex(
region=prepared.region,
diff --git a/modules/map/test/test_downloader.py b/modules/map/test/test_downloader.py
index cc4a69cd..52d90f47 100644
--- a/modules/map/test/test_downloader.py
+++ b/modules/map/test/test_downloader.py
@@ -330,18 +330,18 @@ def test_extract_finalize_builds_search_index(test_session, test_directory, has_
# ---------------------------------------------------------------------------
-def _prepared_extract(map_dir):
+def _prepared_extract(map_dir, sha256: str = None):
return map_extract_downloader.prepare_download(None, Download(
url='https://x/us-2024-01.pmtiles', downloader='map_extract',
- settings={'region': 'us', 'version': '2024-01'},
+ settings={'region': 'us', 'version': '2024-01', 'sha256': sha256},
))
-def _prepared_search_index(map_dir):
+def _prepared_search_index(map_dir, sha256: str = None):
(map_dir / 'us-2024-01.pmtiles').write_bytes(b'pretend pmtiles')
return map_search_index_downloader.prepare_download(None, Download(
url='https://x/us-2024-01.search.db', downloader='map_search_index',
- settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'},
+ settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles', 'sha256': sha256},
))
@@ -521,3 +521,75 @@ async def test_map_download_refuses_unusable_file_name(test_directory, monkeypat
assert executed.error
download_file.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# Hashes from the signed manifest
+# ---------------------------------------------------------------------------
+
+MANIFEST_SHA256 = hashlib.sha256(b'manifest contents').hexdigest()
+
+
+def test_catalog_finalize_copies_manifest_hashes(test_session, test_directory, test_download_manager, monkeypatch):
+ """Each child download gets its file's hash from the signed manifest."""
+ map_dir = test_directory / 'maps'
+ monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
+ pmtiles_url, search_url = 'https://x/us-2024-02.pmtiles', 'https://x/us-2024-02.search.db'
+ executed = ExecutedMapCatalog(
+ manifest={'regions': {'us': pmtiles_url}, 'search_indexes': {'us': search_url}, 'version': '2024-02',
+ 'sha256': {pmtiles_url: MANIFEST_SHA256, search_url: GENUINE_SHA256}},
+ subscribed_regions=[{'region': 'us'}],
+ )
+ download = Download(url='https://x/manifest.json', downloader='map_catalog')
+ result = map_catalog_downloader.finalize_download(test_session, download, executed)
+ assert result.success is True, result.error
+
+ hashes = {i.downloader: i.settings.get('sha256') for i in test_session.query(Download)}
+ assert hashes == {'map_extract': MANIFEST_SHA256, 'map_search_index': GENUINE_SHA256}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('downloader,prepare', [
+ (map_extract_downloader, _prepared_extract),
+ (map_search_index_downloader, _prepared_search_index),
+], ids=['extract', 'search_index'])
+async def test_map_download_uses_the_manifest_hash(test_directory, monkeypatch, downloader, prepare):
+ """A hash from the signed manifest is the one aria2c checks. A meta4 signs only a bare hash, so any genuine
+ meta4 (an older version, another region) would pass its own signature check."""
+ map_dir = test_directory / 'maps'
+ map_dir.mkdir()
+ monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
+ prepared = prepare(map_dir, sha256=MANIFEST_SHA256)
+ url = 'https://x/us-2024-01.pmtiles'
+ # A genuinely signed meta4, but of some other file.
+ meta4 = INJECTED_META4.format(url=url).encode()
+
+ with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)) as get_meta4, \
+ mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \
+ mock.patch.object(downloader, 'download_file', mock.AsyncMock(side_effect=RuntimeError('stop'))) as df:
+ await downloader.execute_download(prepared, make_test_ctx(), Download(url=url))
+
+ get_meta4.assert_not_called()
+ passed = ElementTree.fromstring(df.call_args.kwargs['meta4_xml'])
+ assert [i.text for i in passed.iter('{urn:ietf:params:xml:ns:metalink}hash')] == [MANIFEST_SHA256]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('downloader,prepare', [
+ (map_extract_downloader, _prepared_extract),
+ (map_search_index_downloader, _prepared_search_index),
+], ids=['extract', 'search_index'])
+async def test_map_download_refuses_a_malformed_manifest_hash(test_directory, monkeypatch, downloader, prepare):
+ map_dir = test_directory / 'maps'
+ map_dir.mkdir()
+ monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
+ prepared = prepare(map_dir, sha256='not-a-hash')
+ meta4 = INJECTED_META4.format(url='https://x/a.pmtiles').encode()
+
+ with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \
+ mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \
+ mock.patch.object(downloader, 'download_file', mock.AsyncMock()) as download_file:
+ executed = await downloader.execute_download(prepared, make_test_ctx(), Download(url='https://x/a.pmtiles'))
+
+ assert executed.error
+ download_file.assert_not_called()
diff --git a/modules/map/test/test_hash_map_manifest.py b/modules/map/test/test_hash_map_manifest.py
new file mode 100644
index 00000000..9131e7f2
--- /dev/null
+++ b/modules/map/test/test_hash_map_manifest.py
@@ -0,0 +1,108 @@
+import hashlib
+from unittest import mock
+
+import pytest
+
+from wrolpi.scripts.hash_map_manifest import manifest_hashes
+
+META4 = '''
+
+
+ {sha256}
+ pretend signature
+
+
+'''
+FILES = {'us-1.pmtiles': b'pmtiles', 'us-1.search.db': b'search', 'terrain-z0-8.pmtiles': b'terrain'}
+
+
+def _sha256(contents: bytes) -> str:
+ return hashlib.sha256(contents).hexdigest()
+
+
+@pytest.fixture
+def served(test_directory, simple_file_server):
+ """The published files, served over HTTP, and their manifest."""
+ host, port = simple_file_server.server_address
+ base = f'http://{host}:{port}' # noqa
+ for name, contents in FILES.items():
+ (test_directory / name).write_bytes(contents)
+ manifest = {
+ 'regions': {'us': f'{base}/us-1.pmtiles', 'terrain-z0-8': f'{base}/terrain-z0-8.pmtiles'},
+ 'search_indexes': {'us': f'{base}/us-1.search.db'},
+ }
+ # Each file's genuine, signed meta4.
+ meta4s = {f'{base}/{name}': META4.format(name=name, sha256=_sha256(contents)) for name, contents in FILES.items()}
+ return base, manifest, meta4s
+
+
+async def _hashes(manifest: dict, meta4s: dict, files_directory=None, signature_valid: bool = True):
+ async def get_meta4_contents(url):
+ return meta4s[url].encode() if url in meta4s else None
+
+ with mock.patch('wrolpi.scripts.hash_map_manifest.Downloader.get_meta4_contents', get_meta4_contents), \
+ mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=signature_valid)):
+ return await manifest_hashes(manifest, files_directory)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('local', [False, True], ids=['downloaded', 'local'])
+async def test_manifest_hashes(served, tmp_path, local):
+ """Each hash is of the file's bytes (a local copy of the published files, or downloaded), and matches its signed
+ meta4."""
+ base, manifest, meta4s = served
+ files_directory = None
+ if local:
+ files_directory = tmp_path / 'published'
+ files_directory.mkdir()
+ for name, contents in FILES.items():
+ (files_directory / name).write_bytes(contents)
+ # The local copy is read; the server's file is not.
+ manifest['regions']['us'] = 'http://127.0.0.1:9/us-1.pmtiles'
+ meta4s[manifest['regions']['us']] = meta4s.pop(f'{base}/us-1.pmtiles')
+
+ assert await _hashes(manifest, meta4s, files_directory) == {
+ manifest['regions']['us']: _sha256(b'pmtiles'),
+ f'{base}/terrain-z0-8.pmtiles': _sha256(b'terrain'),
+ f'{base}/us-1.search.db': _sha256(b'search'),
+ }
+
+
+@pytest.mark.asyncio
+async def test_manifest_hashes_refuses_a_missing_meta4(served):
+ base, manifest, meta4s = served
+ del meta4s[f'{base}/us-1.search.db']
+ with pytest.raises(RuntimeError):
+ await _hashes(manifest, meta4s)
+
+
+@pytest.mark.asyncio
+async def test_manifest_hashes_refuses_an_unsigned_meta4(served):
+ _, manifest, meta4s = served
+ with pytest.raises(RuntimeError):
+ await _hashes(manifest, meta4s, signature_valid=False)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('name,old_meta4_name', [
+ # An older version's signed meta4, its unsigned name rewritten to the current file's.
+ ('us-1.search.db', 'us-1.search.db'),
+ # Terrain is unversioned, so an older signed meta4 already has the current name.
+ ('terrain-z0-8.pmtiles', 'terrain-z0-8.pmtiles'),
+])
+async def test_manifest_hashes_refuses_an_older_signed_meta4(served, name, old_meta4_name):
+ """Only a bare hash is signed in a meta4, so only the file's bytes show which file a signed hash belongs to."""
+ base, manifest, meta4s = served
+ meta4s[f'{base}/{name}'] = META4.format(name=old_meta4_name, sha256=_sha256(b'older version'))
+ with pytest.raises(RuntimeError):
+ await _hashes(manifest, meta4s)
+
+
+@pytest.mark.asyncio
+async def test_manifest_hashes_refuses_a_local_copy_that_differs(served, tmp_path):
+ _, manifest, meta4s = served
+ files_directory = tmp_path / 'published'
+ files_directory.mkdir()
+ (files_directory / 'us-1.pmtiles').write_bytes(b'something else')
+ with pytest.raises(RuntimeError):
+ await _hashes(manifest, meta4s, files_directory)
diff --git a/wrolpi/scripts/build_map_regions.py b/wrolpi/scripts/build_map_regions.py
index 17bc6c2e..74bbbf44 100755
--- a/wrolpi/scripts/build_map_regions.py
+++ b/wrolpi/scripts/build_map_regions.py
@@ -10,7 +10,7 @@ Example:
This script:
1. Reads region definitions from modules/map/catalog.py
2. Runs `pmtiles extract` for each bbox region
- 3. Generates manifest.json with direct download URLs for each region
+ 3. Generates manifest.json with direct download URLs and the sha256 of each file, then signs it
4. Prints a summary of all extracted files and sizes
Requirements:
@@ -87,8 +87,8 @@ def generate_meta4(name: str, url: str, sha256: str, size: int, signature: str)
)
-def write_meta4(file_path: str, cdn_url: str):
- """Compute hash, sign it, and write a signed .meta4 file alongside the given file."""
+def write_meta4(file_path: str, cdn_url: str) -> str:
+ """Compute hash, sign it, and write a signed .meta4 file alongside the given file. Returns the hash."""
name = os.path.basename(file_path)
size = os.path.getsize(file_path)
sha256 = sha256_file(file_path)
@@ -99,6 +99,24 @@ def write_meta4(file_path: str, cdn_url: str):
with open(meta4_path, 'w') as f:
f.write(meta4_xml)
print(f' Written {os.path.basename(meta4_path)}')
+ return sha256
+
+
+def sign_manifest(manifest_path: str):
+ """Write the manifest's detached GPG signature beside it, or exit."""
+ sig_path = f'{manifest_path}.sig'
+ print(f'Signing manifest...')
+ result = subprocess.run(
+ ['gpg', '-u', 'roland@learningselfreliance.com', '--detach-sign', '--armor', '-o', sig_path, manifest_path],
+ capture_output=True,
+ )
+ if result.returncode == 0:
+ print(f'Signature written to: {sig_path}')
+ else:
+ print(f'ERROR: GPG signing failed: {result.stderr.decode().strip()}')
+ print(f'Sign manually and re-run, or fix GPG key access:'
+ f'\n gpg -u roland@learningselfreliance.com --detach-sign --armor -o {sig_path} {manifest_path}')
+ sys.exit(1)
def extract_region(planet_path: str, output_path: str, bbox: str, threads: int = 4) -> bool:
@@ -269,6 +287,8 @@ def main():
# Extract each bbox region.
manifest_regions = {}
search_index_urls = {}
+ # WROLPi checks each download against its hash here; a meta4's signed hash is not tied to any one file.
+ hashes = {}
results = []
failed = []
@@ -298,7 +318,7 @@ def main():
size = os.path.getsize(output_path)
cdn_url = f'{cdn_base}/{output_name}'
- write_meta4(output_path, cdn_url)
+ hashes[cdn_url] = write_meta4(output_path, cdn_url)
results.append((r['name'], region, size))
manifest_regions[region] = cdn_url
@@ -308,7 +328,7 @@ def main():
enrich_search_db_with_regions(search_db_path)
search_db_name = os.path.basename(str(search_db_path))
search_cdn_url = f'{cdn_base}/{search_db_name}'
- write_meta4(str(search_db_path), search_cdn_url)
+ hashes[search_cdn_url] = write_meta4(str(search_db_path), search_cdn_url)
search_index_urls[region] = search_cdn_url
# Add terrain to manifest if the file exists in output_dir.
@@ -317,7 +337,7 @@ def main():
terrain_path = os.path.join(output_dir, terrain_name)
if os.path.isfile(terrain_path):
cdn_url = f'{cdn_base}/{terrain_name}'
- write_meta4(terrain_path, cdn_url)
+ hashes[cdn_url] = write_meta4(terrain_path, cdn_url)
manifest_regions[terrain_region['region']] = cdn_url
size = os.path.getsize(terrain_path)
results.append((terrain_region['name'], terrain_region['region'], size))
@@ -336,26 +356,14 @@ def main():
# Add search index URLs.
if search_index_urls:
manifest['search_indexes'] = search_index_urls
+ manifest['sha256'] = hashes
manifest_path = os.path.join(output_dir, 'manifest.json')
with open(manifest_path, 'w') as f:
json.dump(manifest, f, indent=2)
print(f'\nManifest written to: {manifest_path}')
- # Sign manifest with GPG.
- sig_path = f'{manifest_path}.sig'
- print(f'Signing manifest...')
- result = subprocess.run(
- ['gpg', '-u', 'roland@learningselfreliance.com', '--detach-sign', '--armor', '-o', sig_path, manifest_path],
- capture_output=True,
- )
- if result.returncode == 0:
- print(f'Signature written to: {sig_path}')
- else:
- print(f'ERROR: GPG signing failed: {result.stderr.decode().strip()}')
- print(f'Sign manually and re-run, or fix GPG key access:'
- f'\n gpg -u roland@learningselfreliance.com --detach-sign --armor -o {sig_path} {manifest_path}')
- sys.exit(1)
+ sign_manifest(manifest_path)
# Print summary.
print(f'\n{"="*70}')
diff --git a/wrolpi/scripts/hash_map_manifest.py b/wrolpi/scripts/hash_map_manifest.py
new file mode 100755
index 00000000..c92bcede
--- /dev/null
+++ b/wrolpi/scripts/hash_map_manifest.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""Add each file's sha256 to the published map manifest, without rebuilding any maps.
+
+Usage:
+ python wrolpi/scripts/hash_map_manifest.py [--files DIR] [--manifest-url URL]
+
+Each file is hashed from its bytes: the copy in --files (e.g. the build_map_regions.py output_dir) when it has one,
+otherwise downloaded from the CDN. Every hash must match the file's signed .meta4, and the published manifest's
+signature is verified, then the manifest is written to output_dir with a `sha256` of each URL, and signed. Upload
+manifest.json and manifest.json.sig to the CDN.
+
+A .meta4 alone cannot show which file its hash belongs to: only the bare hash is signed, so an older version's .meta4
+(with its unsigned name rewritten) passes its signature check.
+
+Manifests from build_map_regions.py already include the hashes.
+"""
+import argparse
+import asyncio
+import hashlib
+import json
+import os
+import pathlib
+import sys
+
+import requests
+
+# Add project root to path so we can import the map module.
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from modules.map.catalog import MANIFEST_URL
+from modules.map.downloader import meta4_file_name, verified_meta4_sha256
+from modules.map.lib import fetch_manifest
+from wrolpi.downloader import Downloader
+from wrolpi.scripts.build_map_regions import sign_manifest
+
+
+def file_sha256(url: str, files_directory: pathlib.Path = None) -> str:
+ """sha256 of the file's local copy in `files_directory`, if it has one, otherwise of the file downloaded from
+ `url`."""
+ h = hashlib.sha256()
+ if files_directory and (path := files_directory / meta4_file_name(url)).is_file():
+ with path.open('rb') as f:
+ for chunk in iter(lambda: f.read(8 * 1024 * 1024), b''):
+ h.update(chunk)
+ else:
+ with requests.get(url, stream=True, timeout=60) as response:
+ response.raise_for_status()
+ for chunk in response.iter_content(8 * 1024 * 1024):
+ h.update(chunk)
+ return h.hexdigest()
+
+
+async def manifest_hashes(manifest: dict, files_directory: pathlib.Path = None) -> dict:
+ """{url: sha256} of every file in the manifest, hashed from the file's bytes, each matching its signed .meta4."""
+ urls = [*manifest['regions'].values(), *manifest.get('search_indexes', {}).values()]
+ hashes = {}
+ for url in urls:
+ meta4 = await Downloader.get_meta4_contents(url)
+ if not meta4 or not (signed_sha256 := await verified_meta4_sha256(meta4)):
+ raise RuntimeError(f'No validly signed meta4 for {url}')
+ if (sha256 := file_sha256(url, files_directory)) != signed_sha256:
+ raise RuntimeError(f'{url} has sha256 {sha256}, its signed meta4 says {signed_sha256}')
+ hashes[url] = sha256
+ print(f'{sha256} {url}', flush=True)
+ return hashes
+
+
+def main():
+ parser = argparse.ArgumentParser(description='Add each file\'s sha256 to the published map manifest.')
+ parser.add_argument('output_dir', help='Directory to write manifest.json and manifest.json.sig to')
+ parser.add_argument('--files', type=pathlib.Path,
+ help='Directory of the published files; any not in it are downloaded from the CDN')
+ parser.add_argument('--manifest-url', default=MANIFEST_URL, help=f'(default: {MANIFEST_URL})')
+ args = parser.parse_args()
+
+ manifest = asyncio.run(fetch_manifest(args.manifest_url))
+ manifest['sha256'] = asyncio.run(manifest_hashes(manifest, args.files))
+
+ os.makedirs(args.output_dir, exist_ok=True)
+ manifest_path = os.path.join(os.path.abspath(args.output_dir), 'manifest.json')
+ with open(manifest_path, 'w') as f:
+ json.dump(manifest, f, indent=2)
+ print(f'\nManifest written to: {manifest_path}')
+ sign_manifest(manifest_path)
+ print('Upload manifest.json and manifest.json.sig to the CDN.')
+
+
+if __name__ == '__main__':
+ main()