mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
Map downloads are checked against the hash in the signed manifest (#934)
* Map downloads are checked against the hash in the signed manifest A meta4 signs only a bare hash, so any genuinely signed meta4 (an older version of a region, another region) passed its own check. The manifest now lists each file's sha256; it is signed as a whole, so the hash is tied to the URL. The catalog copies each hash into its download's settings, and aria2c checks the file against it. Downloads from a manifest without hashes still require the file's signed meta4. build_map_regions.py writes the hashes, and hash_map_manifest.py adds them to the published manifest from the verified meta4s without rebuilding any maps. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * hash_map_manifest.py hashes each file's bytes instead of trusting its meta4's name Only the bare hash in a meta4 is signed, so an older version's meta4 with its name rewritten (or terrain's, whose name never changes) passed the name and duplicate-hash checks. Each file is now hashed from a local copy of the published files (--files) or downloaded from the CDN, and must match its signed meta4. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
ee6c384dc6
commit
e6b7b9e5dd
5 changed files with 341 additions and 55 deletions
|
|
@ -31,6 +31,7 @@ __all__ = [
|
|||
logger = logger.getChild(__name__)
|
||||
|
||||
METALINK_NS = {'ml': 'urn:ietf:params:xml:ns:metalink'}
|
||||
SHA256_PATTERN = re.compile(r'[0-9a-f]{64}')
|
||||
# aria2c's exit code when a downloaded file does not match the meta4's hash.
|
||||
ARIA2C_CHECKSUM_ERROR = 32
|
||||
# Map files are downloaded here, and moved into the map directory only once aria2c has checked them against the signed
|
||||
|
|
@ -66,7 +67,7 @@ async def verified_meta4_sha256(meta4_contents: bytes) -> Optional[str]:
|
|||
logger.warning(f'Failed to parse meta4 XML: {e}')
|
||||
return None
|
||||
|
||||
if not re.fullmatch(r'[0-9a-f]{64}', sha256_hash):
|
||||
if not SHA256_PATTERN.fullmatch(sha256_hash):
|
||||
logger.warning('meta4 sha-256 hash is malformed')
|
||||
return None
|
||||
|
||||
|
|
@ -82,6 +83,27 @@ async def verified_meta4_sha256(meta4_contents: bytes) -> Optional[str]:
|
|||
return None
|
||||
|
||||
|
||||
async def get_signed_sha256(downloader: Downloader, url: str, manifest_sha256: Optional[str],
|
||||
label: str) -> tuple[Optional[str], Optional[str]]:
|
||||
"""The signed sha-256 hash a map file must match, or an error explaining why there is none.
|
||||
|
||||
The hash from the signed manifest is used when it has one: a meta4 signs only a bare hash, so any genuinely signed
|
||||
meta4 (an older version, another region) would pass its own check. Downloads from a manifest without hashes use
|
||||
the file's signed meta4."""
|
||||
if manifest_sha256:
|
||||
if SHA256_PATTERN.fullmatch(manifest_sha256):
|
||||
return manifest_sha256, None
|
||||
return None, f'Malformed hash in the manifest for {label}'
|
||||
|
||||
meta4_contents = await downloader.get_meta4_contents(url)
|
||||
if not meta4_contents:
|
||||
return None, f'No signed hash (meta4) available for {label}, not downloading'
|
||||
if not (sha256 := await verified_meta4_sha256(meta4_contents)):
|
||||
return None, f'meta4 hash signature verification failed for {label}'
|
||||
logger.info(f'meta4 signature verified for {label}')
|
||||
return sha256, None
|
||||
|
||||
|
||||
def meta4_file_name(url: str) -> Optional[str]:
|
||||
"""The name aria2c saves `url` as, or None if the URL's file name is not a plain one."""
|
||||
name = PurePosixPath(urlparse(url).path).name
|
||||
|
|
@ -175,6 +197,7 @@ class MapCatalogDownloader(Downloader):
|
|||
version = manifest.get('version', '')
|
||||
manifest_regions = manifest.get('regions', {})
|
||||
search_indexes = manifest.get('search_indexes', {})
|
||||
hashes = manifest.get('sha256', {})
|
||||
|
||||
if not manifest_regions:
|
||||
return DownloadResult(success=False, error='Manifest does not contain regions')
|
||||
|
|
@ -221,6 +244,7 @@ class MapCatalogDownloader(Downloader):
|
|||
'region': region,
|
||||
'version': version,
|
||||
'search_index_url': search_indexes.get(region, ''),
|
||||
'sha256': hashes.get(region_url),
|
||||
}
|
||||
child.renew(reset_attempts=True)
|
||||
created += 1
|
||||
|
|
@ -239,7 +263,8 @@ class MapCatalogDownloader(Downloader):
|
|||
session, search_url, reset_attempts=True, override_skip=True)
|
||||
search_child.downloader = 'map_search_index'
|
||||
search_child.location = '/map/manage'
|
||||
search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name}
|
||||
search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name,
|
||||
'sha256': hashes.get(search_url)}
|
||||
search_child.renew(reset_attempts=True)
|
||||
logger.info(f'Created search index download for {region}')
|
||||
|
||||
|
|
@ -273,6 +298,7 @@ class PreparedMapExtract:
|
|||
output_name: str
|
||||
output_path: pathlib.Path
|
||||
tmp_path: pathlib.Path
|
||||
sha256: Optional[str] = None # From the signed manifest.
|
||||
already_done: bool = False
|
||||
error: Optional[str] = None
|
||||
|
||||
|
|
@ -297,9 +323,7 @@ class MapExtractDownloader(Downloader):
|
|||
|
||||
Settings contain: region, version — set by MapCatalogDownloader.
|
||||
|
||||
Fetches the meta4 file once, GPG-verifies the hash signature, then passes
|
||||
the same meta4 bytes to aria2c for download hash verification. This ensures
|
||||
the hash that was GPG-verified is the same hash aria2c checks against.
|
||||
aria2c checks the file against a signed hash (see `get_signed_sha256`), given to it in a meta4 that WROLPi writes.
|
||||
"""
|
||||
|
||||
name = 'map_extract'
|
||||
|
|
@ -329,6 +353,7 @@ class MapExtractDownloader(Downloader):
|
|||
prepared = PreparedMapExtract(
|
||||
region=region, version=version, map_directory=map_directory,
|
||||
output_name=output_name, output_path=output_path, tmp_path=tmp_path,
|
||||
sha256=settings.get('sha256'),
|
||||
)
|
||||
|
||||
# Skip if versioned file already exists.
|
||||
|
|
@ -358,22 +383,13 @@ class MapExtractDownloader(Downloader):
|
|||
# `download` with a URL.
|
||||
url = download.url
|
||||
|
||||
# Fetch meta4 once, GPG-verify the hash signature before downloading. A map file is never downloaded
|
||||
# without a signed hash.
|
||||
meta4_contents = await self.get_meta4_contents(url)
|
||||
if not meta4_contents:
|
||||
# A map file is never downloaded without a signed hash.
|
||||
sha256, error = await get_signed_sha256(self, url, prepared.sha256, prepared.region)
|
||||
if error:
|
||||
return ExecutedMapExtract(
|
||||
region=prepared.region, output_path=prepared.output_path,
|
||||
map_directory=prepared.map_directory,
|
||||
error=f'No signed hash (meta4) available for {prepared.region}, not downloading',
|
||||
map_directory=prepared.map_directory, error=error,
|
||||
)
|
||||
if not (sha256 := await verified_meta4_sha256(meta4_contents)):
|
||||
return ExecutedMapExtract(
|
||||
region=prepared.region, output_path=prepared.output_path,
|
||||
map_directory=prepared.map_directory,
|
||||
error=f'meta4 hash signature verification failed for {prepared.region}',
|
||||
)
|
||||
logger.info(f'meta4 signature verified for {prepared.region}')
|
||||
if not (name := meta4_file_name(url)):
|
||||
return ExecutedMapExtract(
|
||||
region=prepared.region, output_path=prepared.output_path,
|
||||
|
|
@ -476,6 +492,7 @@ class PreparedMapSearchIndex:
|
|||
map_directory: pathlib.Path
|
||||
pmtiles_path: pathlib.Path
|
||||
search_db_path: pathlib.Path
|
||||
sha256: Optional[str] = None # From the signed manifest.
|
||||
already_done: bool = False
|
||||
error: Optional[str] = None
|
||||
|
||||
|
|
@ -509,6 +526,7 @@ class MapSearchIndexDownloader(Downloader):
|
|||
pmtiles_path=map_directory / pmtiles_name if pmtiles_name else map_directory,
|
||||
search_db_path=(map_directory / pmtiles_name).with_suffix('.search.db')
|
||||
if pmtiles_name else map_directory,
|
||||
sha256=settings.get('sha256'),
|
||||
)
|
||||
|
||||
if not region or not pmtiles_name:
|
||||
|
|
@ -535,19 +553,10 @@ class MapSearchIndexDownloader(Downloader):
|
|||
download = download if download is not None else Download(url=prepared.pmtiles_name)
|
||||
url = download.url
|
||||
|
||||
# Fetch and verify meta4 (GPG-signed hash). A search index is never downloaded without a signed hash.
|
||||
meta4_contents = await self.get_meta4_contents(url)
|
||||
if not meta4_contents:
|
||||
return ExecutedMapSearchIndex(
|
||||
region=prepared.region,
|
||||
error=f'No signed hash (meta4) available for {prepared.region} search index, not downloading',
|
||||
)
|
||||
if not (sha256 := await verified_meta4_sha256(meta4_contents)):
|
||||
return ExecutedMapSearchIndex(
|
||||
region=prepared.region,
|
||||
error=f'meta4 signature verification failed for {prepared.region} search index',
|
||||
)
|
||||
logger.info(f'Search index meta4 verified for {prepared.region}')
|
||||
# A search index is never downloaded without a signed hash.
|
||||
sha256, error = await get_signed_sha256(self, url, prepared.sha256, f'{prepared.region} search index')
|
||||
if error:
|
||||
return ExecutedMapSearchIndex(region=prepared.region, error=error)
|
||||
if not (name := meta4_file_name(url)):
|
||||
return ExecutedMapSearchIndex(
|
||||
region=prepared.region,
|
||||
|
|
|
|||
|
|
@ -330,18 +330,18 @@ def test_extract_finalize_builds_search_index(test_session, test_directory, has_
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _prepared_extract(map_dir):
|
||||
def _prepared_extract(map_dir, sha256: str = None):
|
||||
return map_extract_downloader.prepare_download(None, Download(
|
||||
url='https://x/us-2024-01.pmtiles', downloader='map_extract',
|
||||
settings={'region': 'us', 'version': '2024-01'},
|
||||
settings={'region': 'us', 'version': '2024-01', 'sha256': sha256},
|
||||
))
|
||||
|
||||
|
||||
def _prepared_search_index(map_dir):
|
||||
def _prepared_search_index(map_dir, sha256: str = None):
|
||||
(map_dir / 'us-2024-01.pmtiles').write_bytes(b'pretend pmtiles')
|
||||
return map_search_index_downloader.prepare_download(None, Download(
|
||||
url='https://x/us-2024-01.search.db', downloader='map_search_index',
|
||||
settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'},
|
||||
settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles', 'sha256': sha256},
|
||||
))
|
||||
|
||||
|
||||
|
|
@ -521,3 +521,75 @@ async def test_map_download_refuses_unusable_file_name(test_directory, monkeypat
|
|||
|
||||
assert executed.error
|
||||
download_file.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hashes from the signed manifest
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MANIFEST_SHA256 = hashlib.sha256(b'manifest contents').hexdigest()
|
||||
|
||||
|
||||
def test_catalog_finalize_copies_manifest_hashes(test_session, test_directory, test_download_manager, monkeypatch):
|
||||
"""Each child download gets its file's hash from the signed manifest."""
|
||||
map_dir = test_directory / 'maps'
|
||||
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
|
||||
pmtiles_url, search_url = 'https://x/us-2024-02.pmtiles', 'https://x/us-2024-02.search.db'
|
||||
executed = ExecutedMapCatalog(
|
||||
manifest={'regions': {'us': pmtiles_url}, 'search_indexes': {'us': search_url}, 'version': '2024-02',
|
||||
'sha256': {pmtiles_url: MANIFEST_SHA256, search_url: GENUINE_SHA256}},
|
||||
subscribed_regions=[{'region': 'us'}],
|
||||
)
|
||||
download = Download(url='https://x/manifest.json', downloader='map_catalog')
|
||||
result = map_catalog_downloader.finalize_download(test_session, download, executed)
|
||||
assert result.success is True, result.error
|
||||
|
||||
hashes = {i.downloader: i.settings.get('sha256') for i in test_session.query(Download)}
|
||||
assert hashes == {'map_extract': MANIFEST_SHA256, 'map_search_index': GENUINE_SHA256}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('downloader,prepare', [
|
||||
(map_extract_downloader, _prepared_extract),
|
||||
(map_search_index_downloader, _prepared_search_index),
|
||||
], ids=['extract', 'search_index'])
|
||||
async def test_map_download_uses_the_manifest_hash(test_directory, monkeypatch, downloader, prepare):
|
||||
"""A hash from the signed manifest is the one aria2c checks. A meta4 signs only a bare hash, so any genuine
|
||||
meta4 (an older version, another region) would pass its own signature check."""
|
||||
map_dir = test_directory / 'maps'
|
||||
map_dir.mkdir()
|
||||
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
|
||||
prepared = prepare(map_dir, sha256=MANIFEST_SHA256)
|
||||
url = 'https://x/us-2024-01.pmtiles'
|
||||
# A genuinely signed meta4, but of some other file.
|
||||
meta4 = INJECTED_META4.format(url=url).encode()
|
||||
|
||||
with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)) as get_meta4, \
|
||||
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \
|
||||
mock.patch.object(downloader, 'download_file', mock.AsyncMock(side_effect=RuntimeError('stop'))) as df:
|
||||
await downloader.execute_download(prepared, make_test_ctx(), Download(url=url))
|
||||
|
||||
get_meta4.assert_not_called()
|
||||
passed = ElementTree.fromstring(df.call_args.kwargs['meta4_xml'])
|
||||
assert [i.text for i in passed.iter('{urn:ietf:params:xml:ns:metalink}hash')] == [MANIFEST_SHA256]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('downloader,prepare', [
|
||||
(map_extract_downloader, _prepared_extract),
|
||||
(map_search_index_downloader, _prepared_search_index),
|
||||
], ids=['extract', 'search_index'])
|
||||
async def test_map_download_refuses_a_malformed_manifest_hash(test_directory, monkeypatch, downloader, prepare):
|
||||
map_dir = test_directory / 'maps'
|
||||
map_dir.mkdir()
|
||||
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
|
||||
prepared = prepare(map_dir, sha256='not-a-hash')
|
||||
meta4 = INJECTED_META4.format(url='https://x/a.pmtiles').encode()
|
||||
|
||||
with mock.patch.object(downloader, 'get_meta4_contents', mock.AsyncMock(return_value=meta4)), \
|
||||
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=True)), \
|
||||
mock.patch.object(downloader, 'download_file', mock.AsyncMock()) as download_file:
|
||||
executed = await downloader.execute_download(prepared, make_test_ctx(), Download(url='https://x/a.pmtiles'))
|
||||
|
||||
assert executed.error
|
||||
download_file.assert_not_called()
|
||||
|
|
|
|||
108
modules/map/test/test_hash_map_manifest.py
Normal file
108
modules/map/test/test_hash_map_manifest.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
import hashlib
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.scripts.hash_map_manifest import manifest_hashes
|
||||
|
||||
META4 = '''<?xml version="1.0" encoding="UTF-8"?>
|
||||
<metalink xmlns="urn:ietf:params:xml:ns:metalink">
|
||||
<file name="{name}">
|
||||
<hash type="sha-256">{sha256}</hash>
|
||||
<signature mediatype="application/pgp-signature">pretend signature</signature>
|
||||
</file>
|
||||
</metalink>
|
||||
'''
|
||||
FILES = {'us-1.pmtiles': b'pmtiles', 'us-1.search.db': b'search', 'terrain-z0-8.pmtiles': b'terrain'}
|
||||
|
||||
|
||||
def _sha256(contents: bytes) -> str:
|
||||
return hashlib.sha256(contents).hexdigest()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def served(test_directory, simple_file_server):
|
||||
"""The published files, served over HTTP, and their manifest."""
|
||||
host, port = simple_file_server.server_address
|
||||
base = f'http://{host}:{port}' # noqa
|
||||
for name, contents in FILES.items():
|
||||
(test_directory / name).write_bytes(contents)
|
||||
manifest = {
|
||||
'regions': {'us': f'{base}/us-1.pmtiles', 'terrain-z0-8': f'{base}/terrain-z0-8.pmtiles'},
|
||||
'search_indexes': {'us': f'{base}/us-1.search.db'},
|
||||
}
|
||||
# Each file's genuine, signed meta4.
|
||||
meta4s = {f'{base}/{name}': META4.format(name=name, sha256=_sha256(contents)) for name, contents in FILES.items()}
|
||||
return base, manifest, meta4s
|
||||
|
||||
|
||||
async def _hashes(manifest: dict, meta4s: dict, files_directory=None, signature_valid: bool = True):
|
||||
async def get_meta4_contents(url):
|
||||
return meta4s[url].encode() if url in meta4s else None
|
||||
|
||||
with mock.patch('wrolpi.scripts.hash_map_manifest.Downloader.get_meta4_contents', get_meta4_contents), \
|
||||
mock.patch('modules.map.downloader.verify_gpg_signature', mock.AsyncMock(return_value=signature_valid)):
|
||||
return await manifest_hashes(manifest, files_directory)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('local', [False, True], ids=['downloaded', 'local'])
|
||||
async def test_manifest_hashes(served, tmp_path, local):
|
||||
"""Each hash is of the file's bytes (a local copy of the published files, or downloaded), and matches its signed
|
||||
meta4."""
|
||||
base, manifest, meta4s = served
|
||||
files_directory = None
|
||||
if local:
|
||||
files_directory = tmp_path / 'published'
|
||||
files_directory.mkdir()
|
||||
for name, contents in FILES.items():
|
||||
(files_directory / name).write_bytes(contents)
|
||||
# The local copy is read; the server's file is not.
|
||||
manifest['regions']['us'] = 'http://127.0.0.1:9/us-1.pmtiles'
|
||||
meta4s[manifest['regions']['us']] = meta4s.pop(f'{base}/us-1.pmtiles')
|
||||
|
||||
assert await _hashes(manifest, meta4s, files_directory) == {
|
||||
manifest['regions']['us']: _sha256(b'pmtiles'),
|
||||
f'{base}/terrain-z0-8.pmtiles': _sha256(b'terrain'),
|
||||
f'{base}/us-1.search.db': _sha256(b'search'),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_hashes_refuses_a_missing_meta4(served):
|
||||
base, manifest, meta4s = served
|
||||
del meta4s[f'{base}/us-1.search.db']
|
||||
with pytest.raises(RuntimeError):
|
||||
await _hashes(manifest, meta4s)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_hashes_refuses_an_unsigned_meta4(served):
|
||||
_, manifest, meta4s = served
|
||||
with pytest.raises(RuntimeError):
|
||||
await _hashes(manifest, meta4s, signature_valid=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('name,old_meta4_name', [
|
||||
# An older version's signed meta4, its unsigned name rewritten to the current file's.
|
||||
('us-1.search.db', 'us-1.search.db'),
|
||||
# Terrain is unversioned, so an older signed meta4 already has the current name.
|
||||
('terrain-z0-8.pmtiles', 'terrain-z0-8.pmtiles'),
|
||||
])
|
||||
async def test_manifest_hashes_refuses_an_older_signed_meta4(served, name, old_meta4_name):
|
||||
"""Only a bare hash is signed in a meta4, so only the file's bytes show which file a signed hash belongs to."""
|
||||
base, manifest, meta4s = served
|
||||
meta4s[f'{base}/{name}'] = META4.format(name=old_meta4_name, sha256=_sha256(b'older version'))
|
||||
with pytest.raises(RuntimeError):
|
||||
await _hashes(manifest, meta4s)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_hashes_refuses_a_local_copy_that_differs(served, tmp_path):
|
||||
_, manifest, meta4s = served
|
||||
files_directory = tmp_path / 'published'
|
||||
files_directory.mkdir()
|
||||
(files_directory / 'us-1.pmtiles').write_bytes(b'something else')
|
||||
with pytest.raises(RuntimeError):
|
||||
await _hashes(manifest, meta4s, files_directory)
|
||||
|
|
@ -10,7 +10,7 @@ Example:
|
|||
This script:
|
||||
1. Reads region definitions from modules/map/catalog.py
|
||||
2. Runs `pmtiles extract` for each bbox region
|
||||
3. Generates manifest.json with direct download URLs for each region
|
||||
3. Generates manifest.json with direct download URLs and the sha256 of each file, then signs it
|
||||
4. Prints a summary of all extracted files and sizes
|
||||
|
||||
Requirements:
|
||||
|
|
@ -87,8 +87,8 @@ def generate_meta4(name: str, url: str, sha256: str, size: int, signature: str)
|
|||
)
|
||||
|
||||
|
||||
def write_meta4(file_path: str, cdn_url: str):
|
||||
"""Compute hash, sign it, and write a signed .meta4 file alongside the given file."""
|
||||
def write_meta4(file_path: str, cdn_url: str) -> str:
|
||||
"""Compute hash, sign it, and write a signed .meta4 file alongside the given file. Returns the hash."""
|
||||
name = os.path.basename(file_path)
|
||||
size = os.path.getsize(file_path)
|
||||
sha256 = sha256_file(file_path)
|
||||
|
|
@ -99,6 +99,24 @@ def write_meta4(file_path: str, cdn_url: str):
|
|||
with open(meta4_path, 'w') as f:
|
||||
f.write(meta4_xml)
|
||||
print(f' Written {os.path.basename(meta4_path)}')
|
||||
return sha256
|
||||
|
||||
|
||||
def sign_manifest(manifest_path: str):
|
||||
"""Write the manifest's detached GPG signature beside it, or exit."""
|
||||
sig_path = f'{manifest_path}.sig'
|
||||
print(f'Signing manifest...')
|
||||
result = subprocess.run(
|
||||
['gpg', '-u', 'roland@learningselfreliance.com', '--detach-sign', '--armor', '-o', sig_path, manifest_path],
|
||||
capture_output=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print(f'Signature written to: {sig_path}')
|
||||
else:
|
||||
print(f'ERROR: GPG signing failed: {result.stderr.decode().strip()}')
|
||||
print(f'Sign manually and re-run, or fix GPG key access:'
|
||||
f'\n gpg -u roland@learningselfreliance.com --detach-sign --armor -o {sig_path} {manifest_path}')
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def extract_region(planet_path: str, output_path: str, bbox: str, threads: int = 4) -> bool:
|
||||
|
|
@ -269,6 +287,8 @@ def main():
|
|||
# Extract each bbox region.
|
||||
manifest_regions = {}
|
||||
search_index_urls = {}
|
||||
# WROLPi checks each download against its hash here; a meta4's signed hash is not tied to any one file.
|
||||
hashes = {}
|
||||
results = []
|
||||
failed = []
|
||||
|
||||
|
|
@ -298,7 +318,7 @@ def main():
|
|||
|
||||
size = os.path.getsize(output_path)
|
||||
cdn_url = f'{cdn_base}/{output_name}'
|
||||
write_meta4(output_path, cdn_url)
|
||||
hashes[cdn_url] = write_meta4(output_path, cdn_url)
|
||||
results.append((r['name'], region, size))
|
||||
manifest_regions[region] = cdn_url
|
||||
|
||||
|
|
@ -308,7 +328,7 @@ def main():
|
|||
enrich_search_db_with_regions(search_db_path)
|
||||
search_db_name = os.path.basename(str(search_db_path))
|
||||
search_cdn_url = f'{cdn_base}/{search_db_name}'
|
||||
write_meta4(str(search_db_path), search_cdn_url)
|
||||
hashes[search_cdn_url] = write_meta4(str(search_db_path), search_cdn_url)
|
||||
search_index_urls[region] = search_cdn_url
|
||||
|
||||
# Add terrain to manifest if the file exists in output_dir.
|
||||
|
|
@ -317,7 +337,7 @@ def main():
|
|||
terrain_path = os.path.join(output_dir, terrain_name)
|
||||
if os.path.isfile(terrain_path):
|
||||
cdn_url = f'{cdn_base}/{terrain_name}'
|
||||
write_meta4(terrain_path, cdn_url)
|
||||
hashes[cdn_url] = write_meta4(terrain_path, cdn_url)
|
||||
manifest_regions[terrain_region['region']] = cdn_url
|
||||
size = os.path.getsize(terrain_path)
|
||||
results.append((terrain_region['name'], terrain_region['region'], size))
|
||||
|
|
@ -336,26 +356,14 @@ def main():
|
|||
# Add search index URLs.
|
||||
if search_index_urls:
|
||||
manifest['search_indexes'] = search_index_urls
|
||||
manifest['sha256'] = hashes
|
||||
|
||||
manifest_path = os.path.join(output_dir, 'manifest.json')
|
||||
with open(manifest_path, 'w') as f:
|
||||
json.dump(manifest, f, indent=2)
|
||||
print(f'\nManifest written to: {manifest_path}')
|
||||
|
||||
# Sign manifest with GPG.
|
||||
sig_path = f'{manifest_path}.sig'
|
||||
print(f'Signing manifest...')
|
||||
result = subprocess.run(
|
||||
['gpg', '-u', 'roland@learningselfreliance.com', '--detach-sign', '--armor', '-o', sig_path, manifest_path],
|
||||
capture_output=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print(f'Signature written to: {sig_path}')
|
||||
else:
|
||||
print(f'ERROR: GPG signing failed: {result.stderr.decode().strip()}')
|
||||
print(f'Sign manually and re-run, or fix GPG key access:'
|
||||
f'\n gpg -u roland@learningselfreliance.com --detach-sign --armor -o {sig_path} {manifest_path}')
|
||||
sys.exit(1)
|
||||
sign_manifest(manifest_path)
|
||||
|
||||
# Print summary.
|
||||
print(f'\n{"="*70}')
|
||||
|
|
|
|||
89
wrolpi/scripts/hash_map_manifest.py
Executable file
89
wrolpi/scripts/hash_map_manifest.py
Executable file
|
|
@ -0,0 +1,89 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Add each file's sha256 to the published map manifest, without rebuilding any maps.
|
||||
|
||||
Usage:
|
||||
python wrolpi/scripts/hash_map_manifest.py <output_dir> [--files DIR] [--manifest-url URL]
|
||||
|
||||
Each file is hashed from its bytes: the copy in --files (e.g. the build_map_regions.py output_dir) when it has one,
|
||||
otherwise downloaded from the CDN. Every hash must match the file's signed .meta4, and the published manifest's
|
||||
signature is verified, then the manifest is written to output_dir with a `sha256` of each URL, and signed. Upload
|
||||
manifest.json and manifest.json.sig to the CDN.
|
||||
|
||||
A .meta4 alone cannot show which file its hash belongs to: only the bare hash is signed, so an older version's .meta4
|
||||
(with its unsigned name rewritten) passes its signature check.
|
||||
|
||||
Manifests from build_map_regions.py already include the hashes.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import requests
|
||||
|
||||
# Add project root to path so we can import the map module.
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from modules.map.catalog import MANIFEST_URL
|
||||
from modules.map.downloader import meta4_file_name, verified_meta4_sha256
|
||||
from modules.map.lib import fetch_manifest
|
||||
from wrolpi.downloader import Downloader
|
||||
from wrolpi.scripts.build_map_regions import sign_manifest
|
||||
|
||||
|
||||
def file_sha256(url: str, files_directory: pathlib.Path = None) -> str:
|
||||
"""sha256 of the file's local copy in `files_directory`, if it has one, otherwise of the file downloaded from
|
||||
`url`."""
|
||||
h = hashlib.sha256()
|
||||
if files_directory and (path := files_directory / meta4_file_name(url)).is_file():
|
||||
with path.open('rb') as f:
|
||||
for chunk in iter(lambda: f.read(8 * 1024 * 1024), b''):
|
||||
h.update(chunk)
|
||||
else:
|
||||
with requests.get(url, stream=True, timeout=60) as response:
|
||||
response.raise_for_status()
|
||||
for chunk in response.iter_content(8 * 1024 * 1024):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
async def manifest_hashes(manifest: dict, files_directory: pathlib.Path = None) -> dict:
|
||||
"""{url: sha256} of every file in the manifest, hashed from the file's bytes, each matching its signed .meta4."""
|
||||
urls = [*manifest['regions'].values(), *manifest.get('search_indexes', {}).values()]
|
||||
hashes = {}
|
||||
for url in urls:
|
||||
meta4 = await Downloader.get_meta4_contents(url)
|
||||
if not meta4 or not (signed_sha256 := await verified_meta4_sha256(meta4)):
|
||||
raise RuntimeError(f'No validly signed meta4 for {url}')
|
||||
if (sha256 := file_sha256(url, files_directory)) != signed_sha256:
|
||||
raise RuntimeError(f'{url} has sha256 {sha256}, its signed meta4 says {signed_sha256}')
|
||||
hashes[url] = sha256
|
||||
print(f'{sha256} {url}', flush=True)
|
||||
return hashes
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Add each file\'s sha256 to the published map manifest.')
|
||||
parser.add_argument('output_dir', help='Directory to write manifest.json and manifest.json.sig to')
|
||||
parser.add_argument('--files', type=pathlib.Path,
|
||||
help='Directory of the published files; any not in it are downloaded from the CDN')
|
||||
parser.add_argument('--manifest-url', default=MANIFEST_URL, help=f'(default: {MANIFEST_URL})')
|
||||
args = parser.parse_args()
|
||||
|
||||
manifest = asyncio.run(fetch_manifest(args.manifest_url))
|
||||
manifest['sha256'] = asyncio.run(manifest_hashes(manifest, args.files))
|
||||
|
||||
os.makedirs(args.output_dir, exist_ok=True)
|
||||
manifest_path = os.path.join(os.path.abspath(args.output_dir), 'manifest.json')
|
||||
with open(manifest_path, 'w') as f:
|
||||
json.dump(manifest, f, indent=2)
|
||||
print(f'\nManifest written to: {manifest_path}')
|
||||
sign_manifest(manifest_path)
|
||||
print('Upload manifest.json and manifest.json.sig to the CDN.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue