rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Migrating map downloaders.

Browse source
This commit is contained in:
Roland 2026-05-06 19:46:24 -06:00 • committed by lrnselfreliance
parent 9a8ad7c3de
commit c644d83a5b
2 changed files with 596 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

426
modules/map/downloader.py
View file

@ -1,14 +1,29 @@
import pathlib
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any, List, Optional
from xml.etree import ElementTree as ET
from sqlalchemy.orm import Session
from modules.map import lib
from wrolpi.common import logger, verify_gpg_signature
from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult, download_manager
from wrolpi.downloader import (
Download,
DownloadContext,
Downloader,
DownloadResult,
download_manager,
)
__all__ = ['MapCatalogDownloader', 'MapExtractDownloader', 'MapSearchIndexDownloader',
'map_catalog_downloader', 'map_extract_downloader', 'map_search_index_downloader']
__all__ = [
'MapCatalogDownloader', 'MapExtractDownloader', 'MapSearchIndexDownloader',
'PreparedMapCatalog', 'ExecutedMapCatalog',
'PreparedMapExtract', 'ExecutedMapExtract',
'PreparedMapSearchIndex', 'ExecutedMapSearchIndex',
'map_catalog_downloader', 'map_extract_downloader', 'map_search_index_downloader',
]
logger = logger.getChild(__name__)
@ -52,19 +67,65 @@ async def verify_meta4_signature(meta4_contents: bytes) -> bool:
return await verify_gpg_signature(hash_path, sig_path)
# ---------------------------------------------------------------------------
# MapCatalogDownloader
# ---------------------------------------------------------------------------
@dataclass
class PreparedMapCatalog:
"""Plan produced by MapCatalogDownloader.prepare_download.
`subscribed_regions` is the list of {region: ...} dicts the user has chosen to track.
The manifest itself is fetched in execute_download.
"""
url: str
subscribed_regions: List[dict]
@dataclass
class ExecutedMapCatalog:
"""Output of MapCatalogDownloader.execute_download.
Either `manifest` is populated (success) or `error` is set (fetch failed). Child
download creation happens in finalize_download where the manager's session is in
scope.
"""
manifest: Optional[dict]
subscribed_regions: List[dict]
error: Optional[str] = None
class MapCatalogDownloader(Downloader):
"""Fetches the manifest and creates a child Download for each subscribed region."""
name = 'map_catalog'
listable = False
async def do_download(self, download: Download) -> DownloadResult:
def prepare_download(self, session: Session, download: Download) -> PreparedMapCatalog:
subscribed = (download.settings or {}).get('regions', [])
return PreparedMapCatalog(url=download.url, subscribed_regions=subscribed)
async def execute_download(self, prepared: PreparedMapCatalog, ctx: DownloadContext,
download: Download = None) -> ExecutedMapCatalog:
try:
manifest = await lib.fetch_manifest(download.url)
manifest = await lib.fetch_manifest(prepared.url)
except Exception as e:
logger.error(f'Failed to fetch map manifest: {e}')
return DownloadResult(success=False, error=f'Failed to fetch manifest: {e}')
return ExecutedMapCatalog(
manifest=None,
subscribed_regions=prepared.subscribed_regions,
error=f'Failed to fetch manifest: {e}',
)
return ExecutedMapCatalog(manifest=manifest,
subscribed_regions=prepared.subscribed_regions)
def finalize_download(self, session: Session, download: Download,
executed: ExecutedMapCatalog) -> DownloadResult:
if executed.error:
return DownloadResult(success=False, error=executed.error)
manifest = executed.manifest or {}
version = manifest.get('version', '')
manifest_regions = manifest.get('regions', {})
search_indexes = manifest.get('search_indexes', {})
@ -72,8 +133,7 @@ class MapCatalogDownloader(Downloader):
if not manifest_regions:
return DownloadResult(success=False, error='Manifest does not contain regions')
regions = (download.settings or {}).get('regions', [])
if not regions:
if not executed.subscribed_regions:
return DownloadResult(success=True)
map_directory = lib.get_map_directory()
@ -83,60 +143,59 @@ class MapCatalogDownloader(Downloader):
# download. Downloads process in insertion order; this prevents a search-index
# download from running before its companion pmtiles file exists on disk.
pending_search_downloads = []
with get_db_session(commit=True) as session:
for r in regions:
region = r.get('region', '')
region_url = manifest_regions.get(region)
if not region_url:
logger.warning(f'Subscribed region {region!r} not found in manifest — skipping')
continue
for r in executed.subscribed_regions:
region = r.get('region', '')
region_url = manifest_regions.get(region)
if not region_url:
logger.warning(f'Subscribed region {region!r} not found in manifest — skipping')
continue
# Compute the expected PMTiles filename.
if version:
pmtiles_name = f'{region}-{version}.pmtiles'
else:
pmtiles_name = f'{region}.pmtiles'
# Compute the expected PMTiles filename.
if version:
pmtiles_name = f'{region}-{version}.pmtiles'
else:
pmtiles_name = f'{region}.pmtiles'
# Delete stale download for this region if the URL changed (new version).
old = session.query(Download).filter(
Download.downloader == 'map_extract',
Download.settings['region'].astext == region,
Download.url != region_url,
).one_or_none()
if old:
logger.info(f'Deleting stale download for {region}: {old.url}')
session.delete(old)
session.flush()
# Delete stale download for this region if the URL changed (new version).
old = session.query(Download).filter(
Download.downloader == 'map_extract',
Download.settings['region'].astext == region,
Download.url != region_url,
).one_or_none()
if old:
logger.info(f'Deleting stale download for {region}: {old.url}')
session.delete(old)
session.flush()
child = download_manager.get_or_create_download(session, region_url, reset_attempts=True,
override_skip=True)
child.downloader = 'map_extract'
child.location = '/map/manage'
child.settings = {
'region': region,
'version': version,
'search_index_url': search_indexes.get(region, ''),
}
child.renew(reset_attempts=True)
created += 1
child = download_manager.get_or_create_download(session, region_url, reset_attempts=True,
override_skip=True)
child.downloader = 'map_extract'
child.location = '/map/manage'
child.settings = {
'region': region,
'version': version,
'search_index_url': search_indexes.get(region, ''),
}
child.renew(reset_attempts=True)
created += 1
# Queue search index download if the CDN provides one and the
# .search.db file does not already exist on disk.
search_url = search_indexes.get(region)
search_db_path = (map_directory / pmtiles_name).with_suffix('.search.db')
if search_url and not search_db_path.is_file():
pending_search_downloads.append((region, search_url, pmtiles_name))
# Queue search index download if the CDN provides one and the
# .search.db file does not already exist on disk.
search_url = search_indexes.get(region)
search_db_path = (map_directory / pmtiles_name).with_suffix('.search.db')
if search_url and not search_db_path.is_file():
pending_search_downloads.append((region, search_url, pmtiles_name))
# Insert all search-index downloads after every pmtiles download, so they
# process in insertion order behind their companion pmtiles.
for region, search_url, pmtiles_name in pending_search_downloads:
search_child = download_manager.get_or_create_download(
session, search_url, reset_attempts=True, override_skip=True)
search_child.downloader = 'map_search_index'
search_child.location = '/map/manage'
search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name}
search_child.renew(reset_attempts=True)
logger.info(f'Created search index download for {region}')
# Insert all search-index downloads after every pmtiles download, so they
# process in insertion order behind their companion pmtiles.
for region, search_url, pmtiles_name in pending_search_downloads:
search_child = download_manager.get_or_create_download(
session, search_url, reset_attempts=True, override_skip=True)
search_child.downloader = 'map_search_index'
search_child.location = '/map/manage'
search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name}
search_child.renew(reset_attempts=True)
logger.info(f'Created search index download for {region}')
if not created:
return DownloadResult(success=False, error='No matching region URLs found in manifest')
@ -149,6 +208,44 @@ class MapCatalogDownloader(Downloader):
map_catalog_downloader = MapCatalogDownloader()
# ---------------------------------------------------------------------------
# MapExtractDownloader
# ---------------------------------------------------------------------------
@dataclass
class PreparedMapExtract:
"""Plan produced by MapExtractDownloader.prepare_download.
Filenames are computed from settings; `already_done` short-circuits execute when
the output already exists. `error` carries an early validation failure (e.g. no
region in settings).
"""
region: str
version: str
map_directory: pathlib.Path
output_name: str
output_path: pathlib.Path
tmp_path: pathlib.Path
already_done: bool = False
error: Optional[str] = None
@dataclass
class ExecutedMapExtract:
"""Output of MapExtractDownloader.execute_download.
`skipped` covers the already-done short-circuit; `error` covers download/verify
failures. Otherwise the file is at output_path and finalize decides whether to
rebuild the search index locally.
"""
region: str
output_path: pathlib.Path
map_directory: pathlib.Path
skipped: bool = False
error: Optional[str] = None
class MapExtractDownloader(Downloader):
"""Downloads a pre-extracted region file from the CDN via aria2c.
@ -162,16 +259,19 @@ class MapExtractDownloader(Downloader):
name = 'map_extract'
listable = False
async def do_download(self, download: Download) -> DownloadResult:
def prepare_download(self, session: Session, download: Download) -> PreparedMapExtract:
settings = download.settings or {}
region = settings.get('region')
version = settings.get('version', '')
if not region:
return DownloadResult(success=False, error='No region in download settings')
return PreparedMapExtract(
region='', version='', map_directory=pathlib.Path('.'),
output_name='', output_path=pathlib.Path('.'), tmp_path=pathlib.Path('.'),
error='No region in download settings',
)
map_directory = lib.get_map_directory()
# Versioned filename.
if version:
output_name = f'{region}-{version}.pmtiles'
@ -180,75 +280,118 @@ class MapExtractDownloader(Downloader):
output_path = map_directory / output_name
tmp_path = map_directory / f'{output_name}.tmp'
prepared = PreparedMapExtract(
region=region, version=version, map_directory=map_directory,
output_name=output_name, output_path=output_path, tmp_path=tmp_path,
)
# Skip if versioned file already exists.
if output_path.is_file():
logger.info(f'{output_name} already exists, skipping download')
return DownloadResult(success=True, location='/map')
prepared.already_done = True
return prepared
# Clean up any leftover temp file.
if tmp_path.is_file():
tmp_path.unlink()
return prepared
async def execute_download(self, prepared: PreparedMapExtract, ctx: DownloadContext,
download: Download = None) -> ExecutedMapExtract:
if prepared.error:
return ExecutedMapExtract(region=prepared.region, output_path=prepared.output_path,
map_directory=prepared.map_directory, error=prepared.error)
if prepared.already_done:
return ExecutedMapExtract(region=prepared.region, output_path=prepared.output_path,
map_directory=prepared.map_directory, skipped=True)
download = download if download is not None else Download(url=prepared.output_path.name)
# The original download URL is needed for download_file; the manager passes the real
# Download in production, so we read .url from it. In unit tests, callers must supply
# `download` with a URL.
url = download.url
# Fetch meta4 once, GPG-verify the hash signature before downloading.
meta4_contents = await self.get_meta4_contents(download.url)
meta4_contents = await self.get_meta4_contents(url)
if meta4_contents:
if not await verify_meta4_signature(meta4_contents):
return DownloadResult(success=False,
error=f'meta4 hash signature verification failed for {region}')
logger.info(f'meta4 signature verified for {region}')
return ExecutedMapExtract(
region=prepared.region, output_path=prepared.output_path,
map_directory=prepared.map_directory,
error=f'meta4 hash signature verification failed for {prepared.region}',
)
logger.info(f'meta4 signature verified for {prepared.region}')
else:
logger.warning(f'No meta4 available for {region} — downloading without hash verification')
logger.warning(f'No meta4 available for {prepared.region} — downloading without hash verification')
# Download via aria2c, passing the pre-verified meta4 for hash checking.
logger.warning(f'Downloading {region} from {download.url}')
logger.warning(f'Downloading {prepared.region} from {url}')
downloaded_path = None
try:
downloaded_path = await self.download_file(
download, download.url, map_directory,
download, url, prepared.map_directory,
check_for_meta4=False, concurrent=1, meta4_xml=meta4_contents,
ctx=ctx,
)
downloaded_path.rename(tmp_path)
downloaded_path.rename(prepared.tmp_path)
except Exception as e:
logger.error(f'Download failed for {region}: {e}')
tmp_path.unlink(missing_ok=True)
logger.error(f'Download failed for {prepared.region}: {e}')
prepared.tmp_path.unlink(missing_ok=True)
if downloaded_path:
downloaded_path.unlink(missing_ok=True)
return DownloadResult(success=False, error=f'Download failed: {str(e)[:500]}')
return ExecutedMapExtract(
region=prepared.region, output_path=prepared.output_path,
map_directory=prepared.map_directory,
error=f'Download failed: {str(e)[:500]}',
)
if not tmp_path.is_file():
return DownloadResult(success=False, error=f'Output file was not created: {tmp_path}')
if not prepared.tmp_path.is_file():
return ExecutedMapExtract(
region=prepared.region, output_path=prepared.output_path,
map_directory=prepared.map_directory,
error=f'Output file was not created: {prepared.tmp_path}',
)
# Replace old file atomically.
tmp_path.rename(output_path)
prepared.tmp_path.rename(prepared.output_path)
# Delete older versions of this region.
for old_file in map_directory.glob(f'{region}-*.pmtiles'):
if old_file.name != output_name:
for old_file in prepared.map_directory.glob(f'{prepared.region}-*.pmtiles'):
if old_file.name != prepared.output_name:
logger.warning(f'Deleting old map version: {old_file}')
old_file.unlink()
# Also delete unversioned file if it exists.
unversioned = map_directory / f'{region}.pmtiles'
if unversioned.is_file() and unversioned.name != output_name:
unversioned = prepared.map_directory / f'{prepared.region}.pmtiles'
if unversioned.is_file() and unversioned.name != prepared.output_name:
logger.warning(f'Deleting unversioned map file: {unversioned}')
unversioned.unlink()
size = output_path.stat().st_size
logger.warning(f'Successfully downloaded {region} ({size} bytes) to {output_path}')
size = prepared.output_path.stat().st_size
logger.warning(f'Successfully downloaded {prepared.region} ({size} bytes) to {prepared.output_path}')
return ExecutedMapExtract(region=prepared.region, output_path=prepared.output_path,
map_directory=prepared.map_directory)
def finalize_download(self, session: Session, download: Download,
executed: ExecutedMapExtract) -> DownloadResult:
if executed.error:
return DownloadResult(success=False, error=executed.error)
if executed.skipped:
return DownloadResult(success=True, location='/map')
# Build search index locally if tippecanoe is available and no CDN
# search index download already exists for this region.
has_search_download = False
with get_db_session() as session:
has_search_download = session.query(Download).filter(
Download.downloader == 'map_search_index',
Download.settings['region'].astext == region,
).count() > 0
has_search_download = session.query(Download).filter(
Download.downloader == 'map_search_index',
Download.settings['region'].astext == executed.region,
).count() > 0
if not has_search_download:
try:
from modules.map.search import rebuild_search_index
rebuild_search_index(output_path.name)
rebuild_search_index(executed.output_path.name)
except Exception as e:
logger.error(f'Failed to start search index build for {region}: {e}')
logger.error(f'Failed to start search index build for {executed.region}: {e}')
return DownloadResult(success=True, location='/map')
@ -256,6 +399,34 @@ class MapExtractDownloader(Downloader):
map_extract_downloader = MapExtractDownloader()
# ---------------------------------------------------------------------------
# MapSearchIndexDownloader
# ---------------------------------------------------------------------------
@dataclass
class PreparedMapSearchIndex:
"""Plan produced by MapSearchIndexDownloader.prepare_download.
`error` covers missing settings or a missing PMTiles file (the search index can't
exist without one). `already_done` short-circuits when the .search.db is on disk.
"""
region: str
pmtiles_name: str
map_directory: pathlib.Path
pmtiles_path: pathlib.Path
search_db_path: pathlib.Path
already_done: bool = False
error: Optional[str] = None
@dataclass
class ExecutedMapSearchIndex:
region: str
skipped: bool = False
error: Optional[str] = None
class MapSearchIndexDownloader(Downloader):
"""Downloads a pre-built .search.db file from the CDN via aria2c with meta4 verification.
@ -266,54 +437,83 @@ class MapSearchIndexDownloader(Downloader):
name = 'map_search_index'
listable = False
async def do_download(self, download: Download) -> DownloadResult:
def prepare_download(self, session: Session, download: Download) -> PreparedMapSearchIndex:
settings = download.settings or {}
region = settings.get('region', '')
pmtiles_name = settings.get('pmtiles_name', '')
if not region or not pmtiles_name:
return DownloadResult(success=False, error='Missing region or pmtiles_name in settings')
map_directory = lib.get_map_directory()
pmtiles_path = map_directory / pmtiles_name
if not pmtiles_path.is_file():
return DownloadResult(success=False, error=f'PMTiles file not found: {pmtiles_name}')
prepared = PreparedMapSearchIndex(
region=region, pmtiles_name=pmtiles_name, map_directory=map_directory,
pmtiles_path=map_directory / pmtiles_name if pmtiles_name else map_directory,
search_db_path=(map_directory / pmtiles_name).with_suffix('.search.db')
if pmtiles_name else map_directory,
)
# Name the search.db to match the local PMTiles filename.
search_db_path = pmtiles_path.with_suffix('.search.db')
if not region or not pmtiles_name:
prepared.error = 'Missing region or pmtiles_name in settings'
return prepared
if search_db_path.is_file():
if not prepared.pmtiles_path.is_file():
prepared.error = f'PMTiles file not found: {pmtiles_name}'
return prepared
if prepared.search_db_path.is_file():
logger.info(f'Search index already exists for {region}, skipping')
return DownloadResult(success=True, location='/map')
prepared.already_done = True
return prepared
async def execute_download(self, prepared: PreparedMapSearchIndex, ctx: DownloadContext,
download: Download = None) -> ExecutedMapSearchIndex:
if prepared.error:
return ExecutedMapSearchIndex(region=prepared.region, error=prepared.error)
if prepared.already_done:
return ExecutedMapSearchIndex(region=prepared.region, skipped=True)
download = download if download is not None else Download(url=prepared.pmtiles_name)
url = download.url
# Fetch and verify meta4 (GPG-signed hash).
meta4_contents = await self.get_meta4_contents(download.url)
meta4_contents = await self.get_meta4_contents(url)
if meta4_contents:
if not await verify_meta4_signature(meta4_contents):
return DownloadResult(success=False,
error=f'meta4 signature verification failed for {region} search index')
logger.info(f'Search index meta4 verified for {region}')
return ExecutedMapSearchIndex(
region=prepared.region,
error=f'meta4 signature verification failed for {prepared.region} search index',
)
logger.info(f'Search index meta4 verified for {prepared.region}')
# Download via aria2c with meta4 hash verification.
logger.warning(f'Downloading search index for {region} from {download.url}')
logger.warning(f'Downloading search index for {prepared.region} from {url}')
downloaded_path = None
try:
downloaded_path = await self.download_file(
download, download.url, map_directory,
download, url, prepared.map_directory,
check_for_meta4=False, concurrent=1, meta4_xml=meta4_contents,
ctx=ctx,
)
# Rename to match the local PMTiles filename.
downloaded_path.rename(search_db_path)
downloaded_path.rename(prepared.search_db_path)
except Exception as e:
logger.error(f'Search index download failed for {region}: {e}')
logger.error(f'Search index download failed for {prepared.region}: {e}')
if downloaded_path:
downloaded_path.unlink(missing_ok=True)
search_db_path.unlink(missing_ok=True)
return DownloadResult(success=False, error=f'Download failed: {str(e)[:500]}')
prepared.search_db_path.unlink(missing_ok=True)
return ExecutedMapSearchIndex(
region=prepared.region,
error=f'Download failed: {str(e)[:500]}',
)
size = search_db_path.stat().st_size
logger.warning(f'Downloaded search index for {region} ({size} bytes)')
size = prepared.search_db_path.stat().st_size
logger.warning(f'Downloaded search index for {prepared.region} ({size} bytes)')
return ExecutedMapSearchIndex(region=prepared.region)
def finalize_download(self, session: Session, download: Download,
executed: ExecutedMapSearchIndex) -> DownloadResult:
if executed.error:
return DownloadResult(success=False, error=executed.error)
return DownloadResult(success=True, location='/map')

283
modules/map/test/test_downloader.py
View file

@ -0,0 +1,283 @@
"""Phase-split tests for MapCatalogDownloader / MapExtractDownloader / MapSearchIndexDownloader.
These exercise prepare_download and finalize_download in isolation — no async_client,
no test_download_manager, no real subprocess. Execute paths that drive download_file +
GPG verification + filesystem rename are integration territory and are deliberately
omitted; the parent download_file ctx threading is already covered by the FileDownloader
migration tests.
"""
from unittest import mock
import pytest
from modules.archive.conftest import make_test_ctx
from modules.map.downloader import (
ExecutedMapCatalog,
ExecutedMapExtract,
ExecutedMapSearchIndex,
PreparedMapCatalog,
PreparedMapExtract,
PreparedMapSearchIndex,
map_catalog_downloader,
map_extract_downloader,
map_search_index_downloader,
)
from wrolpi.downloader import Download
# ---------------------------------------------------------------------------
# MapCatalogDownloader
# ---------------------------------------------------------------------------
def test_catalog_prepare_extracts_subscribed_regions(test_session):
"""prepare_download surfaces the user's region subscriptions."""
download = Download(
url='https://maps.example.com/manifest.json',
downloader='map_catalog',
settings={'regions': [{'region': 'us'}, {'region': 'eu'}]},
)
prepared = map_catalog_downloader.prepare_download(test_session, download)
assert isinstance(prepared, PreparedMapCatalog)
assert prepared.url == download.url
assert prepared.subscribed_regions == [{'region': 'us'}, {'region': 'eu'}]
def test_catalog_prepare_handles_no_regions_in_settings(test_session):
"""No 'regions' key → empty list (handled in finalize as success no-op)."""
download = Download(
url='https://maps.example.com/manifest.json',
downloader='map_catalog',
settings={},
)
prepared = map_catalog_downloader.prepare_download(test_session, download)
assert prepared.subscribed_regions == []
@pytest.mark.asyncio
async def test_catalog_execute_handles_fetch_failure(test_session):
"""A fetch_manifest exception ends up on executed.error rather than raising."""
async def fake_fetch(url):
raise RuntimeError('CDN down')
prepared = PreparedMapCatalog(url='https://x', subscribed_regions=[{'region': 'us'}])
with mock.patch('modules.map.lib.fetch_manifest', fake_fetch):
executed = await map_catalog_downloader.execute_download(prepared, make_test_ctx())
assert executed.manifest is None
assert 'CDN down' in (executed.error or '')
def test_catalog_finalize_no_subscribed_regions_returns_success(test_session):
"""User has no subscriptions → success no-op (skips DB writes)."""
download = Download(url='https://x', downloader='map_catalog')
executed = ExecutedMapCatalog(
manifest={'regions': {'us': 'https://x/us.pmtiles'}, 'version': '2024-01'},
subscribed_regions=[],
)
result = map_catalog_downloader.finalize_download(test_session, download, executed)
assert result.success is True
def test_catalog_finalize_no_manifest_regions_returns_failure(test_session):
"""Manifest came back without a regions block → failure."""
download = Download(url='https://x', downloader='map_catalog')
executed = ExecutedMapCatalog(
manifest={'regions': {}, 'version': '2024-01'},
subscribed_regions=[{'region': 'us'}],
)
result = map_catalog_downloader.finalize_download(test_session, download, executed)
assert result.success is False
assert 'does not contain regions' in (result.error or '')
def test_catalog_finalize_propagates_executed_error(test_session):
"""An error raised during execute_download flows into the DownloadResult."""
download = Download(url='https://x', downloader='map_catalog')
executed = ExecutedMapCatalog(
manifest=None, subscribed_regions=[{'region': 'us'}],
error='Failed to fetch manifest: timeout',
)
result = map_catalog_downloader.finalize_download(test_session, download, executed)
assert result.success is False
assert 'timeout' in (result.error or '')
# ---------------------------------------------------------------------------
# MapExtractDownloader
# ---------------------------------------------------------------------------
def test_extract_prepare_missing_region_sets_error(test_session, test_directory, monkeypatch):
"""A download without settings.region fails fast in prepare."""
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: test_directory / 'maps')
download = Download(
url='https://x/foo.pmtiles',
downloader='map_extract',
settings={},
)
prepared = map_extract_downloader.prepare_download(test_session, download)
assert prepared.error == 'No region in download settings'
def test_extract_prepare_already_done(test_session, test_directory, monkeypatch):
"""If the versioned output already exists, prepare flags already_done so execute
becomes a no-op."""
map_dir = test_directory / 'maps'
map_dir.mkdir(parents=True, exist_ok=True)
(map_dir / 'us-2024-01.pmtiles').write_bytes(b'pretend pmtiles')
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
download = Download(
url='https://x/us.pmtiles',
downloader='map_extract',
settings={'region': 'us', 'version': '2024-01'},
)
prepared = map_extract_downloader.prepare_download(test_session, download)
assert prepared.already_done is True
assert prepared.error is None
assert prepared.output_name == 'us-2024-01.pmtiles'
def test_extract_prepare_cleans_leftover_tmp(test_session, test_directory, monkeypatch):
"""A leftover .tmp file from a prior crashed run is removed in prepare."""
map_dir = test_directory / 'maps'
map_dir.mkdir(parents=True, exist_ok=True)
leftover = map_dir / 'us-2024-01.pmtiles.tmp'
leftover.write_bytes(b'partial')
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
download = Download(
url='https://x/us.pmtiles',
downloader='map_extract',
settings={'region': 'us', 'version': '2024-01'},
)
prepared = map_extract_downloader.prepare_download(test_session, download)
assert prepared.already_done is False
assert not leftover.exists()
assert prepared.tmp_path == leftover # path is set, just not on disk anymore
def test_extract_finalize_skipped(test_session, test_directory):
"""skipped → success without touching DB."""
download = Download(url='https://x', downloader='map_extract')
executed = ExecutedMapExtract(
region='us', output_path=test_directory / 'us.pmtiles',
map_directory=test_directory, skipped=True,
)
result = map_extract_downloader.finalize_download(test_session, download, executed)
assert result.success is True
assert result.location == '/map'
def test_extract_finalize_propagates_error(test_session, test_directory):
"""An execute-phase error becomes a failure DownloadResult."""
download = Download(url='https://x', downloader='map_extract')
executed = ExecutedMapExtract(
region='us', output_path=test_directory / 'us.pmtiles',
map_directory=test_directory,
error='Download failed: timeout',
)
result = map_extract_downloader.finalize_download(test_session, download, executed)
assert result.success is False
assert 'timeout' in (result.error or '')
# ---------------------------------------------------------------------------
# MapSearchIndexDownloader
# ---------------------------------------------------------------------------
def test_search_index_prepare_missing_settings_sets_error(test_session, test_directory, monkeypatch):
"""No region or pmtiles_name in settings → error."""
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: test_directory / 'maps')
download = Download(
url='https://x/us.search.db',
downloader='map_search_index',
settings={'region': 'us'}, # missing pmtiles_name
)
prepared = map_search_index_downloader.prepare_download(test_session, download)
assert prepared.error == 'Missing region or pmtiles_name in settings'
def test_search_index_prepare_pmtiles_missing_sets_error(test_session, test_directory, monkeypatch):
"""If the companion PMTiles file isn't on disk yet, prepare fails so the manager
can defer + retry."""
map_dir = test_directory / 'maps'
map_dir.mkdir(parents=True, exist_ok=True)
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
download = Download(
url='https://x/us.search.db',
downloader='map_search_index',
settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'},
)
prepared = map_search_index_downloader.prepare_download(test_session, download)
assert prepared.error and 'PMTiles file not found' in prepared.error
def test_search_index_prepare_already_done(test_session, test_directory, monkeypatch):
"""Both pmtiles and search.db exist → already_done short-circuit."""
map_dir = test_directory / 'maps'
map_dir.mkdir(parents=True, exist_ok=True)
(map_dir / 'us-2024-01.pmtiles').write_bytes(b'tiles')
(map_dir / 'us-2024-01.search.db').write_bytes(b'db')
monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir)
download = Download(
url='https://x/us.search.db',
downloader='map_search_index',
settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'},
)
prepared = map_search_index_downloader.prepare_download(test_session, download)
assert prepared.already_done is True
assert prepared.error is None
def test_search_index_finalize_skipped(test_session):
"""skipped → success."""
download = Download(url='https://x', downloader='map_search_index')
executed = ExecutedMapSearchIndex(region='us', skipped=True)
result = map_search_index_downloader.finalize_download(test_session, download, executed)
assert result.success is True
def test_search_index_finalize_error(test_session):
"""Error path returns failure with the error message."""
download = Download(url='https://x', downloader='map_search_index')
executed = ExecutedMapSearchIndex(region='us', error='Download failed: 500')
result = map_search_index_downloader.finalize_download(test_session, download, executed)
assert result.success is False
assert '500' in (result.error or '')