diff --git a/modules/map/downloader.py b/modules/map/downloader.py index b9aadc6d..6ab693ba 100644 --- a/modules/map/downloader.py +++ b/modules/map/downloader.py @@ -1,14 +1,29 @@ +import pathlib import tempfile +from dataclasses import dataclass from pathlib import Path +from typing import Any, List, Optional from xml.etree import ElementTree as ET +from sqlalchemy.orm import Session + from modules.map import lib from wrolpi.common import logger, verify_gpg_signature -from wrolpi.db import get_db_session -from wrolpi.downloader import Downloader, Download, DownloadResult, download_manager +from wrolpi.downloader import ( + Download, + DownloadContext, + Downloader, + DownloadResult, + download_manager, +) -__all__ = ['MapCatalogDownloader', 'MapExtractDownloader', 'MapSearchIndexDownloader', - 'map_catalog_downloader', 'map_extract_downloader', 'map_search_index_downloader'] +__all__ = [ + 'MapCatalogDownloader', 'MapExtractDownloader', 'MapSearchIndexDownloader', + 'PreparedMapCatalog', 'ExecutedMapCatalog', + 'PreparedMapExtract', 'ExecutedMapExtract', + 'PreparedMapSearchIndex', 'ExecutedMapSearchIndex', + 'map_catalog_downloader', 'map_extract_downloader', 'map_search_index_downloader', +] logger = logger.getChild(__name__) @@ -52,19 +67,65 @@ async def verify_meta4_signature(meta4_contents: bytes) -> bool: return await verify_gpg_signature(hash_path, sig_path) +# --------------------------------------------------------------------------- +# MapCatalogDownloader +# --------------------------------------------------------------------------- + + +@dataclass +class PreparedMapCatalog: + """Plan produced by MapCatalogDownloader.prepare_download. + + `subscribed_regions` is the list of {region: ...} dicts the user has chosen to track. + The manifest itself is fetched in execute_download. + """ + url: str + subscribed_regions: List[dict] + + +@dataclass +class ExecutedMapCatalog: + """Output of MapCatalogDownloader.execute_download. + + Either `manifest` is populated (success) or `error` is set (fetch failed). Child + download creation happens in finalize_download where the manager's session is in + scope. + """ + manifest: Optional[dict] + subscribed_regions: List[dict] + error: Optional[str] = None + + class MapCatalogDownloader(Downloader): """Fetches the manifest and creates a child Download for each subscribed region.""" name = 'map_catalog' listable = False - async def do_download(self, download: Download) -> DownloadResult: + def prepare_download(self, session: Session, download: Download) -> PreparedMapCatalog: + subscribed = (download.settings or {}).get('regions', []) + return PreparedMapCatalog(url=download.url, subscribed_regions=subscribed) + + async def execute_download(self, prepared: PreparedMapCatalog, ctx: DownloadContext, + download: Download = None) -> ExecutedMapCatalog: try: - manifest = await lib.fetch_manifest(download.url) + manifest = await lib.fetch_manifest(prepared.url) except Exception as e: logger.error(f'Failed to fetch map manifest: {e}') - return DownloadResult(success=False, error=f'Failed to fetch manifest: {e}') + return ExecutedMapCatalog( + manifest=None, + subscribed_regions=prepared.subscribed_regions, + error=f'Failed to fetch manifest: {e}', + ) + return ExecutedMapCatalog(manifest=manifest, + subscribed_regions=prepared.subscribed_regions) + def finalize_download(self, session: Session, download: Download, + executed: ExecutedMapCatalog) -> DownloadResult: + if executed.error: + return DownloadResult(success=False, error=executed.error) + + manifest = executed.manifest or {} version = manifest.get('version', '') manifest_regions = manifest.get('regions', {}) search_indexes = manifest.get('search_indexes', {}) @@ -72,8 +133,7 @@ class MapCatalogDownloader(Downloader): if not manifest_regions: return DownloadResult(success=False, error='Manifest does not contain regions') - regions = (download.settings or {}).get('regions', []) - if not regions: + if not executed.subscribed_regions: return DownloadResult(success=True) map_directory = lib.get_map_directory() @@ -83,60 +143,59 @@ class MapCatalogDownloader(Downloader): # download. Downloads process in insertion order; this prevents a search-index # download from running before its companion pmtiles file exists on disk. pending_search_downloads = [] - with get_db_session(commit=True) as session: - for r in regions: - region = r.get('region', '') - region_url = manifest_regions.get(region) - if not region_url: - logger.warning(f'Subscribed region {region!r} not found in manifest — skipping') - continue + for r in executed.subscribed_regions: + region = r.get('region', '') + region_url = manifest_regions.get(region) + if not region_url: + logger.warning(f'Subscribed region {region!r} not found in manifest — skipping') + continue - # Compute the expected PMTiles filename. - if version: - pmtiles_name = f'{region}-{version}.pmtiles' - else: - pmtiles_name = f'{region}.pmtiles' + # Compute the expected PMTiles filename. + if version: + pmtiles_name = f'{region}-{version}.pmtiles' + else: + pmtiles_name = f'{region}.pmtiles' - # Delete stale download for this region if the URL changed (new version). - old = session.query(Download).filter( - Download.downloader == 'map_extract', - Download.settings['region'].astext == region, - Download.url != region_url, - ).one_or_none() - if old: - logger.info(f'Deleting stale download for {region}: {old.url}') - session.delete(old) - session.flush() + # Delete stale download for this region if the URL changed (new version). + old = session.query(Download).filter( + Download.downloader == 'map_extract', + Download.settings['region'].astext == region, + Download.url != region_url, + ).one_or_none() + if old: + logger.info(f'Deleting stale download for {region}: {old.url}') + session.delete(old) + session.flush() - child = download_manager.get_or_create_download(session, region_url, reset_attempts=True, - override_skip=True) - child.downloader = 'map_extract' - child.location = '/map/manage' - child.settings = { - 'region': region, - 'version': version, - 'search_index_url': search_indexes.get(region, ''), - } - child.renew(reset_attempts=True) - created += 1 + child = download_manager.get_or_create_download(session, region_url, reset_attempts=True, + override_skip=True) + child.downloader = 'map_extract' + child.location = '/map/manage' + child.settings = { + 'region': region, + 'version': version, + 'search_index_url': search_indexes.get(region, ''), + } + child.renew(reset_attempts=True) + created += 1 - # Queue search index download if the CDN provides one and the - # .search.db file does not already exist on disk. - search_url = search_indexes.get(region) - search_db_path = (map_directory / pmtiles_name).with_suffix('.search.db') - if search_url and not search_db_path.is_file(): - pending_search_downloads.append((region, search_url, pmtiles_name)) + # Queue search index download if the CDN provides one and the + # .search.db file does not already exist on disk. + search_url = search_indexes.get(region) + search_db_path = (map_directory / pmtiles_name).with_suffix('.search.db') + if search_url and not search_db_path.is_file(): + pending_search_downloads.append((region, search_url, pmtiles_name)) - # Insert all search-index downloads after every pmtiles download, so they - # process in insertion order behind their companion pmtiles. - for region, search_url, pmtiles_name in pending_search_downloads: - search_child = download_manager.get_or_create_download( - session, search_url, reset_attempts=True, override_skip=True) - search_child.downloader = 'map_search_index' - search_child.location = '/map/manage' - search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name} - search_child.renew(reset_attempts=True) - logger.info(f'Created search index download for {region}') + # Insert all search-index downloads after every pmtiles download, so they + # process in insertion order behind their companion pmtiles. + for region, search_url, pmtiles_name in pending_search_downloads: + search_child = download_manager.get_or_create_download( + session, search_url, reset_attempts=True, override_skip=True) + search_child.downloader = 'map_search_index' + search_child.location = '/map/manage' + search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name} + search_child.renew(reset_attempts=True) + logger.info(f'Created search index download for {region}') if not created: return DownloadResult(success=False, error='No matching region URLs found in manifest') @@ -149,6 +208,44 @@ class MapCatalogDownloader(Downloader): map_catalog_downloader = MapCatalogDownloader() +# --------------------------------------------------------------------------- +# MapExtractDownloader +# --------------------------------------------------------------------------- + + +@dataclass +class PreparedMapExtract: + """Plan produced by MapExtractDownloader.prepare_download. + + Filenames are computed from settings; `already_done` short-circuits execute when + the output already exists. `error` carries an early validation failure (e.g. no + region in settings). + """ + region: str + version: str + map_directory: pathlib.Path + output_name: str + output_path: pathlib.Path + tmp_path: pathlib.Path + already_done: bool = False + error: Optional[str] = None + + +@dataclass +class ExecutedMapExtract: + """Output of MapExtractDownloader.execute_download. + + `skipped` covers the already-done short-circuit; `error` covers download/verify + failures. Otherwise the file is at output_path and finalize decides whether to + rebuild the search index locally. + """ + region: str + output_path: pathlib.Path + map_directory: pathlib.Path + skipped: bool = False + error: Optional[str] = None + + class MapExtractDownloader(Downloader): """Downloads a pre-extracted region file from the CDN via aria2c. @@ -162,16 +259,19 @@ class MapExtractDownloader(Downloader): name = 'map_extract' listable = False - async def do_download(self, download: Download) -> DownloadResult: + def prepare_download(self, session: Session, download: Download) -> PreparedMapExtract: settings = download.settings or {} region = settings.get('region') version = settings.get('version', '') if not region: - return DownloadResult(success=False, error='No region in download settings') + return PreparedMapExtract( + region='', version='', map_directory=pathlib.Path('.'), + output_name='', output_path=pathlib.Path('.'), tmp_path=pathlib.Path('.'), + error='No region in download settings', + ) map_directory = lib.get_map_directory() - # Versioned filename. if version: output_name = f'{region}-{version}.pmtiles' @@ -180,75 +280,118 @@ class MapExtractDownloader(Downloader): output_path = map_directory / output_name tmp_path = map_directory / f'{output_name}.tmp' + prepared = PreparedMapExtract( + region=region, version=version, map_directory=map_directory, + output_name=output_name, output_path=output_path, tmp_path=tmp_path, + ) + # Skip if versioned file already exists. if output_path.is_file(): logger.info(f'{output_name} already exists, skipping download') - return DownloadResult(success=True, location='/map') + prepared.already_done = True + return prepared # Clean up any leftover temp file. if tmp_path.is_file(): tmp_path.unlink() + return prepared + + async def execute_download(self, prepared: PreparedMapExtract, ctx: DownloadContext, + download: Download = None) -> ExecutedMapExtract: + if prepared.error: + return ExecutedMapExtract(region=prepared.region, output_path=prepared.output_path, + map_directory=prepared.map_directory, error=prepared.error) + if prepared.already_done: + return ExecutedMapExtract(region=prepared.region, output_path=prepared.output_path, + map_directory=prepared.map_directory, skipped=True) + + download = download if download is not None else Download(url=prepared.output_path.name) + # The original download URL is needed for download_file; the manager passes the real + # Download in production, so we read .url from it. In unit tests, callers must supply + # `download` with a URL. + url = download.url + # Fetch meta4 once, GPG-verify the hash signature before downloading. - meta4_contents = await self.get_meta4_contents(download.url) + meta4_contents = await self.get_meta4_contents(url) if meta4_contents: if not await verify_meta4_signature(meta4_contents): - return DownloadResult(success=False, - error=f'meta4 hash signature verification failed for {region}') - logger.info(f'meta4 signature verified for {region}') + return ExecutedMapExtract( + region=prepared.region, output_path=prepared.output_path, + map_directory=prepared.map_directory, + error=f'meta4 hash signature verification failed for {prepared.region}', + ) + logger.info(f'meta4 signature verified for {prepared.region}') else: - logger.warning(f'No meta4 available for {region} — downloading without hash verification') + logger.warning(f'No meta4 available for {prepared.region} — downloading without hash verification') # Download via aria2c, passing the pre-verified meta4 for hash checking. - logger.warning(f'Downloading {region} from {download.url}') + logger.warning(f'Downloading {prepared.region} from {url}') downloaded_path = None try: downloaded_path = await self.download_file( - download, download.url, map_directory, + download, url, prepared.map_directory, check_for_meta4=False, concurrent=1, meta4_xml=meta4_contents, + ctx=ctx, ) - downloaded_path.rename(tmp_path) + downloaded_path.rename(prepared.tmp_path) except Exception as e: - logger.error(f'Download failed for {region}: {e}') - tmp_path.unlink(missing_ok=True) + logger.error(f'Download failed for {prepared.region}: {e}') + prepared.tmp_path.unlink(missing_ok=True) if downloaded_path: downloaded_path.unlink(missing_ok=True) - return DownloadResult(success=False, error=f'Download failed: {str(e)[:500]}') + return ExecutedMapExtract( + region=prepared.region, output_path=prepared.output_path, + map_directory=prepared.map_directory, + error=f'Download failed: {str(e)[:500]}', + ) - if not tmp_path.is_file(): - return DownloadResult(success=False, error=f'Output file was not created: {tmp_path}') + if not prepared.tmp_path.is_file(): + return ExecutedMapExtract( + region=prepared.region, output_path=prepared.output_path, + map_directory=prepared.map_directory, + error=f'Output file was not created: {prepared.tmp_path}', + ) # Replace old file atomically. - tmp_path.rename(output_path) + prepared.tmp_path.rename(prepared.output_path) # Delete older versions of this region. - for old_file in map_directory.glob(f'{region}-*.pmtiles'): - if old_file.name != output_name: + for old_file in prepared.map_directory.glob(f'{prepared.region}-*.pmtiles'): + if old_file.name != prepared.output_name: logger.warning(f'Deleting old map version: {old_file}') old_file.unlink() # Also delete unversioned file if it exists. - unversioned = map_directory / f'{region}.pmtiles' - if unversioned.is_file() and unversioned.name != output_name: + unversioned = prepared.map_directory / f'{prepared.region}.pmtiles' + if unversioned.is_file() and unversioned.name != prepared.output_name: logger.warning(f'Deleting unversioned map file: {unversioned}') unversioned.unlink() - size = output_path.stat().st_size - logger.warning(f'Successfully downloaded {region} ({size} bytes) to {output_path}') + size = prepared.output_path.stat().st_size + logger.warning(f'Successfully downloaded {prepared.region} ({size} bytes) to {prepared.output_path}') + + return ExecutedMapExtract(region=prepared.region, output_path=prepared.output_path, + map_directory=prepared.map_directory) + + def finalize_download(self, session: Session, download: Download, + executed: ExecutedMapExtract) -> DownloadResult: + if executed.error: + return DownloadResult(success=False, error=executed.error) + if executed.skipped: + return DownloadResult(success=True, location='/map') # Build search index locally if tippecanoe is available and no CDN # search index download already exists for this region. - has_search_download = False - with get_db_session() as session: - has_search_download = session.query(Download).filter( - Download.downloader == 'map_search_index', - Download.settings['region'].astext == region, - ).count() > 0 + has_search_download = session.query(Download).filter( + Download.downloader == 'map_search_index', + Download.settings['region'].astext == executed.region, + ).count() > 0 if not has_search_download: try: from modules.map.search import rebuild_search_index - rebuild_search_index(output_path.name) + rebuild_search_index(executed.output_path.name) except Exception as e: - logger.error(f'Failed to start search index build for {region}: {e}') + logger.error(f'Failed to start search index build for {executed.region}: {e}') return DownloadResult(success=True, location='/map') @@ -256,6 +399,34 @@ class MapExtractDownloader(Downloader): map_extract_downloader = MapExtractDownloader() +# --------------------------------------------------------------------------- +# MapSearchIndexDownloader +# --------------------------------------------------------------------------- + + +@dataclass +class PreparedMapSearchIndex: + """Plan produced by MapSearchIndexDownloader.prepare_download. + + `error` covers missing settings or a missing PMTiles file (the search index can't + exist without one). `already_done` short-circuits when the .search.db is on disk. + """ + region: str + pmtiles_name: str + map_directory: pathlib.Path + pmtiles_path: pathlib.Path + search_db_path: pathlib.Path + already_done: bool = False + error: Optional[str] = None + + +@dataclass +class ExecutedMapSearchIndex: + region: str + skipped: bool = False + error: Optional[str] = None + + class MapSearchIndexDownloader(Downloader): """Downloads a pre-built .search.db file from the CDN via aria2c with meta4 verification. @@ -266,54 +437,83 @@ class MapSearchIndexDownloader(Downloader): name = 'map_search_index' listable = False - async def do_download(self, download: Download) -> DownloadResult: + def prepare_download(self, session: Session, download: Download) -> PreparedMapSearchIndex: settings = download.settings or {} region = settings.get('region', '') pmtiles_name = settings.get('pmtiles_name', '') - if not region or not pmtiles_name: - return DownloadResult(success=False, error='Missing region or pmtiles_name in settings') - map_directory = lib.get_map_directory() - pmtiles_path = map_directory / pmtiles_name - if not pmtiles_path.is_file(): - return DownloadResult(success=False, error=f'PMTiles file not found: {pmtiles_name}') + prepared = PreparedMapSearchIndex( + region=region, pmtiles_name=pmtiles_name, map_directory=map_directory, + pmtiles_path=map_directory / pmtiles_name if pmtiles_name else map_directory, + search_db_path=(map_directory / pmtiles_name).with_suffix('.search.db') + if pmtiles_name else map_directory, + ) - # Name the search.db to match the local PMTiles filename. - search_db_path = pmtiles_path.with_suffix('.search.db') + if not region or not pmtiles_name: + prepared.error = 'Missing region or pmtiles_name in settings' + return prepared - if search_db_path.is_file(): + if not prepared.pmtiles_path.is_file(): + prepared.error = f'PMTiles file not found: {pmtiles_name}' + return prepared + + if prepared.search_db_path.is_file(): logger.info(f'Search index already exists for {region}, skipping') - return DownloadResult(success=True, location='/map') + prepared.already_done = True + + return prepared + + async def execute_download(self, prepared: PreparedMapSearchIndex, ctx: DownloadContext, + download: Download = None) -> ExecutedMapSearchIndex: + if prepared.error: + return ExecutedMapSearchIndex(region=prepared.region, error=prepared.error) + if prepared.already_done: + return ExecutedMapSearchIndex(region=prepared.region, skipped=True) + + download = download if download is not None else Download(url=prepared.pmtiles_name) + url = download.url # Fetch and verify meta4 (GPG-signed hash). - meta4_contents = await self.get_meta4_contents(download.url) + meta4_contents = await self.get_meta4_contents(url) if meta4_contents: if not await verify_meta4_signature(meta4_contents): - return DownloadResult(success=False, - error=f'meta4 signature verification failed for {region} search index') - logger.info(f'Search index meta4 verified for {region}') + return ExecutedMapSearchIndex( + region=prepared.region, + error=f'meta4 signature verification failed for {prepared.region} search index', + ) + logger.info(f'Search index meta4 verified for {prepared.region}') # Download via aria2c with meta4 hash verification. - logger.warning(f'Downloading search index for {region} from {download.url}') + logger.warning(f'Downloading search index for {prepared.region} from {url}') downloaded_path = None try: downloaded_path = await self.download_file( - download, download.url, map_directory, + download, url, prepared.map_directory, check_for_meta4=False, concurrent=1, meta4_xml=meta4_contents, + ctx=ctx, ) # Rename to match the local PMTiles filename. - downloaded_path.rename(search_db_path) + downloaded_path.rename(prepared.search_db_path) except Exception as e: - logger.error(f'Search index download failed for {region}: {e}') + logger.error(f'Search index download failed for {prepared.region}: {e}') if downloaded_path: downloaded_path.unlink(missing_ok=True) - search_db_path.unlink(missing_ok=True) - return DownloadResult(success=False, error=f'Download failed: {str(e)[:500]}') + prepared.search_db_path.unlink(missing_ok=True) + return ExecutedMapSearchIndex( + region=prepared.region, + error=f'Download failed: {str(e)[:500]}', + ) - size = search_db_path.stat().st_size - logger.warning(f'Downloaded search index for {region} ({size} bytes)') + size = prepared.search_db_path.stat().st_size + logger.warning(f'Downloaded search index for {prepared.region} ({size} bytes)') + return ExecutedMapSearchIndex(region=prepared.region) + + def finalize_download(self, session: Session, download: Download, + executed: ExecutedMapSearchIndex) -> DownloadResult: + if executed.error: + return DownloadResult(success=False, error=executed.error) return DownloadResult(success=True, location='/map') diff --git a/modules/map/test/test_downloader.py b/modules/map/test/test_downloader.py index e69de29b..91d3f0fd 100644 --- a/modules/map/test/test_downloader.py +++ b/modules/map/test/test_downloader.py @@ -0,0 +1,283 @@ +"""Phase-split tests for MapCatalogDownloader / MapExtractDownloader / MapSearchIndexDownloader. + +These exercise prepare_download and finalize_download in isolation — no async_client, +no test_download_manager, no real subprocess. Execute paths that drive download_file + +GPG verification + filesystem rename are integration territory and are deliberately +omitted; the parent download_file ctx threading is already covered by the FileDownloader +migration tests. +""" +from unittest import mock + +import pytest + +from modules.archive.conftest import make_test_ctx +from modules.map.downloader import ( + ExecutedMapCatalog, + ExecutedMapExtract, + ExecutedMapSearchIndex, + PreparedMapCatalog, + PreparedMapExtract, + PreparedMapSearchIndex, + map_catalog_downloader, + map_extract_downloader, + map_search_index_downloader, +) +from wrolpi.downloader import Download + + +# --------------------------------------------------------------------------- +# MapCatalogDownloader +# --------------------------------------------------------------------------- + + +def test_catalog_prepare_extracts_subscribed_regions(test_session): + """prepare_download surfaces the user's region subscriptions.""" + download = Download( + url='https://maps.example.com/manifest.json', + downloader='map_catalog', + settings={'regions': [{'region': 'us'}, {'region': 'eu'}]}, + ) + + prepared = map_catalog_downloader.prepare_download(test_session, download) + + assert isinstance(prepared, PreparedMapCatalog) + assert prepared.url == download.url + assert prepared.subscribed_regions == [{'region': 'us'}, {'region': 'eu'}] + + +def test_catalog_prepare_handles_no_regions_in_settings(test_session): + """No 'regions' key → empty list (handled in finalize as success no-op).""" + download = Download( + url='https://maps.example.com/manifest.json', + downloader='map_catalog', + settings={}, + ) + + prepared = map_catalog_downloader.prepare_download(test_session, download) + + assert prepared.subscribed_regions == [] + + +@pytest.mark.asyncio +async def test_catalog_execute_handles_fetch_failure(test_session): + """A fetch_manifest exception ends up on executed.error rather than raising.""" + + async def fake_fetch(url): + raise RuntimeError('CDN down') + + prepared = PreparedMapCatalog(url='https://x', subscribed_regions=[{'region': 'us'}]) + + with mock.patch('modules.map.lib.fetch_manifest', fake_fetch): + executed = await map_catalog_downloader.execute_download(prepared, make_test_ctx()) + + assert executed.manifest is None + assert 'CDN down' in (executed.error or '') + + +def test_catalog_finalize_no_subscribed_regions_returns_success(test_session): + """User has no subscriptions → success no-op (skips DB writes).""" + download = Download(url='https://x', downloader='map_catalog') + executed = ExecutedMapCatalog( + manifest={'regions': {'us': 'https://x/us.pmtiles'}, 'version': '2024-01'}, + subscribed_regions=[], + ) + + result = map_catalog_downloader.finalize_download(test_session, download, executed) + + assert result.success is True + + +def test_catalog_finalize_no_manifest_regions_returns_failure(test_session): + """Manifest came back without a regions block → failure.""" + download = Download(url='https://x', downloader='map_catalog') + executed = ExecutedMapCatalog( + manifest={'regions': {}, 'version': '2024-01'}, + subscribed_regions=[{'region': 'us'}], + ) + + result = map_catalog_downloader.finalize_download(test_session, download, executed) + + assert result.success is False + assert 'does not contain regions' in (result.error or '') + + +def test_catalog_finalize_propagates_executed_error(test_session): + """An error raised during execute_download flows into the DownloadResult.""" + download = Download(url='https://x', downloader='map_catalog') + executed = ExecutedMapCatalog( + manifest=None, subscribed_regions=[{'region': 'us'}], + error='Failed to fetch manifest: timeout', + ) + + result = map_catalog_downloader.finalize_download(test_session, download, executed) + + assert result.success is False + assert 'timeout' in (result.error or '') + + +# --------------------------------------------------------------------------- +# MapExtractDownloader +# --------------------------------------------------------------------------- + + +def test_extract_prepare_missing_region_sets_error(test_session, test_directory, monkeypatch): + """A download without settings.region fails fast in prepare.""" + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: test_directory / 'maps') + download = Download( + url='https://x/foo.pmtiles', + downloader='map_extract', + settings={}, + ) + + prepared = map_extract_downloader.prepare_download(test_session, download) + + assert prepared.error == 'No region in download settings' + + +def test_extract_prepare_already_done(test_session, test_directory, monkeypatch): + """If the versioned output already exists, prepare flags already_done so execute + becomes a no-op.""" + map_dir = test_directory / 'maps' + map_dir.mkdir(parents=True, exist_ok=True) + (map_dir / 'us-2024-01.pmtiles').write_bytes(b'pretend pmtiles') + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + + download = Download( + url='https://x/us.pmtiles', + downloader='map_extract', + settings={'region': 'us', 'version': '2024-01'}, + ) + + prepared = map_extract_downloader.prepare_download(test_session, download) + + assert prepared.already_done is True + assert prepared.error is None + assert prepared.output_name == 'us-2024-01.pmtiles' + + +def test_extract_prepare_cleans_leftover_tmp(test_session, test_directory, monkeypatch): + """A leftover .tmp file from a prior crashed run is removed in prepare.""" + map_dir = test_directory / 'maps' + map_dir.mkdir(parents=True, exist_ok=True) + leftover = map_dir / 'us-2024-01.pmtiles.tmp' + leftover.write_bytes(b'partial') + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + + download = Download( + url='https://x/us.pmtiles', + downloader='map_extract', + settings={'region': 'us', 'version': '2024-01'}, + ) + + prepared = map_extract_downloader.prepare_download(test_session, download) + + assert prepared.already_done is False + assert not leftover.exists() + assert prepared.tmp_path == leftover # path is set, just not on disk anymore + + +def test_extract_finalize_skipped(test_session, test_directory): + """skipped → success without touching DB.""" + download = Download(url='https://x', downloader='map_extract') + executed = ExecutedMapExtract( + region='us', output_path=test_directory / 'us.pmtiles', + map_directory=test_directory, skipped=True, + ) + + result = map_extract_downloader.finalize_download(test_session, download, executed) + + assert result.success is True + assert result.location == '/map' + + +def test_extract_finalize_propagates_error(test_session, test_directory): + """An execute-phase error becomes a failure DownloadResult.""" + download = Download(url='https://x', downloader='map_extract') + executed = ExecutedMapExtract( + region='us', output_path=test_directory / 'us.pmtiles', + map_directory=test_directory, + error='Download failed: timeout', + ) + + result = map_extract_downloader.finalize_download(test_session, download, executed) + + assert result.success is False + assert 'timeout' in (result.error or '') + + +# --------------------------------------------------------------------------- +# MapSearchIndexDownloader +# --------------------------------------------------------------------------- + + +def test_search_index_prepare_missing_settings_sets_error(test_session, test_directory, monkeypatch): + """No region or pmtiles_name in settings → error.""" + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: test_directory / 'maps') + download = Download( + url='https://x/us.search.db', + downloader='map_search_index', + settings={'region': 'us'}, # missing pmtiles_name + ) + + prepared = map_search_index_downloader.prepare_download(test_session, download) + + assert prepared.error == 'Missing region or pmtiles_name in settings' + + +def test_search_index_prepare_pmtiles_missing_sets_error(test_session, test_directory, monkeypatch): + """If the companion PMTiles file isn't on disk yet, prepare fails so the manager + can defer + retry.""" + map_dir = test_directory / 'maps' + map_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + + download = Download( + url='https://x/us.search.db', + downloader='map_search_index', + settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'}, + ) + + prepared = map_search_index_downloader.prepare_download(test_session, download) + + assert prepared.error and 'PMTiles file not found' in prepared.error + + +def test_search_index_prepare_already_done(test_session, test_directory, monkeypatch): + """Both pmtiles and search.db exist → already_done short-circuit.""" + map_dir = test_directory / 'maps' + map_dir.mkdir(parents=True, exist_ok=True) + (map_dir / 'us-2024-01.pmtiles').write_bytes(b'tiles') + (map_dir / 'us-2024-01.search.db').write_bytes(b'db') + monkeypatch.setattr('modules.map.lib.get_map_directory', lambda: map_dir) + + download = Download( + url='https://x/us.search.db', + downloader='map_search_index', + settings={'region': 'us', 'pmtiles_name': 'us-2024-01.pmtiles'}, + ) + + prepared = map_search_index_downloader.prepare_download(test_session, download) + + assert prepared.already_done is True + assert prepared.error is None + + +def test_search_index_finalize_skipped(test_session): + """skipped → success.""" + download = Download(url='https://x', downloader='map_search_index') + executed = ExecutedMapSearchIndex(region='us', skipped=True) + + result = map_search_index_downloader.finalize_download(test_session, download, executed) + + assert result.success is True + + +def test_search_index_finalize_error(test_session): + """Error path returns failure with the error message.""" + download = Download(url='https://x', downloader='map_search_index') + executed = ExecutedMapSearchIndex(region='us', error='Download failed: 500') + + result = map_search_index_downloader.finalize_download(test_session, download, executed) + + assert result.success is False + assert '500' in (result.error or '')