Key changes:

- Add MapLibre GL JS viewer with light/dark themes, terrain layer,
  and embedded static map page for coordinate sharing
- Add map pin support with persistent map_pins.yaml config
- Add map region subscriptions and downloader with progress tracking,
  sourcing pre-extracted PMTiles from a cold CDN
- Add map export to PNG and GPS coordinate copy via context menu
- Add MapViewer React component alongside reworked Map page
- Add map catalog and region-building script (build_map_regions.py)
- Remove OSM import/render scripts, leaflet files, Apache configs,
  renderd service, and osm-map-dumper Docker container
- Update Caddyfile to serve PMTiles and map static assets
- Update build/install/repair scripts for Protomaps dependencies
- Drop map_file table, add/drop map_subscription table (alembic)
- Adding button to skip primary drive mounting during onboarding.

.gitignore

@@ -1,3 +1,6 @@
+# Local blobs directory (planet overview, etc.)
+/blobs/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

alembic/env.py

@@ -26,7 +26,6 @@ from wrolpi.collections.models import Collection, CollectionItem  # noqa: F401
 from modules.videos.models import Video, Channel  # noqa: F401
 from modules.archive.models import Archive  # noqa: F401
 from modules.zim.models import Zim, TagZimEntry, ZimSubscription  # noqa: F401
-from modules.map.models import MapFile  # noqa: F401
 from modules.inventory.models import Item, Inventory  # noqa: F401
 target_metadata = Base.metadata
 

alembic/versions/a2b3c4d5e6f7_drop_map_file_table.py

@@ -0,0 +1,29 @@
+"""Drop map_file table.
+
+PMTiles files are now scanned directly from disk; the map_file table is no longer needed.
+
+Revision ID: a2b3c4d5e6f7
+Revises: e7f8a9b0c1d2
+Create Date: 2026-03-29
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'a2b3c4d5e6f7'
+down_revision = 'e7f8a9b0c1d2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_table('map_file')
+
+
+def downgrade():
+    op.execute('''CREATE TABLE map_file (
+        id SERIAL PRIMARY KEY,
+        path TEXT NOT NULL,
+        imported BOOLEAN DEFAULT FALSE,
+        size BIGINT
+    )''')

alembic/versions/b3c4d5e6f7a8_create_map_subscription_table.py

@@ -0,0 +1,26 @@
+"""Create map_subscription table.
+
+Revision ID: b3c4d5e6f7a8
+Revises: a2b3c4d5e6f7
+Create Date: 2026-03-29
+
+"""
+from alembic import op
+
+revision = 'b3c4d5e6f7a8'
+down_revision = 'a2b3c4d5e6f7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute('''CREATE TABLE map_subscription (
+        id SERIAL PRIMARY KEY,
+        name TEXT UNIQUE NOT NULL,
+        region TEXT NOT NULL,
+        download_id INTEGER NOT NULL REFERENCES download(id) ON DELETE CASCADE
+    )''')
+
+
+def downgrade():
+    op.execute('DROP TABLE IF EXISTS map_subscription')

alembic/versions/c4d5e6f7a8b9_drop_map_subscription_table.py

@@ -0,0 +1,28 @@
+"""Drop map_subscription table.
+
+Map subscriptions are now managed via settings.regions on a single Download record.
+
+Revision ID: c4d5e6f7a8b9
+Revises: b3c4d5e6f7a8
+Create Date: 2026-03-31
+
+"""
+from alembic import op
+
+revision = 'c4d5e6f7a8b9'
+down_revision = 'b3c4d5e6f7a8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute('DROP TABLE IF EXISTS map_subscription')
+
+
+def downgrade():
+    op.execute('''CREATE TABLE map_subscription (
+        id SERIAL PRIMARY KEY,
+        name TEXT UNIQUE NOT NULL,
+        region TEXT NOT NULL,
+        download_id INTEGER NOT NULL REFERENCES download(id) ON DELETE CASCADE
+    )''')

app/package.json

@@ -3,13 +3,17 @@
   "version": "0.1.0",
   "private": true,
   "dependencies": {
+    "@acalcutt/maplibre-contour-pmtiles": "^0.1.2",
     "@artsy/fresnel": "^6.1.0",
     "@cypress/react18": "^2.0.1",
     "@testing-library/react": "^13.4.0",
     "@testing-library/user-event": "^13.5.0",
     "cypress": "^13.15.2",
     "lodash": "^4.17.23",
+    "maplibre-gl": "^5.21.1",
     "mathjs": "^14.0.1",
+    "pmtiles": "^4.4.0",
+    "protomaps-themes-base": "^4.5.0",
     "query-string": "^8.1.0",
     "react": "^18.2.0",
     "react-colorful": "^5.6.1",

app/src/api.js

@@ -1485,21 +1485,85 @@ export async function setThrottle(on) {
     }
 }
 
-export async function getMapImportStatus() {
+export async function getMapFiles() {
     const response = await apiGet(`${API_URI}/map/files`);
     if (response.ok) {
         return await response.json();
     } else {
-        const message = await getErrorMessage(response, 'Could not get import status');
+        const message = await getErrorMessage(response, 'Could not get map files');
         toast({type: 'error', title: 'Map Error', description: message, time: 5000});
     }
 }
 
-export async function importMapFiles(paths) {
-    let body = {'files': paths};
-    const response = await apiPost(`${API_URI}/map/import`, body);
+export async function deleteMapFile(filename) {
+    const response = await apiDelete(`${API_URI}/map/files/${encodeURIComponent(filename)}`);
     if (!response.ok) {
-        const message = await getErrorMessage(response, 'Could not start import!  See server logs.');
+        const message = await getErrorMessage(response, 'Could not delete map file');
+        toast({type: 'error', title: 'Map Error', description: message, time: 5000});
+    }
+}
+
+export async function fetchMapSubscriptions() {
+    const response = await apiGet(`${API_URI}/map/subscribe`);
+    if (response.ok) {
+        return await response.json();
+    } else {
+        const message = await getErrorMessage(response, 'Could not get map subscriptions');
+        toast({type: 'error', title: 'Map Error', description: message, time: 5000});
+    }
+}
+
+export async function mapSubscribe(name, region) {
+    const body = {name, region};
+    const response = await apiPost(`${API_URI}/map/subscribe`, body);
+    if (response.ok) {
+        return true;
+    } else {
+        const message = await getErrorMessage(response, 'Could not subscribe to map region');
+        toast({type: 'error', title: 'Map Error', description: message, time: 5000});
+        return false;
+    }
+}
+
+export async function mapUnsubscribe(region) {
+    const response = await apiDelete(`${API_URI}/map/subscribe/${encodeURIComponent(region)}`);
+    if (!response.ok) {
+        const message = await getErrorMessage(response, 'Could not unsubscribe from map region');
+        toast({type: 'error', title: 'Map Error', description: message, time: 5000});
+    }
+}
+
+export async function getMapPins() {
+    const response = await apiGet(`${API_URI}/map/pins`);
+    if (response.ok) {
+        return await response.json();
+    }
+}
+
+export async function addMapPin(lat, lon, label, color) {
+    const body = {lat, lon, label, color};
+    const response = await apiPost(`${API_URI}/map/pins`, body);
+    if (response.ok) {
+        return await response.json();
+    } else {
+        const message = await getErrorMessage(response, 'Could not add map pin');
+        toast({type: 'error', title: 'Map Error', description: message, time: 5000});
+    }
+}
+
+export async function deleteMapPin(pinId) {
+    const response = await apiDelete(`${API_URI}/map/pins/${pinId}`);
+    if (!response.ok) {
+        const message = await getErrorMessage(response, 'Could not delete map pin');
+        toast({type: 'error', title: 'Map Error', description: message, time: 5000});
+    }
+}
+
+export async function updateMapPin(pinId, label, color) {
+    const body = {label, color};
+    const response = await apiPut(`${API_URI}/map/pins/${pinId}`, body);
+    if (!response.ok) {
+        const message = await getErrorMessage(response, 'Could not update map pin');
         toast({type: 'error', title: 'Map Error', description: message, time: 5000});
     }
 }

app/src/components/Common.js

@@ -2054,10 +2054,10 @@ export function IframeViewer({title, src, fallback, style, timeout = 5000}) {
 
     const iframeMedia = <>
         <Media at='mobile'>
-            <iframe title={title} src={src} style={mobileStyle}/>
+            <iframe title={title} src={src} style={mobileStyle} allow="geolocation"/>
         </Media>
         <Media greaterThan='mobile'>
-            <iframe title={title} src={src} style={tabletStyle}/>
+            <iframe title={title} src={src} style={tabletStyle} allow="geolocation"/>
         </Media>
     </>;
 

app/src/components/Map.js

@@ -1,302 +1,610 @@
-import React, {useContext} from "react";
+import React, {useEffect, useState} from "react";
 import {
     APIButton,
     ErrorMessage,
     HandPointMessage,
     humanFileSize,
-    IframeViewer,
     InfoMessage,
-    InfoPopup,
     PageContainer,
     TabLinks,
     useTitle,
     WROLModeMessage
 } from "./Common";
-import {Route, Routes} from "react-router";
-import {getMapImportStatus, importMapFiles} from "../api";
+import {Route, Routes, useNavigate} from "react-router";
 import {
-    Checkbox,
+    deleteMapFile, deleteMapPin, fetchMapSubscriptions, getMapFiles, getMapPins,
+    mapSubscribe, mapUnsubscribe, updateMapPin
+} from "../api";
+import {
+    Button,
     Divider,
-    Icon as SIcon,
-    Loader as SLoader,
+    Input,
     TableCell,
-    TableFooter,
-    TableHeaderCell,
     TableRow
 } from "semantic-ui-react";
 import {SortableTable} from "./SortableTable";
 import Message from "semantic-ui-react/dist/commonjs/collections/Message";
-import {Header, Loader, Segment} from "./Theme";
-import {StatusContext} from "../contexts/contexts";
-import _ from "lodash";
+import {Header} from "./Theme";
+import {Media} from "../contexts/contexts";
 import {MAP_VIEWER_URI} from "./Vars";
+import {Modal} from "./Theme";
+import MapViewer from "./MapViewer";
+import maplibregl from "maplibre-gl";
+import {Protocol} from "pmtiles";
+import layers from "protomaps-themes-base";
 
-function DockerMapImportWarning() {
-    const {status} = useContext(StatusContext);
-    if (status['dockerized']) {
-        return <ErrorMessage storageName='hint_map_docker_warning'>
-            <Message.Header>
-                Maps are not fully supported in a Docker container!
-            </Message.Header>
+// Ensure PMTiles protocol is registered (may already be registered by MapViewer).
+try { maplibregl.addProtocol("pmtiles", new Protocol().tile); } catch (e) { /* already registered */ }
 
-            <p><b>Only one PBF can be imported and displayed in the docker container.</b></p>
+const MAP_OVERVIEW_URL = "/blobs/map-overview.pmtiles";
 
-            <p>To import a map file, run the following docker-compose commands:</p>
-            <pre>  docker-compose stop map</pre>
-            <pre>  docker-compose rm map</pre>
-            <pre>  docker-compose run --rm -v /absolute/path/to/map.osm.pbf:/data.osm.pbf
-                    -v openstreetmap-data:/var/lib/postgresql/12/main map import
-                </pre>
-
-            <p>Be sure to change <b>/absolute/path/to/map.osm.pbf</b>!</p>
-
-            <p>After you have imported a new PBF file, you need to clear the rendered tile cache:</p>
-            <pre>  docker volume rm openstreetmap-rendered-tiles</pre>
-            <pre>  docker volume create openstreetmap-rendered-tiles</pre>
-
-            <p>Start your map container:</p>
-            <pre>  docker-compose up -d map</pre>
-
-            <Divider/>
-
-            <p>You can merge multiple PBF files using osmium (the merged file can then be imported):</p>
-            <pre>  osmium merge file1.osm.pbf file2.osm.pbf -o merged.osm.pbf</pre>
-        </ErrorMessage>;
-    }
-    return <></>;
+// Detect full planet files: "planet-20260329.pmtiles" or bare date like "20260329.pmtiles".
+function isPlanetFile(name) {
+    if (name.startsWith("planet-")) return true;
+    return /^\d{8}\.pmtiles$/.test(name);
 }
 
-function DownloadMessage() {
-    return <InfoMessage storageName='hint_map_download'>
-        <Message.Header>Where do I get map files?</Message.Header>
-
-        <p>You can download map files from&nbsp;
-            <a href='https://download.geofabrik.de/' rel='noopener noreferrer' target='_blank'>https://download.geofabrik.de/</a>
-        </p>
-
-        <p><b>Download only the areas you need</b>. Large regions like all of Asia, or the entire
-            planet are most likely <b>too large</b> and won't render quickly. It is recommend to only
-            import files less than 1GB on a Raspberry Pi.</p>
-
-        <p>Only <b>*.osm.pbf</b> files are supported!</p>
-
-        <p>Place downloaded map files into <b>map/pbf</b> so they can be imported here.</p>
-    </InfoMessage>
-}
-
-const ViewerMessage = () => {
-    return <HandPointMessage>
-        <p>You can view the Map at <a href={MAP_VIEWER_URI}>{MAP_VIEWER_URI}</a></p>
-    </HandPointMessage>
-}
-
-function SlowImportMessage() {
-    const {status} = useContext(StatusContext);
-    if (status && status['cpu_stats'] && status['cpu_stats']['temperature'] >= 80) {
-        return <Message warning icon>
-            <SIcon name='exclamation'/>
-            <Message.Content>
-                <Message.Header>
-                    CPU temperature is too high
-                </Message.Header>
-                <p>Any import may take much more time because the CPU is throttled.</p>
-                <small>If your import takes more time than estimated, you will need an aftermarket cooler.</small>
-            </Message.Content>
-        </Message>
-    }
-    return <></>
-}
-
-class ManageMap extends React.Component {
-    constructor(props) {
-        super(props);
-        this.state = {
-            files: null,
-            pending: null,
-            import_running: false,
-            selectedPaths: [],
-            dockerized: null,
+// Resolve the best available base map source for preview maps.
+// Checks for a user's planet file first, falls back to the overview blob.
+async function getPreviewMapSource() {
+    try {
+        const data = await getMapFiles();
+        if (data?.files) {
+            for (const f of data.files) {
+                if (isPlanetFile(f.name)) {
+                    return {type: 'vector', url: `pmtiles:///media/map/${f.name}`};
+                }
+            }
         }
-    }
+    } catch (e) { /* ignore */ }
+    return {type: 'vector', url: `pmtiles://${MAP_OVERVIEW_URL}`};
+}
 
-    async componentDidMount() {
-        await this.fetchImportStatus();
-        this.intervalId = setInterval(this.fetchImportStatus, 1000 * 30);
-    }
+function RegionPreviewModal({bbox, name, open, onClose}) {
+    const mapContainer = React.useRef(null);
 
-    componentWillUnmount() {
-        clearInterval(this.intervalId);
-    }
+    React.useEffect(() => {
+        if (!open || !mapContainer.current || !bbox) return;
 
-    fetchImportStatus = async () => {
-        try {
-            const importStatus = await getMapImportStatus();
-            this.setState({
-                files: importStatus['files'],
-                pending: importStatus['pending'],
-                import_running: importStatus['import_running'],
-                dockerized: importStatus['dockerized'],
+        const [minLon, minLat, maxLon, maxLat] = bbox.split(',').map(Number);
+        const centerLon = (minLon + maxLon) / 2;
+        const centerLat = (minLat + maxLat) / 2;
+
+        let map;
+        getPreviewMapSource().then(source => {
+            if (!mapContainer.current) return;
+            map = new maplibregl.Map({
+                container: mapContainer.current,
+                style: {
+                    version: 8,
+                    sources: {'basemap': source},
+                    layers: layers('basemap', 'light'),
+                    glyphs: '/map-assets/fonts/{fontstack}/{range}.pbf',
+                    sprite: `${window.location.origin}/map-assets/sprites/light`,
+                },
+                center: [centerLon, centerLat],
+                zoom: 1,
+                attributionControl: false,
             });
+
+            map.on('load', () => {
+                map.addSource('bbox', {
+                    type: 'geojson',
+                    data: {
+                        type: 'Feature',
+                        geometry: {
+                            type: 'Polygon',
+                            coordinates: [[
+                                [minLon, minLat],
+                                [maxLon, minLat],
+                                [maxLon, maxLat],
+                                [minLon, maxLat],
+                                [minLon, minLat],
+                            ]],
+                        },
+                    },
+                });
+                map.addLayer({
+                    id: 'bbox-fill',
+                    type: 'fill',
+                    source: 'bbox',
+                    paint: {'fill-color': '#6435c9', 'fill-opacity': 0.2},
+                });
+                map.addLayer({
+                    id: 'bbox-outline',
+                    type: 'line',
+                    source: 'bbox',
+                    paint: {'line-color': '#6435c9', 'line-width': 2},
+                });
+                map.fitBounds([[minLon, minLat], [maxLon, maxLat]], {padding: 40});
+            });
+        });
+
+        return () => { if (map) map.remove(); };
+    }, [open, bbox]);
+
+    return <Modal open={open} onClose={onClose} size='large' closeIcon>
+        <Modal.Header>{name}</Modal.Header>
+        <Modal.Content>
+            <div ref={mapContainer} style={{width: '100%', height: '50vh'}}/>
+        </Modal.Content>
+    </Modal>;
+}
+
+// Random-ish distinct colors for region boxes.
+const REGION_COLORS = [
+    '#6435c9', '#e03997', '#00b5ad', '#2185d0', '#f2711c', '#b5cc18',
+    '#a333c8', '#21ba45', '#db2828', '#fbbd08', '#a5673f', '#767676',
+];
+
+function AllRegionsPreviewModal({catalog, open, onClose}) {
+    const mapContainer = React.useRef(null);
+
+    React.useEffect(() => {
+        if (!open || !mapContainer.current) return;
+
+        let map;
+        getPreviewMapSource().then(source => {
+            if (!mapContainer.current) return;
+            map = new maplibregl.Map({
+                container: mapContainer.current,
+                style: {
+                    version: 8,
+                    sources: {'basemap': source},
+                    layers: layers('basemap', 'light'),
+                    glyphs: '/map-assets/fonts/{fontstack}/{range}.pbf',
+                    sprite: `${window.location.origin}/map-assets/sprites/light`,
+                },
+                center: [0, 20],
+                zoom: 1,
+                attributionControl: false,
+            });
+
+            map.on('load', () => {
+                const regions = catalog.filter(r => r.bbox);
+                regions.forEach((region, i) => {
+                    const [minLon, minLat, maxLon, maxLat] = region.bbox.split(',').map(Number);
+                    const color = REGION_COLORS[i % REGION_COLORS.length];
+                    const id = `region-${i}`;
+
+                    map.addSource(id, {
+                        type: 'geojson',
+                        data: {
+                            type: 'Feature',
+                            geometry: {
+                                type: 'Polygon',
+                                coordinates: [[
+                                    [minLon, minLat], [maxLon, minLat],
+                                    [maxLon, maxLat], [minLon, maxLat],
+                                    [minLon, minLat],
+                                ]],
+                            },
+                            properties: {name: region.name},
+                        },
+                    });
+                    map.addLayer({
+                        id: `${id}-fill`, type: 'fill', source: id,
+                        paint: {'fill-color': color, 'fill-opacity': 0.15},
+                    });
+                    map.addLayer({
+                        id: `${id}-outline`, type: 'line', source: id,
+                        paint: {'line-color': color, 'line-width': 1.5},
+                    });
+                    map.addLayer({
+                        id: `${id}-label`, type: 'symbol', source: id,
+                        layout: {
+                            'text-field': ['get', 'name'],
+                            'text-size': 11,
+                            'text-allow-overlap': true,
+                        },
+                        paint: {
+                            'text-color': color,
+                            'text-halo-color': 'white',
+                            'text-halo-width': 1,
+                        },
+                    });
+                });
+            });
+        });
+
+        return () => { if (map) map.remove(); };
+    }, [open, catalog]);
+
+    return <Modal open={open} onClose={onClose} size='fullscreen' closeIcon>
+        <Modal.Header>All Map Regions</Modal.Header>
+        <Modal.Content>
+            <div ref={mapContainer} style={{width: '100%', height: '75vh'}}/>
+        </Modal.Content>
+    </Modal>;
+}
+
+function MapCatalogRow({item, subscribedRegions, fetchData}) {
+    const {name, region, size_estimate, bbox} = item;
+    const isSubscribed = subscribedRegions.has(region);
+    const [previewOpen, setPreviewOpen] = useState(false);
+
+    const handleButton = async () => {
+        if (isSubscribed) {
+            await mapUnsubscribe(region);
+        } else {
+            await mapSubscribe(name, region);
+        }
+        if (fetchData) {
+            await fetchData();
+        }
+    };
+
+    return <TableRow>
+        <TableCell>{name}</TableCell>
+        <TableCell>{humanFileSize(size_estimate)}</TableCell>
+        <TableCell collapsing>
+            {bbox && <Button
+                size='tiny'
+                icon='eye'
+                onClick={() => setPreviewOpen(true)}
+            />}
+            <Button
+                color={isSubscribed ? 'red' : 'violet'}
+                size='tiny'
+                onClick={handleButton}
+            >
+                {isSubscribed ? 'Unsubscribe' : 'Subscribe'}
+            </Button>
+        </TableCell>
+        {previewOpen && <RegionPreviewModal
+            bbox={bbox}
+            name={name}
+            open={previewOpen}
+            onClose={() => setPreviewOpen(false)}
+        />}
+    </TableRow>;
+}
+
+function ManageMap() {
+    const [files, setFiles] = useState(null);
+    const [catalog, setCatalog] = useState([]);
+    const [subscriptions, setSubscriptions] = useState([]);
+    const [allRegionsOpen, setAllRegionsOpen] = useState(false);
+
+    const fetchFiles = async () => {
+        try {
+            const data = await getMapFiles();
+            if (data) {
+                setFiles(data.files || []);
+            }
         } catch (e) {
             console.error(e);
-            this.setState({files: undefined});
+            setFiles(undefined);
         }
-    }
+    };
 
-    import = async (e) => {
-        if (e) {
-            e.preventDefault();
-        }
-
-        this.setState({'ready': false});
-        const {files, selectedPaths} = this.state;
-        if (_.isEmpty(selectedPaths)) {
-            // No files are selected, import all.
-            let paths = [];
-            for (let i = 0; i < files.length; i++) {
-                paths = paths.concat([files[i].path]);
+    const fetchSubscriptionData = async () => {
+        try {
+            const data = await fetchMapSubscriptions();
+            if (data) {
+                setCatalog(data.catalog || []);
+                setSubscriptions(data.subscriptions || []);
             }
-            await importMapFiles(paths);
-        } else {
-            await importMapFiles(selectedPaths);
+        } catch (e) {
+            console.error(e);
         }
-        await this.fetchImportStatus();
-    }
+    };
 
-    handleCheckbox = (checked, pbf) => {
-        let {selectedPaths} = this.state;
-        if (checked === true) {
-            selectedPaths = selectedPaths.concat([pbf.path]);
-        } else {
-            const index = selectedPaths.indexOf(pbf.path);
-            if (index > -1) {
-                selectedPaths.splice(index, 1);
-            }
-        }
-        this.setState({selectedPaths});
-    }
+    useEffect(() => {
+        fetchFiles();
+        fetchSubscriptionData();
+        const interval = setInterval(fetchFiles, 30000);
+        return () => clearInterval(interval);
+    }, []);
 
-    render() {
-        const {files, selectedPaths, import_running, pending, dockerized} = this.state;
+    const handleDelete = async (filename) => {
+        await deleteMapFile(filename);
+        await fetchFiles();
+    };
 
-        const importingMessage = pending ?
-            pending.length > 1 ? 'Importing...' : `Importing ${pending[0]}`
-            : null;
+    // --- Files table ---
+    const fileHeaders = [
+        {key: 'name', text: 'File', sortBy: i => i.name.toLowerCase()},
+        {key: 'size', text: 'Size', sortBy: i => i.size || 0},
+        {key: 'actions', text: '', sortBy: null},
+    ];
 
-        let disabled = dockerized || _.isEmpty(files) || import_running;
-        let importButton = <APIButton
-            color='violet'
-            disabled={disabled}
-            onClick={this.import}
-        >
-            {!_.isEmpty(selectedPaths) ? 'Import Selected' : 'Import All'}
-        </APIButton>;
-
-        let spaceInfoPopup = <InfoPopup
-            content='Upon importing, a PBF file will consume more disk space than the original file.'/>;
-        let timeInfoPopup = <InfoPopup content='Estimated for a Raspberry Pi 4'/>;
-
-        const tableHeaders = [
-            {key: 'select', text: '', sortBy: null},
-            {key: 'path', text: 'PBF File', sortBy: i => i.path.toLowerCase()},
-            {key: 'imported', text: 'Imported', sortBy: i => i.imported ? 1 : 0},
-            {key: 'size', text: 'Size', sortBy: i => i.size || 0},
-            {key: 'space', text: <>Space Required {spaceInfoPopup}</>, sortBy: i => i.size || 0},
-            {key: 'time', text: <>Time to Import {timeInfoPopup}</>, sortBy: i => i.seconds_to_import || 0},
-        ];
-
-        const rowFunc = (pbf) => {
-            let ref = React.createRef();
-            const {size, path, imported, time_to_import} = pbf;
-            let sizeCells = (<>
-                <TableCell>{humanFileSize(size)}</TableCell>
-                <TableCell>{humanFileSize(size * 25)}</TableCell>
-                <TableCell>{time_to_import}</TableCell>
-            </>);
-            return <TableRow>
-                <TableCell collapsing>
-                    <Checkbox
-                        disabled={disabled}
-                        ref={ref}
-                        onChange={(e, data) => this.handleCheckbox(data.checked, pbf)}
-                    />
-                </TableCell>
-                <TableCell>
-                    <a href={`/media/${path}`}>{path}</a>
-                </TableCell>
-                <TableCell>
-                    {pending && pending.indexOf(path) >= 0 ?
-                        <SLoader active inline size='mini'/> :
-                        imported ? 'yes' : 'no'}
-                </TableCell>
-                {size !== null ? sizeCells : <TableCell colSpan={3}/>}
-            </TableRow>;
-        };
-
-        const emptyRow = <TableRow>
-            <TableCell/><TableCell colSpan={5}>No PBF map files were found in <b>map/pbf</b></TableCell>
+    const fileRowFunc = (file) => {
+        return <TableRow key={file.name}>
+            <TableCell>{file.name}</TableCell>
+            <TableCell>{humanFileSize(file.size)}</TableCell>
+            <TableCell collapsing>
+                <APIButton
+                    color='red'
+                    size='tiny'
+                    confirmContent='Delete this map file?'
+                    confirmButton='Delete'
+                    onClick={() => handleDelete(file.name)}
+                >
+                    Delete
+                </APIButton>
+            </TableCell>
         </TableRow>;
+    };
 
-        const footer = <TableFooter>
-            <TableRow>
-                <TableHeaderCell/>
-                <TableHeaderCell colSpan={5}>
-                    {importButton}
-                </TableHeaderCell>
-            </TableRow>
-        </TableFooter>;
+    const emptyRow = <TableRow>
+        <TableCell colSpan={3}>No PMTiles map files were found in <b>map/</b></TableCell>
+    </TableRow>;
 
-        // Handle error state (files === undefined)
-        if (files === undefined) {
-            return <PageContainer>
-                <WROLModeMessage content='Cannot modify Map'/>
-                <DockerMapImportWarning/>
-                <SlowImportMessage/>
-                <ErrorMessage>Could not fetch map files</ErrorMessage>
-                <DownloadMessage/>
-                <ViewerMessage/>
-            </PageContainer>;
-        }
+    // --- Catalog table ---
+    const catalogHeaders = [
+        {key: 'name', text: 'Region', sortBy: i => i.name.toLowerCase()},
+        {key: 'size', text: 'Estimated Size', sortBy: i => i.size_estimate || 0},
+        {key: 'actions', text: '', sortBy: null},
+    ];
 
+    const catalogRowFunc = (item) => {
+        return <MapCatalogRow
+            key={item.region}
+            item={item}
+            subscribedRegions={new Set(subscriptions.map(s => s.region))}
+            fetchData={fetchSubscriptionData}
+        />;
+    };
+
+    if (files === undefined) {
         return <PageContainer>
             <WROLModeMessage content='Cannot modify Map'/>
-            <DockerMapImportWarning/>
-            <SlowImportMessage/>
-            <Loader size='large' active={import_running} inline='centered'>
-                {importingMessage}
-            </Loader>
-            <SortableTable
-                tableHeaders={tableHeaders}
-                data={files}
-                rowFunc={rowFunc}
-                rowKey='path'
-                defaultSortColumn='path'
-                emptyRow={emptyRow}
-                footer={footer}
-            />
-            <DownloadMessage/>
-            <ViewerMessage/>
-        </PageContainer>
+            <ErrorMessage>Could not fetch map files</ErrorMessage>
+        </PageContainer>;
     }
+
+    return <PageContainer>
+        <WROLModeMessage content='Cannot modify Map'/>
+
+        <Header as='h3'>Map Files</Header>
+        <SortableTable
+            tableHeaders={fileHeaders}
+            data={files}
+            rowFunc={fileRowFunc}
+            rowKey='name'
+            defaultSortColumn='name'
+            emptyRow={emptyRow}
+        />
+
+        <Divider/>
+
+        <div style={{display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginBottom: '0.5em'}}>
+            <Header as='h3'>Map Subscriptions</Header>
+            <Button size='tiny' icon='globe' content='View All Regions' onClick={() => setAllRegionsOpen(true)}/>
+        </div>
+
+        <AllRegionsPreviewModal catalog={catalog} open={allRegionsOpen} onClose={() => setAllRegionsOpen(false)}/>
+
+        <SortableTable
+            tableHeaders={catalogHeaders}
+            data={catalog}
+            rowFunc={catalogRowFunc}
+            rowKey='region'
+            defaultSortColumn='name'
+        />
+
+        <InfoMessage storageName='hint_map_subscribe'>
+            <Message.Header>How to get map files</Message.Header>
+            <p>Subscribe to a map region to automatically download it. Subscriptions will
+                periodically check for updated maps. Downloads may take a long time depending on
+                the region size and your internet speed. You can monitor progress on
+                the <b>Downloads</b> page.</p>
+            <p>You can also manually extract a region using the <b>pmtiles</b> command-line tool
+                and place the resulting file into the <b>map/</b> directory:</p>
+            <pre>  pmtiles extract https://build.protomaps.com/YYYYMMDD.pmtiles region.pmtiles \{'\n'}    --bbox="-125.0,24.4,-66.9,49.4"</pre>
+            <p>Find the latest build date at&nbsp;
+                <a href='https://maps.protomaps.com/builds/' rel='noopener noreferrer'
+                   target='_blank'>maps.protomaps.com/builds</a>
+            </p>
+        </InfoMessage>
+
+        <HandPointMessage>
+            <p>You can also view the map at <a href={MAP_VIEWER_URI}>{MAP_VIEWER_URI}</a></p>
+        </HandPointMessage>
+    </PageContainer>
+}
+
+const PIN_COLORS = ["red", "blue", "green", "yellow", "orange", "purple"];
+
+function PinEditRow({pin, onSave, onCancel}) {
+    const [label, setLabel] = useState(pin.label);
+    const [color, setColor] = useState(pin.color);
+
+    return <TableRow>
+        <TableCell collapsing>
+            <div style={{display: "flex", gap: 4}}>
+                {PIN_COLORS.map(c =>
+                    <div
+                        key={c}
+                        onClick={() => setColor(c)}
+                        style={{
+                            width: 20, height: 20, borderRadius: "50%", background: c, cursor: "pointer",
+                            border: color === c ? "3px solid #333" : "2px solid #ccc",
+                        }}
+                    />
+                )}
+            </div>
+        </TableCell>
+        <TableCell>
+            <Input
+                size='small'
+                value={label}
+                onChange={(e) => setLabel(e.target.value)}
+                onKeyDown={(e) => e.key === 'Enter' && label.trim() && onSave(pin.id, label.trim(), color)}
+            />
+        </TableCell>
+        <TableCell>{pin.lat.toFixed(4)}, {pin.lon.toFixed(4)}</TableCell>
+        <Media greaterThan='mobile'><TableCell>{pin.created}</TableCell></Media>
+        <TableCell collapsing>
+            <Button size='tiny' color='green' onClick={() => label.trim() && onSave(pin.id, label.trim(), color)}>Save</Button>
+            <Button size='tiny' onClick={onCancel}>Cancel</Button>
+        </TableCell>
+    </TableRow>;
+}
+
+function MapPins() {
+    const [pins, setPins] = useState([]);
+    const [filter, setFilter] = useState('');
+    const [editingId, setEditingId] = useState(null);
+    const navigate = useNavigate();
+
+    const fetchPins = async () => {
+        try {
+            const data = await getMapPins();
+            if (data) setPins(data.pins || []);
+        } catch (e) {
+            console.error(e);
+        }
+    };
+
+    useEffect(() => {
+        fetchPins();
+    }, []);
+
+    const handleDelete = async (pinId) => {
+        await deleteMapPin(pinId);
+        setEditingId(null);
+        await fetchPins();
+    };
+
+    const handleSave = async (pinId, label, color) => {
+        await updateMapPin(pinId, label, color);
+        setEditingId(null);
+        await fetchPins();
+    };
+
+    const handleNavigate = (pin) => {
+        navigate(`/map?lat=${pin.lat}&lon=${pin.lon}&z=14`);
+    };
+
+    // --- Desktop headers and row ---
+    const fullHeaders = [
+        {key: 'color', text: '', sortBy: i => i.color},
+        {key: 'label', text: 'Label', sortBy: i => i.label.toLowerCase()},
+        {key: 'coords', text: 'Coordinates', sortBy: i => i.lat},
+        {key: 'created', text: 'Created', sortBy: i => i.created},
+        {key: 'actions', text: '', sortBy: null},
+    ];
+
+    const fullRowFunc = (pin) => {
+        if (editingId === pin.id) {
+            return <PinEditRow
+                key={pin.id}
+                pin={pin}
+                onSave={handleSave}
+                onCancel={() => setEditingId(null)}
+            />;
+        }
+        return <TableRow key={pin.id}>
+            <TableCell collapsing>
+                <div style={{
+                    width: 16, height: 16, borderRadius: "50%",
+                    background: pin.color || "red", border: "1px solid #ccc",
+                }}/>
+            </TableCell>
+            <TableCell>{pin.label}</TableCell>
+            <TableCell>
+                <a href="#" onClick={(e) => {e.preventDefault(); handleNavigate(pin);}}>
+                    {pin.lat.toFixed(4)}, {pin.lon.toFixed(4)}
+                </a>
+            </TableCell>
+            <TableCell>{pin.created}</TableCell>
+            <TableCell collapsing>
+                <Button size='tiny' icon='edit' onClick={() => setEditingId(pin.id)}/>
+                <APIButton
+                    size='tiny'
+                    color='red'
+                    icon='trash'
+                    confirmContent='Delete this pin?'
+                    confirmButton='Delete'
+                    onClick={() => handleDelete(pin.id)}
+                />
+            </TableCell>
+        </TableRow>;
+    };
+
+    // --- Mobile headers and row ---
+    const mobileHeaders = [
+        {key: 'color', text: '', sortBy: i => i.color},
+        {key: 'label', text: 'Label', sortBy: i => i.label.toLowerCase()},
+        {key: 'actions', text: '', sortBy: null},
+    ];
+
+    const mobileRowFunc = (pin) => {
+        return <TableRow key={pin.id}>
+            <TableCell collapsing>
+                <div style={{
+                    width: 16, height: 16, borderRadius: "50%",
+                    background: pin.color || "red", border: "1px solid #ccc",
+                }}/>
+            </TableCell>
+            <TableCell>
+                <a href="#" onClick={(e) => {e.preventDefault(); handleNavigate(pin);}}>
+                    {pin.label}
+                </a>
+                <div style={{fontSize: 11, color: '#888'}}>{pin.lat.toFixed(4)}, {pin.lon.toFixed(4)}</div>
+            </TableCell>
+            <TableCell collapsing>
+                <APIButton
+                    size='tiny'
+                    color='red'
+                    icon='trash'
+                    confirmContent='Delete this pin?'
+                    confirmButton='Delete'
+                    onClick={() => handleDelete(pin.id)}
+                />
+            </TableCell>
+        </TableRow>;
+    };
+
+    const emptyRow = <TableRow>
+        <TableCell colSpan={5}>No pins. Right-click on the map to add one.</TableCell>
+    </TableRow>;
+
+    const lowerFilter = filter.toLowerCase();
+    const filteredPins = lowerFilter
+        ? pins.filter(p =>
+            (p.label || '').toLowerCase().includes(lowerFilter) ||
+            `${p.lat.toFixed(4)}, ${p.lon.toFixed(4)}`.includes(lowerFilter)
+        )
+        : pins;
+
+    return <PageContainer>
+        <Input
+            icon='search'
+            placeholder='Filter by label or coordinates...'
+            value={filter}
+            onChange={(e) => setFilter(e.target.value)}
+            style={{marginBottom: '1em', width: '100%'}}
+        />
+        <Media at='mobile'>
+            <SortableTable
+                tableProps={{unstackable: true}}
+                tableHeaders={mobileHeaders}
+                data={filteredPins}
+                rowFunc={mobileRowFunc}
+                rowKey='label'
+                defaultSortColumn='label'
+                emptyRow={emptyRow}
+            />
+        </Media>
+        <Media greaterThanOrEqual='tablet'>
+            <SortableTable
+                tableProps={{unstackable: true}}
+                tableHeaders={fullHeaders}
+                data={filteredPins}
+                rowFunc={fullRowFunc}
+                rowKey='label'
+                defaultSortColumn='label'
+                emptyRow={emptyRow}
+            />
+        </Media>
+    </PageContainer>;
 }
 
 function MapPage() {
-    const fallback = <Segment>
-        <Header as='h3'>Failed to fetch Map service.</Header>
-        <p>You may need to give permission to access the page: <a href={MAP_VIEWER_URI}>{MAP_VIEWER_URI}</a></p>
-
-        <p>Map startup is delayed on boot, if you have just started your WROLPi try again in a few minutes.</p>
-
-        <p>If the above does not work, try starting the service:</p>
-        <pre>sudo systemctl start renderd</pre>
-        <pre>sudo systemctl start apache2</pre>
-
-        <p>Check the logs</p>
-        <pre>journalctl -u renderd</pre>
-        <pre>sudo tail -f /var/log/apache2/*log</pre>
-    </Segment>;
-
-    return <IframeViewer title='map' src={MAP_VIEWER_URI} fallback={fallback}/>
+    return <MapViewer/>;
 }
 
 export function MapRoute() {
@@ -304,6 +612,7 @@ export function MapRoute() {
 
     const links = [
         {text: 'Map', to: '/map', key: 'map', end: true},
+        {text: 'Pins', to: '/map/pins', key: 'pins'},
         {text: 'Manage', to: '/map/manage', key: 'manage'},
     ];
 
@@ -311,7 +620,8 @@ export function MapRoute() {
         <TabLinks links={links}/>
         <Routes>
             <Route path='/' exact element={<MapPage/>}/>
+            <Route path='pins' exact element={<MapPins/>}/>
             <Route path='manage' exact element={<ManageMap/>}/>
         </Routes>
     </div>
-}
+}

app/src/components/MapViewer.js

@@ -0,0 +1,781 @@
+import React, {useCallback, useContext, useEffect, useRef, useState} from "react";
+import {useSearchParams} from "react-router";
+import maplibregl from "maplibre-gl";
+import "maplibre-gl/dist/maplibre-gl.css";
+import {Protocol} from "pmtiles";
+import layers from "protomaps-themes-base";
+import mlcontour from "@acalcutt/maplibre-contour-pmtiles";
+import {addMapPin, deleteMapPin, getMapFiles, getMapPins} from "../api";
+import {MAP_VIEWER_URI} from "./Vars";
+import {Checkbox, Icon, Segment} from "semantic-ui-react";
+import {Header} from "./Theme";
+import {ThemeContext} from "../contexts/contexts";
+
+// Terrain DEM file prefix for hillshade and contours.
+const TERRAIN_PREFIX = "terrain-";
+
+// Detect full planet files: "planet-20260329.pmtiles" or bare date like "20260329.pmtiles".
+function isPlanetFile(name) {
+    if (name.startsWith("planet-")) return true;
+    return /^\d{8}\.pmtiles$/.test(name);
+}
+
+// Layer groups matching the standalone viewer.
+const LAYER_GROUPS = {
+    "Hillshade": ["hillshade"],
+    "Contours": ["contour-lines", "contour-labels"],
+    "Water": ["water"],
+    "Land Cover": [
+        "landcover", "landuse_park", "landuse_urban_green", "landuse_beach",
+        "landuse_aerodrome", "landuse_hospital", "landuse_industrial",
+        "landuse_pedestrian", "landuse_pier", "landuse_school", "landuse_zoo",
+    ],
+    "Buildings": ["buildings"],
+    "Roads - Highway": ["roads_highway"],
+    "Roads - Major": ["roads_major", "roads_link"],
+    "Roads - Minor": ["roads_minor", "roads_other"],
+    "Roads - Tunnels": ["roads_tunnels"],
+    "Roads - Bridges": ["roads_bridges"],
+    "Rail & Runways": ["roads_rail", "roads_runway", "roads_taxiway", "landuse_runway"],
+    "Boundaries": ["boundaries"],
+    "Labels": ["roads_labels", "places_", "address_label", "pois", "water_label", "water_waterway_label"],
+};
+
+// Register PMTiles protocol once.
+const protocol = new Protocol();
+maplibregl.addProtocol("pmtiles", protocol.tile);
+
+// Planet overview blob — provides global basemap at zoom 0-6.
+const MAP_OVERVIEW_URL = "/blobs/map-overview.pmtiles";
+
+// Layers to skip from non-primary sources (they would paint over the primary source's content).
+const SKIP_SECONDARY = new Set(["background", "earth"]);
+
+// Set up contour DEM source (must be called once before buildStyle).
+let demSource = null;
+let terrainUrl = null;
+function initContourSource(filename) {
+    terrainUrl = `/media/map/${filename}`;
+    if (demSource) return;
+    demSource = new mlcontour.DemSource({
+        url: `pmtiles://${window.location.origin}${terrainUrl}`,
+        encoding: "terrarium",
+        maxzoom: 8,
+        worker: true,
+    });
+    demSource.setupMaplibre(maplibregl);
+}
+
+// sources: [{name, url}] where name is used as the MapLibre source ID and url is the pmtiles:// URL.
+// hasTerrain: whether to include hillshade and contour layers.
+// scaleUnit: "metric" or "imperial" — affects contour intervals and labels.
+function buildStyle(sources, flavor = "light", hasTerrain = false, scaleUnit = "metric") {
+    const styleSources = {};
+    const allLayers = [];
+
+    for (let i = 0; i < sources.length; i++) {
+        const {name, url} = sources[i];
+        styleSources[name] = {type: "vector", url};
+        const fileLayers = layers(name, flavor);
+        if (i === 0) {
+            // Primary source: use layer IDs as-is.
+            allLayers.push(...fileLayers);
+        } else {
+            // Additional sources: prefix layer IDs and skip background/earth.
+            for (const layer of fileLayers) {
+                if (SKIP_SECONDARY.has(layer.id)) continue;
+                allLayers.push({...layer, id: `${name}:${layer.id}`});
+            }
+        }
+    }
+
+    // Add terrain sources and layers if available.
+    if (hasTerrain && demSource) {
+        // Read saved visibility to set initial layer state correctly.
+        const savedVis = getInitialVisibility();
+        const hillshadeVis = savedVis["Hillshade"] ? "visible" : "none";
+        const contourVis = savedVis["Contours"] ? "visible" : "none";
+
+        // Hillshade from raster-dem source.
+        styleSources["terrain-dem"] = {
+            type: "raster-dem",
+            url: `pmtiles://${terrainUrl}`,
+            encoding: "terrarium",
+            tileSize: 512,
+        };
+
+        // Insert hillshade early (after earth/landcover, before roads).
+        const hillshadeSearch = allLayers.findIndex(l => l.id === "buildings");
+        const hillshadeIdx = hillshadeSearch === -1 ? allLayers.length : hillshadeSearch;
+        allLayers.splice(hillshadeIdx, 0, {
+            id: "hillshade",
+            type: "hillshade",
+            source: "terrain-dem",
+            layout: {visibility: hillshadeVis},
+            paint: {
+                "hillshade-shadow-color": "#000000",
+                "hillshade-illumination-anchor": "viewport",
+                "hillshade-exaggeration": 0.3,
+            },
+        });
+
+        // Contour lines from on-the-fly generation.
+        const isImperial = scaleUnit === "imperial";
+        const contourUrl = demSource.contourProtocolUrl({
+            overzoom: 1,
+            contourLayer: "contours",
+            elevationKey: "ele",
+            levelKey: "level",
+            multiplier: isImperial ? 3.28084 : 1, // meters to feet conversion
+            thresholds: isImperial ? {
+                // [minor, major] contour intervals in feet per zoom level.
+                4: [1000, 5000],
+                5: [500, 2500],
+                6: [200, 1000],
+                7: [200, 1000],
+                8: [100, 500],
+            } : {
+                // [minor, major] contour intervals in meters per zoom level.
+                4: [500, 2500],
+                5: [200, 1000],
+                6: [100, 500],
+                7: [100, 500],
+                8: [50, 200],
+            },
+        });
+        const elevUnit = isImperial ? "ft" : "m";
+        styleSources["contours"] = {
+            type: "vector",
+            tiles: [contourUrl],
+            maxzoom: 8,
+        };
+
+        allLayers.push(
+            {
+                id: "contour-lines",
+                type: "line",
+                source: "contours",
+                "source-layer": "contours",
+                filter: ["!=", ["get", "level"], 1],
+                layout: {visibility: contourVis},
+                paint: {"line-color": "rgba(150,100,50,0.3)", "line-width": 0.5},
+            },
+            {
+                id: "contour-lines-major",
+                type: "line",
+                source: "contours",
+                "source-layer": "contours",
+                filter: ["==", ["get", "level"], 1],
+                layout: {visibility: contourVis},
+                paint: {"line-color": "rgba(150,100,50,0.5)", "line-width": 1},
+            },
+            {
+                id: "contour-labels",
+                type: "symbol",
+                source: "contours",
+                "source-layer": "contours",
+                filter: ["==", ["get", "level"], 1],
+                layout: {
+                    visibility: contourVis,
+                    "symbol-placement": "line",
+                    "text-field": ["concat", ["get", "ele"], elevUnit],
+                    "text-size": 10,
+                },
+                paint: {
+                    "text-color": flavor === "dark" ? "rgba(210,180,140,0.9)" : "rgba(150,100,50,0.8)",
+                    "text-halo-color": flavor === "dark" ? "rgba(0,0,0,0.8)" : "rgba(255,255,255,0.8)",
+                    "text-halo-width": 1,
+                },
+            },
+        );
+    }
+
+    return {
+        version: 8,
+        glyphs: "/map-assets/fonts/{fontstack}/{range}.pbf",
+        sprite: `${window.location.origin}/map-assets/sprites/${flavor}`,
+        sources: styleSources,
+        layers: allLayers,
+    };
+}
+
+// Groups that start hidden by default (used when no saved state exists).
+const DEFAULT_HIDDEN = new Set(["Hillshade", "Contours"]);
+const STORAGE_KEY_LAYERS = "wrolpi-map-layers";
+const STORAGE_KEY_SCALE = "wrolpi-map-scale";
+
+function getInitialVisibility() {
+    try {
+        const saved = localStorage.getItem(STORAGE_KEY_LAYERS);
+        if (saved) return JSON.parse(saved);
+    } catch (e) { /* ignore */ }
+    const v = {};
+    for (const g of Object.keys(LAYER_GROUPS)) v[g] = !DEFAULT_HIDDEN.has(g);
+    return v;
+}
+
+function LayerControl({map, scaleUnit, onScaleUnitChange, visibilityRef}) {
+    const [visibility, setVisibility] = useState(() => {
+        const v = getInitialVisibility();
+        if (visibilityRef) visibilityRef.current = v;
+        return v;
+    });
+    const [collapsed, setCollapsed] = useState(true);
+
+    const toggleGroup = useCallback((groupName, checked) => {
+        if (!map) return;
+        const prefixes = LAYER_GROUPS[groupName];
+        const allMapLayers = map.getStyle()?.layers || [];
+        for (const layer of allMapLayers) {
+            // Match both plain IDs and prefixed IDs (e.g., "water" and "file.pmtiles:water").
+            const baseId = layer.id.includes(":") ? layer.id.split(":").slice(1).join(":") : layer.id;
+            if (prefixes.some(p => baseId.includes(p))) {
+                map.setLayoutProperty(layer.id, "visibility", checked ? "visible" : "none");
+            }
+        }
+        setVisibility(prev => {
+            const next = {...prev, [groupName]: checked};
+            if (visibilityRef) visibilityRef.current = next;
+            try { localStorage.setItem(STORAGE_KEY_LAYERS, JSON.stringify(next)); } catch (e) { /* ignore */ }
+            return next;
+        });
+    }, [map, visibilityRef]);
+
+    const panelStyle = {
+        position: "absolute",
+        top: 10,
+        left: 10,
+        zIndex: 1000,
+        background: "white",
+        borderRadius: 8,
+        boxShadow: "0 2px 8px rgba(0,0,0,0.2)",
+        maxHeight: "calc(100vh - 120px)",
+        overflowY: "auto",
+        minWidth: 200,
+        fontSize: 13,
+    };
+
+    return <div style={panelStyle}>
+        <div
+            style={{
+                padding: "8px 12px", cursor: "pointer", fontWeight: 600,
+                display: "flex", justifyContent: "space-between", alignItems: "center",
+                borderBottom: collapsed ? "none" : "1px solid #eee", userSelect: "none",
+            }}
+            onClick={() => setCollapsed(c => !c)}
+        >
+            <span>Layers</span>
+            <Icon name={collapsed ? "chevron right" : "chevron down"} size="small"/>
+        </div>
+        {!collapsed && <div style={{padding: "4px 12px 8px"}}>
+            {Object.keys(LAYER_GROUPS).map(name =>
+                <div key={name} style={{padding: "2px 0"}}>
+                    <Checkbox
+                        label={name}
+                        checked={visibility[name]}
+                        onChange={(e, {checked}) => toggleGroup(name, checked)}
+                    />
+                </div>
+            )}
+            <div style={{borderTop: "1px solid #eee", marginTop: 6, paddingTop: 6}}>
+                <Checkbox
+                    toggle
+                    label={scaleUnit === "imperial" ? "Imperial" : "Metric"}
+                    checked={scaleUnit === "imperial"}
+                    onChange={(e, {checked}) => onScaleUnitChange(checked ? "imperial" : "metric")}
+                />
+            </div>
+        </div>}
+    </div>;
+}
+
+const PIN_COLORS = ["red", "blue", "green", "yellow", "orange", "purple"];
+
+function AddPinDialog({lat, lon, onSubmit, onCancel}) {
+    const [label, setLabel] = useState("");
+    const [color, setColor] = useState("red");
+
+    return <div style={{
+        position: "absolute", top: "50%", left: "50%", transform: "translate(-50%, -50%)",
+        zIndex: 1002, background: "white", borderRadius: 8, boxShadow: "0 4px 16px rgba(0,0,0,0.3)",
+        padding: 16, minWidth: 250,
+    }}>
+        <div style={{fontWeight: 600, marginBottom: 8}}>Add Pin</div>
+        <div style={{fontSize: 12, color: "#666", marginBottom: 8}}>{lat.toFixed(4)}, {lon.toFixed(4)}</div>
+        <input
+            type="text"
+            placeholder="Label"
+            value={label}
+            onChange={e => setLabel(e.target.value)}
+            autoFocus
+            onKeyDown={e => e.key === "Enter" && label.trim() && onSubmit(label.trim(), color)}
+            style={{width: "100%", padding: "6px 8px", marginBottom: 8, border: "1px solid #ccc", borderRadius: 4}}
+        />
+        <div style={{display: "flex", gap: 6, marginBottom: 12}}>
+            {PIN_COLORS.map(c =>
+                <div
+                    key={c}
+                    onClick={() => setColor(c)}
+                    style={{
+                        width: 24, height: 24, borderRadius: "50%", background: c, cursor: "pointer",
+                        border: color === c ? "3px solid #333" : "2px solid #ccc",
+                    }}
+                />
+            )}
+        </div>
+        <div style={{display: "flex", gap: 8, justifyContent: "flex-end"}}>
+            <button onClick={onCancel} style={{padding: "4px 12px", cursor: "pointer"}}>Cancel</button>
+            <button
+                onClick={() => label.trim() && onSubmit(label.trim(), color)}
+                disabled={!label.trim()}
+                style={{padding: "4px 12px", cursor: "pointer", background: "#6435c9", color: "white", border: "none", borderRadius: 4}}
+            >
+                Add
+            </button>
+        </div>
+    </div>;
+}
+
+export default function MapViewer() {
+    const mapContainer = useRef(null);
+    const mapRef = useRef(null);
+    const scaleControlRef = useRef(null);
+    const activeFilesRef = useRef([]);
+    const hasTerrainRef = useRef(false);
+    const layerVisibilityRef = useRef(null);
+    const [mapReady, setMapReady] = useState(false);
+    const [error, setError] = useState(null);
+    const [contextMenu, setContextMenu] = useState(null);
+    const [addingPin, setAddingPin] = useState(null); // {lat, lon} when pin dialog is open
+    const markersRef = useRef([]);
+    const pinsVisibleRef = useRef(true);
+    const [scaleUnit, setScaleUnit] = useState(() => {
+        try { return localStorage.getItem(STORAGE_KEY_SCALE) || "metric"; } catch (e) { return "metric"; }
+    });
+    const [searchParams, setSearchParams] = useSearchParams();
+    const {theme} = useContext(ThemeContext);
+
+    // Parse initial coordinates from URL.
+    const initialLat = parseFloat(searchParams.get("lat")) || 0;
+    const initialLon = parseFloat(searchParams.get("lon")) || 0;
+    const initialZoom = parseFloat(searchParams.get("z")) || 2;
+
+    useEffect(() => {
+        if (!mapContainer.current) return;
+
+        let map;
+        let destroyed = false;
+
+        const init = async () => {
+            const mapSources = [];
+
+            // Fetch user's PMTiles files first to determine what base layer is needed.
+            let hasTerrain = false;
+            let terrainFileName = null;
+            let planetFile = null;
+            try {
+                const data = await getMapFiles();
+                if (data?.files) {
+                    const regionFiles = [];
+                    for (const f of data.files) {
+                        if (f.name.startsWith(TERRAIN_PREFIX) && f.name.endsWith('.pmtiles')) {
+                            hasTerrain = true;
+                            terrainFileName = f.name;
+                        } else if (isPlanetFile(f.name)) {
+                            if (!planetFile || f.size > planetFile.size) {
+                                planetFile = f;
+                            }
+                        } else {
+                            regionFiles.push(f);
+                        }
+                    }
+
+                    if (planetFile) {
+                        // Planet covers everything — sole vector source, no overview needed.
+                        mapSources.push({name: planetFile.name, url: `pmtiles:///media/map/${planetFile.name}`});
+                    } else {
+                        // Use regional extracts.
+                        for (const f of regionFiles) {
+                            mapSources.push({name: f.name, url: `pmtiles:///media/map/${f.name}`});
+                        }
+                        // Add planet overview blob as low-zoom base layer (if available).
+                        try {
+                            const resp = await fetch(MAP_OVERVIEW_URL, {method: "HEAD"});
+                            if (resp.ok) {
+                                mapSources.unshift({name: "map-overview", url: `pmtiles://${MAP_OVERVIEW_URL}`});
+                            }
+                        } catch (e) {
+                            console.debug("Planet overview not available:", e);
+                        }
+                    }
+                }
+            } catch (e) {
+                console.error("Failed to fetch map files:", e);
+            }
+
+            if (destroyed) return;
+            activeFilesRef.current = mapSources;
+            hasTerrainRef.current = hasTerrain;
+
+            // Initialize contour source if terrain data is available.
+            if (hasTerrain) {
+                initContourSource(terrainFileName);
+            }
+
+            try {
+                map = new maplibregl.Map({
+                    container: mapContainer.current,
+                    style: buildStyle(mapSources, theme, hasTerrain, scaleUnit),
+                    center: [initialLon, initialLat],
+                    zoom: initialZoom,
+                    attributionControl: true,
+                    preserveDrawingBuffer: true,
+                });
+
+                map.addControl(new maplibregl.NavigationControl(), "top-right");
+                map.addControl(new maplibregl.FullscreenControl(), "top-right");
+                const sc = new maplibregl.ScaleControl({maxWidth: 200, unit: scaleUnit});
+                map.addControl(sc, "bottom-left");
+                scaleControlRef.current = sc;
+                map.addControl(new maplibregl.GeolocateControl({
+                    positionOptions: {enableHighAccuracy: true},
+                    trackUserLocation: true,
+                    showUserHeading: true,
+                    showAccuracyCircle: true,
+                }), "top-right");
+
+                // Pin toggle button.
+                const pinToggle = document.createElement("button");
+                pinToggle.type = "button";
+                pinToggle.title = "Toggle pins";
+                pinToggle.textContent = "📍";
+                pinToggle.style.cssText = "font-size:18px;width:29px;height:29px;border:none;cursor:pointer;background:white;border-radius:4px;display:flex;align-items:center;justify-content:center;";
+                pinToggle.addEventListener("click", () => {
+                    pinsVisibleRef.current = !pinsVisibleRef.current;
+                    const display = pinsVisibleRef.current ? "" : "none";
+                    pinToggle.style.opacity = pinsVisibleRef.current ? "1" : "0.5";
+                    for (const m of markersRef.current) {
+                        m.getElement().style.display = display;
+                    }
+                });
+                const pinControl = {
+                    onAdd: () => {
+                        const container = document.createElement("div");
+                        container.className = "maplibregl-ctrl maplibregl-ctrl-group";
+                        container.appendChild(pinToggle);
+                        return container;
+                    },
+                    onRemove: () => {},
+                };
+                map.addControl(pinControl, "top-right");
+
+                // Right-click context menu (desktop).
+                map.on("contextmenu", (e) => {
+                    if (destroyed) return;
+                    setContextMenu({
+                        x: e.point.x,
+                        y: e.point.y,
+                        lat: e.lngLat.lat,
+                        lon: e.lngLat.lng,
+                    });
+                });
+                map.on("click", () => setContextMenu(null));
+                map.on("movestart", () => setContextMenu(null));
+
+                // Long-press context menu (mobile).
+                let longPressTimer = null;
+                let touchStartPos = null;
+                const canvas = map.getCanvas();
+                canvas.addEventListener("touchstart", (e) => {
+                    if (e.touches.length !== 1) return;
+                    touchStartPos = {x: e.touches[0].clientX, y: e.touches[0].clientY};
+                    longPressTimer = setTimeout(() => {
+                        const rect = canvas.getBoundingClientRect();
+                        const point = {x: touchStartPos.x - rect.left, y: touchStartPos.y - rect.top};
+                        const lngLat = map.unproject(point);
+                        setContextMenu({x: point.x, y: point.y, lat: lngLat.lat, lon: lngLat.lng});
+                    }, 500);
+                }, {passive: true});
+                canvas.addEventListener("touchmove", (e) => {
+                    if (longPressTimer && touchStartPos) {
+                        const dx = e.touches[0].clientX - touchStartPos.x;
+                        const dy = e.touches[0].clientY - touchStartPos.y;
+                        if (Math.abs(dx) > 10 || Math.abs(dy) > 10) {
+                            clearTimeout(longPressTimer);
+                            longPressTimer = null;
+                        }
+                    }
+                });
+                canvas.addEventListener("touchend", () => {
+                    clearTimeout(longPressTimer);
+                    longPressTimer = null;
+                });
+
+                // Update URL on pan/zoom (only while mounted).
+                map.on("moveend", () => {
+                    if (destroyed) return;
+                    const center = map.getCenter();
+                    const zoom = map.getZoom();
+                    window.history.replaceState(null, '',
+                        `${window.location.pathname}?lat=${center.lat.toFixed(4)}&lon=${center.lng.toFixed(4)}&z=${zoom.toFixed(1)}`
+                    );
+                });
+
+                map.on("load", () => {
+                    if (!destroyed) {
+                        mapRef.current = map;
+                        // Apply saved layer visibility on initial load.
+                        const savedVis = getInitialVisibility();
+                        const allMapLayers = map.getStyle()?.layers || [];
+                        for (const [groupName, visible] of Object.entries(savedVis)) {
+                            const prefixes = LAYER_GROUPS[groupName];
+                            if (!prefixes) continue;
+                            for (const layer of allMapLayers) {
+                                const baseId = layer.id.includes(":") ? layer.id.split(":").slice(1).join(":") : layer.id;
+                                if (prefixes.some(p => baseId.includes(p))) {
+                                    map.setLayoutProperty(layer.id, "visibility", visible ? "visible" : "none");
+                                }
+                            }
+                        }
+                        setMapReady(true);
+                    }
+                });
+            } catch (e) {
+                console.error("MapLibre failed to initialize:", e);
+                if (!destroyed) setError(e.message);
+            }
+        };
+
+        init();
+
+        return () => {
+            destroyed = true;
+            if (map) {
+                map.remove();
+            }
+            mapRef.current = null;
+            setMapReady(false);
+            // Clean up map params from URL so they don't leak to other pages.
+            const url = new URL(window.location);
+            url.searchParams.delete("lat");
+            url.searchParams.delete("lon");
+            url.searchParams.delete("z");
+            window.history.replaceState(null, '', url.pathname + url.search);
+        };
+    }, []); // eslint-disable-line react-hooks/exhaustive-deps
+
+    // Rebuild style when theme changes.
+    useEffect(() => {
+        const map = mapRef.current;
+        if (!map || !mapReady) return;
+
+        const center = map.getCenter();
+        const zoom = map.getZoom();
+        map.setStyle(buildStyle(activeFilesRef.current, theme, hasTerrainRef.current, scaleUnit));
+        map.once("style.load", () => {
+            map.setCenter(center);
+            map.setZoom(zoom);
+            // Re-apply layer visibility state after style rebuild.
+            const vis = layerVisibilityRef.current;
+            if (vis) {
+                const allMapLayers = map.getStyle()?.layers || [];
+                for (const [groupName, visible] of Object.entries(vis)) {
+                    const prefixes = LAYER_GROUPS[groupName];
+                    if (!prefixes) continue;
+                    for (const layer of allMapLayers) {
+                        const baseId = layer.id.includes(":") ? layer.id.split(":").slice(1).join(":") : layer.id;
+                        if (prefixes.some(p => baseId.includes(p))) {
+                            map.setLayoutProperty(layer.id, "visibility", visible ? "visible" : "none");
+                        }
+                    }
+                }
+            }
+        });
+    }, [theme, mapReady, scaleUnit]);
+
+    const handleScaleUnitChange = useCallback((unit) => {
+        const map = mapRef.current;
+        const sc = scaleControlRef.current;
+        if (map && sc) {
+            map.removeControl(sc);
+            const newSc = new maplibregl.ScaleControl({maxWidth: 200, unit});
+            map.addControl(newSc, "bottom-left");
+            scaleControlRef.current = newSc;
+        }
+        try { localStorage.setItem(STORAGE_KEY_SCALE, unit); } catch (e) { /* ignore */ }
+        setScaleUnit(unit);
+    }, []);
+
+    const handleCopyCoords = useCallback(() => {
+        if (!contextMenu) return;
+        // Match decimal precision to current zoom level so pasting into search doesn't over-zoom.
+        const map = mapRef.current;
+        const zoom = map ? map.getZoom() : 10;
+        const decimals = zoom < 4 ? 1 : zoom < 8 ? 2 : zoom < 12 ? 3 : 4;
+        const text = `${contextMenu.lat.toFixed(decimals)}, ${contextMenu.lon.toFixed(decimals)}`;
+        navigator.clipboard.writeText(text).catch(() => {
+            window.prompt("Copy coordinates:", text);
+        });
+        setContextMenu(null);
+    }, [contextMenu]);
+
+    const handleExportPng = useCallback(() => {
+        const map = mapRef.current;
+        if (!map) return;
+        setContextMenu(null);
+        // Force a fresh render, then capture.
+        map.once("render", function() {
+            const c = map.getCenter();
+            const z = map.getZoom();
+            const filename = `wrolpi-map_${c.lat.toFixed(4)}_${c.lng.toFixed(4)}_z${z.toFixed(0)}.png`;
+            map.getCanvas().toBlob(function(blob) {
+                if (!blob) return;
+                const url = URL.createObjectURL(blob);
+                const a = document.createElement("a");
+                a.href = url;
+                a.download = filename;
+                a.click();
+                URL.revokeObjectURL(url);
+            });
+        });
+        map.triggerRepaint();
+    }, []);
+
+    // --- Pins ---
+    const loadPins = useCallback(async () => {
+        const map = mapRef.current;
+        if (!map) return;
+
+        // Remove existing markers.
+        for (const m of markersRef.current) m.remove();
+        markersRef.current = [];
+
+        const data = await getMapPins();
+        if (!data?.pins) return;
+
+        data.pins.forEach((pin, index) => {
+            const el = document.createElement("div");
+            el.style.width = "24px";
+            el.style.height = "24px";
+            el.style.borderRadius = "50% 50% 50% 0";
+            el.style.background = pin.color || "red";
+            el.style.border = "2px solid white";
+            el.style.boxShadow = "0 1px 4px rgba(0,0,0,0.4)";
+            el.style.transform = "rotate(-45deg)";
+            el.style.cursor = "pointer";
+
+            const popup = new maplibregl.Popup({offset: 15}).setHTML(
+                `<div style="font-size:13px">` +
+                `<strong>${pin.label || "Pin"}</strong><br/>` +
+                `<small>${pin.lat.toFixed(4)}, ${pin.lon.toFixed(4)}</small><br/>` +
+                `<button onclick="window.__deleteMapPin(${pin.id})" ` +
+                `style="margin-top:4px;padding:2px 8px;cursor:pointer;color:red;border:1px solid red;background:none;border-radius:3px;">` +
+                `Delete</button></div>`
+            );
+
+            const marker = new maplibregl.Marker({element: el})
+                .setLngLat([pin.lon, pin.lat])
+                .setPopup(popup)
+                .addTo(map);
+
+            if (!pinsVisibleRef.current) {
+                marker.getElement().style.display = "none";
+            }
+            markersRef.current.push(marker);
+        });
+    }, []);
+
+    // Global callback for popup delete buttons.
+    useEffect(() => {
+        window.__deleteMapPin = async (pinId) => {
+            await deleteMapPin(pinId);
+            await loadPins();
+        };
+        return () => { delete window.__deleteMapPin; };
+    }, [loadPins]);
+
+    // Load pins when map is ready.
+    useEffect(() => {
+        if (mapReady) loadPins();
+    }, [mapReady, loadPins]);
+
+    const handleAddPin = useCallback(() => {
+        if (!contextMenu) return;
+        setAddingPin({lat: contextMenu.lat, lon: contextMenu.lon});
+        setContextMenu(null);
+    }, [contextMenu]);
+
+    const handleSubmitPin = useCallback(async (label, color) => {
+        if (!addingPin) return;
+        const {lat, lon} = addingPin;
+        setAddingPin(null);  // Prevent double-submit.
+        await addMapPin(lat, lon, label, color);
+        await loadPins();
+    }, [addingPin, loadPins]);
+
+    if (error) {
+        return <Segment>
+            <Header as="h3">Map failed to load</Header>
+            <p>Your browser may not support WebGL. You can access the standalone map viewer
+                at <a href={MAP_VIEWER_URI}>{MAP_VIEWER_URI}</a></p>
+            <p>Error: {error}</p>
+        </Segment>;
+    }
+
+    return <div ref={mapContainer} style={{position: "relative", width: "100%", height: "85vh"}}>
+        {mapReady && <LayerControl map={mapRef.current} scaleUnit={scaleUnit} onScaleUnitChange={handleScaleUnitChange} visibilityRef={layerVisibilityRef}/>}
+        {contextMenu && <div
+            style={{
+                position: "absolute",
+                left: contextMenu.x,
+                top: contextMenu.y,
+                zIndex: 1001,
+                background: "white",
+                borderRadius: 6,
+                boxShadow: "0 2px 8px rgba(0,0,0,0.3)",
+                padding: "4px 0",
+                fontSize: 13,
+                minWidth: 180,
+            }}
+        >
+            <div style={{padding: "6px 12px", color: "#666", fontSize: 12, borderBottom: "1px solid #eee"}}>
+                {(() => {
+                    const map = mapRef.current;
+                    const zoom = map ? map.getZoom() : 10;
+                    const d = zoom < 4 ? 1 : zoom < 8 ? 2 : zoom < 12 ? 3 : 4;
+                    return `${contextMenu.lat.toFixed(d)}, ${contextMenu.lon.toFixed(d)}`;
+                })()}
+            </div>
+            <div
+                style={{padding: "8px 12px", cursor: "pointer", userSelect: "none"}}
+                onMouseEnter={e => e.target.style.background = "#f0f0f0"}
+                onMouseLeave={e => e.target.style.background = "transparent"}
+                onClick={handleCopyCoords}
+            >
+                Copy GPS coordinates
+            </div>
+            <div
+                style={{padding: "8px 12px", cursor: "pointer", userSelect: "none"}}
+                onMouseEnter={e => e.target.style.background = "#f0f0f0"}
+                onMouseLeave={e => e.target.style.background = "transparent"}
+                onClick={handleExportPng}
+            >
+                Export as PNG
+            </div>
+            <div
+                style={{padding: "8px 12px", cursor: "pointer", userSelect: "none"}}
+                onMouseEnter={e => e.target.style.background = "#f0f0f0"}
+                onMouseLeave={e => e.target.style.background = "transparent"}
+                onClick={handleAddPin}
+            >
+                Add Pin Here
+            </div>
+        </div>}
+        {addingPin && <AddPinDialog
+            lat={addingPin.lat}
+            lon={addingPin.lon}
+            onSubmit={handleSubmitPin}
+            onCancel={() => setAddingPin(null)}
+        />}
+    </div>;
+}

app/src/components/Nav.js

@@ -224,8 +224,6 @@ export function NavBar() {
             } else {
                 processingLink = '/files';
             }
-        } else if (status.flags.map_importing) {
-            processingLink = '/map/manage';
         }
     }
     const processingIcon = processingLink &&

app/src/components/Search.js

@@ -325,6 +325,31 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
             results.apps = {name: 'Apps', results: matchingApps.slice(0, 5)};
         }
 
+        // Detect coordinates (e.g., "40.76, -111.89" or "40.76 -111.89").
+        const coordMatch = searchStr && searchStr.trim().match(
+            /^(-?\d+\.?\d*)[,\s]+(-?\d+\.?\d*)$/
+        );
+        if (coordMatch) {
+            const lat = parseFloat(coordMatch[1]);
+            const lon = parseFloat(coordMatch[2]);
+            if (lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180) {
+                // Derive zoom from decimal precision: 0dp=3, 1dp=7, 2dp=10, 3dp=13, 4dp=16, 5+=18
+                const maxDecimals = Math.max(
+                    (coordMatch[1].split('.')[1] || '').length,
+                    (coordMatch[2].split('.')[1] || '').length,
+                );
+                const zoom = Math.min([3, 7, 10, 13, 16, 18][maxDecimals] || 18, 18);
+                results.map = {
+                    name: 'Map', results: [{
+                        type: 'map',
+                        title: `${lat}, ${lon}`,
+                        description: 'View on map',
+                        location: `/map?lat=${lat}&lon=${lon}&z=${zoom}`,
+                    }]
+                };
+            }
+        }
+
         setSuggestionsResults(results);
         setSuggestionsSums({
             fileGroups: newSuggestions.fileGroups,

app/src/components/Vars.js

@@ -127,6 +127,8 @@ export const Downloaders = {
     File: 'file',
     KiwixCatalog: 'kiwix_catalog',
     KiwixZim: 'kiwix_zim',
+    MapCatalog: 'map_catalog',
+    MapExtract: 'map_extract',
     RSS: 'rss',
     ScrapeHtml: 'scrape_html',
     Video: 'video',

app/src/components/admin/Downloads.js

@@ -243,7 +243,8 @@ function RecurringDownloadRow({download, fetchDownloads, onDelete}) {
         </Modal.Actions>
     </Modal>;
 
-    const editButton = <Button icon='edit' onClick={handleEditOpen}/>;
+    const hideEdit = downloader === Downloaders.MapCatalog || downloader === Downloaders.MapExtract;
+    const editButton = hideEdit ? null : <Button icon='edit' onClick={handleEditOpen}/>;
 
     const restartButton = <APIButton
         color='green'

controller/defaults.py

@@ -75,23 +75,6 @@ DEFAULT_CONFIG = {
             "use_https": True,
             "description": "Kiwix/Zim server",
         },
-        # Map services
-        {
-            "name": "renderd",
-            "systemd_name": "renderd",
-            "port": None,
-            "viewable": False,
-            "use_https": False,
-            "description": "Map tile rendering daemon",
-        },
-        {
-            "name": "apache2",
-            "systemd_name": "apache2",
-            "port": 8084,
-            "viewable": True,
-            "use_https": True,
-            "description": "Map tile web server",
-        },
         # System services
         {
             "name": "postgresql",

controller/lib/admin.py

@@ -683,8 +683,6 @@ async def restart_all_services() -> dict:
         "wrolpi-app",
         "wrolpi-kiwix",
         "wrolpi-help",
-        "renderd",
-        "apache2",
         # Controller restarts itself last (scheduled after response)
     ]
 

controller/templates/index.html

@@ -787,6 +787,10 @@
             <span class="loading"></span>
         </div>
         <p id="onboarding-no-drives" style="display: none;">No unmounted drives detected. Plug in a USB drive and refresh.</p>
+        <div style="margin-top: 1.5rem; padding-top: 1rem; border-top: 1px solid rgba(255,255,255,0.2);">
+            <p style="margin-bottom: 0.5rem; font-size: 0.9em; opacity: 0.8;">No external drive? You can still set up WROLPi using internal storage.</p>
+            <button class="btn btn-secondary" onclick="repairWithoutDrive()">Run repair without external drive</button>
+        </div>
     </div>
 
     <!-- Step 2: Confirm -->
@@ -1872,6 +1876,12 @@
 
                 currentScriptOutput = status.script_name;
 
+                // Hide onboarding while a script is running
+                const onboardingEl = document.getElementById('onboarding-banner');
+                if (onboardingEl) {
+                    onboardingEl.style.display = 'none';
+                }
+
                 // Disable run buttons while script is running
                 document.querySelectorAll('.script-row .btn').forEach(btn => {
                     btn.disabled = true;
@@ -1881,6 +1891,12 @@
                 banner.classList.remove('show');
                 currentScriptOutput = null;
 
+                // Re-check if onboarding is still needed (e.g. after repair finishes)
+                const onboardingEl = document.getElementById('onboarding-banner');
+                if (onboardingEl) {
+                    loadOnboardingCandidates();
+                }
+
                 // Re-enable run buttons
                 document.querySelectorAll('.script-row').forEach(row => {
                     if (!row.classList.contains('unavailable')) {
@@ -2105,6 +2121,9 @@
             const candidates = await resp.json();
             container.innerHTML = '';
 
+            // Re-show the onboarding banner (may have been hidden while a script was running)
+            document.getElementById('onboarding-banner').style.display = '';
+
             if (candidates.length === 0) {
                 noDrives.style.display = 'block';
                 return;
@@ -2229,6 +2248,35 @@
         loadOnboardingCandidates();
     }
 
+    async function repairWithoutDrive() {
+        showOnboardingStep('progress');
+        document.querySelector('#onboarding-progress h2').textContent = 'Running Repair...';
+        document.querySelector('#onboarding-progress p').textContent = 'Starting repair without external drive. Please wait...';
+
+        try {
+            const resp = await fetch('api/scripts/repair/start', {
+                method: 'POST',
+                headers: {'Content-Type': 'application/json', 'Accept': 'application/json'},
+                signal: AbortSignal.timeout(60000)
+            });
+
+            if (!resp.ok) {
+                const data = await resp.json().catch(() => ({}));
+                throw new Error(data.detail || 'Failed to start repair');
+            }
+
+            showOnboardingStep('complete');
+            document.getElementById('onboarding-complete-msg').textContent =
+                'Repair started without external drive. The main UI will be available shortly.';
+            setTimeout(() => {
+                window.location.href = '?fallback=true';
+            }, 10000);
+        } catch (e) {
+            document.getElementById('onboarding-error-msg').textContent = e.message;
+            showOnboardingStep('error');
+        }
+    }
+
     function resetOnboarding() {
         cancelOnboarding();
     }

controller/test/test_defaults.py

@@ -57,7 +57,7 @@ class TestManagedServicesConfig:
     def test_managed_services_count(self):
         """Should have expected number of managed services."""
         services = DEFAULT_CONFIG["managed_services"]
-        assert len(services) == 14
+        assert len(services) == 12
 
     @pytest.mark.parametrize("name,expected", [
         ("wrolpi-api",
@@ -81,7 +81,6 @@ class TestManagedServicesConfig:
     @pytest.mark.parametrize("name,use_https", [
         ("wrolpi-help", True),
         ("wrolpi-kiwix", True),
-        ("apache2", True),
         ("wrolpi-api", False),
         ("wrolpi-app", False),
         ("postgresql", False),

controller/test/test_systemd.py

@@ -134,7 +134,6 @@ class TestGetServiceStatus:
     @pytest.mark.parametrize("service_name,expected_https", [
         ("wrolpi-help", True),
         ("wrolpi-kiwix", True),
-        ("apache2", True),
         ("wrolpi-api", False),
     ])
     def test_returns_use_https_field(self, service_name, expected_https):

debian-live-config/build.sh

@@ -37,40 +37,7 @@ if [ $EUID != 0 ]; then
   exit $?
 fi
 
-if [ ! -f "${SCRIPT_DIR}/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump" ]; then
-  echo "config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump does not exist!"
-  exit 1
-fi
-
-# Validate dump file format - must be PostgreSQL custom format (PGDMP magic)
-DUMP_FILE="${SCRIPT_DIR}/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump"
-DUMP_HEADER=$(head -c 5 "${DUMP_FILE}" 2>/dev/null | od -An -tx1 | tr -d ' \n')
-
-if [[ "${DUMP_HEADER:0:6}" == "504744" ]]; then
-  echo "map-db-gis.dump: Valid PostgreSQL custom format"
-elif [[ "${DUMP_HEADER:0:6}" == "1f8b08" ]]; then
-  # Check if gzipped content is custom format
-  INNER_HEADER=$(gunzip -c "${DUMP_FILE}" 2>/dev/null | head -c 5 | od -An -tx1 | tr -d ' \n')
-  if [[ "${INNER_HEADER:0:6}" == "504744" ]]; then
-    echo "map-db-gis.dump: Valid gzipped PostgreSQL custom format"
-  else
-    echo "ERROR: map-db-gis.dump is gzipped but does NOT contain PostgreSQL custom format!"
-    echo "Inner header: ${INNER_HEADER} (expected to start with 504744 for 'PGDMP')"
-    echo "The dump was likely created incorrectly. Use:"
-    echo "  docker run --rm -v /path/to/file.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper > map-db-gis.dump"
-    exit 1
-  fi
-else
-  echo "ERROR: map-db-gis.dump has invalid format!"
-  echo "Header: ${DUMP_HEADER} (expected to start with 504744 for 'PGDMP' or 1f8b08 for gzip)"
-  echo "Create the dump using:"
-  echo "  docker run --rm -v /path/to/file.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper > map-db-gis.dump"
-  exit 1
-fi
-
 # Clear out old builds.
-# Remove immutable attribute from protected files before cleanup
-chattr -i "${BUILD_DIR}"/chroot/opt/wrolpi-blobs/* 2>/dev/null || :
 [ -d "${BUILD_DIR}" ] && rm -rf "${BUILD_DIR}"
 mkdir "${BUILD_DIR}"
 cd "${BUILD_DIR}" || (echo "Work directory must exist" && exit 1)
@@ -104,8 +71,38 @@ lb config \
 rsync -a "${SCRIPT_DIR}/config" "${BUILD_DIR}/"
 
 # Write branch config for hook script.
+mkdir -p "${BUILD_DIR}/config/includes.chroot/opt/wrolpi-blobs"
 echo "${BRANCH}" > "${BUILD_DIR}/config/includes.chroot/opt/wrolpi-branch"
 
+# Cache directory for downloaded blobs (persists across builds).
+BLOB_CACHE="${SCRIPT_DIR}/files"
+mkdir -p "${BLOB_CACHE}"
+
+# Download map fonts (skip if cached).
+if [ ! -f "${BLOB_CACHE}/map-fonts.tar.gz" ]; then
+  echo "Downloading map fonts..."
+  mkdir -p /tmp/map-fonts-dl
+  curl -fsSL https://github.com/protomaps/basemaps-assets/archive/refs/heads/main.tar.gz \
+    | tar -xz --strip-components=1 -C /tmp/map-fonts-dl basemaps-assets-main/fonts
+  tar -czf "${BLOB_CACHE}/map-fonts.tar.gz" -C /tmp/map-fonts-dl fonts
+  rm -rf /tmp/map-fonts-dl
+fi
+[[ -f "${BLOB_CACHE}/map-fonts.tar.gz" && -s "${BLOB_CACHE}/map-fonts.tar.gz" ]] || \
+  (echo "ERROR: Failed to download map fonts!" && exit 1)
+
+# Download map overview (skip if cached).
+if [ ! -f "${BLOB_CACHE}/map-overview.pmtiles" ]; then
+  echo "Downloading map overview..."
+  curl -fsSL https://wrolpi.nyc3.cdn.digitaloceanspaces.com/maps/map-overview.pmtiles \
+    -o "${BLOB_CACHE}/map-overview.pmtiles"
+fi
+[[ -f "${BLOB_CACHE}/map-overview.pmtiles" && -s "${BLOB_CACHE}/map-overview.pmtiles" ]] || \
+  (echo "ERROR: Failed to download map overview!" && exit 1)
+
+# Copy cached blobs into the live image.
+cp "${BLOB_CACHE}/map-fonts.tar.gz" "${BUILD_DIR}/config/includes.chroot/opt/wrolpi-blobs/"
+cp "${BLOB_CACHE}/map-overview.pmtiles" "${BUILD_DIR}/config/includes.chroot/opt/wrolpi-blobs/"
+
 time nice -n 18 lb build 2>&1 | tee "${SCRIPT_DIR}/build.log"
 
 grep "9999-wrolpi.hook.chroot completed" "${SCRIPT_DIR}/build.log" >/dev/null 2>&1 || (echo "build hook failed!" && exit 1)

debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot

@@ -30,7 +30,7 @@ node --version
 npm install
 
 # Install Node console commands.
-npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0
+npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6
 
 # Install Deno runtime.
 DENO_VERSION="v2.2.4"
@@ -65,6 +65,7 @@ apt-get install -y caddy
 # Configure Caddy.
 mkdir -p /etc/caddy
 cp /opt/wrolpi/etc/raspberrypios/Caddyfile /etc/caddy/Caddyfile
+mkdir -p /var/www
 cp /opt/wrolpi/etc/raspberrypios/50x.html /var/www/50x.html
 cp /opt/wrolpi/etc/raspberrypios/landing.html /var/www/landing.html
 
@@ -74,27 +75,20 @@ chmod 0440 /etc/sudoers.d/90-wrolpi
 # Verify this new file is valid.
 visudo -c -f /etc/sudoers.d/90-wrolpi
 
-# Create postgres cluster for map.
-pg_createcluster 15 map --port=5433 -e utf8
-
-git clone https://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto
-git config --global --add safe.directory /opt/openstreetmap-carto
-sed -i '/dbname: "gis"/a \    port: 5433' /opt/openstreetmap-carto/project.mml
-(cd /opt/openstreetmap-carto && carto project.mml >/opt/openstreetmap-carto/mapnik.xml)
-
-# Change default postgresql port to 5432.
-sed -i 's/port = 5433/port = 5432/' /etc/postgresql/15/main/postgresql.conf
-
 mkdir /media/wrolpi
 
+# Verify blob files were copied in by includes.chroot (downloaded by build.sh).
+[[ -f /opt/wrolpi-blobs/map-fonts.tar.gz && -s /opt/wrolpi-blobs/map-fonts.tar.gz ]] || \
+  (echo "ERROR: map-fonts.tar.gz missing from /opt/wrolpi-blobs!" && exit 1)
+[[ -f /opt/wrolpi-blobs/map-overview.pmtiles && -s /opt/wrolpi-blobs/map-overview.pmtiles ]] || \
+  (echo "ERROR: map-overview.pmtiles missing from /opt/wrolpi-blobs!" && exit 1)
+
 # Install WROLPi Help.
 /opt/wrolpi/scripts/install_help_service.sh
 
 cp /opt/wrolpi/etc/raspberrypios/wrolpi-*.service /etc/systemd/system/
 cp /opt/wrolpi/etc/raspberrypios/wrolpi-*.timer /etc/systemd/system/
 cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/
-cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf
-
 # Copy MOTD scripts, delete original.
 cp /opt/wrolpi/etc/raspberrypios/motd/20-wrolpi.motd /etc/update-motd.d/20-wrolpi
 cp /opt/wrolpi/etc/raspberrypios/motd/30-wrolpi-first.motd /etc/update-motd.d/30-wrolpi
@@ -116,10 +110,11 @@ systemctl enable wrolpi-cert-renew.timer
 systemctl enable wrolpi-cert-renew.timer
 
 # Protect blob files from accidental modification or deletion.
-# These files are critical for initializing WROLPi.
-chown -R root:root /opt/wrolpi-blobs
-chmod 444 /opt/wrolpi-blobs/*
-chattr +i /opt/wrolpi-blobs/*
+if [ -d /opt/wrolpi-blobs ] && ls /opt/wrolpi-blobs/* >/dev/null 2>&1; then
+  chown -R root:root /opt/wrolpi-blobs
+  chmod 444 /opt/wrolpi-blobs/*
+  chattr +i /opt/wrolpi-blobs/*
+fi
 
 set +x
 

debian-live-config/config/package-lists/wrolpi.list.chroot

@@ -1,6 +1,3 @@
-apache2
-apache2-dev
-apache2-doc
 apt-transport-https
 aria2
 autoconf
@@ -24,7 +21,6 @@ fonts-noto-hinted
 fonts-noto-unhinted
 gcc
 gcc-doc
-gdal-bin
 git
 git-doc
 gnupg-agent
@@ -35,58 +31,36 @@ iotop
 iperf3
 kiwix
 kiwix-tools
-libagg-dev
-libapache2-mod-tile
 libboost-all-dev
 libbz2-dev
 libcairo2-dev
-libcairomm-1.0-dev
 libcurl4-gnutls-dev
 libffi-dev
 libfreetype6-dev
-libgdal-dev
 libgeos++-dev
 libgeos-dev
 libicu-dev
-libiniparser-dev
-liblua5.1-0-dev
-liblua5.2-dev
-libmapnik-dev
-libmemcached-dev
 libpq-dev
 libproj-dev
-librados-dev
 libreoffice
-libsensors-dev
 libssl-dev
 libtiff5-dev
 libtool
 libxml2-dev
-lua5.1
-mapnik-utils
 netcat-openbsd
 network-manager
 nicstat
 nodejs
 npm
-osm2pgsql
-osmium-tool
 pkg-config
-postgis
-postgis-doc
 postgresql-15
-postgresql-15-postgis-3
-postgresql-15-postgis-3-scripts
 postgresql-doc
-protobuf-c-compiler
 pv
 python3-dev
 python3-doc
 python3-full
-python3-mapnik
 python3-pip
 python3-venv
-renderd
 rsync
 samba
 smartmontools

debian-live-config/files/.gitignore

@@ -0,0 +1,2 @@
+*.pmtiles
+*.tar.gz

docker-compose.yml

@@ -105,26 +105,11 @@ services:
       # Force React to use Caddy port to connect to it's websocket.
       - WDS_SOCKET_PORT=0
 
-  map: # https://github.com/Overv/openstreetmap-tile-server
-    image: overv/openstreetmap-tile-server:v1.6.0
-    volumes:
-      - 'openstreetmap-data:/var/lib/postgresql/12/main'
-      - 'openstreetmap-rendered-tiles:/var/lib/mod_tile'
-      - './modules/map/leaflet.html:/var/www/html/index.html'
-      - './modules/map/leaflet.js:/var/www/html/leaflet.js'
-      - './modules/map/leaflet.css:/var/www/html/leaflet.css'
-    command: 'run'
-    shm_size: 1g  # Increase shared size to render large map tiles
-    healthcheck:
-      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:80' ]
-      interval: 1m
-
   web: # Caddy reverse proxy - handles HTTPS for all services.
     depends_on:
       - app
       - zim
       - help
-      - map
     build:
       context: .
       dockerfile: docker/web/Dockerfile
@@ -139,6 +124,8 @@ services:
       - './icon.ico:/etc/caddy/favicon.ico'
       - './icon.png:/etc/caddy/icon.png'
       - './test:/opt/media'
+      - './modules/map/static:/opt/wrolpi/modules/map/static:ro'
+      - './blobs:/opt/wrolpi-blobs:ro'
     environment:
       - MEDIA_DIRECTORY=/opt/media
       # Set EXTRA_SANS to include the host's LAN IP in the TLS certificate.
@@ -181,9 +168,3 @@ services:
     healthcheck:
       test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8086' ]
       interval: 1m
-
-volumes:
-  openstreetmap-data:
-    external: true
-  openstreetmap-rendered-tiles:
-    external: true

docker/help/wrolpi-help

@@ -1 +1 @@
-Subproject commit d2415ae19fe02edfb8b54d22830a2043bd4c4e45
+Subproject commit 5e9a4306585ea8e8099f7efd8411def97230a49f

docker/osm-map-dumper/Dockerfile

@@ -1,4 +0,0 @@
-FROM overv/openstreetmap-tile-server:2.2.0
-
-COPY run.sh /
-ENTRYPOINT ["/run.sh"]

docker/osm-map-dumper/README.md

@@ -1,14 +0,0 @@
-# OSM Map Dumper
-
-A simple container which initializes planetary data, imports an Open Street Map PBF file, then dumps the Postgresql
-database contents.  Contents are dumped to stdout using `pg_dump`, suitable for `pg_restore`.
-
-Warning: Typically adds 2GB of planetary data.
-
-## Usage
-
-Import `map.osm.pbf`, dump to `output.dump`
-
-`# docker run --rm -v /absolute/path/to/your/map.osm.pbf:/data/region.osm.pbf $(docker build -q .) > output.dump`
-
-See: https://hub.docker.com/r/lrnselfreliance/osm-map-dumper

docker/osm-map-dumper/run.sh

@@ -1,98 +0,0 @@
-#!/bin/bash
-# Taken from https://github.com/Overv/openstreetmap-tile-server and modified.
-
-set -euo pipefail
-
-function createPostgresConfig() {
-  cp /etc/postgresql/14/main/postgresql.custom.conf.tmpl /etc/postgresql/14/main/conf.d/postgresql.custom.conf
-  # shellcheck disable=SC2024
-  sudo -u postgres echo "autovacuum = $AUTOVACUUM" >>/etc/postgresql/14/main/conf.d/postgresql.custom.conf
-}
-
-function setPostgresPassword() {
-  sudo -u postgres psql -c "ALTER USER renderer PASSWORD '${PGPASSWORD:-renderer}'" >/dev/null
-}
-
-if [ -d /data/region.osm.pbf ]; then
-  echo >&2 "ERROR: Found directory at /data/region.osm.pbf.  Did you mount the PBF file correctly?"
-  exit 2
-fi
-
-if [ ! -f /data/region.osm.pbf ]; then
-  echo >&2 "WARNING: No import file at /data/region.osm.pbf."
-  echo >&2 "   Mount it: -v /absolute/path/to/your.osm.pbf:/data/region.osm.pbf"
-  exit 3
-fi
-
-# if there is no custom style mounted, then use osm-carto
-if [ ! "$(ls -A /data/style/)" ]; then
-  mv /home/renderer/src/openstreetmap-carto-backup/* /data/style/
-fi
-
-# carto build
-if [ ! -f /data/style/mapnik.xml ]; then
-  cd /data/style/
-  carto ${NAME_MML:-project.mml} >mapnik.xml
-fi
-
-# Ensure that database directory is in right state
-mkdir -p /data/database/postgres/
-chown renderer: /data/database/
-chown -R postgres: /var/lib/postgresql /data/database/postgres/
-if [ ! -f /data/database/postgres/PG_VERSION ]; then
-  sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8" >/dev/null
-fi
-
-# Initialize PostgreSQL
-echo >&2 "Initializing map tables"
-createPostgresConfig
-service postgresql start >/dev/null
-sudo -u postgres createuser renderer >/dev/null
-sudo -u postgres createdb -E UTF8 -O renderer gis >/dev/null
-sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;" >/dev/null
-sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore;" >/dev/null
-sudo -u postgres psql -d gis -c "ALTER TABLE geometry_columns OWNER TO renderer;" >/dev/null
-sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO renderer;" >/dev/null
-setPostgresPassword
-
-# copy polygon file if available
-if [ -f /data/region.poly ]; then
-  cp /data/region.poly /data/database/region.poly
-  chown renderer: /data/database/region.poly
-fi
-
-# Import data
-echo >&2 "Importing your map data"
-sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore \
-  --tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua} \
-  --number-processes ${THREADS:-4} \
-  -S /data/style/${NAME_STYLE:-openstreetmap-carto.style} \
-  /data/region.osm.pbf \
-  ${OSM2PGSQL_EXTRA_ARGS:-} \
-  >/dev/null
-
-# Create indexes
-if [ -f /data/style/${NAME_SQL:-indexes.sql} ]; then
-  sudo -u postgres psql -d gis -f /data/style/${NAME_SQL:-indexes.sql} >/dev/null
-fi
-
-# Import external data
-echo >&2 "Importing planetary data"
-chown -R renderer: /home/renderer/src/ /data/style/
-if [ -f /data/style/scripts/get-external-data.py ] && [ -f /data/style/external-data.yml ]; then
-  sudo -E -u renderer python3 /data/style/scripts/get-external-data.py \
-    -c /data/style/external-data.yml -D /data/style/data >/dev/null
-fi
-
-echo >&2 "Dumping the DB"
-sudo -u postgres pg_dump \
-  --create \
-  --clean \
-  --format custom \
-  --compress 9 \
-  --no-owner \
-  --no-comments \
-  --encoding UTF-8 \
-  --dbname gis
-
-service postgresql stop

docker/web/Caddyfile

@@ -60,6 +60,18 @@
 	}
 
 
+	# WROLPi blobs (planet overview map, fonts, etc.)
+	handle_path /blobs/* {
+		root * /opt/wrolpi-blobs
+		file_server
+	}
+
+	# Map static assets (fonts, sprites) for embedded MapLibre viewer.
+	handle_path /map-assets/* {
+		root * /opt/wrolpi/modules/map/static
+		file_server
+	}
+
 	# Custom MIME types not in Caddy's defaults.
 	@yaml path *.yaml *.yml
 	header @yaml Content-Type "text/yaml"
@@ -116,10 +128,33 @@
 	reverse_proxy help:80
 }
 
-# Map HTTPS proxy (replaces map_https container).
+# Map viewer - serves MapLibre static assets and PMTiles data via Caddy.
 :8084 {
 	tls /etc/ssl/caddy/cert.crt /etc/ssl/caddy/cert.key
 	header ?Access-Control-Allow-Origin "*"
-	header ?Access-Control-Allow-Methods "GET, OPTIONS"
-	reverse_proxy map:80
+	header ?Access-Control-Allow-Methods "GET, HEAD, OPTIONS"
+	header ?Access-Control-Expose-Headers "Content-Range, Content-Length"
+
+	# PMTiles file list from WROLPi API.
+	handle /api/map/* {
+		reverse_proxy api:8081
+	}
+
+	# WROLPi blobs (planet overview map).
+	handle_path /blobs/* {
+		root * /opt/wrolpi-blobs
+		file_server
+	}
+
+	# PMTiles data files from media directory (Range Requests supported by default).
+	handle_path /data/* {
+		root * /opt/media/map
+		file_server
+	}
+
+	# MapLibre static assets (HTML, JS, CSS, fonts, sprites).
+	handle {
+		root * /opt/wrolpi/modules/map/static
+		file_server
+	}
 }

etc/debian12/000-default.conf

@@ -1,43 +0,0 @@
-<VirtualHost *:9084>
-    # The ServerName directive sets the request scheme, hostname and port that
-    # the server uses to identify itself. This is used when creating
-    # redirection URLs. In the context of virtual hosts, the ServerName
-    # specifies what hostname must appear in the request's Host: header to
-    # match this virtual host. For the default virtual host (this file) this
-    # value is not decisive as it is used as a last resort host regardless.
-    # However, you must set it for any further virtual host explicitly.
-    #ServerName www.example.com
-
-    ServerAdmin webmaster@localhost
-
-    LoadTileConfigFile /etc/renderd.conf
-    ModTileRenderdSocketName /var/run/renderd/renderd.sock
-    # Timeout before giving up for a tile to be rendered
-    ModTileRequestTimeout 0
-    # Timeout before giving up for a tile to be rendered that is otherwise missing
-    ModTileMissingRequestTimeout 30
-
-    DocumentRoot /var/www/html
-
-    Header set Access-Control-Allow-Origin "*"
-    Header set Access-Control-Allow-Methods "POST, GET, OPTIONS, DELETE, PUT"
-    Header set Access-Control-Allow-Headers "Content-Type"
-
-    # Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
-    # error, crit, alert, emerg.
-    # It is also possible to configure the loglevel for particular
-    # modules, e.g.
-    #LogLevel info ssl:warn
-
-    ErrorLog ${APACHE_LOG_DIR}/error.log
-    CustomLog ${APACHE_LOG_DIR}/access.log combined
-
-    # For most configuration files from conf-available/, which are
-    # enabled or disabled at a global level, it is possible to
-    # include a line for only one particular virtual host. For example the
-    # following line enables the CGI configuration for this host only
-    # after it has been globally disabled with "a2disconf".
-    #Include conf-available/serve-cgi-bin.conf
-</VirtualHost>
-
-# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

etc/raspberrypios/000-default.conf

@@ -1,43 +0,0 @@
-<VirtualHost *:9084>
-    # The ServerName directive sets the request scheme, hostname and port that
-    # the server uses to identify itself. This is used when creating
-    # redirection URLs. In the context of virtual hosts, the ServerName
-    # specifies what hostname must appear in the request's Host: header to
-    # match this virtual host. For the default virtual host (this file) this
-    # value is not decisive as it is used as a last resort host regardless.
-    # However, you must set it for any further virtual host explicitly.
-    #ServerName www.example.com
-
-    ServerAdmin webmaster@localhost
-
-    LoadTileConfigFile /etc/renderd.conf
-    ModTileRenderdSocketName /run/renderd/renderd.sock
-    # Timeout before giving up for a tile to be rendered
-    ModTileRequestTimeout 0
-    # Timeout before giving up for a tile to be rendered that is otherwise missing
-    ModTileMissingRequestTimeout 30
-
-    DocumentRoot /var/www/html
-
-    Header set Access-Control-Allow-Origin "*"
-    Header set Access-Control-Allow-Methods "POST, GET, OPTIONS, DELETE, PUT"
-    Header set Access-Control-Allow-Headers "Content-Type"
-
-    # Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
-    # error, crit, alert, emerg.
-    # It is also possible to configure the loglevel for particular
-    # modules, e.g.
-    #LogLevel info ssl:warn
-
-    ErrorLog ${APACHE_LOG_DIR}/error.log
-    CustomLog ${APACHE_LOG_DIR}/access.log combined
-
-    # For most configuration files from conf-available/, which are
-    # enabled or disabled at a global level, it is possible to
-    # include a line for only one particular virtual host. For example the
-    # following line enables the CGI configuration for this host only
-    # after it has been globally disabled with "a2disconf".
-    #Include conf-available/serve-cgi-bin.conf
-</VirtualHost>
-
-# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

etc/raspberrypios/50x.html

@@ -62,8 +62,6 @@
     <li><a href="file:///usr/share/doc/python3-doc/html/index.html">Python 3</a></li>
     <li><a href="file:///usr/share/doc/postgresql-doc-15/html/index.html">Postgresql</a></li>
     <li><a href="file:///usr/share/doc/nodejs/api/index.html">NodeJS</a></li>
-    <li><a href="file:///usr/share/doc/apache2-doc/manual/en/index.html">Apache2</a></li>
-    <li><a href="file:///usr/share/doc/postgis-doc/postgis.html">Postgis</a></li>
     <li><a href="file:///usr/share/doc/ffmpeg/manual/index.html">FFMPEG</a></li>
     <li><a href="file:///usr/share/doc/gcc-10-doc/gcc.html">GCC</a></li>
     <li><a href="file:///usr/share/doc/git-doc/git.html">Git</a></li>

etc/raspberrypios/90-wrolpi

@@ -1,7 +1,4 @@
 %wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set
-%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart renderd.service
-%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop renderd.service
-%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start renderd.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-kiwix.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-kiwix.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-kiwix.service
@@ -11,7 +8,6 @@
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-app.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-app.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-app.service
-%wrolpi ALL= NOPASSWD:/opt/wrolpi/scripts/import_map.sh
 %wrolpi ALL= NOPASSWD:/opt/wrolpi/upgrade.sh *
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-upgrade.service
 %wrolpi ALL= NOPASSWD:/usr/sbin/shutdown

etc/raspberrypios/Caddyfile

@@ -60,6 +60,18 @@
 	}
 
 
+	# WROLPi blobs (planet overview map, fonts, etc.)
+	handle_path /blobs/* {
+		root * /opt/wrolpi-blobs
+		file_server
+	}
+
+	# Map static assets (fonts, sprites) for embedded MapLibre viewer.
+	handle_path /map-assets/* {
+		root * /opt/wrolpi/modules/map/static
+		file_server
+	}
+
 	# Custom MIME types not in Caddy's defaults.
 	@yaml path *.yaml *.yml
 	header @yaml Content-Type "text/yaml"
@@ -116,10 +128,33 @@
 	reverse_proxy 0.0.0.0:9086
 }
 
-# Map HTTPS proxy (Apache2 HTTP-only on 9084).
+# Map viewer - serves MapLibre static assets and PMTiles data via Caddy.
 :8084 {
 	tls /etc/ssl/wrolpi/cert.crt /etc/ssl/wrolpi/cert.key
 	header ?Access-Control-Allow-Origin "*"
-	header ?Access-Control-Allow-Methods "GET, OPTIONS"
-	reverse_proxy 0.0.0.0:9084
+	header ?Access-Control-Allow-Methods "GET, HEAD, OPTIONS"
+	header ?Access-Control-Expose-Headers "Content-Range, Content-Length"
+
+	# PMTiles file list from WROLPi API.
+	handle /api/map/* {
+		reverse_proxy 0.0.0.0:8081
+	}
+
+	# WROLPi blobs (planet overview map).
+	handle_path /blobs/* {
+		root * /opt/wrolpi-blobs
+		file_server
+	}
+
+	# PMTiles data files from media directory (Range Requests supported by default).
+	handle_path /data/* {
+		root * /media/wrolpi/map
+		file_server
+	}
+
+	# MapLibre static assets (HTML, JS, CSS, fonts, sprites).
+	handle {
+		root * /opt/wrolpi/modules/map/static
+		file_server
+	}
 }

etc/raspberrypios/index.html

@@ -1,37 +0,0 @@
-<!DOCTYPE HTML>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <script src="/leaflet.js"></script>
-    <link rel="stylesheet" href="/leaflet.css" />
-    <style>
-      html, body {
-        height: 100%;
-        padding: 0;
-        margin: 0;
-      }
-      #map {
-        /* configure the size of the map */
-        width: 100%;
-        height: 100%;
-      }
-    </style>
-  </head>
-  <body>
-    <div id="map"></div>
-    <script>
-      // initialize Leaflet
-      var map = L.map('map').setView({lon: 0, lat: 0}, 2);
-
-      // add the OpenStreetMap tiles
-      L.tileLayer(`https://${window.location.hostname}:8084/hot/{z}/{x}/{y}.png`, {
-        maxZoom: 19,
-        attribution: '&copy; <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
-      }).addTo(map);
-
-      // show the scale bar on the lower left corner
-      L.control.scale({imperial: true, metric: true}).addTo(map);
-    </script>
-  </body>
-</html>

etc/raspberrypios/mod_tile.conf

@@ -1 +0,0 @@
-LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so

etc/raspberrypios/ports.conf

@@ -1 +0,0 @@
-Listen 9084

etc/raspberrypios/postgresql@15-map.service.d/override.conf

@@ -1,3 +0,0 @@
-[Service]
-# WROLPi's map uses a lot of processing power.  Change niceness of map database to prevent blocking of other services.
-Nice=19

etc/raspberrypios/renderd.conf

@@ -1,18 +0,0 @@
-[renderd]
-stats_file=/run/renderd/renderd.stats
-socketname=/run/renderd/renderd.sock
-num_threads=4
-tile_dir=/var/cache/renderd/tiles
-
-[mapnik]
-plugins_dir=/usr/lib/mapnik/3.1/input
-font_dir=/usr/share/fonts/truetype
-font_dir_recurse=true
-
-[ajt]
-URI=/hot/
-TILEDIR=/var/cache/renderd/tiles
-XML=/opt/openstreetmap-carto/mapnik.xml
-HOST=localhost
-TILESIZE=256
-MAXZOOM=20

etc/raspberrypios/renderd.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Daemon that renders map tiles using mapnik
-Documentation=man:renderd
-After=network.target auditd.service
-
-[Service]
-# Delay startup so WROLPi App and API have time to start.
-ExecStartPre=/bin/sleep 30
-# Map is uses a lot of resources, but not a high priority.
-Nice=19
-ExecStart=/usr/bin/renderd -f
-User=_renderd
-
-[Install]
-WantedBy=multi-user.target

etc/raspberrypios/wrolpi.target

@@ -1,3 +1,3 @@
 [Unit]
 Description=WROLPi services group
-Wants=wrolpi-controller.service wrolpi-api.service wrolpi-app.service caddy.service postgresql.service wrolpi-kiwix.service renderd.service apache2.service postgresql@15-map.service
+Wants=wrolpi-controller.service wrolpi-api.service wrolpi-app.service caddy.service postgresql.service wrolpi-kiwix.service

help.sh

@@ -290,40 +290,27 @@ fi
 echo
 # Map
 
-if netstat -ant | grep LISTEN | grep -E '[:\.]5433\b' >/dev/null; then
-  echo "OK: Port 5433 is occupied"
+if curl -k -s https://0.0.0.0:8084 --max-time 5 2>/dev/null | grep -i "WROLPi Map" >/dev/null; then
+  echo "OK: Map viewer responded"
 else
-  echo "FAILED: Port 5433 is not occupied"
+  echo "FAILED: Map viewer did not respond on port 8084"
 fi
 
-if sudo -iu postgres psql --port=5433 -l 2>/dev/null | grep gis >/dev/null; then
-  echo "OK: Found map database"
+check_directory /opt/wrolpi/modules/map/static "Map static assets exist" "Map static assets missing at /opt/wrolpi/modules/map/static"
 
-  if sudo -iu postgres psql --port=5433 gis -c '\d' | grep water_polygons >/dev/null; then
-    echo "OK: Map database is initialized"
-  else
-    echo "FAILED: Map database is not initialized"
-  fi
+if ls /media/wrolpi/map/*.pmtiles >/dev/null 2>&1; then
+  echo "OK: PMTiles map files found"
+  ls -lh /media/wrolpi/map/*.pmtiles 2>/dev/null | awk '{print "  " $NF " (" $5 ")"}'
 else
-  echo "FAILED: Unable to find map database"
+  echo "Note: No PMTiles map files found in /media/wrolpi/map/"
 fi
 
-if systemctl list-unit-files "*renderd*" >/dev/null; then
-  echo "OK: renderd systemd exists"
+if [ -f /opt/wrolpi/.protomaps_installed ]; then
+  echo "OK: Protomaps migration completed"
 else
-  echo "FAILED: renderd systemd does not exist"
+  echo "Note: Protomaps migration has not been run (old map stack may still be active)"
 fi
 
-if curl -k -s https://0.0.0.0:8084 --max-time 5 2>/dev/null | grep -i openstreetmap >/dev/null; then
-  echo "OK: Map app responded"
-else
-  echo "FAILED: Map app did not respond"
-fi
-
-check_file /opt/wrolpi-blobs/map-db-gis.dump "Map initialization blob exists" "Map initialization blob does not exist"
-
-check_file /var/www/html/leaflet.js "Leaflet.js exists" "Leaflet.js does not exist"
-
 echo
 # Kiwix
 

modules/map/__init__.py

@@ -0,0 +1 @@
+from .downloader import MapCatalogDownloader  # noqa

modules/map/api.py

@@ -1,51 +1,100 @@
 from http import HTTPStatus
-from pathlib import Path
 
 from sanic import Request, response, Blueprint
 from sanic_ext import validate
 from sanic_ext.extensions.openapi import openapi
 
 from modules.map import lib, schema
-from wrolpi import flags
+from modules.map.pins import get_map_pins_config
 from wrolpi.api_utils import json_response
-from wrolpi.common import wrol_mode_check, get_media_directory, background_task
-from wrolpi.errors import ValidationError
-from wrolpi.vars import DOCKERIZED
+from wrolpi.common import wrol_mode_check
+from wrolpi.events import Events
 
 map_bp = Blueprint('Map', '/api/map')
 
 
-@map_bp.post('/import')
-@openapi.definition(
-    summary='Import PBF/dump map files',
-    body=schema.ImportPost,
-)
-@validate(schema.ImportPost)
-@wrol_mode_check
-async def import_pbfs(_: Request, body: schema.ImportPost):
-    if flags.map_importing.is_set():
-        return response.json({'error': 'Map import already running'}, HTTPStatus.CONFLICT)
-
-    paths = [i for i in body.files if i]
-    if not paths:
-        raise ValidationError('No PBF or dump files were provided')
-
-    background_task(lib.import_files(paths))
-    return response.empty()
-
-
 @map_bp.get('/files')
-@openapi.description('Find any map files, get their import status')
-def get_files_status(request: Request):
-    paths = lib.get_import_status(request.ctx.session)
-    paths = sorted(paths, key=lambda i: str(i.path))
-    pending = request.app.shared_ctx.map_importing.get('pending')
-    if pending:
-        pending = [Path(i).relative_to(get_media_directory()) for i in pending]
-    body = dict(
-        files=paths,
-        pending=pending,
-        import_running=flags.map_importing.is_set(),
-        dockerized=DOCKERIZED,
-    )
-    return json_response(body, HTTPStatus.OK)
+@openapi.description('List available PMTiles map files')
+def get_files(_: Request):
+    files = lib.get_pmtiles_files()
+    return json_response(dict(files=files), HTTPStatus.OK)
+
+
+@map_bp.delete('/files/<filename:str>')
+@openapi.description('Delete a PMTiles map file')
+@wrol_mode_check
+async def delete_file(_: Request, filename: str):
+    try:
+        deleted = lib.delete_pmtiles_file(filename)
+    except ValueError as e:
+        return response.json({'error': str(e)}, HTTPStatus.BAD_REQUEST)
+
+    if deleted:
+        return response.empty()
+    return response.json({'error': 'File not found'}, HTTPStatus.NOT_FOUND)
+
+
+@map_bp.get('/subscribe')
+@openapi.description('Get map catalog and current subscriptions')
+def get_subscriptions(request: Request):
+    catalog = lib.get_map_catalog()
+    subscriptions = lib.get_map_subscriptions(request.ctx.session)
+    return json_response(dict(catalog=catalog, subscriptions=subscriptions), HTTPStatus.OK)
+
+
+@map_bp.post('/subscribe')
+@openapi.description('Subscribe to a map region for automatic downloads')
+@validate(schema.MapSubscribeRequest)
+@wrol_mode_check
+async def post_subscribe(request: Request, body: schema.MapSubscribeRequest):
+    try:
+        await lib.subscribe(request.ctx.session, body.name, body.region)
+    except ValueError as e:
+        return response.json({'error': str(e)}, HTTPStatus.BAD_REQUEST)
+
+    Events.send_created(f'Map subscription created for {body.name}')
+    return response.empty(HTTPStatus.CREATED)
+
+
+@map_bp.delete('/subscribe/<region:str>')
+@openapi.description('Unsubscribe from a map region')
+@wrol_mode_check
+async def delete_subscription(request: Request, region: str):
+    try:
+        await lib.unsubscribe(request.ctx.session, region)
+    except ValueError as e:
+        return response.json({'error': str(e)}, HTTPStatus.NOT_FOUND)
+
+    return response.empty(HTTPStatus.NO_CONTENT)
+
+
+@map_bp.get('/pins')
+@openapi.description('Get all map pins')
+def get_pins(_: Request):
+    pins = get_map_pins_config().pins
+    return json_response(dict(pins=pins), HTTPStatus.OK)
+
+
+@map_bp.post('/pins')
+@openapi.description('Add a map pin')
+@validate(schema.MapPinRequest)
+async def add_pin(_: Request, body: schema.MapPinRequest):
+    pin = get_map_pins_config().add_pin(body.lat, body.lon, body.label, body.color)
+    return json_response(dict(pin=pin), HTTPStatus.CREATED)
+
+
+@map_bp.delete('/pins/<pin_id:int>')
+@openapi.description('Delete a map pin by ID')
+async def delete_pin(_: Request, pin_id: int):
+    if get_map_pins_config().delete_pin(pin_id):
+        return response.empty(HTTPStatus.NO_CONTENT)
+    return response.json({'error': 'Pin not found'}, HTTPStatus.NOT_FOUND)
+
+
+@map_bp.put('/pins/<pin_id:int>')
+@openapi.description('Update a map pin by ID')
+@validate(schema.MapPinUpdateRequest)
+async def update_pin(_: Request, pin_id: int, body: schema.MapPinUpdateRequest):
+    if get_map_pins_config().update_pin(pin_id, body.label, body.color):
+        return response.empty(HTTPStatus.NO_CONTENT)
+    return response.json({'error': 'Pin not found'}, HTTPStatus.NOT_FOUND)

modules/map/catalog.py

@@ -0,0 +1,233 @@
+GB = 1_000_000_000
+MB = 1_000_000
+
+# Manifest URL — points to a JSON file on the CDN that contains the current planet file URL.
+MANIFEST_URL = 'https://wrolpi.nyc3.cdn.digitaloceanspaces.com/maps/manifest.json'
+
+# Predefined regions with bounding boxes for pmtiles extract.
+# These regions collectively cover the entire planet.
+# All regions are kept under ~9 GB to avoid OOM on 4 GB Raspberry Pis.
+# Sizes measured from the 20260329 planet file.
+MAP_REGIONS = [
+    # United States (split into 3 for Pi compatibility)
+    {
+        'name': 'United States (West)',
+        'region': 'us-west',
+        'bbox': '-125.0,24.4,-104.0,49.4',
+        'size_estimate': 5 * GB,
+    },
+    {
+        'name': 'United States (Central)',
+        'region': 'us-central',
+        'bbox': '-104.0,24.4,-87.0,49.4',
+        'size_estimate': 5 * GB,
+    },
+    {
+        'name': 'United States (East)',
+        'region': 'us-east',
+        'bbox': '-87.0,24.4,-66.9,49.4',
+        'size_estimate': 8 * GB,
+    },
+    {
+        'name': 'Alaska',
+        'region': 'us-alaska',
+        'bbox': '-180.0,51.0,-129.0,71.5',
+        'size_estimate': int(1.4 * GB),
+    },
+    # Canada (split into 3)
+    {
+        'name': 'Canada (West)',
+        'region': 'canada-west',
+        'bbox': '-141.0,48.0,-110.0,83.5',
+        'size_estimate': 5 * GB,
+    },
+    {
+        'name': 'Canada (Central)',
+        'region': 'canada-central',
+        'bbox': '-110.0,48.0,-80.0,83.5',
+        'size_estimate': 6 * GB,
+    },
+    {
+        'name': 'Canada (East)',
+        'region': 'canada-east',
+        'bbox': '-80.0,41.7,-52.0,83.5',
+        'size_estimate': 6 * GB,
+    },
+    {
+        'name': 'Mexico',
+        'region': 'mexico',
+        'bbox': '-118.5,14.5,-86.7,32.8',
+        'size_estimate': int(2.5 * GB),
+    },
+    # Central & South America
+    {
+        'name': 'Central America & Caribbean',
+        'region': 'central-america',
+        'bbox': '-92.2,7.2,-59.4,23.5',
+        'size_estimate': int(1.4 * GB),
+    },
+    {
+        'name': 'South America',
+        'region': 'south-america',
+        'bbox': '-81.4,-56.0,-34.7,12.5',
+        'size_estimate': int(7.8 * GB),
+    },
+    # Europe (split into 9 grid boxes — dense OSM coverage requires small regions)
+    {
+        'name': 'Europe (NW: UK, Ireland, Iceland)',
+        'region': 'europe-nw',
+        'bbox': '-25.0,48.0,5.0,72.0',
+        'size_estimate': int(5.5 * GB),
+    },
+    {
+        'name': 'Europe (SW: Spain, Portugal)',
+        'region': 'europe-sw',
+        'bbox': '-25.0,34.0,5.0,48.0',
+        'size_estimate': int(5.6 * GB),
+    },
+    {
+        'name': 'Europe (N-Central: Scandinavia)',
+        'region': 'europe-n-central',
+        'bbox': '5.0,55.0,15.0,72.0',
+        'size_estimate': int(3.6 * GB),
+    },
+    {
+        'name': 'Europe (Central: Germany, Benelux)',
+        'region': 'europe-central',
+        'bbox': '5.0,48.0,15.0,55.0',
+        'size_estimate': int(6.9 * GB),
+    },
+    {
+        'name': 'Europe (S-Central: Italy, France)',
+        'region': 'europe-s-central',
+        'bbox': '5.0,34.0,15.0,48.0',
+        'size_estimate': int(4.9 * GB),
+    },
+    {
+        'name': 'Europe (NE-Central: Poland, Baltics)',
+        'region': 'europe-ne-central',
+        'bbox': '15.0,48.0,25.0,72.0',
+        'size_estimate': int(7.2 * GB),
+    },
+    {
+        'name': 'Europe (SE-Central: Balkans, Greece)',
+        'region': 'europe-se-central',
+        'bbox': '15.0,34.0,25.0,48.0',
+        'size_estimate': int(3.2 * GB),
+    },
+    {
+        'name': 'Europe (NE: Ukraine, Belarus)',
+        'region': 'europe-ne',
+        'bbox': '25.0,48.0,45.0,72.0',
+        'size_estimate': int(7.4 * GB),
+    },
+    {
+        'name': 'Europe (SE: Turkey, Black Sea)',
+        'region': 'europe-se',
+        'bbox': '25.0,34.0,45.0,48.0',
+        'size_estimate': int(3.4 * GB),
+    },
+    # Africa (split into 2)
+    {
+        'name': 'Africa (North)',
+        'region': 'africa-north',
+        'bbox': '-18.0,0.0,52.0,37.5',
+        'size_estimate': 9 * GB,
+    },
+    {
+        'name': 'Africa (South)',
+        'region': 'africa-south',
+        'bbox': '-18.0,-35.0,52.0,0.0',
+        'size_estimate': 5 * GB,
+    },
+    # Middle East
+    {
+        'name': 'Middle East',
+        'region': 'middle-east',
+        'bbox': '25.0,12.0,63.0,42.0',
+        'size_estimate': int(4.1 * GB),
+    },
+    # Russia & Northern Asia (split into 3)
+    {
+        'name': 'Russia (West)',
+        'region': 'russia-west',
+        'bbox': '27.0,41.0,45.0,82.0',
+        'size_estimate': 8 * GB,
+    },
+    {
+        'name': 'Russia (Central)',
+        'region': 'russia-central',
+        'bbox': '45.0,41.0,100.0,82.0',
+        'size_estimate': 3 * GB,
+    },
+    {
+        'name': 'Russia (East)',
+        'region': 'russia-east',
+        'bbox': '100.0,41.0,180.0,82.0',
+        'size_estimate': 4 * GB,
+    },
+    # South Asia
+    {
+        'name': 'South Asia',
+        'region': 'south-asia',
+        'bbox': '60.0,5.0,98.0,40.0',
+        'size_estimate': int(5.5 * GB),
+    },
+    # East Asia (split into 2)
+    {
+        'name': 'East Asia (North)',
+        'region': 'east-asia-north',
+        'bbox': '73.0,35.0,150.0,53.5',
+        'size_estimate': 6 * GB,
+    },
+    {
+        'name': 'East Asia (South)',
+        'region': 'east-asia-south',
+        'bbox': '73.0,18.0,150.0,35.0',
+        'size_estimate': 8 * GB,
+    },
+    # Southeast Asia
+    {
+        'name': 'Southeast Asia',
+        'region': 'southeast-asia',
+        'bbox': '92.0,-11.0,141.0,28.5',
+        'size_estimate': 6 * GB,
+    },
+    # Oceania & Pacific
+    {
+        'name': 'Oceania & Australia',
+        'region': 'oceania',
+        'bbox': '110.0,-47.0,180.0,0.0',
+        'size_estimate': int(3.7 * GB),
+    },
+    {
+        'name': 'Pacific Islands',
+        'region': 'pacific-islands',
+        'bbox': '-180.0,-47.0,-120.0,28.5',
+        'size_estimate': 88 * MB,
+    },
+    # Arctic & Antarctic
+    {
+        'name': 'Greenland',
+        'region': 'greenland',
+        'bbox': '-74.0,59.0,-10.0,84.0',
+        'size_estimate': int(1.5 * GB),
+    },
+    {
+        'name': 'Antarctica',
+        'region': 'antarctica',
+        'bbox': '-180.0,-90.0,180.0,-60.0',
+        'size_estimate': 423 * MB,
+    },
+    # Terrain (DEM) — direct download, no bbox extraction.
+    {
+        'name': 'Terrain (Global Hillshade & Contours)',
+        'region': 'terrain-z0-8',
+        'bbox': None,
+        'size_estimate': 12 * GB,
+        'terrain': True,
+    },
+]
+
+MAP_REGIONS_BY_NAME = {r['name']: r for r in MAP_REGIONS}
+MAP_REGIONS_BY_REGION = {r['region']: r for r in MAP_REGIONS}

modules/map/downloader.py

@@ -0,0 +1,207 @@
+import tempfile
+from pathlib import Path
+from xml.etree import ElementTree as ET
+
+from modules.map import lib
+from wrolpi.common import logger, verify_gpg_signature
+from wrolpi.db import get_db_session
+from wrolpi.downloader import Downloader, Download, DownloadResult, download_manager
+
+__all__ = ['MapCatalogDownloader', 'MapExtractDownloader', 'map_catalog_downloader', 'map_extract_downloader']
+
+logger = logger.getChild(__name__)
+
+METALINK_NS = {'ml': 'urn:ietf:params:xml:ns:metalink'}
+
+
+async def verify_meta4_signature(meta4_contents: bytes) -> bool:
+    """Parse meta4 XML bytes, extract the hash and its GPG signature, and verify.
+
+    Returns True if the signature is valid, False if invalid or missing."""
+    try:
+        root = ET.fromstring(meta4_contents)
+        file_el = root.find('ml:file', METALINK_NS)
+        if file_el is None:
+            logger.warning('meta4 missing <file> element')
+            return False
+
+        hash_el = file_el.find('ml:hash[@type="sha-256"]', METALINK_NS)
+        sig_el = file_el.find('ml:signature', METALINK_NS)
+
+        if hash_el is None or hash_el.text is None:
+            logger.warning('meta4 missing <hash type="sha-256"> element')
+            return False
+        if sig_el is None or sig_el.text is None:
+            logger.warning('meta4 missing <signature> element — skipping verification')
+            return False
+
+        sha256_hash = hash_el.text.strip()
+        signature = sig_el.text.strip()
+    except ET.ParseError as e:
+        logger.warning(f'Failed to parse meta4 XML: {e}')
+        return False
+
+    # Write hash and signature to temp files for GPG verification.
+    with tempfile.TemporaryDirectory() as tmpdir:
+        hash_path = Path(tmpdir) / 'hash'
+        sig_path = Path(tmpdir) / 'hash.sig'
+        hash_path.write_text(sha256_hash)
+        sig_path.write_text(signature)
+
+        return await verify_gpg_signature(hash_path, sig_path)
+
+
+class MapCatalogDownloader(Downloader):
+    """Fetches the manifest and creates a child Download for each subscribed region."""
+
+    name = 'map_catalog'
+    listable = False
+
+    async def do_download(self, download: Download) -> DownloadResult:
+        try:
+            manifest = await lib.fetch_manifest(download.url)
+        except Exception as e:
+            logger.error(f'Failed to fetch map manifest: {e}')
+            return DownloadResult(success=False, error=f'Failed to fetch manifest: {e}')
+
+        version = manifest.get('version', '')
+        manifest_regions = manifest.get('regions', {})
+
+        if not manifest_regions:
+            return DownloadResult(success=False, error='Manifest does not contain regions')
+
+        regions = (download.settings or {}).get('regions', [])
+        if not regions:
+            return DownloadResult(success=True)
+
+        created = 0
+        with get_db_session(commit=True) as session:
+            for r in regions:
+                region = r.get('region', '')
+                region_url = manifest_regions.get(region)
+                if not region_url:
+                    logger.warning(f'Subscribed region {region!r} not found in manifest — skipping')
+                    continue
+
+                # Delete stale download for this region if the URL changed (new version).
+                old = session.query(Download).filter(
+                    Download.downloader == 'map_extract',
+                    Download.settings['region'].astext == region,
+                    Download.url != region_url,
+                ).one_or_none()
+                if old:
+                    logger.info(f'Deleting stale download for {region}: {old.url}')
+                    session.delete(old)
+                    session.flush()
+
+                child = download_manager.get_or_create_download(session, region_url, reset_attempts=True,
+                                                               override_skip=True)
+                child.downloader = 'map_extract'
+                child.location = '/map/manage'
+                child.settings = {
+                    'region': region,
+                    'version': version,
+                }
+                child.renew(reset_attempts=True)
+                created += 1
+
+        if not created:
+            return DownloadResult(success=False, error='No matching region URLs found in manifest')
+
+        logger.info(f'Created {created} map region downloads')
+        return DownloadResult(success=True, location='/map/manage')
+
+
+map_catalog_downloader = MapCatalogDownloader()
+
+
+class MapExtractDownloader(Downloader):
+    """Downloads a pre-extracted region file from the CDN via aria2c.
+
+    Settings contain: region, version — set by MapCatalogDownloader.
+
+    Fetches the meta4 file once, GPG-verifies the hash signature, then passes
+    the same meta4 bytes to aria2c for download hash verification. This ensures
+    the hash that was GPG-verified is the same hash aria2c checks against.
+    """
+
+    name = 'map_extract'
+    listable = False
+
+    async def do_download(self, download: Download) -> DownloadResult:
+        settings = download.settings or {}
+        region = settings.get('region')
+        version = settings.get('version', '')
+
+        if not region:
+            return DownloadResult(success=False, error='No region in download settings')
+
+        map_directory = lib.get_map_directory()
+
+        # Versioned filename.
+        if version:
+            output_name = f'{region}-{version}.pmtiles'
+        else:
+            output_name = f'{region}.pmtiles'
+        output_path = map_directory / output_name
+        tmp_path = map_directory / f'{output_name}.tmp'
+
+        # Skip if versioned file already exists.
+        if output_path.is_file():
+            logger.info(f'{output_name} already exists, skipping download')
+            return DownloadResult(success=True, location='/map')
+
+        # Clean up any leftover temp file.
+        if tmp_path.is_file():
+            tmp_path.unlink()
+
+        # Fetch meta4 once, GPG-verify the hash signature before downloading.
+        meta4_contents = await self.get_meta4_contents(download.url)
+        if meta4_contents:
+            if not await verify_meta4_signature(meta4_contents):
+                return DownloadResult(success=False,
+                                     error=f'meta4 hash signature verification failed for {region}')
+            logger.info(f'meta4 signature verified for {region}')
+        else:
+            logger.warning(f'No meta4 available for {region} — downloading without hash verification')
+
+        # Download via aria2c, passing the pre-verified meta4 for hash checking.
+        logger.warning(f'Downloading {region} from {download.url}')
+        downloaded_path = None
+        try:
+            downloaded_path = await self.download_file(
+                download, download.url, map_directory,
+                check_for_meta4=False, concurrent=1, meta4_xml=meta4_contents,
+            )
+            downloaded_path.rename(tmp_path)
+        except Exception as e:
+            logger.error(f'Download failed for {region}: {e}')
+            tmp_path.unlink(missing_ok=True)
+            if downloaded_path:
+                downloaded_path.unlink(missing_ok=True)
+            return DownloadResult(success=False, error=f'Download failed: {str(e)[:500]}')
+
+        if not tmp_path.is_file():
+            return DownloadResult(success=False, error=f'Output file was not created: {tmp_path}')
+
+        # Replace old file atomically.
+        tmp_path.rename(output_path)
+
+        # Delete older versions of this region.
+        for old_file in map_directory.glob(f'{region}-*.pmtiles'):
+            if old_file.name != output_name:
+                logger.warning(f'Deleting old map version: {old_file}')
+                old_file.unlink()
+        # Also delete unversioned file if it exists.
+        unversioned = map_directory / f'{region}.pmtiles'
+        if unversioned.is_file() and unversioned.name != output_name:
+            logger.warning(f'Deleting unversioned map file: {unversioned}')
+            unversioned.unlink()
+
+        size = output_path.stat().st_size
+        logger.warning(f'Successfully downloaded {region} ({size} bytes) to {output_path}')
+
+        return DownloadResult(success=True, location='/map')
+
+
+map_extract_downloader = MapExtractDownloader()

modules/map/leaflet.css

@@ -1,758 +0,0 @@
-/* required styles */
-/* @preserve
- * Leaflet 1.7.1, a JS library for interactive maps. http://leafletjs.com
- * (c) 2010-2019 Vladimir Agafonkin, (c) 2010-2011 CloudMade
- */
-
-.leaflet-pane,
-.leaflet-tile,
-.leaflet-marker-icon,
-.leaflet-marker-shadow,
-.leaflet-tile-container,
-.leaflet-pane > svg,
-.leaflet-pane > canvas,
-.leaflet-zoom-box,
-.leaflet-image-layer,
-.leaflet-layer {
-    position: absolute;
-    left: 0;
-    top: 0;
-}
-
-.leaflet-container {
-    overflow: hidden;
-}
-
-.leaflet-tile,
-.leaflet-marker-icon,
-.leaflet-marker-shadow {
-    -webkit-user-select: none;
-    -moz-user-select: none;
-    user-select: none;
-    -webkit-user-drag: none;
-}
-
-/* Prevents IE11 from highlighting tiles in blue */
-.leaflet-tile::selection {
-    background: transparent;
-}
-
-/* Safari renders non-retina tile on retina better with this, but Chrome is worse */
-.leaflet-safari .leaflet-tile {
-    image-rendering: -webkit-optimize-contrast;
-}
-
-/* hack that prevents hw layers "stretching" when loading new tiles */
-.leaflet-safari .leaflet-tile-container {
-    width: 1600px;
-    height: 1600px;
-    -webkit-transform-origin: 0 0;
-}
-
-.leaflet-marker-icon,
-.leaflet-marker-shadow {
-    display: block;
-}
-
-/* .leaflet-container svg: reset svg max-width decleration shipped in Joomla! (joomla.org) 3.x */
-/* .leaflet-container img: map is broken in FF if you have max-width: 100% on tiles */
-.leaflet-container .leaflet-overlay-pane svg,
-.leaflet-container .leaflet-marker-pane img,
-.leaflet-container .leaflet-shadow-pane img,
-.leaflet-container .leaflet-tile-pane img,
-.leaflet-container img.leaflet-image-layer,
-.leaflet-container .leaflet-tile {
-    max-width: none !important;
-    max-height: none !important;
-}
-
-.leaflet-container.leaflet-touch-zoom {
-    -ms-touch-action: pan-x pan-y;
-    touch-action: pan-x pan-y;
-}
-
-.leaflet-container.leaflet-touch-drag {
-    -ms-touch-action: pinch-zoom;
-    /* Fallback for FF which doesn't support pinch-zoom */
-    touch-action: none;
-    touch-action: pinch-zoom;
-}
-
-.leaflet-container.leaflet-touch-drag.leaflet-touch-zoom {
-    -ms-touch-action: none;
-    touch-action: none;
-}
-
-.leaflet-container {
-    -webkit-tap-highlight-color: transparent;
-}
-
-.leaflet-container a {
-    -webkit-tap-highlight-color: rgba(51, 181, 229, 0.4);
-}
-
-.leaflet-tile {
-    filter: inherit;
-    visibility: hidden;
-}
-
-.leaflet-tile-loaded {
-    visibility: inherit;
-}
-
-.leaflet-zoom-box {
-    width: 0;
-    height: 0;
-    -moz-box-sizing: border-box;
-    box-sizing: border-box;
-    z-index: 800;
-}
-
-/* workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=888319 */
-.leaflet-overlay-pane svg {
-    -moz-user-select: none;
-}
-
-.leaflet-pane {
-    z-index: 400;
-}
-
-.leaflet-tile-pane {
-    z-index: 200;
-}
-
-.leaflet-overlay-pane {
-    z-index: 400;
-}
-
-.leaflet-shadow-pane {
-    z-index: 500;
-}
-
-.leaflet-marker-pane {
-    z-index: 600;
-}
-
-.leaflet-tooltip-pane {
-    z-index: 650;
-}
-
-.leaflet-popup-pane {
-    z-index: 700;
-}
-
-.leaflet-map-pane canvas {
-    z-index: 100;
-}
-
-.leaflet-map-pane svg {
-    z-index: 200;
-}
-
-.leaflet-vml-shape {
-    width: 1px;
-    height: 1px;
-}
-
-.lvml {
-    behavior: url(#default#VML);
-    display: inline-block;
-    position: absolute;
-}
-
-
-/* control positioning */
-
-.leaflet-control {
-    position: relative;
-    z-index: 800;
-    pointer-events: visiblePainted; /* IE 9-10 doesn't have auto */
-    pointer-events: auto;
-}
-
-.leaflet-top,
-.leaflet-bottom {
-    position: absolute;
-    z-index: 1000;
-    pointer-events: none;
-}
-
-.leaflet-top {
-    top: 0;
-}
-
-.leaflet-right {
-    right: 0;
-}
-
-.leaflet-bottom {
-    bottom: 0;
-}
-
-.leaflet-left {
-    left: 0;
-}
-
-.leaflet-control {
-    float: left;
-    clear: both;
-}
-
-.leaflet-right .leaflet-control {
-    float: right;
-}
-
-.leaflet-top .leaflet-control {
-    margin-top: 10px;
-}
-
-.leaflet-bottom .leaflet-control {
-    margin-bottom: 10px;
-}
-
-.leaflet-left .leaflet-control {
-    margin-left: 10px;
-}
-
-.leaflet-right .leaflet-control {
-    margin-right: 10px;
-}
-
-
-/* zoom and fade animations */
-
-.leaflet-fade-anim .leaflet-tile {
-    will-change: opacity;
-}
-
-.leaflet-fade-anim .leaflet-popup {
-    opacity: 0;
-    -webkit-transition: opacity 0.2s linear;
-    -moz-transition: opacity 0.2s linear;
-    transition: opacity 0.2s linear;
-}
-
-.leaflet-fade-anim .leaflet-map-pane .leaflet-popup {
-    opacity: 1;
-}
-
-.leaflet-zoom-animated {
-    -webkit-transform-origin: 0 0;
-    -ms-transform-origin: 0 0;
-    transform-origin: 0 0;
-}
-
-.leaflet-zoom-anim .leaflet-zoom-animated {
-    will-change: transform;
-}
-
-.leaflet-zoom-anim .leaflet-zoom-animated {
-    -webkit-transition: -webkit-transform 0.25s cubic-bezier(0, 0, 0.25, 1);
-    -moz-transition: -moz-transform 0.25s cubic-bezier(0, 0, 0.25, 1);
-    transition: transform 0.25s cubic-bezier(0, 0, 0.25, 1);
-}
-
-.leaflet-zoom-anim .leaflet-tile,
-.leaflet-pan-anim .leaflet-tile {
-    -webkit-transition: none;
-    -moz-transition: none;
-    transition: none;
-}
-
-.leaflet-zoom-anim .leaflet-zoom-hide {
-    visibility: hidden;
-}
-
-
-/* cursors */
-
-.leaflet-interactive {
-    cursor: pointer;
-}
-
-.leaflet-grab {
-    cursor: -webkit-grab;
-    cursor: -moz-grab;
-    cursor: grab;
-}
-
-.leaflet-crosshair,
-.leaflet-crosshair .leaflet-interactive {
-    cursor: crosshair;
-}
-
-.leaflet-popup-pane,
-.leaflet-control {
-    cursor: auto;
-}
-
-.leaflet-dragging .leaflet-grab,
-.leaflet-dragging .leaflet-grab .leaflet-interactive,
-.leaflet-dragging .leaflet-marker-draggable {
-    cursor: move;
-    cursor: -webkit-grabbing;
-    cursor: -moz-grabbing;
-    cursor: grabbing;
-}
-
-/* marker & overlays interactivity */
-.leaflet-marker-icon,
-.leaflet-marker-shadow,
-.leaflet-image-layer,
-.leaflet-pane > svg path,
-.leaflet-tile-container {
-    pointer-events: none;
-}
-
-.leaflet-marker-icon.leaflet-interactive,
-.leaflet-image-layer.leaflet-interactive,
-.leaflet-pane > svg path.leaflet-interactive,
-svg.leaflet-image-layer.leaflet-interactive path {
-    pointer-events: visiblePainted; /* IE 9-10 doesn't have auto */
-    pointer-events: auto;
-}
-
-/* visual tweaks */
-
-.leaflet-container {
-    background: #ddd;
-    outline: 0;
-}
-
-.leaflet-container a {
-    color: #0078A8;
-}
-
-.leaflet-container a.leaflet-active {
-    outline: 2px solid orange;
-}
-
-.leaflet-zoom-box {
-    border: 2px dotted #38f;
-    background: rgba(255, 255, 255, 0.5);
-}
-
-
-/* general typography */
-.leaflet-container {
-    font: 12px/1.5 "Helvetica Neue", Arial, Helvetica, sans-serif;
-}
-
-
-/* general toolbar styles */
-
-.leaflet-bar {
-    box-shadow: 0 1px 5px rgba(0, 0, 0, 0.65);
-    border-radius: 4px;
-}
-
-.leaflet-bar a,
-.leaflet-bar a:hover {
-    background-color: #fff;
-    border-bottom: 1px solid #ccc;
-    width: 26px;
-    height: 26px;
-    line-height: 26px;
-    display: block;
-    text-align: center;
-    text-decoration: none;
-    color: black;
-}
-
-.leaflet-bar a,
-.leaflet-control-layers-toggle {
-    background-position: 50% 50%;
-    background-repeat: no-repeat;
-    display: block;
-}
-
-.leaflet-bar a:hover {
-    background-color: #f4f4f4;
-}
-
-.leaflet-bar a:first-child {
-    border-top-left-radius: 4px;
-    border-top-right-radius: 4px;
-}
-
-.leaflet-bar a:last-child {
-    border-bottom-left-radius: 4px;
-    border-bottom-right-radius: 4px;
-    border-bottom: none;
-}
-
-.leaflet-bar a.leaflet-disabled {
-    cursor: default;
-    background-color: #f4f4f4;
-    color: #bbb;
-}
-
-.leaflet-touch .leaflet-bar a {
-    width: 30px;
-    height: 30px;
-    line-height: 30px;
-}
-
-.leaflet-touch .leaflet-bar a:first-child {
-    border-top-left-radius: 2px;
-    border-top-right-radius: 2px;
-}
-
-.leaflet-touch .leaflet-bar a:last-child {
-    border-bottom-left-radius: 2px;
-    border-bottom-right-radius: 2px;
-}
-
-/* zoom control */
-
-.leaflet-control-zoom-in,
-.leaflet-control-zoom-out {
-    font: bold 18px 'Lucida Console', Monaco, monospace;
-    text-indent: 1px;
-}
-
-.leaflet-touch .leaflet-control-zoom-in, .leaflet-touch .leaflet-control-zoom-out {
-    font-size: 22px;
-}
-
-
-/* layers control */
-
-.leaflet-control-layers {
-    box-shadow: 0 1px 5px rgba(0, 0, 0, 0.4);
-    background: #fff;
-    border-radius: 5px;
-}
-
-.leaflet-control-layers-toggle {
-    background-image: url(images/layers.png);
-    width: 36px;
-    height: 36px;
-}
-
-.leaflet-retina .leaflet-control-layers-toggle {
-    background-image: url(images/layers-2x.png);
-    background-size: 26px 26px;
-}
-
-.leaflet-touch .leaflet-control-layers-toggle {
-    width: 44px;
-    height: 44px;
-}
-
-.leaflet-control-layers .leaflet-control-layers-list,
-.leaflet-control-layers-expanded .leaflet-control-layers-toggle {
-    display: none;
-}
-
-.leaflet-control-layers-expanded .leaflet-control-layers-list {
-    display: block;
-    position: relative;
-}
-
-.leaflet-control-layers-expanded {
-    padding: 6px 10px 6px 6px;
-    color: #333;
-    background: #fff;
-}
-
-.leaflet-control-layers-scrollbar {
-    overflow-y: scroll;
-    overflow-x: hidden;
-    padding-right: 5px;
-}
-
-.leaflet-control-layers-selector {
-    margin-top: 2px;
-    position: relative;
-    top: 1px;
-}
-
-.leaflet-control-layers label {
-    display: block;
-}
-
-.leaflet-control-layers-separator {
-    height: 0;
-    border-top: 1px solid #ddd;
-    margin: 5px -10px 5px -6px;
-}
-
-/* Default icon URLs */
-.leaflet-default-icon-path {
-    background-image: url(images/marker-icon.png);
-}
-
-
-/* attribution and scale controls */
-
-.leaflet-container .leaflet-control-attribution {
-    background: #fff;
-    background: rgba(255, 255, 255, 0.7);
-    margin: 0;
-}
-
-.leaflet-control-attribution,
-.leaflet-control-scale-line {
-    padding: 0 5px;
-    color: #333;
-}
-
-.leaflet-control-attribution a {
-    text-decoration: none;
-}
-
-.leaflet-control-attribution a:hover {
-    text-decoration: underline;
-}
-
-.leaflet-container .leaflet-control-attribution,
-.leaflet-container .leaflet-control-scale {
-    font-size: 11px;
-}
-
-.leaflet-left .leaflet-control-scale {
-    margin-left: 5px;
-}
-
-.leaflet-bottom .leaflet-control-scale {
-    margin-bottom: 5px;
-}
-
-.leaflet-control-scale-line {
-    border: 2px solid #777;
-    border-top: none;
-    line-height: 1.1;
-    padding: 2px 5px 1px;
-    font-size: 11px;
-    white-space: nowrap;
-    overflow: hidden;
-    -moz-box-sizing: border-box;
-    box-sizing: border-box;
-
-    background: #fff;
-    background: rgba(255, 255, 255, 0.5);
-}
-
-.leaflet-control-scale-line:not(:first-child) {
-    border-top: 2px solid #777;
-    border-bottom: none;
-    margin-top: -2px;
-}
-
-.leaflet-control-scale-line:not(:first-child):not(:last-child) {
-    border-bottom: 2px solid #777;
-}
-
-.leaflet-touch .leaflet-control-attribution,
-.leaflet-touch .leaflet-control-layers,
-.leaflet-touch .leaflet-bar {
-    box-shadow: none;
-}
-
-.leaflet-touch .leaflet-control-layers,
-.leaflet-touch .leaflet-bar {
-    border: 2px solid rgba(0, 0, 0, 0.2);
-    background-clip: padding-box;
-}
-
-
-/* popup */
-
-.leaflet-popup {
-    position: absolute;
-    text-align: center;
-    margin-bottom: 20px;
-}
-
-.leaflet-popup-content-wrapper {
-    padding: 1px;
-    text-align: left;
-    border-radius: 12px;
-}
-
-.leaflet-popup-content {
-    margin: 13px 19px;
-    line-height: 1.4;
-}
-
-.leaflet-popup-content p {
-    margin: 18px 0;
-}
-
-.leaflet-popup-tip-container {
-    width: 40px;
-    height: 20px;
-    position: absolute;
-    left: 50%;
-    margin-left: -20px;
-    overflow: hidden;
-    pointer-events: none;
-}
-
-.leaflet-popup-tip {
-    width: 17px;
-    height: 17px;
-    padding: 1px;
-
-    margin: -10px auto 0;
-
-    -webkit-transform: rotate(45deg);
-    -moz-transform: rotate(45deg);
-    -ms-transform: rotate(45deg);
-    transform: rotate(45deg);
-}
-
-.leaflet-popup-content-wrapper,
-.leaflet-popup-tip {
-    background: white;
-    color: #333;
-    box-shadow: 0 3px 14px rgba(0, 0, 0, 0.4);
-}
-
-.leaflet-container a.leaflet-popup-close-button {
-    position: absolute;
-    top: 0;
-    right: 0;
-    padding: 4px 4px 0 0;
-    border: none;
-    text-align: center;
-    width: 18px;
-    height: 14px;
-    font: 16px/14px Tahoma, Verdana, sans-serif;
-    color: #c3c3c3;
-    text-decoration: none;
-    font-weight: bold;
-    background: transparent;
-}
-
-.leaflet-container a.leaflet-popup-close-button:hover {
-    color: #999;
-}
-
-.leaflet-popup-scrolled {
-    overflow: auto;
-    border-bottom: 1px solid #ddd;
-    border-top: 1px solid #ddd;
-}
-
-.leaflet-oldie .leaflet-popup-content-wrapper {
-    -ms-zoom: 1;
-}
-
-.leaflet-oldie .leaflet-popup-tip {
-    width: 24px;
-    margin: 0 auto;
-
-    -ms-filter: "progid:DXImageTransform.Microsoft.Matrix(M11=0.70710678, M12=0.70710678, M21=-0.70710678, M22=0.70710678)";
-    filter: progid:DXImageTransform.Microsoft.Matrix(M11=0.70710678, M12=0.70710678, M21=-0.70710678, M22=0.70710678);
-}
-
-.leaflet-oldie .leaflet-popup-tip-container {
-    margin-top: -1px;
-}
-
-.leaflet-oldie .leaflet-control-zoom,
-.leaflet-oldie .leaflet-control-layers,
-.leaflet-oldie .leaflet-popup-content-wrapper,
-.leaflet-oldie .leaflet-popup-tip {
-    border: 1px solid #999;
-}
-
-
-/* div icon */
-
-.leaflet-div-icon {
-    background: #fff;
-    border: 1px solid #666;
-}
-
-
-/* Tooltip */
-/* Base styles for the element that has a tooltip */
-.leaflet-tooltip {
-    position: absolute;
-    padding: 6px;
-    background-color: #fff;
-    border: 1px solid #fff;
-    border-radius: 3px;
-    color: #222;
-    white-space: nowrap;
-    -webkit-user-select: none;
-    -moz-user-select: none;
-    -ms-user-select: none;
-    user-select: none;
-    pointer-events: none;
-    box-shadow: 0 1px 3px rgba(0, 0, 0, 0.4);
-}
-
-.leaflet-tooltip.leaflet-clickable {
-    cursor: pointer;
-    pointer-events: auto;
-}
-
-.leaflet-tooltip-top:before,
-.leaflet-tooltip-bottom:before,
-.leaflet-tooltip-left:before,
-.leaflet-tooltip-right:before {
-    position: absolute;
-    pointer-events: none;
-    border: 6px solid transparent;
-    background: transparent;
-    content: "";
-}
-
-/* Directions */
-
-.leaflet-tooltip-bottom {
-    margin-top: 6px;
-}
-
-.leaflet-tooltip-top {
-    margin-top: -6px;
-}
-
-.leaflet-tooltip-bottom:before,
-.leaflet-tooltip-top:before {
-    left: 50%;
-    margin-left: -6px;
-}
-
-.leaflet-tooltip-top:before {
-    bottom: 0;
-    margin-bottom: -12px;
-    border-top-color: #fff;
-}
-
-.leaflet-tooltip-bottom:before {
-    top: 0;
-    margin-top: -12px;
-    margin-left: -6px;
-    border-bottom-color: #fff;
-}
-
-.leaflet-tooltip-left {
-    margin-left: -6px;
-}
-
-.leaflet-tooltip-right {
-    margin-left: 6px;
-}
-
-.leaflet-tooltip-left:before,
-.leaflet-tooltip-right:before {
-    top: 50%;
-    margin-top: -6px;
-}
-
-.leaflet-tooltip-left:before {
-    right: 0;
-    margin-right: -12px;
-    border-left-color: #fff;
-}
-
-.leaflet-tooltip-right:before {
-    left: 0;
-    margin-left: -12px;
-    border-right-color: #fff;
-}

modules/map/leaflet.html

@@ -1,39 +0,0 @@
-<!DOCTYPE HTML>
-<html lang="en">
-<head>
-    <meta charset="utf-8"/>
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <script src="/leaflet.js"></script>
-    <link rel="stylesheet" href="/leaflet.css"/>
-    <title>Leaflet Map</title>
-    <style>
-        html, body {
-            height: 100%;
-            padding: 0;
-            margin: 0;
-        }
-
-        #map {
-            /* configure the size of the map */
-            width: 100%;
-            height: 100%;
-        }
-    </style>
-</head>
-<body>
-<div id="map"></div>
-<script>
-    // initialize Leaflet
-    var map = L.map('map').setView({lon: 0, lat: 0}, 2);
-
-    // add the OpenStreetMap tiles
-    L.tileLayer(`/tile/{z}/{x}/{y}.png`, {
-        maxZoom: 19,
-        attribution: '&copy; <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
-    }).addTo(map);
-
-    // show the scale bar on the lower left corner
-    L.control.scale({imperial: true, metric: true}).addTo(map);
-</script>
-</body>
-</html>

modules/map/lib.py

@@ -1,19 +1,13 @@
-import subprocess
-from datetime import timedelta
+import json
+import tempfile
 from pathlib import Path
-from typing import List
+from typing import List, Dict
 
 from sqlalchemy.orm import Session
 
-from modules.map.models import MapFile
-from wrolpi import flags
-from wrolpi.api_utils import api_app
-from wrolpi.cmd import SUDO_BIN, run_command
-from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config
-from wrolpi.dates import timedelta_to_timestamp, seconds_to_timestamp, now
-from wrolpi.db import get_db_session
-from wrolpi.events import Events
-from wrolpi.vars import PROJECT_DIR, IS_RPI5
+from modules.map.catalog import MAP_REGIONS, MAP_REGIONS_BY_NAME, MAP_REGIONS_BY_REGION, MANIFEST_URL
+from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config, aiohttp_get, verify_gpg_signature
+from wrolpi.downloader import DownloadFrequency, Download, save_downloads_config, download_manager
 
 logger = logger.getChild(__name__)
 
@@ -25,229 +19,151 @@ def get_map_directory() -> Path:
     return map_directory
 
 
-def is_pbf_file(pbf: Path) -> bool:
-    """Uses file command to check type of file.  Returns True if a file is an OpenStreetMap PBF file."""
-    cmd = ('/usr/bin/file', pbf)
-    try:
-        output = subprocess.check_output(cmd)
-    except FileNotFoundError:
-        return False
-
-    return 'OpenStreetMap Protocolbuffer Binary Format' in output.decode()
-
-
-def is_dump_file(path: Path) -> bool:
-    """Uses file command to check the type of file.  Returns True if a file is a Postgresql dump file."""
-    cmd = ('/usr/bin/file', path)
-    try:
-        output = subprocess.check_output(cmd)
-    except FileNotFoundError:
-        return False
-
-    return 'PostgreSQL custom database dump' in output.decode()
-
-
-def get_map_paths() -> List[Path]:
-    """Find all pbf/dump files in the map directory."""
+def get_pmtiles_files() -> List[dict]:
+    """Find all .pmtiles files in the map directory and return their metadata."""
     map_directory = get_map_directory()
     paths = walk(map_directory)
 
-    def is_valid(path: Path) -> bool:
-        if path.is_file():
-            if str(path).endswith('.osm.pbf') and is_pbf_file(path):
-                return True
-            elif path.suffix == '.dump' and is_dump_file(path):
-                return True
-        return False
+    files = []
+    for path in sorted(paths):
+        if path.is_file() and path.suffix == '.pmtiles':
+            stat = path.stat()
+            files.append(dict(
+                name=path.name,
+                path=str(path.relative_to(get_media_directory())),
+                size=stat.st_size,
+                mtime=stat.st_mtime,
+            ))
 
-    return list(filter(is_valid, paths))
+    return files
 
 
-def get_or_create_map_file(pbf_path: Path, session: Session) -> MapFile:
-    """Finds the MapFile row in the DB, or creates one."""
-    map_file: MapFile = session.query(MapFile).filter_by(path=str(pbf_path)).one_or_none()
-    if map_file:
-        return map_file
+def delete_pmtiles_file(filename: str) -> bool:
+    """Delete a PMTiles file from the map directory.  Returns True if the file was deleted."""
+    map_directory = get_map_directory()
+    # Prevent path traversal.
+    path = (map_directory / filename).resolve()
+    if not path.is_relative_to(map_directory.resolve()):
+        raise ValueError(f'Invalid filename: {filename}')
 
-    map_file = MapFile(path=pbf_path, size=pbf_path.stat().st_size)
-    session.add(map_file)
-    return map_file
+    if path.is_file() and path.suffix == '.pmtiles':
+        path.unlink()
+        logger.warning(f'Deleted PMTiles file: {path}')
+        return True
+
+    return False
 
 
-import_logger = logger.getChild('import')
+def get_map_catalog() -> List[dict]:
+    """Return the list of available map regions."""
+    return MAP_REGIONS.copy()
 
 
-async def import_files(paths: List[str]):
-    if flags.map_importing.is_set():
-        import_logger.error('Map import already running...')
+def _get_map_download(session: Session) -> Download | None:
+    """Find the single recurring map download, if it exists."""
+    return session.query(Download).filter_by(url=MANIFEST_URL, downloader='map_catalog').one_or_none()
+
+
+def _get_or_create_map_download(session: Session) -> Download:
+    """Get or create the single recurring map download."""
+    download = _get_map_download(session)
+    if download:
+        return download
+
+    download = download_manager.get_or_create_download(session, MANIFEST_URL, reset_attempts=True,
+                                                       override_skip=True)
+    download.downloader = 'map_catalog'
+    download.sub_downloader = 'map_extract'
+    download.frequency = DownloadFrequency.days180
+    download.settings = {'regions': []}
+    download.location = '/map/manage'
+    download.attempts = 0
+    session.flush([download])
+    return download
+
+
+def get_map_subscriptions(session: Session) -> List[dict]:
+    """Return the list of subscribed regions."""
+    regions = []
+    download = _get_map_download(session)
+    if download and download.settings:
+        regions.extend(download.settings.get('regions', []))
+    return regions
+
+
+async def subscribe(session: Session, name: str, region: str):
+    """Subscribe to a map region."""
+    if name not in MAP_REGIONS_BY_NAME:
+        raise ValueError(f'{name!r} is not a valid map region name')
+
+    region_info = MAP_REGIONS_BY_NAME[name]
+    if region != region_info['region']:
+        raise ValueError(f'Region {region!r} does not match name {name!r}')
+
+    download = _get_or_create_map_download(session)
+    regions = list(download.settings.get('regions', []))
+
+    # Don't add duplicates.
+    if any(r['region'] == region for r in regions):
         return
 
-    with flags.map_importing:
-        paths = [get_media_directory() / i for i in paths]
-        import_logger.warning(f'Importing: {", ".join(map(str, paths))}')
-
-        dumps = [i for i in paths if i.suffix == '.dump']
-        pbfs = [i for i in paths if i.suffix == '.pbf']
-
-        start = now()
-        any_success = False
-        try:
-            if pbfs:
-                with get_db_session(commit=True) as session:
-                    # Any previously imported PBFs are no longer imported.
-                    for pbf_file in session.query(MapFile):
-                        pbf_file.imported = False
-
-                success = False
-                try:
-                    api_app.shared_ctx.map_importing.update(dict(
-                        pending=list(pbfs),
-                    ))
-                    await run_import_command(*pbfs)
-                    success = True
-                    any_success = True
-                except Exception as e:
-                    import_logger.warning('Failed to run import', exc_info=e)
-                finally:
-                    api_app.shared_ctx.map_importing.update(dict(
-                        pending=None,
-                    ))
-
-                if success:
-                    with get_db_session(commit=True) as session:
-                        # Any previously imported PBFs are no longer imported.
-                        for pbf_path in pbfs:
-                            pbf_file = get_or_create_map_file(pbf_path, session)
-                            pbf_file.imported = True
-
-            for path in dumps:
-                # Import each dump individually.
-                if not path.is_file():
-                    import_logger.fatal(f'Map file does not exist! {path}')
-                    continue
-
-                with get_db_session() as session:
-                    map_file = session.query(MapFile).filter_by(path=path).one_or_none()
-                    if map_file and map_file.imported:
-                        # Don't import a map file twice.
-                        import_logger.debug(f'{path} is already imported')
-                        continue
-
-                success = False
-                try:
-                    api_app.shared_ctx.map_importing.update(dict(
-                        pending=str(path),
-                    ))
-                    await run_import_command(path)
-                    success = True
-                    any_success = True
-                except Exception as e:
-                    import_logger.warning('Failed to run import', exc_info=e)
-                finally:
-                    api_app.shared_ctx.map_importing.update(dict(
-                        pending=None,
-                    ))
-
-                if success:
-                    with get_db_session(commit=True) as session:
-                        map_file = get_or_create_map_file(path, session)
-                        map_file.imported = True
-        finally:
-            elapsed = (now() - start).total_seconds()
-            if any_success:
-                Events.send_map_import_complete(f'Map import completed; took {seconds_to_timestamp(elapsed)}')
-            else:
-                Events.send_map_import_failed(
-                    f'Map import failed; took {seconds_to_timestamp(elapsed)} See server logs.')
-
-
-async def run_import_command(*paths: Path):
-    """Run the map import script on the provided paths.
-
-    Can only import a single *.dump file, or a list of *.osm.pbf files.  They cannot be mixed.
-    """
-    paths = [i.absolute() for i in paths]
-    dumps = [i for i in paths if i.suffix == '.dump']
-    pbfs = [i for i in paths if i.suffix == '.pbf']
-
-    # Only import a single dump, or a list of pbfs.  No other combinations acceptable.
-    if not paths:
-        raise ValueError('Must import a file')
-    if not dumps and not pbfs:
-        raise ValueError('Cannot import unknown file!')
-    if dumps and pbfs:
-        raise ValueError('Cannot import mixed dumps and pbfs.')
-    if len(dumps) > 1:
-        raise ValueError('Cannot import more than one dump')
-    if dumps and not is_dump_file(dumps[0]):
-        import_logger.warning(f'Could not import non-dump file: {dumps}')
-        raise ValueError('Invalid dump file')
-    if pbfs:
-        for path in pbfs:
-            if not is_pbf_file(path):
-                import_logger.warning(f'Could not import non-pbf file: {path}')
-                raise ValueError('Invalid PBF file')
-
-    paths = [str(i) for i in paths]
-    # Run with sudo so renderd can be restarted.
-    cmd = (SUDO_BIN, f'{PROJECT_DIR}/scripts/import_map.sh', *paths)
-    result = await run_command(
-        cmd,
-        timeout=0,  # No timeout, map importing could take days.
-    )
-
-    success = 'Successful' if result.return_code == 0 else 'Unsuccessful'
-    import_logger.info(
-        f'{success} import of {repr(paths)} took {timedelta_to_timestamp(timedelta(seconds=result.elapsed))}')
-    if result.return_code != 0:
-        # Log all lines.  Truncate long lines.
-        for line in result.stdout.decode().splitlines():
-            import_logger.debug(line[:500])
-        for line in result.stderr.decode().splitlines():
-            import_logger.error(line[:500])
-        raise ValueError(f'Importing map file failed with return code {result.return_code}')
-
-
-def get_import_status(session: Session) -> List[MapFile]:
-    paths = get_map_paths()
-    map_paths = []
-    for path in paths:
-        map_file = get_or_create_map_file(path, session)
-        map_paths.append(map_file)
+    entry = {'region': region}
+    if region_info.get('terrain'):
+        entry['terrain'] = True
 
+    regions.append(entry)
+    download.settings = {**download.settings, 'regions': regions}
+    download.renew(reset_attempts=True)
     session.commit()
-    return map_paths
+    save_downloads_config.activate_switch()
 
 
-# Calculated using many tests on a well-cooled RPi4 (4GB).
-RPI4_PBF_BYTES_PER_SECOND = 175473
-RPI4_COEFFICIENTS = [3.25099914e-15, 8.88717664e-06, -3.59754030e+02]
-# Calculated using many tests on a well-cooled RPi5 (8GB).
-RPI5_PBF_BYTES_PER_SECOND = 136003
-RPI5_COEFFICIENTS = [1.66813827e-15, 2.35907825e-06, 5.37276181e+01]
+async def unsubscribe(session: Session, region: str):
+    """Remove a region from the subscriptions."""
+    download = _get_map_download(session)
+    if not download:
+        raise ValueError(f'No map subscriptions exist')
+
+    regions = [r for r in download.settings.get('regions', []) if r['region'] != region]
+    if regions:
+        download.settings = {**download.settings, 'regions': regions}
+    else:
+        # No more subscriptions — delete the download.
+        session.delete(download)
+    session.commit()
+    save_downloads_config.activate_switch()
 
 
-def seconds_to_import_rpi4(size_in_bytes: int) -> int:
-    if size_in_bytes > 1_000_000_000:
-        # Use exponential curve for large files.
-        a, b, c = RPI4_COEFFICIENTS
-        return int(a * size_in_bytes ** 2 + b * size_in_bytes + c)
-    # Use simpler equation for small files.
-    return max(int(size_in_bytes // RPI4_PBF_BYTES_PER_SECOND), 0)
+async def fetch_manifest(url: str = None) -> dict:
+    """Fetch the map manifest JSON from the CDN and verify its GPG signature."""
+    url = url or MANIFEST_URL
+    sig_url = f'{url}.sig'
 
+    # Fetch manifest.
+    async with aiohttp_get(url, timeout=30) as response:
+        if response.status != 200:
+            raise RuntimeError(f'Failed to fetch manifest: HTTP {response.status}')
+        manifest_bytes = await response.content.read()
 
-def seconds_to_import_rpi5(size_in_bytes: int) -> int:
-    if size_in_bytes > 1_000_000_000:
-        # Use exponential curve for large files.
-        a, b, c = RPI5_COEFFICIENTS
-        return int(a * size_in_bytes ** 2 + b * size_in_bytes + c)
-    return max(int(size_in_bytes // RPI5_PBF_BYTES_PER_SECOND), 0)
+    # Fetch detached signature.
+    try:
+        async with aiohttp_get(sig_url, timeout=30) as response:
+            if response.status != 200:
+                raise RuntimeError(f'Failed to fetch manifest signature: HTTP {response.status}')
+            signature_bytes = await response.content.read()
+    except Exception as e:
+        logger.warning(f'Could not fetch manifest signature from {sig_url}: {e}')
+        raise RuntimeError(f'Manifest signature not available: {e}')
 
+    # Write to temp files and verify signature against on-disk files.
+    with tempfile.TemporaryDirectory() as tmpdir:
+        manifest_path = Path(tmpdir) / 'manifest.json'
+        sig_path = Path(tmpdir) / 'manifest.json.sig'
+        manifest_path.write_bytes(manifest_bytes)
+        sig_path.write_bytes(signature_bytes)
 
-def seconds_to_import(size_in_bytes: int, is_rpi_5: bool = IS_RPI5) -> int:
-    """Attempt to predict how long it will take an RPi4 to import a given PBF file."""
-    if is_rpi_5:
-        return seconds_to_import_rpi5(size_in_bytes)
+        if not await verify_gpg_signature(manifest_path, sig_path):
+            raise RuntimeError('Manifest GPG signature verification failed — file may be tampered with')
 
-    # Default to RPi4, because it's the most conservative estimate.
-    return seconds_to_import_rpi4(size_in_bytes)
+    logger.info('Manifest GPG signature verified successfully')
+    return json.loads(manifest_bytes)

modules/map/models.py

@@ -1,28 +1,2 @@
-from sqlalchemy import Column, Integer, Boolean, BigInteger
-
-from modules.map import lib
-from wrolpi import dates
-from wrolpi.common import Base, ModelHelper
-from wrolpi.media_path import MediaPathType
-
-
-class MapFile(Base, ModelHelper):
-    __tablename__ = 'map_file'
-    id = Column(Integer, primary_key=True)
-
-    path = Column(MediaPathType, nullable=False)
-    imported = Column(Boolean, default=False)
-    size = Column(BigInteger)
-
-    def __repr__(self):
-        return f'<MapFile {self.path} imported={self.imported}>'
-
-    def __json__(self) -> dict:
-        return dict(
-            id=self.id,
-            imported=self.imported,
-            path=self.path,
-            seconds_to_import=(seconds := lib.seconds_to_import(self.size)),
-            size=self.size,
-            time_to_import=dates.seconds_to_timestamp(seconds),
-        )
+# MapSubscription model has been removed.
+# Map subscriptions are managed via settings.regions on a single Download record.

modules/map/pins.py

@@ -0,0 +1,100 @@
+from dataclasses import dataclass, field
+from typing import List
+
+from wrolpi.common import ConfigFile, logger
+from wrolpi.dates import now
+from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
+
+logger = logger.getChild(__name__)
+
+
+@dataclass
+class MapPinsConfigValidator:
+    version: int = None
+    pins: list = field(default_factory=list)
+
+
+class MapPinsConfig(ConfigFile):
+    file_name = 'map_pins.yaml'
+    default_config = dict(
+        version=0,
+        pins=[],
+    )
+    validator = MapPinsConfigValidator
+
+    def import_config(self, file=None, send_events=False):
+        super().import_config(file, send_events)
+        self.successful_import = True
+
+    @property
+    def pins(self) -> List[dict]:
+        return list(self._config.get('pins', []))
+
+    @pins.setter
+    def pins(self, value: List[dict]):
+        self.update({'pins': value})
+
+    def _next_pin_id(self) -> int:
+        """Return the next available pin ID (max existing ID + 1)."""
+        pins = self.pins
+        if not pins:
+            return 1
+        return max(p.get('id', 0) for p in pins) + 1
+
+    def add_pin(self, lat: float, lon: float, label: str, color: str = 'red') -> dict:
+        pin = dict(
+            id=self._next_pin_id(),
+            lat=round(lat, 6),
+            lon=round(lon, 6),
+            label=label,
+            color=color,
+            created=str(now().date()),
+        )
+        pins = self.pins
+        pins.append(pin)
+        self.pins = pins
+        save_map_pins_config.activate_switch()
+        return pin
+
+    def delete_pin(self, pin_id: int) -> bool:
+        pins = self.pins
+        new_pins = [p for p in pins if p.get('id') != pin_id]
+        if len(new_pins) == len(pins):
+            return False
+        self.pins = new_pins
+        save_map_pins_config.activate_switch()
+        return True
+
+    def update_pin(self, pin_id: int, label: str = None, color: str = None) -> bool:
+        pins = self.pins
+        for pin in pins:
+            if pin.get('id') == pin_id:
+                if label is not None:
+                    pin['label'] = label
+                if color is not None:
+                    pin['color'] = color
+                self.pins = pins
+                save_map_pins_config.activate_switch()
+                return True
+        return False
+
+
+MAP_PINS_CONFIG: MapPinsConfig = MapPinsConfig()
+
+# Test override.
+TEST_MAP_PINS_CONFIG = None
+
+
+def get_map_pins_config() -> MapPinsConfig:
+    global TEST_MAP_PINS_CONFIG
+    if isinstance(TEST_MAP_PINS_CONFIG, ConfigFile):
+        return TEST_MAP_PINS_CONFIG
+    return MAP_PINS_CONFIG
+
+
+@register_switch_handler('save_map_pins_config')
+def save_map_pins_config():
+    get_map_pins_config().save()
+
+
+save_map_pins_config: ActivateSwitchMethod

modules/map/schema.py

@@ -1,7 +1,21 @@
 from dataclasses import dataclass
-from typing import List
 
 
 @dataclass
-class ImportPost:
-    files: List[str]
+class MapSubscribeRequest:
+    name: str
+    region: str
+
+
+@dataclass
+class MapPinRequest:
+    lat: float
+    lon: float
+    label: str
+    color: str = "red"
+
+
+@dataclass
+class MapPinUpdateRequest:
+    label: str = None
+    color: str = None

modules/map/static/fonts/OFL.txt

@@ -0,0 +1,93 @@
+Copyright 2022 The Noto Project Authors (https://github.com/notofonts)
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+https://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded, 
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

modules/map/static/index.html

@@ -0,0 +1,739 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
+    <title>WROLPi Map</title>
+    <link rel="stylesheet" href="/libs/maplibre-gl.css"/>
+    <style>
+        * { margin: 0; padding: 0; box-sizing: border-box; }
+        html, body { height: 100%; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; }
+
+        #map { width: 100%; height: 100%; }
+
+        /* Layer control panel */
+        #layer-panel {
+            position: absolute;
+            top: 10px;
+            left: 10px;
+            background: white;
+            border-radius: 8px;
+            box-shadow: 0 2px 8px rgba(0,0,0,0.2);
+            z-index: 1000;
+            max-height: calc(100vh - 20px);
+            overflow-y: auto;
+            min-width: 220px;
+            font-size: 14px;
+        }
+
+        #layer-toggle {
+            padding: 10px 14px;
+            cursor: pointer;
+            font-weight: 600;
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            border-bottom: 1px solid #eee;
+            user-select: none;
+        }
+        #layer-toggle:hover { background: #f5f5f5; }
+        #layer-toggle .arrow { transition: transform 0.2s; }
+        #layer-toggle .arrow.collapsed { transform: rotate(-90deg); }
+
+        #layer-content { padding: 8px 0; }
+        #layer-content.hidden { display: none; }
+
+        .layer-group { padding: 4px 14px; }
+        .layer-group-header {
+            display: flex;
+            align-items: center;
+            gap: 6px;
+            padding: 4px 0;
+            cursor: pointer;
+            font-weight: 500;
+            user-select: none;
+        }
+        .layer-group-header:hover { color: #1a73e8; }
+        .layer-group-header input { cursor: pointer; width: 16px; height: 16px; }
+
+        /* File selector */
+        #file-panel {
+            position: absolute;
+            top: 10px;
+            right: 50px;
+            background: white;
+            border-radius: 8px;
+            box-shadow: 0 2px 8px rgba(0,0,0,0.2);
+            z-index: 1000;
+            padding: 10px 14px;
+            font-size: 14px;
+            max-width: 300px;
+        }
+        #file-panel h3 { margin-bottom: 8px; font-size: 14px; }
+        .file-item {
+            display: flex;
+            align-items: center;
+            gap: 6px;
+            padding: 3px 0;
+        }
+        .file-item input { cursor: pointer; width: 16px; height: 16px; }
+        .file-item label { cursor: pointer; word-break: break-all; }
+
+        #no-files-msg {
+            color: #888;
+            font-style: italic;
+            padding: 4px 0;
+        }
+
+        /* Status bar */
+        #status {
+            position: absolute;
+            bottom: 30px;
+            left: 50%;
+            transform: translateX(-50%);
+            background: rgba(0,0,0,0.75);
+            color: white;
+            padding: 8px 16px;
+            border-radius: 6px;
+            font-size: 13px;
+            z-index: 1000;
+            display: none;
+        }
+
+        /* Context menu */
+        #context-menu {
+            position: absolute;
+            z-index: 1001;
+            background: white;
+            border-radius: 6px;
+            box-shadow: 0 2px 8px rgba(0,0,0,0.3);
+            padding: 4px 0;
+            font-size: 13px;
+            min-width: 180;
+            display: none;
+        }
+        #context-menu .coords {
+            padding: 6px 12px;
+            color: #666;
+            font-size: 12px;
+            border-bottom: 1px solid #eee;
+        }
+        #context-menu .action {
+            padding: 8px 12px;
+            cursor: pointer;
+            user-select: none;
+        }
+        #context-menu .action:hover { background: #f0f0f0; }
+    </style>
+</head>
+<body>
+<div id="map"></div>
+
+<div id="layer-panel">
+    <div id="layer-toggle">
+        <span>Layers</span>
+        <span class="arrow">&#9660;</span>
+    </div>
+    <div id="layer-content"></div>
+</div>
+
+<div id="file-panel">
+    <h3>Map Files</h3>
+    <div id="file-list">
+        <div id="no-files-msg">Scanning...</div>
+    </div>
+</div>
+
+<div id="status"></div>
+
+<div id="context-menu">
+    <div class="coords" id="context-coords"></div>
+    <div class="action" id="context-copy">Copy GPS coordinates</div>
+    <div class="action" id="context-export">Export as PNG</div>
+</div>
+
+<script src="/libs/pmtiles.js"></script>
+<script src="/libs/maplibre-gl.js"></script>
+<script src="/libs/protomaps-themes-base.js"></script>
+<script src="/libs/maplibre-contour-pmtiles.js"></script>
+<script>
+(function() {
+    // --- Constants ---
+    var MAP_OVERVIEW_URL = "/blobs/map-overview.pmtiles";
+    var TERRAIN_PREFIX = "terrain-";
+
+    function isPlanetFile(name) {
+        if (name.indexOf("planet-") === 0) return true;
+        return /^\d{8}\.pmtiles$/.test(name);
+    }
+    var terrainFileName = null;
+    var STORAGE_KEY_LAYERS = "wrolpi-map-layers";
+    var STORAGE_KEY_SCALE = "wrolpi-map-scale";
+    var DEFAULT_HIDDEN = {"Hillshade": true, "Contours": true};
+
+    // --- PMTiles protocol ---
+    var protocol = new pmtiles.Protocol();
+    maplibregl.addProtocol("pmtiles", protocol.tile);
+
+    // --- Layer group definitions ---
+    var LAYER_GROUPS = {
+        "Hillshade": ["hillshade"],
+        "Contours": ["contour-lines", "contour-labels"],
+        "Water": ["water"],
+        "Land Cover": [
+            "landcover", "landuse_park", "landuse_urban_green", "landuse_beach",
+            "landuse_aerodrome", "landuse_hospital", "landuse_industrial",
+            "landuse_pedestrian", "landuse_pier", "landuse_school", "landuse_zoo"
+        ],
+        "Buildings": ["buildings"],
+        "Roads - Highway": ["roads_highway"],
+        "Roads - Major": ["roads_major", "roads_link"],
+        "Roads - Minor": ["roads_minor", "roads_other"],
+        "Roads - Tunnels": ["roads_tunnels"],
+        "Roads - Bridges": ["roads_bridges"],
+        "Rail & Runways": ["roads_rail", "roads_runway", "roads_taxiway", "landuse_runway"],
+        "Boundaries": ["boundaries"],
+        "Labels": ["roads_labels", "places_", "address_label", "pois", "water_label", "water_waterway_label"],
+    };
+
+    // --- State ---
+    var map;
+    var scaleControl;
+    var demSource = null;
+    var activeSources = {};
+    var hasPlanetOverview = false;
+    var hasTerrain = false;
+    var contextMenuData = null;
+
+    // Load saved state from localStorage.
+    function getSavedVisibility() {
+        try {
+            var saved = localStorage.getItem(STORAGE_KEY_LAYERS);
+            if (saved) return JSON.parse(saved);
+        } catch(e) {}
+        var v = {};
+        for (var g in LAYER_GROUPS) v[g] = !DEFAULT_HIDDEN[g];
+        return v;
+    }
+    function getSavedScale() {
+        try { return localStorage.getItem(STORAGE_KEY_SCALE) || "metric"; } catch(e) { return "metric"; }
+    }
+
+    var groupVisibility = getSavedVisibility();
+    var currentScale = getSavedScale();
+
+    function getMapFlavor() {
+        return window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light";
+    }
+
+    // --- Build style ---
+    function buildStyle() {
+        var flavor = getMapFlavor();
+        var skipSecondary = {"background": true, "earth": true};
+        var sources = {};
+        var allLayers = [];
+        var si = 0;
+
+        // Planet overview as the primary base layer.
+        if (hasPlanetOverview) {
+            sources["map-overview"] = {
+                type: "vector",
+                url: "pmtiles://" + MAP_OVERVIEW_URL,
+            };
+            var baseLayers = protomaps_themes_base.default("map-overview", flavor);
+            allLayers = allLayers.concat(baseLayers);
+            si = 1;
+        }
+
+        // User files as additional sources.
+        var sourceNames = Object.keys(activeSources);
+        for (var fi = 0; fi < sourceNames.length; fi++) {
+            var filename = sourceNames[fi];
+            sources[filename] = {
+                type: "vector",
+                url: "pmtiles:///data/" + filename,
+            };
+            var fileLayers = protomaps_themes_base.default(filename, flavor);
+            if (si === 0 && fi === 0) {
+                allLayers = allLayers.concat(fileLayers);
+            } else {
+                for (var li = 0; li < fileLayers.length; li++) {
+                    if (skipSecondary[fileLayers[li].id]) continue;
+                    var layer = Object.assign({}, fileLayers[li]);
+                    layer.id = filename + ":" + layer.id;
+                    allLayers.push(layer);
+                }
+            }
+        }
+
+        // Terrain (hillshade + contours).
+        if (hasTerrain && demSource) {
+            var savedVis = getSavedVisibility();
+            var hillshadeVis = savedVis["Hillshade"] ? "visible" : "none";
+            var contourVis = savedVis["Contours"] ? "visible" : "none";
+            var isImperial = currentScale === "imperial";
+            var elevUnit = isImperial ? "ft" : "m";
+
+            sources["terrain-dem"] = {
+                type: "raster-dem",
+                url: "pmtiles://" + window.location.origin + "/data/" + terrainFileName,
+                encoding: "terrarium",
+                tileSize: 512,
+            };
+
+            var hillshadeSearch = allLayers.findIndex(function(l) { return l.id === "buildings"; });
+            var hillshadeIdx = hillshadeSearch === -1 ? allLayers.length : hillshadeSearch;
+            allLayers.splice(hillshadeIdx, 0, {
+                id: "hillshade",
+                type: "hillshade",
+                source: "terrain-dem",
+                layout: {visibility: hillshadeVis},
+                paint: {
+                    "hillshade-shadow-color": "#000000",
+                    "hillshade-illumination-anchor": "viewport",
+                    "hillshade-exaggeration": 0.3,
+                },
+            });
+
+            var contourUrl = demSource.contourProtocolUrl({
+                overzoom: 1,
+                contourLayer: "contours",
+                elevationKey: "ele",
+                levelKey: "level",
+                multiplier: isImperial ? 3.28084 : 1,
+                thresholds: isImperial ? {
+                    4: [1000, 5000], 5: [500, 2500], 6: [200, 1000], 7: [200, 1000], 8: [100, 500],
+                } : {
+                    4: [500, 2500], 5: [200, 1000], 6: [100, 500], 7: [100, 500], 8: [50, 200],
+                },
+            });
+            sources["contours"] = { type: "vector", tiles: [contourUrl], maxzoom: 8 };
+
+            var textColor = flavor === "dark" ? "rgba(210,180,140,0.9)" : "rgba(150,100,50,0.8)";
+            var haloColor = flavor === "dark" ? "rgba(0,0,0,0.8)" : "rgba(255,255,255,0.8)";
+
+            allLayers.push(
+                {
+                    id: "contour-lines", type: "line", source: "contours", "source-layer": "contours",
+                    filter: ["!=", ["get", "level"], 1],
+                    layout: {visibility: contourVis},
+                    paint: {"line-color": "rgba(150,100,50,0.3)", "line-width": 0.5},
+                },
+                {
+                    id: "contour-lines-major", type: "line", source: "contours", "source-layer": "contours",
+                    filter: ["==", ["get", "level"], 1],
+                    layout: {visibility: contourVis},
+                    paint: {"line-color": "rgba(150,100,50,0.5)", "line-width": 1},
+                },
+                {
+                    id: "contour-labels", type: "symbol", source: "contours", "source-layer": "contours",
+                    filter: ["==", ["get", "level"], 1],
+                    layout: {
+                        visibility: contourVis,
+                        "symbol-placement": "line",
+                        "text-field": ["concat", ["get", "ele"], elevUnit],
+                        "text-size": 10,
+                    },
+                    paint: {"text-color": textColor, "text-halo-color": haloColor, "text-halo-width": 1},
+                }
+            );
+        }
+
+        return {
+            version: 8,
+            glyphs: "/fonts/{fontstack}/{range}.pbf",
+            sprite: window.location.origin + "/sprites/" + flavor,
+            sources: sources,
+            layers: allLayers,
+        };
+    }
+
+    // --- Parse URL hash for coordinates ---
+    function parseHash() {
+        var hash = window.location.hash.replace("#", "");
+        if (!hash) return null;
+        var parts = hash.split(",");
+        if (parts.length >= 2) {
+            var lat = parseFloat(parts[0]);
+            var lon = parseFloat(parts[1]);
+            var zoom = parts.length >= 3 ? parseFloat(parts[2]) : 10;
+            if (!isNaN(lat) && !isNaN(lon) && lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180) {
+                return {lat: lat, lon: lon, zoom: zoom};
+            }
+        }
+        return null;
+    }
+
+    function initMap() {
+        var initial = parseHash();
+        var center = initial ? [initial.lon, initial.lat] : [0, 0];
+        var zoom = initial ? initial.zoom : 2;
+
+        map = new maplibregl.Map({
+            container: "map",
+            style: buildStyle(),
+            center: center,
+            zoom: zoom,
+            attributionControl: true,
+            preserveDrawingBuffer: true,
+        });
+
+        map.addControl(new maplibregl.NavigationControl(), "top-right");
+        map.addControl(new maplibregl.FullscreenControl(), "top-right");
+        scaleControl = new maplibregl.ScaleControl({maxWidth: 200, unit: currentScale});
+        map.addControl(scaleControl, "bottom-left");
+
+        var geolocate = new maplibregl.GeolocateControl({
+            positionOptions: {enableHighAccuracy: true},
+            trackUserLocation: true,
+            showUserHeading: true,
+            showAccuracyCircle: true,
+        });
+        map.addControl(geolocate, "top-right");
+
+        // Update URL hash on pan/zoom.
+        map.on("moveend", function() {
+            var c = map.getCenter();
+            var z = map.getZoom();
+            window.location.hash = c.lat.toFixed(4) + "," + c.lng.toFixed(4) + "," + z.toFixed(1);
+        });
+
+        // Right-click context menu.
+        map.on("contextmenu", function(e) {
+            contextMenuData = {lat: e.lngLat.lat, lon: e.lngLat.lng};
+            var menu = document.getElementById("context-menu");
+            var z = map.getZoom();
+            var d = z < 4 ? 1 : z < 8 ? 2 : z < 12 ? 3 : 4;
+            document.getElementById("context-coords").textContent =
+                contextMenuData.lat.toFixed(d) + ", " + contextMenuData.lon.toFixed(d);
+            menu.style.left = e.point.x + "px";
+            menu.style.top = e.point.y + "px";
+            menu.style.display = "block";
+        });
+        map.on("click", function() { document.getElementById("context-menu").style.display = "none"; });
+        map.on("movestart", function() { document.getElementById("context-menu").style.display = "none"; });
+
+        // Long-press context menu (mobile).
+        var longPressTimer = null;
+        var touchStartPos = null;
+        var canvas = map.getCanvas();
+        canvas.addEventListener("touchstart", function(e) {
+            if (e.touches.length !== 1) return;
+            touchStartPos = {x: e.touches[0].clientX, y: e.touches[0].clientY};
+            longPressTimer = setTimeout(function() {
+                var rect = canvas.getBoundingClientRect();
+                var px = touchStartPos.x - rect.left;
+                var py = touchStartPos.y - rect.top;
+                var lngLat = map.unproject([px, py]);
+                contextMenuData = {lat: lngLat.lat, lon: lngLat.lng};
+                var menu = document.getElementById("context-menu");
+                var z = map.getZoom();
+                var d = z < 4 ? 1 : z < 8 ? 2 : z < 12 ? 3 : 4;
+                document.getElementById("context-coords").textContent =
+                    contextMenuData.lat.toFixed(d) + ", " + contextMenuData.lon.toFixed(d);
+                menu.style.left = px + "px";
+                menu.style.top = py + "px";
+                menu.style.display = "block";
+            }, 500);
+        }, {passive: true});
+        canvas.addEventListener("touchmove", function(e) {
+            if (longPressTimer && touchStartPos) {
+                var dx = e.touches[0].clientX - touchStartPos.x;
+                var dy = e.touches[0].clientY - touchStartPos.y;
+                if (Math.abs(dx) > 10 || Math.abs(dy) > 10) {
+                    clearTimeout(longPressTimer);
+                    longPressTimer = null;
+                }
+            }
+        });
+        canvas.addEventListener("touchend", function() {
+            clearTimeout(longPressTimer);
+            longPressTimer = null;
+        });
+
+        map.on("load", function() {
+            buildLayerControls();
+        });
+    }
+
+    // --- Context menu copy ---
+    document.getElementById("context-copy").addEventListener("click", function() {
+        if (!contextMenuData || !map) return;
+        var z = map.getZoom();
+        var d = z < 4 ? 1 : z < 8 ? 2 : z < 12 ? 3 : 4;
+        var text = contextMenuData.lat.toFixed(d) + ", " + contextMenuData.lon.toFixed(d);
+        if (navigator.clipboard) {
+            navigator.clipboard.writeText(text).catch(function() {
+                window.prompt("Copy coordinates:", text);
+            });
+        } else {
+            window.prompt("Copy coordinates:", text);
+        }
+        document.getElementById("context-menu").style.display = "none";
+    });
+
+    // --- Context menu export ---
+    document.getElementById("context-export").addEventListener("click", function() {
+        if (!map) return;
+        document.getElementById("context-menu").style.display = "none";
+        // Force a fresh render, then capture.
+        map.once("render", function() {
+            var c = map.getCenter();
+            var z = map.getZoom();
+            var filename = "wrolpi-map_" + c.lat.toFixed(4) + "_" + c.lng.toFixed(4) + "_z" + z.toFixed(0) + ".png";
+            map.getCanvas().toBlob(function(blob) {
+                if (!blob) return;
+                var url = URL.createObjectURL(blob);
+                var a = document.createElement("a");
+                a.href = url;
+                a.download = filename;
+                a.click();
+                URL.revokeObjectURL(url);
+            });
+        });
+        map.triggerRepaint();
+    });
+
+    // --- Layer control ---
+    function matchesGroup(layerId, prefixes) {
+        var baseId = layerId.indexOf(":") >= 0 ? layerId.split(":").slice(1).join(":") : layerId;
+        return prefixes.some(function(prefix) {
+            return baseId.indexOf(prefix) >= 0;
+        });
+    }
+
+    function setGroupVisibility(groupName, visible) {
+        groupVisibility[groupName] = visible;
+        try { localStorage.setItem(STORAGE_KEY_LAYERS, JSON.stringify(groupVisibility)); } catch(e) {}
+        if (!map || !map.getStyle()) return;
+        var prefixes = LAYER_GROUPS[groupName];
+        var allLayers = map.getStyle().layers || [];
+        for (var i = 0; i < allLayers.length; i++) {
+            if (matchesGroup(allLayers[i].id, prefixes)) {
+                map.setLayoutProperty(allLayers[i].id, "visibility", visible ? "visible" : "none");
+            }
+        }
+    }
+
+    function buildLayerControls() {
+        var container = document.getElementById("layer-content");
+        container.innerHTML = "";
+
+        for (var groupName in LAYER_GROUPS) {
+            var div = document.createElement("div");
+            div.className = "layer-group";
+
+            var header = document.createElement("label");
+            header.className = "layer-group-header";
+
+            var checkbox = document.createElement("input");
+            checkbox.type = "checkbox";
+            checkbox.checked = groupVisibility[groupName] !== false;
+            checkbox.addEventListener("change", (function(name) {
+                return function(e) { setGroupVisibility(name, e.target.checked); };
+            })(groupName));
+
+            var label = document.createTextNode(groupName);
+            header.appendChild(checkbox);
+            header.appendChild(label);
+            div.appendChild(header);
+            container.appendChild(div);
+        }
+
+        // Scale unit toggle.
+        var scaleDiv = document.createElement("div");
+        scaleDiv.className = "layer-group";
+        scaleDiv.style.borderTop = "1px solid #eee";
+        scaleDiv.style.marginTop = "4px";
+        scaleDiv.style.paddingTop = "4px";
+
+        var scaleLabel = document.createElement("label");
+        scaleLabel.className = "layer-group-header";
+
+        var scaleCheckbox = document.createElement("input");
+        scaleCheckbox.type = "checkbox";
+        scaleCheckbox.checked = currentScale === "imperial";
+        scaleCheckbox.addEventListener("change", function(e) {
+            currentScale = e.target.checked ? "imperial" : "metric";
+            scaleText.textContent = e.target.checked ? "Imperial" : "Metric";
+            try { localStorage.setItem(STORAGE_KEY_SCALE, currentScale); } catch(e2) {}
+            if (map && scaleControl) {
+                map.removeControl(scaleControl);
+                scaleControl = new maplibregl.ScaleControl({maxWidth: 200, unit: currentScale});
+                map.addControl(scaleControl, "bottom-left");
+            }
+            // Rebuild style for contour unit change.
+            if (hasTerrain) reloadMap();
+        });
+
+        var scaleText = document.createTextNode(currentScale === "imperial" ? "Imperial" : "Metric");
+        scaleLabel.appendChild(scaleCheckbox);
+        scaleLabel.appendChild(scaleText);
+        scaleDiv.appendChild(scaleLabel);
+        container.appendChild(scaleDiv);
+    }
+
+    // Toggle layer panel
+    document.getElementById("layer-toggle").addEventListener("click", function() {
+        var content = document.getElementById("layer-content");
+        var arrow = this.querySelector(".arrow");
+        content.classList.toggle("hidden");
+        arrow.classList.toggle("collapsed");
+    });
+
+    // --- File management ---
+    function showStatus(msg) {
+        var el = document.getElementById("status");
+        el.textContent = msg;
+        el.style.display = "block";
+        setTimeout(function() { el.style.display = "none"; }, 3000);
+    }
+
+    function reloadMap() {
+        if (!map) { initMap(); return; }
+
+        var center = map.getCenter();
+        var zoom = map.getZoom();
+        var bearing = map.getBearing();
+        var pitch = map.getPitch();
+
+        map.setStyle(buildStyle());
+
+        map.once("style.load", function() {
+            map.setCenter(center);
+            map.setZoom(zoom);
+            map.setBearing(bearing);
+            map.setPitch(pitch);
+            // Re-apply visibility for groups that were toggled off.
+            for (var groupName in groupVisibility) {
+                if (!groupVisibility[groupName]) {
+                    setGroupVisibility(groupName, false);
+                }
+            }
+            buildLayerControls();
+        });
+    }
+
+    function toggleFile(filename, enabled) {
+        if (enabled) { activeSources[filename] = true; }
+        else { delete activeSources[filename]; }
+        reloadMap();
+        showStatus(enabled ? "Added " + filename : "Removed " + filename);
+    }
+
+    function renderFileList(files) {
+        var container = document.getElementById("file-list");
+        container.innerHTML = "";
+
+        if (files.length === 0) {
+            container.innerHTML = '<div id="no-files-msg">No .pmtiles files found in map/ directory</div>';
+            return;
+        }
+
+        files.forEach(function(file) {
+            var div = document.createElement("div");
+            div.className = "file-item";
+
+            var checkbox = document.createElement("input");
+            checkbox.type = "checkbox";
+            checkbox.id = "file-" + file;
+            checkbox.checked = !!activeSources[file];
+            checkbox.addEventListener("change", function(e) {
+                toggleFile(file, e.target.checked);
+            });
+
+            var label = document.createElement("label");
+            label.htmlFor = "file-" + file;
+            label.textContent = file;
+
+            div.appendChild(checkbox);
+            div.appendChild(label);
+            container.appendChild(div);
+        });
+    }
+
+    function scanFiles() {
+        // Fetch user's map files first to determine what base layer is needed.
+        fetch("/api/map/files")
+            .then(function(r) { return r.json(); })
+            .then(function(data) {
+                var allFiles = (data.files || []).sort(function(a, b) { return a.name.localeCompare(b.name); });
+
+                // Separate files into planet, terrain, and regional extracts.
+                var planetFile = null;
+                var regionFiles = [];
+                for (var i = 0; i < allFiles.length; i++) {
+                    var f = allFiles[i];
+                    if (f.name.startsWith(TERRAIN_PREFIX) && f.name.endsWith('.pmtiles')) {
+                        terrainFileName = f.name;
+                        hasTerrain = true;
+                    } else if (isPlanetFile(f.name)) {
+                        if (!planetFile || f.size > planetFile.size) {
+                            planetFile = f;
+                        }
+                    } else {
+                        regionFiles.push(f);
+                    }
+                }
+
+                var pmtilesFiles;
+                if (planetFile) {
+                    // Planet covers everything — sole vector source, no overview needed.
+                    pmtilesFiles = [planetFile.name];
+                    hasPlanetOverview = false;
+                    return pmtilesFiles;
+                }
+
+                // Use regional extracts.
+                pmtilesFiles = regionFiles.map(function(f) { return f.name; });
+
+                // Check for planet overview blob as low-zoom base layer.
+                return fetch(MAP_OVERVIEW_URL, {method: "HEAD"})
+                    .then(function(r) { hasPlanetOverview = r.ok; })
+                    .catch(function() { hasPlanetOverview = false; })
+                    .then(function() { return pmtilesFiles; });
+            })
+            .then(function(pmtilesFiles) {
+
+                // Initialize contour source if terrain available.
+                if (hasTerrain && !demSource) {
+                    demSource = new mlcontour.DemSource({
+                        url: "pmtiles://" + window.location.origin + "/data/" + terrainFileName,
+                        encoding: "terrarium",
+                        maxzoom: 8,
+                        worker: true,
+                    });
+                    demSource.setupMaplibre(maplibregl);
+                }
+
+                renderFileList(pmtilesFiles);
+
+                if (pmtilesFiles.length > 0 && Object.keys(activeSources).length === 0) {
+                    activeSources[pmtilesFiles[0]] = true;
+                    renderFileList(pmtilesFiles);
+                    reloadMap();
+                } else if (hasPlanetOverview || hasTerrain) {
+                    reloadMap();
+                } else if (Object.keys(activeSources).length === 0) {
+                    initMap();
+                }
+            })
+            .catch(function(err) {
+                console.error("Failed to scan files:", err);
+                document.getElementById("file-list").innerHTML =
+                    '<div id="no-files-msg">Error scanning for map files</div>';
+                initMap();
+            });
+    }
+
+    // --- Listen for system theme changes ---
+    window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change", function() {
+        if (map) reloadMap();
+    });
+
+    // --- Start ---
+    scanFiles();
+})();
+</script>
+</body>
+</html>

modules/map/static/sprites/dark.json

@@ -0,0 +1 @@
+{"NL:S-road-1char":{"height":21,"pixelRatio":1,"width":21,"x":0,"y":36},"NL:S-road-2char":{"height":21,"pixelRatio":1,"width":22,"x":38,"y":21},"NL:S-road-3char":{"height":21,"pixelRatio":1,"width":25,"x":202,"y":18},"NL:S-road-4char":{"height":21,"pixelRatio":1,"width":29,"x":173,"y":0},"NL:S-road-5char":{"height":21,"pixelRatio":1,"width":32,"x":79,"y":0},"US:I-1char":{"height":21,"pixelRatio":1,"width":20,"x":84,"y":21},"US:I-2char":{"height":21,"pixelRatio":1,"width":23,"x":227,"y":18},"US:I-3char":{"height":21,"pixelRatio":1,"width":30,"x":143,"y":0},"US:I-4char":{"height":21,"pixelRatio":1,"width":32,"x":111,"y":0},"US:I-5char":{"height":21,"pixelRatio":1,"width":34,"x":45,"y":0},"aerodrome":{"height":19,"pixelRatio":1,"width":19,"x":60,"y":39},"animal":{"height":19,"pixelRatio":1,"width":19,"x":21,"y":42},"arrow":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":57},"artwork":{"height":19,"pixelRatio":1,"width":19,"x":40,"y":42},"attraction":{"height":19,"pixelRatio":1,"width":19,"x":104,"y":21},"bar":{"height":19,"pixelRatio":1,"width":19,"x":123,"y":21},"beach":{"height":19,"pixelRatio":1,"width":19,"x":142,"y":21},"beauty":{"height":19,"pixelRatio":1,"width":19,"x":161,"y":21},"bench":{"height":19,"pixelRatio":1,"width":19,"x":180,"y":21},"books":{"height":19,"pixelRatio":1,"width":19,"x":199,"y":39},"building":{"height":17,"pixelRatio":1,"width":17,"x":193,"y":77},"bus_stop":{"height":19,"pixelRatio":1,"width":19,"x":218,"y":39},"cafe":{"height":19,"pixelRatio":1,"width":19,"x":237,"y":39},"capital":{"height":10,"pixelRatio":1,"width":10,"x":210,"y":77},"clothes":{"height":19,"pixelRatio":1,"width":19,"x":104,"y":40},"convenience":{"height":19,"pixelRatio":1,"width":19,"x":79,"y":42},"drinking_water":{"height":19,"pixelRatio":1,"width":19,"x":59,"y":58},"electronics":{"height":19,"pixelRatio":1,"width":19,"x":19,"y":61},"fast_food":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":76},"ferry_terminal":{"height":19,"pixelRatio":1,"width":19,"x":38,"y":61},"forest":{"height":19,"pixelRatio":1,"width":19,"x":123,"y":40},"garden":{"height":19,"pixelRatio":1,"width":19,"x":142,"y":40},"generic_shield-1char":{"height":18,"pixelRatio":1,"width":19,"x":235,"y":0},"generic_shield-2char":{"height":18,"pixelRatio":1,"width":24,"x":60,"y":21},"generic_shield-3char":{"height":18,"pixelRatio":1,"width":33,"x":202,"y":0},"generic_shield-4char":{"height":18,"pixelRatio":1,"width":38,"x":0,"y":18},"generic_shield-5char":{"height":18,"pixelRatio":1,"width":45,"x":0,"y":0},"library":{"height":19,"pixelRatio":1,"width":19,"x":161,"y":40},"marina":{"height":19,"pixelRatio":1,"width":19,"x":180,"y":40},"museum":{"height":19,"pixelRatio":1,"width":19,"x":199,"y":58},"park":{"height":19,"pixelRatio":1,"width":19,"x":218,"y":58},"peak":{"height":19,"pixelRatio":1,"width":19,"x":237,"y":58},"post_office":{"height":19,"pixelRatio":1,"width":19,"x":98,"y":59},"restaurant":{"height":19,"pixelRatio":1,"width":19,"x":78,"y":61},"school":{"height":19,"pixelRatio":1,"width":19,"x":57,"y":77},"stadium":{"height":19,"pixelRatio":1,"width":19,"x":19,"y":80},"supermarket":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":95},"theatre":{"height":19,"pixelRatio":1,"width":19,"x":38,"y":80},"toilets":{"height":19,"pixelRatio":1,"width":19,"x":117,"y":59},"townspot":{"height":10,"pixelRatio":1,"width":10,"x":220,"y":77},"train_station":{"height":19,"pixelRatio":1,"width":19,"x":136,"y":59},"university":{"height":19,"pixelRatio":1,"width":19,"x":155,"y":59},"zoo":{"height":19,"pixelRatio":1,"width":19,"x":174,"y":59}}

modules/map/static/sprites/dark@2x.json

@@ -0,0 +1 @@
+{"NL:S-road-1char":{"height":42,"pixelRatio":2,"width":42,"x":0,"y":72},"NL:S-road-2char":{"height":42,"pixelRatio":2,"width":44,"x":76,"y":42},"NL:S-road-3char":{"height":42,"pixelRatio":2,"width":50,"x":404,"y":36},"NL:S-road-4char":{"height":42,"pixelRatio":2,"width":58,"x":346,"y":0},"NL:S-road-5char":{"height":42,"pixelRatio":2,"width":64,"x":158,"y":0},"US:I-1char":{"height":42,"pixelRatio":2,"width":40,"x":168,"y":42},"US:I-2char":{"height":42,"pixelRatio":2,"width":46,"x":454,"y":36},"US:I-3char":{"height":42,"pixelRatio":2,"width":60,"x":286,"y":0},"US:I-4char":{"height":42,"pixelRatio":2,"width":64,"x":222,"y":0},"US:I-5char":{"height":42,"pixelRatio":2,"width":68,"x":90,"y":0},"aerodrome":{"height":38,"pixelRatio":2,"width":38,"x":120,"y":78},"animal":{"height":38,"pixelRatio":2,"width":38,"x":42,"y":84},"arrow":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":114},"artwork":{"height":38,"pixelRatio":2,"width":38,"x":80,"y":84},"attraction":{"height":38,"pixelRatio":2,"width":38,"x":208,"y":42},"bar":{"height":38,"pixelRatio":2,"width":38,"x":246,"y":42},"beach":{"height":38,"pixelRatio":2,"width":38,"x":284,"y":42},"beauty":{"height":38,"pixelRatio":2,"width":38,"x":322,"y":42},"bench":{"height":38,"pixelRatio":2,"width":38,"x":360,"y":42},"books":{"height":38,"pixelRatio":2,"width":38,"x":398,"y":78},"building":{"height":34,"pixelRatio":2,"width":34,"x":386,"y":154},"bus_stop":{"height":38,"pixelRatio":2,"width":38,"x":436,"y":78},"cafe":{"height":38,"pixelRatio":2,"width":38,"x":474,"y":78},"capital":{"height":20,"pixelRatio":2,"width":20,"x":420,"y":154},"clothes":{"height":38,"pixelRatio":2,"width":38,"x":208,"y":80},"convenience":{"height":38,"pixelRatio":2,"width":38,"x":158,"y":84},"drinking_water":{"height":38,"pixelRatio":2,"width":38,"x":118,"y":116},"electronics":{"height":38,"pixelRatio":2,"width":38,"x":38,"y":122},"fast_food":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":152},"ferry_terminal":{"height":38,"pixelRatio":2,"width":38,"x":76,"y":122},"forest":{"height":38,"pixelRatio":2,"width":38,"x":246,"y":80},"garden":{"height":38,"pixelRatio":2,"width":38,"x":284,"y":80},"generic_shield-1char":{"height":36,"pixelRatio":2,"width":38,"x":470,"y":0},"generic_shield-2char":{"height":36,"pixelRatio":2,"width":48,"x":120,"y":42},"generic_shield-3char":{"height":36,"pixelRatio":2,"width":66,"x":404,"y":0},"generic_shield-4char":{"height":36,"pixelRatio":2,"width":76,"x":0,"y":36},"generic_shield-5char":{"height":36,"pixelRatio":2,"width":90,"x":0,"y":0},"library":{"height":38,"pixelRatio":2,"width":38,"x":322,"y":80},"marina":{"height":38,"pixelRatio":2,"width":38,"x":360,"y":80},"museum":{"height":38,"pixelRatio":2,"width":38,"x":398,"y":116},"park":{"height":38,"pixelRatio":2,"width":38,"x":436,"y":116},"peak":{"height":38,"pixelRatio":2,"width":38,"x":474,"y":116},"post_office":{"height":38,"pixelRatio":2,"width":38,"x":196,"y":118},"restaurant":{"height":38,"pixelRatio":2,"width":38,"x":156,"y":122},"school":{"height":38,"pixelRatio":2,"width":38,"x":114,"y":154},"stadium":{"height":38,"pixelRatio":2,"width":38,"x":38,"y":160},"supermarket":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":190},"theatre":{"height":38,"pixelRatio":2,"width":38,"x":76,"y":160},"toilets":{"height":38,"pixelRatio":2,"width":38,"x":234,"y":118},"townspot":{"height":20,"pixelRatio":2,"width":20,"x":440,"y":154},"train_station":{"height":38,"pixelRatio":2,"width":38,"x":272,"y":118},"university":{"height":38,"pixelRatio":2,"width":38,"x":310,"y":118},"zoo":{"height":38,"pixelRatio":2,"width":38,"x":348,"y":118}}

modules/map/static/sprites/light.json

@@ -0,0 +1 @@
+{"NL:S-road-1char":{"height":21,"pixelRatio":1,"width":21,"x":0,"y":36},"NL:S-road-2char":{"height":21,"pixelRatio":1,"width":22,"x":38,"y":21},"NL:S-road-3char":{"height":21,"pixelRatio":1,"width":25,"x":202,"y":18},"NL:S-road-4char":{"height":21,"pixelRatio":1,"width":29,"x":173,"y":0},"NL:S-road-5char":{"height":21,"pixelRatio":1,"width":32,"x":79,"y":0},"US:I-1char":{"height":21,"pixelRatio":1,"width":20,"x":84,"y":21},"US:I-2char":{"height":21,"pixelRatio":1,"width":23,"x":227,"y":18},"US:I-3char":{"height":21,"pixelRatio":1,"width":30,"x":143,"y":0},"US:I-4char":{"height":21,"pixelRatio":1,"width":32,"x":111,"y":0},"US:I-5char":{"height":21,"pixelRatio":1,"width":34,"x":45,"y":0},"aerodrome":{"height":19,"pixelRatio":1,"width":19,"x":60,"y":39},"animal":{"height":19,"pixelRatio":1,"width":19,"x":21,"y":42},"arrow":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":57},"artwork":{"height":19,"pixelRatio":1,"width":19,"x":40,"y":42},"attraction":{"height":19,"pixelRatio":1,"width":19,"x":104,"y":21},"bar":{"height":19,"pixelRatio":1,"width":19,"x":123,"y":21},"beach":{"height":19,"pixelRatio":1,"width":19,"x":142,"y":21},"beauty":{"height":19,"pixelRatio":1,"width":19,"x":161,"y":21},"bench":{"height":19,"pixelRatio":1,"width":19,"x":180,"y":21},"books":{"height":19,"pixelRatio":1,"width":19,"x":199,"y":39},"building":{"height":17,"pixelRatio":1,"width":17,"x":193,"y":77},"bus_stop":{"height":19,"pixelRatio":1,"width":19,"x":218,"y":39},"cafe":{"height":19,"pixelRatio":1,"width":19,"x":237,"y":39},"capital":{"height":10,"pixelRatio":1,"width":10,"x":210,"y":77},"clothes":{"height":19,"pixelRatio":1,"width":19,"x":104,"y":40},"convenience":{"height":19,"pixelRatio":1,"width":19,"x":79,"y":42},"drinking_water":{"height":19,"pixelRatio":1,"width":19,"x":59,"y":58},"electronics":{"height":19,"pixelRatio":1,"width":19,"x":19,"y":61},"fast_food":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":76},"ferry_terminal":{"height":19,"pixelRatio":1,"width":19,"x":38,"y":61},"forest":{"height":19,"pixelRatio":1,"width":19,"x":123,"y":40},"garden":{"height":19,"pixelRatio":1,"width":19,"x":142,"y":40},"generic_shield-1char":{"height":18,"pixelRatio":1,"width":19,"x":235,"y":0},"generic_shield-2char":{"height":18,"pixelRatio":1,"width":24,"x":60,"y":21},"generic_shield-3char":{"height":18,"pixelRatio":1,"width":33,"x":202,"y":0},"generic_shield-4char":{"height":18,"pixelRatio":1,"width":38,"x":0,"y":18},"generic_shield-5char":{"height":18,"pixelRatio":1,"width":45,"x":0,"y":0},"library":{"height":19,"pixelRatio":1,"width":19,"x":161,"y":40},"marina":{"height":19,"pixelRatio":1,"width":19,"x":180,"y":40},"museum":{"height":19,"pixelRatio":1,"width":19,"x":199,"y":58},"park":{"height":19,"pixelRatio":1,"width":19,"x":218,"y":58},"peak":{"height":19,"pixelRatio":1,"width":19,"x":237,"y":58},"post_office":{"height":19,"pixelRatio":1,"width":19,"x":98,"y":59},"restaurant":{"height":19,"pixelRatio":1,"width":19,"x":78,"y":61},"school":{"height":19,"pixelRatio":1,"width":19,"x":57,"y":77},"stadium":{"height":19,"pixelRatio":1,"width":19,"x":19,"y":80},"supermarket":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":95},"theatre":{"height":19,"pixelRatio":1,"width":19,"x":38,"y":80},"toilets":{"height":19,"pixelRatio":1,"width":19,"x":117,"y":59},"townspot":{"height":10,"pixelRatio":1,"width":10,"x":220,"y":77},"train_station":{"height":19,"pixelRatio":1,"width":19,"x":136,"y":59},"university":{"height":19,"pixelRatio":1,"width":19,"x":155,"y":59},"zoo":{"height":19,"pixelRatio":1,"width":19,"x":174,"y":59}}

modules/map/static/sprites/light@2x.json

@@ -0,0 +1 @@
+{"NL:S-road-1char":{"height":42,"pixelRatio":2,"width":42,"x":0,"y":72},"NL:S-road-2char":{"height":42,"pixelRatio":2,"width":44,"x":76,"y":42},"NL:S-road-3char":{"height":42,"pixelRatio":2,"width":50,"x":404,"y":36},"NL:S-road-4char":{"height":42,"pixelRatio":2,"width":58,"x":346,"y":0},"NL:S-road-5char":{"height":42,"pixelRatio":2,"width":64,"x":158,"y":0},"US:I-1char":{"height":42,"pixelRatio":2,"width":40,"x":168,"y":42},"US:I-2char":{"height":42,"pixelRatio":2,"width":46,"x":454,"y":36},"US:I-3char":{"height":42,"pixelRatio":2,"width":60,"x":286,"y":0},"US:I-4char":{"height":42,"pixelRatio":2,"width":64,"x":222,"y":0},"US:I-5char":{"height":42,"pixelRatio":2,"width":68,"x":90,"y":0},"aerodrome":{"height":38,"pixelRatio":2,"width":38,"x":120,"y":78},"animal":{"height":38,"pixelRatio":2,"width":38,"x":42,"y":84},"arrow":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":114},"artwork":{"height":38,"pixelRatio":2,"width":38,"x":80,"y":84},"attraction":{"height":38,"pixelRatio":2,"width":38,"x":208,"y":42},"bar":{"height":38,"pixelRatio":2,"width":38,"x":246,"y":42},"beach":{"height":38,"pixelRatio":2,"width":38,"x":284,"y":42},"beauty":{"height":38,"pixelRatio":2,"width":38,"x":322,"y":42},"bench":{"height":38,"pixelRatio":2,"width":38,"x":360,"y":42},"books":{"height":38,"pixelRatio":2,"width":38,"x":398,"y":78},"building":{"height":34,"pixelRatio":2,"width":34,"x":386,"y":154},"bus_stop":{"height":38,"pixelRatio":2,"width":38,"x":436,"y":78},"cafe":{"height":38,"pixelRatio":2,"width":38,"x":474,"y":78},"capital":{"height":20,"pixelRatio":2,"width":20,"x":420,"y":154},"clothes":{"height":38,"pixelRatio":2,"width":38,"x":208,"y":80},"convenience":{"height":38,"pixelRatio":2,"width":38,"x":158,"y":84},"drinking_water":{"height":38,"pixelRatio":2,"width":38,"x":118,"y":116},"electronics":{"height":38,"pixelRatio":2,"width":38,"x":38,"y":122},"fast_food":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":152},"ferry_terminal":{"height":38,"pixelRatio":2,"width":38,"x":76,"y":122},"forest":{"height":38,"pixelRatio":2,"width":38,"x":246,"y":80},"garden":{"height":38,"pixelRatio":2,"width":38,"x":284,"y":80},"generic_shield-1char":{"height":36,"pixelRatio":2,"width":38,"x":470,"y":0},"generic_shield-2char":{"height":36,"pixelRatio":2,"width":48,"x":120,"y":42},"generic_shield-3char":{"height":36,"pixelRatio":2,"width":66,"x":404,"y":0},"generic_shield-4char":{"height":36,"pixelRatio":2,"width":76,"x":0,"y":36},"generic_shield-5char":{"height":36,"pixelRatio":2,"width":90,"x":0,"y":0},"library":{"height":38,"pixelRatio":2,"width":38,"x":322,"y":80},"marina":{"height":38,"pixelRatio":2,"width":38,"x":360,"y":80},"museum":{"height":38,"pixelRatio":2,"width":38,"x":398,"y":116},"park":{"height":38,"pixelRatio":2,"width":38,"x":436,"y":116},"peak":{"height":38,"pixelRatio":2,"width":38,"x":474,"y":116},"post_office":{"height":38,"pixelRatio":2,"width":38,"x":196,"y":118},"restaurant":{"height":38,"pixelRatio":2,"width":38,"x":156,"y":122},"school":{"height":38,"pixelRatio":2,"width":38,"x":114,"y":154},"stadium":{"height":38,"pixelRatio":2,"width":38,"x":38,"y":160},"supermarket":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":190},"theatre":{"height":38,"pixelRatio":2,"width":38,"x":76,"y":160},"toilets":{"height":38,"pixelRatio":2,"width":38,"x":234,"y":118},"townspot":{"height":20,"pixelRatio":2,"width":20,"x":440,"y":154},"train_station":{"height":38,"pixelRatio":2,"width":38,"x":272,"y":118},"university":{"height":38,"pixelRatio":2,"width":38,"x":310,"y":118},"zoo":{"height":38,"pixelRatio":2,"width":38,"x":348,"y":118}}

modules/map/test/test_api.py

@@ -1,135 +1,130 @@
 import json
 from http import HTTPStatus
-from itertools import zip_longest
 
-import mock
 import pytest
 
 
-async def assert_map_file_status(async_client, expected, status_code=HTTPStatus.OK):
+@pytest.mark.asyncio
+async def test_get_files_empty(async_client, test_directory):
+    """An empty map directory returns no files."""
     request, response = await async_client.get('/api/map/files')
-    assert response.status_code == status_code
-    for pbf, expected_ in zip_longest(response.json['files'], expected):
-        imported = 'to be imported' if expected_['imported'] else 'to NOT be imported'
-        if not pbf['imported'] == expected_['imported']:
-            raise AssertionError(f'Expected {pbf["path"]} {imported}')
-        assert pbf['path'].endswith(expected_['path'])
-    assert response.json['pending'] is None
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['files'] == []
 
 
 @pytest.mark.asyncio
-@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
-@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
-async def test_status_and_import(async_client, test_session, make_files_structure, mock_run_command):
-    """PBF files can be imported, and the status of the import can be monitored."""
-    pbf1, pbf2, dump1 = make_files_structure([
-        'map/pbf/some-country.osm.pbf',
-        'map/pbf/other-country.osm.pbf',
-        'map/dump/dumpy.dump',
+async def test_get_files(async_client, test_directory, make_files_structure):
+    """PMTiles files in the map directory are listed."""
+    make_files_structure([
+        'map/usa.pmtiles',
+        'map/oregon.pmtiles',
+        'map/not-a-map.txt',
     ])
 
-    # The PBF files can be found, but have not yet been imported.
-    expected = [
-        {'imported': False, 'path': 'map/dump/dumpy.dump'},
-        {'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
-        {'imported': False, 'path': 'map/pbf/some-country.osm.pbf'},
-    ]
-    await assert_map_file_status(async_client, expected)
-
-    body = {'files': [str(pbf1), str(pbf2)]}
-    request, response = await async_client.post('/api/map/import', content=json.dumps(body))
-    assert response.status_code == HTTPStatus.NO_CONTENT
-
-    # Can import multiple pbf files.
-    expected = [
-        {'imported': False, 'path': 'map/dump/dumpy.dump'},
-        {'imported': True, 'path': 'map/pbf/other-country.osm.pbf'},
-        {'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
-    ]
-    await assert_map_file_status(async_client, expected)
-
-    body = {'files': [str(pbf1), ], }
-    request, response = await async_client.post('/api/map/import', content=json.dumps(body))
-    assert response.status_code == HTTPStatus.NO_CONTENT
-
-    # Importing one pbf resets the import statuses.
-    expected = [
-        {'imported': False, 'path': 'map/dump/dumpy.dump'},
-        {'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
-        {'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
-    ]
-    await assert_map_file_status(async_client, expected)
-
-    body = {'files': [str(dump1), ]}
-    request, response = await async_client.post('/api/map/import', content=json.dumps(body))
-    assert response.status_code == HTTPStatus.NO_CONTENT
-
-    # Importing a dump also resets statuses.
-    expected = [
-        {'imported': True, 'path': 'map/dump/dumpy.dump'},
-        {'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
-        {'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
-    ]
-    await assert_map_file_status(async_client, expected)
+    request, response = await async_client.get('/api/map/files')
+    assert response.status_code == HTTPStatus.OK
+    files = response.json['files']
+    names = [f['name'] for f in files]
+    assert 'oregon.pmtiles' in names
+    assert 'usa.pmtiles' in names
+    assert 'not-a-map.txt' not in names
 
 
 @pytest.mark.asyncio
-@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
-@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
-async def test_multiple_import(async_client, test_session, test_directory, make_files_structure, mock_run_command):
-    """Multiple PBFs can be imported.  Importing a second time overwrites the previous imports."""
-    pbf1, pbf2, pbf3, dump = make_files_structure([
-        'map/pbf/country1.osm.pbf',
-        'map/pbf/country2.osm.pbf',
-        'map/pbf/country3.osm.pbf',
-        'map/dumpy.dump',
-    ])
+async def test_delete_file(async_client, test_directory, make_files_structure):
+    """A PMTiles file can be deleted."""
+    pmtiles_file, = make_files_structure(['map/oregon.pmtiles'])
 
-    body = {'files': [str(pbf2), str(pbf1)]}
-    request, response = await async_client.post('/api/map/import', content=json.dumps(body))
+    # File exists.
+    request, response = await async_client.get('/api/map/files')
+    assert len(response.json['files']) == 1
+
+    # Delete the file.
+    request, response = await async_client.delete('/api/map/files/oregon.pmtiles')
     assert response.status_code == HTTPStatus.NO_CONTENT
 
-    # Two PBFs can be merged and imported.
-    expected = [
-        {'imported': False, 'path': 'map/dumpy.dump'},
-        {'imported': True, 'path': 'map/pbf/country1.osm.pbf'},
-        {'imported': True, 'path': 'map/pbf/country2.osm.pbf'},
-        {'imported': False, 'path': 'map/pbf/country3.osm.pbf'},
-    ]
-    await assert_map_file_status(async_client, expected)
-
-    body = {'files': [str(pbf3), ]}
-    request, response = await async_client.post('/api/map/import', content=json.dumps(body))
-    assert response.status_code == HTTPStatus.NO_CONTENT
-
-    # New imports overwrite old imports.
-    expected = [
-        {'imported': False, 'path': 'map/dumpy.dump'},
-        {'imported': False, 'path': 'map/pbf/country1.osm.pbf'},
-        {'imported': False, 'path': 'map/pbf/country2.osm.pbf'},
-        {'imported': True, 'path': 'map/pbf/country3.osm.pbf'},
-    ]
-    await assert_map_file_status(async_client, expected)
-
-    # Can't mix PBF and dump.
-    body = {'files': [str(dump), ]}
-    request, response = await async_client.post('/api/map/import', content=json.dumps(body))
-    assert response.status_code == HTTPStatus.NO_CONTENT
-
-    # Dump does not overwrite.
-    expected = [
-        {'imported': True, 'path': 'map/dumpy.dump'},
-        {'imported': False, 'path': 'map/pbf/country1.osm.pbf'},
-        {'imported': False, 'path': 'map/pbf/country2.osm.pbf'},
-        {'imported': True, 'path': 'map/pbf/country3.osm.pbf'},
-    ]
-    await assert_map_file_status(async_client, expected)
+    # File is gone.
+    request, response = await async_client.get('/api/map/files')
+    assert len(response.json['files']) == 0
 
 
 @pytest.mark.asyncio
-async def test_empty_import(async_client, test_session, test_directory):
-    """Some files must be requested."""
-    body = {'files': []}
-    request, response = await async_client.post('/api/map/import', content=json.dumps(body))
+async def test_delete_nonexistent_file(async_client, test_directory):
+    """Deleting a nonexistent file returns 404."""
+    request, response = await async_client.delete('/api/map/files/nonexistent.pmtiles')
+    assert response.status_code == HTTPStatus.NOT_FOUND
+
+
+@pytest.mark.asyncio
+async def test_delete_path_traversal(async_client, test_directory, make_files_structure):
+    """Path traversal attempts are rejected."""
+    request, response = await async_client.delete('/api/map/files/..%2F..%2Fetc%2Fpasswd')
+    # Path traversal is blocked — either 400 (caught by validation) or 404 (file not found).
+    assert response.status_code in (HTTPStatus.BAD_REQUEST, HTTPStatus.NOT_FOUND)
+
+
+@pytest.mark.asyncio
+async def test_get_subscriptions_empty(async_client, test_session, test_directory):
+    """Catalog and empty subscriptions can be fetched."""
+    request, response = await async_client.get('/api/map/subscribe')
+    assert response.status_code == HTTPStatus.OK
+    assert 'catalog' in response.json
+    assert 'subscriptions' in response.json
+    assert len(response.json['catalog']) > 0
+    assert len(response.json['subscriptions']) == 0
+
+
+@pytest.mark.asyncio
+async def test_subscribe_and_unsubscribe(async_client, test_session, test_directory):
+    """A map region can be subscribed to and unsubscribed from."""
+    body = json.dumps({'name': 'Alaska', 'region': 'us-alaska'})
+    request, response = await async_client.post('/api/map/subscribe', content=body)
+    assert response.status_code == HTTPStatus.CREATED
+
+    # Subscription appears in the list.
+    request, response = await async_client.get('/api/map/subscribe')
+    assert response.status_code == HTTPStatus.OK
+    regions = [s['region'] for s in response.json['subscriptions']]
+    assert 'us-alaska' in regions
+
+    # Unsubscribe.
+    request, response = await async_client.delete('/api/map/subscribe/us-alaska')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+
+    # Subscription is gone.
+    request, response = await async_client.get('/api/map/subscribe')
+    assert len(response.json['subscriptions']) == 0
+
+
+@pytest.mark.asyncio
+async def test_subscribe_multiple_regions(async_client, test_session, test_directory):
+    """Multiple regions share a single Download record."""
+    body = json.dumps({'name': 'Alaska', 'region': 'us-alaska'})
+    request, response = await async_client.post('/api/map/subscribe', content=body)
+    assert response.status_code == HTTPStatus.CREATED
+
+    body = json.dumps({'name': 'United States (West)', 'region': 'us-west'})
+    request, response = await async_client.post('/api/map/subscribe', content=body)
+    assert response.status_code == HTTPStatus.CREATED
+
+    request, response = await async_client.get('/api/map/subscribe')
+    regions = [s['region'] for s in response.json['subscriptions']]
+    assert 'us-alaska' in regions
+    assert 'us-west' in regions
+
+    # Unsubscribe one — the other remains.
+    request, response = await async_client.delete('/api/map/subscribe/us-alaska')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+
+    request, response = await async_client.get('/api/map/subscribe')
+    regions = [s['region'] for s in response.json['subscriptions']]
+    assert 'us-alaska' not in regions
+    assert 'us-west' in regions
+
+
+@pytest.mark.asyncio
+async def test_subscribe_invalid_region(async_client, test_session, test_directory):
+    """Subscribing to an invalid region returns an error."""
+    body = json.dumps({'name': 'Atlantis', 'region': 'atlantis'})
+    request, response = await async_client.post('/api/map/subscribe', content=body)
     assert response.status_code == HTTPStatus.BAD_REQUEST
-    assert 'validate' in response.json['message']

modules/map/test/test_lib.py

@@ -6,98 +6,159 @@ from modules.map import lib
 from wrolpi.common import get_wrolpi_config
 
 
-def test_get_map_paths(test_directory, make_files_structure):
-    pbf, dump = make_files_structure([
-        'map/pbf/map.osm.pbf',
-        'map/pbf/map.dump',
+def test_get_pmtiles_files_empty(test_directory):
+    """No files returns an empty list."""
+    assert lib.get_pmtiles_files() == []
+
+
+def test_get_pmtiles_files(test_directory, make_files_structure):
+    """PMTiles files are found, other files are ignored."""
+    make_files_structure([
+        'map/usa.pmtiles',
+        'map/oregon.pmtiles',
+        'map/old-data.osm.pbf',
+        'map/readme.txt',
     ])
 
-    with mock.patch('modules.map.lib.subprocess.check_output') as mock_check_output:
-        mock_check_output.return_value = b'OpenStreetMap Protocolbuffer Binary Format'
-        assert lib.get_map_paths() == [pbf, ]
+    files = lib.get_pmtiles_files()
+    names = [f['name'] for f in files]
+    assert 'oregon.pmtiles' in names
+    assert 'usa.pmtiles' in names
+    assert 'old-data.osm.pbf' not in names
+    assert 'readme.txt' not in names
 
-    with mock.patch('modules.map.lib.subprocess.check_output') as mock_check_output:
-        mock_check_output.return_value = b'PostgreSQL custom database dump'
-        assert lib.get_map_paths() == [dump, ]
+    # Each file has expected keys.
+    for f in files:
+        assert 'name' in f
+        assert 'path' in f
+        assert 'size' in f
+        assert 'mtime' in f
+
+
+def test_delete_pmtiles_file(test_directory, make_files_structure):
+    """A PMTiles file can be deleted."""
+    pmtiles_file, = make_files_structure(['map/oregon.pmtiles'])
+    assert pmtiles_file.is_file()
+
+    assert lib.delete_pmtiles_file('oregon.pmtiles') is True
+    assert not pmtiles_file.is_file()
+
+
+def test_delete_nonexistent_file(test_directory):
+    """Deleting a nonexistent file returns False."""
+    assert lib.delete_pmtiles_file('nonexistent.pmtiles') is False
+
+
+def test_delete_path_traversal(test_directory, make_files_structure):
+    """Path traversal is rejected."""
+    make_files_structure(['map/safe.pmtiles'])
+
+    with pytest.raises(ValueError):
+        lib.delete_pmtiles_file('../../etc/passwd')
+
+
+def test_get_map_catalog(test_directory):
+    """Catalog returns predefined regions."""
+    catalog = lib.get_map_catalog()
+    assert len(catalog) > 0
+    names = [r['name'] for r in catalog]
+    assert 'Alaska' in names  # Verify a known region exists
+    for region in catalog:
+        assert 'name' in region
+        assert 'region' in region
+        assert 'size_estimate' in region
 
 
 @pytest.mark.asyncio
-async def test_run_import_command(test_directory, mock_run_command, run_command_bad_result):
-    """
-    Import map files.  Files to import are checked for validity before importing.  Any errors returned by
-    asyncio.create_subprocess_shell are caught and reported.
-    """
-    foo_pbf = test_directory / 'foo.osm.pbf'
-    bar_pbf = test_directory / 'bar.osm.pbf'
+async def test_subscribe(async_client, test_session, test_directory):
+    """A subscription adds a region to the download settings."""
+    await lib.subscribe(test_session, 'Alaska', 'us-alaska')
+    subs = lib.get_map_subscriptions(test_session)
+    regions = [s['region'] for s in subs]
+    assert 'us-alaska' in regions
+
+
+@pytest.mark.asyncio
+async def test_subscribe_multiple(async_client, test_session, test_directory):
+    """Multiple subscriptions share one Download."""
+    await lib.subscribe(test_session, 'Alaska', 'us-alaska')
+    await lib.subscribe(test_session, 'United States (West)', 'us-west')
+    subs = lib.get_map_subscriptions(test_session)
+    regions = [s['region'] for s in subs]
+    assert 'us-alaska' in regions
+    assert 'us-west' in regions
+
+
+@pytest.mark.asyncio
+async def test_subscribe_duplicate(async_client, test_session, test_directory):
+    """Subscribing to the same region twice doesn't duplicate."""
+    await lib.subscribe(test_session, 'Alaska', 'us-alaska')
+    await lib.subscribe(test_session, 'Alaska', 'us-alaska')
+    subs = lib.get_map_subscriptions(test_session)
+    assert len([s for s in subs if s['region'] == 'us-alaska']) == 1
+
+
+@pytest.mark.asyncio
+async def test_subscribe_invalid(async_client, test_session, test_directory):
+    """Invalid region name raises ValueError."""
     with pytest.raises(ValueError):
-        await lib.run_import_command(foo_pbf)
-
-    foo_pbf.touch()
-    with pytest.raises(ValueError):
-        await lib.run_import_command(foo_pbf)
-
-    dump_file = test_directory / 'dumpy.dump'
-    dump_file.touch()
-
-    with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
-        # Run the import, it succeeds.
-        mock_is_pbf_file.return_value = True
-        await lib.run_import_command(foo_pbf)
-
-    # Can import more than one pbf.
-    with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
-        mock_is_pbf_file.return_value = True
-        await lib.run_import_command(foo_pbf, bar_pbf)
-
-    with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
-        # Run the import, it fails.
-        mock_run_command(run_command_bad_result)
-        mock_is_pbf_file.return_value = True
-        with pytest.raises(ValueError):
-            await lib.run_import_command(foo_pbf)
-
-    with pytest.raises(ValueError) as e:
-        await lib.run_import_command()
-    assert 'Must import a file' in str(e)
-
-    # Can't import more than one dump.
-    with pytest.raises(ValueError) as e:
-        await lib.run_import_command(dump_file, dump_file)
-    assert 'more than one' in str(e)
+        await lib.subscribe(test_session, 'Atlantis', 'atlantis')
 
 
-@pytest.mark.parametrize('size,expected', [
-    (0, 0),
-    (-1, 0),
-    (17737381, 101),
-    (63434267, 361),
-    (87745484, 500),
-    (116318111, 662),
-    (136372996, 777),
-    (1936075318, 29032),
-    (2676094489, 46705),
-    (3392375001, 67202),
-    (11346305075, 519006),
-])
-def test_seconds_to_import_rpi4(size, expected):
-    assert lib.seconds_to_import(size) == expected
+@pytest.mark.asyncio
+async def test_unsubscribe(async_client, test_session, test_directory):
+    """Unsubscribing removes a region."""
+    await lib.subscribe(test_session, 'Alaska', 'us-alaska')
+    await lib.subscribe(test_session, 'United States (West)', 'us-west')
+
+    await lib.unsubscribe(test_session, 'us-alaska')
+    subs = lib.get_map_subscriptions(test_session)
+    regions = [s['region'] for s in subs]
+    assert 'us-alaska' not in regions
+    assert 'us-west' in regions
 
 
-@pytest.mark.parametrize('size,expected', [
-    (0, 0),
-    (-1, 0),
-    (17737381, 130),
-    (63434267, 466),
-    (87745484, 645),
-    (116318111, 855),
-    (136372996, 1002),
-    (1936075318, 10873),
-    (2676094489, 18313),
-    (3392375001, 27253),
-    (11346305075, 241574),
-])
-def test_seconds_to_import_rpi5(size, expected):
-    assert lib.seconds_to_import(size, True) == expected
+@pytest.mark.asyncio
+async def test_unsubscribe_last_deletes_download(async_client, test_session, test_directory):
+    """Unsubscribing the last region deletes the Download."""
+    await lib.subscribe(test_session, 'Alaska', 'us-alaska')
+    await lib.unsubscribe(test_session, 'us-alaska')
+    assert lib.get_map_subscriptions(test_session) == []
+    assert lib._get_map_download(test_session) is None
+
+
+@pytest.mark.asyncio
+async def test_subscribe_renews_download(async_client, test_session, test_directory):
+    """Subscribing renews the download so it runs immediately."""
+    await lib.subscribe(test_session, 'Alaska', 'us-alaska')
+    download = lib._get_map_download(test_session)
+    assert download.status == 'new'
+
+    # Simulate the download completing.
+    download.status = 'complete'
+    test_session.commit()
+
+    # Subscribe to another region — download should be renewed.
+    await lib.subscribe(test_session, 'United States (West)', 'us-west')
+    download = lib._get_map_download(test_session)
+    assert download.status == 'new'
+
+
+@pytest.mark.asyncio
+async def test_subscribe_saves_config(async_client, test_session, test_directory):
+    """Subscribing triggers a download config save."""
+    with mock.patch('modules.map.lib.save_downloads_config') as mock_save:
+        await lib.subscribe(test_session, 'Alaska', 'us-alaska')
+        mock_save.activate_switch.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_unsubscribe_saves_config(async_client, test_session, test_directory):
+    """Unsubscribing triggers a download config save."""
+    await lib.subscribe(test_session, 'Alaska', 'us-alaska')
+    with mock.patch('modules.map.lib.save_downloads_config') as mock_save:
+        await lib.unsubscribe(test_session, 'us-alaska')
+        mock_save.activate_switch.assert_called_once()
 
 
 def test_get_custom_map_directory(async_client, test_directory, test_wrolpi_config):

modules/videos/downloader.py

@@ -26,7 +26,7 @@ from wrolpi.dates import now
 from wrolpi.db import get_db_session
 from wrolpi.downloader import Downloader, Download, DownloadResult, \
     clear_download_progress, _parse_size, make_progress_callback, get_download_cache_config
-from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError
+from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError, ValidationError
 from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
 from wrolpi.files.models import FileGroup
 from wrolpi.files.worker import file_worker
@@ -34,7 +34,8 @@ from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
 from .channel.lib import create_channel, get_channel
 from .common import get_no_channel_directory, update_view_counts_and_censored, \
     ffmpeg_video_complete
-from .errors import UnknownChannel
+from .errors import UnknownChannel, ChannelDirectoryConflict, ChannelNameConflict, ChannelURLConflict, \
+    ChannelSourceIdConflict
 from .cookies import cookies_unlocked, cookies_for_download
 from .lib import get_videos_downloader_config, get_effective_video_settings, CHANNEL_INHERITABLE_SETTINGS, YDL, \
     ydl_logger, format_videos_destination, get_yt_dlp_http_headers, get_yt_dlp_sleep_opts
@@ -419,7 +420,8 @@ class ChannelDownloader(Downloader, ABC):
             raise ValueError(f'Could not find name')
         channel_source_id = info.get('channel_id') or info.get('id')
         channel_url = info.get('channel_url') or info.get('uploader_url') or download.url
-        channel = get_or_create_channel(session, channel_source_id, channel_url, name, channel_tag_name)
+        channel = get_or_create_channel(session, channel_source_id, channel_url, name, channel_tag_name,
+                                        destination=download.destination)
         channel.dict()  # get all attributes while we have the session.
 
         # Link this channel download to the channel's collection.
@@ -872,8 +874,8 @@ class VideoDownloader(Downloader, ABC):
             # Try to find the channel via info_json from yt-dlp.
             try:
                 channel = get_or_create_channel(session, source_id=channel_source_id, url=channel_url,
-                                                name=channel_name,
-                                                tag_name=channel_tag_name)
+                                                name=channel_name, tag_name=channel_tag_name,
+                                                destination=download.destination)
                 if channel:
                     found_channel = 'yt_dlp'
                     logger.debug(f'Found a channel with source_id {channel=}')
@@ -1096,11 +1098,12 @@ video_downloader = VideoDownloader()
 
 
 def get_or_create_channel(session: Session, source_id: str = None, url: str = None, name: str = None,
-                          tag_name: str = None) -> Channel:
+                          tag_name: str = None, destination: pathlib.Path = None) -> Channel:
     """
     Attempt to find a Channel using the provided params.  The params are in order of reliability.
 
-    Creates a new Channel if one cannot be found.
+    Creates a new Channel if one cannot be found.  If a destination is provided, the Channel will be created in that
+    directory instead of the default videos destination.
     """
     try:
         channel = get_channel(session, source_id=source_id, url=url, name=name, return_dict=False)
@@ -1113,18 +1116,25 @@ def get_or_create_channel(session: Session, source_id: str = None, url: str = No
     if not name:
         raise UnknownChannel(f'Cannot create channel without a name')
 
-    # Channel does not exist.  Create one in the video directory.
-    channel_directory = format_videos_destination(name, tag_name, url)
+    # Channel does not exist.  Create one using the destination if provided, otherwise the default video directory.
+    if destination:
+        channel_directory = get_absolute_media_path(destination)
+    else:
+        channel_directory = format_videos_destination(name, tag_name, url)
     if not channel_directory.is_dir():
         channel_directory.mkdir(parents=True)
     data = ChannelPostRequest(
         source_id=source_id,
         name=name,
         url=url,
-        directory=str(channel_directory.relative_to(get_media_directory())),
+        directory=str(channel_directory),
         tag_name=tag_name,
     )
-    channel = create_channel(session, data=data, return_dict=False)
+    try:
+        channel = create_channel(session, data=data, return_dict=False)
+    except (ValidationError, ChannelDirectoryConflict, ChannelNameConflict, ChannelURLConflict, ChannelSourceIdConflict) as e:
+        raise UnrecoverableDownloadError(
+            f'Cannot create channel {name!r}: conflicts with an existing channel') from e
     # Create the directory now that the channel is approved.
     channel_directory.mkdir(exist_ok=True)
     # Get all properties while we have the session.

modules/videos/test/test_downloader.py

@@ -14,8 +14,8 @@ from modules.videos.downloader import VideoDownloader, \
 from modules.videos.lib import get_videos_downloader_config
 from modules.videos.models import Channel, Video
 from wrolpi.conftest import test_directory, await_switches
-from wrolpi.downloader import Download, DownloadResult
-from wrolpi.errors import InvalidDownload
+from wrolpi.downloader import Download, DownloadResult, get_download_manager_config
+from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
 from wrolpi.vars import PROJECT_DIR
 
 example_video_json = {
@@ -308,6 +308,84 @@ async def test_get_or_create_channel_youtube_channel_url(async_client, test_sess
     assert result.id == channel.id
 
 
+@pytest.mark.asyncio
+async def test_get_or_create_channel_with_destination(async_client, test_session, test_directory):
+    """A new Channel should be created at the destination directory when provided."""
+    destination = test_directory / 'custom/my_channel'
+
+    channel = get_or_create_channel(test_session, source_id='abc', name='My Channel',
+                                    url='https://example.com', destination=destination)
+    assert channel.name == 'My Channel'
+    assert channel.directory == destination
+    assert destination.is_dir()
+
+
+@pytest.mark.asyncio
+async def test_get_or_create_channel_destination_conflict(async_client, test_session, test_directory):
+    """Creating a channel at a destination that conflicts with an existing channel should raise
+    UnrecoverableDownloadError."""
+    from wrolpi.collections import Collection
+
+    # Create an existing channel at the destination directory.
+    existing_dir = test_directory / 'videos/existing'
+    existing_dir.mkdir(parents=True)
+    collection = Collection(name='Existing Channel', kind='channel', directory=existing_dir)
+    test_session.add(collection)
+    test_session.flush([collection])
+    channel = Channel(collection_id=collection.id, source_id='existing')
+    test_session.add(channel)
+    test_session.commit()
+
+    # Attempting to create a new channel at the same directory should fail.
+    with pytest.raises(UnrecoverableDownloadError):
+        get_or_create_channel(test_session, source_id='new_id', name='New Channel',
+                              url='https://example.com', destination=existing_dir)
+
+
+@pytest.mark.asyncio
+async def test_get_or_create_channel_destination_nested_conflict(async_client, test_session, test_directory):
+    """Creating a channel at a destination nested inside an existing channel's directory should raise
+    UnrecoverableDownloadError."""
+    from wrolpi.collections import Collection
+
+    parent_dir = test_directory / 'videos/parent_channel'
+    parent_dir.mkdir(parents=True)
+    collection = Collection(name='Parent Channel', kind='channel', directory=parent_dir)
+    test_session.add(collection)
+    test_session.flush([collection])
+    channel = Channel(collection_id=collection.id, source_id='parent')
+    test_session.add(channel)
+    test_session.commit()
+
+    # Nested directory should conflict.
+    nested_dir = parent_dir / 'nested'
+    with pytest.raises(UnrecoverableDownloadError):
+        get_or_create_channel(test_session, source_id='nested_id', name='Nested Channel',
+                              url='https://example.net', destination=nested_dir)
+
+
+@pytest.mark.asyncio
+async def test_get_or_create_channel_existing_ignores_destination(async_client, test_session, test_directory):
+    """When a channel already exists (matched by source_id), the destination parameter should be ignored."""
+    from wrolpi.collections import Collection
+
+    existing_dir = test_directory / 'videos/existing'
+    existing_dir.mkdir(parents=True)
+    collection = Collection(name='Existing', kind='channel', directory=existing_dir)
+    test_session.add(collection)
+    test_session.flush([collection])
+    channel = Channel(collection_id=collection.id, source_id='match_me', url='https://example.com')
+    test_session.add(channel)
+    test_session.commit()
+
+    # Even with a different destination, the existing channel is returned.
+    other_dir = test_directory / 'videos/other'
+    result = get_or_create_channel(test_session, source_id='match_me', name='Existing',
+                                   destination=other_dir)
+    assert result.id == channel.id
+    assert result.directory == existing_dir
+
+
 def test_channel_downloader_hidden(video_download_manager):
     """
     ChannelDownloader should not be presented to the User.
@@ -1050,3 +1128,40 @@ async def test_fetch_video_duration_cache_eviction(test_download_cache_config):
     assert 'https://example.com/video0' not in durations_dict
     # Second oldest (video1) is still present.
     assert durations_dict['https://example.com/video1'] == 1
+
+
+@pytest.mark.asyncio
+async def test_channel_download_preserves_skip_list(test_session, test_directory, simple_channel,
+                                                    video_download_manager, video_file,
+                                                    mock_video_extract_info, mock_video_prepare_filename,
+                                                    await_switches, mock_video_process_runner,
+                                                    test_download_manager_config):
+    """A skipped video URL should remain in the skip list when its channel downloads again.
+
+    Regression: child video downloads created by ChannelDownloader used reset_attempts=True which
+    caused create_downloads to remove the URL from the skip list."""
+    skipped_url = 'https://youtube.com/watch?v=video_1_url'
+    url = 'https://www.youtube.com/c/ExampleChannel/videos'
+
+    # Skip video_1_url — simulating user skipping a video.
+    get_download_manager_config().skip_urls = [skipped_url]
+    assert get_download_manager_config().skip_urls == [skipped_url]
+
+    mock_video_prepare_filename.return_value = video_file
+    mock_video_extract_info.return_value = example_channel_json
+    with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
+        mock_get_channel.return_value = simple_channel
+        simple_channel.get_or_create_download(test_session, url, 60)
+        # Download channel catalog.
+        await video_download_manager.wait_for_all_downloads()
+        # Download the child videos.
+        await video_download_manager.wait_for_all_downloads()
+
+    # The skipped URL must still be in the skip list.
+    assert skipped_url in get_download_manager_config().skip_urls, \
+        'Channel download removed a URL from the skip list!'
+
+    # Only video_2 should have been downloaded, not the skipped video_1.
+    once_downloads = video_download_manager.get_once_downloads(test_session)
+    video_downloads = [i for i in once_downloads if 'watch' in i.url]
+    assert {i.url for i in video_downloads} == {'https://youtube.com/watch?v=video_2_url'}

modules/zim/models.py

@@ -401,7 +401,8 @@ class ZimSubscription(Base):
     def change_download(self, session: Session, url: str, frequency: int):
         # A Zim file is found using the KiwixCatalogDownloader, then handled by the KiwixZimDownloader.
         old_url = self.download.url if self.download else url
-        download = download_manager.get_or_create_download(session, old_url, reset_attempts=True)
+        download = download_manager.get_or_create_download(session, old_url, reset_attempts=True,
+                                                               override_skip=True)
         download.url = url
         download.frequency = frequency
         download.downloader = 'kiwix_catalog'

pi-gen/build.sh

@@ -37,37 +37,6 @@ if [ $EUID != 0 ]; then
   exit $?
 fi
 
-if [ ! -f "${SCRIPT_DIR}/stage2/04-wrolpi/files/map-db-gis.dump" ]; then
-  echo "stage2/04-wrolpi/files/map-db-gis.dump does not exist!"
-  exit 1
-fi
-
-# Validate dump file format - must be PostgreSQL custom format (PGDMP magic)
-DUMP_FILE="${SCRIPT_DIR}/stage2/04-wrolpi/files/map-db-gis.dump"
-DUMP_HEADER=$(head -c 5 "${DUMP_FILE}" 2>/dev/null | od -An -tx1 | tr -d ' \n')
-
-if [[ "${DUMP_HEADER:0:6}" == "504744" ]]; then
-  echo "map-db-gis.dump: Valid PostgreSQL custom format"
-elif [[ "${DUMP_HEADER:0:6}" == "1f8b08" ]]; then
-  # Check if gzipped content is custom format
-  INNER_HEADER=$(gunzip -c "${DUMP_FILE}" 2>/dev/null | head -c 5 | od -An -tx1 | tr -d ' \n')
-  if [[ "${INNER_HEADER:0:6}" == "504744" ]]; then
-    echo "map-db-gis.dump: Valid gzipped PostgreSQL custom format"
-  else
-    echo "ERROR: map-db-gis.dump is gzipped but does NOT contain PostgreSQL custom format!"
-    echo "Inner header: ${INNER_HEADER} (expected to start with 504744 for 'PGDMP')"
-    echo "The dump was likely created incorrectly. Use:"
-    echo "  docker run --rm -v /path/to/file.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper > map-db-gis.dump"
-    exit 1
-  fi
-else
-  echo "ERROR: map-db-gis.dump has invalid format!"
-  echo "Header: ${DUMP_HEADER} (expected to start with 504744 for 'PGDMP' or 1f8b08 for gzip)"
-  echo "Create the dump using:"
-  echo "  docker run --rm -v /path/to/file.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper > map-db-gis.dump"
-  exit 1
-fi
-
 CHROOTFS="${BUILD_DIR}/work/WROLPi/stage0/rootfs"
 
 cleanup() {
@@ -99,6 +68,31 @@ trap cleanup EXIT
 set -e
 set -x
 
+# Cache directory for downloaded blobs (persists across builds).
+BLOB_CACHE="${SCRIPT_DIR}/stage2/04-wrolpi/files"
+mkdir -p "${BLOB_CACHE}"
+
+# Download map fonts (skip if cached).
+if [ ! -f "${BLOB_CACHE}/map-fonts.tar.gz" ]; then
+  echo "Downloading map fonts..."
+  mkdir -p /tmp/map-fonts-dl
+  curl -fsSL https://github.com/protomaps/basemaps-assets/archive/refs/heads/main.tar.gz \
+    | tar -xz --strip-components=1 -C /tmp/map-fonts-dl basemaps-assets-main/fonts
+  tar -czf "${BLOB_CACHE}/map-fonts.tar.gz" -C /tmp/map-fonts-dl fonts
+  rm -rf /tmp/map-fonts-dl
+fi
+[[ -f "${BLOB_CACHE}/map-fonts.tar.gz" && -s "${BLOB_CACHE}/map-fonts.tar.gz" ]] || \
+  (echo "ERROR: Failed to download map fonts!" && exit 1)
+
+# Download map overview (skip if cached).
+if [ ! -f "${BLOB_CACHE}/map-overview.pmtiles" ]; then
+  echo "Downloading map overview..."
+  curl -fsSL https://wrolpi.nyc3.cdn.digitaloceanspaces.com/maps/map-overview.pmtiles \
+    -o "${BLOB_CACHE}/map-overview.pmtiles"
+fi
+[[ -f "${BLOB_CACHE}/map-overview.pmtiles" && -s "${BLOB_CACHE}/map-overview.pmtiles" ]] || \
+  (echo "ERROR: Failed to download map overview!" && exit 1)
+
 # Get the latest pi-gen code.
 wget https://github.com/RPi-Distro/pi-gen/archive/refs/heads/bookworm-arm64.zip -O /var/tmp/bookworm-arm64.zip
 unzip /var/tmp/bookworm-arm64.zip -d /var/tmp
@@ -106,7 +100,7 @@ mv /var/tmp/pi-gen-bookworm-arm64 /var/tmp/wrolpi-build-pi-gen
 
 # Copy the configuration and build files into the pi-gen directory.
 cp "${SCRIPT_DIR}/config.txt" "${BUILD_DIR}/config.txt"
-echo "WROLPI_BRANCH=${BRANCH}" >> "${BUILD_DIR}/config.txt"
+echo "export WROLPI_BRANCH=${BRANCH}" >> "${BUILD_DIR}/config.txt"
 rsync -a "${SCRIPT_DIR}"/stage2/* "${BUILD_DIR}/stage2/"
 
 # We only need to build the Lite and Desktop images.

pi-gen/stage2/04-wrolpi/01-packages-nr

@@ -1,6 +1,3 @@
-apache2
-apache2-dev
-apache2-doc
 apt-transport-https
 aria2
 autoconf
@@ -24,7 +21,6 @@ fonts-noto-hinted
 fonts-noto-unhinted
 gcc
 gcc-doc
-gdal-bin
 git
 git-doc
 gnupg-agent
@@ -35,57 +31,35 @@ iotop
 iperf3
 kiwix
 kiwix-tools
-libagg-dev
-libapache2-mod-tile
 libboost-all-dev
 libbz2-dev
 libcairo2-dev
-libcairomm-1.0-dev
 libcurl4-gnutls-dev
 libffi-dev
 libfreetype6-dev
-libgdal-dev
 libgeos++-dev
 libgeos-dev
 libicu-dev
-libiniparser-dev
-liblua5.1-0-dev
-liblua5.2-dev
-libmapnik-dev
-libmemcached-dev
 libpq-dev
 libproj-dev
-librados-dev
-libsensors-dev
 libssl-dev
 libtiff5-dev
 libtool
 libxml2-dev
-lua5.1
-mapnik-utils
 netcat-openbsd
 network-manager
 nicstat
 nodejs
 npm
-osm2pgsql
-osmium-tool
 pkg-config
-postgis
-postgis-doc
 postgresql-15
-postgresql-15-postgis-3
-postgresql-15-postgis-3-scripts
 postgresql-doc
-protobuf-c-compiler
 pv
 python3-dev
 python3-doc
 python3-full
-python3-mapnik
 python3-pip
 python3-venv
-renderd
 rsync
 samba
 smartmontools

pi-gen/stage2/04-wrolpi/03-run-chroot.sh

@@ -25,7 +25,7 @@ useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
 sed -i 's/port =.*/port = 5432/' /etc/postgresql/15/main/postgresql.conf
 
 # Install Node console commands.
-npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0
+npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6
 
 # Install Deno runtime.
 DENO_VERSION="v2.2.4"
@@ -68,6 +68,7 @@ apt-get install -y caddy
 # Configure Caddy.
 mkdir -p /etc/caddy
 cp /opt/wrolpi/etc/raspberrypios/Caddyfile /etc/caddy/Caddyfile
+mkdir -p /var/www
 cp /opt/wrolpi/etc/raspberrypios/50x.html /var/www/50x.html
 cp /opt/wrolpi/etc/raspberrypios/landing.html /var/www/landing.html
 

pi-gen/stage2/04-wrolpi/04-run-chroot.sh

@@ -1,23 +1,10 @@
 #! /usr/bin/env bash
-# Install and configure Renderd for Open Street Map.
+# Configure WROLPi on Raspberry Pi OS.
 set -e
 set -x
 
-# Create postgres cluster for map.
-pg_createcluster 15 map --port=5433 -e utf8
-
-# Create mapnik config.
-git clone https://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto
-git config --global --add safe.directory /opt/openstreetmap-carto
-(cd /opt/openstreetmap-carto && git fetch && git checkout master && git reset --hard origin/master && git pull --ff)
-chown -R wrolpi:wrolpi /opt/openstreetmap-carto
-# Append port line after dbname configuration line.
-sed -i '/dbname: "gis"/a \    port: 5433' /opt/openstreetmap-carto/project.mml
-(cd /opt/openstreetmap-carto/ && carto project.mml >mapnik.xml)
-
-# All users can access wrolpi and map database.
+# All users can access the wrolpi database.
 cat >/etc/skel/.pgpass <<'EOF'
-127.0.0.1:5433:gis:_renderd:wrolpi
 127.0.0.1:5432:wrolpi:wrolpi:wrolpi
 EOF
 chmod 0600 /etc/skel/.pgpass
@@ -30,23 +17,11 @@ EOF
 mkdir /etc/skel/Desktop
 cp /opt/wrolpi/etc/raspberrypios/*desktop /etc/skel/Desktop
 
-# Configure renderd (map).
-cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf
-# Configure Apache2 to listen on 8084.  Import renderd into apache.
-cp /opt/wrolpi/etc/raspberrypios/ports.conf /etc/apache2/ports.conf
-cp /opt/wrolpi/etc/raspberrypios/000-default.conf /etc/apache2/sites-available/000-default.conf
-cp /opt/wrolpi/etc/raspberrypios/mod_tile.conf /etc/apache2/conf-enabled/mod_tile.conf
-# Copy Leaflet files to Apache's directory so they can be used offline.
-cp /opt/wrolpi/etc/raspberrypios/index.html \
-  /opt/wrolpi/modules/map/leaflet.js \
-  /opt/wrolpi/modules/map/leaflet.css /var/www/html/
-chmod 644 /var/www/html/*
-
-# Allow wrolpi user to delete map tiles.
-usermod -aG _renderd wrolpi
-
-[ ! -d /var/cache/renderd/tiles ] && mkdir /var/cache/renderd/tiles
-chown -R _renderd:_renderd /var/cache/renderd/tiles
+# Verify blob files were copied in by 02-run.sh (downloaded by build.sh).
+[[ -f /opt/wrolpi-blobs/map-fonts.tar.gz && -s /opt/wrolpi-blobs/map-fonts.tar.gz ]] || \
+  (echo "ERROR: map-fonts.tar.gz missing from /opt/wrolpi-blobs!" && exit 1)
+[[ -f /opt/wrolpi-blobs/map-overview.pmtiles && -s /opt/wrolpi-blobs/map-overview.pmtiles ]] || \
+  (echo "ERROR: map-overview.pmtiles missing from /opt/wrolpi-blobs!" && exit 1)
 
 # Install WROLPi Help.
 /opt/wrolpi/scripts/install_help_service.sh
@@ -56,10 +31,11 @@ mkdir -p /media/wrolpi
 chown -R wrolpi:wrolpi /media/wrolpi /home/wrolpi /opt/wrolpi*
 
 # Protect blob files from accidental modification or deletion.
-# These files are critical for initializing WROLPi.
-chown -R root:root /opt/wrolpi-blobs
-chmod 444 /opt/wrolpi-blobs/*
-chattr +i /opt/wrolpi-blobs/*
+if [ -d /opt/wrolpi-blobs ] && ls /opt/wrolpi-blobs/* >/dev/null 2>&1; then
+  chown -R root:root /opt/wrolpi-blobs
+  chmod 444 /opt/wrolpi-blobs/*
+  chattr +i /opt/wrolpi-blobs/*
+fi
 
 set +x
 

pi-gen/stage2/04-wrolpi/files/.gitignore

@@ -1,2 +1,4 @@
 *.dump.gz
 *.dump
+*.pmtiles
+*.tar.gz

repair.sh

@@ -23,8 +23,6 @@ systemctl stop wrolpi-api.service || :
 systemctl stop wrolpi-app.service || :
 systemctl stop wrolpi-kiwix.service || :
 systemctl stop caddy || :
-systemctl stop renderd || :
-systemctl stop apache2 || :
 
 # Clear Python bytecode cache from virtual environments to fix potential corruption.
 echo "Clearing Python bytecode cache..."
@@ -38,7 +36,6 @@ find /opt/wrolpi-help/venv -name "*.pyc" -delete 2>/dev/null || :
 # Create the WROLPi user
 grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
 [ -f /home/wrolpi/.pgpass ] || cat >/home/wrolpi/.pgpass <<'EOF'
-127.0.0.1:5433:gis:_renderd:wrolpi
 127.0.0.1:5432:wrolpi:wrolpi:wrolpi
 EOF
 chmod 0600 /home/wrolpi/.pgpass
@@ -51,12 +48,20 @@ chown -R wrolpi:wrolpi /opt/wrolpi
 # Copy configs to system.
 mkdir -p /etc/caddy
 cp /opt/wrolpi/etc/raspberrypios/Caddyfile /etc/caddy/Caddyfile
+mkdir -p /var/www
 cp /opt/wrolpi/etc/raspberrypios/50x.html /var/www/50x.html
 cp /opt/wrolpi/etc/raspberrypios/landing.html /var/www/landing.html
 
 # Generate certificate for HTTPS.
 /opt/wrolpi/scripts/generate_certificates.sh
 
+# Install map fonts from blob (needed for MapLibre map labels).
+MAP_FONTS_BLOB=/opt/wrolpi-blobs/map-fonts.tar.gz
+if [ -f "${MAP_FONTS_BLOB}" ]; then
+  mkdir -p /opt/wrolpi/modules/map/static/fonts
+  tar -xzf "${MAP_FONTS_BLOB}" -C /opt/wrolpi/modules/map/static/
+fi
+
 # Start Caddy quickly so user can access Controller from React UI
 systemctl start caddy || :
 
@@ -119,20 +124,6 @@ fi
 # Start Controller so user can monitor repair in UI.
 systemctl start wrolpi-controller
 
-# Copy config files necessary for map.
-cp -r /opt/wrolpi/etc/raspberrypios/postgresql@15-map.service.d /etc/systemd/system/
-cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf
-cp /opt/wrolpi/etc/raspberrypios/renderd.service /lib/systemd/system/renderd.service
-# Configure Apache2 to listen on 8084.
-cp /opt/wrolpi/etc/raspberrypios/ports.conf /etc/apache2/ports.conf
-# Copy Leaflet files to Apache's directory so they can be used offline.
-cp /opt/wrolpi/etc/raspberrypios/index.html \
-  /opt/wrolpi/modules/map/leaflet.js \
-  /opt/wrolpi/modules/map/leaflet.css /var/www/html/
-chmod 644 /var/www/html/*
-a2enmod headers
-
-systemctl enable renderd
 /usr/bin/systemctl daemon-reload
 
 # WROLPi needs a few privileged commands.
@@ -146,40 +137,6 @@ visudo -c -f /etc/sudoers.d/90-wrolpi
 # wrolpi user is superuser so they can import maps.
 sudo -iu postgres psql -c "alter user wrolpi with superuser"
 
-# Configure renderd.
-if [ ! -d /opt/openstreetmap-carto ]; then
-  echo "/opt/openstreetmap-carto does not exist!  Has install completed?" && exit 1
-fi
-
-cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf
-# Enable mod_tile.
-cp /opt/wrolpi/etc/raspberrypios/mod_tile.conf /etc/apache2/conf-available/mod_tile.conf
-
-carto -v
-
-# Initialize the map if it has not been initialized.
-systemctl enable postgresql@15-map.service || :
-systemctl start postgresql@15-map.service || :
-sudo -iu postgres psql -c '\l' --port=5433 | grep -q gis || yes | /opt/wrolpi/scripts/initialize_map_db.sh
-# Update openstreetmap-carto to use the new database.
-grep -q 'port: 5433' /opt/openstreetmap-carto/project.mml || (
-  # Append port line after dbname configuration line.
-  sed -i '/dbname: "gis"/a \    port: 5433' /opt/openstreetmap-carto/project.mml
-  (cd /opt/openstreetmap-carto/ && carto project.mml >mapnik.xml)
-)
-chown -R _renderd:_renderd /opt/openstreetmap-carto
-# Disable JIT as recommended by mod_tile
-sed -i 's/#jit =.*/jit = off/' /etc/postgresql/15/map/postgresql.conf
-
-cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf
-# Configure Apache2 to listen on 8084.
-cp /opt/wrolpi/etc/raspberrypios/ports.conf /etc/apache2/ports.conf
-if [[ ${rpi} == true ]]; then
-  cp /opt/wrolpi/etc/raspberrypios/000-default.conf /etc/apache2/sites-available/000-default.conf
-else
-  cp /opt/wrolpi/etc/debian12/000-default.conf /etc/apache2/sites-available/000-default.conf
-fi
-
 # Create the media directory.  This should be mounted by the maintainer.
 [ -d /media/wrolpi ] || mkdir /media/wrolpi
 # Create config directory if external drive is mounted, and is empty.
@@ -192,6 +149,8 @@ fi
 
 # Change owner of the media directory, ignore any errors because the drive may be fat/exfat/etc.
 chown wrolpi:wrolpi /media/wrolpi 2>/dev/null || echo "Ignoring failure to change media directory permissions."
+# Ensure the config directory is owned by wrolpi so the API can write config files.
+[ -d /media/wrolpi/config ] && chown -R wrolpi:wrolpi /media/wrolpi/config 2>/dev/null || :
 
 # Remove immutable flag from blob files before chown (may not exist or may not be set).
 chattr -i /opt/wrolpi-blobs/* 2>/dev/null || :
@@ -200,7 +159,7 @@ chown -R wrolpi:wrolpi /home/wrolpi /opt/wrolpi*
 
 # Protect blob files from accidental modification or deletion.
 # These files are critical for initializing WROLPi.
-if [ -d /opt/wrolpi-blobs ]; then
+if [ -d /opt/wrolpi-blobs ] && ls /opt/wrolpi-blobs/* >/dev/null 2>&1; then
   chown -R root:root /opt/wrolpi-blobs
   chmod 444 /opt/wrolpi-blobs/*
   chattr +i /opt/wrolpi-blobs/*
@@ -223,7 +182,6 @@ if [[ ${rpi} == true ]]; then
   cp /opt/wrolpi/etc/raspberrypios/*.desktop /etc/skel/Desktop/
 fi
 
-systemctl restart renderd
 systemctl restart wrolpi-help
 systemctl start wrolpi.target
 

scripts/clear_map_cache.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-source /opt/wrolpi/wrolpi/scripts/lib.sh
-
-# Re-execute this script if it wasn't called with sudo.
-if [ $EUID != 0 ]; then
-  sudo "$0" "$@"
-  exit $?
-fi
-
-yes_or_no "Are you sure you want to delete map cache files?" || exit 0
-
-set -x
-
-# Clear map tile cache only after successful import.
-[ -d /var/lib/mod_tile/ajt ] && rm -rf /var/lib/mod_tile/ajt && chown -R _renderd:_renderd /var/lib/mod_tile
-[ -d /var/cache/renderd/tiles/ajt ] && rm -rf /var/cache/renderd/tiles/ajt && chown -R _renderd:_renderd /var/cache/renderd
-
-# Restart renderd to forget the cache.
-systemctl restart renderd

scripts/import_map.sh

@@ -1,112 +0,0 @@
-#!/usr/bin/env bash
-
-MERGED_TMP_FILE=/tmp/wrolpi-merged.osm.pbf
-
-if [ ! -f /opt/openstreetmap-carto/openstreetmap-carto.lua ]; then
-  echo "openstreetmap-carto.lua file is missing.  Has install been completed?"
-  exit 1
-fi
-
-if [ ! -f /opt/openstreetmap-carto/openstreetmap-carto.style ]; then
-  echo "openstreetmap-carto.style file is missing.  Has install been completed?"
-  exit 2
-fi
-
-if [ "${1}" == "" ]; then
-  echo "Missing file argument."
-  exit 3
-fi
-
-set -x
-
-HAS_ICE_SHEET=false
-sudo -iu postgres psql --port=5433 gis -c '\d' | grep icesheet_polygons && HAS_ICE_SHEET=true
-if [ ${HAS_ICE_SHEET} = false ]; then
-  echo "WROLPi: Missing icesheet_polygons.  Run reset_map.sh"
-  exit 4
-fi
-
-# Re-execute this script if it wasn't called with sudo.
-if [ $EUID != 0 ]; then
-  sudo "$0" "$@"
-  exit $?
-fi
-
-function cleanup() {
-  if [ -f ${MERGED_TMP_FILE} ]; then
-    rm ${MERGED_TMP_FILE}
-  fi
-}
-
-trap cleanup EXIT
-cleanup
-
-systemctl stop renderd
-
-if [[ ${1} == *.osm.pbf ]]; then
-  # Import PBF files.
-  for i in "$@"; do
-    if [[ $i != *.osm.pbf ]]; then
-      echo "WROLPi: Cannot mix file types"
-      exit 5
-    fi
-    if [ ! -f "${i}" ]; then
-      echo "WROLPi: File does not exist! ${i}"
-      exit 6
-    fi
-  done
-
-  # Use 1/4 of the RAM to import.  1/2 causes crashes on RPi.
-  RAM_KB=$(grep MemTotal /proc/meminfo | awk '{print $2}')
-  MAX_CACHE=$((RAM_KB / 1024 / 4))
-
-  pbf_path=${1}
-  pbf_paths=($@)
-  if [[ ${#pbf_paths[@]} -gt 1 ]]; then
-    # More than one PBF, merge them before import.
-    nice -n 18 osmium merge $@ -o ${MERGED_TMP_FILE}
-    pbf_path=${MERGED_TMP_FILE}
-  fi
-
-  # Import as postgres user, then change ownership to _renderd.
-  nice -n 18 sudo -iu postgres osm2pgsql -d gis --create --slim -G --hstore --port=5433 \
-    --tag-transform-script /opt/openstreetmap-carto/openstreetmap-carto.lua \
-    -C ${MAX_CACHE} \
-    --number-processes 3 \
-    -S /opt/openstreetmap-carto/openstreetmap-carto.style \
-    "${pbf_path}"
-  # Assign all data to _renderd so the map can be rendered.
-  sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE external_data OWNER TO _renderd'
-  sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd'
-  sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd'
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE icesheet_outlines OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE icesheet_polygons OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE ne_100m_admin_0_boundary_lines_land OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_line OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_nodes OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_point OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_polygon OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_rels OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_roads OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_ways OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE simplified_water_polygons OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO _renderd"
-  sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE water_polygons OWNER TO _renderd"
-elif [[ ${1} == *.dump ]]; then
-  if [ ! -f "${1}" ]; then
-    echo "WROLPi: File does not exist! ${1}"
-    exit 7
-  fi
-  # Import a Postgresql dump.
-  nice -n 18 pg_restore --port=5433 -j3 --no-owner -d gis -U _renderd -h 127.0.0.1 "${1}"
-else
-  echo "WROLPi: Cannot import unknown file"
-  exit 8
-fi
-
-# Clear map tile cache only after successful import.
-yes | /bin/bash /opt/wrolpi/scripts/clear_map_cache.sh
-
-# Use wget to fetch the first few layers of map tiles so the map is ready to use.
-# Failure does not mean the import failed.
-bash /opt/wrolpi/wrolpi/scripts/initialize_map_tiles.sh || :

scripts/initialize_map_db.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-# Creates the gis (map) database and _renderd database user if they don't exist.
-# WARNING: This will overwrite the map database data.
-
-source /opt/wrolpi/wrolpi/scripts/lib.sh
-
-sudo -iu postgres psql -c '\l' 2>/dev/null | grep gis >/dev/null && (
-  yes_or_no "Are you sure you want to initialize the map database?  Data from imported maps will be lost." || exit 0
-)
-
-set -x
-
-systemctl stop renderd
-
-# Create map database if it does not exist.
-pg_createcluster 15 map --port=5433 --start -e utf8
-
-# Create gis (map) database.
-sudo -iu postgres psql -c '\l' --port=5433 | grep gis || (
-  sudo -iu postgres createuser --port=5433 _renderd
-  sudo -iu postgres psql --port=5433 -c "alter user _renderd password 'wrolpi'"
-  sudo -iu postgres createdb --port=5433 -E UTF8 -O _renderd --template=template0 gis
-  sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION postgis'
-  sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION hstore'
-  echo "Created gis database"
-)
-# Restore initial dump (handle both gzipped and non-gzipped custom format).
-DUMP_FILE=/opt/wrolpi-blobs/map-db-gis.dump
-if file "${DUMP_FILE}" | grep -q "gzip"; then
-  echo "Restoring gzipped dump..."
-  gunzip -c "${DUMP_FILE}" | sudo -iu postgres pg_restore --port=5433 --no-owner --role=_renderd -d gis
-else
-  echo "Restoring dump..."
-  sudo -iu postgres pg_restore --port=5433 --no-owner --role=_renderd -d gis -j3 "${DUMP_FILE}"
-fi
-sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd'
-sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd'
-sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE spatial_ref_sys OWNER TO _renderd'
-
-# Clear map tile cache files.
-yes | /bin/bash /opt/wrolpi/scripts/clear_map_cache.sh

scripts/install_debian_12.sh

@@ -14,7 +14,7 @@ node -v
 npm -v
 
 # Install serve, and archiving tools.
-single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0
+single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6
 
 # Build React app in background job.
 cd /opt/wrolpi/app || exit 5
@@ -27,18 +27,6 @@ python3 -m venv /opt/wrolpi/venv
 # Create the WROLPi user
 grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
 
-# Get map dependencies.
-[ -d /opt/openstreetmap-carto ] && chown -R wrolpi:wrolpi /opt/openstreetmap-carto
-git clone https://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto || :
-git config --global --add safe.directory /opt/openstreetmap-carto
-(cd /opt/openstreetmap-carto && git fetch && git checkout master && git reset --hard origin/master && git pull --ff)
-chown -R _renderd:_renderd /opt/openstreetmap-carto
-
-# Get map initialization dump.
-if [[ ! -f /opt/wrolpi-blobs/map-db-gis.dump || ! -s /opt/wrolpi-blobs/map-db-gis.dump ]]; then
-  wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O /opt/wrolpi-blobs/map-db-gis.dump
-fi
-
 # Install Caddy from official apt repo if not already installed.
 if ! command -v caddy &>/dev/null; then
   curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg

scripts/install_protomaps.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# Installs PMTiles tools and migrates from the old Leaflet/renderd/Apache2 map stack.
+# This script is idempotent — safe to run multiple times.
+# Called by upgrade.sh during the upgrade process.
+
+set -e
+set -x
+
+# --- Install/upgrade pmtiles CLI ---
+
+install_pmtiles() {
+    PMTILES_VERSION="v1.30.1"
+    ARCH=$(uname -m)
+    if [ "$ARCH" = "aarch64" ]; then
+        PMTILES_ARCH="arm64"
+    elif [ "$ARCH" = "x86_64" ]; then
+        PMTILES_ARCH="amd64"
+    else
+        echo "Unsupported architecture for pmtiles: $ARCH"
+        return 1
+    fi
+
+    # Check if already installed at correct version.
+    if command -v pmtiles &>/dev/null; then
+        CURRENT=$(pmtiles version 2>&1 | head -1 | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' || echo "")
+        if [ "$CURRENT" = "$PMTILES_VERSION" ]; then
+            echo "pmtiles ${PMTILES_VERSION} already installed"
+            return 0
+        fi
+    fi
+
+    echo "Installing pmtiles ${PMTILES_VERSION} for ${ARCH}..."
+    curl -fsSL "https://github.com/protomaps/go-pmtiles/releases/download/${PMTILES_VERSION}/go-pmtiles_${PMTILES_VERSION#v}_Linux_${PMTILES_ARCH}.tar.gz" \
+      | tar -xz -C /usr/local/bin/ pmtiles
+    chmod +x /usr/local/bin/pmtiles
+    pmtiles version
+}
+
+install_pmtiles || echo "pmtiles CLI installation failed, continuing..."
+
+# --- Migrate from old map stack (runs once) ---
+
+MARKER_FILE="/opt/wrolpi/.protomaps_installed"
+
+if [ -f "${MARKER_FILE}" ]; then
+  echo "Protomaps migration already completed."
+  exit 0
+fi
+
+echo "Migrating map stack to Protomaps (MapLibre GL JS + PMTiles)..."
+
+# --- Step 1: Stop and disable old map services ---
+
+echo "Stopping old map services..."
+systemctl stop renderd 2>/dev/null || :
+systemctl stop apache2 2>/dev/null || :
+systemctl stop postgresql@15-map.service 2>/dev/null || :
+
+systemctl disable renderd 2>/dev/null || :
+systemctl disable apache2 2>/dev/null || :
+systemctl disable postgresql@15-map.service 2>/dev/null || :
+
+# --- Step 2: Uninstall old map packages ---
+
+echo "Removing old map packages..."
+apt-get remove -y --purge apache2 apache2-dev apache2-doc libapache2-mod-tile renderd osm2pgsql 2>/dev/null || :
+apt-get autoremove -y 2>/dev/null || :
+
+# --- Step 3: Drop the map PostgreSQL cluster ---
+
+echo "Removing map PostgreSQL cluster..."
+if pg_lsclusters 2>/dev/null | grep -q "15.*map"; then
+  pg_dropcluster --stop 15 map 2>/dev/null || :
+fi
+
+# Remove the map database blob (no longer needed, saves ~1.2 GB).
+if [ -f /opt/wrolpi-blobs/map-db-gis.dump ]; then
+  chattr -i /opt/wrolpi-blobs/map-db-gis.dump 2>/dev/null || :
+  rm -f /opt/wrolpi-blobs/map-db-gis.dump
+  echo "Removed map-db-gis.dump blob."
+fi
+
+# --- Step 4: Clean up old config files ---
+
+rm -f /etc/renderd.conf
+rm -f /etc/apache2/ports.conf
+rm -f /etc/apache2/conf-available/mod_tile.conf
+rm -f /etc/apache2/sites-available/000-default.conf
+rm -rf /etc/systemd/system/postgresql@15-map.service.d
+rm -f /var/www/html/leaflet.js /var/www/html/leaflet.css
+# Remove old .pgpass entry for the map database.
+if [ -f /home/wrolpi/.pgpass ]; then
+  sed -i '/5433/d' /home/wrolpi/.pgpass
+fi
+
+# --- Step 5: Update wrolpi.target to remove old services ---
+
+# The target file is copied from the repo by repair.sh, but update the installed copy now.
+if [ -f /etc/systemd/system/wrolpi.target ]; then
+  sed -i 's/ renderd\.service//' /etc/systemd/system/wrolpi.target
+  sed -i 's/ apache2\.service//' /etc/systemd/system/wrolpi.target
+  sed -i 's/ postgresql@15-map\.service//' /etc/systemd/system/wrolpi.target
+fi
+
+systemctl daemon-reload
+
+# --- Step 6: Mark migration as complete ---
+
+touch "${MARKER_FILE}"
+chown wrolpi:wrolpi "${MARKER_FILE}"
+
+echo "Protomaps migration completed successfully."
+echo "Map files (.pmtiles) should be placed in /media/wrolpi/map/"
+echo "The map viewer is served by Caddy on port 8084."

scripts/install_raspberrypios.sh

@@ -14,7 +14,7 @@ node -v
 npm -v
 
 # Install serve, and archiving tools.
-single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 carto@1.2.0
+single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6
 
 # Build React app in background job.
 cd /opt/wrolpi/app || exit 5
@@ -27,18 +27,6 @@ python3 -m venv /opt/wrolpi/venv
 # Create the WROLPi user
 grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
 
-# Get map dependencies.
-[ -d /opt/openstreetmap-carto ] && chown -R wrolpi:wrolpi /opt/openstreetmap-carto
-git clone https://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto || :
-git config --global --add safe.directory /opt/openstreetmap-carto
-(cd /opt/openstreetmap-carto && git fetch && git checkout master && git reset --hard origin/master && git pull --ff)
-chown -R _renderd:_renderd /opt/openstreetmap-carto
-
-# Get map initialization dump.
-if [[ ! -f /opt/wrolpi-blobs/map-db-gis.dump || ! -s /opt/wrolpi-blobs/map-db-gis.dump ]]; then
-  wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O /opt/wrolpi-blobs/map-db-gis.dump
-fi
-
 # Install Caddy from official apt repo if not already installed.
 if ! command -v caddy &>/dev/null; then
   curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg

scripts/map_download_tiles.py

@@ -1,107 +0,0 @@
-#! /usr/bin/env python3
-"""Used to download many map tiles from the Map service into the current directory.
-
-Example:  To download all tiles of the US state of Kansas, run the script like so:
-
-/opt/wrolpi/venv/bin/python /opt/wrolpi/scripts/map_download_tiles.py 40.011647534727125 -102.06383712982996 37.030883772380065 -94.61534551981345
-
-The script will download zoom levels 6 to 12 by default, this will show most of a US state down to major streets.
-
-Changing zoom levels can be exponential!
-"""
-import argparse
-import pathlib
-from math import log, tan, cos, pi
-
-import requests
-import urllib3
-from tqdm import tqdm
-
-# urllib3 reports a warning whenever you make an insecure request, the line below disables that functionality.
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-
-def latitude_to_tile_y(lat: float, zoom: int) -> int:
-    return int((1.0 - log(tan(lat * pi / 180.0) + 1.0 / cos(lat * pi / 180.0)) / pi) / 2.0 * (1 << zoom))
-
-
-def longitude_to_tile_x(lon: float, zoom: int) -> int:
-    return int((lon + 180.0) / 360.0 * (1 << zoom))
-
-
-def get_tile(host: str, zoom: int, x: int, y: int) -> bytes:
-    response = requests.get(f"{host}/{zoom}/{x}/{y}.png", verify=False, timeout=5 * 60)
-    if response.status_code != 200:
-        raise RuntimeError(f"Request failed with status code {response.status_code}")
-    return response.content
-
-
-def main(
-        host: str,
-        zoom_start: int,
-        zoom_stop: int,
-        west_latitude: float,
-        west_longitude: float,
-        east_latitude: float,
-        east_longitude: float,
-):
-    grand_total = 0
-    for zoom in range(zoom_start, zoom_stop + 1):
-        west_y = latitude_to_tile_y(west_latitude, zoom)
-        west_x = longitude_to_tile_x(west_longitude, zoom)
-        east_y = latitude_to_tile_y(east_latitude, zoom)
-        east_x = longitude_to_tile_x(east_longitude, zoom)
-        grand_total += len(list(range(west_y, east_y))) * len(list(range(west_x, east_x)))
-    print(f'Expecting to download {grand_total} tiles...')
-
-    for zoom in range(zoom_start, zoom_stop + 1):
-        west_y = latitude_to_tile_y(west_latitude, zoom)
-        west_x = longitude_to_tile_x(west_longitude, zoom)
-        east_y = latitude_to_tile_y(east_latitude, zoom)
-        east_x = longitude_to_tile_x(east_longitude, zoom)
-        total = len(list(range(west_y, east_y))) * len(list(range(west_x, east_x)))
-        with tqdm(total=total) as pbar:
-            pbar.set_description(f'zoom {zoom}')
-            for x in range(west_x, east_x + 1):
-                pathlib.Path(f'{zoom}/{x}').mkdir(parents=True, exist_ok=True)
-                for y in range(west_y, east_y + 1):
-                    pbar.update(1)
-                    file = pathlib.Path(f'{zoom}/{x}/{y}.png')
-                    if not file.exists() or file.stat().st_size == 0:
-                        tile = get_tile(host, zoom, x, y)
-                        file.write_bytes(tile)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Download map tiles from WROLPi")
-    parser.add_argument('-H', '--host', type=str,
-                        default='https://localhost:8084/hot',
-                        help='The URL of your map tiles server',
-                        )
-    parser.add_argument('-a', '--zoom-start', type=int, default=0,
-                        help='The highest zoom to download.  0 will display the entire planet, 6 will display display most of a US state.')
-    parser.add_argument('-o', '--zoom-stop', type=int, default=12,
-                        help='The lowest zoom to download.  12 will show major streets, 13 will show small streets, 14 will show walking paths.')
-    parser.add_argument('west_latitude', type=float,
-                        help='The west-most (leftmost) latitude of the square you want to download.  Example: 40.011647534727125',
-                        )
-    parser.add_argument('west_longitude', type=float,
-                        help='The west-most (leftmost) longitude of the square you want to download.  Example: -102.06383712982996',
-                        )
-    parser.add_argument('east_latitude', type=float,
-                        help='The west-most (leftmost) longitude of the square you want to download.  Example: 37.030883772380065',
-                        )
-    parser.add_argument('east_longitude', type=float,
-                        help='The west-most (leftmost) longitude of the square you want to download.  Example: -94.61534551981345',
-                        )
-    args = parser.parse_args()
-
-    main(
-        host=args.host,
-        zoom_start=args.zoom_start,
-        zoom_stop=args.zoom_stop,
-        west_latitude=args.west_latitude,
-        west_longitude=args.west_longitude,
-        east_latitude=args.east_latitude,
-        east_longitude=args.east_longitude,
-    )

scripts/reset_map.sh

@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-
-source /opt/wrolpi/wrolpi/scripts/lib.sh
-
-# Re-execute this script if it wasn't called with sudo.
-if [ $EUID != 0 ]; then
-  sudo "$0" "$@"
-  exit $?
-fi
-
-# Change directory to avoid "sudo" home directory warning.
-cd /tmp
-
-if sudo -iu postgres psql --port=5433 -l 2>/dev/null | grep gis >/dev/null; then
-  yes_or_no "Are you sure you want to delete the map database and cache?  This can't be undone!" || exit 0
-fi
-
-function cleanup() {
-  sudo rm -f /tmp/tile.png /tmp/tile.md5
-}
-
-trap cleanup EXIT
-cleanup
-
-set -e
-set -x
-
-systemctl stop renderd
-
-# Delete old map database, if it exists.
-sudo -iu postgres dropdb --port=5432 gis || :
-sudo -iu postgres dropuser --port=5432 _renderd || :
-# Delete new map database, it will be created again.
-sudo -iu postgres dropdb --port=5433 gis || :
-sudo -iu postgres dropuser --port=5433 _renderd || :
-
-# Reset "imported" status of any map files in API DB.
-sudo -iu postgres psql -d wrolpi -c "UPDATE map_file SET imported=false"
-
-yes | /opt/wrolpi/scripts/initialize_map_db.sh
-
-# Use curl to fetch the first few layers of map tiles so the map is ready to use.
-bash /opt/wrolpi/wrolpi/scripts/initialize_map_tiles.sh
-
-# Check the hash value of a Washington DC tile from /opt/wrolpi-blobs/map-db-gis.dump
-wget "https://127.0.0.1:8084/hot/13/2345/3134.png" \
-  --quiet \
-  --no-check-certificate \
-  --timeout 600 \
-  -O /tmp/tile.png
-set +x
-echo "58b261940abaf50229544839fc55c83d  /tmp/tile.png" > /tmp/tile.md5
-if md5sum -c /tmp/tile.md5 >/dev/null 2>&1 ; then
-  echo "Map tile was rendered correctly"
-else
-  echo "Map tile was not rendered correctly!"
-  exit 1
-fi
-
-echo "Map has been reset."

scripts/upgrade.sh

@@ -87,19 +87,27 @@ find /opt/wrolpi/venv -name "*.pyc" -delete 2>/dev/null || :
 # Upgrade WROLPi Help.
 /opt/wrolpi/scripts/install_help_service.sh || echo "Help install failed."
 
-# Get map db blob, if it is missing.
-MAP_DB_BLOB=/opt/wrolpi-blobs/map-db-gis.dump
-if [[ ! -f ${MAP_DB_BLOB} || ! -s ${MAP_DB_BLOB} ]]; then
-  echo "Downloading new map blob (1.2 GB)..."
-  wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O ${MAP_DB_BLOB}
-fi
-if [[ -f /opt/wrolpi-blobs/gis-map.dump.gz && -s ${MAP_DB_BLOB} ]]; then
-  # Remove old blob now that we have the new one.
-  rm /opt/wrolpi-blobs/gis-map.dump.gz
-fi
+# Migrate from old Leaflet/renderd/Apache2 map stack to MapLibre GL JS + PMTiles.
+/opt/wrolpi/scripts/install_protomaps.sh || echo "Protomaps migration failed."
 
-# Migrate map DB if necessary.  Do this before repair because it will reset map if map db is empty.
-/opt/wrolpi/wrolpi/scripts/migrate_map_db.sh || echo "Map DB migration failed."
+# Download map fonts blob if missing.
+MAP_FONTS_BLOB=/opt/wrolpi-blobs/map-fonts.tar.gz
+if [[ ! -f ${MAP_FONTS_BLOB} || ! -s ${MAP_FONTS_BLOB} ]]; then
+  echo "Downloading map fonts..."
+  mkdir -p /tmp/map-fonts-dl
+  curl -fsSL https://github.com/protomaps/basemaps-assets/archive/refs/heads/main.tar.gz \
+    | tar -xz --strip-components=1 -C /tmp/map-fonts-dl basemaps-assets-main/fonts
+  tar -czf "${MAP_FONTS_BLOB}" -C /tmp/map-fonts-dl fonts
+  rm -rf /tmp/map-fonts-dl
+fi || echo "Map fonts download failed, continuing..."
+
+# Download map overview blob if missing (42 MB, provides global zoom 0-6).
+MAP_OVERVIEW_BLOB=/opt/wrolpi-blobs/map-overview.pmtiles
+if [[ ! -f ${MAP_OVERVIEW_BLOB} || ! -s ${MAP_OVERVIEW_BLOB} ]]; then
+  echo "Downloading map overview..."
+  curl -fsSL https://wrolpi.nyc3.cdn.digitaloceanspaces.com/maps/map-overview.pmtiles \
+    -o "${MAP_OVERVIEW_BLOB}"
+fi || echo "Map overview download failed, continuing..."
 
 # Migrate from nginx to Caddy if necessary.
 if ! command -v caddy &>/dev/null; then

wrolpi/cmd.py

@@ -157,7 +157,7 @@ TESTING_RUN_COMMAND_RESULT = None
 
 async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | str = None,
                       timeout: int = 600, log_command: bool = True,
-                      stdout_callback: callable = None) -> CommandResult:
+                      stdout_callback: callable = None, env: dict = None) -> CommandResult:
     """Run a shell command, return the results (stdout/stderr/return code).
 
     When stdout_callback is provided, stdout is piped and each line is passed to the callback
@@ -165,6 +165,7 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
 
     :param log_command: Enable debug logging.
     :param stdout_callback: Called with each decoded stdout line (str). May be None.
+    :param env: Environment variables for the subprocess. If None, inherits the current environment.
     """
     if not isinstance(cmd, (list, tuple)):
         raise RuntimeError('Command must be a list or tuple')
@@ -203,20 +204,43 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
                 stdout=stdout_arg,
                 stderr=stderr_fh,
                 cwd=cwd,
+                env=env,
             )
             pid = proc.pid
 
             if streaming:
                 async def _read_stdout():
+                    buf = ''
                     while True:
-                        line = await proc.stdout.readline()
-                        if not line:
+                        chunk = await proc.stdout.read(4096)
+                        if not chunk:
+                            # Flush remaining buffer.
+                            if buf.strip():
+                                try:
+                                    stdout_callback(buf.strip())
+                                except Exception as e:
+                                    logger.warning(f'run_command: stdout_callback error: {e}')
                             break
-                        stdout_buf.extend(line)
-                        try:
-                            stdout_callback(line.decode(errors='replace').rstrip('\n'))
-                        except Exception as e:
-                            logger.warning(f'run_command: stdout_callback error: {e}')
+                        stdout_buf.extend(chunk)
+                        buf += chunk.decode(errors='replace')
+                        # Split on both \n and \r to handle progress bars.
+                        while '\n' in buf or '\r' in buf:
+                            # Find the earliest delimiter.
+                            ni = buf.find('\n')
+                            ri = buf.find('\r')
+                            if ni == -1:
+                                idx, skip = ri, 1
+                            elif ri == -1:
+                                idx, skip = ni, 1
+                            else:
+                                idx, skip = min(ni, ri), 1
+                            line = buf[:idx]
+                            buf = buf[idx + skip:]
+                            if line.strip():
+                                try:
+                                    stdout_callback(line.strip())
+                                except Exception as e:
+                                    logger.warning(f'run_command: stdout_callback error: {e}')
 
                 reader_task = asyncio.create_task(_read_stdout())
 

wrolpi/common.py

@@ -855,6 +855,10 @@ def get_all_configs() -> Dict[str, ConfigFile]:
     if _collections_config:
         all_configs[_collections_config.file_name] = _collections_config
 
+    from modules.map.pins import get_map_pins_config
+    if map_pins_config := get_map_pins_config():
+        all_configs[map_pins_config.file_name] = map_pins_config
+
     return all_configs
 
 
@@ -937,6 +941,16 @@ async def import_all_db_configs() -> dict[str, bool]:
         logger.warning(f'Failed to import collections config: {e}')
         results['collections'] = False
 
+    # Map pins (YAML-only, no DB)
+    try:
+        from modules.map.pins import get_map_pins_config
+        get_map_pins_config().import_config()
+        results['map_pins'] = True
+        logger.debug('map pins config imported')
+    except Exception as e:
+        logger.warning(f'Failed to import map pins config: {e}')
+        results['map_pins'] = False
+
     return results
 
 
@@ -2662,3 +2676,33 @@ def create_empty_config_files() -> list[str]:
                 created.append(get_inventories_config().get_file().name)
 
     return created
+
+
+GPG_PUBLIC_KEY = Path(__file__).parent / 'roland@learningselfreliance.com.gpg'
+
+
+async def verify_gpg_signature(data_path: Path, signature_path: Path) -> bool:
+    """Verify a detached GPG signature against the shipped WROLPi public key.
+
+    Accepts file paths so that large files are never loaded into memory.
+    Uses a temporary GNUPGHOME to avoid polluting the system keyring.
+    Returns True if the signature is valid, False otherwise."""
+    from wrolpi.cmd import run_command
+
+    with tempfile.TemporaryDirectory() as gnupghome:
+        env = {**os.environ, 'GNUPGHOME': gnupghome}
+
+        # Import the trusted public key.
+        result = await run_command(
+            ('gpg', '--batch', '--quiet', '--import', str(GPG_PUBLIC_KEY)),
+            timeout=10, env=env, log_command=False,
+        )
+        if result.return_code != 0:
+            logger.error(f'Failed to import GPG key: {result.stderr.decode()}')
+            return False
+
+        result = await run_command(
+            ('gpg', '--batch', '--verify', str(signature_path), str(data_path)),
+            timeout=10, env=env, log_command=False,
+        )
+        return result.return_code == 0

wrolpi/contexts.py

@@ -33,11 +33,11 @@ def attach_shared_contexts(app: Sanic):
     app.shared_ctx.archive_downloader_config = manager.dict()
     app.shared_ctx.download_cache_config = manager.dict()
     app.shared_ctx.collections_config = manager.dict()
+    app.shared_ctx.map_pins_config = manager.dict()
     # Shared dicts.
     app.shared_ctx.refresh = manager.dict()
     app.shared_ctx.uploaded_files = manager.dict()
     app.shared_ctx.status = manager.dict()
-    app.shared_ctx.map_importing = manager.dict()
     app.shared_ctx.cache = manager.dict()
     # Secure cookies shared storage
     app.shared_ctx.secure_cookies = manager.dict()
@@ -123,7 +123,6 @@ def reset_shared_contexts(app: Sanic):
         commits_behind=0,
         git_branch=None,
     ))
-    app.shared_ctx.map_importing.clear()
     app.shared_ctx.cache.clear()
     # Secure cookies
     app.shared_ctx.secure_cookies.clear()
@@ -271,3 +270,9 @@ def initialize_configs_contexts(app: Sanic):
         _collections_config.initialize(app.shared_ctx.collections_config)
     except Exception as e:
         logger.error(f'Failed to initialize in-memory collections config: {e}')
+
+    try:
+        from modules.map.pins import MAP_PINS_CONFIG
+        MAP_PINS_CONFIG.initialize(app.shared_ctx.map_pins_config)
+    except Exception as e:
+        logger.error(f'Failed to initialize in-memory map pins config: {e}')

wrolpi/downloader.py

@@ -114,10 +114,15 @@ def make_progress_callback(download_id: int, parse_fn: callable) -> callable:
         now_ = time.monotonic()
         if now_ - last_update[0] < 1.0:
             return
-        progress = parse_fn(line)
-        if progress:
-            last_update[0] = now_
-            set_download_progress(download_id, progress)
+        # Some tools (e.g., pmtiles) use \r for progress updates within a single line.
+        # Parse the last non-empty segment.
+        segments = [s.strip() for s in line.split('\r') if s.strip()]
+        for segment in reversed(segments):
+            progress = parse_fn(segment)
+            if progress:
+                last_update[0] = now_
+                set_download_progress(download_id, progress)
+                return
 
     return on_stdout
 
@@ -390,7 +395,7 @@ class Downloader:
 
     async def process_runner(self, download: Download, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path,
                              timeout: int = None, debug: bool = True,
-                             stdout_callback: callable = None) -> CommandResult:
+                             stdout_callback: callable = None, env: dict = None) -> CommandResult:
         """
         Run a subprocess using the provided arguments.  This process can be killed by the Download Manager.
 
@@ -399,9 +404,14 @@ class Downloader:
         if cwd and not cwd.is_dir():
             raise RuntimeError('cwd directory does not exist')
 
-        timeout = get_wrolpi_config().download_timeout or timeout or self.timeout
+        config_timeout = get_wrolpi_config().download_timeout
+        # 0 means no timeout. Prefer config timeout, then argument, then class default.
+        if config_timeout:
+            timeout = config_timeout
+        elif timeout is None:
+            timeout = self.timeout
         coro = run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug,
-                           stdout_callback=stdout_callback)
+                           stdout_callback=stdout_callback, env=env)
         result = await self.cancel_wrapper(coro, download)
 
         if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
@@ -475,8 +485,9 @@ class Downloader:
 
         return None
 
-    async def download_file(self, download: Download, url: str, destination: pathlib.Path, check_for_meta4: bool = True) \
-            -> pathlib.Path:
+    async def download_file(self, download: Download, url: str, destination: pathlib.Path,
+                            check_for_meta4: bool = True, concurrent: int = 3,
+                            meta4_xml: bytes = None) -> pathlib.Path:
         from wrolpi.files.lib import glob_shared_stem
 
         if not ARIA2C_PATH:
@@ -484,8 +495,9 @@ class Downloader:
 
         # Log download time.
         with timer('download_file', level='info'):
-            meta4_contents = None
-            if check_for_meta4:
+            # Use caller-provided meta4 (pre-verified), or auto-discover from CDN.
+            meta4_contents = meta4_xml
+            if not meta4_contents and check_for_meta4:
                 meta4_contents = await self.get_meta4_contents(url)
 
             info = await get_download_info(url)
@@ -496,10 +508,11 @@ class Downloader:
             with tempfile.NamedTemporaryFile(suffix='.meta4') as meta4_path:
                 cmd = (
                     ARIA2C_PATH,
-                    '-l', '-',
+                    '-l', '-',  # Log to stdout for progress parsing.
+                    '--log-level=notice',  # Only notice-level logs (skip debug spam).
                     '-c',  # Continue downloading a partially downloaded file.
-                    '-j3',  # concurrent downloads
-                    '-s3',  # split jobs
+                    f'-j{concurrent}',  # concurrent connections
+                    f'-s{concurrent}',  # split into segments
                     '-d', destination,
                     url
                 )
@@ -520,13 +533,20 @@ class Downloader:
                     result = await self.process_runner(download, cmd, destination, stdout_callback=on_stdout)
                 finally:
                     clear_download_progress(download.id)
-                error = result.stderr.decode()
+                stderr = result.stderr.decode() if getattr(result, 'stderr', None) else ''
+                stdout = result.stdout.decode() if getattr(result, 'stdout', None) else ''
 
                 if result.return_code != 0:
-                    raise DownloadError(f'{error}\n\nFileDownloader failed with return code {result.return_code}')
+                    output = stderr.strip() or stdout.strip() or '(no output captured)'
+                    logger.error(f'FileDownloader failed (rc={result.return_code}):'
+                                 f'\nstderr: {stderr[-1000:]}'
+                                 f'\nstdout: {stdout[-1000:]}')
+                    raise DownloadError(
+                        f'{output}\n\nFileDownloader failed with return code {result.return_code}'
+                    )
 
                 if not output_path.is_file():
-                    raise DownloadError(f'{error}\n\nOutput file not found: {output_path}')
+                    raise DownloadError(f'{stderr}\n\nOutput file not found: {output_path}')
 
             # Delete any trailing meta4 files.
             matching_files = glob_shared_stem(output_path)
@@ -702,16 +722,19 @@ class DownloadManager:
             return downloader
         raise InvalidDownload(f'Cannot find downloader with name {name}')
 
-    def get_or_create_download(self, session: Session, url: str, reset_attempts: bool = False) -> Download:
+    def get_or_create_download(self, session: Session, url: str, reset_attempts: bool = False,
+                               override_skip: bool = False) -> Download:
         """Get a Download by its URL, if it cannot be found create one."""
         if not url:
             raise ValueError('Download must have a URL')
 
         if download := Download.get_by_url(session, url):
+            if reset_attempts:
+                download.attempts = 0
             return download
 
         if self.is_skipped(url):
-            if reset_attempts is True:
+            if override_skip is True:
                 self.remove_from_skip_list(url)
             else:
                 raise InvalidDownload(
@@ -722,7 +745,8 @@ class DownloadManager:
 
     def create_downloads(self, session: Session, urls: List[str], downloader_name: str,
                          reset_attempts: bool = False, sub_downloader_name: str = None,
-                         destination: str | pathlib.Path = None, tag_names: List[str] = None, settings: dict = None) \
+                         destination: str | pathlib.Path = None, tag_names: List[str] = None, settings: dict = None,
+                         override_skip: bool = False) \
             -> List[Download]:
         """Schedule all URLs for download.  If one cannot be downloaded, none will be added."""
         if not urls or not all(urls):
@@ -749,14 +773,14 @@ class DownloadManager:
                 destination = get_absolute_media_path(destination)
 
         for url in urls:
-            if url in get_download_manager_config().skip_urls and reset_attempts:
+            if url in get_download_manager_config().skip_urls and override_skip:
                 # User manually entered this download, remove it from the skip list.
                 self.remove_from_skip_list(url)
             elif url in get_download_manager_config().skip_urls:
                 self.log_warning(f'Skipping {url} because it is in the download_manager.yaml skip list.')
                 continue
 
-            download = self.get_or_create_download(session, url, reset_attempts=reset_attempts)
+            download = self.get_or_create_download(session, url, override_skip=override_skip)
             # Download may have failed, try again.
             download.renew(reset_attempts=reset_attempts)
             download.destination = destination or None
@@ -788,17 +812,20 @@ class DownloadManager:
 
     def create_download(self, session: Session, url: str, downloader_name: str, reset_attempts: bool = False,
                         sub_downloader_name: str = None, destination: str | pathlib.Path = None,
-                        tag_names: List[str] = None, settings: Dict = None) -> Download:
+                        tag_names: List[str] = None, settings: Dict = None,
+                        override_skip: bool = False) -> Download:
         """Schedule a URL for download.  If the URL failed previously, it may be retried."""
         downloads = self.create_downloads(session, [url], downloader_name=downloader_name,
                                           reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name,
-                                          destination=destination, tag_names=tag_names, settings=settings)
+                                          destination=destination, tag_names=tag_names, settings=settings,
+                                          override_skip=override_skip)
         return downloads[0]
 
     def recurring_download(self, session: Session, url: str, frequency: int, downloader_name: str,
                            sub_downloader_name: str = None, reset_attempts: bool = False,
                            destination: str | pathlib.Path = None, tag_names: List[str] = None,
-                           settings: Dict = None, collection_id: int = None) -> Download:
+                           settings: Dict = None, collection_id: int = None,
+                           override_skip: bool = False) -> Download:
         """Schedule a recurring download."""
         if not frequency or not isinstance(frequency, int):
             raise ValueError('Recurring download must have a frequency!')
@@ -809,7 +836,8 @@ class DownloadManager:
 
         download, = self.create_downloads(session, [url, ], downloader_name=downloader_name,
                                           reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name,
-                                          destination=destination, tag_names=tag_names, settings=settings)
+                                          destination=destination, tag_names=tag_names, settings=settings,
+                                          override_skip=override_skip)
         download.frequency = frequency
         if collection_id:
             download.collection_id = collection_id
@@ -1483,7 +1511,7 @@ async def signal_download_download(download_id: int, download_url: str):
                     child_settings = dict(result.settings) if result.settings else {}
                     child_settings['parent_download_url'] = download.url
                     download_manager.create_downloads(session, urls, downloader_name=download.sub_downloader,
-                                                      settings=child_settings)
+                                                      settings=child_settings, reset_attempts=True)
 
             # Remove this domain from the running list.
             download_manager._delete_processing_domain(download_domain)

wrolpi/flags.py

@@ -108,8 +108,6 @@ class Flag:
 db_up = Flag('db_up')
 # Outdated Zims need to be removed.
 outdated_zims = Flag('outdated_zims', store_db=True)
-# Map Importing
-map_importing = Flag('map_importing')
 # Used to disable or enable downloading when Internet is down.
 have_internet = Flag('have_internet')
 
@@ -147,7 +145,6 @@ def get_flags() -> dict:
         file_worker_modeling=file_worker_modeling.is_set(),
         global_refresh_active=global_refresh_active.is_set(),
         have_internet=have_internet.is_set(),
-        map_importing=map_importing.is_set(),
         outdated_zims=outdated_zims.is_set(),
         refresh_complete=refresh_complete.is_set(),
     )

wrolpi/root_api.py

@@ -306,7 +306,7 @@ async def post_download(request: Request, body: schema.DownloadRequest):
     kwargs = dict(downloader_name=body.downloader,
                   sub_downloader_name=body.sub_downloader, reset_attempts=True,
                   destination=body.destination, tag_names=body.tag_names,
-                  settings=body.settings)
+                  settings=body.settings, override_skip=True)
     if body.frequency:
         download_manager.recurring_download(session, body.urls[0], body.frequency, collection_id=body.collection_id,
                                             **kwargs)

wrolpi/schema.py

@@ -104,7 +104,6 @@ class FlagsStatusResponse:
     file_worker_modeling: bool
     global_refresh_active: bool
     have_internet: bool
-    map_importing: bool
     outdated_zims: bool
     refresh_complete: bool
 

wrolpi/scripts/build_map_regions.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+"""Extract all map regions from a planet PMTiles file and generate a CDN manifest.
+
+Usage:
+    python scripts/build_map_regions.py <planet.pmtiles> <version> <output_dir> [--cdn-base URL]
+
+Example:
+    python scripts/build_map_regions.py planet-20260329.pmtiles 20260329 ./cdn-output/
+
+This script:
+  1. Reads region definitions from modules/map/catalog.py
+  2. Runs `pmtiles extract` for each bbox region
+  3. Generates manifest.json with direct download URLs for each region
+  4. Prints a summary of all extracted files and sizes
+
+Requirements:
+  - pmtiles CLI installed (https://github.com/protomaps/go-pmtiles)
+  - Planet PMTiles file already downloaded
+  - Terrain file must be placed in output_dir manually (not extracted from planet)
+"""
+import argparse
+import hashlib
+import json
+import os
+import subprocess
+import sys
+import time
+
+# Add project root to path so we can import catalog.
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from modules.map.catalog import MAP_REGIONS
+
+DEFAULT_CDN_BASE = 'https://wrolpi.nyc3.cdn.digitaloceanspaces.com/maps'
+
+
+def format_size(size_bytes: int) -> str:
+    if size_bytes >= 1_000_000_000:
+        return f'{size_bytes / 1_000_000_000:.1f} GB'
+    elif size_bytes >= 1_000_000:
+        return f'{size_bytes / 1_000_000:.1f} MB'
+    elif size_bytes >= 1_000:
+        return f'{size_bytes / 1_000:.1f} kB'
+    return f'{size_bytes} B'
+
+
+def sha256_file(file_path: str) -> str:
+    """Compute SHA-256 hash of a file using chunked reading (safe for multi-GB files)."""
+    h = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(8 * 1024 * 1024), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def sign_hash(sha256_hex: str) -> str:
+    """GPG-sign a SHA-256 hash string, return armored detached signature."""
+    result = subprocess.run(
+        ['gpg', '--batch', '-u', 'roland@learningselfreliance.com', '--detach-sign', '--armor'],
+        input=sha256_hex.encode(), capture_output=True,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f'GPG signing failed: {result.stderr.decode().strip()}')
+    return result.stdout.decode()
+
+
+def generate_meta4(name: str, url: str, sha256: str, size: int, signature: str) -> str:
+    """Generate Metalink4 XML with hash, size, url, and GPG signature of the hash."""
+    return (
+        f'<?xml version="1.0" encoding="UTF-8"?>\n'
+        f'<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
+        f'  <file name="{name}">\n'
+        f'    <size>{size}</size>\n'
+        f'    <hash type="sha-256">{sha256}</hash>\n'
+        f'    <signature mediatype="application/pgp-signature">{signature}</signature>\n'
+        f'    <url>{url}</url>\n'
+        f'  </file>\n'
+        f'</metalink>\n'
+    )
+
+
+def write_meta4(file_path: str, cdn_url: str):
+    """Compute hash, sign it, and write a signed .meta4 file alongside the given file."""
+    name = os.path.basename(file_path)
+    size = os.path.getsize(file_path)
+    sha256 = sha256_file(file_path)
+    print(f'  Signing hash for {name}...')
+    signature = sign_hash(sha256)
+    meta4_xml = generate_meta4(name, cdn_url, sha256, size, signature)
+    meta4_path = f'{file_path}.meta4'
+    with open(meta4_path, 'w') as f:
+        f.write(meta4_xml)
+    print(f'  Written {os.path.basename(meta4_path)}')
+
+
+def extract_region(planet_path: str, output_path: str, bbox: str, threads: int = 4) -> bool:
+    """Run pmtiles extract for a single region. Returns True on success."""
+    cmd = ['pmtiles', 'extract', planet_path, output_path, f'--bbox={bbox}',
+           f'--download-threads={threads}']
+    print(f'  Running: {" ".join(cmd)}')
+    result = subprocess.run(cmd, capture_output=False)
+    return result.returncode == 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Extract map regions and generate CDN manifest.')
+    parser.add_argument('planet', help='Path to the planet PMTiles file')
+    parser.add_argument('version', help='Version string (e.g. 20260329)')
+    parser.add_argument('output_dir', help='Output directory for extracted files and manifest')
+    parser.add_argument('--cdn-base', default=DEFAULT_CDN_BASE,
+                        help=f'CDN base URL (default: {DEFAULT_CDN_BASE})')
+    parser.add_argument('--threads', type=int, default=4,
+                        help='Download threads for pmtiles extract (default: 4)')
+    args = parser.parse_args()
+
+    planet_path = os.path.abspath(args.planet)
+    output_dir = os.path.abspath(args.output_dir)
+    cdn_base = args.cdn_base.rstrip('/')
+
+    if not os.path.isfile(planet_path):
+        print(f'Error: Planet file not found: {planet_path}')
+        sys.exit(1)
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Separate regions by type.
+    bbox_regions = []
+    terrain_region = None
+
+    for r in MAP_REGIONS:
+        if r.get('terrain'):
+            terrain_region = r
+        elif r.get('bbox'):
+            bbox_regions.append(r)
+
+    print(f'Planet file: {planet_path}')
+    print(f'Version: {args.version}')
+    print(f'Output dir: {output_dir}')
+    print(f'CDN base: {cdn_base}')
+    print(f'Regions to extract: {len(bbox_regions)}')
+    print()
+
+    # Extract each bbox region.
+    manifest_regions = {}
+    results = []
+    failed = []
+
+    for i, r in enumerate(bbox_regions, 1):
+        region = r['region']
+        bbox = r['bbox']
+        output_name = f'{region}-{args.version}.pmtiles'
+        output_path = os.path.join(output_dir, output_name)
+
+        print(f'[{i}/{len(bbox_regions)}] Extracting {r["name"]} ({region})...')
+
+        if os.path.isfile(output_path):
+            size = os.path.getsize(output_path)
+            print(f'  Already exists ({format_size(size)}), skipping.')
+        else:
+            start = time.time()
+            success = extract_region(planet_path, output_path, bbox, threads=args.threads)
+            elapsed = time.time() - start
+
+            if not success:
+                print(f'  FAILED after {elapsed:.0f}s')
+                failed.append(region)
+                continue
+
+            size = os.path.getsize(output_path)
+            print(f'  Done in {elapsed:.0f}s ({format_size(size)})')
+
+        size = os.path.getsize(output_path)
+        cdn_url = f'{cdn_base}/{output_name}'
+        write_meta4(output_path, cdn_url)
+        results.append((r['name'], region, size))
+        manifest_regions[region] = cdn_url
+
+    # Add terrain to manifest if the file exists in output_dir.
+    if terrain_region:
+        terrain_name = f'{terrain_region["region"]}.pmtiles'
+        terrain_path = os.path.join(output_dir, terrain_name)
+        if os.path.isfile(terrain_path):
+            cdn_url = f'{cdn_base}/{terrain_name}'
+            write_meta4(terrain_path, cdn_url)
+            manifest_regions[terrain_region['region']] = cdn_url
+            size = os.path.getsize(terrain_path)
+            results.append((terrain_region['name'], terrain_region['region'], size))
+            print(f'Terrain file found: {terrain_name} ({format_size(size)})')
+        else:
+            print(f'Warning: Terrain file not found at {terrain_path} — not included in manifest.')
+
+    # Build manifest.
+    manifest = {
+        'version': args.version,
+        'regions': manifest_regions,
+    }
+    # Add terrain URL if present.
+    if terrain_region and terrain_region['region'] in manifest_regions:
+        manifest['terrain_url'] = manifest_regions[terrain_region['region']]
+
+    manifest_path = os.path.join(output_dir, 'manifest.json')
+    with open(manifest_path, 'w') as f:
+        json.dump(manifest, f, indent=2)
+    print(f'\nManifest written to: {manifest_path}')
+
+    # Sign manifest with GPG.
+    sig_path = f'{manifest_path}.sig'
+    print(f'Signing manifest...')
+    result = subprocess.run(
+        ['gpg', '-u', 'roland@learningselfreliance.com', '--detach-sign', '--armor', '-o', sig_path, manifest_path],
+        capture_output=True,
+    )
+    if result.returncode == 0:
+        print(f'Signature written to: {sig_path}')
+    else:
+        print(f'ERROR: GPG signing failed: {result.stderr.decode().strip()}')
+        print(f'Sign manually and re-run, or fix GPG key access:'
+              f'\n  gpg -u roland@learningselfreliance.com --detach-sign --armor -o {sig_path} {manifest_path}')
+        sys.exit(1)
+
+    # Print summary.
+    print(f'\n{"="*70}')
+    print(f'{"Region":<45} {"Size":>12}')
+    print(f'{"-"*45} {"-"*12}')
+    total_size = 0
+    for name, region, size in results:
+        print(f'{name:<45} {format_size(size):>12}')
+        total_size += size
+    print(f'{"-"*45} {"-"*12}')
+    print(f'{"Total":<45} {format_size(total_size):>12}')
+    print(f'{"="*70}')
+
+    if failed:
+        print(f'\nFailed regions ({len(failed)}): {", ".join(failed)}')
+        sys.exit(1)
+
+    print(f'\nAll {len(bbox_regions)} regions extracted successfully.')
+    print(f'Upload contents of {output_dir} to {cdn_base}')
+
+
+if __name__ == '__main__':
+    main()

wrolpi/scripts/initialize_map_tiles.sh

@@ -1,86 +0,0 @@
-#! /usr/bin/env bash
-# Use curl to fetch the first few layers of map tiles so the map is ready to use.
-
-HOST="https://127.0.0.1:8084"
-
-Help() {
-  # Display Help
-  echo "Fetch several lays of map tiles to initialize map tile data."
-  echo
-  echo "Syntax: initialize_map_tiles.sh [-h] [-H HOST]"
-  echo "options:"
-  echo "h     Print this help."
-  echo "b     The hostname to fetch tiles from. (default: ${HOST})"
-  echo
-}
-
-while getopts ":hH:" option; do
-  case $option in
-  h) # display Help
-    Help
-    exit
-    ;;
-  H)
-    HOST="${OPTARG}"
-    ;;
-  *) # invalid argument(s)
-    echo "Error: Invalid option"
-    exit 1
-    ;;
-  esac
-done
-
-# Try to fetch the first tile multiple times if the service is starting, or if new map was imported.
-ATTEMPTS=100
-for i in {1..100}; do
-    # Try to fetch the first tile.
-    wget "${HOST}/hot/3/0/0.png" \
-      --quiet \
-      --tries=1 \
-      --no-check-certificate \
-      --timeout 5 -O /dev/null || {
-        echo "Waiting for map to come up, sleeping for 5 seconds..."
-        sleep 5
-        continue
-    }
-    # If we reach here, wget succeeded, so we exit the loop
-    break
-done
-
-# If we've made it through all attempts, curl never succeeded
-if [[ $i -eq $ATTEMPTS ]]; then
-    echo "Failed to fetch the image after $ATTEMPTS attempts."
-    echo "  You can try resetting your map using /opt/wrolpi/scripts/reset_map.sh then importing a map again."
-    exit 1
-fi
-
-echo "Fetching first map tile succeeded."
-
-
-# Function to fetch and count downloads
-fetch_and_count() {
-    local level=$1
-    local max_x=$2
-    local max_y=$3
-
-    local success_count=0
-    local failure_count=0
-
-    for x in $(seq 0 $max_x); do
-        for y in $(seq 0 $max_y); do
-            if wget -q --spider --no-check-certificate "${HOST}/hot/${level}/${x}/${y}.png"; then
-                ((success_count++))
-            else
-                ((failure_count++))
-            fi
-        done
-    done
-
-    echo "Level $level - Successful downloads: $success_count"
-}
-
-# Fetch all tiles from each level so the map is ready to use.
-fetch_and_count 3 7 7
-fetch_and_count 4 15 15
-fetch_and_count 5 31 31
-fetch_and_count 6 41 41

wrolpi/scripts/migrate_map_db.sh

@@ -1,58 +0,0 @@
-#! /usr/bin/env bash
-# Moves the `gis` map from default database on 5432 to a new database on 5433.
-# This script should only need to be run once.  If you are having issues with your map, try
-# /opt/wrolpi/scripts/reset_map.sh
-
-# Re-execute this script if it wasn't called with sudo.
-if [ $EUID != 0 ]; then
-  sudo "$0" "$@"
-  exit $?
-fi
-
-sudo -iu postgres psql -c '\l' --port=5433 2>/dev/null | grep -q gis && (
-  echo "Map DB already migrated."
-  exit 0
-)
-
-function cleanup() {
-  if [ -e /tmp/map.dump ]; then
-    rm -rf /tmp/map.dump
-  fi
-}
-
-trap cleanup EXIT
-cleanup
-
-systemctl stop renderd
-
-# Create new cluster for the map.
-pg_createcluster 15 map --port=5433 --start -e utf8
-# Disable JIT as recommended by mod_tile
-sed -i 's/#jit =.*/jit = off/' /etc/postgresql/15/map/postgresql.conf
-
-# Create user permissions for map user.
-sudo -iu postgres psql -c '\l' --port=5433 | grep gis || (
-  sudo -iu postgres createuser --port=5433 _renderd
-  sudo -iu postgres psql --port=5433 -c "alter user _renderd password 'wrolpi'"
-  sudo -iu postgres createdb --port=5433 -E UTF8 -O _renderd gis
-  sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION postgis'
-  sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION hstore'
-)
-
-# Override niceness of new map database.
-cp -r /opt/wrolpi/etc/raspberrypios/postgresql@15-map.service.d /etc/systemd/system/
-systemctl daemon-reload
-systemctl restart postgresql@15-map.service
-
-# Dump old DB, then restore in new DB.
->&2 echo "Dumping map database, this may take several hours if your map is large..."
-sudo -iu postgres pg_dump -Fd gis --port=5432 -j3 -f /tmp/map.dump &&  # 5432 is old DB.
-  sudo -iu postgres pg_restore --port=5433 -j3 -d gis /tmp/map.dump && # 5433 is new DB.
-  sudo -iu postgres dropdb --port=5432 gis
-
-# Update openstreetmap-carto to use the new database.
-grep -q 'port: 5433' /opt/openstreetmap-carto/project.mml || (
-  # Append port line after dbname configuration line.
-  sed -i '/dbname: "gis"/a \    port: 5433' /opt/openstreetmap-carto/project.mml
-  (cd /opt/openstreetmap-carto/ && carto project.mml >mapnik.xml)
-)