diff --git a/README.md b/README.md
index bd2f80e8..9b16faa0 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ network so that any user with a laptop/tablet/phone can connect and use the libr
| Web Archives | yes | yes | yes | [Singlefile](https://github.com/gildas-lormeau/SingleFile) |
| Wikipedia | yes | yes | yes | [Kiwix](https://www.kiwix.org) |
| eBooks | yes | yes | planned | EPUB/PDF |
-| Map | yes | planned | yes | [Protomaps](https://protomaps.com/) |
+| Map | yes | yes | yes | [Protomaps](https://protomaps.com/) |
| Podcasts/Audio | yes | planned | planned | |
# Demo
diff --git a/app/src/Events.js b/app/src/Events.js
index fc9990bf..be4b6795 100644
--- a/app/src/Events.js
+++ b/app/src/Events.js
@@ -54,6 +54,14 @@ function handleEvents(events) {
eventToast('Map import failed', message, 'error');
}
+ if (event === 'map_search_complete') {
+ eventToast('Map Search Index', message, 'success');
+ }
+
+ if (event === 'map_search_failed') {
+ eventToast('Map Search Index Failed', message, 'error');
+ }
+
if (event === 'user_notify_message') {
console.log(message, url);
eventToast(
diff --git a/app/src/api.js b/app/src/api.js
index a0e87203..cb795d77 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -1580,6 +1580,44 @@ export async function setMapDefaultLocation(lat, lon, zoom) {
}
}
+export async function searchMap(query, limit = 12, offset = 0, lat = null, lon = null) {
+ let url = `${API_URI}/map/search?q=${encodeURIComponent(query)}&limit=${limit}&offset=${offset}`;
+ if (lat !== null && lon !== null) {
+ url += `&lat=${lat}&lon=${lon}`;
+ }
+ const response = await apiGet(url);
+ if (!response.ok) {
+ return {results: [], total: 0};
+ }
+ return await response.json();
+}
+
+export async function getMapSearchStatus() {
+ const response = await apiGet(`${API_URI}/map/search/status`);
+ if (!response.ok) {
+ return {indexed: [], missing: []};
+ }
+ return await response.json();
+}
+
+export async function searchEstimateMap(search_str) {
+ if (!search_str) return {mapPlaces: 0};
+ const response = await apiGet(`${API_URI}/map/search/estimate?q=${encodeURIComponent(search_str)}`);
+ if (!response.ok) return {mapPlaces: 0};
+ const data = await response.json();
+ return {mapPlaces: data.map_places || 0};
+}
+
+export async function rebuildMapSearchIndex(filename) {
+ const response = await apiPost(`${API_URI}/map/search/rebuild/${encodeURIComponent(filename)}`);
+ if (!response.ok) {
+ const message = await getErrorMessage(response, 'Could not rebuild search index');
+ toast({type: 'error', title: 'Map Error', description: message, time: 5000});
+ } else {
+ toast({type: 'success', title: 'Search Index', description: 'Search index build started', time: 3000});
+ }
+}
+
export async function clearCompletedDownloads() {
const response = await apiPost(`${API_URI}/download/clear_completed`);
if (!response.ok) {
diff --git a/app/src/components/Map.js b/app/src/components/Map.js
index 20b02f18..8bbc10de 100644
--- a/app/src/components/Map.js
+++ b/app/src/components/Map.js
@@ -13,7 +13,7 @@ import {
import {Route, Routes, useNavigate} from "react-router";
import {
deleteMapFile, deleteMapPin, fetchMapSubscriptions, getMapFiles, getMapPins,
- mapSubscribe, mapUnsubscribe, updateMapPin
+ mapSubscribe, mapUnsubscribe, rebuildMapSearchIndex, updateMapPin
} from "../api";
import {
Button,
@@ -25,7 +25,7 @@ import {
import {SortableTable} from "./SortableTable";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {Header} from "./Theme";
-import {Media} from "../contexts/contexts";
+import {Media, StatusContext} from "../contexts/contexts";
import {MAP_VIEWER_URI} from "./Vars";
import {Modal} from "./Theme";
import MapViewer from "./MapViewer";
@@ -265,6 +265,8 @@ function ManageMap() {
const [catalog, setCatalog] = useState([]);
const [subscriptions, setSubscriptions] = useState([]);
const [allRegionsOpen, setAllRegionsOpen] = useState(false);
+ const {status} = React.useContext(StatusContext);
+ const searchBuilding = status?.flags?.map_search_building;
const fetchFiles = async () => {
try {
@@ -306,13 +308,31 @@ function ManageMap() {
const fileHeaders = [
{key: 'name', text: 'File', sortBy: i => i.name.toLowerCase()},
{key: 'size', text: 'Size', sortBy: i => i.size || 0},
+ {key: 'search', text: 'Search', sortBy: null},
{key: 'actions', text: '', sortBy: null},
];
+ const handleBuildSearch = async (filename) => {
+ await rebuildMapSearchIndex(filename);
+ await fetchFiles();
+ };
+
const fileRowFunc = (file) => {
return
{file.name}
{humanFileSize(file.size)}
+
+ {file.name.startsWith('terrain')
+ ? null
+ : file.has_search_index
+ ?
+ : handleBuildSearch(file.name)}/>
+ }
+
- No PMTiles map files were found in map/
+ No PMTiles map files were found in map/
;
// --- Catalog table ---
diff --git a/app/src/components/MapSearchView.js b/app/src/components/MapSearchView.js
new file mode 100644
index 00000000..0437d3ca
--- /dev/null
+++ b/app/src/components/MapSearchView.js
@@ -0,0 +1,133 @@
+import React, {useEffect, useState} from "react";
+import {Link, useNavigate} from "react-router";
+import {CardContent, CardDescription, CardGroup, CardHeader, CardMeta} from "semantic-ui-react";
+import {getMapSearchStatus, searchMap} from "../api";
+import {Card, Header, Loader, Segment} from "./Theme";
+import {Paginator} from "./Common";
+import {useLatestRequest, usePages} from "../hooks/customHooks";
+import {QueryContext, ThemeContext} from "../contexts/contexts";
+import {MAP_VIEWER_URI} from "./Vars";
+
+function formatPopulation(pop) {
+ if (!pop) return null;
+ if (pop >= 1_000_000) return `${(pop / 1_000_000).toFixed(1)}M`;
+ if (pop >= 1_000) return `${Math.round(pop / 1_000)}K`;
+ return `${pop}`;
+}
+
+function MapPlaceCard({place, index}) {
+ const navigate = useNavigate();
+ const {s} = React.useContext(ThemeContext);
+ const [iframeSrc, setIframeSrc] = useState(null);
+
+ const zoom = place.min_zoom ? Math.max(8, Math.round(13 - place.min_zoom / 3)) : 11;
+ const embedSrc = `${MAP_VIEWER_URI}/embed.html#${place.lat},${place.lon},${zoom}`;
+
+ // Stagger iframe loading to avoid overwhelming the map viewer with simultaneous requests.
+ useEffect(() => {
+ const timer = setTimeout(() => setIframeSrc(embedSrc), 100 + index * 200);
+ return () => clearTimeout(timer);
+ }, [embedSrc, index]);
+
+ const handleClick = () => {
+ navigate(`/map?lat=${place.lat}&lon=${place.lon}&z=${zoom}`);
+ };
+
+ const subtitle = [];
+ if (place.region) subtitle.push(place.region);
+ const detail = place.kind_detail || place.kind || "";
+ if (detail && !place.region) subtitle.push(detail.replace(/_/g, " "));
+ const pop = formatPopulation(place.population);
+ if (pop) subtitle.push(`pop. ${pop}`);
+
+ return
+
+ {iframeSrc && }
+
+
+ {place.name}
+ {subtitle.length > 0 && {subtitle.join(" \u00b7 ")}}
+
+ {place.lat.toFixed(4)}, {place.lon.toFixed(4)}
+
+
+ ;
+}
+
+export function MapSearchView() {
+ const {searchParams} = React.useContext(QueryContext);
+ const searchStr = searchParams.get("q") || "";
+ const {offset, limit, activePage, setPage, totalPages, setTotal} = usePages(12);
+ // useLatestRequest discards stale responses and catches errors, so rapid pagination
+ // can't show out-of-order results and a network failure can't leave the spinner stuck.
+ const {data, sendRequest, loading} = useLatestRequest(300);
+ const [hasMaps, setHasMaps] = useState(null);
+
+ useEffect(() => {
+ let cancelled = false;
+ (async () => {
+ const status = await getMapSearchStatus();
+ if (!cancelled) {
+ const installed = (status.indexed || []).length + (status.missing || []).length;
+ setHasMaps(installed > 0);
+ }
+ })();
+ return () => {
+ cancelled = true;
+ };
+ }, []);
+
+ useEffect(() => {
+ if (!searchStr || searchStr.length < 2) {
+ setTotal(0);
+ return;
+ }
+ sendRequest(async () => await searchMap(searchStr, limit, offset));
+ }, [searchStr, offset, limit, sendRequest]);
+
+ const results = (data && data.results) || [];
+ useEffect(() => {
+ setTotal((data && data.total) || 0);
+ }, [data, setTotal]);
+
+ if (hasMaps === false) {
+ return
+
+ Subscribe to a map region to search for places.
+ Go to Map Manager
+ ;
+ }
+
+ if (loading) {
+ return ;
+ }
+
+ if (!searchStr) {
+ return Enter a search query to find map locations.;
+ }
+
+ if (results.length === 0) {
+ return No map locations found for "{searchStr}".;
+ }
+
+ return
+
+ {results.map((place, i) =>
+
+ )}
+
+
+
+
+ ;
+}
diff --git a/app/src/components/MapViewer.js b/app/src/components/MapViewer.js
index fac3910b..d805f229 100644
--- a/app/src/components/MapViewer.js
+++ b/app/src/components/MapViewer.js
@@ -5,7 +5,7 @@ import "maplibre-gl/dist/maplibre-gl.css";
import {Protocol} from "pmtiles";
import layers from "protomaps-themes-base";
import mlcontour from "@acalcutt/maplibre-contour-pmtiles";
-import {addMapPin, deleteMapPin, getMapFiles, getMapPins, setMapDefaultLocation} from "../api";
+import {addMapPin, deleteMapPin, getMapFiles, getMapPins, searchMap, setMapDefaultLocation} from "../api";
import {MAP_VIEWER_URI} from "./Vars";
import {Checkbox, Icon, Segment} from "semantic-ui-react";
import {Header} from "./Theme";
@@ -221,6 +221,21 @@ function LayerControl({map, scaleUnit, onScaleUnitChange, visibilityRef}) {
return v;
});
const [collapsed, setCollapsed] = useState(true);
+ const [isMobile, setIsMobile] = useState(
+ () => typeof window !== "undefined" && window.matchMedia("(max-width: 768px)").matches
+ );
+
+ useEffect(() => {
+ if (typeof window === "undefined") return;
+ const mq = window.matchMedia("(max-width: 768px)");
+ const handler = (e) => setIsMobile(e.matches);
+ mq.addEventListener ? mq.addEventListener("change", handler) : mq.addListener(handler);
+ return () => {
+ mq.removeEventListener ? mq.removeEventListener("change", handler) : mq.removeListener(handler);
+ };
+ }, []);
+
+ const iconOnly = isMobile && collapsed;
const toggleGroup = useCallback((groupName, checked) => {
if (!map) return;
@@ -251,21 +266,35 @@ function LayerControl({map, scaleUnit, onScaleUnitChange, visibilityRef}) {
boxShadow: "0 2px 8px rgba(0,0,0,0.2)",
maxHeight: "calc(100vh - 120px)",
overflowY: "auto",
- minWidth: 200,
+ minWidth: iconOnly ? 0 : 200,
fontSize: 13,
};
+ const headerStyle = iconOnly
+ ? {
+ padding: 0, cursor: "pointer", userSelect: "none",
+ display: "flex", alignItems: "center", justifyContent: "center",
+ width: 40, height: 40,
+ }
+ : {
+ padding: "8px 12px", cursor: "pointer", fontWeight: 600,
+ display: "flex", justifyContent: "space-between", alignItems: "center",
+ borderBottom: collapsed ? "none" : "1px solid #eee", userSelect: "none",
+ };
+
return
setCollapsed(c => !c)}
+ title="Layers"
+ aria-label="Layers"
>
- Layers
-
+ {iconOnly
+ ?
+ : <>
+ Layers
+
+ >}
{!collapsed &&
{Object.keys(LAYER_GROUPS).map(name =>
@@ -336,6 +365,150 @@ function AddPinDialog({lat, lon, onSubmit, onCancel}) {
;
}
+// Coordinate regex: "45.5, -122.6" or "45.5 -122.6"
+const COORD_REGEX = /^\s*(-?\d+\.?\d*)\s*[,\s]\s*(-?\d+\.?\d*)\s*$/;
+
+function MapSearch({map}) {
+ const [query, setQuery] = useState("");
+ const [results, setResults] = useState([]);
+ const [showResults, setShowResults] = useState(false);
+ const [searchMarker, setSearchMarker] = useState(null);
+ const debounceRef = useRef(null);
+ const containerRef = useRef(null);
+
+ // Close results when clicking outside.
+ useEffect(() => {
+ const handler = (e) => {
+ if (containerRef.current && !containerRef.current.contains(e.target)) {
+ setShowResults(false);
+ }
+ };
+ document.addEventListener("click", handler);
+ return () => document.removeEventListener("click", handler);
+ }, []);
+
+ const doSearch = useCallback((q) => {
+ if (!q || q.length < 2) {
+ setResults([]);
+ setShowResults(false);
+ return;
+ }
+
+ // Check for coordinate input first.
+ const coordMatch = q.match(COORD_REGEX);
+ if (coordMatch) {
+ const lat = parseFloat(coordMatch[1]);
+ const lon = parseFloat(coordMatch[2]);
+ if (lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180) {
+ setResults([{name: `${lat}, ${lon}`, kind: "coordinates", lat, lon, min_zoom: 0, source: "input"}]);
+ setShowResults(true);
+ return;
+ }
+ }
+
+ // Get current map center for proximity bias.
+ let lat = null, lon = null;
+ if (map) {
+ const center = map.getCenter();
+ lat = center.lat;
+ lon = center.lng;
+ }
+
+ searchMap(q, 8, 0, lat, lon).then(data => {
+ setResults(data.results || []);
+ setShowResults((data.results || []).length > 0);
+ });
+ }, [map]);
+
+ const handleInput = useCallback((e) => {
+ const q = e.target.value;
+ setQuery(q);
+ clearTimeout(debounceRef.current);
+ debounceRef.current = setTimeout(() => doSearch(q), 300);
+ }, [doSearch]);
+
+ const handleSelect = useCallback((result) => {
+ if (!map) return;
+
+ // Remove old search marker.
+ if (searchMarker) {
+ searchMarker.remove();
+ }
+
+ const zoom = result.kind === "coordinates" ? 14 : Math.max(map.getZoom(), 12);
+ map.flyTo({center: [result.lon, result.lat], zoom});
+
+ const marker = new maplibregl.Marker({color: "#6435c9"})
+ .setLngLat([result.lon, result.lat])
+ .setPopup(new maplibregl.Popup({offset: 25}).setText(result.name))
+ .addTo(map);
+ marker.togglePopup();
+ setSearchMarker(marker);
+
+ setShowResults(false);
+ setQuery(result.name);
+ }, [map, searchMarker]);
+
+ const handleKeyDown = useCallback((e) => {
+ if (e.key === "Escape") {
+ setShowResults(false);
+ e.target.blur();
+ }
+ }, []);
+
+ const formatPopulation = (pop) => {
+ if (!pop) return "";
+ if (pop >= 1_000_000) return `${(pop / 1_000_000).toFixed(1)}M`;
+ if (pop >= 1_000) return `${(pop / 1_000).toFixed(0)}K`;
+ return `${pop}`;
+ };
+
+ const resultSubtext = (r) => {
+ if (r.kind === "coordinates") return "";
+ const parts = [];
+ if (r.region) parts.push(r.region);
+ const detail = r.kind_detail || r.kind || "";
+ if (detail && !r.region) parts.push(detail.replace(/_/g, " "));
+ const pop = formatPopulation(r.population);
+ if (pop) parts.push(`pop. ${pop}`);
+ return parts.join(" \u00b7 ");
+ };
+
+ return
+
results.length > 0 && setShowResults(true)}
+ onKeyDown={handleKeyDown}
+ style={{
+ width: "100%", padding: "8px 12px", border: "1px solid #ccc", borderRadius: 4,
+ fontSize: 14, boxShadow: "0 2px 6px rgba(0,0,0,0.2)", background: "white",
+ }}
+ />
+ {showResults && results.length > 0 &&
+ {results.map((r, i) =>
handleSelect(r)}
+ onMouseEnter={e => e.target.style.background = "#f0f0f0"}
+ onMouseLeave={e => e.target.style.background = "transparent"}
+ style={{padding: "8px 12px", cursor: "pointer", borderBottom: "1px solid #eee"}}
+ >
+
{r.name}
+ {resultSubtext(r) &&
{resultSubtext(r)}
}
+
)}
+
}
+
;
+}
+
export default function MapViewer() {
const mapContainer = useRef(null);
const mapRef = useRef(null);
@@ -742,6 +915,7 @@ export default function MapViewer() {
}
return
+ {mapReady &&
}
{mapReady && }
{contextMenu && {
if (searchStr || (tagNames && tagNames.length > 0)) {
@@ -141,6 +143,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
if (searchStr && searchStr.length > 0) {
// We can't search Channels/Domains without some string to filter by.
sendGeneralReqeust(async () => await searchSuggestions(searchStr));
+ sendMapRequest(async () => await searchEstimateMap(searchStr));
}
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag));
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
@@ -195,6 +198,14 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
}
}, [setSuggestions, otherData]);
+ React.useEffect(() => {
+ if (!_.isEmpty(mapData)) {
+ setSuggestions((prevState) => {
+ return {...prevState, mapPlaces: mapData.mapPlaces}
+ });
+ }
+ }, [setSuggestions, mapData]);
+
return {suggestions, loading: generalLoading || zimLoading || filesLoading}
}
@@ -329,6 +340,18 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
const coordMatch = searchStr && searchStr.trim().match(
/^(-?\d+\.?\d*)[,\s]+(-?\d+\.?\d*)$/
);
+ // Map places suggestion in dropdown.
+ if (newSuggestions.mapPlaces > 0) {
+ results['map places'] = {
+ name: 'Map Places', results: [{
+ type: 'search',
+ title: `Map: ${newSuggestions.mapPlaces} places`,
+ description: 'Search map locations',
+ location: `/search/map?q=${encodeURIComponent(searchStr)}`,
+ }]
+ };
+ }
+
if (coordMatch) {
const lat = parseFloat(coordMatch[1]);
const lon = parseFloat(coordMatch[2]);
@@ -355,6 +378,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
fileGroups: newSuggestions.fileGroups,
zims: zimSum,
otherSum: otherSum,
+ mapPlaces: newSuggestions.mapPlaces || 0,
channels: newSuggestions.channels.length,
domains: newSuggestions.domains.length,
authors: newSuggestions.authors?.length,
@@ -441,11 +465,13 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
const filesTabName = Files ;
const zimsTabName = Zims ;
+ const mapTabName = Map ;
const othersTabName = Other ;
const links = [
{text: filesTabName, to: `/search${search}`, key: 'filesSearch_', end: true},
{text: zimsTabName, to: `/search/zim${search}`, key: 'zimsSearch'},
+ {text: mapTabName, to: `/search/map${search}`, key: 'mapSearch'},
{text: othersTabName, to: `/search/other${search}`, key: 'othersSearch'},
];
@@ -454,6 +480,7 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
}/>
}/>
+ }/>
}/>
diff --git a/controller/templates/index.html b/controller/templates/index.html
index 8e3216ed..c51b99b7 100644
--- a/controller/templates/index.html
+++ b/controller/templates/index.html
@@ -881,6 +881,16 @@
{% endif %}
+{% if not docker_mode %}
+
+
Script Running
+
+
+
+
+
+{% endif %}
+
{% set app_ready = services | selectattr('name', 'in', ['caddy', 'web']) | selectattr('status', 'equalto', 'running') | list | length > 0 %}
Proceed to WROLPi
@@ -892,16 +902,6 @@
Main UI is unavailable. Using Controller fallback. Will redirect when ready...
-{% if not docker_mode %}
-
-
Script Running
-
-
-
-
-
-{% endif %}
-
System Status
diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index ccbd7d10..45909ca0 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -3,7 +3,14 @@ ENV DOCKER=true
WORKDIR /opt/wrolpi
# Update dependencies for the services install.
-RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unrar-free unzip
+RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unrar-free unzip \
+ build-essential libsqlite3-dev zlib1g-dev git
+
+# Install tippecanoe (for map search index building via tippecanoe-decode).
+RUN git clone --depth 1 --branch 2.79.0 https://github.com/felt/tippecanoe.git /tmp/tippecanoe \
+ && cd /tmp/tippecanoe && make -j$(nproc) && make install \
+ && rm -rf /tmp/tippecanoe \
+ && tippecanoe-decode --help 2>&1 | head -1
RUN ffmpeg -version
# Install Deno runtime for yt-dlp
diff --git a/modules/map/api.py b/modules/map/api.py
index 0d875426..2e2f72d8 100644
--- a/modules/map/api.py
+++ b/modules/map/api.py
@@ -4,7 +4,7 @@ from sanic import Request, response, Blueprint
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
-from modules.map import lib, schema
+from modules.map import lib, schema, search
from modules.map.pins import get_map_pins_config
from wrolpi.api_utils import json_response
from wrolpi.common import wrol_mode_check
@@ -98,3 +98,63 @@ async def update_pin(_: Request, pin_id: int, body: schema.MapPinUpdateRequest):
if get_map_pins_config().update_pin(pin_id, body.label, body.color):
return response.empty(HTTPStatus.NO_CONTENT)
return response.json({'error': 'Pin not found'}, HTTPStatus.NOT_FOUND)
+
+
+@map_bp.get('/search')
+@openapi.description('Search for places in map search indexes')
+async def search_places(request: Request):
+ q = request.args.get('q', '')
+ if not q or len(q.strip()) < 1:
+ return json_response({'error': 'Query parameter "q" is required'}, HTTPStatus.BAD_REQUEST)
+
+ try:
+ limit = int(request.args.get('limit', 12))
+ offset = int(request.args.get('offset', 0))
+ lat = float(request.args.get('lat')) if request.args.get('lat') is not None else None
+ lon = float(request.args.get('lon')) if request.args.get('lon') is not None else None
+ except (ValueError, TypeError):
+ return json_response({'error': 'Invalid query parameters'}, HTTPStatus.BAD_REQUEST)
+
+ data = search.search_places(q.strip(), limit=limit, offset=offset, lat=lat, lon=lon)
+ return json_response(data, HTTPStatus.OK)
+
+
+@map_bp.get('/search/estimate')
+@openapi.description('Get estimated count of map search results')
+async def get_search_estimate(request: Request):
+ q = request.args.get('q', '')
+ if not q or len(q.strip()) < 1:
+ return json_response({'map_places': 0}, HTTPStatus.OK)
+ count = search.search_places_count(q.strip())
+ return json_response({'map_places': count}, HTTPStatus.OK)
+
+
+@map_bp.get('/search/status')
+@openapi.description('Get status of map search indexes')
+async def get_search_status(_: Request):
+ status = search.get_search_status()
+ return json_response(status, HTTPStatus.OK)
+
+
+@map_bp.post('/search/rebuild')
+@openapi.description('Rebuild all map search indexes')
+@wrol_mode_check
+async def rebuild_search_indexes(_: Request):
+ started = search.rebuild_all_search_indexes()
+ if started:
+ return json_response({'message': 'Search index rebuild started'}, HTTPStatus.ACCEPTED)
+ return json_response({'error': 'No PMTiles files found'}, HTTPStatus.NOT_FOUND)
+
+
+@map_bp.post('/search/rebuild/')
+@openapi.description('Rebuild search index for a single PMTiles file')
+@wrol_mode_check
+async def rebuild_single_search_index(_: Request, filename: str):
+ try:
+ started = search.rebuild_search_index(filename)
+ except ValueError as e:
+ return response.json({'error': str(e)}, HTTPStatus.BAD_REQUEST)
+
+ if started:
+ return json_response({'message': f'Search index rebuild started for {filename}'}, HTTPStatus.ACCEPTED)
+ return json_response({'error': 'File not found'}, HTTPStatus.NOT_FOUND)
diff --git a/modules/map/downloader.py b/modules/map/downloader.py
index 5373bc34..b9aadc6d 100644
--- a/modules/map/downloader.py
+++ b/modules/map/downloader.py
@@ -7,7 +7,8 @@ from wrolpi.common import logger, verify_gpg_signature
from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult, download_manager
-__all__ = ['MapCatalogDownloader', 'MapExtractDownloader', 'map_catalog_downloader', 'map_extract_downloader']
+__all__ = ['MapCatalogDownloader', 'MapExtractDownloader', 'MapSearchIndexDownloader',
+ 'map_catalog_downloader', 'map_extract_downloader', 'map_search_index_downloader']
logger = logger.getChild(__name__)
@@ -66,6 +67,7 @@ class MapCatalogDownloader(Downloader):
version = manifest.get('version', '')
manifest_regions = manifest.get('regions', {})
+ search_indexes = manifest.get('search_indexes', {})
if not manifest_regions:
return DownloadResult(success=False, error='Manifest does not contain regions')
@@ -74,7 +76,13 @@ class MapCatalogDownloader(Downloader):
if not regions:
return DownloadResult(success=True)
+ map_directory = lib.get_map_directory()
+
created = 0
+ # Defer search-index downloads so they are all inserted AFTER every pmtiles
+ # download. Downloads process in insertion order; this prevents a search-index
+ # download from running before its companion pmtiles file exists on disk.
+ pending_search_downloads = []
with get_db_session(commit=True) as session:
for r in regions:
region = r.get('region', '')
@@ -83,6 +91,12 @@ class MapCatalogDownloader(Downloader):
logger.warning(f'Subscribed region {region!r} not found in manifest — skipping')
continue
+ # Compute the expected PMTiles filename.
+ if version:
+ pmtiles_name = f'{region}-{version}.pmtiles'
+ else:
+ pmtiles_name = f'{region}.pmtiles'
+
# Delete stale download for this region if the URL changed (new version).
old = session.query(Download).filter(
Download.downloader == 'map_extract',
@@ -101,14 +115,34 @@ class MapCatalogDownloader(Downloader):
child.settings = {
'region': region,
'version': version,
+ 'search_index_url': search_indexes.get(region, ''),
}
child.renew(reset_attempts=True)
created += 1
+ # Queue search index download if the CDN provides one and the
+ # .search.db file does not already exist on disk.
+ search_url = search_indexes.get(region)
+ search_db_path = (map_directory / pmtiles_name).with_suffix('.search.db')
+ if search_url and not search_db_path.is_file():
+ pending_search_downloads.append((region, search_url, pmtiles_name))
+
+ # Insert all search-index downloads after every pmtiles download, so they
+ # process in insertion order behind their companion pmtiles.
+ for region, search_url, pmtiles_name in pending_search_downloads:
+ search_child = download_manager.get_or_create_download(
+ session, search_url, reset_attempts=True, override_skip=True)
+ search_child.downloader = 'map_search_index'
+ search_child.location = '/map/manage'
+ search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name}
+ search_child.renew(reset_attempts=True)
+ logger.info(f'Created search index download for {region}')
+
if not created:
return DownloadResult(success=False, error='No matching region URLs found in manifest')
logger.info(f'Created {created} map region downloads')
+
return DownloadResult(success=True, location='/map/manage')
@@ -201,7 +235,86 @@ class MapExtractDownloader(Downloader):
size = output_path.stat().st_size
logger.warning(f'Successfully downloaded {region} ({size} bytes) to {output_path}')
+ # Build search index locally if tippecanoe is available and no CDN
+ # search index download already exists for this region.
+ has_search_download = False
+ with get_db_session() as session:
+ has_search_download = session.query(Download).filter(
+ Download.downloader == 'map_search_index',
+ Download.settings['region'].astext == region,
+ ).count() > 0
+ if not has_search_download:
+ try:
+ from modules.map.search import rebuild_search_index
+ rebuild_search_index(output_path.name)
+ except Exception as e:
+ logger.error(f'Failed to start search index build for {region}: {e}')
+
return DownloadResult(success=True, location='/map')
map_extract_downloader = MapExtractDownloader()
+
+
+class MapSearchIndexDownloader(Downloader):
+ """Downloads a pre-built .search.db file from the CDN via aria2c with meta4 verification.
+
+ Created by MapCatalogDownloader alongside map_extract downloads for each
+ subscribed region. Will wait (fail + retry) until the PMTiles file exists.
+ """
+
+ name = 'map_search_index'
+ listable = False
+
+ async def do_download(self, download: Download) -> DownloadResult:
+ settings = download.settings or {}
+ region = settings.get('region', '')
+ pmtiles_name = settings.get('pmtiles_name', '')
+
+ if not region or not pmtiles_name:
+ return DownloadResult(success=False, error='Missing region or pmtiles_name in settings')
+
+ map_directory = lib.get_map_directory()
+ pmtiles_path = map_directory / pmtiles_name
+
+ if not pmtiles_path.is_file():
+ return DownloadResult(success=False, error=f'PMTiles file not found: {pmtiles_name}')
+
+ # Name the search.db to match the local PMTiles filename.
+ search_db_path = pmtiles_path.with_suffix('.search.db')
+
+ if search_db_path.is_file():
+ logger.info(f'Search index already exists for {region}, skipping')
+ return DownloadResult(success=True, location='/map')
+
+ # Fetch and verify meta4 (GPG-signed hash).
+ meta4_contents = await self.get_meta4_contents(download.url)
+ if meta4_contents:
+ if not await verify_meta4_signature(meta4_contents):
+ return DownloadResult(success=False,
+ error=f'meta4 signature verification failed for {region} search index')
+ logger.info(f'Search index meta4 verified for {region}')
+
+ # Download via aria2c with meta4 hash verification.
+ logger.warning(f'Downloading search index for {region} from {download.url}')
+ downloaded_path = None
+ try:
+ downloaded_path = await self.download_file(
+ download, download.url, map_directory,
+ check_for_meta4=False, concurrent=1, meta4_xml=meta4_contents,
+ )
+ # Rename to match the local PMTiles filename.
+ downloaded_path.rename(search_db_path)
+ except Exception as e:
+ logger.error(f'Search index download failed for {region}: {e}')
+ if downloaded_path:
+ downloaded_path.unlink(missing_ok=True)
+ search_db_path.unlink(missing_ok=True)
+ return DownloadResult(success=False, error=f'Download failed: {str(e)[:500]}')
+
+ size = search_db_path.stat().st_size
+ logger.warning(f'Downloaded search index for {region} ({size} bytes)')
+ return DownloadResult(success=True, location='/map')
+
+
+map_search_index_downloader = MapSearchIndexDownloader()
diff --git a/modules/map/lib.py b/modules/map/lib.py
index 7890d2e6..5e706b41 100644
--- a/modules/map/lib.py
+++ b/modules/map/lib.py
@@ -33,13 +33,16 @@ def get_pmtiles_files() -> List[dict]:
path=str(path.relative_to(get_media_directory())),
size=stat.st_size,
mtime=stat.st_mtime,
+ has_search_index=path.with_suffix('.search.db').is_file(),
))
return files
def delete_pmtiles_file(filename: str) -> bool:
- """Delete a PMTiles file from the map directory. Returns True if the file was deleted."""
+ """Delete a PMTiles file from the map directory. Returns True if the file was deleted.
+
+ Also deletes the companion .search.db file if it exists."""
map_directory = get_map_directory()
# Prevent path traversal.
path = (map_directory / filename).resolve()
@@ -49,6 +52,11 @@ def delete_pmtiles_file(filename: str) -> bool:
if path.is_file() and path.suffix == '.pmtiles':
path.unlink()
logger.warning(f'Deleted PMTiles file: {path}')
+ # Also delete companion search index.
+ search_db = path.with_suffix('.search.db')
+ if search_db.is_file():
+ search_db.unlink()
+ logger.warning(f'Deleted search index: {search_db}')
return True
return False
@@ -73,7 +81,6 @@ def _get_or_create_map_download(session: Session) -> Download:
download = download_manager.get_or_create_download(session, MANIFEST_URL, reset_attempts=True,
override_skip=True)
download.downloader = 'map_catalog'
- download.sub_downloader = 'map_extract'
download.frequency = DownloadFrequency.days180
download.settings = {'regions': []}
download.location = '/map/manage'
diff --git a/modules/map/search.py b/modules/map/search.py
new file mode 100644
index 00000000..8806d7b8
--- /dev/null
+++ b/modules/map/search.py
@@ -0,0 +1,295 @@
+import asyncio
+import contextlib
+import math
+import shutil
+import sqlite3
+import sys
+from pathlib import Path
+from typing import List
+
+from modules.map.lib import get_map_directory
+from wrolpi.common import logger, background_task
+from wrolpi.events import Events
+from wrolpi.flags import map_search_building
+
+logger = logger.getChild(__name__)
+
+BUILD_SCRIPT = Path(__file__).parent.parent.parent / 'wrolpi' / 'scripts' / 'build_map_search.py'
+
+
+def get_search_db_files() -> List[Path]:
+ """Find all .search.db files in the map directory."""
+ map_directory = get_map_directory()
+ return sorted(map_directory.glob('*.search.db'))
+
+
+def search_places(query: str, limit: int = 12, offset: int = 0, lat: float = None, lon: float = None) -> dict:
+ """Search all .search.db files for places matching the query.
+
+ Results are ranked by importance (min_zoom) and optionally by proximity to lat/lon.
+ Returns {"results": [...], "total": N}.
+ """
+ db_files = get_search_db_files()
+ if not db_files:
+ return {'results': [], 'total': 0}
+
+ results = []
+
+ # Cap the number of rows fetched per DB to bound memory on large regional indexes
+ # (e.g. a prefix query like "S*" can match hundreds of thousands of rows). The
+ # cap is a multiple of (offset + limit) to keep enough headroom for ranking and
+ # cross-DB deduplication before pagination.
+ per_db_cap = max((offset + limit) * 5, 100)
+
+ # FTS5 match query with prefix matching. Escape embedded double quotes.
+ escaped = query.replace('"', '""')
+ fts_query = f'"{escaped}"*'
+
+ for db_path in db_files:
+ try:
+ with contextlib.closing(sqlite3.connect(f'file:{db_path}?mode=ro', uri=True)) as conn:
+ conn.row_factory = sqlite3.Row
+
+ rows = conn.execute("""
+ SELECT p.name, p.kind, p.lat, p.lon, p.min_zoom, p.source,
+ p.kind_detail, p.population, p.region,
+ places_fts.rank AS fts_rank
+ FROM places_fts
+ JOIN places p ON p.id = places_fts.rowid
+ WHERE places_fts MATCH ?
+ ORDER BY places_fts.rank
+ LIMIT ?
+ """, (fts_query, per_db_cap)).fetchall()
+
+ for row in rows:
+ result = dict(row)
+ if lat is not None and lon is not None:
+ dlat = result['lat'] - lat
+ dlon = result['lon'] - lon
+ result['distance'] = math.sqrt(dlat ** 2 + dlon ** 2)
+ results.append(result)
+ except sqlite3.OperationalError as e:
+ logger.warning(f'Search error in {db_path.name}: {e}')
+ continue
+
+ # Fuzzy fallback: if FTS5 found nothing, try LIKE substring matching.
+ if not results:
+ like_pattern = f'%{query}%'
+ for db_path in db_files:
+ try:
+ with contextlib.closing(sqlite3.connect(f'file:{db_path}?mode=ro', uri=True)) as conn:
+ conn.row_factory = sqlite3.Row
+
+ rows = conn.execute("""
+ SELECT p.name, p.kind, p.lat, p.lon, p.min_zoom, p.source,
+ p.kind_detail, p.population, p.region, 0 AS fts_rank
+ FROM places p
+ WHERE p.name LIKE ?
+ ORDER BY p.min_zoom, p.population DESC
+ LIMIT ?
+ """, (like_pattern, per_db_cap)).fetchall()
+
+ for row in rows:
+ result = dict(row)
+ if lat is not None and lon is not None:
+ dlat = result['lat'] - lat
+ dlon = result['lon'] - lon
+ result['distance'] = math.sqrt(dlat ** 2 + dlon ** 2)
+ results.append(result)
+ except sqlite3.OperationalError as e:
+ logger.warning(f'LIKE search error in {db_path.name}: {e}')
+ continue
+
+ # Sort by importance (lower min_zoom = more important), then by proximity or FTS rank.
+ # Coerce NULL min_zoom (possible from older schemas) to a default; `dict.get` returns
+ # None rather than the default for explicit NULL values.
+ if lat is not None and lon is not None:
+ results.sort(key=lambda r: (r.get('min_zoom') or 10, r.get('distance') or 0))
+ else:
+ results.sort(key=lambda r: (r.get('min_zoom') or 10, r.get('fts_rank') or 0))
+
+ # Deduplicate across sources: same name+kind at ~same location keeps highest population.
+ seen = {}
+ deduped = []
+ for r in results:
+ key = (r['name'], r['kind'], round(r['lat'], 2), round(r['lon'], 2))
+ if key not in seen:
+ seen[key] = len(deduped)
+ deduped.append(r)
+ else:
+ # Keep the entry with higher population.
+ existing = deduped[seen[key]]
+ if (r.get('population') or 0) > (existing.get('population') or 0):
+ deduped[seen[key]] = r
+
+ # Use the DB-wide count for `total` so pagination isn't capped by `per_db_cap`.
+ # Fall back to the deduped length if the count query fails (e.g. the per-DB cap
+ # was never hit, in which case they agree anyway).
+ if len(results) >= per_db_cap:
+ total = search_places_count(query)
+ else:
+ total = len(deduped)
+
+ # Remove internal ranking fields before returning.
+ final = []
+ for r in deduped[offset:offset + limit]:
+ r.pop('fts_rank', None)
+ r.pop('distance', None)
+ final.append(r)
+
+ return {'results': final, 'total': total}
+
+
+def search_places_count(query: str) -> int:
+ """Return the count of places matching the query across all search indexes."""
+ db_files = get_search_db_files()
+ if not db_files:
+ return 0
+
+ escaped = query.replace('"', '""')
+ fts_query = f'"{escaped}"*'
+ total = 0
+
+ for db_path in db_files:
+ try:
+ with contextlib.closing(sqlite3.connect(f'file:{db_path}?mode=ro', uri=True)) as conn:
+ row = conn.execute("""
+ SELECT COUNT(*) FROM places_fts WHERE places_fts MATCH ?
+ """, (fts_query,)).fetchone()
+ total += row[0] if row else 0
+ except sqlite3.OperationalError:
+ continue
+
+ # LIKE fallback count if FTS5 found nothing.
+ if total == 0:
+ like_pattern = f'%{query}%'
+ for db_path in db_files:
+ try:
+ with contextlib.closing(sqlite3.connect(f'file:{db_path}?mode=ro', uri=True)) as conn:
+ row = conn.execute("SELECT COUNT(*) FROM places WHERE name LIKE ?",
+ (like_pattern,)).fetchone()
+ total += row[0] if row else 0
+ except sqlite3.OperationalError:
+ continue
+
+ return total
+
+
+def get_search_status() -> dict:
+ """Return status of search indexes: which PMTiles files have indexes and which don't."""
+ map_directory = get_map_directory()
+
+ pmtiles_files = sorted(map_directory.glob('*.pmtiles'))
+ # Strip the double .search.db extension to get the PMTiles stem.
+ search_db_files = {p.name.removesuffix('.search.db') for p in get_search_db_files()}
+
+ indexed = []
+ missing = []
+
+ for pmtiles_path in pmtiles_files:
+ info = {'name': pmtiles_path.name, 'stem': pmtiles_path.stem}
+ if pmtiles_path.stem in search_db_files:
+ db_path = pmtiles_path.with_suffix('.search.db')
+ info['search_db_size'] = db_path.stat().st_size
+ indexed.append(info)
+ else:
+ missing.append(info)
+
+ return {
+ 'indexed': indexed,
+ 'missing': missing,
+ }
+
+
+def _check_tippecanoe() -> bool:
+ """Return True if tippecanoe-decode is on PATH. If missing, log and send an Event."""
+ if shutil.which('tippecanoe-decode'):
+ return True
+ msg = 'tippecanoe-decode not found on PATH — map search index cannot be built'
+ logger.error(msg)
+ Events.send_tippecanoe_missing(msg)
+ return False
+
+
+def _should_enrich() -> bool:
+ """Return True if Wikidata enrichment should be attempted (WROL mode is not active)."""
+ try:
+ from wrolpi.common import get_wrolpi_config
+ return not get_wrolpi_config().wrol_mode
+ except Exception:
+ return False
+
+
+async def _run_build(cmd: list, description: str):
+ """Run a build subprocess with the map_search_building flag set.
+
+ The flag is automatically cleared when the build finishes (success or failure).
+ """
+ with map_search_building:
+ logger.warning(f'Starting search index build: {description}')
+ proc = await asyncio.create_subprocess_exec(
+ *cmd,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.STDOUT,
+ )
+ stdout, _ = await proc.communicate()
+ output = stdout.decode()[-500:] if stdout else ''
+ if proc.returncode == 0:
+ logger.warning(f'Search index build completed: {description}')
+ Events.send_map_search_complete(f'Search index built for {description}')
+ else:
+ logger.error(f'Search index build failed for {description} (exit {proc.returncode}):\n{output}')
+ Events.send_map_search_failed(f'Search index build failed for {description}')
+
+
+def rebuild_search_index(pmtiles_name: str, max_zoom: int = 10) -> bool:
+ """Launch the build script as a background task for a single PMTiles file.
+
+ Returns True if the build was started, False if the file doesn't exist.
+ """
+ map_directory = get_map_directory()
+ pmtiles_path = (map_directory / pmtiles_name).resolve()
+
+ if not pmtiles_path.is_relative_to(map_directory.resolve()):
+ raise ValueError(f'Invalid filename: {pmtiles_name}')
+
+ if not pmtiles_path.is_file():
+ return False
+
+ if map_search_building.is_set():
+ logger.warning(f'Search index build already running — ignoring request for {pmtiles_name}')
+ return False
+
+ if not _check_tippecanoe():
+ return False
+
+ cmd = [sys.executable, str(BUILD_SCRIPT), str(pmtiles_path), '--max-zoom', str(max_zoom)]
+ if _should_enrich():
+ cmd.append('--enrich')
+
+ background_task(_run_build(cmd, pmtiles_name))
+ return True
+
+
+def rebuild_all_search_indexes(max_zoom: int = 10) -> bool:
+ """Launch the build script for the entire map directory as a background task."""
+ map_directory = get_map_directory()
+
+ if not any(map_directory.glob('*.pmtiles')):
+ return False
+
+ if map_search_building.is_set():
+ logger.warning('Search index build already running — ignoring rebuild-all request')
+ return False
+
+ if not _check_tippecanoe():
+ return False
+
+ cmd = [sys.executable, str(BUILD_SCRIPT), str(map_directory), '--max-zoom', str(max_zoom)]
+ if _should_enrich():
+ cmd.append('--enrich')
+
+ background_task(_run_build(cmd, 'all map files'))
+ return True
+
+
diff --git a/modules/map/static/embed.html b/modules/map/static/embed.html
new file mode 100644
index 00000000..3587e2e5
--- /dev/null
+++ b/modules/map/static/embed.html
@@ -0,0 +1,66 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/modules/map/static/index.html b/modules/map/static/index.html
index 8a0c70a9..0fc07b89 100644
--- a/modules/map/static/index.html
+++ b/modules/map/static/index.html
@@ -124,6 +124,43 @@
user-select: none;
}
#context-menu .action:hover { background: #f0f0f0; }
+
+ /* Search */
+ #search-container {
+ position: absolute;
+ top: 10px;
+ left: 50%;
+ transform: translateX(-50%);
+ z-index: 1000;
+ width: 320px;
+ max-width: calc(100% - 100px);
+ }
+ #search-input {
+ width: 100%;
+ padding: 8px 12px;
+ border: 1px solid #ccc;
+ border-radius: 4px;
+ font-size: 14px;
+ box-shadow: 0 2px 6px rgba(0,0,0,0.2);
+ }
+ #search-results {
+ display: none;
+ background: white;
+ border: 1px solid #ccc;
+ border-top: none;
+ border-radius: 0 0 4px 4px;
+ max-height: 300px;
+ overflow-y: auto;
+ box-shadow: 0 4px 8px rgba(0,0,0,0.2);
+ }
+ .search-result {
+ padding: 8px 12px;
+ cursor: pointer;
+ border-bottom: 1px solid #eee;
+ }
+ .search-result:hover { background: #f0f0f0; }
+ .search-result-name { font-weight: 500; }
+ .search-result-kind { font-size: 11px; color: #888; }
@@ -146,6 +183,11 @@
+
+