mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-25 18:40:06 -04:00
Map search added to global search. Preview cards are displayed for all results in new Map tab.
Map viewer has simple search. Map search databases can be downloaded from CDN, or built locally. Map searches can be built offline, but will not have enriched things like state names downloaded from Wikidata.
This commit is contained in:
parent
aaa3c0aff7
commit
c726654de9
21 changed files with 1831 additions and 62 deletions
|
|
@ -52,7 +52,7 @@ network so that any user with a laptop/tablet/phone can connect and use the libr
|
|||
| Web Archives | yes | yes | yes | [Singlefile](https://github.com/gildas-lormeau/SingleFile) |
|
||||
| Wikipedia | yes | yes | yes | [Kiwix](https://www.kiwix.org) |
|
||||
| eBooks | yes | yes | planned | EPUB/PDF |
|
||||
| Map | yes | planned | yes | [Protomaps](https://protomaps.com/) |
|
||||
| Map | yes | yes | yes | [Protomaps](https://protomaps.com/) |
|
||||
| Podcasts/Audio | yes | planned | planned | |
|
||||
|
||||
# Demo
|
||||
|
|
|
|||
|
|
@ -54,6 +54,14 @@ function handleEvents(events) {
|
|||
eventToast('Map import failed', message, 'error');
|
||||
}
|
||||
|
||||
if (event === 'map_search_complete') {
|
||||
eventToast('Map Search Index', message, 'success');
|
||||
}
|
||||
|
||||
if (event === 'map_search_failed') {
|
||||
eventToast('Map Search Index Failed', message, 'error');
|
||||
}
|
||||
|
||||
if (event === 'user_notify_message') {
|
||||
console.log(message, url);
|
||||
eventToast(
|
||||
|
|
|
|||
|
|
@ -1580,6 +1580,44 @@ export async function setMapDefaultLocation(lat, lon, zoom) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function searchMap(query, limit = 12, offset = 0, lat = null, lon = null) {
|
||||
let url = `${API_URI}/map/search?q=${encodeURIComponent(query)}&limit=${limit}&offset=${offset}`;
|
||||
if (lat !== null && lon !== null) {
|
||||
url += `&lat=${lat}&lon=${lon}`;
|
||||
}
|
||||
const response = await apiGet(url);
|
||||
if (!response.ok) {
|
||||
return {results: [], total: 0};
|
||||
}
|
||||
return await response.json();
|
||||
}
|
||||
|
||||
export async function getMapSearchStatus() {
|
||||
const response = await apiGet(`${API_URI}/map/search/status`);
|
||||
if (!response.ok) {
|
||||
return {indexed: [], missing: []};
|
||||
}
|
||||
return await response.json();
|
||||
}
|
||||
|
||||
export async function searchEstimateMap(search_str) {
|
||||
if (!search_str) return {mapPlaces: 0};
|
||||
const response = await apiGet(`${API_URI}/map/search/estimate?q=${encodeURIComponent(search_str)}`);
|
||||
if (!response.ok) return {mapPlaces: 0};
|
||||
const data = await response.json();
|
||||
return {mapPlaces: data.map_places || 0};
|
||||
}
|
||||
|
||||
export async function rebuildMapSearchIndex(filename) {
|
||||
const response = await apiPost(`${API_URI}/map/search/rebuild/${encodeURIComponent(filename)}`);
|
||||
if (!response.ok) {
|
||||
const message = await getErrorMessage(response, 'Could not rebuild search index');
|
||||
toast({type: 'error', title: 'Map Error', description: message, time: 5000});
|
||||
} else {
|
||||
toast({type: 'success', title: 'Search Index', description: 'Search index build started', time: 3000});
|
||||
}
|
||||
}
|
||||
|
||||
export async function clearCompletedDownloads() {
|
||||
const response = await apiPost(`${API_URI}/download/clear_completed`);
|
||||
if (!response.ok) {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import {
|
|||
import {Route, Routes, useNavigate} from "react-router";
|
||||
import {
|
||||
deleteMapFile, deleteMapPin, fetchMapSubscriptions, getMapFiles, getMapPins,
|
||||
mapSubscribe, mapUnsubscribe, updateMapPin
|
||||
mapSubscribe, mapUnsubscribe, rebuildMapSearchIndex, updateMapPin
|
||||
} from "../api";
|
||||
import {
|
||||
Button,
|
||||
|
|
@ -25,7 +25,7 @@ import {
|
|||
import {SortableTable} from "./SortableTable";
|
||||
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
|
||||
import {Header} from "./Theme";
|
||||
import {Media} from "../contexts/contexts";
|
||||
import {Media, StatusContext} from "../contexts/contexts";
|
||||
import {MAP_VIEWER_URI} from "./Vars";
|
||||
import {Modal} from "./Theme";
|
||||
import MapViewer from "./MapViewer";
|
||||
|
|
@ -265,6 +265,8 @@ function ManageMap() {
|
|||
const [catalog, setCatalog] = useState([]);
|
||||
const [subscriptions, setSubscriptions] = useState([]);
|
||||
const [allRegionsOpen, setAllRegionsOpen] = useState(false);
|
||||
const {status} = React.useContext(StatusContext);
|
||||
const searchBuilding = status?.flags?.map_search_building;
|
||||
|
||||
const fetchFiles = async () => {
|
||||
try {
|
||||
|
|
@ -306,13 +308,31 @@ function ManageMap() {
|
|||
const fileHeaders = [
|
||||
{key: 'name', text: 'File', sortBy: i => i.name.toLowerCase()},
|
||||
{key: 'size', text: 'Size', sortBy: i => i.size || 0},
|
||||
{key: 'search', text: 'Search', sortBy: null},
|
||||
{key: 'actions', text: '', sortBy: null},
|
||||
];
|
||||
|
||||
const handleBuildSearch = async (filename) => {
|
||||
await rebuildMapSearchIndex(filename);
|
||||
await fetchFiles();
|
||||
};
|
||||
|
||||
const fileRowFunc = (file) => {
|
||||
return <TableRow key={file.name}>
|
||||
<TableCell>{file.name}</TableCell>
|
||||
<TableCell>{humanFileSize(file.size)}</TableCell>
|
||||
<TableCell collapsing>
|
||||
{file.name.startsWith('terrain')
|
||||
? null
|
||||
: file.has_search_index
|
||||
? <Button size='tiny' icon='check' content='Built' disabled color='green'/>
|
||||
: <APIButton size='tiny' icon='search'
|
||||
content={searchBuilding ? 'Building...' : 'Build'}
|
||||
disabled={searchBuilding}
|
||||
loading={searchBuilding}
|
||||
onClick={() => handleBuildSearch(file.name)}/>
|
||||
}
|
||||
</TableCell>
|
||||
<TableCell collapsing>
|
||||
<APIButton
|
||||
color='red'
|
||||
|
|
@ -328,7 +348,7 @@ function ManageMap() {
|
|||
};
|
||||
|
||||
const emptyRow = <TableRow>
|
||||
<TableCell colSpan={3}>No PMTiles map files were found in <b>map/</b></TableCell>
|
||||
<TableCell colSpan={4}>No PMTiles map files were found in <b>map/</b></TableCell>
|
||||
</TableRow>;
|
||||
|
||||
// --- Catalog table ---
|
||||
|
|
|
|||
133
app/src/components/MapSearchView.js
Normal file
133
app/src/components/MapSearchView.js
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
import React, {useEffect, useState} from "react";
|
||||
import {Link, useNavigate} from "react-router";
|
||||
import {CardContent, CardDescription, CardGroup, CardHeader, CardMeta} from "semantic-ui-react";
|
||||
import {getMapSearchStatus, searchMap} from "../api";
|
||||
import {Card, Header, Loader, Segment} from "./Theme";
|
||||
import {Paginator} from "./Common";
|
||||
import {useLatestRequest, usePages} from "../hooks/customHooks";
|
||||
import {QueryContext, ThemeContext} from "../contexts/contexts";
|
||||
import {MAP_VIEWER_URI} from "./Vars";
|
||||
|
||||
function formatPopulation(pop) {
|
||||
if (!pop) return null;
|
||||
if (pop >= 1_000_000) return `${(pop / 1_000_000).toFixed(1)}M`;
|
||||
if (pop >= 1_000) return `${Math.round(pop / 1_000)}K`;
|
||||
return `${pop}`;
|
||||
}
|
||||
|
||||
function MapPlaceCard({place, index}) {
|
||||
const navigate = useNavigate();
|
||||
const {s} = React.useContext(ThemeContext);
|
||||
const [iframeSrc, setIframeSrc] = useState(null);
|
||||
|
||||
const zoom = place.min_zoom ? Math.max(8, Math.round(13 - place.min_zoom / 3)) : 11;
|
||||
const embedSrc = `${MAP_VIEWER_URI}/embed.html#${place.lat},${place.lon},${zoom}`;
|
||||
|
||||
// Stagger iframe loading to avoid overwhelming the map viewer with simultaneous requests.
|
||||
useEffect(() => {
|
||||
const timer = setTimeout(() => setIframeSrc(embedSrc), 100 + index * 200);
|
||||
return () => clearTimeout(timer);
|
||||
}, [embedSrc, index]);
|
||||
|
||||
const handleClick = () => {
|
||||
navigate(`/map?lat=${place.lat}&lon=${place.lon}&z=${zoom}`);
|
||||
};
|
||||
|
||||
const subtitle = [];
|
||||
if (place.region) subtitle.push(place.region);
|
||||
const detail = place.kind_detail || place.kind || "";
|
||||
if (detail && !place.region) subtitle.push(detail.replace(/_/g, " "));
|
||||
const pop = formatPopulation(place.population);
|
||||
if (pop) subtitle.push(`pop. ${pop}`);
|
||||
|
||||
return <Card onClick={handleClick} style={{cursor: "pointer"}}>
|
||||
<div style={{height: 150, overflow: "hidden", position: "relative", background: "#1a1a2e"}}>
|
||||
{iframeSrc && <iframe
|
||||
src={iframeSrc}
|
||||
title={place.name}
|
||||
loading="lazy"
|
||||
style={{
|
||||
width: "100%",
|
||||
height: "100%",
|
||||
border: "none",
|
||||
pointerEvents: "none",
|
||||
}}
|
||||
/>}
|
||||
</div>
|
||||
<CardContent {...s}>
|
||||
<CardHeader {...s}>{place.name}</CardHeader>
|
||||
{subtitle.length > 0 && <CardMeta {...s}>{subtitle.join(" \u00b7 ")}</CardMeta>}
|
||||
<CardDescription {...s}>
|
||||
{place.lat.toFixed(4)}, {place.lon.toFixed(4)}
|
||||
</CardDescription>
|
||||
</CardContent>
|
||||
</Card>;
|
||||
}
|
||||
|
||||
export function MapSearchView() {
|
||||
const {searchParams} = React.useContext(QueryContext);
|
||||
const searchStr = searchParams.get("q") || "";
|
||||
const {offset, limit, activePage, setPage, totalPages, setTotal} = usePages(12);
|
||||
// useLatestRequest discards stale responses and catches errors, so rapid pagination
|
||||
// can't show out-of-order results and a network failure can't leave the spinner stuck.
|
||||
const {data, sendRequest, loading} = useLatestRequest(300);
|
||||
const [hasMaps, setHasMaps] = useState(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
(async () => {
|
||||
const status = await getMapSearchStatus();
|
||||
if (!cancelled) {
|
||||
const installed = (status.indexed || []).length + (status.missing || []).length;
|
||||
setHasMaps(installed > 0);
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!searchStr || searchStr.length < 2) {
|
||||
setTotal(0);
|
||||
return;
|
||||
}
|
||||
sendRequest(async () => await searchMap(searchStr, limit, offset));
|
||||
}, [searchStr, offset, limit, sendRequest]);
|
||||
|
||||
const results = (data && data.results) || [];
|
||||
useEffect(() => {
|
||||
setTotal((data && data.total) || 0);
|
||||
}, [data, setTotal]);
|
||||
|
||||
if (hasMaps === false) {
|
||||
return <Segment>
|
||||
<Header as="h4">No maps are installed.</Header>
|
||||
<p>Subscribe to a map region to search for places.</p>
|
||||
<Link to="/map/manage">Go to Map Manager</Link>
|
||||
</Segment>;
|
||||
}
|
||||
|
||||
if (loading) {
|
||||
return <Segment><Loader active inline="centered"/></Segment>;
|
||||
}
|
||||
|
||||
if (!searchStr) {
|
||||
return <Segment><Header as="h4">Enter a search query to find map locations.</Header></Segment>;
|
||||
}
|
||||
|
||||
if (results.length === 0) {
|
||||
return <Segment><Header as="h4">No map locations found for "{searchStr}".</Header></Segment>;
|
||||
}
|
||||
|
||||
return <React.Fragment>
|
||||
<CardGroup itemsPerRow={3} stackable style={{marginTop: "1em"}}>
|
||||
{results.map((place, i) =>
|
||||
<MapPlaceCard key={`${place.name}-${place.lat}-${place.lon}-${i}`} place={place} index={i}/>
|
||||
)}
|
||||
</CardGroup>
|
||||
<center style={{marginTop: '2em'}}>
|
||||
<Paginator activePage={activePage} totalPages={totalPages} onPageChange={setPage}/>
|
||||
</center>
|
||||
</React.Fragment>;
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ import "maplibre-gl/dist/maplibre-gl.css";
|
|||
import {Protocol} from "pmtiles";
|
||||
import layers from "protomaps-themes-base";
|
||||
import mlcontour from "@acalcutt/maplibre-contour-pmtiles";
|
||||
import {addMapPin, deleteMapPin, getMapFiles, getMapPins, setMapDefaultLocation} from "../api";
|
||||
import {addMapPin, deleteMapPin, getMapFiles, getMapPins, searchMap, setMapDefaultLocation} from "../api";
|
||||
import {MAP_VIEWER_URI} from "./Vars";
|
||||
import {Checkbox, Icon, Segment} from "semantic-ui-react";
|
||||
import {Header} from "./Theme";
|
||||
|
|
@ -221,6 +221,21 @@ function LayerControl({map, scaleUnit, onScaleUnitChange, visibilityRef}) {
|
|||
return v;
|
||||
});
|
||||
const [collapsed, setCollapsed] = useState(true);
|
||||
const [isMobile, setIsMobile] = useState(
|
||||
() => typeof window !== "undefined" && window.matchMedia("(max-width: 768px)").matches
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof window === "undefined") return;
|
||||
const mq = window.matchMedia("(max-width: 768px)");
|
||||
const handler = (e) => setIsMobile(e.matches);
|
||||
mq.addEventListener ? mq.addEventListener("change", handler) : mq.addListener(handler);
|
||||
return () => {
|
||||
mq.removeEventListener ? mq.removeEventListener("change", handler) : mq.removeListener(handler);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const iconOnly = isMobile && collapsed;
|
||||
|
||||
const toggleGroup = useCallback((groupName, checked) => {
|
||||
if (!map) return;
|
||||
|
|
@ -251,21 +266,35 @@ function LayerControl({map, scaleUnit, onScaleUnitChange, visibilityRef}) {
|
|||
boxShadow: "0 2px 8px rgba(0,0,0,0.2)",
|
||||
maxHeight: "calc(100vh - 120px)",
|
||||
overflowY: "auto",
|
||||
minWidth: 200,
|
||||
minWidth: iconOnly ? 0 : 200,
|
||||
fontSize: 13,
|
||||
};
|
||||
|
||||
const headerStyle = iconOnly
|
||||
? {
|
||||
padding: 0, cursor: "pointer", userSelect: "none",
|
||||
display: "flex", alignItems: "center", justifyContent: "center",
|
||||
width: 40, height: 40,
|
||||
}
|
||||
: {
|
||||
padding: "8px 12px", cursor: "pointer", fontWeight: 600,
|
||||
display: "flex", justifyContent: "space-between", alignItems: "center",
|
||||
borderBottom: collapsed ? "none" : "1px solid #eee", userSelect: "none",
|
||||
};
|
||||
|
||||
return <div style={panelStyle}>
|
||||
<div
|
||||
style={{
|
||||
padding: "8px 12px", cursor: "pointer", fontWeight: 600,
|
||||
display: "flex", justifyContent: "space-between", alignItems: "center",
|
||||
borderBottom: collapsed ? "none" : "1px solid #eee", userSelect: "none",
|
||||
}}
|
||||
style={headerStyle}
|
||||
onClick={() => setCollapsed(c => !c)}
|
||||
title="Layers"
|
||||
aria-label="Layers"
|
||||
>
|
||||
<span>Layers</span>
|
||||
<Icon name={collapsed ? "chevron right" : "chevron down"} size="small"/>
|
||||
{iconOnly
|
||||
? <Icon name="map outline" size="large" style={{margin: 0}}/>
|
||||
: <>
|
||||
<span>Layers</span>
|
||||
<Icon name={collapsed ? "chevron right" : "chevron down"} size="small"/>
|
||||
</>}
|
||||
</div>
|
||||
{!collapsed && <div style={{padding: "4px 12px 8px"}}>
|
||||
{Object.keys(LAYER_GROUPS).map(name =>
|
||||
|
|
@ -336,6 +365,150 @@ function AddPinDialog({lat, lon, onSubmit, onCancel}) {
|
|||
</div>;
|
||||
}
|
||||
|
||||
// Coordinate regex: "45.5, -122.6" or "45.5 -122.6"
|
||||
const COORD_REGEX = /^\s*(-?\d+\.?\d*)\s*[,\s]\s*(-?\d+\.?\d*)\s*$/;
|
||||
|
||||
function MapSearch({map}) {
|
||||
const [query, setQuery] = useState("");
|
||||
const [results, setResults] = useState([]);
|
||||
const [showResults, setShowResults] = useState(false);
|
||||
const [searchMarker, setSearchMarker] = useState(null);
|
||||
const debounceRef = useRef(null);
|
||||
const containerRef = useRef(null);
|
||||
|
||||
// Close results when clicking outside.
|
||||
useEffect(() => {
|
||||
const handler = (e) => {
|
||||
if (containerRef.current && !containerRef.current.contains(e.target)) {
|
||||
setShowResults(false);
|
||||
}
|
||||
};
|
||||
document.addEventListener("click", handler);
|
||||
return () => document.removeEventListener("click", handler);
|
||||
}, []);
|
||||
|
||||
const doSearch = useCallback((q) => {
|
||||
if (!q || q.length < 2) {
|
||||
setResults([]);
|
||||
setShowResults(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check for coordinate input first.
|
||||
const coordMatch = q.match(COORD_REGEX);
|
||||
if (coordMatch) {
|
||||
const lat = parseFloat(coordMatch[1]);
|
||||
const lon = parseFloat(coordMatch[2]);
|
||||
if (lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180) {
|
||||
setResults([{name: `${lat}, ${lon}`, kind: "coordinates", lat, lon, min_zoom: 0, source: "input"}]);
|
||||
setShowResults(true);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Get current map center for proximity bias.
|
||||
let lat = null, lon = null;
|
||||
if (map) {
|
||||
const center = map.getCenter();
|
||||
lat = center.lat;
|
||||
lon = center.lng;
|
||||
}
|
||||
|
||||
searchMap(q, 8, 0, lat, lon).then(data => {
|
||||
setResults(data.results || []);
|
||||
setShowResults((data.results || []).length > 0);
|
||||
});
|
||||
}, [map]);
|
||||
|
||||
const handleInput = useCallback((e) => {
|
||||
const q = e.target.value;
|
||||
setQuery(q);
|
||||
clearTimeout(debounceRef.current);
|
||||
debounceRef.current = setTimeout(() => doSearch(q), 300);
|
||||
}, [doSearch]);
|
||||
|
||||
const handleSelect = useCallback((result) => {
|
||||
if (!map) return;
|
||||
|
||||
// Remove old search marker.
|
||||
if (searchMarker) {
|
||||
searchMarker.remove();
|
||||
}
|
||||
|
||||
const zoom = result.kind === "coordinates" ? 14 : Math.max(map.getZoom(), 12);
|
||||
map.flyTo({center: [result.lon, result.lat], zoom});
|
||||
|
||||
const marker = new maplibregl.Marker({color: "#6435c9"})
|
||||
.setLngLat([result.lon, result.lat])
|
||||
.setPopup(new maplibregl.Popup({offset: 25}).setText(result.name))
|
||||
.addTo(map);
|
||||
marker.togglePopup();
|
||||
setSearchMarker(marker);
|
||||
|
||||
setShowResults(false);
|
||||
setQuery(result.name);
|
||||
}, [map, searchMarker]);
|
||||
|
||||
const handleKeyDown = useCallback((e) => {
|
||||
if (e.key === "Escape") {
|
||||
setShowResults(false);
|
||||
e.target.blur();
|
||||
}
|
||||
}, []);
|
||||
|
||||
const formatPopulation = (pop) => {
|
||||
if (!pop) return "";
|
||||
if (pop >= 1_000_000) return `${(pop / 1_000_000).toFixed(1)}M`;
|
||||
if (pop >= 1_000) return `${(pop / 1_000).toFixed(0)}K`;
|
||||
return `${pop}`;
|
||||
};
|
||||
|
||||
const resultSubtext = (r) => {
|
||||
if (r.kind === "coordinates") return "";
|
||||
const parts = [];
|
||||
if (r.region) parts.push(r.region);
|
||||
const detail = r.kind_detail || r.kind || "";
|
||||
if (detail && !r.region) parts.push(detail.replace(/_/g, " "));
|
||||
const pop = formatPopulation(r.population);
|
||||
if (pop) parts.push(`pop. ${pop}`);
|
||||
return parts.join(" \u00b7 ");
|
||||
};
|
||||
|
||||
return <div ref={containerRef} style={{
|
||||
position: "absolute", top: 10, left: "50%", transform: "translateX(-50%)",
|
||||
zIndex: 5, width: 320, maxWidth: "calc(100% - 100px)",
|
||||
}}>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Search places..."
|
||||
value={query}
|
||||
onChange={handleInput}
|
||||
onFocus={() => results.length > 0 && setShowResults(true)}
|
||||
onKeyDown={handleKeyDown}
|
||||
style={{
|
||||
width: "100%", padding: "8px 12px", border: "1px solid #ccc", borderRadius: 4,
|
||||
fontSize: 14, boxShadow: "0 2px 6px rgba(0,0,0,0.2)", background: "white",
|
||||
}}
|
||||
/>
|
||||
{showResults && results.length > 0 && <div style={{
|
||||
background: "white", border: "1px solid #ccc", borderTop: "none",
|
||||
borderRadius: "0 0 4px 4px", maxHeight: 300, overflowY: "auto",
|
||||
boxShadow: "0 4px 8px rgba(0,0,0,0.2)",
|
||||
}}>
|
||||
{results.map((r, i) => <div
|
||||
key={i}
|
||||
onClick={() => handleSelect(r)}
|
||||
onMouseEnter={e => e.target.style.background = "#f0f0f0"}
|
||||
onMouseLeave={e => e.target.style.background = "transparent"}
|
||||
style={{padding: "8px 12px", cursor: "pointer", borderBottom: "1px solid #eee"}}
|
||||
>
|
||||
<div style={{fontWeight: 500}}>{r.name}</div>
|
||||
{resultSubtext(r) && <div style={{fontSize: 11, color: "#888"}}>{resultSubtext(r)}</div>}
|
||||
</div>)}
|
||||
</div>}
|
||||
</div>;
|
||||
}
|
||||
|
||||
export default function MapViewer() {
|
||||
const mapContainer = useRef(null);
|
||||
const mapRef = useRef(null);
|
||||
|
|
@ -742,6 +915,7 @@ export default function MapViewer() {
|
|||
}
|
||||
|
||||
return <div ref={mapContainer} style={{position: "relative", width: "100%", height: "85vh"}}>
|
||||
{mapReady && <MapSearch map={mapRef.current}/>}
|
||||
{mapReady && <LayerControl map={mapRef.current} scaleUnit={scaleUnit} onScaleUnitChange={handleScaleUnitChange} visibilityRef={layerVisibilityRef}/>}
|
||||
{contextMenu && <div
|
||||
style={{
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ import {FilesSearchView} from "./Files";
|
|||
import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchFilter} from "../hooks/customHooks";
|
||||
import {ShortcutHint} from "./ShortcutHint";
|
||||
import {ZimSearchView} from "./Zim";
|
||||
import {searchEstimateFiles, searchEstimateOthers, searchEstimateZims, searchSuggestions} from "../api";
|
||||
import {MapSearchView} from "./MapSearchView";
|
||||
import {searchEstimateFiles, searchEstimateMap, searchEstimateOthers, searchEstimateZims, searchSuggestions} from "../api";
|
||||
import {filterToMimetypes, fuzzyMatch, normalizeEstimate, SearchResultsInput, TabLinks} from "./Common";
|
||||
import _ from "lodash";
|
||||
import {TagsContext} from "../Tags";
|
||||
|
|
@ -133,6 +134,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
|
|||
// Zims are slow, so they are separate.
|
||||
const {data: zimData, sendRequest: sendZimRequest, loading: zimLoading} = useLatestRequest(500);
|
||||
const {data: otherData, sendRequest: sendOtherRequest, loading: otherLoading} = useLatestRequest(500);
|
||||
const {data: mapData, sendRequest: sendMapRequest, loading: mapLoading} = useLatestRequest(500);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (searchStr || (tagNames && tagNames.length > 0)) {
|
||||
|
|
@ -141,6 +143,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
|
|||
if (searchStr && searchStr.length > 0) {
|
||||
// We can't search Channels/Domains without some string to filter by.
|
||||
sendGeneralReqeust(async () => await searchSuggestions(searchStr));
|
||||
sendMapRequest(async () => await searchEstimateMap(searchStr));
|
||||
}
|
||||
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag));
|
||||
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
|
||||
|
|
@ -195,6 +198,14 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
|
|||
}
|
||||
}, [setSuggestions, otherData]);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!_.isEmpty(mapData)) {
|
||||
setSuggestions((prevState) => {
|
||||
return {...prevState, mapPlaces: mapData.mapPlaces}
|
||||
});
|
||||
}
|
||||
}, [setSuggestions, mapData]);
|
||||
|
||||
return {suggestions, loading: generalLoading || zimLoading || filesLoading}
|
||||
}
|
||||
|
||||
|
|
@ -329,6 +340,18 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
const coordMatch = searchStr && searchStr.trim().match(
|
||||
/^(-?\d+\.?\d*)[,\s]+(-?\d+\.?\d*)$/
|
||||
);
|
||||
// Map places suggestion in dropdown.
|
||||
if (newSuggestions.mapPlaces > 0) {
|
||||
results['map places'] = {
|
||||
name: 'Map Places', results: [{
|
||||
type: 'search',
|
||||
title: `Map: ${newSuggestions.mapPlaces} places`,
|
||||
description: 'Search map locations',
|
||||
location: `/search/map?q=${encodeURIComponent(searchStr)}`,
|
||||
}]
|
||||
};
|
||||
}
|
||||
|
||||
if (coordMatch) {
|
||||
const lat = parseFloat(coordMatch[1]);
|
||||
const lon = parseFloat(coordMatch[2]);
|
||||
|
|
@ -355,6 +378,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
fileGroups: newSuggestions.fileGroups,
|
||||
zims: zimSum,
|
||||
otherSum: otherSum,
|
||||
mapPlaces: newSuggestions.mapPlaces || 0,
|
||||
channels: newSuggestions.channels.length,
|
||||
domains: newSuggestions.domains.length,
|
||||
authors: newSuggestions.authors?.length,
|
||||
|
|
@ -441,11 +465,13 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
|
|||
|
||||
const filesTabName = <span>Files <Label>{normalizeEstimate(suggestionsSums?.fileGroups)}</Label></span>;
|
||||
const zimsTabName = <span>Zims <Label>{normalizeEstimate(suggestionsSums?.zims)}</Label></span>;
|
||||
const mapTabName = <span>Map <Label>{normalizeEstimate(suggestionsSums?.mapPlaces)}</Label></span>;
|
||||
const othersTabName = <span>Other <Label>{normalizeEstimate(suggestionsSums?.otherSum)}</Label></span>;
|
||||
|
||||
const links = [
|
||||
{text: filesTabName, to: `/search${search}`, key: 'filesSearch_', end: true},
|
||||
{text: zimsTabName, to: `/search/zim${search}`, key: 'zimsSearch'},
|
||||
{text: mapTabName, to: `/search/map${search}`, key: 'mapSearch'},
|
||||
{text: othersTabName, to: `/search/other${search}`, key: 'othersSearch'},
|
||||
];
|
||||
|
||||
|
|
@ -454,6 +480,7 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
|
|||
<Routes>
|
||||
<Route path='/*' element={<FilesSearchView/>}/>
|
||||
<Route path='/zim' exact element={<ZimSearchView suggestions={suggestions} loading={loading}/>}/>
|
||||
<Route path='/map' exact element={<MapSearchView/>}/>
|
||||
<Route path='/other' exact element={<OtherSearchView loading={loading}/>}/>
|
||||
</Routes>
|
||||
</React.Fragment>
|
||||
|
|
|
|||
|
|
@ -881,6 +881,16 @@
|
|||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if not docker_mode %}
|
||||
<div id="script-running-banner" class="script-running-banner">
|
||||
<h2 id="script-running-name">Script Running</h2>
|
||||
<span class="loading"></span>
|
||||
<div class="actions">
|
||||
<button class="btn btn-info" onclick="showScriptOutput()">View Output</button>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% set app_ready = services | selectattr('name', 'in', ['caddy', 'web']) | selectattr('status', 'equalto', 'running') | list | length > 0 %}
|
||||
<div style="margin-bottom: 1rem;">
|
||||
<a id="proceed-btn" class="btn btn-success" style="{% if not app_ready %}display: none;{% endif %} font-size: 1.1rem; padding: 0.6rem 1.5rem;" href="#">Proceed to WROLPi</a>
|
||||
|
|
@ -892,16 +902,6 @@
|
|||
Main UI is unavailable. Using Controller fallback. Will redirect when ready...
|
||||
</div>
|
||||
|
||||
{% if not docker_mode %}
|
||||
<div id="script-running-banner" class="script-running-banner">
|
||||
<h2 id="script-running-name">Script Running</h2>
|
||||
<span class="loading"></span>
|
||||
<div class="actions">
|
||||
<button class="btn btn-info" onclick="showScriptOutput()">View Output</button>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<main>
|
||||
<section id="status">
|
||||
<h2>System Status</h2>
|
||||
|
|
|
|||
|
|
@ -3,7 +3,14 @@ ENV DOCKER=true
|
|||
WORKDIR /opt/wrolpi
|
||||
|
||||
# Update dependencies for the services install.
|
||||
RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unrar-free unzip
|
||||
RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unrar-free unzip \
|
||||
build-essential libsqlite3-dev zlib1g-dev git
|
||||
|
||||
# Install tippecanoe (for map search index building via tippecanoe-decode).
|
||||
RUN git clone --depth 1 --branch 2.79.0 https://github.com/felt/tippecanoe.git /tmp/tippecanoe \
|
||||
&& cd /tmp/tippecanoe && make -j$(nproc) && make install \
|
||||
&& rm -rf /tmp/tippecanoe \
|
||||
&& tippecanoe-decode --help 2>&1 | head -1
|
||||
RUN ffmpeg -version
|
||||
|
||||
# Install Deno runtime for yt-dlp
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from sanic import Request, response, Blueprint
|
|||
from sanic_ext import validate
|
||||
from sanic_ext.extensions.openapi import openapi
|
||||
|
||||
from modules.map import lib, schema
|
||||
from modules.map import lib, schema, search
|
||||
from modules.map.pins import get_map_pins_config
|
||||
from wrolpi.api_utils import json_response
|
||||
from wrolpi.common import wrol_mode_check
|
||||
|
|
@ -98,3 +98,63 @@ async def update_pin(_: Request, pin_id: int, body: schema.MapPinUpdateRequest):
|
|||
if get_map_pins_config().update_pin(pin_id, body.label, body.color):
|
||||
return response.empty(HTTPStatus.NO_CONTENT)
|
||||
return response.json({'error': 'Pin not found'}, HTTPStatus.NOT_FOUND)
|
||||
|
||||
|
||||
@map_bp.get('/search')
|
||||
@openapi.description('Search for places in map search indexes')
|
||||
async def search_places(request: Request):
|
||||
q = request.args.get('q', '')
|
||||
if not q or len(q.strip()) < 1:
|
||||
return json_response({'error': 'Query parameter "q" is required'}, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
try:
|
||||
limit = int(request.args.get('limit', 12))
|
||||
offset = int(request.args.get('offset', 0))
|
||||
lat = float(request.args.get('lat')) if request.args.get('lat') is not None else None
|
||||
lon = float(request.args.get('lon')) if request.args.get('lon') is not None else None
|
||||
except (ValueError, TypeError):
|
||||
return json_response({'error': 'Invalid query parameters'}, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
data = search.search_places(q.strip(), limit=limit, offset=offset, lat=lat, lon=lon)
|
||||
return json_response(data, HTTPStatus.OK)
|
||||
|
||||
|
||||
@map_bp.get('/search/estimate')
|
||||
@openapi.description('Get estimated count of map search results')
|
||||
async def get_search_estimate(request: Request):
|
||||
q = request.args.get('q', '')
|
||||
if not q or len(q.strip()) < 1:
|
||||
return json_response({'map_places': 0}, HTTPStatus.OK)
|
||||
count = search.search_places_count(q.strip())
|
||||
return json_response({'map_places': count}, HTTPStatus.OK)
|
||||
|
||||
|
||||
@map_bp.get('/search/status')
|
||||
@openapi.description('Get status of map search indexes')
|
||||
async def get_search_status(_: Request):
|
||||
status = search.get_search_status()
|
||||
return json_response(status, HTTPStatus.OK)
|
||||
|
||||
|
||||
@map_bp.post('/search/rebuild')
|
||||
@openapi.description('Rebuild all map search indexes')
|
||||
@wrol_mode_check
|
||||
async def rebuild_search_indexes(_: Request):
|
||||
started = search.rebuild_all_search_indexes()
|
||||
if started:
|
||||
return json_response({'message': 'Search index rebuild started'}, HTTPStatus.ACCEPTED)
|
||||
return json_response({'error': 'No PMTiles files found'}, HTTPStatus.NOT_FOUND)
|
||||
|
||||
|
||||
@map_bp.post('/search/rebuild/<filename:str>')
|
||||
@openapi.description('Rebuild search index for a single PMTiles file')
|
||||
@wrol_mode_check
|
||||
async def rebuild_single_search_index(_: Request, filename: str):
|
||||
try:
|
||||
started = search.rebuild_search_index(filename)
|
||||
except ValueError as e:
|
||||
return response.json({'error': str(e)}, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
if started:
|
||||
return json_response({'message': f'Search index rebuild started for {filename}'}, HTTPStatus.ACCEPTED)
|
||||
return json_response({'error': 'File not found'}, HTTPStatus.NOT_FOUND)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ from wrolpi.common import logger, verify_gpg_signature
|
|||
from wrolpi.db import get_db_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult, download_manager
|
||||
|
||||
__all__ = ['MapCatalogDownloader', 'MapExtractDownloader', 'map_catalog_downloader', 'map_extract_downloader']
|
||||
__all__ = ['MapCatalogDownloader', 'MapExtractDownloader', 'MapSearchIndexDownloader',
|
||||
'map_catalog_downloader', 'map_extract_downloader', 'map_search_index_downloader']
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -66,6 +67,7 @@ class MapCatalogDownloader(Downloader):
|
|||
|
||||
version = manifest.get('version', '')
|
||||
manifest_regions = manifest.get('regions', {})
|
||||
search_indexes = manifest.get('search_indexes', {})
|
||||
|
||||
if not manifest_regions:
|
||||
return DownloadResult(success=False, error='Manifest does not contain regions')
|
||||
|
|
@ -74,7 +76,13 @@ class MapCatalogDownloader(Downloader):
|
|||
if not regions:
|
||||
return DownloadResult(success=True)
|
||||
|
||||
map_directory = lib.get_map_directory()
|
||||
|
||||
created = 0
|
||||
# Defer search-index downloads so they are all inserted AFTER every pmtiles
|
||||
# download. Downloads process in insertion order; this prevents a search-index
|
||||
# download from running before its companion pmtiles file exists on disk.
|
||||
pending_search_downloads = []
|
||||
with get_db_session(commit=True) as session:
|
||||
for r in regions:
|
||||
region = r.get('region', '')
|
||||
|
|
@ -83,6 +91,12 @@ class MapCatalogDownloader(Downloader):
|
|||
logger.warning(f'Subscribed region {region!r} not found in manifest — skipping')
|
||||
continue
|
||||
|
||||
# Compute the expected PMTiles filename.
|
||||
if version:
|
||||
pmtiles_name = f'{region}-{version}.pmtiles'
|
||||
else:
|
||||
pmtiles_name = f'{region}.pmtiles'
|
||||
|
||||
# Delete stale download for this region if the URL changed (new version).
|
||||
old = session.query(Download).filter(
|
||||
Download.downloader == 'map_extract',
|
||||
|
|
@ -101,14 +115,34 @@ class MapCatalogDownloader(Downloader):
|
|||
child.settings = {
|
||||
'region': region,
|
||||
'version': version,
|
||||
'search_index_url': search_indexes.get(region, ''),
|
||||
}
|
||||
child.renew(reset_attempts=True)
|
||||
created += 1
|
||||
|
||||
# Queue search index download if the CDN provides one and the
|
||||
# .search.db file does not already exist on disk.
|
||||
search_url = search_indexes.get(region)
|
||||
search_db_path = (map_directory / pmtiles_name).with_suffix('.search.db')
|
||||
if search_url and not search_db_path.is_file():
|
||||
pending_search_downloads.append((region, search_url, pmtiles_name))
|
||||
|
||||
# Insert all search-index downloads after every pmtiles download, so they
|
||||
# process in insertion order behind their companion pmtiles.
|
||||
for region, search_url, pmtiles_name in pending_search_downloads:
|
||||
search_child = download_manager.get_or_create_download(
|
||||
session, search_url, reset_attempts=True, override_skip=True)
|
||||
search_child.downloader = 'map_search_index'
|
||||
search_child.location = '/map/manage'
|
||||
search_child.settings = {'region': region, 'pmtiles_name': pmtiles_name}
|
||||
search_child.renew(reset_attempts=True)
|
||||
logger.info(f'Created search index download for {region}')
|
||||
|
||||
if not created:
|
||||
return DownloadResult(success=False, error='No matching region URLs found in manifest')
|
||||
|
||||
logger.info(f'Created {created} map region downloads')
|
||||
|
||||
return DownloadResult(success=True, location='/map/manage')
|
||||
|
||||
|
||||
|
|
@ -201,7 +235,86 @@ class MapExtractDownloader(Downloader):
|
|||
size = output_path.stat().st_size
|
||||
logger.warning(f'Successfully downloaded {region} ({size} bytes) to {output_path}')
|
||||
|
||||
# Build search index locally if tippecanoe is available and no CDN
|
||||
# search index download already exists for this region.
|
||||
has_search_download = False
|
||||
with get_db_session() as session:
|
||||
has_search_download = session.query(Download).filter(
|
||||
Download.downloader == 'map_search_index',
|
||||
Download.settings['region'].astext == region,
|
||||
).count() > 0
|
||||
if not has_search_download:
|
||||
try:
|
||||
from modules.map.search import rebuild_search_index
|
||||
rebuild_search_index(output_path.name)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to start search index build for {region}: {e}')
|
||||
|
||||
return DownloadResult(success=True, location='/map')
|
||||
|
||||
|
||||
map_extract_downloader = MapExtractDownloader()
|
||||
|
||||
|
||||
class MapSearchIndexDownloader(Downloader):
|
||||
"""Downloads a pre-built .search.db file from the CDN via aria2c with meta4 verification.
|
||||
|
||||
Created by MapCatalogDownloader alongside map_extract downloads for each
|
||||
subscribed region. Will wait (fail + retry) until the PMTiles file exists.
|
||||
"""
|
||||
|
||||
name = 'map_search_index'
|
||||
listable = False
|
||||
|
||||
async def do_download(self, download: Download) -> DownloadResult:
|
||||
settings = download.settings or {}
|
||||
region = settings.get('region', '')
|
||||
pmtiles_name = settings.get('pmtiles_name', '')
|
||||
|
||||
if not region or not pmtiles_name:
|
||||
return DownloadResult(success=False, error='Missing region or pmtiles_name in settings')
|
||||
|
||||
map_directory = lib.get_map_directory()
|
||||
pmtiles_path = map_directory / pmtiles_name
|
||||
|
||||
if not pmtiles_path.is_file():
|
||||
return DownloadResult(success=False, error=f'PMTiles file not found: {pmtiles_name}')
|
||||
|
||||
# Name the search.db to match the local PMTiles filename.
|
||||
search_db_path = pmtiles_path.with_suffix('.search.db')
|
||||
|
||||
if search_db_path.is_file():
|
||||
logger.info(f'Search index already exists for {region}, skipping')
|
||||
return DownloadResult(success=True, location='/map')
|
||||
|
||||
# Fetch and verify meta4 (GPG-signed hash).
|
||||
meta4_contents = await self.get_meta4_contents(download.url)
|
||||
if meta4_contents:
|
||||
if not await verify_meta4_signature(meta4_contents):
|
||||
return DownloadResult(success=False,
|
||||
error=f'meta4 signature verification failed for {region} search index')
|
||||
logger.info(f'Search index meta4 verified for {region}')
|
||||
|
||||
# Download via aria2c with meta4 hash verification.
|
||||
logger.warning(f'Downloading search index for {region} from {download.url}')
|
||||
downloaded_path = None
|
||||
try:
|
||||
downloaded_path = await self.download_file(
|
||||
download, download.url, map_directory,
|
||||
check_for_meta4=False, concurrent=1, meta4_xml=meta4_contents,
|
||||
)
|
||||
# Rename to match the local PMTiles filename.
|
||||
downloaded_path.rename(search_db_path)
|
||||
except Exception as e:
|
||||
logger.error(f'Search index download failed for {region}: {e}')
|
||||
if downloaded_path:
|
||||
downloaded_path.unlink(missing_ok=True)
|
||||
search_db_path.unlink(missing_ok=True)
|
||||
return DownloadResult(success=False, error=f'Download failed: {str(e)[:500]}')
|
||||
|
||||
size = search_db_path.stat().st_size
|
||||
logger.warning(f'Downloaded search index for {region} ({size} bytes)')
|
||||
return DownloadResult(success=True, location='/map')
|
||||
|
||||
|
||||
map_search_index_downloader = MapSearchIndexDownloader()
|
||||
|
|
|
|||
|
|
@ -33,13 +33,16 @@ def get_pmtiles_files() -> List[dict]:
|
|||
path=str(path.relative_to(get_media_directory())),
|
||||
size=stat.st_size,
|
||||
mtime=stat.st_mtime,
|
||||
has_search_index=path.with_suffix('.search.db').is_file(),
|
||||
))
|
||||
|
||||
return files
|
||||
|
||||
|
||||
def delete_pmtiles_file(filename: str) -> bool:
|
||||
"""Delete a PMTiles file from the map directory. Returns True if the file was deleted."""
|
||||
"""Delete a PMTiles file from the map directory. Returns True if the file was deleted.
|
||||
|
||||
Also deletes the companion .search.db file if it exists."""
|
||||
map_directory = get_map_directory()
|
||||
# Prevent path traversal.
|
||||
path = (map_directory / filename).resolve()
|
||||
|
|
@ -49,6 +52,11 @@ def delete_pmtiles_file(filename: str) -> bool:
|
|||
if path.is_file() and path.suffix == '.pmtiles':
|
||||
path.unlink()
|
||||
logger.warning(f'Deleted PMTiles file: {path}')
|
||||
# Also delete companion search index.
|
||||
search_db = path.with_suffix('.search.db')
|
||||
if search_db.is_file():
|
||||
search_db.unlink()
|
||||
logger.warning(f'Deleted search index: {search_db}')
|
||||
return True
|
||||
|
||||
return False
|
||||
|
|
@ -73,7 +81,6 @@ def _get_or_create_map_download(session: Session) -> Download:
|
|||
download = download_manager.get_or_create_download(session, MANIFEST_URL, reset_attempts=True,
|
||||
override_skip=True)
|
||||
download.downloader = 'map_catalog'
|
||||
download.sub_downloader = 'map_extract'
|
||||
download.frequency = DownloadFrequency.days180
|
||||
download.settings = {'regions': []}
|
||||
download.location = '/map/manage'
|
||||
|
|
|
|||
295
modules/map/search.py
Normal file
295
modules/map/search.py
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import math
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from modules.map.lib import get_map_directory
|
||||
from wrolpi.common import logger, background_task
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.flags import map_search_building
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
BUILD_SCRIPT = Path(__file__).parent.parent.parent / 'wrolpi' / 'scripts' / 'build_map_search.py'
|
||||
|
||||
|
||||
def get_search_db_files() -> List[Path]:
|
||||
"""Find all .search.db files in the map directory."""
|
||||
map_directory = get_map_directory()
|
||||
return sorted(map_directory.glob('*.search.db'))
|
||||
|
||||
|
||||
def search_places(query: str, limit: int = 12, offset: int = 0, lat: float = None, lon: float = None) -> dict:
|
||||
"""Search all .search.db files for places matching the query.
|
||||
|
||||
Results are ranked by importance (min_zoom) and optionally by proximity to lat/lon.
|
||||
Returns {"results": [...], "total": N}.
|
||||
"""
|
||||
db_files = get_search_db_files()
|
||||
if not db_files:
|
||||
return {'results': [], 'total': 0}
|
||||
|
||||
results = []
|
||||
|
||||
# Cap the number of rows fetched per DB to bound memory on large regional indexes
|
||||
# (e.g. a prefix query like "S*" can match hundreds of thousands of rows). The
|
||||
# cap is a multiple of (offset + limit) to keep enough headroom for ranking and
|
||||
# cross-DB deduplication before pagination.
|
||||
per_db_cap = max((offset + limit) * 5, 100)
|
||||
|
||||
# FTS5 match query with prefix matching. Escape embedded double quotes.
|
||||
escaped = query.replace('"', '""')
|
||||
fts_query = f'"{escaped}"*'
|
||||
|
||||
for db_path in db_files:
|
||||
try:
|
||||
with contextlib.closing(sqlite3.connect(f'file:{db_path}?mode=ro', uri=True)) as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
rows = conn.execute("""
|
||||
SELECT p.name, p.kind, p.lat, p.lon, p.min_zoom, p.source,
|
||||
p.kind_detail, p.population, p.region,
|
||||
places_fts.rank AS fts_rank
|
||||
FROM places_fts
|
||||
JOIN places p ON p.id = places_fts.rowid
|
||||
WHERE places_fts MATCH ?
|
||||
ORDER BY places_fts.rank
|
||||
LIMIT ?
|
||||
""", (fts_query, per_db_cap)).fetchall()
|
||||
|
||||
for row in rows:
|
||||
result = dict(row)
|
||||
if lat is not None and lon is not None:
|
||||
dlat = result['lat'] - lat
|
||||
dlon = result['lon'] - lon
|
||||
result['distance'] = math.sqrt(dlat ** 2 + dlon ** 2)
|
||||
results.append(result)
|
||||
except sqlite3.OperationalError as e:
|
||||
logger.warning(f'Search error in {db_path.name}: {e}')
|
||||
continue
|
||||
|
||||
# Fuzzy fallback: if FTS5 found nothing, try LIKE substring matching.
|
||||
if not results:
|
||||
like_pattern = f'%{query}%'
|
||||
for db_path in db_files:
|
||||
try:
|
||||
with contextlib.closing(sqlite3.connect(f'file:{db_path}?mode=ro', uri=True)) as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
rows = conn.execute("""
|
||||
SELECT p.name, p.kind, p.lat, p.lon, p.min_zoom, p.source,
|
||||
p.kind_detail, p.population, p.region, 0 AS fts_rank
|
||||
FROM places p
|
||||
WHERE p.name LIKE ?
|
||||
ORDER BY p.min_zoom, p.population DESC
|
||||
LIMIT ?
|
||||
""", (like_pattern, per_db_cap)).fetchall()
|
||||
|
||||
for row in rows:
|
||||
result = dict(row)
|
||||
if lat is not None and lon is not None:
|
||||
dlat = result['lat'] - lat
|
||||
dlon = result['lon'] - lon
|
||||
result['distance'] = math.sqrt(dlat ** 2 + dlon ** 2)
|
||||
results.append(result)
|
||||
except sqlite3.OperationalError as e:
|
||||
logger.warning(f'LIKE search error in {db_path.name}: {e}')
|
||||
continue
|
||||
|
||||
# Sort by importance (lower min_zoom = more important), then by proximity or FTS rank.
|
||||
# Coerce NULL min_zoom (possible from older schemas) to a default; `dict.get` returns
|
||||
# None rather than the default for explicit NULL values.
|
||||
if lat is not None and lon is not None:
|
||||
results.sort(key=lambda r: (r.get('min_zoom') or 10, r.get('distance') or 0))
|
||||
else:
|
||||
results.sort(key=lambda r: (r.get('min_zoom') or 10, r.get('fts_rank') or 0))
|
||||
|
||||
# Deduplicate across sources: same name+kind at ~same location keeps highest population.
|
||||
seen = {}
|
||||
deduped = []
|
||||
for r in results:
|
||||
key = (r['name'], r['kind'], round(r['lat'], 2), round(r['lon'], 2))
|
||||
if key not in seen:
|
||||
seen[key] = len(deduped)
|
||||
deduped.append(r)
|
||||
else:
|
||||
# Keep the entry with higher population.
|
||||
existing = deduped[seen[key]]
|
||||
if (r.get('population') or 0) > (existing.get('population') or 0):
|
||||
deduped[seen[key]] = r
|
||||
|
||||
# Use the DB-wide count for `total` so pagination isn't capped by `per_db_cap`.
|
||||
# Fall back to the deduped length if the count query fails (e.g. the per-DB cap
|
||||
# was never hit, in which case they agree anyway).
|
||||
if len(results) >= per_db_cap:
|
||||
total = search_places_count(query)
|
||||
else:
|
||||
total = len(deduped)
|
||||
|
||||
# Remove internal ranking fields before returning.
|
||||
final = []
|
||||
for r in deduped[offset:offset + limit]:
|
||||
r.pop('fts_rank', None)
|
||||
r.pop('distance', None)
|
||||
final.append(r)
|
||||
|
||||
return {'results': final, 'total': total}
|
||||
|
||||
|
||||
def search_places_count(query: str) -> int:
|
||||
"""Return the count of places matching the query across all search indexes."""
|
||||
db_files = get_search_db_files()
|
||||
if not db_files:
|
||||
return 0
|
||||
|
||||
escaped = query.replace('"', '""')
|
||||
fts_query = f'"{escaped}"*'
|
||||
total = 0
|
||||
|
||||
for db_path in db_files:
|
||||
try:
|
||||
with contextlib.closing(sqlite3.connect(f'file:{db_path}?mode=ro', uri=True)) as conn:
|
||||
row = conn.execute("""
|
||||
SELECT COUNT(*) FROM places_fts WHERE places_fts MATCH ?
|
||||
""", (fts_query,)).fetchone()
|
||||
total += row[0] if row else 0
|
||||
except sqlite3.OperationalError:
|
||||
continue
|
||||
|
||||
# LIKE fallback count if FTS5 found nothing.
|
||||
if total == 0:
|
||||
like_pattern = f'%{query}%'
|
||||
for db_path in db_files:
|
||||
try:
|
||||
with contextlib.closing(sqlite3.connect(f'file:{db_path}?mode=ro', uri=True)) as conn:
|
||||
row = conn.execute("SELECT COUNT(*) FROM places WHERE name LIKE ?",
|
||||
(like_pattern,)).fetchone()
|
||||
total += row[0] if row else 0
|
||||
except sqlite3.OperationalError:
|
||||
continue
|
||||
|
||||
return total
|
||||
|
||||
|
||||
def get_search_status() -> dict:
|
||||
"""Return status of search indexes: which PMTiles files have indexes and which don't."""
|
||||
map_directory = get_map_directory()
|
||||
|
||||
pmtiles_files = sorted(map_directory.glob('*.pmtiles'))
|
||||
# Strip the double .search.db extension to get the PMTiles stem.
|
||||
search_db_files = {p.name.removesuffix('.search.db') for p in get_search_db_files()}
|
||||
|
||||
indexed = []
|
||||
missing = []
|
||||
|
||||
for pmtiles_path in pmtiles_files:
|
||||
info = {'name': pmtiles_path.name, 'stem': pmtiles_path.stem}
|
||||
if pmtiles_path.stem in search_db_files:
|
||||
db_path = pmtiles_path.with_suffix('.search.db')
|
||||
info['search_db_size'] = db_path.stat().st_size
|
||||
indexed.append(info)
|
||||
else:
|
||||
missing.append(info)
|
||||
|
||||
return {
|
||||
'indexed': indexed,
|
||||
'missing': missing,
|
||||
}
|
||||
|
||||
|
||||
def _check_tippecanoe() -> bool:
|
||||
"""Return True if tippecanoe-decode is on PATH. If missing, log and send an Event."""
|
||||
if shutil.which('tippecanoe-decode'):
|
||||
return True
|
||||
msg = 'tippecanoe-decode not found on PATH — map search index cannot be built'
|
||||
logger.error(msg)
|
||||
Events.send_tippecanoe_missing(msg)
|
||||
return False
|
||||
|
||||
|
||||
def _should_enrich() -> bool:
|
||||
"""Return True if Wikidata enrichment should be attempted (WROL mode is not active)."""
|
||||
try:
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
return not get_wrolpi_config().wrol_mode
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def _run_build(cmd: list, description: str):
|
||||
"""Run a build subprocess with the map_search_building flag set.
|
||||
|
||||
The flag is automatically cleared when the build finishes (success or failure).
|
||||
"""
|
||||
with map_search_building:
|
||||
logger.warning(f'Starting search index build: {description}')
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
stdout, _ = await proc.communicate()
|
||||
output = stdout.decode()[-500:] if stdout else ''
|
||||
if proc.returncode == 0:
|
||||
logger.warning(f'Search index build completed: {description}')
|
||||
Events.send_map_search_complete(f'Search index built for {description}')
|
||||
else:
|
||||
logger.error(f'Search index build failed for {description} (exit {proc.returncode}):\n{output}')
|
||||
Events.send_map_search_failed(f'Search index build failed for {description}')
|
||||
|
||||
|
||||
def rebuild_search_index(pmtiles_name: str, max_zoom: int = 10) -> bool:
|
||||
"""Launch the build script as a background task for a single PMTiles file.
|
||||
|
||||
Returns True if the build was started, False if the file doesn't exist.
|
||||
"""
|
||||
map_directory = get_map_directory()
|
||||
pmtiles_path = (map_directory / pmtiles_name).resolve()
|
||||
|
||||
if not pmtiles_path.is_relative_to(map_directory.resolve()):
|
||||
raise ValueError(f'Invalid filename: {pmtiles_name}')
|
||||
|
||||
if not pmtiles_path.is_file():
|
||||
return False
|
||||
|
||||
if map_search_building.is_set():
|
||||
logger.warning(f'Search index build already running — ignoring request for {pmtiles_name}')
|
||||
return False
|
||||
|
||||
if not _check_tippecanoe():
|
||||
return False
|
||||
|
||||
cmd = [sys.executable, str(BUILD_SCRIPT), str(pmtiles_path), '--max-zoom', str(max_zoom)]
|
||||
if _should_enrich():
|
||||
cmd.append('--enrich')
|
||||
|
||||
background_task(_run_build(cmd, pmtiles_name))
|
||||
return True
|
||||
|
||||
|
||||
def rebuild_all_search_indexes(max_zoom: int = 10) -> bool:
|
||||
"""Launch the build script for the entire map directory as a background task."""
|
||||
map_directory = get_map_directory()
|
||||
|
||||
if not any(map_directory.glob('*.pmtiles')):
|
||||
return False
|
||||
|
||||
if map_search_building.is_set():
|
||||
logger.warning('Search index build already running — ignoring rebuild-all request')
|
||||
return False
|
||||
|
||||
if not _check_tippecanoe():
|
||||
return False
|
||||
|
||||
cmd = [sys.executable, str(BUILD_SCRIPT), str(map_directory), '--max-zoom', str(max_zoom)]
|
||||
if _should_enrich():
|
||||
cmd.append('--enrich')
|
||||
|
||||
background_task(_run_build(cmd, 'all map files'))
|
||||
return True
|
||||
|
||||
|
||||
66
modules/map/static/embed.html
Normal file
66
modules/map/static/embed.html
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<link rel="stylesheet" href="/libs/maplibre-gl.css"/>
|
||||
<style>* { margin: 0; padding: 0; } html, body, #map { width: 100%; height: 100%; }</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="map"></div>
|
||||
<script src="/libs/pmtiles.js"></script>
|
||||
<script src="/libs/maplibre-gl.js"></script>
|
||||
<script src="/libs/protomaps-themes-base.js"></script>
|
||||
<script>
|
||||
(function() {
|
||||
var protocol = new pmtiles.Protocol();
|
||||
maplibregl.addProtocol("pmtiles", protocol.tile);
|
||||
|
||||
var hash = window.location.hash.replace("#", "");
|
||||
var parts = hash ? hash.split(",") : [];
|
||||
var lat = parseFloat(parts[0]) || 0;
|
||||
var lon = parseFloat(parts[1]) || 0;
|
||||
var zoom = parseFloat(parts[2]) || 4;
|
||||
|
||||
var dark = window.matchMedia("(prefers-color-scheme: dark)").matches;
|
||||
var flavor = dark ? "dark" : "light";
|
||||
|
||||
fetch("/api/map/files").then(function(r) { return r.json(); }).then(function(data) {
|
||||
var sources = {};
|
||||
var allLayers = [];
|
||||
|
||||
// Add user's PMTiles files (skip terrain). Includes map-overview if present.
|
||||
var files = (data.files || []).filter(function(f) {
|
||||
return f.name.indexOf("terrain") !== 0;
|
||||
});
|
||||
files.forEach(function(f, i) {
|
||||
sources[f.name] = {type: "vector", url: "pmtiles:///data/" + f.name};
|
||||
var layers = protomaps_themes_base.default(f.name, flavor);
|
||||
if (i > 0) {
|
||||
layers.forEach(function(l) { l.id = f.name + ":" + l.id; });
|
||||
}
|
||||
allLayers = allLayers.concat(layers);
|
||||
});
|
||||
|
||||
new maplibregl.Map({
|
||||
container: "map",
|
||||
style: {
|
||||
version: 8,
|
||||
glyphs: "/fonts/{fontstack}/{range}.pbf",
|
||||
sprite: window.location.origin + "/sprites/" + flavor,
|
||||
sources: sources,
|
||||
layers: allLayers,
|
||||
},
|
||||
center: [lon, lat],
|
||||
zoom: zoom,
|
||||
attributionControl: false,
|
||||
interactive: false,
|
||||
});
|
||||
}).catch(function(e) {
|
||||
console.error("Embed map error:", e);
|
||||
document.getElementById("map").textContent = "Map unavailable";
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -124,6 +124,43 @@
|
|||
user-select: none;
|
||||
}
|
||||
#context-menu .action:hover { background: #f0f0f0; }
|
||||
|
||||
/* Search */
|
||||
#search-container {
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
z-index: 1000;
|
||||
width: 320px;
|
||||
max-width: calc(100% - 100px);
|
||||
}
|
||||
#search-input {
|
||||
width: 100%;
|
||||
padding: 8px 12px;
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 4px;
|
||||
font-size: 14px;
|
||||
box-shadow: 0 2px 6px rgba(0,0,0,0.2);
|
||||
}
|
||||
#search-results {
|
||||
display: none;
|
||||
background: white;
|
||||
border: 1px solid #ccc;
|
||||
border-top: none;
|
||||
border-radius: 0 0 4px 4px;
|
||||
max-height: 300px;
|
||||
overflow-y: auto;
|
||||
box-shadow: 0 4px 8px rgba(0,0,0,0.2);
|
||||
}
|
||||
.search-result {
|
||||
padding: 8px 12px;
|
||||
cursor: pointer;
|
||||
border-bottom: 1px solid #eee;
|
||||
}
|
||||
.search-result:hover { background: #f0f0f0; }
|
||||
.search-result-name { font-weight: 500; }
|
||||
.search-result-kind { font-size: 11px; color: #888; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
|
@ -146,6 +183,11 @@
|
|||
|
||||
<div id="status"></div>
|
||||
|
||||
<div id="search-container">
|
||||
<input type="text" id="search-input" placeholder="Search places...">
|
||||
<div id="search-results"></div>
|
||||
</div>
|
||||
|
||||
<div id="context-menu">
|
||||
<div class="coords" id="context-coords"></div>
|
||||
<div class="action" id="context-copy">Copy GPS coordinates</div>
|
||||
|
|
@ -417,40 +459,39 @@
|
|||
var longPressTimer = null;
|
||||
var touchStartPos = null;
|
||||
var canvas = map.getCanvas();
|
||||
canvas.addEventListener("touchstart", function(e) {
|
||||
if (e.touches.length !== 1) return;
|
||||
touchStartPos = {x: e.touches[0].clientX, y: e.touches[0].clientY};
|
||||
longPressTimer = setTimeout(function() {
|
||||
var rect = canvas.getBoundingClientRect();
|
||||
var px = touchStartPos.x - rect.left;
|
||||
var py = touchStartPos.y - rect.top;
|
||||
var lngLat = map.unproject([px, py]);
|
||||
contextMenuData = {lat: lngLat.lat, lon: lngLat.lng};
|
||||
var menu = document.getElementById("context-menu");
|
||||
var z = map.getZoom();
|
||||
var d = z < 4 ? 1 : z < 8 ? 2 : z < 12 ? 3 : 4;
|
||||
document.getElementById("context-coords").textContent =
|
||||
contextMenuData.lat.toFixed(d) + ", " + contextMenuData.lon.toFixed(d);
|
||||
menu.style.left = px + "px";
|
||||
menu.style.top = py + "px";
|
||||
menu.style.display = "block";
|
||||
}, 500);
|
||||
}, {passive: true});
|
||||
canvas.addEventListener("touchmove", function(e) {
|
||||
if (longPressTimer && touchStartPos) {
|
||||
var dx = e.touches[0].clientX - touchStartPos.x;
|
||||
var dy = e.touches[0].clientY - touchStartPos.y;
|
||||
if (Math.abs(dx) > 10 || Math.abs(dy) > 10) {
|
||||
clearTimeout(longPressTimer);
|
||||
longPressTimer = null;
|
||||
canvas.addEventListener("touchstart", function(e) {
|
||||
if (e.touches.length !== 1) return;
|
||||
touchStartPos = {x: e.touches[0].clientX, y: e.touches[0].clientY};
|
||||
longPressTimer = setTimeout(function() {
|
||||
var rect = canvas.getBoundingClientRect();
|
||||
var px = touchStartPos.x - rect.left;
|
||||
var py = touchStartPos.y - rect.top;
|
||||
var lngLat = map.unproject([px, py]);
|
||||
contextMenuData = {lat: lngLat.lat, lon: lngLat.lng};
|
||||
var menu = document.getElementById("context-menu");
|
||||
var z = map.getZoom();
|
||||
var d = z < 4 ? 1 : z < 8 ? 2 : z < 12 ? 3 : 4;
|
||||
document.getElementById("context-coords").textContent =
|
||||
contextMenuData.lat.toFixed(d) + ", " + contextMenuData.lon.toFixed(d);
|
||||
menu.style.left = px + "px";
|
||||
menu.style.top = py + "px";
|
||||
menu.style.display = "block";
|
||||
}, 500);
|
||||
}, {passive: true});
|
||||
canvas.addEventListener("touchmove", function(e) {
|
||||
if (longPressTimer && touchStartPos) {
|
||||
var dx = e.touches[0].clientX - touchStartPos.x;
|
||||
var dy = e.touches[0].clientY - touchStartPos.y;
|
||||
if (Math.abs(dx) > 10 || Math.abs(dy) > 10) {
|
||||
clearTimeout(longPressTimer);
|
||||
longPressTimer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
canvas.addEventListener("touchend", function() {
|
||||
clearTimeout(longPressTimer);
|
||||
longPressTimer = null;
|
||||
});
|
||||
|
||||
});
|
||||
canvas.addEventListener("touchend", function() {
|
||||
clearTimeout(longPressTimer);
|
||||
longPressTimer = null;
|
||||
});
|
||||
map.on("load", function() {
|
||||
buildLayerControls();
|
||||
});
|
||||
|
|
@ -731,6 +772,117 @@
|
|||
if (map) reloadMap();
|
||||
});
|
||||
|
||||
// --- Search ---
|
||||
var searchInput = document.getElementById("search-input");
|
||||
var searchResults = document.getElementById("search-results");
|
||||
var searchTimeout = null;
|
||||
var searchMarker = null;
|
||||
var coordRegex = /^\s*(-?\d+\.?\d*)\s*[,\s]\s*(-?\d+\.?\d*)\s*$/;
|
||||
|
||||
searchInput.addEventListener("input", function() {
|
||||
clearTimeout(searchTimeout);
|
||||
var query = this.value.trim();
|
||||
if (query.length < 2) {
|
||||
searchResults.style.display = "none";
|
||||
return;
|
||||
}
|
||||
|
||||
// Check for coordinate input.
|
||||
var coordMatch = query.match(coordRegex);
|
||||
if (coordMatch) {
|
||||
var cLat = parseFloat(coordMatch[1]);
|
||||
var cLon = parseFloat(coordMatch[2]);
|
||||
if (cLat >= -90 && cLat <= 90 && cLon >= -180 && cLon <= 180) {
|
||||
searchResults.innerHTML = "";
|
||||
var div = document.createElement("div");
|
||||
div.className = "search-result";
|
||||
var nameEl = document.createElement("div");
|
||||
nameEl.className = "search-result-name";
|
||||
nameEl.textContent = cLat + ", " + cLon;
|
||||
div.appendChild(nameEl);
|
||||
var kindEl = document.createElement("div");
|
||||
kindEl.className = "search-result-kind";
|
||||
kindEl.textContent = "coordinates";
|
||||
div.appendChild(kindEl);
|
||||
div.onclick = function() { goToResult(cLat, cLon, cLat + ", " + cLon); };
|
||||
searchResults.appendChild(div);
|
||||
searchResults.style.display = "block";
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
searchTimeout = setTimeout(function() {
|
||||
var url = "/api/map/search?q=" + encodeURIComponent(query) + "&limit=8";
|
||||
if (map) {
|
||||
var center = map.getCenter();
|
||||
url += "&lat=" + center.lat + "&lon=" + center.lng;
|
||||
}
|
||||
fetch(url).then(function(r) { return r.json(); }).then(function(data) {
|
||||
searchResults.innerHTML = "";
|
||||
var results = data.results || [];
|
||||
if (results.length > 0) {
|
||||
results.forEach(function(r) {
|
||||
var div = document.createElement("div");
|
||||
div.className = "search-result";
|
||||
var parts = [];
|
||||
if (r.region) parts.push(r.region);
|
||||
var detail = r.kind_detail || r.kind || "";
|
||||
if (detail && !r.region) parts.push(detail.replace(/_/g, " "));
|
||||
if (r.population) {
|
||||
var pop = r.population >= 1000000 ? (r.population / 1000000).toFixed(1) + "M"
|
||||
: r.population >= 1000 ? Math.round(r.population / 1000) + "K"
|
||||
: "" + r.population;
|
||||
parts.push("pop. " + pop);
|
||||
}
|
||||
var subtitle = parts.join(" \u00b7 ");
|
||||
var nameEl = document.createElement("div");
|
||||
nameEl.className = "search-result-name";
|
||||
nameEl.textContent = r.name;
|
||||
div.appendChild(nameEl);
|
||||
if (subtitle) {
|
||||
var kindEl = document.createElement("div");
|
||||
kindEl.className = "search-result-kind";
|
||||
kindEl.textContent = subtitle;
|
||||
div.appendChild(kindEl);
|
||||
}
|
||||
div.onclick = function() { goToResult(r.lat, r.lon, r.name); };
|
||||
searchResults.appendChild(div);
|
||||
});
|
||||
searchResults.style.display = "block";
|
||||
} else {
|
||||
searchResults.style.display = "none";
|
||||
}
|
||||
}).catch(function() { searchResults.style.display = "none"; });
|
||||
}, 300);
|
||||
});
|
||||
|
||||
searchInput.addEventListener("keydown", function(e) {
|
||||
if (e.key === "Escape") {
|
||||
searchResults.style.display = "none";
|
||||
searchInput.blur();
|
||||
}
|
||||
});
|
||||
|
||||
document.addEventListener("click", function(e) {
|
||||
if (!document.getElementById("search-container").contains(e.target)) {
|
||||
searchResults.style.display = "none";
|
||||
}
|
||||
});
|
||||
|
||||
function goToResult(lat, lon, name) {
|
||||
if (!map) return;
|
||||
if (searchMarker) { searchMarker.remove(); searchMarker = null; }
|
||||
var zoom = Math.max(map.getZoom(), 12);
|
||||
map.flyTo({center: [lon, lat], zoom: zoom});
|
||||
searchMarker = new maplibregl.Marker({color: "#6435c9"})
|
||||
.setLngLat([lon, lat])
|
||||
.setPopup(new maplibregl.Popup({offset: 25}).setText(name))
|
||||
.addTo(map);
|
||||
searchMarker.togglePopup();
|
||||
searchResults.style.display = "none";
|
||||
searchInput.value = name;
|
||||
}
|
||||
|
||||
// --- Start ---
|
||||
scanFiles();
|
||||
})();
|
||||
|
|
|
|||
166
modules/map/test/test_search.py
Normal file
166
modules/map/test/test_search.py
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
import sqlite3
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _create_test_search_db(path: Path, features: list):
|
||||
"""Create a minimal .search.db file with given features for testing."""
|
||||
conn = sqlite3.connect(str(path))
|
||||
conn.execute('PRAGMA journal_mode=WAL')
|
||||
conn.execute("""
|
||||
CREATE TABLE places (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
kind TEXT,
|
||||
lat REAL NOT NULL,
|
||||
lon REAL NOT NULL,
|
||||
min_zoom REAL,
|
||||
source TEXT,
|
||||
kind_detail TEXT,
|
||||
population INTEGER,
|
||||
wikidata TEXT,
|
||||
region TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE VIRTUAL TABLE places_fts USING fts5(
|
||||
name, kind, content=places, content_rowid=id
|
||||
)
|
||||
""")
|
||||
conn.executemany(
|
||||
'INSERT INTO places (name, kind, lat, lon, min_zoom, source, kind_detail, population, region)'
|
||||
' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
||||
features,
|
||||
)
|
||||
conn.execute("INSERT INTO places_fts(places_fts) VALUES('rebuild')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def search_db(test_directory):
|
||||
"""Create a test search database with known features."""
|
||||
map_dir = test_directory / 'map'
|
||||
map_dir.mkdir(exist_ok=True)
|
||||
|
||||
features = [
|
||||
('Portland', 'locality', 45.5217, -122.6733, 4.0, 'test', 'city', 652503, 'Oregon'),
|
||||
('Portland', 'locality', 43.6559, -70.2576, 5.0, 'test', 'city', 78408, 'Maine'),
|
||||
('San Francisco', 'locality', 37.7881, -122.4097, 3.0, 'test', 'city', 873965, 'California'),
|
||||
('Springfield', 'locality', 39.7990, -89.6436, 6.0, 'test', 'city', 114230, 'Illinois'),
|
||||
('Springfield', 'locality', 37.2085, -93.2918, 6.0, 'test', 'city', 169176, 'Missouri'),
|
||||
('Yellowstone National Park', 'nature_reserve', 44.6222, -110.5616, 8.0, 'test', '', None, None),
|
||||
]
|
||||
db_path = map_dir / 'test.search.db'
|
||||
_create_test_search_db(db_path, features)
|
||||
return db_path
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_requires_query(async_client, test_directory):
|
||||
"""Search endpoint requires a query parameter."""
|
||||
request, response = await async_client.get('/api/map/search')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert 'error' in response.json
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_empty_query(async_client, test_directory):
|
||||
"""Search endpoint rejects empty query."""
|
||||
request, response = await async_client.get('/api/map/search?q=')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_no_indexes(async_client, test_directory):
|
||||
"""Search returns empty results when no search indexes exist."""
|
||||
request, response = await async_client.get('/api/map/search?q=portland')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['results'] == []
|
||||
assert response.json['total'] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_returns_results(async_client, test_directory, search_db):
|
||||
"""Search returns matching results from the index."""
|
||||
request, response = await async_client.get('/api/map/search?q=Portland')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
results = response.json['results']
|
||||
assert len(results) >= 1
|
||||
assert response.json['total'] >= 1
|
||||
assert results[0]['name'] == 'Portland'
|
||||
assert 'lat' in results[0]
|
||||
assert 'lon' in results[0]
|
||||
assert 'kind' in results[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_with_coordinates(async_client, test_directory, search_db):
|
||||
"""Search with lat/lon returns results sorted by proximity."""
|
||||
# Bias toward Portland, OR (45.5, -122.6)
|
||||
request, response = await async_client.get('/api/map/search?q=Portland&lat=45.5&lon=-122.6')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
results = response.json['results']
|
||||
assert len(results) >= 2
|
||||
# Portland, OR should be first (closest to bias point).
|
||||
assert results[0]['lat'] == pytest.approx(45.5217, abs=0.1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_prefix_matching(async_client, test_directory, search_db):
|
||||
"""Search supports prefix matching."""
|
||||
request, response = await async_client.get('/api/map/search?q=Spring')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
results = response.json['results']
|
||||
assert any(r['name'] == 'Springfield' for r in results)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_status_no_files(async_client, test_directory):
|
||||
"""Status endpoint returns empty when no map files exist."""
|
||||
request, response = await async_client.get('/api/map/search/status')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['indexed'] == []
|
||||
assert response.json['missing'] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_status_with_index(async_client, test_directory, search_db, make_files_structure):
|
||||
"""Status endpoint shows which files have indexes."""
|
||||
# Create a PMTiles file that matches the search.db stem.
|
||||
make_files_structure(['map/test.pmtiles'])
|
||||
|
||||
request, response = await async_client.get('/api/map/search/status')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['indexed']) == 1
|
||||
assert response.json['indexed'][0]['stem'] == 'test'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_status_missing_index(async_client, test_directory, make_files_structure):
|
||||
"""Status endpoint shows PMTiles files without search indexes."""
|
||||
make_files_structure(['map/usa.pmtiles'])
|
||||
|
||||
request, response = await async_client.get('/api/map/search/status')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['missing']) == 1
|
||||
assert response.json['missing'][0]['stem'] == 'usa'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_pmtiles_also_deletes_search_db(async_client, test_directory, search_db, make_files_structure):
|
||||
"""Deleting a PMTiles file also deletes its companion search.db."""
|
||||
make_files_structure(['map/test.pmtiles'])
|
||||
|
||||
# Both files exist.
|
||||
assert search_db.exists()
|
||||
|
||||
# Delete the PMTiles file.
|
||||
request, response = await async_client.delete('/api/map/files/test.pmtiles')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Search DB should also be deleted.
|
||||
assert not search_db.exists()
|
||||
|
|
@ -38,6 +38,33 @@ install_pmtiles() {
|
|||
|
||||
install_pmtiles || echo "pmtiles CLI installation failed, continuing..."
|
||||
|
||||
# --- Install/upgrade tippecanoe (for map search index building) ---
|
||||
|
||||
install_tippecanoe() {
|
||||
TIPPECANOE_VERSION="2.79.0"
|
||||
|
||||
# Check if already installed at correct version.
|
||||
if command -v tippecanoe-decode &>/dev/null; then
|
||||
CURRENT=$(tippecanoe --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || echo "")
|
||||
if [ "$CURRENT" = "$TIPPECANOE_VERSION" ]; then
|
||||
echo "tippecanoe ${TIPPECANOE_VERSION} already installed"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Building tippecanoe ${TIPPECANOE_VERSION}..."
|
||||
apt-get install -y build-essential libsqlite3-dev zlib1g-dev
|
||||
|
||||
TIPPECANOE_BUILD_DIR=$(mktemp -d)
|
||||
curl -fsSL "https://github.com/felt/tippecanoe/archive/refs/tags/${TIPPECANOE_VERSION}.tar.gz" \
|
||||
| tar -xz -C "${TIPPECANOE_BUILD_DIR}" --strip-components=1
|
||||
cd "${TIPPECANOE_BUILD_DIR}" && make -j$(nproc) && make install
|
||||
rm -rf "${TIPPECANOE_BUILD_DIR}"
|
||||
tippecanoe --version
|
||||
}
|
||||
|
||||
install_tippecanoe || echo "tippecanoe installation failed, continuing..."
|
||||
|
||||
# --- Migrate from old map stack (runs once) ---
|
||||
|
||||
MARKER_FILE="/opt/wrolpi/.protomaps_installed"
|
||||
|
|
|
|||
|
|
@ -72,6 +72,18 @@ class Events:
|
|||
def send_map_import_failed(message: str = None):
|
||||
send_event('map_import_failed', message, subject='map')
|
||||
|
||||
@staticmethod
|
||||
def send_map_search_complete(message: str = None):
|
||||
send_event('map_search_complete', message, subject='map')
|
||||
|
||||
@staticmethod
|
||||
def send_map_search_failed(message: str = None):
|
||||
send_event('map_search_failed', message, subject='map')
|
||||
|
||||
@staticmethod
|
||||
def send_tippecanoe_missing(message: str = None):
|
||||
send_event('tippecanoe_missing', message, subject='map')
|
||||
|
||||
@staticmethod
|
||||
def send_shutdown(message: str = None):
|
||||
send_event('shutdown', message, subject='shutdown')
|
||||
|
|
|
|||
|
|
@ -123,6 +123,9 @@ file_worker_cleanup = Flag('file_worker_cleanup')
|
|||
# A global refresh is currently running.
|
||||
global_refresh_active = Flag('global_refresh_active')
|
||||
|
||||
# Map search index is being built.
|
||||
map_search_building = Flag('map_search_building')
|
||||
|
||||
# The global refresh has been performed. This is False on a fresh instance of WROLPi.
|
||||
refresh_complete = Flag('refresh_complete', store_db=True)
|
||||
|
||||
|
|
@ -145,6 +148,7 @@ def get_flags() -> dict:
|
|||
file_worker_modeling=file_worker_modeling.is_set(),
|
||||
global_refresh_active=global_refresh_active.is_set(),
|
||||
have_internet=have_internet.is_set(),
|
||||
map_search_building=map_search_building.is_set(),
|
||||
outdated_zims=outdated_zims.is_set(),
|
||||
refresh_complete=refresh_complete.is_set(),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -22,14 +22,22 @@ import argparse
|
|||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
# Add project root to path so we can import catalog.
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from modules.map.catalog import MAP_REGIONS
|
||||
from wrolpi.scripts.build_map_search import build_search_index
|
||||
|
||||
SPARQL_URL = 'https://query.wikidata.org/sparql'
|
||||
SPARQL_BATCH_SIZE = 2000
|
||||
SPARQL_DELAY = 2 # seconds between batches
|
||||
|
||||
DEFAULT_CDN_BASE = 'https://wrolpi.nyc3.cdn.digitaloceanspaces.com/maps'
|
||||
|
||||
|
|
@ -102,6 +110,124 @@ def extract_region(planet_path: str, output_path: str, bbox: str, threads: int =
|
|||
return result.returncode == 0
|
||||
|
||||
|
||||
def _parse_sparql_bindings(bindings):
|
||||
"""Parse SPARQL bindings into a dict of qid -> region label."""
|
||||
# Substrings that indicate a county-level region (use parent instead).
|
||||
county_indicators = ('county', 'parish', 'borough', 'municipality',
|
||||
'district', 'canton', 'comarca', 'département',
|
||||
'united counties')
|
||||
|
||||
results = {}
|
||||
for b in bindings:
|
||||
entity_uri = b.get('entity', {}).get('value', '')
|
||||
qid = entity_uri.rsplit('/', 1)[-1] if entity_uri else ''
|
||||
parent_label = b.get('parentLabel', {}).get('value', '')
|
||||
region_label = b.get('regionLabel', {}).get('value', '')
|
||||
|
||||
# Default to the direct P131 region (usually state or province).
|
||||
# If it looks like a county, use the parent (state) instead.
|
||||
if region_label and any(ind in region_label.lower() for ind in county_indicators):
|
||||
best_label = parent_label or region_label
|
||||
else:
|
||||
best_label = region_label or parent_label
|
||||
|
||||
if qid and best_label and qid not in results:
|
||||
results[qid] = best_label
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _query_sparql_batch(qid_list, min_batch_size=100):
|
||||
"""Query Wikidata SPARQL for a batch of entity IDs. Retries with smaller chunks on failure."""
|
||||
values = ' '.join(f'wd:{qid}' for qid in qid_list)
|
||||
sparql = f"""
|
||||
SELECT ?entity ?regionLabel ?parentLabel WHERE {{
|
||||
VALUES ?entity {{ {values} }}
|
||||
?entity wdt:P131 ?region .
|
||||
OPTIONAL {{ ?region wdt:P131 ?parent . }}
|
||||
SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en" . }}
|
||||
}}
|
||||
"""
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
SPARQL_URL,
|
||||
data={'query': sparql, 'format': 'json'},
|
||||
headers={'User-Agent': 'WROLPi/1.0 (https://wrolpi.org)'},
|
||||
timeout=90,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
bindings = resp.json().get('results', {}).get('bindings', [])
|
||||
return bindings
|
||||
except Exception as e:
|
||||
if len(qid_list) <= min_batch_size:
|
||||
print(f' WARNING: batch of {len(qid_list)} failed, skipping: {e}', flush=True)
|
||||
return []
|
||||
|
||||
# Split in half and retry.
|
||||
mid = len(qid_list) // 2
|
||||
print(f' Batch of {len(qid_list)} failed, retrying as 2x{mid}...', flush=True)
|
||||
time.sleep(SPARQL_DELAY)
|
||||
left = _query_sparql_batch(qid_list[:mid], min_batch_size)
|
||||
time.sleep(SPARQL_DELAY)
|
||||
right = _query_sparql_batch(qid_list[mid:], min_batch_size)
|
||||
return left + right
|
||||
|
||||
|
||||
def enrich_search_db_with_regions(search_db_path):
|
||||
"""Query Wikidata SPARQL to look up the administrative region for each place and update the search.db.
|
||||
|
||||
Batches queries in chunks of SPARQL_BATCH_SIZE to stay within Wikidata's 60s timeout.
|
||||
Safe to re-run — only queries places that don't already have a region.
|
||||
"""
|
||||
conn = sqlite3.connect(str(search_db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
# Only collect wikidata IDs that still need enrichment (region is NULL or empty).
|
||||
rows = conn.execute(
|
||||
"SELECT id, wikidata FROM places WHERE wikidata != '' AND wikidata IS NOT NULL"
|
||||
" AND (region IS NULL OR region = '')"
|
||||
).fetchall()
|
||||
qids = {row['wikidata']: row['id'] for row in rows if row['wikidata']}
|
||||
|
||||
if not qids:
|
||||
print(f' No wikidata IDs to enrich in {search_db_path}')
|
||||
conn.close()
|
||||
return
|
||||
|
||||
print(f' Enriching {len(qids)} places with Wikidata regions...', flush=True)
|
||||
|
||||
# Batch SPARQL queries.
|
||||
qid_list = list(qids.keys())
|
||||
region_map = {} # qid -> region label
|
||||
|
||||
for i in range(0, len(qid_list), SPARQL_BATCH_SIZE):
|
||||
batch = qid_list[i:i + SPARQL_BATCH_SIZE]
|
||||
batch_num = i // SPARQL_BATCH_SIZE + 1
|
||||
total_batches = (len(qid_list) + SPARQL_BATCH_SIZE - 1) // SPARQL_BATCH_SIZE
|
||||
print(f' SPARQL batch {batch_num}/{total_batches} ({len(batch)} IDs)...', flush=True)
|
||||
|
||||
bindings = _query_sparql_batch(batch)
|
||||
parsed = _parse_sparql_bindings(bindings)
|
||||
region_map.update(parsed)
|
||||
|
||||
print(f' Got {len(bindings)} results', flush=True)
|
||||
|
||||
if i + SPARQL_BATCH_SIZE < len(qid_list):
|
||||
time.sleep(SPARQL_DELAY)
|
||||
|
||||
# Update the search.db with region labels.
|
||||
updated = 0
|
||||
for qid, region_label in region_map.items():
|
||||
if qid in qids:
|
||||
conn.execute('UPDATE places SET region = ? WHERE wikidata = ?', (region_label, qid))
|
||||
updated += 1
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f' Enriched {updated}/{len(qids)} places with region names', flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Extract map regions and generate CDN manifest.')
|
||||
parser.add_argument('planet', help='Path to the planet PMTiles file')
|
||||
|
|
@ -142,6 +268,7 @@ def main():
|
|||
|
||||
# Extract each bbox region.
|
||||
manifest_regions = {}
|
||||
search_index_urls = {}
|
||||
results = []
|
||||
failed = []
|
||||
|
||||
|
|
@ -175,6 +302,15 @@ def main():
|
|||
results.append((r['name'], region, size))
|
||||
manifest_regions[region] = cdn_url
|
||||
|
||||
# Build search index for this region, then enrich with Wikidata region names.
|
||||
search_db_path = build_search_index(output_path)
|
||||
if search_db_path:
|
||||
enrich_search_db_with_regions(search_db_path)
|
||||
search_db_name = os.path.basename(str(search_db_path))
|
||||
search_cdn_url = f'{cdn_base}/{search_db_name}'
|
||||
write_meta4(str(search_db_path), search_cdn_url)
|
||||
search_index_urls[region] = search_cdn_url
|
||||
|
||||
# Add terrain to manifest if the file exists in output_dir.
|
||||
if terrain_region:
|
||||
terrain_name = f'{terrain_region["region"]}.pmtiles'
|
||||
|
|
@ -197,6 +333,9 @@ def main():
|
|||
# Add terrain URL if present.
|
||||
if terrain_region and terrain_region['region'] in manifest_regions:
|
||||
manifest['terrain_url'] = manifest_regions[terrain_region['region']]
|
||||
# Add search index URLs.
|
||||
if search_index_urls:
|
||||
manifest['search_indexes'] = search_index_urls
|
||||
|
||||
manifest_path = os.path.join(output_dir, 'manifest.json')
|
||||
with open(manifest_path, 'w') as f:
|
||||
|
|
|
|||
321
wrolpi/scripts/build_map_search.py
Executable file
321
wrolpi/scripts/build_map_search.py
Executable file
|
|
@ -0,0 +1,321 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Build SQLite FTS5 search indexes from PMTiles vector tile files.
|
||||
|
||||
Uses tippecanoe-decode (C++) to extract place names and POIs from the `places`
|
||||
and `pois` layers in Protomaps basemap PMTiles files, then writes them into a
|
||||
SQLite FTS5 database for fast offline search.
|
||||
|
||||
Usage:
|
||||
# Build index for a single file:
|
||||
python scripts/build_map_search.py test/map/usa.pmtiles
|
||||
|
||||
# Build indexes for all .pmtiles files in a directory:
|
||||
python scripts/build_map_search.py test/map/
|
||||
|
||||
# Customize max zoom (default 10, higher = more POIs but slower):
|
||||
python scripts/build_map_search.py test/map/usa.pmtiles --max-zoom 12
|
||||
|
||||
Requirements:
|
||||
- tippecanoe-decode (part of felt/tippecanoe) installed and on PATH
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# Layers in the Protomaps basemap that contain searchable features.
|
||||
SEARCHABLE_LAYERS = {'places', 'pois'}
|
||||
|
||||
MIN_ZOOM = 2
|
||||
DEFAULT_MAX_ZOOM = 10
|
||||
|
||||
# Planet files are too large to process — they should use regional extracts instead.
|
||||
PLANET_RE = re.compile(r'^\d{8}$')
|
||||
|
||||
|
||||
def is_planet_file(name):
|
||||
"""Detect full planet files: 'planet-*.pmtiles' or bare date like '20260329.pmtiles'."""
|
||||
stem = Path(name).stem if '.' in name else name
|
||||
return stem.startswith('planet-') or bool(PLANET_RE.match(stem))
|
||||
|
||||
|
||||
def extract_features(pmtiles_path, source_name, min_zoom, max_zoom):
|
||||
"""Extract place/POI features from a PMTiles file using tippecanoe-decode.
|
||||
|
||||
Returns a list of (name, kind, lat, lon, min_zoom, source, kind_detail, population, wikidata) tuples.
|
||||
"""
|
||||
layers_args = []
|
||||
for layer in sorted(SEARCHABLE_LAYERS):
|
||||
layers_args.extend(['-l', layer])
|
||||
|
||||
cmd = [
|
||||
'tippecanoe-decode',
|
||||
*layers_args,
|
||||
'-Z', str(min_zoom),
|
||||
'-z', str(max_zoom),
|
||||
str(pmtiles_path),
|
||||
]
|
||||
|
||||
print(f' Running: {" ".join(cmd)}', flush=True)
|
||||
t0 = time.time()
|
||||
|
||||
# Stream tippecanoe-decode stdout to a temp file instead of capturing into RAM.
|
||||
# The decoded GeoJSON for regional PMTiles can reach hundreds of MB, which would
|
||||
# exhaust memory on a Raspberry Pi if held as a Python string alongside the
|
||||
# parsed dict (see PR #405 review).
|
||||
tmp_fd, tmp_path = tempfile.mkstemp(suffix='.json', prefix='tippecanoe-decode-')
|
||||
try:
|
||||
with os.fdopen(tmp_fd, 'w') as out:
|
||||
result = subprocess.run(cmd, stdout=out, stderr=subprocess.PIPE, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f'tippecanoe-decode failed: {result.stderr.strip()}')
|
||||
|
||||
elapsed = time.time() - t0
|
||||
print(f' tippecanoe-decode completed in {elapsed:.1f}s', flush=True)
|
||||
|
||||
# Parse the nested GeoJSON output.
|
||||
# Structure: FeatureCollection > tile FeatureCollections > layer FeatureCollections > features.
|
||||
t0 = time.time()
|
||||
with open(tmp_path) as f:
|
||||
fc = json.load(f)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
seen = set()
|
||||
features = []
|
||||
|
||||
for tile_fc in fc.get('features', []):
|
||||
for layer_fc in tile_fc.get('features', []):
|
||||
layer_name = layer_fc.get('properties', {}).get('layer', '')
|
||||
if layer_name not in SEARCHABLE_LAYERS:
|
||||
continue
|
||||
|
||||
for feature in layer_fc.get('features', []):
|
||||
props = feature.get('properties', {})
|
||||
name = props.get('name')
|
||||
if not name:
|
||||
continue
|
||||
|
||||
geom = feature.get('geometry', {})
|
||||
coords = geom.get('coordinates')
|
||||
if not coords:
|
||||
continue
|
||||
|
||||
kind = props.get('kind', layer_name)
|
||||
kind_detail = props.get('kind_detail', '')
|
||||
population = props.get('population')
|
||||
wikidata = props.get('wikidata', '')
|
||||
min_z = props.get('min_zoom', 0)
|
||||
|
||||
# tippecanoe-decode outputs [lon, lat] in real coordinates.
|
||||
geom_type = geom.get('type')
|
||||
if geom_type == 'Point':
|
||||
lon, lat = coords[0], coords[1]
|
||||
elif geom_type == 'MultiPoint' and coords:
|
||||
lon, lat = coords[0][0], coords[0][1]
|
||||
elif geom_type == 'Polygon' and coords and coords[0]:
|
||||
ring = coords[0]
|
||||
lon = sum(c[0] for c in ring) / len(ring)
|
||||
lat = sum(c[1] for c in ring) / len(ring)
|
||||
else:
|
||||
continue
|
||||
|
||||
lat = round(lat, 6)
|
||||
lon = round(lon, 6)
|
||||
|
||||
# Deduplicate: same name+kind within ~1.1km collapses to one entry.
|
||||
# When a duplicate is found, keep the one with higher population (more data).
|
||||
key = (name, kind, round(lat, 2), round(lon, 2))
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
features.append((name, kind, lat, lon, min_z, source_name, kind_detail, population, wikidata))
|
||||
|
||||
elapsed = time.time() - t0
|
||||
print(f' Parsed {len(features)} unique features ({elapsed:.1f}s)', flush=True)
|
||||
return features
|
||||
|
||||
|
||||
def write_search_db(features, output_path):
|
||||
"""Write features to a SQLite FTS5 database."""
|
||||
conn = sqlite3.connect(str(output_path))
|
||||
conn.execute('PRAGMA journal_mode=WAL')
|
||||
conn.execute("""
|
||||
CREATE TABLE places (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
kind TEXT,
|
||||
lat REAL NOT NULL,
|
||||
lon REAL NOT NULL,
|
||||
min_zoom REAL,
|
||||
source TEXT,
|
||||
kind_detail TEXT,
|
||||
population INTEGER,
|
||||
wikidata TEXT,
|
||||
region TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE VIRTUAL TABLE places_fts USING fts5(
|
||||
name, kind, content=places, content_rowid=id
|
||||
)
|
||||
""")
|
||||
|
||||
conn.executemany(
|
||||
'INSERT INTO places (name, kind, lat, lon, min_zoom, source, kind_detail, population, wikidata)'
|
||||
' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
||||
features,
|
||||
)
|
||||
conn.execute("INSERT INTO places_fts(places_fts) VALUES('rebuild')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def build_search_index(pmtiles_path, output_path=None, max_zoom=DEFAULT_MAX_ZOOM):
|
||||
"""Build a search index for a single PMTiles file.
|
||||
|
||||
Returns the output path on success, or None on failure.
|
||||
"""
|
||||
pmtiles_path = Path(pmtiles_path).resolve()
|
||||
|
||||
if is_planet_file(pmtiles_path.name):
|
||||
print(f'Skipping planet file {pmtiles_path.name} — use regional extracts instead.', flush=True)
|
||||
return None
|
||||
|
||||
if output_path is None:
|
||||
output_path = pmtiles_path.with_suffix('.search.db')
|
||||
else:
|
||||
output_path = Path(output_path).resolve()
|
||||
|
||||
source_name = pmtiles_path.stem
|
||||
|
||||
print(f'Building search index for {pmtiles_path.name}...', flush=True)
|
||||
print(f' Output: {output_path}', flush=True)
|
||||
print(f' Zoom range: z{MIN_ZOOM}-z{max_zoom}', flush=True)
|
||||
|
||||
if output_path.exists():
|
||||
print(f' Search index already exists, skipping extraction.', flush=True)
|
||||
return output_path
|
||||
|
||||
t_start = time.time()
|
||||
|
||||
try:
|
||||
features = extract_features(pmtiles_path, source_name, MIN_ZOOM, max_zoom)
|
||||
except RuntimeError as e:
|
||||
print(f' ERROR: {e}', flush=True)
|
||||
return None
|
||||
|
||||
if not features:
|
||||
print(f' No features found, skipping.', flush=True)
|
||||
return None
|
||||
|
||||
write_search_db(features, output_path)
|
||||
|
||||
size_mb = output_path.stat().st_size / (1024 * 1024)
|
||||
total_elapsed = time.time() - t_start
|
||||
print(f' Done: {len(features)} features, {size_mb:.1f} MB, {total_elapsed:.1f}s', flush=True)
|
||||
return output_path
|
||||
|
||||
|
||||
def build_all_in_directory(directory, max_zoom=DEFAULT_MAX_ZOOM):
|
||||
"""Build search indexes for all .pmtiles files in a directory.
|
||||
|
||||
Skips terrain files (which contain no place data).
|
||||
"""
|
||||
directory = Path(directory).resolve()
|
||||
pmtiles_files = sorted(directory.glob('*.pmtiles'))
|
||||
|
||||
if not pmtiles_files:
|
||||
print(f'No .pmtiles files found in {directory}')
|
||||
return []
|
||||
|
||||
# Skip terrain and planet files.
|
||||
pmtiles_files = [p for p in pmtiles_files
|
||||
if 'terrain' not in p.stem.lower() and not is_planet_file(p.name)]
|
||||
|
||||
print(f'Found {len(pmtiles_files)} PMTiles files in {directory}')
|
||||
results = []
|
||||
|
||||
for i, pmtiles_path in enumerate(pmtiles_files, 1):
|
||||
print(f'\n[{i}/{len(pmtiles_files)}] {pmtiles_path.name}', flush=True)
|
||||
output = build_search_index(pmtiles_path, max_zoom=max_zoom)
|
||||
if output:
|
||||
results.append(output)
|
||||
|
||||
print(f'\nBuilt {len(results)}/{len(pmtiles_files)} search indexes.')
|
||||
return results
|
||||
|
||||
|
||||
def _enrich_results(search_db_paths):
|
||||
"""Attempt to enrich search databases with Wikidata region names. Fails gracefully."""
|
||||
try:
|
||||
from wrolpi.scripts.build_map_regions import enrich_search_db_with_regions
|
||||
except ImportError:
|
||||
print('Warning: could not import enrichment function, skipping.', flush=True)
|
||||
return
|
||||
|
||||
for db_path in search_db_paths:
|
||||
try:
|
||||
enrich_search_db_with_regions(str(db_path))
|
||||
except Exception as e:
|
||||
print(f' Warning: enrichment failed for {db_path}: {e}', flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Build SQLite FTS5 search indexes from PMTiles files.',
|
||||
)
|
||||
parser.add_argument(
|
||||
'path', type=Path,
|
||||
help='PMTiles file or directory containing PMTiles files',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--output', type=Path, default=None,
|
||||
help='Output .search.db path (only for single file mode)',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--max-zoom', type=int, default=DEFAULT_MAX_ZOOM,
|
||||
help=f'Maximum zoom level to scan (default: {DEFAULT_MAX_ZOOM})',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--enrich', action='store_true',
|
||||
help='Enrich with Wikidata region names (requires internet)',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not shutil.which('tippecanoe-decode'):
|
||||
print('Error: tippecanoe-decode not found on PATH.', file=sys.stderr)
|
||||
print('Install tippecanoe: https://github.com/felt/tippecanoe', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
path = args.path.resolve()
|
||||
|
||||
if path.is_dir():
|
||||
if args.output:
|
||||
print('Error: --output cannot be used with a directory.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
results = build_all_in_directory(path, max_zoom=args.max_zoom)
|
||||
if args.enrich:
|
||||
_enrich_results(results)
|
||||
elif path.is_file() and path.suffix == '.pmtiles':
|
||||
result = build_search_index(path, output_path=args.output, max_zoom=args.max_zoom)
|
||||
if not result:
|
||||
sys.exit(1)
|
||||
if args.enrich:
|
||||
_enrich_results([result])
|
||||
else:
|
||||
print(f'Error: {path} is not a .pmtiles file or directory.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue