rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 02:48:38 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Key changes:

Browse source 
- Add MapLibre GL JS viewer with light/dark themes, terrain layer,
  and embedded static map page for coordinate sharing
- Add map pin support with persistent map_pins.yaml config
- Add map region subscriptions and downloader with progress tracking,
  sourcing pre-extracted PMTiles from a cold CDN
- Add map export to PNG and GPS coordinate copy via context menu
- Add MapViewer React component alongside reworked Map page
- Add map catalog and region-building script (build_map_regions.py)
- Remove OSM import/render scripts, leaflet files, Apache configs,
  renderd service, and osm-map-dumper Docker container
- Update Caddyfile to serve PMTiles and map static assets
- Update build/install/repair scripts for Protomaps dependencies
- Drop map_file table, add/drop map_subscription table (alembic)
- Adding button to skip primary drive mounting during onboarding.
This commit is contained in:
Roland 2026-03-29 18:58:31 -06:00
parent 0a5fe7eebe
commit b8a532c8d9
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
102 changed files with 5033 additions and 2712 deletions
Download patch file Download diff file Expand all files Collapse all files

1
modules/map/__init__.py
View file

@ -0,0 +1 @@
from .downloader import MapCatalogDownloader # noqa

125
modules/map/api.py
View file

@ -1,51 +1,100 @@
from http import HTTPStatus
from pathlib import Path
from sanic import Request, response, Blueprint
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from modules.map import lib, schema
from wrolpi import flags
from modules.map.pins import get_map_pins_config
from wrolpi.api_utils import json_response
from wrolpi.common import wrol_mode_check, get_media_directory, background_task
from wrolpi.errors import ValidationError
from wrolpi.vars import DOCKERIZED
from wrolpi.common import wrol_mode_check
from wrolpi.events import Events
map_bp = Blueprint('Map', '/api/map')
@map_bp.post('/import')
@openapi.definition(
summary='Import PBF/dump map files',
body=schema.ImportPost,
)
@validate(schema.ImportPost)
@wrol_mode_check
async def import_pbfs(_: Request, body: schema.ImportPost):
if flags.map_importing.is_set():
return response.json({'error': 'Map import already running'}, HTTPStatus.CONFLICT)
paths = [i for i in body.files if i]
if not paths:
raise ValidationError('No PBF or dump files were provided')
background_task(lib.import_files(paths))
return response.empty()
@map_bp.get('/files')
@openapi.description('Find any map files, get their import status')
def get_files_status(request: Request):
paths = lib.get_import_status(request.ctx.session)
paths = sorted(paths, key=lambda i: str(i.path))
pending = request.app.shared_ctx.map_importing.get('pending')
if pending:
pending = [Path(i).relative_to(get_media_directory()) for i in pending]
body = dict(
files=paths,
pending=pending,
import_running=flags.map_importing.is_set(),
dockerized=DOCKERIZED,
)
return json_response(body, HTTPStatus.OK)
@openapi.description('List available PMTiles map files')
def get_files(_: Request):
files = lib.get_pmtiles_files()
return json_response(dict(files=files), HTTPStatus.OK)
@map_bp.delete('/files/<filename:str>')
@openapi.description('Delete a PMTiles map file')
@wrol_mode_check
async def delete_file(_: Request, filename: str):
try:
deleted = lib.delete_pmtiles_file(filename)
except ValueError as e:
return response.json({'error': str(e)}, HTTPStatus.BAD_REQUEST)
if deleted:
return response.empty()
return response.json({'error': 'File not found'}, HTTPStatus.NOT_FOUND)
@map_bp.get('/subscribe')
@openapi.description('Get map catalog and current subscriptions')
def get_subscriptions(request: Request):
catalog = lib.get_map_catalog()
subscriptions = lib.get_map_subscriptions(request.ctx.session)
return json_response(dict(catalog=catalog, subscriptions=subscriptions), HTTPStatus.OK)
@map_bp.post('/subscribe')
@openapi.description('Subscribe to a map region for automatic downloads')
@validate(schema.MapSubscribeRequest)
@wrol_mode_check
async def post_subscribe(request: Request, body: schema.MapSubscribeRequest):
try:
await lib.subscribe(request.ctx.session, body.name, body.region)
except ValueError as e:
return response.json({'error': str(e)}, HTTPStatus.BAD_REQUEST)
Events.send_created(f'Map subscription created for {body.name}')
return response.empty(HTTPStatus.CREATED)
@map_bp.delete('/subscribe/<region:str>')
@openapi.description('Unsubscribe from a map region')
@wrol_mode_check
async def delete_subscription(request: Request, region: str):
try:
await lib.unsubscribe(request.ctx.session, region)
except ValueError as e:
return response.json({'error': str(e)}, HTTPStatus.NOT_FOUND)
return response.empty(HTTPStatus.NO_CONTENT)
@map_bp.get('/pins')
@openapi.description('Get all map pins')
def get_pins(_: Request):
pins = get_map_pins_config().pins
return json_response(dict(pins=pins), HTTPStatus.OK)
@map_bp.post('/pins')
@openapi.description('Add a map pin')
@validate(schema.MapPinRequest)
async def add_pin(_: Request, body: schema.MapPinRequest):
pin = get_map_pins_config().add_pin(body.lat, body.lon, body.label, body.color)
return json_response(dict(pin=pin), HTTPStatus.CREATED)
@map_bp.delete('/pins/<pin_id:int>')
@openapi.description('Delete a map pin by ID')
async def delete_pin(_: Request, pin_id: int):
if get_map_pins_config().delete_pin(pin_id):
return response.empty(HTTPStatus.NO_CONTENT)
return response.json({'error': 'Pin not found'}, HTTPStatus.NOT_FOUND)
@map_bp.put('/pins/<pin_id:int>')
@openapi.description('Update a map pin by ID')
@validate(schema.MapPinUpdateRequest)
async def update_pin(_: Request, pin_id: int, body: schema.MapPinUpdateRequest):
if get_map_pins_config().update_pin(pin_id, body.label, body.color):
return response.empty(HTTPStatus.NO_CONTENT)
return response.json({'error': 'Pin not found'}, HTTPStatus.NOT_FOUND)

233
modules/map/catalog.py Normal file
View file

@ -0,0 +1,233 @@
GB = 1_000_000_000
MB = 1_000_000
# Manifest URL — points to a JSON file on the CDN that contains the current planet file URL.
MANIFEST_URL = 'https://wrolpi.nyc3.cdn.digitaloceanspaces.com/maps/manifest.json'
# Predefined regions with bounding boxes for pmtiles extract.
# These regions collectively cover the entire planet.
# All regions are kept under ~9 GB to avoid OOM on 4 GB Raspberry Pis.
# Sizes measured from the 20260329 planet file.
MAP_REGIONS = [
# United States (split into 3 for Pi compatibility)
{
'name': 'United States (West)',
'region': 'us-west',
'bbox': '-125.0,24.4,-104.0,49.4',
'size_estimate': 5 * GB,
},
{
'name': 'United States (Central)',
'region': 'us-central',
'bbox': '-104.0,24.4,-87.0,49.4',
'size_estimate': 5 * GB,
},
{
'name': 'United States (East)',
'region': 'us-east',
'bbox': '-87.0,24.4,-66.9,49.4',
'size_estimate': 8 * GB,
},
{
'name': 'Alaska',
'region': 'us-alaska',
'bbox': '-180.0,51.0,-129.0,71.5',
'size_estimate': int(1.4 * GB),
},
# Canada (split into 3)
{
'name': 'Canada (West)',
'region': 'canada-west',
'bbox': '-141.0,48.0,-110.0,83.5',
'size_estimate': 5 * GB,
},
{
'name': 'Canada (Central)',
'region': 'canada-central',
'bbox': '-110.0,48.0,-80.0,83.5',
'size_estimate': 6 * GB,
},
{
'name': 'Canada (East)',
'region': 'canada-east',
'bbox': '-80.0,41.7,-52.0,83.5',
'size_estimate': 6 * GB,
},
{
'name': 'Mexico',
'region': 'mexico',
'bbox': '-118.5,14.5,-86.7,32.8',
'size_estimate': int(2.5 * GB),
},
# Central & South America
{
'name': 'Central America & Caribbean',
'region': 'central-america',
'bbox': '-92.2,7.2,-59.4,23.5',
'size_estimate': int(1.4 * GB),
},
{
'name': 'South America',
'region': 'south-america',
'bbox': '-81.4,-56.0,-34.7,12.5',
'size_estimate': int(7.8 * GB),
},
# Europe (split into 9 grid boxes — dense OSM coverage requires small regions)
{
'name': 'Europe (NW: UK, Ireland, Iceland)',
'region': 'europe-nw',
'bbox': '-25.0,48.0,5.0,72.0',
'size_estimate': int(5.5 * GB),
},
{
'name': 'Europe (SW: Spain, Portugal)',
'region': 'europe-sw',
'bbox': '-25.0,34.0,5.0,48.0',
'size_estimate': int(5.6 * GB),
},
{
'name': 'Europe (N-Central: Scandinavia)',
'region': 'europe-n-central',
'bbox': '5.0,55.0,15.0,72.0',
'size_estimate': int(3.6 * GB),
},
{
'name': 'Europe (Central: Germany, Benelux)',
'region': 'europe-central',
'bbox': '5.0,48.0,15.0,55.0',
'size_estimate': int(6.9 * GB),
},
{
'name': 'Europe (S-Central: Italy, France)',
'region': 'europe-s-central',
'bbox': '5.0,34.0,15.0,48.0',
'size_estimate': int(4.9 * GB),
},
{
'name': 'Europe (NE-Central: Poland, Baltics)',
'region': 'europe-ne-central',
'bbox': '15.0,48.0,25.0,72.0',
'size_estimate': int(7.2 * GB),
},
{
'name': 'Europe (SE-Central: Balkans, Greece)',
'region': 'europe-se-central',
'bbox': '15.0,34.0,25.0,48.0',
'size_estimate': int(3.2 * GB),
},
{
'name': 'Europe (NE: Ukraine, Belarus)',
'region': 'europe-ne',
'bbox': '25.0,48.0,45.0,72.0',
'size_estimate': int(7.4 * GB),
},
{
'name': 'Europe (SE: Turkey, Black Sea)',
'region': 'europe-se',
'bbox': '25.0,34.0,45.0,48.0',
'size_estimate': int(3.4 * GB),
},
# Africa (split into 2)
{
'name': 'Africa (North)',
'region': 'africa-north',
'bbox': '-18.0,0.0,52.0,37.5',
'size_estimate': 9 * GB,
},
{
'name': 'Africa (South)',
'region': 'africa-south',
'bbox': '-18.0,-35.0,52.0,0.0',
'size_estimate': 5 * GB,
},
# Middle East
{
'name': 'Middle East',
'region': 'middle-east',
'bbox': '25.0,12.0,63.0,42.0',
'size_estimate': int(4.1 * GB),
},
# Russia & Northern Asia (split into 3)
{
'name': 'Russia (West)',
'region': 'russia-west',
'bbox': '27.0,41.0,45.0,82.0',
'size_estimate': 8 * GB,
},
{
'name': 'Russia (Central)',
'region': 'russia-central',
'bbox': '45.0,41.0,100.0,82.0',
'size_estimate': 3 * GB,
},
{
'name': 'Russia (East)',
'region': 'russia-east',
'bbox': '100.0,41.0,180.0,82.0',
'size_estimate': 4 * GB,
},
# South Asia
{
'name': 'South Asia',
'region': 'south-asia',
'bbox': '60.0,5.0,98.0,40.0',
'size_estimate': int(5.5 * GB),
},
# East Asia (split into 2)
{
'name': 'East Asia (North)',
'region': 'east-asia-north',
'bbox': '73.0,35.0,150.0,53.5',
'size_estimate': 6 * GB,
},
{
'name': 'East Asia (South)',
'region': 'east-asia-south',
'bbox': '73.0,18.0,150.0,35.0',
'size_estimate': 8 * GB,
},
# Southeast Asia
{
'name': 'Southeast Asia',
'region': 'southeast-asia',
'bbox': '92.0,-11.0,141.0,28.5',
'size_estimate': 6 * GB,
},
# Oceania & Pacific
{
'name': 'Oceania & Australia',
'region': 'oceania',
'bbox': '110.0,-47.0,180.0,0.0',
'size_estimate': int(3.7 * GB),
},
{
'name': 'Pacific Islands',
'region': 'pacific-islands',
'bbox': '-180.0,-47.0,-120.0,28.5',
'size_estimate': 88 * MB,
},
# Arctic & Antarctic
{
'name': 'Greenland',
'region': 'greenland',
'bbox': '-74.0,59.0,-10.0,84.0',
'size_estimate': int(1.5 * GB),
},
{
'name': 'Antarctica',
'region': 'antarctica',
'bbox': '-180.0,-90.0,180.0,-60.0',
'size_estimate': 423 * MB,
},
# Terrain (DEM) — direct download, no bbox extraction.
{
'name': 'Terrain (Global Hillshade & Contours)',
'region': 'terrain-z0-8',
'bbox': None,
'size_estimate': 12 * GB,
'terrain': True,
},
]
MAP_REGIONS_BY_NAME = {r['name']: r for r in MAP_REGIONS}
MAP_REGIONS_BY_REGION = {r['region']: r for r in MAP_REGIONS}

207
modules/map/downloader.py Normal file
View file

@ -0,0 +1,207 @@
import tempfile
from pathlib import Path
from xml.etree import ElementTree as ET
from modules.map import lib
from wrolpi.common import logger, verify_gpg_signature
from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult, download_manager
__all__ = ['MapCatalogDownloader', 'MapExtractDownloader', 'map_catalog_downloader', 'map_extract_downloader']
logger = logger.getChild(__name__)
METALINK_NS = {'ml': 'urn:ietf:params:xml:ns:metalink'}
async def verify_meta4_signature(meta4_contents: bytes) -> bool:
"""Parse meta4 XML bytes, extract the hash and its GPG signature, and verify.
Returns True if the signature is valid, False if invalid or missing."""
try:
root = ET.fromstring(meta4_contents)
file_el = root.find('ml:file', METALINK_NS)
if file_el is None:
logger.warning('meta4 missing <file> element')
return False
hash_el = file_el.find('ml:hash[@type="sha-256"]', METALINK_NS)
sig_el = file_el.find('ml:signature', METALINK_NS)
if hash_el is None or hash_el.text is None:
logger.warning('meta4 missing <hash type="sha-256"> element')
return False
if sig_el is None or sig_el.text is None:
logger.warning('meta4 missing <signature> element — skipping verification')
return False
sha256_hash = hash_el.text.strip()
signature = sig_el.text.strip()
except ET.ParseError as e:
logger.warning(f'Failed to parse meta4 XML: {e}')
return False
# Write hash and signature to temp files for GPG verification.
with tempfile.TemporaryDirectory() as tmpdir:
hash_path = Path(tmpdir) / 'hash'
sig_path = Path(tmpdir) / 'hash.sig'
hash_path.write_text(sha256_hash)
sig_path.write_text(signature)
return await verify_gpg_signature(hash_path, sig_path)
class MapCatalogDownloader(Downloader):
"""Fetches the manifest and creates a child Download for each subscribed region."""
name = 'map_catalog'
listable = False
async def do_download(self, download: Download) -> DownloadResult:
try:
manifest = await lib.fetch_manifest(download.url)
except Exception as e:
logger.error(f'Failed to fetch map manifest: {e}')
return DownloadResult(success=False, error=f'Failed to fetch manifest: {e}')
version = manifest.get('version', '')
manifest_regions = manifest.get('regions', {})
if not manifest_regions:
return DownloadResult(success=False, error='Manifest does not contain regions')
regions = (download.settings or {}).get('regions', [])
if not regions:
return DownloadResult(success=True)
created = 0
with get_db_session(commit=True) as session:
for r in regions:
region = r.get('region', '')
region_url = manifest_regions.get(region)
if not region_url:
logger.warning(f'Subscribed region {region!r} not found in manifest — skipping')
continue
# Delete stale download for this region if the URL changed (new version).
old = session.query(Download).filter(
Download.downloader == 'map_extract',
Download.settings['region'].astext == region,
Download.url != region_url,
).one_or_none()
if old:
logger.info(f'Deleting stale download for {region}: {old.url}')
session.delete(old)
session.flush()
child = download_manager.get_or_create_download(session, region_url, reset_attempts=True,
override_skip=True)
child.downloader = 'map_extract'
child.location = '/map/manage'
child.settings = {
'region': region,
'version': version,
}
child.renew(reset_attempts=True)
created += 1
if not created:
return DownloadResult(success=False, error='No matching region URLs found in manifest')
logger.info(f'Created {created} map region downloads')
return DownloadResult(success=True, location='/map/manage')
map_catalog_downloader = MapCatalogDownloader()
class MapExtractDownloader(Downloader):
"""Downloads a pre-extracted region file from the CDN via aria2c.
Settings contain: region, version — set by MapCatalogDownloader.
Fetches the meta4 file once, GPG-verifies the hash signature, then passes
the same meta4 bytes to aria2c for download hash verification. This ensures
the hash that was GPG-verified is the same hash aria2c checks against.
"""
name = 'map_extract'
listable = False
async def do_download(self, download: Download) -> DownloadResult:
settings = download.settings or {}
region = settings.get('region')
version = settings.get('version', '')
if not region:
return DownloadResult(success=False, error='No region in download settings')
map_directory = lib.get_map_directory()
# Versioned filename.
if version:
output_name = f'{region}-{version}.pmtiles'
else:
output_name = f'{region}.pmtiles'
output_path = map_directory / output_name
tmp_path = map_directory / f'{output_name}.tmp'
# Skip if versioned file already exists.
if output_path.is_file():
logger.info(f'{output_name} already exists, skipping download')
return DownloadResult(success=True, location='/map')
# Clean up any leftover temp file.
if tmp_path.is_file():
tmp_path.unlink()
# Fetch meta4 once, GPG-verify the hash signature before downloading.
meta4_contents = await self.get_meta4_contents(download.url)
if meta4_contents:
if not await verify_meta4_signature(meta4_contents):
return DownloadResult(success=False,
error=f'meta4 hash signature verification failed for {region}')
logger.info(f'meta4 signature verified for {region}')
else:
logger.warning(f'No meta4 available for {region} — downloading without hash verification')
# Download via aria2c, passing the pre-verified meta4 for hash checking.
logger.warning(f'Downloading {region} from {download.url}')
downloaded_path = None
try:
downloaded_path = await self.download_file(
download, download.url, map_directory,
check_for_meta4=False, concurrent=1, meta4_xml=meta4_contents,
)
downloaded_path.rename(tmp_path)
except Exception as e:
logger.error(f'Download failed for {region}: {e}')
tmp_path.unlink(missing_ok=True)
if downloaded_path:
downloaded_path.unlink(missing_ok=True)
return DownloadResult(success=False, error=f'Download failed: {str(e)[:500]}')
if not tmp_path.is_file():
return DownloadResult(success=False, error=f'Output file was not created: {tmp_path}')
# Replace old file atomically.
tmp_path.rename(output_path)
# Delete older versions of this region.
for old_file in map_directory.glob(f'{region}-*.pmtiles'):
if old_file.name != output_name:
logger.warning(f'Deleting old map version: {old_file}')
old_file.unlink()
# Also delete unversioned file if it exists.
unversioned = map_directory / f'{region}.pmtiles'
if unversioned.is_file() and unversioned.name != output_name:
logger.warning(f'Deleting unversioned map file: {unversioned}')
unversioned.unlink()
size = output_path.stat().st_size
logger.warning(f'Successfully downloaded {region} ({size} bytes) to {output_path}')
return DownloadResult(success=True, location='/map')
map_extract_downloader = MapExtractDownloader()

758
modules/map/leaflet.css
View file

@ -1,758 +0,0 @@
/* required styles */
/* @preserve
* Leaflet 1.7.1, a JS library for interactive maps. http://leafletjs.com
* (c) 2010-2019 Vladimir Agafonkin, (c) 2010-2011 CloudMade
*/
.leaflet-pane,
.leaflet-tile,
.leaflet-marker-icon,
.leaflet-marker-shadow,
.leaflet-tile-container,
.leaflet-pane > svg,
.leaflet-pane > canvas,
.leaflet-zoom-box,
.leaflet-image-layer,
.leaflet-layer {
position: absolute;
left: 0;
top: 0;
}
.leaflet-container {
overflow: hidden;
}
.leaflet-tile,
.leaflet-marker-icon,
.leaflet-marker-shadow {
-webkit-user-select: none;
-moz-user-select: none;
user-select: none;
-webkit-user-drag: none;
}
/* Prevents IE11 from highlighting tiles in blue */
.leaflet-tile::selection {
background: transparent;
}
/* Safari renders non-retina tile on retina better with this, but Chrome is worse */
.leaflet-safari .leaflet-tile {
image-rendering: -webkit-optimize-contrast;
}
/* hack that prevents hw layers "stretching" when loading new tiles */
.leaflet-safari .leaflet-tile-container {
width: 1600px;
height: 1600px;
-webkit-transform-origin: 0 0;
}
.leaflet-marker-icon,
.leaflet-marker-shadow {
display: block;
}
/* .leaflet-container svg: reset svg max-width decleration shipped in Joomla! (joomla.org) 3.x */
/* .leaflet-container img: map is broken in FF if you have max-width: 100% on tiles */
.leaflet-container .leaflet-overlay-pane svg,
.leaflet-container .leaflet-marker-pane img,
.leaflet-container .leaflet-shadow-pane img,
.leaflet-container .leaflet-tile-pane img,
.leaflet-container img.leaflet-image-layer,
.leaflet-container .leaflet-tile {
max-width: none !important;
max-height: none !important;
}
.leaflet-container.leaflet-touch-zoom {
-ms-touch-action: pan-x pan-y;
touch-action: pan-x pan-y;
}
.leaflet-container.leaflet-touch-drag {
-ms-touch-action: pinch-zoom;
/* Fallback for FF which doesn't support pinch-zoom */
touch-action: none;
touch-action: pinch-zoom;
}
.leaflet-container.leaflet-touch-drag.leaflet-touch-zoom {
-ms-touch-action: none;
touch-action: none;
}
.leaflet-container {
-webkit-tap-highlight-color: transparent;
}
.leaflet-container a {
-webkit-tap-highlight-color: rgba(51, 181, 229, 0.4);
}
.leaflet-tile {
filter: inherit;
visibility: hidden;
}
.leaflet-tile-loaded {
visibility: inherit;
}
.leaflet-zoom-box {
width: 0;
height: 0;
-moz-box-sizing: border-box;
box-sizing: border-box;
z-index: 800;
}
/* workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=888319 */
.leaflet-overlay-pane svg {
-moz-user-select: none;
}
.leaflet-pane {
z-index: 400;
}
.leaflet-tile-pane {
z-index: 200;
}
.leaflet-overlay-pane {
z-index: 400;
}
.leaflet-shadow-pane {
z-index: 500;
}
.leaflet-marker-pane {
z-index: 600;
}
.leaflet-tooltip-pane {
z-index: 650;
}
.leaflet-popup-pane {
z-index: 700;
}
.leaflet-map-pane canvas {
z-index: 100;
}
.leaflet-map-pane svg {
z-index: 200;
}
.leaflet-vml-shape {
width: 1px;
height: 1px;
}
.lvml {
behavior: url(#default#VML);
display: inline-block;
position: absolute;
}
/* control positioning */
.leaflet-control {
position: relative;
z-index: 800;
pointer-events: visiblePainted; /* IE 9-10 doesn't have auto */
pointer-events: auto;
}
.leaflet-top,
.leaflet-bottom {
position: absolute;
z-index: 1000;
pointer-events: none;
}
.leaflet-top {
top: 0;
}
.leaflet-right {
right: 0;
}
.leaflet-bottom {
bottom: 0;
}
.leaflet-left {
left: 0;
}
.leaflet-control {
float: left;
clear: both;
}
.leaflet-right .leaflet-control {
float: right;
}
.leaflet-top .leaflet-control {
margin-top: 10px;
}
.leaflet-bottom .leaflet-control {
margin-bottom: 10px;
}
.leaflet-left .leaflet-control {
margin-left: 10px;
}
.leaflet-right .leaflet-control {
margin-right: 10px;
}
/* zoom and fade animations */
.leaflet-fade-anim .leaflet-tile {
will-change: opacity;
}
.leaflet-fade-anim .leaflet-popup {
opacity: 0;
-webkit-transition: opacity 0.2s linear;
-moz-transition: opacity 0.2s linear;
transition: opacity 0.2s linear;
}
.leaflet-fade-anim .leaflet-map-pane .leaflet-popup {
opacity: 1;
}
.leaflet-zoom-animated {
-webkit-transform-origin: 0 0;
-ms-transform-origin: 0 0;
transform-origin: 0 0;
}
.leaflet-zoom-anim .leaflet-zoom-animated {
will-change: transform;
}
.leaflet-zoom-anim .leaflet-zoom-animated {
-webkit-transition: -webkit-transform 0.25s cubic-bezier(0, 0, 0.25, 1);
-moz-transition: -moz-transform 0.25s cubic-bezier(0, 0, 0.25, 1);
transition: transform 0.25s cubic-bezier(0, 0, 0.25, 1);
}
.leaflet-zoom-anim .leaflet-tile,
.leaflet-pan-anim .leaflet-tile {
-webkit-transition: none;
-moz-transition: none;
transition: none;
}
.leaflet-zoom-anim .leaflet-zoom-hide {
visibility: hidden;
}
/* cursors */
.leaflet-interactive {
cursor: pointer;
}
.leaflet-grab {
cursor: -webkit-grab;
cursor: -moz-grab;
cursor: grab;
}
.leaflet-crosshair,
.leaflet-crosshair .leaflet-interactive {
cursor: crosshair;
}
.leaflet-popup-pane,
.leaflet-control {
cursor: auto;
}
.leaflet-dragging .leaflet-grab,
.leaflet-dragging .leaflet-grab .leaflet-interactive,
.leaflet-dragging .leaflet-marker-draggable {
cursor: move;
cursor: -webkit-grabbing;
cursor: -moz-grabbing;
cursor: grabbing;
}
/* marker & overlays interactivity */
.leaflet-marker-icon,
.leaflet-marker-shadow,
.leaflet-image-layer,
.leaflet-pane > svg path,
.leaflet-tile-container {
pointer-events: none;
}
.leaflet-marker-icon.leaflet-interactive,
.leaflet-image-layer.leaflet-interactive,
.leaflet-pane > svg path.leaflet-interactive,
svg.leaflet-image-layer.leaflet-interactive path {
pointer-events: visiblePainted; /* IE 9-10 doesn't have auto */
pointer-events: auto;
}
/* visual tweaks */
.leaflet-container {
background: #ddd;
outline: 0;
}
.leaflet-container a {
color: #0078A8;
}
.leaflet-container a.leaflet-active {
outline: 2px solid orange;
}
.leaflet-zoom-box {
border: 2px dotted #38f;
background: rgba(255, 255, 255, 0.5);
}
/* general typography */
.leaflet-container {
font: 12px/1.5 "Helvetica Neue", Arial, Helvetica, sans-serif;
}
/* general toolbar styles */
.leaflet-bar {
box-shadow: 0 1px 5px rgba(0, 0, 0, 0.65);
border-radius: 4px;
}
.leaflet-bar a,
.leaflet-bar a:hover {
background-color: #fff;
border-bottom: 1px solid #ccc;
width: 26px;
height: 26px;
line-height: 26px;
display: block;
text-align: center;
text-decoration: none;
color: black;
}
.leaflet-bar a,
.leaflet-control-layers-toggle {
background-position: 50% 50%;
background-repeat: no-repeat;
display: block;
}
.leaflet-bar a:hover {
background-color: #f4f4f4;
}
.leaflet-bar a:first-child {
border-top-left-radius: 4px;
border-top-right-radius: 4px;
}
.leaflet-bar a:last-child {
border-bottom-left-radius: 4px;
border-bottom-right-radius: 4px;
border-bottom: none;
}
.leaflet-bar a.leaflet-disabled {
cursor: default;
background-color: #f4f4f4;
color: #bbb;
}
.leaflet-touch .leaflet-bar a {
width: 30px;
height: 30px;
line-height: 30px;
}
.leaflet-touch .leaflet-bar a:first-child {
border-top-left-radius: 2px;
border-top-right-radius: 2px;
}
.leaflet-touch .leaflet-bar a:last-child {
border-bottom-left-radius: 2px;
border-bottom-right-radius: 2px;
}
/* zoom control */
.leaflet-control-zoom-in,
.leaflet-control-zoom-out {
font: bold 18px 'Lucida Console', Monaco, monospace;
text-indent: 1px;
}
.leaflet-touch .leaflet-control-zoom-in, .leaflet-touch .leaflet-control-zoom-out {
font-size: 22px;
}
/* layers control */
.leaflet-control-layers {
box-shadow: 0 1px 5px rgba(0, 0, 0, 0.4);
background: #fff;
border-radius: 5px;
}
.leaflet-control-layers-toggle {
background-image: url(images/layers.png);
width: 36px;
height: 36px;
}
.leaflet-retina .leaflet-control-layers-toggle {
background-image: url(images/layers-2x.png);
background-size: 26px 26px;
}
.leaflet-touch .leaflet-control-layers-toggle {
width: 44px;
height: 44px;
}
.leaflet-control-layers .leaflet-control-layers-list,
.leaflet-control-layers-expanded .leaflet-control-layers-toggle {
display: none;
}
.leaflet-control-layers-expanded .leaflet-control-layers-list {
display: block;
position: relative;
}
.leaflet-control-layers-expanded {
padding: 6px 10px 6px 6px;
color: #333;
background: #fff;
}
.leaflet-control-layers-scrollbar {
overflow-y: scroll;
overflow-x: hidden;
padding-right: 5px;
}
.leaflet-control-layers-selector {
margin-top: 2px;
position: relative;
top: 1px;
}
.leaflet-control-layers label {
display: block;
}
.leaflet-control-layers-separator {
height: 0;
border-top: 1px solid #ddd;
margin: 5px -10px 5px -6px;
}
/* Default icon URLs */
.leaflet-default-icon-path {
background-image: url(images/marker-icon.png);
}
/* attribution and scale controls */
.leaflet-container .leaflet-control-attribution {
background: #fff;
background: rgba(255, 255, 255, 0.7);
margin: 0;
}
.leaflet-control-attribution,
.leaflet-control-scale-line {
padding: 0 5px;
color: #333;
}
.leaflet-control-attribution a {
text-decoration: none;
}
.leaflet-control-attribution a:hover {
text-decoration: underline;
}
.leaflet-container .leaflet-control-attribution,
.leaflet-container .leaflet-control-scale {
font-size: 11px;
}
.leaflet-left .leaflet-control-scale {
margin-left: 5px;
}
.leaflet-bottom .leaflet-control-scale {
margin-bottom: 5px;
}
.leaflet-control-scale-line {
border: 2px solid #777;
border-top: none;
line-height: 1.1;
padding: 2px 5px 1px;
font-size: 11px;
white-space: nowrap;
overflow: hidden;
-moz-box-sizing: border-box;
box-sizing: border-box;
background: #fff;
background: rgba(255, 255, 255, 0.5);
}
.leaflet-control-scale-line:not(:first-child) {
border-top: 2px solid #777;
border-bottom: none;
margin-top: -2px;
}
.leaflet-control-scale-line:not(:first-child):not(:last-child) {
border-bottom: 2px solid #777;
}
.leaflet-touch .leaflet-control-attribution,
.leaflet-touch .leaflet-control-layers,
.leaflet-touch .leaflet-bar {
box-shadow: none;
}
.leaflet-touch .leaflet-control-layers,
.leaflet-touch .leaflet-bar {
border: 2px solid rgba(0, 0, 0, 0.2);
background-clip: padding-box;
}
/* popup */
.leaflet-popup {
position: absolute;
text-align: center;
margin-bottom: 20px;
}
.leaflet-popup-content-wrapper {
padding: 1px;
text-align: left;
border-radius: 12px;
}
.leaflet-popup-content {
margin: 13px 19px;
line-height: 1.4;
}
.leaflet-popup-content p {
margin: 18px 0;
}
.leaflet-popup-tip-container {
width: 40px;
height: 20px;
position: absolute;
left: 50%;
margin-left: -20px;
overflow: hidden;
pointer-events: none;
}
.leaflet-popup-tip {
width: 17px;
height: 17px;
padding: 1px;
margin: -10px auto 0;
-webkit-transform: rotate(45deg);
-moz-transform: rotate(45deg);
-ms-transform: rotate(45deg);
transform: rotate(45deg);
}
.leaflet-popup-content-wrapper,
.leaflet-popup-tip {
background: white;
color: #333;
box-shadow: 0 3px 14px rgba(0, 0, 0, 0.4);
}
.leaflet-container a.leaflet-popup-close-button {
position: absolute;
top: 0;
right: 0;
padding: 4px 4px 0 0;
border: none;
text-align: center;
width: 18px;
height: 14px;
font: 16px/14px Tahoma, Verdana, sans-serif;
color: #c3c3c3;
text-decoration: none;
font-weight: bold;
background: transparent;
}
.leaflet-container a.leaflet-popup-close-button:hover {
color: #999;
}
.leaflet-popup-scrolled {
overflow: auto;
border-bottom: 1px solid #ddd;
border-top: 1px solid #ddd;
}
.leaflet-oldie .leaflet-popup-content-wrapper {
-ms-zoom: 1;
}
.leaflet-oldie .leaflet-popup-tip {
width: 24px;
margin: 0 auto;
-ms-filter: "progid:DXImageTransform.Microsoft.Matrix(M11=0.70710678, M12=0.70710678, M21=-0.70710678, M22=0.70710678)";
filter: progid:DXImageTransform.Microsoft.Matrix(M11=0.70710678, M12=0.70710678, M21=-0.70710678, M22=0.70710678);
}
.leaflet-oldie .leaflet-popup-tip-container {
margin-top: -1px;
}
.leaflet-oldie .leaflet-control-zoom,
.leaflet-oldie .leaflet-control-layers,
.leaflet-oldie .leaflet-popup-content-wrapper,
.leaflet-oldie .leaflet-popup-tip {
border: 1px solid #999;
}
/* div icon */
.leaflet-div-icon {
background: #fff;
border: 1px solid #666;
}
/* Tooltip */
/* Base styles for the element that has a tooltip */
.leaflet-tooltip {
position: absolute;
padding: 6px;
background-color: #fff;
border: 1px solid #fff;
border-radius: 3px;
color: #222;
white-space: nowrap;
-webkit-user-select: none;
-moz-user-select: none;
-ms-user-select: none;
user-select: none;
pointer-events: none;
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.4);
}
.leaflet-tooltip.leaflet-clickable {
cursor: pointer;
pointer-events: auto;
}
.leaflet-tooltip-top:before,
.leaflet-tooltip-bottom:before,
.leaflet-tooltip-left:before,
.leaflet-tooltip-right:before {
position: absolute;
pointer-events: none;
border: 6px solid transparent;
background: transparent;
content: "";
}
/* Directions */
.leaflet-tooltip-bottom {
margin-top: 6px;
}
.leaflet-tooltip-top {
margin-top: -6px;
}
.leaflet-tooltip-bottom:before,
.leaflet-tooltip-top:before {
left: 50%;
margin-left: -6px;
}
.leaflet-tooltip-top:before {
bottom: 0;
margin-bottom: -12px;
border-top-color: #fff;
}
.leaflet-tooltip-bottom:before {
top: 0;
margin-top: -12px;
margin-left: -6px;
border-bottom-color: #fff;
}
.leaflet-tooltip-left {
margin-left: -6px;
}
.leaflet-tooltip-right {
margin-left: 6px;
}
.leaflet-tooltip-left:before,
.leaflet-tooltip-right:before {
top: 50%;
margin-top: -6px;
}
.leaflet-tooltip-left:before {
right: 0;
margin-right: -12px;
border-left-color: #fff;
}
.leaflet-tooltip-right:before {
left: 0;
margin-left: -12px;
border-right-color: #fff;
}

39
modules/map/leaflet.html
View file

@ -1,39 +0,0 @@
<!DOCTYPE HTML>
<html lang="en">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<script src="/leaflet.js"></script>
<link rel="stylesheet" href="/leaflet.css"/>
<title>Leaflet Map</title>
<style>
html, body {
height: 100%;
padding: 0;
margin: 0;
}
#map {
/* configure the size of the map */
width: 100%;
height: 100%;
}
</style>
</head>
<body>
<div id="map"></div>
<script>
// initialize Leaflet
var map = L.map('map').setView({lon: 0, lat: 0}, 2);
// add the OpenStreetMap tiles
L.tileLayer(`/tile/{z}/{x}/{y}.png`, {
maxZoom: 19,
attribution: '&copy; <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
}).addTo(map);
// show the scale bar on the lower left corner
L.control.scale({imperial: true, metric: true}).addTo(map);
</script>
</body>
</html>

6
modules/map/leaflet.js
View file

File diff suppressed because one or more lines are too long

344
modules/map/lib.py
View file

@ -1,19 +1,13 @@
import subprocess
from datetime import timedelta
import json
import tempfile
from pathlib import Path
from typing import List
from typing import List, Dict
from sqlalchemy.orm import Session
from modules.map.models import MapFile
from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.cmd import SUDO_BIN, run_command
from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config
from wrolpi.dates import timedelta_to_timestamp, seconds_to_timestamp, now
from wrolpi.db import get_db_session
from wrolpi.events import Events
from wrolpi.vars import PROJECT_DIR, IS_RPI5
from modules.map.catalog import MAP_REGIONS, MAP_REGIONS_BY_NAME, MAP_REGIONS_BY_REGION, MANIFEST_URL
from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config, aiohttp_get, verify_gpg_signature
from wrolpi.downloader import DownloadFrequency, Download, save_downloads_config, download_manager
logger = logger.getChild(__name__)
@ -25,229 +19,151 @@ def get_map_directory() -> Path:
return map_directory
def is_pbf_file(pbf: Path) -> bool:
"""Uses file command to check type of file. Returns True if a file is an OpenStreetMap PBF file."""
cmd = ('/usr/bin/file', pbf)
try:
output = subprocess.check_output(cmd)
except FileNotFoundError:
return False
return 'OpenStreetMap Protocolbuffer Binary Format' in output.decode()
def is_dump_file(path: Path) -> bool:
"""Uses file command to check the type of file. Returns True if a file is a Postgresql dump file."""
cmd = ('/usr/bin/file', path)
try:
output = subprocess.check_output(cmd)
except FileNotFoundError:
return False
return 'PostgreSQL custom database dump' in output.decode()
def get_map_paths() -> List[Path]:
"""Find all pbf/dump files in the map directory."""
def get_pmtiles_files() -> List[dict]:
"""Find all .pmtiles files in the map directory and return their metadata."""
map_directory = get_map_directory()
paths = walk(map_directory)
def is_valid(path: Path) -> bool:
if path.is_file():
if str(path).endswith('.osm.pbf') and is_pbf_file(path):
return True
elif path.suffix == '.dump' and is_dump_file(path):
return True
return False
files = []
for path in sorted(paths):
if path.is_file() and path.suffix == '.pmtiles':
stat = path.stat()
files.append(dict(
name=path.name,
path=str(path.relative_to(get_media_directory())),
size=stat.st_size,
mtime=stat.st_mtime,
))
return list(filter(is_valid, paths))
return files
def get_or_create_map_file(pbf_path: Path, session: Session) -> MapFile:
"""Finds the MapFile row in the DB, or creates one."""
map_file: MapFile = session.query(MapFile).filter_by(path=str(pbf_path)).one_or_none()
if map_file:
return map_file
def delete_pmtiles_file(filename: str) -> bool:
"""Delete a PMTiles file from the map directory. Returns True if the file was deleted."""
map_directory = get_map_directory()
# Prevent path traversal.
path = (map_directory / filename).resolve()
if not path.is_relative_to(map_directory.resolve()):
raise ValueError(f'Invalid filename: {filename}')
map_file = MapFile(path=pbf_path, size=pbf_path.stat().st_size)
session.add(map_file)
return map_file
if path.is_file() and path.suffix == '.pmtiles':
path.unlink()
logger.warning(f'Deleted PMTiles file: {path}')
return True
return False
import_logger = logger.getChild('import')
def get_map_catalog() -> List[dict]:
"""Return the list of available map regions."""
return MAP_REGIONS.copy()
async def import_files(paths: List[str]):
if flags.map_importing.is_set():
import_logger.error('Map import already running...')
def _get_map_download(session: Session) -> Download | None:
"""Find the single recurring map download, if it exists."""
return session.query(Download).filter_by(url=MANIFEST_URL, downloader='map_catalog').one_or_none()
def _get_or_create_map_download(session: Session) -> Download:
"""Get or create the single recurring map download."""
download = _get_map_download(session)
if download:
return download
download = download_manager.get_or_create_download(session, MANIFEST_URL, reset_attempts=True,
override_skip=True)
download.downloader = 'map_catalog'
download.sub_downloader = 'map_extract'
download.frequency = DownloadFrequency.days180
download.settings = {'regions': []}
download.location = '/map/manage'
download.attempts = 0
session.flush([download])
return download
def get_map_subscriptions(session: Session) -> List[dict]:
"""Return the list of subscribed regions."""
regions = []
download = _get_map_download(session)
if download and download.settings:
regions.extend(download.settings.get('regions', []))
return regions
async def subscribe(session: Session, name: str, region: str):
"""Subscribe to a map region."""
if name not in MAP_REGIONS_BY_NAME:
raise ValueError(f'{name!r} is not a valid map region name')
region_info = MAP_REGIONS_BY_NAME[name]
if region != region_info['region']:
raise ValueError(f'Region {region!r} does not match name {name!r}')
download = _get_or_create_map_download(session)
regions = list(download.settings.get('regions', []))
# Don't add duplicates.
if any(r['region'] == region for r in regions):
return
with flags.map_importing:
paths = [get_media_directory() / i for i in paths]
import_logger.warning(f'Importing: {", ".join(map(str, paths))}')
dumps = [i for i in paths if i.suffix == '.dump']
pbfs = [i for i in paths if i.suffix == '.pbf']
start = now()
any_success = False
try:
if pbfs:
with get_db_session(commit=True) as session:
# Any previously imported PBFs are no longer imported.
for pbf_file in session.query(MapFile):
pbf_file.imported = False
success = False
try:
api_app.shared_ctx.map_importing.update(dict(
pending=list(pbfs),
))
await run_import_command(*pbfs)
success = True
any_success = True
except Exception as e:
import_logger.warning('Failed to run import', exc_info=e)
finally:
api_app.shared_ctx.map_importing.update(dict(
pending=None,
))
if success:
with get_db_session(commit=True) as session:
# Any previously imported PBFs are no longer imported.
for pbf_path in pbfs:
pbf_file = get_or_create_map_file(pbf_path, session)
pbf_file.imported = True
for path in dumps:
# Import each dump individually.
if not path.is_file():
import_logger.fatal(f'Map file does not exist! {path}')
continue
with get_db_session() as session:
map_file = session.query(MapFile).filter_by(path=path).one_or_none()
if map_file and map_file.imported:
# Don't import a map file twice.
import_logger.debug(f'{path} is already imported')
continue
success = False
try:
api_app.shared_ctx.map_importing.update(dict(
pending=str(path),
))
await run_import_command(path)
success = True
any_success = True
except Exception as e:
import_logger.warning('Failed to run import', exc_info=e)
finally:
api_app.shared_ctx.map_importing.update(dict(
pending=None,
))
if success:
with get_db_session(commit=True) as session:
map_file = get_or_create_map_file(path, session)
map_file.imported = True
finally:
elapsed = (now() - start).total_seconds()
if any_success:
Events.send_map_import_complete(f'Map import completed; took {seconds_to_timestamp(elapsed)}')
else:
Events.send_map_import_failed(
f'Map import failed; took {seconds_to_timestamp(elapsed)} See server logs.')
async def run_import_command(*paths: Path):
"""Run the map import script on the provided paths.
Can only import a single *.dump file, or a list of *.osm.pbf files. They cannot be mixed.
"""
paths = [i.absolute() for i in paths]
dumps = [i for i in paths if i.suffix == '.dump']
pbfs = [i for i in paths if i.suffix == '.pbf']
# Only import a single dump, or a list of pbfs. No other combinations acceptable.
if not paths:
raise ValueError('Must import a file')
if not dumps and not pbfs:
raise ValueError('Cannot import unknown file!')
if dumps and pbfs:
raise ValueError('Cannot import mixed dumps and pbfs.')
if len(dumps) > 1:
raise ValueError('Cannot import more than one dump')
if dumps and not is_dump_file(dumps[0]):
import_logger.warning(f'Could not import non-dump file: {dumps}')
raise ValueError('Invalid dump file')
if pbfs:
for path in pbfs:
if not is_pbf_file(path):
import_logger.warning(f'Could not import non-pbf file: {path}')
raise ValueError('Invalid PBF file')
paths = [str(i) for i in paths]
# Run with sudo so renderd can be restarted.
cmd = (SUDO_BIN, f'{PROJECT_DIR}/scripts/import_map.sh', *paths)
result = await run_command(
cmd,
timeout=0, # No timeout, map importing could take days.
)
success = 'Successful' if result.return_code == 0 else 'Unsuccessful'
import_logger.info(
f'{success} import of {repr(paths)} took {timedelta_to_timestamp(timedelta(seconds=result.elapsed))}')
if result.return_code != 0:
# Log all lines. Truncate long lines.
for line in result.stdout.decode().splitlines():
import_logger.debug(line[:500])
for line in result.stderr.decode().splitlines():
import_logger.error(line[:500])
raise ValueError(f'Importing map file failed with return code {result.return_code}')
def get_import_status(session: Session) -> List[MapFile]:
paths = get_map_paths()
map_paths = []
for path in paths:
map_file = get_or_create_map_file(path, session)
map_paths.append(map_file)
entry = {'region': region}
if region_info.get('terrain'):
entry['terrain'] = True
regions.append(entry)
download.settings = {**download.settings, 'regions': regions}
download.renew(reset_attempts=True)
session.commit()
return map_paths
save_downloads_config.activate_switch()
# Calculated using many tests on a well-cooled RPi4 (4GB).
RPI4_PBF_BYTES_PER_SECOND = 175473
RPI4_COEFFICIENTS = [3.25099914e-15, 8.88717664e-06, -3.59754030e+02]
# Calculated using many tests on a well-cooled RPi5 (8GB).
RPI5_PBF_BYTES_PER_SECOND = 136003
RPI5_COEFFICIENTS = [1.66813827e-15, 2.35907825e-06, 5.37276181e+01]
async def unsubscribe(session: Session, region: str):
"""Remove a region from the subscriptions."""
download = _get_map_download(session)
if not download:
raise ValueError(f'No map subscriptions exist')
regions = [r for r in download.settings.get('regions', []) if r['region'] != region]
if regions:
download.settings = {**download.settings, 'regions': regions}
else:
# No more subscriptions — delete the download.
session.delete(download)
session.commit()
save_downloads_config.activate_switch()
def seconds_to_import_rpi4(size_in_bytes: int) -> int:
if size_in_bytes > 1_000_000_000:
# Use exponential curve for large files.
a, b, c = RPI4_COEFFICIENTS
return int(a * size_in_bytes ** 2 + b * size_in_bytes + c)
# Use simpler equation for small files.
return max(int(size_in_bytes // RPI4_PBF_BYTES_PER_SECOND), 0)
async def fetch_manifest(url: str = None) -> dict:
"""Fetch the map manifest JSON from the CDN and verify its GPG signature."""
url = url or MANIFEST_URL
sig_url = f'{url}.sig'
# Fetch manifest.
async with aiohttp_get(url, timeout=30) as response:
if response.status != 200:
raise RuntimeError(f'Failed to fetch manifest: HTTP {response.status}')
manifest_bytes = await response.content.read()
def seconds_to_import_rpi5(size_in_bytes: int) -> int:
if size_in_bytes > 1_000_000_000:
# Use exponential curve for large files.
a, b, c = RPI5_COEFFICIENTS
return int(a * size_in_bytes ** 2 + b * size_in_bytes + c)
return max(int(size_in_bytes // RPI5_PBF_BYTES_PER_SECOND), 0)
# Fetch detached signature.
try:
async with aiohttp_get(sig_url, timeout=30) as response:
if response.status != 200:
raise RuntimeError(f'Failed to fetch manifest signature: HTTP {response.status}')
signature_bytes = await response.content.read()
except Exception as e:
logger.warning(f'Could not fetch manifest signature from {sig_url}: {e}')
raise RuntimeError(f'Manifest signature not available: {e}')
# Write to temp files and verify signature against on-disk files.
with tempfile.TemporaryDirectory() as tmpdir:
manifest_path = Path(tmpdir) / 'manifest.json'
sig_path = Path(tmpdir) / 'manifest.json.sig'
manifest_path.write_bytes(manifest_bytes)
sig_path.write_bytes(signature_bytes)
def seconds_to_import(size_in_bytes: int, is_rpi_5: bool = IS_RPI5) -> int:
"""Attempt to predict how long it will take an RPi4 to import a given PBF file."""
if is_rpi_5:
return seconds_to_import_rpi5(size_in_bytes)
if not await verify_gpg_signature(manifest_path, sig_path):
raise RuntimeError('Manifest GPG signature verification failed — file may be tampered with')
# Default to RPi4, because it's the most conservative estimate.
return seconds_to_import_rpi4(size_in_bytes)
logger.info('Manifest GPG signature verified successfully')
return json.loads(manifest_bytes)

30
modules/map/models.py
View file

@ -1,28 +1,2 @@
from sqlalchemy import Column, Integer, Boolean, BigInteger
from modules.map import lib
from wrolpi import dates
from wrolpi.common import Base, ModelHelper
from wrolpi.media_path import MediaPathType
class MapFile(Base, ModelHelper):
__tablename__ = 'map_file'
id = Column(Integer, primary_key=True)
path = Column(MediaPathType, nullable=False)
imported = Column(Boolean, default=False)
size = Column(BigInteger)
def __repr__(self):
return f'<MapFile {self.path} imported={self.imported}>'
def __json__(self) -> dict:
return dict(
id=self.id,
imported=self.imported,
path=self.path,
seconds_to_import=(seconds := lib.seconds_to_import(self.size)),
size=self.size,
time_to_import=dates.seconds_to_timestamp(seconds),
)
# MapSubscription model has been removed.
# Map subscriptions are managed via settings.regions on a single Download record.

100
modules/map/pins.py Normal file
View file

@ -0,0 +1,100 @@
from dataclasses import dataclass, field
from typing import List
from wrolpi.common import ConfigFile, logger
from wrolpi.dates import now
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
logger = logger.getChild(__name__)
@dataclass
class MapPinsConfigValidator:
version: int = None
pins: list = field(default_factory=list)
class MapPinsConfig(ConfigFile):
file_name = 'map_pins.yaml'
default_config = dict(
version=0,
pins=[],
)
validator = MapPinsConfigValidator
def import_config(self, file=None, send_events=False):
super().import_config(file, send_events)
self.successful_import = True
@property
def pins(self) -> List[dict]:
return list(self._config.get('pins', []))
@pins.setter
def pins(self, value: List[dict]):
self.update({'pins': value})
def _next_pin_id(self) -> int:
"""Return the next available pin ID (max existing ID + 1)."""
pins = self.pins
if not pins:
return 1
return max(p.get('id', 0) for p in pins) + 1
def add_pin(self, lat: float, lon: float, label: str, color: str = 'red') -> dict:
pin = dict(
id=self._next_pin_id(),
lat=round(lat, 6),
lon=round(lon, 6),
label=label,
color=color,
created=str(now().date()),
)
pins = self.pins
pins.append(pin)
self.pins = pins
save_map_pins_config.activate_switch()
return pin
def delete_pin(self, pin_id: int) -> bool:
pins = self.pins
new_pins = [p for p in pins if p.get('id') != pin_id]
if len(new_pins) == len(pins):
return False
self.pins = new_pins
save_map_pins_config.activate_switch()
return True
def update_pin(self, pin_id: int, label: str = None, color: str = None) -> bool:
pins = self.pins
for pin in pins:
if pin.get('id') == pin_id:
if label is not None:
pin['label'] = label
if color is not None:
pin['color'] = color
self.pins = pins
save_map_pins_config.activate_switch()
return True
return False
MAP_PINS_CONFIG: MapPinsConfig = MapPinsConfig()
# Test override.
TEST_MAP_PINS_CONFIG = None
def get_map_pins_config() -> MapPinsConfig:
global TEST_MAP_PINS_CONFIG
if isinstance(TEST_MAP_PINS_CONFIG, ConfigFile):
return TEST_MAP_PINS_CONFIG
return MAP_PINS_CONFIG
@register_switch_handler('save_map_pins_config')
def save_map_pins_config():
get_map_pins_config().save()
save_map_pins_config: ActivateSwitchMethod

20
modules/map/schema.py
View file

@ -1,7 +1,21 @@
from dataclasses import dataclass
from typing import List
@dataclass
class ImportPost:
files: List[str]
class MapSubscribeRequest:
name: str
region: str
@dataclass
class MapPinRequest:
lat: float
lon: float
label: str
color: str = "red"
@dataclass
class MapPinUpdateRequest:
label: str = None
color: str = None

93
modules/map/static/fonts/OFL.txt Normal file
View file

@ -0,0 +1,93 @@
Copyright 2022 The Noto Project Authors (https://github.com/notofonts)
This Font Software is licensed under the SIL Open Font License, Version 1.1.
This license is copied below, and is also available with a FAQ at:
https://scripts.sil.org/OFL
-----------------------------------------------------------
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
-----------------------------------------------------------
PREAMBLE
The goals of the Open Font License (OFL) are to stimulate worldwide
development of collaborative font projects, to support the font creation
efforts of academic and linguistic communities, and to provide a free and
open framework in which fonts may be shared and improved in partnership
with others.
The OFL allows the licensed fonts to be used, studied, modified and
redistributed freely as long as they are not sold by themselves. The
fonts, including any derivative works, can be bundled, embedded,
redistributed and/or sold with any software provided that any reserved
names are not used by derivative works. The fonts and derivatives,
however, cannot be released under any other type of license. The
requirement for fonts to remain under this license does not apply
to any document created using the fonts or their derivatives.
DEFINITIONS
"Font Software" refers to the set of files released by the Copyright
Holder(s) under this license and clearly marked as such. This may
include source files, build scripts and documentation.
"Reserved Font Name" refers to any names specified as such after the
copyright statement(s).
"Original Version" refers to the collection of Font Software components as
distributed by the Copyright Holder(s).
"Modified Version" refers to any derivative made by adding to, deleting,
or substituting -- in part or in whole -- any of the components of the
Original Version, by changing formats or by porting the Font Software to a
new environment.
"Author" refers to any designer, engineer, programmer, technical
writer or other person who contributed to the Font Software.
PERMISSION & CONDITIONS
Permission is hereby granted, free of charge, to any person obtaining
a copy of the Font Software, to use, study, copy, merge, embed, modify,
redistribute, and sell modified and unmodified copies of the Font
Software, subject to the following conditions:
1) Neither the Font Software nor any of its individual components,
in Original or Modified Versions, may be sold by itself.
2) Original or Modified Versions of the Font Software may be bundled,
redistributed and/or sold with any software, provided that each copy
contains the above copyright notice and this license. These can be
included either as stand-alone text files, human-readable headers or
in the appropriate machine-readable metadata fields within text or
binary files as long as those fields can be easily viewed by the user.
3) No Modified Version of the Font Software may use the Reserved Font
Name(s) unless explicit written permission is granted by the corresponding
Copyright Holder. This restriction only applies to the primary font name as
presented to the users.
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
Software shall not be used to promote, endorse or advertise any
Modified Version, except to acknowledge the contribution(s) of the
Copyright Holder(s) and the Author(s) or with their explicit written
permission.
5) The Font Software, modified or unmodified, in part or in whole,
must be distributed entirely under this license, and must not be
distributed under any other license. The requirement for fonts to
remain under this license does not apply to any document created
using the Font Software.
TERMINATION
This license becomes null and void if any of the above conditions are
not met.
DISCLAIMER
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
OTHER DEALINGS IN THE FONT SOFTWARE.

739
modules/map/static/index.html Normal file
View file

@ -0,0 +1,739 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
<title>WROLPi Map</title>
<link rel="stylesheet" href="/libs/maplibre-gl.css"/>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
html, body { height: 100%; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; }
#map { width: 100%; height: 100%; }
/* Layer control panel */
#layer-panel {
position: absolute;
top: 10px;
left: 10px;
background: white;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0,0,0,0.2);
z-index: 1000;
max-height: calc(100vh - 20px);
overflow-y: auto;
min-width: 220px;
font-size: 14px;
}
#layer-toggle {
padding: 10px 14px;
cursor: pointer;
font-weight: 600;
display: flex;
justify-content: space-between;
align-items: center;
border-bottom: 1px solid #eee;
user-select: none;
}
#layer-toggle:hover { background: #f5f5f5; }
#layer-toggle .arrow { transition: transform 0.2s; }
#layer-toggle .arrow.collapsed { transform: rotate(-90deg); }
#layer-content { padding: 8px 0; }
#layer-content.hidden { display: none; }
.layer-group { padding: 4px 14px; }
.layer-group-header {
display: flex;
align-items: center;
gap: 6px;
padding: 4px 0;
cursor: pointer;
font-weight: 500;
user-select: none;
}
.layer-group-header:hover { color: #1a73e8; }
.layer-group-header input { cursor: pointer; width: 16px; height: 16px; }
/* File selector */
#file-panel {
position: absolute;
top: 10px;
right: 50px;
background: white;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0,0,0,0.2);
z-index: 1000;
padding: 10px 14px;
font-size: 14px;
max-width: 300px;
}
#file-panel h3 { margin-bottom: 8px; font-size: 14px; }
.file-item {
display: flex;
align-items: center;
gap: 6px;
padding: 3px 0;
}
.file-item input { cursor: pointer; width: 16px; height: 16px; }
.file-item label { cursor: pointer; word-break: break-all; }
#no-files-msg {
color: #888;
font-style: italic;
padding: 4px 0;
}
/* Status bar */
#status {
position: absolute;
bottom: 30px;
left: 50%;
transform: translateX(-50%);
background: rgba(0,0,0,0.75);
color: white;
padding: 8px 16px;
border-radius: 6px;
font-size: 13px;
z-index: 1000;
display: none;
}
/* Context menu */
#context-menu {
position: absolute;
z-index: 1001;
background: white;
border-radius: 6px;
box-shadow: 0 2px 8px rgba(0,0,0,0.3);
padding: 4px 0;
font-size: 13px;
min-width: 180;
display: none;
}
#context-menu .coords {
padding: 6px 12px;
color: #666;
font-size: 12px;
border-bottom: 1px solid #eee;
}
#context-menu .action {
padding: 8px 12px;
cursor: pointer;
user-select: none;
}
#context-menu .action:hover { background: #f0f0f0; }
</style>
</head>
<body>
<div id="map"></div>
<div id="layer-panel">
<div id="layer-toggle">
<span>Layers</span>
<span class="arrow">&#9660;</span>
</div>
<div id="layer-content"></div>
</div>
<div id="file-panel">
<h3>Map Files</h3>
<div id="file-list">
<div id="no-files-msg">Scanning...</div>
</div>
</div>
<div id="status"></div>
<div id="context-menu">
<div class="coords" id="context-coords"></div>
<div class="action" id="context-copy">Copy GPS coordinates</div>
<div class="action" id="context-export">Export as PNG</div>
</div>
<script src="/libs/pmtiles.js"></script>
<script src="/libs/maplibre-gl.js"></script>
<script src="/libs/protomaps-themes-base.js"></script>
<script src="/libs/maplibre-contour-pmtiles.js"></script>
<script>
(function() {
// --- Constants ---
var MAP_OVERVIEW_URL = "/blobs/map-overview.pmtiles";
var TERRAIN_PREFIX = "terrain-";
function isPlanetFile(name) {
if (name.indexOf("planet-") === 0) return true;
return /^\d{8}\.pmtiles$/.test(name);
}
var terrainFileName = null;
var STORAGE_KEY_LAYERS = "wrolpi-map-layers";
var STORAGE_KEY_SCALE = "wrolpi-map-scale";
var DEFAULT_HIDDEN = {"Hillshade": true, "Contours": true};
// --- PMTiles protocol ---
var protocol = new pmtiles.Protocol();
maplibregl.addProtocol("pmtiles", protocol.tile);
// --- Layer group definitions ---
var LAYER_GROUPS = {
"Hillshade": ["hillshade"],
"Contours": ["contour-lines", "contour-labels"],
"Water": ["water"],
"Land Cover": [
"landcover", "landuse_park", "landuse_urban_green", "landuse_beach",
"landuse_aerodrome", "landuse_hospital", "landuse_industrial",
"landuse_pedestrian", "landuse_pier", "landuse_school", "landuse_zoo"
],
"Buildings": ["buildings"],
"Roads - Highway": ["roads_highway"],
"Roads - Major": ["roads_major", "roads_link"],
"Roads - Minor": ["roads_minor", "roads_other"],
"Roads - Tunnels": ["roads_tunnels"],
"Roads - Bridges": ["roads_bridges"],
"Rail & Runways": ["roads_rail", "roads_runway", "roads_taxiway", "landuse_runway"],
"Boundaries": ["boundaries"],
"Labels": ["roads_labels", "places_", "address_label", "pois", "water_label", "water_waterway_label"],
};
// --- State ---
var map;
var scaleControl;
var demSource = null;
var activeSources = {};
var hasPlanetOverview = false;
var hasTerrain = false;
var contextMenuData = null;
// Load saved state from localStorage.
function getSavedVisibility() {
try {
var saved = localStorage.getItem(STORAGE_KEY_LAYERS);
if (saved) return JSON.parse(saved);
} catch(e) {}
var v = {};
for (var g in LAYER_GROUPS) v[g] = !DEFAULT_HIDDEN[g];
return v;
}
function getSavedScale() {
try { return localStorage.getItem(STORAGE_KEY_SCALE) || "metric"; } catch(e) { return "metric"; }
}
var groupVisibility = getSavedVisibility();
var currentScale = getSavedScale();
function getMapFlavor() {
return window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light";
}
// --- Build style ---
function buildStyle() {
var flavor = getMapFlavor();
var skipSecondary = {"background": true, "earth": true};
var sources = {};
var allLayers = [];
var si = 0;
// Planet overview as the primary base layer.
if (hasPlanetOverview) {
sources["map-overview"] = {
type: "vector",
url: "pmtiles://" + MAP_OVERVIEW_URL,
};
var baseLayers = protomaps_themes_base.default("map-overview", flavor);
allLayers = allLayers.concat(baseLayers);
si = 1;
}
// User files as additional sources.
var sourceNames = Object.keys(activeSources);
for (var fi = 0; fi < sourceNames.length; fi++) {
var filename = sourceNames[fi];
sources[filename] = {
type: "vector",
url: "pmtiles:///data/" + filename,
};
var fileLayers = protomaps_themes_base.default(filename, flavor);
if (si === 0 && fi === 0) {
allLayers = allLayers.concat(fileLayers);
} else {
for (var li = 0; li < fileLayers.length; li++) {
if (skipSecondary[fileLayers[li].id]) continue;
var layer = Object.assign({}, fileLayers[li]);
layer.id = filename + ":" + layer.id;
allLayers.push(layer);
}
}
}
// Terrain (hillshade + contours).
if (hasTerrain && demSource) {
var savedVis = getSavedVisibility();
var hillshadeVis = savedVis["Hillshade"] ? "visible" : "none";
var contourVis = savedVis["Contours"] ? "visible" : "none";
var isImperial = currentScale === "imperial";
var elevUnit = isImperial ? "ft" : "m";
sources["terrain-dem"] = {
type: "raster-dem",
url: "pmtiles://" + window.location.origin + "/data/" + terrainFileName,
encoding: "terrarium",
tileSize: 512,
};
var hillshadeSearch = allLayers.findIndex(function(l) { return l.id === "buildings"; });
var hillshadeIdx = hillshadeSearch === -1 ? allLayers.length : hillshadeSearch;
allLayers.splice(hillshadeIdx, 0, {
id: "hillshade",
type: "hillshade",
source: "terrain-dem",
layout: {visibility: hillshadeVis},
paint: {
"hillshade-shadow-color": "#000000",
"hillshade-illumination-anchor": "viewport",
"hillshade-exaggeration": 0.3,
},
});
var contourUrl = demSource.contourProtocolUrl({
overzoom: 1,
contourLayer: "contours",
elevationKey: "ele",
levelKey: "level",
multiplier: isImperial ? 3.28084 : 1,
thresholds: isImperial ? {
4: [1000, 5000], 5: [500, 2500], 6: [200, 1000], 7: [200, 1000], 8: [100, 500],
} : {
4: [500, 2500], 5: [200, 1000], 6: [100, 500], 7: [100, 500], 8: [50, 200],
},
});
sources["contours"] = { type: "vector", tiles: [contourUrl], maxzoom: 8 };
var textColor = flavor === "dark" ? "rgba(210,180,140,0.9)" : "rgba(150,100,50,0.8)";
var haloColor = flavor === "dark" ? "rgba(0,0,0,0.8)" : "rgba(255,255,255,0.8)";
allLayers.push(
{
id: "contour-lines", type: "line", source: "contours", "source-layer": "contours",
filter: ["!=", ["get", "level"], 1],
layout: {visibility: contourVis},
paint: {"line-color": "rgba(150,100,50,0.3)", "line-width": 0.5},
},
{
id: "contour-lines-major", type: "line", source: "contours", "source-layer": "contours",
filter: ["==", ["get", "level"], 1],
layout: {visibility: contourVis},
paint: {"line-color": "rgba(150,100,50,0.5)", "line-width": 1},
},
{
id: "contour-labels", type: "symbol", source: "contours", "source-layer": "contours",
filter: ["==", ["get", "level"], 1],
layout: {
visibility: contourVis,
"symbol-placement": "line",
"text-field": ["concat", ["get", "ele"], elevUnit],
"text-size": 10,
},
paint: {"text-color": textColor, "text-halo-color": haloColor, "text-halo-width": 1},
}
);
}
return {
version: 8,
glyphs: "/fonts/{fontstack}/{range}.pbf",
sprite: window.location.origin + "/sprites/" + flavor,
sources: sources,
layers: allLayers,
};
}
// --- Parse URL hash for coordinates ---
function parseHash() {
var hash = window.location.hash.replace("#", "");
if (!hash) return null;
var parts = hash.split(",");
if (parts.length >= 2) {
var lat = parseFloat(parts[0]);
var lon = parseFloat(parts[1]);
var zoom = parts.length >= 3 ? parseFloat(parts[2]) : 10;
if (!isNaN(lat) && !isNaN(lon) && lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180) {
return {lat: lat, lon: lon, zoom: zoom};
}
}
return null;
}
function initMap() {
var initial = parseHash();
var center = initial ? [initial.lon, initial.lat] : [0, 0];
var zoom = initial ? initial.zoom : 2;
map = new maplibregl.Map({
container: "map",
style: buildStyle(),
center: center,
zoom: zoom,
attributionControl: true,
preserveDrawingBuffer: true,
});
map.addControl(new maplibregl.NavigationControl(), "top-right");
map.addControl(new maplibregl.FullscreenControl(), "top-right");
scaleControl = new maplibregl.ScaleControl({maxWidth: 200, unit: currentScale});
map.addControl(scaleControl, "bottom-left");
var geolocate = new maplibregl.GeolocateControl({
positionOptions: {enableHighAccuracy: true},
trackUserLocation: true,
showUserHeading: true,
showAccuracyCircle: true,
});
map.addControl(geolocate, "top-right");
// Update URL hash on pan/zoom.
map.on("moveend", function() {
var c = map.getCenter();
var z = map.getZoom();
window.location.hash = c.lat.toFixed(4) + "," + c.lng.toFixed(4) + "," + z.toFixed(1);
});
// Right-click context menu.
map.on("contextmenu", function(e) {
contextMenuData = {lat: e.lngLat.lat, lon: e.lngLat.lng};
var menu = document.getElementById("context-menu");
var z = map.getZoom();
var d = z < 4 ? 1 : z < 8 ? 2 : z < 12 ? 3 : 4;
document.getElementById("context-coords").textContent =
contextMenuData.lat.toFixed(d) + ", " + contextMenuData.lon.toFixed(d);
menu.style.left = e.point.x + "px";
menu.style.top = e.point.y + "px";
menu.style.display = "block";
});
map.on("click", function() { document.getElementById("context-menu").style.display = "none"; });
map.on("movestart", function() { document.getElementById("context-menu").style.display = "none"; });
// Long-press context menu (mobile).
var longPressTimer = null;
var touchStartPos = null;
var canvas = map.getCanvas();
canvas.addEventListener("touchstart", function(e) {
if (e.touches.length !== 1) return;
touchStartPos = {x: e.touches[0].clientX, y: e.touches[0].clientY};
longPressTimer = setTimeout(function() {
var rect = canvas.getBoundingClientRect();
var px = touchStartPos.x - rect.left;
var py = touchStartPos.y - rect.top;
var lngLat = map.unproject([px, py]);
contextMenuData = {lat: lngLat.lat, lon: lngLat.lng};
var menu = document.getElementById("context-menu");
var z = map.getZoom();
var d = z < 4 ? 1 : z < 8 ? 2 : z < 12 ? 3 : 4;
document.getElementById("context-coords").textContent =
contextMenuData.lat.toFixed(d) + ", " + contextMenuData.lon.toFixed(d);
menu.style.left = px + "px";
menu.style.top = py + "px";
menu.style.display = "block";
}, 500);
}, {passive: true});
canvas.addEventListener("touchmove", function(e) {
if (longPressTimer && touchStartPos) {
var dx = e.touches[0].clientX - touchStartPos.x;
var dy = e.touches[0].clientY - touchStartPos.y;
if (Math.abs(dx) > 10 || Math.abs(dy) > 10) {
clearTimeout(longPressTimer);
longPressTimer = null;
}
}
});
canvas.addEventListener("touchend", function() {
clearTimeout(longPressTimer);
longPressTimer = null;
});
map.on("load", function() {
buildLayerControls();
});
}
// --- Context menu copy ---
document.getElementById("context-copy").addEventListener("click", function() {
if (!contextMenuData || !map) return;
var z = map.getZoom();
var d = z < 4 ? 1 : z < 8 ? 2 : z < 12 ? 3 : 4;
var text = contextMenuData.lat.toFixed(d) + ", " + contextMenuData.lon.toFixed(d);
if (navigator.clipboard) {
navigator.clipboard.writeText(text).catch(function() {
window.prompt("Copy coordinates:", text);
});
} else {
window.prompt("Copy coordinates:", text);
}
document.getElementById("context-menu").style.display = "none";
});
// --- Context menu export ---
document.getElementById("context-export").addEventListener("click", function() {
if (!map) return;
document.getElementById("context-menu").style.display = "none";
// Force a fresh render, then capture.
map.once("render", function() {
var c = map.getCenter();
var z = map.getZoom();
var filename = "wrolpi-map_" + c.lat.toFixed(4) + "_" + c.lng.toFixed(4) + "_z" + z.toFixed(0) + ".png";
map.getCanvas().toBlob(function(blob) {
if (!blob) return;
var url = URL.createObjectURL(blob);
var a = document.createElement("a");
a.href = url;
a.download = filename;
a.click();
URL.revokeObjectURL(url);
});
});
map.triggerRepaint();
});
// --- Layer control ---
function matchesGroup(layerId, prefixes) {
var baseId = layerId.indexOf(":") >= 0 ? layerId.split(":").slice(1).join(":") : layerId;
return prefixes.some(function(prefix) {
return baseId.indexOf(prefix) >= 0;
});
}
function setGroupVisibility(groupName, visible) {
groupVisibility[groupName] = visible;
try { localStorage.setItem(STORAGE_KEY_LAYERS, JSON.stringify(groupVisibility)); } catch(e) {}
if (!map || !map.getStyle()) return;
var prefixes = LAYER_GROUPS[groupName];
var allLayers = map.getStyle().layers || [];
for (var i = 0; i < allLayers.length; i++) {
if (matchesGroup(allLayers[i].id, prefixes)) {
map.setLayoutProperty(allLayers[i].id, "visibility", visible ? "visible" : "none");
}
}
}
function buildLayerControls() {
var container = document.getElementById("layer-content");
container.innerHTML = "";
for (var groupName in LAYER_GROUPS) {
var div = document.createElement("div");
div.className = "layer-group";
var header = document.createElement("label");
header.className = "layer-group-header";
var checkbox = document.createElement("input");
checkbox.type = "checkbox";
checkbox.checked = groupVisibility[groupName] !== false;
checkbox.addEventListener("change", (function(name) {
return function(e) { setGroupVisibility(name, e.target.checked); };
})(groupName));
var label = document.createTextNode(groupName);
header.appendChild(checkbox);
header.appendChild(label);
div.appendChild(header);
container.appendChild(div);
}
// Scale unit toggle.
var scaleDiv = document.createElement("div");
scaleDiv.className = "layer-group";
scaleDiv.style.borderTop = "1px solid #eee";
scaleDiv.style.marginTop = "4px";
scaleDiv.style.paddingTop = "4px";
var scaleLabel = document.createElement("label");
scaleLabel.className = "layer-group-header";
var scaleCheckbox = document.createElement("input");
scaleCheckbox.type = "checkbox";
scaleCheckbox.checked = currentScale === "imperial";
scaleCheckbox.addEventListener("change", function(e) {
currentScale = e.target.checked ? "imperial" : "metric";
scaleText.textContent = e.target.checked ? "Imperial" : "Metric";
try { localStorage.setItem(STORAGE_KEY_SCALE, currentScale); } catch(e2) {}
if (map && scaleControl) {
map.removeControl(scaleControl);
scaleControl = new maplibregl.ScaleControl({maxWidth: 200, unit: currentScale});
map.addControl(scaleControl, "bottom-left");
}
// Rebuild style for contour unit change.
if (hasTerrain) reloadMap();
});
var scaleText = document.createTextNode(currentScale === "imperial" ? "Imperial" : "Metric");
scaleLabel.appendChild(scaleCheckbox);
scaleLabel.appendChild(scaleText);
scaleDiv.appendChild(scaleLabel);
container.appendChild(scaleDiv);
}
// Toggle layer panel
document.getElementById("layer-toggle").addEventListener("click", function() {
var content = document.getElementById("layer-content");
var arrow = this.querySelector(".arrow");
content.classList.toggle("hidden");
arrow.classList.toggle("collapsed");
});
// --- File management ---
function showStatus(msg) {
var el = document.getElementById("status");
el.textContent = msg;
el.style.display = "block";
setTimeout(function() { el.style.display = "none"; }, 3000);
}
function reloadMap() {
if (!map) { initMap(); return; }
var center = map.getCenter();
var zoom = map.getZoom();
var bearing = map.getBearing();
var pitch = map.getPitch();
map.setStyle(buildStyle());
map.once("style.load", function() {
map.setCenter(center);
map.setZoom(zoom);
map.setBearing(bearing);
map.setPitch(pitch);
// Re-apply visibility for groups that were toggled off.
for (var groupName in groupVisibility) {
if (!groupVisibility[groupName]) {
setGroupVisibility(groupName, false);
}
}
buildLayerControls();
});
}
function toggleFile(filename, enabled) {
if (enabled) { activeSources[filename] = true; }
else { delete activeSources[filename]; }
reloadMap();
showStatus(enabled ? "Added " + filename : "Removed " + filename);
}
function renderFileList(files) {
var container = document.getElementById("file-list");
container.innerHTML = "";
if (files.length === 0) {
container.innerHTML = '<div id="no-files-msg">No .pmtiles files found in map/ directory</div>';
return;
}
files.forEach(function(file) {
var div = document.createElement("div");
div.className = "file-item";
var checkbox = document.createElement("input");
checkbox.type = "checkbox";
checkbox.id = "file-" + file;
checkbox.checked = !!activeSources[file];
checkbox.addEventListener("change", function(e) {
toggleFile(file, e.target.checked);
});
var label = document.createElement("label");
label.htmlFor = "file-" + file;
label.textContent = file;
div.appendChild(checkbox);
div.appendChild(label);
container.appendChild(div);
});
}
function scanFiles() {
// Fetch user's map files first to determine what base layer is needed.
fetch("/api/map/files")
.then(function(r) { return r.json(); })
.then(function(data) {
var allFiles = (data.files || []).sort(function(a, b) { return a.name.localeCompare(b.name); });
// Separate files into planet, terrain, and regional extracts.
var planetFile = null;
var regionFiles = [];
for (var i = 0; i < allFiles.length; i++) {
var f = allFiles[i];
if (f.name.startsWith(TERRAIN_PREFIX) && f.name.endsWith('.pmtiles')) {
terrainFileName = f.name;
hasTerrain = true;
} else if (isPlanetFile(f.name)) {
if (!planetFile || f.size > planetFile.size) {
planetFile = f;
}
} else {
regionFiles.push(f);
}
}
var pmtilesFiles;
if (planetFile) {
// Planet covers everything — sole vector source, no overview needed.
pmtilesFiles = [planetFile.name];
hasPlanetOverview = false;
return pmtilesFiles;
}
// Use regional extracts.
pmtilesFiles = regionFiles.map(function(f) { return f.name; });
// Check for planet overview blob as low-zoom base layer.
return fetch(MAP_OVERVIEW_URL, {method: "HEAD"})
.then(function(r) { hasPlanetOverview = r.ok; })
.catch(function() { hasPlanetOverview = false; })
.then(function() { return pmtilesFiles; });
})
.then(function(pmtilesFiles) {
// Initialize contour source if terrain available.
if (hasTerrain && !demSource) {
demSource = new mlcontour.DemSource({
url: "pmtiles://" + window.location.origin + "/data/" + terrainFileName,
encoding: "terrarium",
maxzoom: 8,
worker: true,
});
demSource.setupMaplibre(maplibregl);
}
renderFileList(pmtilesFiles);
if (pmtilesFiles.length > 0 && Object.keys(activeSources).length === 0) {
activeSources[pmtilesFiles[0]] = true;
renderFileList(pmtilesFiles);
reloadMap();
} else if (hasPlanetOverview || hasTerrain) {
reloadMap();
} else if (Object.keys(activeSources).length === 0) {
initMap();
}
})
.catch(function(err) {
console.error("Failed to scan files:", err);
document.getElementById("file-list").innerHTML =
'<div id="no-files-msg">Error scanning for map files</div>';
initMap();
});
}
// --- Listen for system theme changes ---
window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change", function() {
if (map) reloadMap();
});
// --- Start ---
scanFiles();
})();
</script>
</body>
</html>

1
modules/map/static/libs/maplibre-contour-pmtiles.js Normal file
View file

File diff suppressed because one or more lines are too long

1
modules/map/static/libs/maplibre-gl.css Normal file
View file

File diff suppressed because one or more lines are too long

59
modules/map/static/libs/maplibre-gl.js Normal file
View file

File diff suppressed because one or more lines are too long

2
modules/map/static/libs/pmtiles.js Normal file
View file

File diff suppressed because one or more lines are too long

16
modules/map/static/libs/protomaps-themes-base.js Normal file
View file

File diff suppressed because one or more lines are too long

1
modules/map/static/sprites/dark.json Normal file
View file

@ -0,0 +1 @@
{"NL:S-road-1char":{"height":21,"pixelRatio":1,"width":21,"x":0,"y":36},"NL:S-road-2char":{"height":21,"pixelRatio":1,"width":22,"x":38,"y":21},"NL:S-road-3char":{"height":21,"pixelRatio":1,"width":25,"x":202,"y":18},"NL:S-road-4char":{"height":21,"pixelRatio":1,"width":29,"x":173,"y":0},"NL:S-road-5char":{"height":21,"pixelRatio":1,"width":32,"x":79,"y":0},"US:I-1char":{"height":21,"pixelRatio":1,"width":20,"x":84,"y":21},"US:I-2char":{"height":21,"pixelRatio":1,"width":23,"x":227,"y":18},"US:I-3char":{"height":21,"pixelRatio":1,"width":30,"x":143,"y":0},"US:I-4char":{"height":21,"pixelRatio":1,"width":32,"x":111,"y":0},"US:I-5char":{"height":21,"pixelRatio":1,"width":34,"x":45,"y":0},"aerodrome":{"height":19,"pixelRatio":1,"width":19,"x":60,"y":39},"animal":{"height":19,"pixelRatio":1,"width":19,"x":21,"y":42},"arrow":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":57},"artwork":{"height":19,"pixelRatio":1,"width":19,"x":40,"y":42},"attraction":{"height":19,"pixelRatio":1,"width":19,"x":104,"y":21},"bar":{"height":19,"pixelRatio":1,"width":19,"x":123,"y":21},"beach":{"height":19,"pixelRatio":1,"width":19,"x":142,"y":21},"beauty":{"height":19,"pixelRatio":1,"width":19,"x":161,"y":21},"bench":{"height":19,"pixelRatio":1,"width":19,"x":180,"y":21},"books":{"height":19,"pixelRatio":1,"width":19,"x":199,"y":39},"building":{"height":17,"pixelRatio":1,"width":17,"x":193,"y":77},"bus_stop":{"height":19,"pixelRatio":1,"width":19,"x":218,"y":39},"cafe":{"height":19,"pixelRatio":1,"width":19,"x":237,"y":39},"capital":{"height":10,"pixelRatio":1,"width":10,"x":210,"y":77},"clothes":{"height":19,"pixelRatio":1,"width":19,"x":104,"y":40},"convenience":{"height":19,"pixelRatio":1,"width":19,"x":79,"y":42},"drinking_water":{"height":19,"pixelRatio":1,"width":19,"x":59,"y":58},"electronics":{"height":19,"pixelRatio":1,"width":19,"x":19,"y":61},"fast_food":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":76},"ferry_terminal":{"height":19,"pixelRatio":1,"width":19,"x":38,"y":61},"forest":{"height":19,"pixelRatio":1,"width":19,"x":123,"y":40},"garden":{"height":19,"pixelRatio":1,"width":19,"x":142,"y":40},"generic_shield-1char":{"height":18,"pixelRatio":1,"width":19,"x":235,"y":0},"generic_shield-2char":{"height":18,"pixelRatio":1,"width":24,"x":60,"y":21},"generic_shield-3char":{"height":18,"pixelRatio":1,"width":33,"x":202,"y":0},"generic_shield-4char":{"height":18,"pixelRatio":1,"width":38,"x":0,"y":18},"generic_shield-5char":{"height":18,"pixelRatio":1,"width":45,"x":0,"y":0},"library":{"height":19,"pixelRatio":1,"width":19,"x":161,"y":40},"marina":{"height":19,"pixelRatio":1,"width":19,"x":180,"y":40},"museum":{"height":19,"pixelRatio":1,"width":19,"x":199,"y":58},"park":{"height":19,"pixelRatio":1,"width":19,"x":218,"y":58},"peak":{"height":19,"pixelRatio":1,"width":19,"x":237,"y":58},"post_office":{"height":19,"pixelRatio":1,"width":19,"x":98,"y":59},"restaurant":{"height":19,"pixelRatio":1,"width":19,"x":78,"y":61},"school":{"height":19,"pixelRatio":1,"width":19,"x":57,"y":77},"stadium":{"height":19,"pixelRatio":1,"width":19,"x":19,"y":80},"supermarket":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":95},"theatre":{"height":19,"pixelRatio":1,"width":19,"x":38,"y":80},"toilets":{"height":19,"pixelRatio":1,"width":19,"x":117,"y":59},"townspot":{"height":10,"pixelRatio":1,"width":10,"x":220,"y":77},"train_station":{"height":19,"pixelRatio":1,"width":19,"x":136,"y":59},"university":{"height":19,"pixelRatio":1,"width":19,"x":155,"y":59},"zoo":{"height":19,"pixelRatio":1,"width":19,"x":174,"y":59}}

BIN
modules/map/static/sprites/dark.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 16 KiB

1
modules/map/static/sprites/dark@2x.json Normal file
View file

@ -0,0 +1 @@
{"NL:S-road-1char":{"height":42,"pixelRatio":2,"width":42,"x":0,"y":72},"NL:S-road-2char":{"height":42,"pixelRatio":2,"width":44,"x":76,"y":42},"NL:S-road-3char":{"height":42,"pixelRatio":2,"width":50,"x":404,"y":36},"NL:S-road-4char":{"height":42,"pixelRatio":2,"width":58,"x":346,"y":0},"NL:S-road-5char":{"height":42,"pixelRatio":2,"width":64,"x":158,"y":0},"US:I-1char":{"height":42,"pixelRatio":2,"width":40,"x":168,"y":42},"US:I-2char":{"height":42,"pixelRatio":2,"width":46,"x":454,"y":36},"US:I-3char":{"height":42,"pixelRatio":2,"width":60,"x":286,"y":0},"US:I-4char":{"height":42,"pixelRatio":2,"width":64,"x":222,"y":0},"US:I-5char":{"height":42,"pixelRatio":2,"width":68,"x":90,"y":0},"aerodrome":{"height":38,"pixelRatio":2,"width":38,"x":120,"y":78},"animal":{"height":38,"pixelRatio":2,"width":38,"x":42,"y":84},"arrow":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":114},"artwork":{"height":38,"pixelRatio":2,"width":38,"x":80,"y":84},"attraction":{"height":38,"pixelRatio":2,"width":38,"x":208,"y":42},"bar":{"height":38,"pixelRatio":2,"width":38,"x":246,"y":42},"beach":{"height":38,"pixelRatio":2,"width":38,"x":284,"y":42},"beauty":{"height":38,"pixelRatio":2,"width":38,"x":322,"y":42},"bench":{"height":38,"pixelRatio":2,"width":38,"x":360,"y":42},"books":{"height":38,"pixelRatio":2,"width":38,"x":398,"y":78},"building":{"height":34,"pixelRatio":2,"width":34,"x":386,"y":154},"bus_stop":{"height":38,"pixelRatio":2,"width":38,"x":436,"y":78},"cafe":{"height":38,"pixelRatio":2,"width":38,"x":474,"y":78},"capital":{"height":20,"pixelRatio":2,"width":20,"x":420,"y":154},"clothes":{"height":38,"pixelRatio":2,"width":38,"x":208,"y":80},"convenience":{"height":38,"pixelRatio":2,"width":38,"x":158,"y":84},"drinking_water":{"height":38,"pixelRatio":2,"width":38,"x":118,"y":116},"electronics":{"height":38,"pixelRatio":2,"width":38,"x":38,"y":122},"fast_food":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":152},"ferry_terminal":{"height":38,"pixelRatio":2,"width":38,"x":76,"y":122},"forest":{"height":38,"pixelRatio":2,"width":38,"x":246,"y":80},"garden":{"height":38,"pixelRatio":2,"width":38,"x":284,"y":80},"generic_shield-1char":{"height":36,"pixelRatio":2,"width":38,"x":470,"y":0},"generic_shield-2char":{"height":36,"pixelRatio":2,"width":48,"x":120,"y":42},"generic_shield-3char":{"height":36,"pixelRatio":2,"width":66,"x":404,"y":0},"generic_shield-4char":{"height":36,"pixelRatio":2,"width":76,"x":0,"y":36},"generic_shield-5char":{"height":36,"pixelRatio":2,"width":90,"x":0,"y":0},"library":{"height":38,"pixelRatio":2,"width":38,"x":322,"y":80},"marina":{"height":38,"pixelRatio":2,"width":38,"x":360,"y":80},"museum":{"height":38,"pixelRatio":2,"width":38,"x":398,"y":116},"park":{"height":38,"pixelRatio":2,"width":38,"x":436,"y":116},"peak":{"height":38,"pixelRatio":2,"width":38,"x":474,"y":116},"post_office":{"height":38,"pixelRatio":2,"width":38,"x":196,"y":118},"restaurant":{"height":38,"pixelRatio":2,"width":38,"x":156,"y":122},"school":{"height":38,"pixelRatio":2,"width":38,"x":114,"y":154},"stadium":{"height":38,"pixelRatio":2,"width":38,"x":38,"y":160},"supermarket":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":190},"theatre":{"height":38,"pixelRatio":2,"width":38,"x":76,"y":160},"toilets":{"height":38,"pixelRatio":2,"width":38,"x":234,"y":118},"townspot":{"height":20,"pixelRatio":2,"width":20,"x":440,"y":154},"train_station":{"height":38,"pixelRatio":2,"width":38,"x":272,"y":118},"university":{"height":38,"pixelRatio":2,"width":38,"x":310,"y":118},"zoo":{"height":38,"pixelRatio":2,"width":38,"x":348,"y":118}}

BIN
modules/map/static/sprites/dark@2x.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 28 KiB

1
modules/map/static/sprites/light.json Normal file
View file

@ -0,0 +1 @@
{"NL:S-road-1char":{"height":21,"pixelRatio":1,"width":21,"x":0,"y":36},"NL:S-road-2char":{"height":21,"pixelRatio":1,"width":22,"x":38,"y":21},"NL:S-road-3char":{"height":21,"pixelRatio":1,"width":25,"x":202,"y":18},"NL:S-road-4char":{"height":21,"pixelRatio":1,"width":29,"x":173,"y":0},"NL:S-road-5char":{"height":21,"pixelRatio":1,"width":32,"x":79,"y":0},"US:I-1char":{"height":21,"pixelRatio":1,"width":20,"x":84,"y":21},"US:I-2char":{"height":21,"pixelRatio":1,"width":23,"x":227,"y":18},"US:I-3char":{"height":21,"pixelRatio":1,"width":30,"x":143,"y":0},"US:I-4char":{"height":21,"pixelRatio":1,"width":32,"x":111,"y":0},"US:I-5char":{"height":21,"pixelRatio":1,"width":34,"x":45,"y":0},"aerodrome":{"height":19,"pixelRatio":1,"width":19,"x":60,"y":39},"animal":{"height":19,"pixelRatio":1,"width":19,"x":21,"y":42},"arrow":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":57},"artwork":{"height":19,"pixelRatio":1,"width":19,"x":40,"y":42},"attraction":{"height":19,"pixelRatio":1,"width":19,"x":104,"y":21},"bar":{"height":19,"pixelRatio":1,"width":19,"x":123,"y":21},"beach":{"height":19,"pixelRatio":1,"width":19,"x":142,"y":21},"beauty":{"height":19,"pixelRatio":1,"width":19,"x":161,"y":21},"bench":{"height":19,"pixelRatio":1,"width":19,"x":180,"y":21},"books":{"height":19,"pixelRatio":1,"width":19,"x":199,"y":39},"building":{"height":17,"pixelRatio":1,"width":17,"x":193,"y":77},"bus_stop":{"height":19,"pixelRatio":1,"width":19,"x":218,"y":39},"cafe":{"height":19,"pixelRatio":1,"width":19,"x":237,"y":39},"capital":{"height":10,"pixelRatio":1,"width":10,"x":210,"y":77},"clothes":{"height":19,"pixelRatio":1,"width":19,"x":104,"y":40},"convenience":{"height":19,"pixelRatio":1,"width":19,"x":79,"y":42},"drinking_water":{"height":19,"pixelRatio":1,"width":19,"x":59,"y":58},"electronics":{"height":19,"pixelRatio":1,"width":19,"x":19,"y":61},"fast_food":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":76},"ferry_terminal":{"height":19,"pixelRatio":1,"width":19,"x":38,"y":61},"forest":{"height":19,"pixelRatio":1,"width":19,"x":123,"y":40},"garden":{"height":19,"pixelRatio":1,"width":19,"x":142,"y":40},"generic_shield-1char":{"height":18,"pixelRatio":1,"width":19,"x":235,"y":0},"generic_shield-2char":{"height":18,"pixelRatio":1,"width":24,"x":60,"y":21},"generic_shield-3char":{"height":18,"pixelRatio":1,"width":33,"x":202,"y":0},"generic_shield-4char":{"height":18,"pixelRatio":1,"width":38,"x":0,"y":18},"generic_shield-5char":{"height":18,"pixelRatio":1,"width":45,"x":0,"y":0},"library":{"height":19,"pixelRatio":1,"width":19,"x":161,"y":40},"marina":{"height":19,"pixelRatio":1,"width":19,"x":180,"y":40},"museum":{"height":19,"pixelRatio":1,"width":19,"x":199,"y":58},"park":{"height":19,"pixelRatio":1,"width":19,"x":218,"y":58},"peak":{"height":19,"pixelRatio":1,"width":19,"x":237,"y":58},"post_office":{"height":19,"pixelRatio":1,"width":19,"x":98,"y":59},"restaurant":{"height":19,"pixelRatio":1,"width":19,"x":78,"y":61},"school":{"height":19,"pixelRatio":1,"width":19,"x":57,"y":77},"stadium":{"height":19,"pixelRatio":1,"width":19,"x":19,"y":80},"supermarket":{"height":19,"pixelRatio":1,"width":19,"x":0,"y":95},"theatre":{"height":19,"pixelRatio":1,"width":19,"x":38,"y":80},"toilets":{"height":19,"pixelRatio":1,"width":19,"x":117,"y":59},"townspot":{"height":10,"pixelRatio":1,"width":10,"x":220,"y":77},"train_station":{"height":19,"pixelRatio":1,"width":19,"x":136,"y":59},"university":{"height":19,"pixelRatio":1,"width":19,"x":155,"y":59},"zoo":{"height":19,"pixelRatio":1,"width":19,"x":174,"y":59}}

BIN
modules/map/static/sprites/light.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 16 KiB

1
modules/map/static/sprites/light@2x.json Normal file
View file

@ -0,0 +1 @@
{"NL:S-road-1char":{"height":42,"pixelRatio":2,"width":42,"x":0,"y":72},"NL:S-road-2char":{"height":42,"pixelRatio":2,"width":44,"x":76,"y":42},"NL:S-road-3char":{"height":42,"pixelRatio":2,"width":50,"x":404,"y":36},"NL:S-road-4char":{"height":42,"pixelRatio":2,"width":58,"x":346,"y":0},"NL:S-road-5char":{"height":42,"pixelRatio":2,"width":64,"x":158,"y":0},"US:I-1char":{"height":42,"pixelRatio":2,"width":40,"x":168,"y":42},"US:I-2char":{"height":42,"pixelRatio":2,"width":46,"x":454,"y":36},"US:I-3char":{"height":42,"pixelRatio":2,"width":60,"x":286,"y":0},"US:I-4char":{"height":42,"pixelRatio":2,"width":64,"x":222,"y":0},"US:I-5char":{"height":42,"pixelRatio":2,"width":68,"x":90,"y":0},"aerodrome":{"height":38,"pixelRatio":2,"width":38,"x":120,"y":78},"animal":{"height":38,"pixelRatio":2,"width":38,"x":42,"y":84},"arrow":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":114},"artwork":{"height":38,"pixelRatio":2,"width":38,"x":80,"y":84},"attraction":{"height":38,"pixelRatio":2,"width":38,"x":208,"y":42},"bar":{"height":38,"pixelRatio":2,"width":38,"x":246,"y":42},"beach":{"height":38,"pixelRatio":2,"width":38,"x":284,"y":42},"beauty":{"height":38,"pixelRatio":2,"width":38,"x":322,"y":42},"bench":{"height":38,"pixelRatio":2,"width":38,"x":360,"y":42},"books":{"height":38,"pixelRatio":2,"width":38,"x":398,"y":78},"building":{"height":34,"pixelRatio":2,"width":34,"x":386,"y":154},"bus_stop":{"height":38,"pixelRatio":2,"width":38,"x":436,"y":78},"cafe":{"height":38,"pixelRatio":2,"width":38,"x":474,"y":78},"capital":{"height":20,"pixelRatio":2,"width":20,"x":420,"y":154},"clothes":{"height":38,"pixelRatio":2,"width":38,"x":208,"y":80},"convenience":{"height":38,"pixelRatio":2,"width":38,"x":158,"y":84},"drinking_water":{"height":38,"pixelRatio":2,"width":38,"x":118,"y":116},"electronics":{"height":38,"pixelRatio":2,"width":38,"x":38,"y":122},"fast_food":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":152},"ferry_terminal":{"height":38,"pixelRatio":2,"width":38,"x":76,"y":122},"forest":{"height":38,"pixelRatio":2,"width":38,"x":246,"y":80},"garden":{"height":38,"pixelRatio":2,"width":38,"x":284,"y":80},"generic_shield-1char":{"height":36,"pixelRatio":2,"width":38,"x":470,"y":0},"generic_shield-2char":{"height":36,"pixelRatio":2,"width":48,"x":120,"y":42},"generic_shield-3char":{"height":36,"pixelRatio":2,"width":66,"x":404,"y":0},"generic_shield-4char":{"height":36,"pixelRatio":2,"width":76,"x":0,"y":36},"generic_shield-5char":{"height":36,"pixelRatio":2,"width":90,"x":0,"y":0},"library":{"height":38,"pixelRatio":2,"width":38,"x":322,"y":80},"marina":{"height":38,"pixelRatio":2,"width":38,"x":360,"y":80},"museum":{"height":38,"pixelRatio":2,"width":38,"x":398,"y":116},"park":{"height":38,"pixelRatio":2,"width":38,"x":436,"y":116},"peak":{"height":38,"pixelRatio":2,"width":38,"x":474,"y":116},"post_office":{"height":38,"pixelRatio":2,"width":38,"x":196,"y":118},"restaurant":{"height":38,"pixelRatio":2,"width":38,"x":156,"y":122},"school":{"height":38,"pixelRatio":2,"width":38,"x":114,"y":154},"stadium":{"height":38,"pixelRatio":2,"width":38,"x":38,"y":160},"supermarket":{"height":38,"pixelRatio":2,"width":38,"x":0,"y":190},"theatre":{"height":38,"pixelRatio":2,"width":38,"x":76,"y":160},"toilets":{"height":38,"pixelRatio":2,"width":38,"x":234,"y":118},"townspot":{"height":20,"pixelRatio":2,"width":20,"x":440,"y":154},"train_station":{"height":38,"pixelRatio":2,"width":38,"x":272,"y":118},"university":{"height":38,"pixelRatio":2,"width":38,"x":310,"y":118},"zoo":{"height":38,"pixelRatio":2,"width":38,"x":348,"y":118}}

BIN
modules/map/static/sprites/light@2x.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 28 KiB

221
modules/map/test/test_api.py
View file

@ -1,135 +1,130 @@
import json
from http import HTTPStatus
from itertools import zip_longest
import mock
import pytest
async def assert_map_file_status(async_client, expected, status_code=HTTPStatus.OK):
@pytest.mark.asyncio
async def test_get_files_empty(async_client, test_directory):
"""An empty map directory returns no files."""
request, response = await async_client.get('/api/map/files')
assert response.status_code == status_code
for pbf, expected_ in zip_longest(response.json['files'], expected):
imported = 'to be imported' if expected_['imported'] else 'to NOT be imported'
if not pbf['imported'] == expected_['imported']:
raise AssertionError(f'Expected {pbf["path"]} {imported}')
assert pbf['path'].endswith(expected_['path'])
assert response.json['pending'] is None
assert response.status_code == HTTPStatus.OK
assert response.json['files'] == []
@pytest.mark.asyncio
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
async def test_status_and_import(async_client, test_session, make_files_structure, mock_run_command):
"""PBF files can be imported, and the status of the import can be monitored."""
pbf1, pbf2, dump1 = make_files_structure([
'map/pbf/some-country.osm.pbf',
'map/pbf/other-country.osm.pbf',
'map/dump/dumpy.dump',
async def test_get_files(async_client, test_directory, make_files_structure):
"""PMTiles files in the map directory are listed."""
make_files_structure([
'map/usa.pmtiles',
'map/oregon.pmtiles',
'map/not-a-map.txt',
])
# The PBF files can be found, but have not yet been imported.
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': False, 'path': 'map/pbf/some-country.osm.pbf'},
]
await assert_map_file_status(async_client, expected)
body = {'files': [str(pbf1), str(pbf2)]}
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Can import multiple pbf files.
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
{'imported': True, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
await assert_map_file_status(async_client, expected)
body = {'files': [str(pbf1), ], }
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Importing one pbf resets the import statuses.
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
await assert_map_file_status(async_client, expected)
body = {'files': [str(dump1), ]}
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Importing a dump also resets statuses.
expected = [
{'imported': True, 'path': 'map/dump/dumpy.dump'},
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
await assert_map_file_status(async_client, expected)
request, response = await async_client.get('/api/map/files')
assert response.status_code == HTTPStatus.OK
files = response.json['files']
names = [f['name'] for f in files]
assert 'oregon.pmtiles' in names
assert 'usa.pmtiles' in names
assert 'not-a-map.txt' not in names
@pytest.mark.asyncio
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
async def test_multiple_import(async_client, test_session, test_directory, make_files_structure, mock_run_command):
"""Multiple PBFs can be imported. Importing a second time overwrites the previous imports."""
pbf1, pbf2, pbf3, dump = make_files_structure([
'map/pbf/country1.osm.pbf',
'map/pbf/country2.osm.pbf',
'map/pbf/country3.osm.pbf',
'map/dumpy.dump',
])
async def test_delete_file(async_client, test_directory, make_files_structure):
"""A PMTiles file can be deleted."""
pmtiles_file, = make_files_structure(['map/oregon.pmtiles'])
body = {'files': [str(pbf2), str(pbf1)]}
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
# File exists.
request, response = await async_client.get('/api/map/files')
assert len(response.json['files']) == 1
# Delete the file.
request, response = await async_client.delete('/api/map/files/oregon.pmtiles')
assert response.status_code == HTTPStatus.NO_CONTENT
# Two PBFs can be merged and imported.
expected = [
{'imported': False, 'path': 'map/dumpy.dump'},
{'imported': True, 'path': 'map/pbf/country1.osm.pbf'},
{'imported': True, 'path': 'map/pbf/country2.osm.pbf'},
{'imported': False, 'path': 'map/pbf/country3.osm.pbf'},
]
await assert_map_file_status(async_client, expected)
body = {'files': [str(pbf3), ]}
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# New imports overwrite old imports.
expected = [
{'imported': False, 'path': 'map/dumpy.dump'},
{'imported': False, 'path': 'map/pbf/country1.osm.pbf'},
{'imported': False, 'path': 'map/pbf/country2.osm.pbf'},
{'imported': True, 'path': 'map/pbf/country3.osm.pbf'},
]
await assert_map_file_status(async_client, expected)
# Can't mix PBF and dump.
body = {'files': [str(dump), ]}
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Dump does not overwrite.
expected = [
{'imported': True, 'path': 'map/dumpy.dump'},
{'imported': False, 'path': 'map/pbf/country1.osm.pbf'},
{'imported': False, 'path': 'map/pbf/country2.osm.pbf'},
{'imported': True, 'path': 'map/pbf/country3.osm.pbf'},
]
await assert_map_file_status(async_client, expected)
# File is gone.
request, response = await async_client.get('/api/map/files')
assert len(response.json['files']) == 0
@pytest.mark.asyncio
async def test_empty_import(async_client, test_session, test_directory):
"""Some files must be requested."""
body = {'files': []}
request, response = await async_client.post('/api/map/import', content=json.dumps(body))
async def test_delete_nonexistent_file(async_client, test_directory):
"""Deleting a nonexistent file returns 404."""
request, response = await async_client.delete('/api/map/files/nonexistent.pmtiles')
assert response.status_code == HTTPStatus.NOT_FOUND
@pytest.mark.asyncio
async def test_delete_path_traversal(async_client, test_directory, make_files_structure):
"""Path traversal attempts are rejected."""
request, response = await async_client.delete('/api/map/files/..%2F..%2Fetc%2Fpasswd')
# Path traversal is blocked — either 400 (caught by validation) or 404 (file not found).
assert response.status_code in (HTTPStatus.BAD_REQUEST, HTTPStatus.NOT_FOUND)
@pytest.mark.asyncio
async def test_get_subscriptions_empty(async_client, test_session, test_directory):
"""Catalog and empty subscriptions can be fetched."""
request, response = await async_client.get('/api/map/subscribe')
assert response.status_code == HTTPStatus.OK
assert 'catalog' in response.json
assert 'subscriptions' in response.json
assert len(response.json['catalog']) > 0
assert len(response.json['subscriptions']) == 0
@pytest.mark.asyncio
async def test_subscribe_and_unsubscribe(async_client, test_session, test_directory):
"""A map region can be subscribed to and unsubscribed from."""
body = json.dumps({'name': 'Alaska', 'region': 'us-alaska'})
request, response = await async_client.post('/api/map/subscribe', content=body)
assert response.status_code == HTTPStatus.CREATED
# Subscription appears in the list.
request, response = await async_client.get('/api/map/subscribe')
assert response.status_code == HTTPStatus.OK
regions = [s['region'] for s in response.json['subscriptions']]
assert 'us-alaska' in regions
# Unsubscribe.
request, response = await async_client.delete('/api/map/subscribe/us-alaska')
assert response.status_code == HTTPStatus.NO_CONTENT
# Subscription is gone.
request, response = await async_client.get('/api/map/subscribe')
assert len(response.json['subscriptions']) == 0
@pytest.mark.asyncio
async def test_subscribe_multiple_regions(async_client, test_session, test_directory):
"""Multiple regions share a single Download record."""
body = json.dumps({'name': 'Alaska', 'region': 'us-alaska'})
request, response = await async_client.post('/api/map/subscribe', content=body)
assert response.status_code == HTTPStatus.CREATED
body = json.dumps({'name': 'United States (West)', 'region': 'us-west'})
request, response = await async_client.post('/api/map/subscribe', content=body)
assert response.status_code == HTTPStatus.CREATED
request, response = await async_client.get('/api/map/subscribe')
regions = [s['region'] for s in response.json['subscriptions']]
assert 'us-alaska' in regions
assert 'us-west' in regions
# Unsubscribe one — the other remains.
request, response = await async_client.delete('/api/map/subscribe/us-alaska')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = await async_client.get('/api/map/subscribe')
regions = [s['region'] for s in response.json['subscriptions']]
assert 'us-alaska' not in regions
assert 'us-west' in regions
@pytest.mark.asyncio
async def test_subscribe_invalid_region(async_client, test_session, test_directory):
"""Subscribing to an invalid region returns an error."""
body = json.dumps({'name': 'Atlantis', 'region': 'atlantis'})
request, response = await async_client.post('/api/map/subscribe', content=body)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'validate' in response.json['message']

0
modules/map/test/test_downloader.py Normal file
View file

223
modules/map/test/test_lib.py
View file

@ -6,98 +6,159 @@ from modules.map import lib
from wrolpi.common import get_wrolpi_config
def test_get_map_paths(test_directory, make_files_structure):
pbf, dump = make_files_structure([
'map/pbf/map.osm.pbf',
'map/pbf/map.dump',
def test_get_pmtiles_files_empty(test_directory):
"""No files returns an empty list."""
assert lib.get_pmtiles_files() == []
def test_get_pmtiles_files(test_directory, make_files_structure):
"""PMTiles files are found, other files are ignored."""
make_files_structure([
'map/usa.pmtiles',
'map/oregon.pmtiles',
'map/old-data.osm.pbf',
'map/readme.txt',
])
with mock.patch('modules.map.lib.subprocess.check_output') as mock_check_output:
mock_check_output.return_value = b'OpenStreetMap Protocolbuffer Binary Format'
assert lib.get_map_paths() == [pbf, ]
files = lib.get_pmtiles_files()
names = [f['name'] for f in files]
assert 'oregon.pmtiles' in names
assert 'usa.pmtiles' in names
assert 'old-data.osm.pbf' not in names
assert 'readme.txt' not in names
with mock.patch('modules.map.lib.subprocess.check_output') as mock_check_output:
mock_check_output.return_value = b'PostgreSQL custom database dump'
assert lib.get_map_paths() == [dump, ]
# Each file has expected keys.
for f in files:
assert 'name' in f
assert 'path' in f
assert 'size' in f
assert 'mtime' in f
def test_delete_pmtiles_file(test_directory, make_files_structure):
"""A PMTiles file can be deleted."""
pmtiles_file, = make_files_structure(['map/oregon.pmtiles'])
assert pmtiles_file.is_file()
assert lib.delete_pmtiles_file('oregon.pmtiles') is True
assert not pmtiles_file.is_file()
def test_delete_nonexistent_file(test_directory):
"""Deleting a nonexistent file returns False."""
assert lib.delete_pmtiles_file('nonexistent.pmtiles') is False
def test_delete_path_traversal(test_directory, make_files_structure):
"""Path traversal is rejected."""
make_files_structure(['map/safe.pmtiles'])
with pytest.raises(ValueError):
lib.delete_pmtiles_file('../../etc/passwd')
def test_get_map_catalog(test_directory):
"""Catalog returns predefined regions."""
catalog = lib.get_map_catalog()
assert len(catalog) > 0
names = [r['name'] for r in catalog]
assert 'Alaska' in names # Verify a known region exists
for region in catalog:
assert 'name' in region
assert 'region' in region
assert 'size_estimate' in region
@pytest.mark.asyncio
async def test_run_import_command(test_directory, mock_run_command, run_command_bad_result):
"""
Import map files. Files to import are checked for validity before importing. Any errors returned by
asyncio.create_subprocess_shell are caught and reported.
"""
foo_pbf = test_directory / 'foo.osm.pbf'
bar_pbf = test_directory / 'bar.osm.pbf'
async def test_subscribe(async_client, test_session, test_directory):
"""A subscription adds a region to the download settings."""
await lib.subscribe(test_session, 'Alaska', 'us-alaska')
subs = lib.get_map_subscriptions(test_session)
regions = [s['region'] for s in subs]
assert 'us-alaska' in regions
@pytest.mark.asyncio
async def test_subscribe_multiple(async_client, test_session, test_directory):
"""Multiple subscriptions share one Download."""
await lib.subscribe(test_session, 'Alaska', 'us-alaska')
await lib.subscribe(test_session, 'United States (West)', 'us-west')
subs = lib.get_map_subscriptions(test_session)
regions = [s['region'] for s in subs]
assert 'us-alaska' in regions
assert 'us-west' in regions
@pytest.mark.asyncio
async def test_subscribe_duplicate(async_client, test_session, test_directory):
"""Subscribing to the same region twice doesn't duplicate."""
await lib.subscribe(test_session, 'Alaska', 'us-alaska')
await lib.subscribe(test_session, 'Alaska', 'us-alaska')
subs = lib.get_map_subscriptions(test_session)
assert len([s for s in subs if s['region'] == 'us-alaska']) == 1
@pytest.mark.asyncio
async def test_subscribe_invalid(async_client, test_session, test_directory):
"""Invalid region name raises ValueError."""
with pytest.raises(ValueError):
await lib.run_import_command(foo_pbf)
foo_pbf.touch()
with pytest.raises(ValueError):
await lib.run_import_command(foo_pbf)
dump_file = test_directory / 'dumpy.dump'
dump_file.touch()
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
# Run the import, it succeeds.
mock_is_pbf_file.return_value = True
await lib.run_import_command(foo_pbf)
# Can import more than one pbf.
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
mock_is_pbf_file.return_value = True
await lib.run_import_command(foo_pbf, bar_pbf)
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
# Run the import, it fails.
mock_run_command(run_command_bad_result)
mock_is_pbf_file.return_value = True
with pytest.raises(ValueError):
await lib.run_import_command(foo_pbf)
with pytest.raises(ValueError) as e:
await lib.run_import_command()
assert 'Must import a file' in str(e)
# Can't import more than one dump.
with pytest.raises(ValueError) as e:
await lib.run_import_command(dump_file, dump_file)
assert 'more than one' in str(e)
await lib.subscribe(test_session, 'Atlantis', 'atlantis')
@pytest.mark.parametrize('size,expected', [
(0, 0),
(-1, 0),
(17737381, 101),
(63434267, 361),
(87745484, 500),
(116318111, 662),
(136372996, 777),
(1936075318, 29032),
(2676094489, 46705),
(3392375001, 67202),
(11346305075, 519006),
])
def test_seconds_to_import_rpi4(size, expected):
assert lib.seconds_to_import(size) == expected
@pytest.mark.asyncio
async def test_unsubscribe(async_client, test_session, test_directory):
"""Unsubscribing removes a region."""
await lib.subscribe(test_session, 'Alaska', 'us-alaska')
await lib.subscribe(test_session, 'United States (West)', 'us-west')
await lib.unsubscribe(test_session, 'us-alaska')
subs = lib.get_map_subscriptions(test_session)
regions = [s['region'] for s in subs]
assert 'us-alaska' not in regions
assert 'us-west' in regions
@pytest.mark.parametrize('size,expected', [
(0, 0),
(-1, 0),
(17737381, 130),
(63434267, 466),
(87745484, 645),
(116318111, 855),
(136372996, 1002),
(1936075318, 10873),
(2676094489, 18313),
(3392375001, 27253),
(11346305075, 241574),
])
def test_seconds_to_import_rpi5(size, expected):
assert lib.seconds_to_import(size, True) == expected
@pytest.mark.asyncio
async def test_unsubscribe_last_deletes_download(async_client, test_session, test_directory):
"""Unsubscribing the last region deletes the Download."""
await lib.subscribe(test_session, 'Alaska', 'us-alaska')
await lib.unsubscribe(test_session, 'us-alaska')
assert lib.get_map_subscriptions(test_session) == []
assert lib._get_map_download(test_session) is None
@pytest.mark.asyncio
async def test_subscribe_renews_download(async_client, test_session, test_directory):
"""Subscribing renews the download so it runs immediately."""
await lib.subscribe(test_session, 'Alaska', 'us-alaska')
download = lib._get_map_download(test_session)
assert download.status == 'new'
# Simulate the download completing.
download.status = 'complete'
test_session.commit()
# Subscribe to another region — download should be renewed.
await lib.subscribe(test_session, 'United States (West)', 'us-west')
download = lib._get_map_download(test_session)
assert download.status == 'new'
@pytest.mark.asyncio
async def test_subscribe_saves_config(async_client, test_session, test_directory):
"""Subscribing triggers a download config save."""
with mock.patch('modules.map.lib.save_downloads_config') as mock_save:
await lib.subscribe(test_session, 'Alaska', 'us-alaska')
mock_save.activate_switch.assert_called_once()
@pytest.mark.asyncio
async def test_unsubscribe_saves_config(async_client, test_session, test_directory):
"""Unsubscribing triggers a download config save."""
await lib.subscribe(test_session, 'Alaska', 'us-alaska')
with mock.patch('modules.map.lib.save_downloads_config') as mock_save:
await lib.unsubscribe(test_session, 'us-alaska')
mock_save.activate_switch.assert_called_once()
def test_get_custom_map_directory(async_client, test_directory, test_wrolpi_config):

32
modules/videos/downloader.py
View file

@ -26,7 +26,7 @@ from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult, \
clear_download_progress, _parse_size, make_progress_callback, get_download_cache_config
from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError
from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError, ValidationError
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
from wrolpi.files.worker import file_worker
@ -34,7 +34,8 @@ from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
from .channel.lib import create_channel, get_channel
from .common import get_no_channel_directory, update_view_counts_and_censored, \
ffmpeg_video_complete
from .errors import UnknownChannel
from .errors import UnknownChannel, ChannelDirectoryConflict, ChannelNameConflict, ChannelURLConflict, \
ChannelSourceIdConflict
from .cookies import cookies_unlocked, cookies_for_download
from .lib import get_videos_downloader_config, get_effective_video_settings, CHANNEL_INHERITABLE_SETTINGS, YDL, \
ydl_logger, format_videos_destination, get_yt_dlp_http_headers, get_yt_dlp_sleep_opts
@ -419,7 +420,8 @@ class ChannelDownloader(Downloader, ABC):
raise ValueError(f'Could not find name')
channel_source_id = info.get('channel_id') or info.get('id')
channel_url = info.get('channel_url') or info.get('uploader_url') or download.url
channel = get_or_create_channel(session, channel_source_id, channel_url, name, channel_tag_name)
channel = get_or_create_channel(session, channel_source_id, channel_url, name, channel_tag_name,
destination=download.destination)
channel.dict() # get all attributes while we have the session.
# Link this channel download to the channel's collection.
@ -872,8 +874,8 @@ class VideoDownloader(Downloader, ABC):
# Try to find the channel via info_json from yt-dlp.
try:
channel = get_or_create_channel(session, source_id=channel_source_id, url=channel_url,
name=channel_name,
tag_name=channel_tag_name)
name=channel_name, tag_name=channel_tag_name,
destination=download.destination)
if channel:
found_channel = 'yt_dlp'
logger.debug(f'Found a channel with source_id {channel=}')
@ -1096,11 +1098,12 @@ video_downloader = VideoDownloader()
def get_or_create_channel(session: Session, source_id: str = None, url: str = None, name: str = None,
tag_name: str = None) -> Channel:
tag_name: str = None, destination: pathlib.Path = None) -> Channel:
"""
Attempt to find a Channel using the provided params. The params are in order of reliability.
Creates a new Channel if one cannot be found.
Creates a new Channel if one cannot be found. If a destination is provided, the Channel will be created in that
directory instead of the default videos destination.
"""
try:
channel = get_channel(session, source_id=source_id, url=url, name=name, return_dict=False)
@ -1113,18 +1116,25 @@ def get_or_create_channel(session: Session, source_id: str = None, url: str = No
if not name:
raise UnknownChannel(f'Cannot create channel without a name')
# Channel does not exist. Create one in the video directory.
channel_directory = format_videos_destination(name, tag_name, url)
# Channel does not exist. Create one using the destination if provided, otherwise the default video directory.
if destination:
channel_directory = get_absolute_media_path(destination)
else:
channel_directory = format_videos_destination(name, tag_name, url)
if not channel_directory.is_dir():
channel_directory.mkdir(parents=True)
data = ChannelPostRequest(
source_id=source_id,
name=name,
url=url,
directory=str(channel_directory.relative_to(get_media_directory())),
directory=str(channel_directory),
tag_name=tag_name,
)
channel = create_channel(session, data=data, return_dict=False)
try:
channel = create_channel(session, data=data, return_dict=False)
except (ValidationError, ChannelDirectoryConflict, ChannelNameConflict, ChannelURLConflict, ChannelSourceIdConflict) as e:
raise UnrecoverableDownloadError(
f'Cannot create channel {name!r}: conflicts with an existing channel') from e
# Create the directory now that the channel is approved.
channel_directory.mkdir(exist_ok=True)
# Get all properties while we have the session.

119
modules/videos/test/test_downloader.py
View file

@ -14,8 +14,8 @@ from modules.videos.downloader import VideoDownloader, \
from modules.videos.lib import get_videos_downloader_config
from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory, await_switches
from wrolpi.downloader import Download, DownloadResult
from wrolpi.errors import InvalidDownload
from wrolpi.downloader import Download, DownloadResult, get_download_manager_config
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
from wrolpi.vars import PROJECT_DIR
example_video_json = {
@ -308,6 +308,84 @@ async def test_get_or_create_channel_youtube_channel_url(async_client, test_sess
assert result.id == channel.id
@pytest.mark.asyncio
async def test_get_or_create_channel_with_destination(async_client, test_session, test_directory):
"""A new Channel should be created at the destination directory when provided."""
destination = test_directory / 'custom/my_channel'
channel = get_or_create_channel(test_session, source_id='abc', name='My Channel',
url='https://example.com', destination=destination)
assert channel.name == 'My Channel'
assert channel.directory == destination
assert destination.is_dir()
@pytest.mark.asyncio
async def test_get_or_create_channel_destination_conflict(async_client, test_session, test_directory):
"""Creating a channel at a destination that conflicts with an existing channel should raise
UnrecoverableDownloadError."""
from wrolpi.collections import Collection
# Create an existing channel at the destination directory.
existing_dir = test_directory / 'videos/existing'
existing_dir.mkdir(parents=True)
collection = Collection(name='Existing Channel', kind='channel', directory=existing_dir)
test_session.add(collection)
test_session.flush([collection])
channel = Channel(collection_id=collection.id, source_id='existing')
test_session.add(channel)
test_session.commit()
# Attempting to create a new channel at the same directory should fail.
with pytest.raises(UnrecoverableDownloadError):
get_or_create_channel(test_session, source_id='new_id', name='New Channel',
url='https://example.com', destination=existing_dir)
@pytest.mark.asyncio
async def test_get_or_create_channel_destination_nested_conflict(async_client, test_session, test_directory):
"""Creating a channel at a destination nested inside an existing channel's directory should raise
UnrecoverableDownloadError."""
from wrolpi.collections import Collection
parent_dir = test_directory / 'videos/parent_channel'
parent_dir.mkdir(parents=True)
collection = Collection(name='Parent Channel', kind='channel', directory=parent_dir)
test_session.add(collection)
test_session.flush([collection])
channel = Channel(collection_id=collection.id, source_id='parent')
test_session.add(channel)
test_session.commit()
# Nested directory should conflict.
nested_dir = parent_dir / 'nested'
with pytest.raises(UnrecoverableDownloadError):
get_or_create_channel(test_session, source_id='nested_id', name='Nested Channel',
url='https://example.net', destination=nested_dir)
@pytest.mark.asyncio
async def test_get_or_create_channel_existing_ignores_destination(async_client, test_session, test_directory):
"""When a channel already exists (matched by source_id), the destination parameter should be ignored."""
from wrolpi.collections import Collection
existing_dir = test_directory / 'videos/existing'
existing_dir.mkdir(parents=True)
collection = Collection(name='Existing', kind='channel', directory=existing_dir)
test_session.add(collection)
test_session.flush([collection])
channel = Channel(collection_id=collection.id, source_id='match_me', url='https://example.com')
test_session.add(channel)
test_session.commit()
# Even with a different destination, the existing channel is returned.
other_dir = test_directory / 'videos/other'
result = get_or_create_channel(test_session, source_id='match_me', name='Existing',
destination=other_dir)
assert result.id == channel.id
assert result.directory == existing_dir
def test_channel_downloader_hidden(video_download_manager):
"""
ChannelDownloader should not be presented to the User.
@ -1050,3 +1128,40 @@ async def test_fetch_video_duration_cache_eviction(test_download_cache_config):
assert 'https://example.com/video0' not in durations_dict
# Second oldest (video1) is still present.
assert durations_dict['https://example.com/video1'] == 1
@pytest.mark.asyncio
async def test_channel_download_preserves_skip_list(test_session, test_directory, simple_channel,
video_download_manager, video_file,
mock_video_extract_info, mock_video_prepare_filename,
await_switches, mock_video_process_runner,
test_download_manager_config):
"""A skipped video URL should remain in the skip list when its channel downloads again.
Regression: child video downloads created by ChannelDownloader used reset_attempts=True which
caused create_downloads to remove the URL from the skip list."""
skipped_url = 'https://youtube.com/watch?v=video_1_url'
url = 'https://www.youtube.com/c/ExampleChannel/videos'
# Skip video_1_url — simulating user skipping a video.
get_download_manager_config().skip_urls = [skipped_url]
assert get_download_manager_config().skip_urls == [skipped_url]
mock_video_prepare_filename.return_value = video_file
mock_video_extract_info.return_value = example_channel_json
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
mock_get_channel.return_value = simple_channel
simple_channel.get_or_create_download(test_session, url, 60)
# Download channel catalog.
await video_download_manager.wait_for_all_downloads()
# Download the child videos.
await video_download_manager.wait_for_all_downloads()
# The skipped URL must still be in the skip list.
assert skipped_url in get_download_manager_config().skip_urls, \
'Channel download removed a URL from the skip list!'
# Only video_2 should have been downloaded, not the skipped video_1.
once_downloads = video_download_manager.get_once_downloads(test_session)
video_downloads = [i for i in once_downloads if 'watch' in i.url]
assert {i.url for i in video_downloads} == {'https://youtube.com/watch?v=video_2_url'}

3
modules/zim/models.py
View file

@ -401,7 +401,8 @@ class ZimSubscription(Base):
def change_download(self, session: Session, url: str, frequency: int):
# A Zim file is found using the KiwixCatalogDownloader, then handled by the KiwixZimDownloader.
old_url = self.download.url if self.download else url
download = download_manager.get_or_create_download(session, old_url, reset_attempts=True)
download = download_manager.get_or_create_download(session, old_url, reset_attempts=True,
override_skip=True)
download.url = url
download.frequency = frequency
download.downloader = 'kiwix_catalog'