mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
Any PMTiles file under the map directory, at any depth, can be shown. The API reads each file's header and metadata and says what it is (basemap, terrain, overlay, unreadable). Overlays are offered in the layer control under the name the file gives itself, hidden until turned on, with a checkbox per vector layer, and are drawn generically. Map files are named by their path within the map directory everywhere, and unfinished downloads are no longer listed. A file can be ignored from the Manage page; ignored files are kept in a new maps.yaml config. PMTiles metadata is not decompressed past its size limit, and the standalone viewer encodes map file paths.
195 lines
6.9 KiB
Python
195 lines
6.9 KiB
Python
"""Read the header and metadata of a PMTiles (version 3) file.
|
|
|
|
A PMTiles file starts with a fixed 127 byte header, which says where the JSON metadata is. Neither needs the tile
|
|
data, so describing a file is two small reads.
|
|
|
|
https://github.com/protomaps/PMTiles/blob/main/spec/v3/spec.md
|
|
"""
|
|
import json
|
|
import struct
|
|
import zlib
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
from wrolpi.common import logger
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
__all__ = ['read_pmtiles_info', 'classify', 'describe', 'is_searchable',
|
|
'KIND_BASEMAP', 'KIND_OVERLAY', 'KIND_TERRAIN', 'KIND_UNKNOWN']
|
|
|
|
MAGIC = b'PMTiles'
|
|
HEADER_SIZE = 127
|
|
# Metadata larger than this (as stored, or once decompressed) is not read; the file is described by its header
|
|
# alone. A map file is something a person downloaded: a few bytes of it must not be able to fill the memory.
|
|
MAX_METADATA_SIZE = 16 * 1024 * 1024
|
|
|
|
COMPRESSION_NONE = 1
|
|
COMPRESSION_GZIP = 2
|
|
|
|
TILE_TYPES = {
|
|
1: 'mvt',
|
|
2: 'png',
|
|
3: 'jpeg',
|
|
4: 'webp',
|
|
5: 'avif',
|
|
}
|
|
|
|
KIND_BASEMAP = 'basemap'
|
|
KIND_OVERLAY = 'overlay'
|
|
KIND_TERRAIN = 'terrain'
|
|
KIND_UNKNOWN = 'unknown'
|
|
|
|
# A terrain (elevation) file is named, its tiles are images like any other raster file.
|
|
TERRAIN_PREFIX = 'terrain-'
|
|
# Layers every Protomaps basemap has; the map's basemap theme can only draw this schema.
|
|
BASEMAP_LAYERS = frozenset({'earth', 'water', 'roads'})
|
|
# Layers the place search index is built from.
|
|
SEARCHABLE_LAYERS = frozenset({'places', 'pois'})
|
|
|
|
# (path, mtime_ns, size) -> info. A file is only read again when it changes.
|
|
_CACHE: Dict[Tuple[str, int, int], Optional[dict]] = dict()
|
|
|
|
|
|
def _tilestats_geometries(metadata: dict) -> Dict[str, str]:
|
|
"""The geometry type of each layer, when the file's generator recorded it (tippecanoe does)."""
|
|
tilestats = metadata.get('tilestats')
|
|
layers = tilestats.get('layers') if isinstance(tilestats, dict) else None
|
|
geometries = dict()
|
|
for layer in layers if isinstance(layers, list) else []:
|
|
if isinstance(layer, dict) and layer.get('layer') and isinstance(layer.get('geometry'), str):
|
|
geometries[str(layer['layer'])] = layer['geometry']
|
|
return geometries
|
|
|
|
|
|
def _vector_layers(metadata: dict) -> List[dict]:
|
|
geometries = _tilestats_geometries(metadata)
|
|
layers = []
|
|
vector_layers = metadata.get('vector_layers')
|
|
for layer in vector_layers if isinstance(vector_layers, list) else []:
|
|
if not isinstance(layer, dict) or not layer.get('id'):
|
|
continue
|
|
id_ = str(layer['id'])
|
|
layers.append(dict(
|
|
id=id_,
|
|
minzoom=layer.get('minzoom'),
|
|
maxzoom=layer.get('maxzoom'),
|
|
description=layer.get('description') or None,
|
|
geometry=geometries.get(id_),
|
|
))
|
|
return layers
|
|
|
|
|
|
def _text(value) -> Optional[str]:
|
|
return str(value) if value not in (None, '') else None
|
|
|
|
|
|
def _gunzip(raw: bytes) -> bytes:
|
|
"""Decompress gzip data, refusing to produce more than MAX_METADATA_SIZE."""
|
|
decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
|
|
data = decompressor.decompress(raw, MAX_METADATA_SIZE)
|
|
if decompressor.unconsumed_tail:
|
|
raise ValueError(f'metadata is larger than {MAX_METADATA_SIZE} bytes')
|
|
return data
|
|
|
|
|
|
def _read(path: Path) -> Optional[dict]:
|
|
with path.open('rb') as fh:
|
|
header = fh.read(HEADER_SIZE)
|
|
if len(header) < HEADER_SIZE or header[:7] != MAGIC or header[7] != 3:
|
|
return None
|
|
|
|
metadata_offset, metadata_length = struct.unpack_from('<QQ', header, 24)
|
|
internal_compression, _, tile_type, min_zoom, max_zoom = struct.unpack_from('<5B', header, 97)
|
|
min_lon, min_lat, max_lon, max_lat = (i / 10_000_000 for i in struct.unpack_from('<4i', header, 102))
|
|
|
|
metadata = dict()
|
|
if 0 < metadata_length <= MAX_METADATA_SIZE \
|
|
and internal_compression in (COMPRESSION_NONE, COMPRESSION_GZIP):
|
|
fh.seek(metadata_offset)
|
|
raw = fh.read(metadata_length)
|
|
try:
|
|
if internal_compression == COMPRESSION_GZIP:
|
|
raw = _gunzip(raw)
|
|
metadata = json.loads(raw)
|
|
except (zlib.error, ValueError) as e:
|
|
logger.warning(f'Unable to read PMTiles metadata of {path}: {e}')
|
|
if not isinstance(metadata, dict):
|
|
metadata = dict()
|
|
|
|
return dict(
|
|
tile_type=TILE_TYPES.get(tile_type, 'unknown'),
|
|
min_zoom=min_zoom,
|
|
max_zoom=max_zoom,
|
|
bounds=[min_lon, min_lat, max_lon, max_lat],
|
|
title=_text(metadata.get('name')),
|
|
description=_text(metadata.get('description')),
|
|
attribution=_text(metadata.get('attribution')),
|
|
metadata_type=_text(metadata.get('type')),
|
|
vector_layers=_vector_layers(metadata),
|
|
)
|
|
|
|
|
|
def read_pmtiles_info(path: Path) -> Optional[dict]:
|
|
"""Describe a PMTiles file from its header and metadata. Returns None if the file is not a readable PMTiles
|
|
version 3 file; never raises."""
|
|
try:
|
|
stat = path.stat()
|
|
except OSError:
|
|
return None
|
|
|
|
key = (str(path), stat.st_mtime_ns, stat.st_size)
|
|
if key not in _CACHE:
|
|
try:
|
|
info = _read(path)
|
|
except (OSError, struct.error) as e:
|
|
logger.warning(f'Unable to read PMTiles file {path}: {e}')
|
|
info = None
|
|
# Forget what this path used to be.
|
|
for old in [i for i in _CACHE if i[0] == key[0]]:
|
|
del _CACHE[old]
|
|
_CACHE[key] = info
|
|
info = _CACHE[key]
|
|
return dict(info, vector_layers=[dict(i) for i in info['vector_layers']]) if info else None
|
|
|
|
|
|
def _layer_ids(info: Optional[dict]) -> set:
|
|
return {i['id'] for i in info['vector_layers']} if info else set()
|
|
|
|
|
|
def classify(name: str, info: Optional[dict]) -> str:
|
|
"""How the map draws a file: `terrain` (hillshade and contours), `basemap` (Protomaps schema, drawn with the
|
|
basemap theme), `overlay` (anything else, vector or raster), or `unknown` (unreadable, never drawn)."""
|
|
if name.startswith(TERRAIN_PREFIX):
|
|
return KIND_TERRAIN
|
|
if not info:
|
|
return KIND_UNKNOWN
|
|
if info['tile_type'] == 'mvt' and BASEMAP_LAYERS <= _layer_ids(info):
|
|
return KIND_BASEMAP
|
|
return KIND_OVERLAY
|
|
|
|
|
|
def is_searchable(path: Path) -> bool:
|
|
"""Whether a place search index can be built from this file."""
|
|
info = read_pmtiles_info(path)
|
|
return classify(path.name, info) == KIND_BASEMAP and bool(SEARCHABLE_LAYERS & _layer_ids(info))
|
|
|
|
|
|
EMPTY_INFO = dict(
|
|
tile_type=None,
|
|
min_zoom=None,
|
|
max_zoom=None,
|
|
bounds=None,
|
|
title=None,
|
|
description=None,
|
|
attribution=None,
|
|
metadata_type=None,
|
|
vector_layers=[],
|
|
)
|
|
|
|
|
|
def describe(path: Path) -> dict:
|
|
"""The `kind` of a PMTiles file and what its header and metadata say, with empty values for an unreadable
|
|
file."""
|
|
info = read_pmtiles_info(path)
|
|
return dict(info or EMPTY_INFO, kind=classify(path.name, info))
|