Reworking map import to handle postgres data dumps.

This commit is contained in:
Roland 2022-03-31 07:31:15 -06:00
parent be4c3a5021
commit 5178f72b66
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
11 changed files with 194 additions and 109 deletions

View file

@ -513,12 +513,12 @@ export async function setThrottle(on) {
}
export async function getMapImportStatus() {
let response = await apiGet(`${API_URI}/map/pbf`);
let response = await apiGet(`${API_URI}/map/files`);
return await response.json();
}
export async function importPbfs(pbfs) {
let body = {'pbfs': pbfs};
export async function importMapFiles(paths) {
let body = {'files': paths};
let response = await apiPost(`${API_URI}/map/import`, body);
if (response.status === 204) {
return null;

View file

@ -1,7 +1,7 @@
import React from "react";
import {HelpPopup, humanFileSize, minutesToTimestamp, PageContainer, TabLinks, WROLModeMessage} from "./Common";
import {Route} from "react-router-dom";
import {getMapImportStatus, importPbfs} from "../api";
import {getMapImportStatus, importMapFiles} from "../api";
import Table from "semantic-ui-react/dist/commonjs/collections/Table";
import {Button, Checkbox, Icon, Loader, Placeholder} from "semantic-ui-react";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
@ -10,7 +10,7 @@ class ManageMap extends React.Component {
constructor(props) {
super(props);
this.state = {
pbfs: null,
files: null,
importing: null,
import_running: false,
selectedPaths: [],
@ -30,7 +30,7 @@ class ManageMap extends React.Component {
fetchImportStatus = async () => {
const importStatus = await getMapImportStatus();
this.setState({
pbfs: importStatus['pbfs'],
files: importStatus['files'],
importing: importStatus['importing'],
import_running: importStatus['import_running'],
dockerized: importStatus['dockerized'],
@ -41,16 +41,16 @@ class ManageMap extends React.Component {
e.preventDefault();
this.setState({'ready': false});
const {pbfs, selectedPaths} = this.state;
const {files, selectedPaths} = this.state;
if (!selectedPaths || selectedPaths.length === 0) {
// No PBFs are selected, import all.
// No files are selected, import all.
let paths = [];
for (let i = 0; i < pbfs.length; i++) {
paths = paths.concat([pbfs[i].path]);
for (let i = 0; i < files.length; i++) {
paths = paths.concat([files[i].path]);
}
await importPbfs(paths);
await importMapFiles(paths);
} else {
await importPbfs(selectedPaths);
await importMapFiles(selectedPaths);
}
await this.fetchImportStatus();
}
@ -97,7 +97,7 @@ class ManageMap extends React.Component {
}
render() {
const {pbfs, selectedPaths, import_running, importing, dockerized} = this.state;
const {files, selectedPaths, import_running, importing, dockerized} = this.state;
let dockerWarning;
if (dockerized === true) {
@ -154,7 +154,7 @@ class ManageMap extends React.Component {
<Loader size='large' active={import_running} inline='centered'>Importing: {importing}</Loader>
);
let disabled = dockerized || !pbfs || pbfs.length === 0 || import_running;
let disabled = dockerized || !files || files.length === 0 || import_running;
let importButton = <Button
primary
disabled={disabled}
@ -164,7 +164,7 @@ class ManageMap extends React.Component {
</Button>;
let rows;
if (!pbfs) {
if (!files) {
// Fetch request is not complete.
rows = <Table.Row>
<Table.Cell/>
@ -175,12 +175,12 @@ class ManageMap extends React.Component {
</Placeholder>
</Table.Cell>
</Table.Row>;
} else if (pbfs.length === 0) {
} else if (files.length === 0) {
rows = <Table.Row>
<Table.Cell/><Table.Cell colSpan={5}>No PBF map files were found in <b>map/pbf</b></Table.Cell>
</Table.Row>;
} else {
rows = pbfs.map(i => this.tableRow(i, disabled));
rows = files.map(i => this.tableRow(i, disabled));
}
let spaceHelpPopup = <HelpPopup

View file

@ -17,36 +17,37 @@ bp = get_blueprint('Map', '/api/map')
@bp.post('/import')
@openapi.definition(
summary='Import PBF map files',
summary='Import PBF/dump map files',
body=schema.ImportPost,
)
@validate(schema.ImportPost)
@wrol_mode_check
async def import_pbfs(request: Request, body: schema.ImportPost):
async def import_pbfs(_: Request, body: schema.ImportPost):
if lib.IMPORT_EVENT.is_set():
return response.json({'error': 'Map import already running'}, HTTPStatus.CONFLICT)
pbfs = [i for i in body.pbfs if i]
if not pbfs:
raise ValidationError('No PBF files were provided')
paths = [i for i in body.files if i]
if not paths:
raise ValidationError('No PBF or dump files were provided')
coro = lib.import_files(paths)
if PYTEST:
await lib.import_pbfs(pbfs)
await coro
else:
asyncio.create_task(lib.import_pbfs(pbfs))
asyncio.create_task(coro)
return response.empty()
@bp.get('/pbf')
@openapi.description('Find any PBF map files, get their import status')
def pbf(request: Request):
pbfs = lib.get_pbf_import_status()
pbfs = sorted(pbfs, key=lambda i: str(i.path))
importing = lib.IMPORTING.get('pbf')
@bp.get('/files')
@openapi.description('Find any map files, get their import status')
def pbf(_: Request):
paths = lib.get_import_status()
paths = sorted(paths, key=lambda i: str(i.path))
importing = lib.IMPORTING.get('path')
if importing:
importing = Path(importing).relative_to(get_media_directory())
body = dict(
pbfs=pbfs,
files=paths,
importing=importing,
import_running=lib.IMPORT_EVENT.is_set(),
dockerized=DOCKERIZED,

View file

@ -17,18 +17,14 @@ logger = logger.getChild(__name__)
IMPORT_EVENT = Event()
IMPORTING = Manager().dict()
IMPORTING['pbf'] = None
IMPORTING['path'] = None
def get_map_directory() -> Path:
return get_media_directory() / 'map'
def get_pbf_directory() -> Path:
pbf_directory = get_map_directory() / 'pbf'
if not pbf_directory.is_dir():
pbf_directory.mkdir(parents=True)
return pbf_directory
map_directory = get_media_directory() / 'map'
if not map_directory.is_dir():
map_directory.mkdir()
return map_directory
def is_pbf_file(pbf: Path) -> bool:
@ -42,11 +38,31 @@ def is_pbf_file(pbf: Path) -> bool:
return 'OpenStreetMap Protocolbuffer Binary Format' in output.decode()
def get_pbf_paths() -> List[Path]:
"""Find all PBF files in the map/pbf directory"""
pbf_directory = get_pbf_directory()
paths = walk(pbf_directory)
return list(filter(lambda i: i.is_file() and str(i).endswith('.osm.pbf') and is_pbf_file(i), paths))
def is_dump_file(path: Path) -> bool:
"""Uses file command to check type of a file. Returns True if a file is a Postgresql dump file."""
cmd = ('/usr/bin/file', path)
try:
output = subprocess.check_output(cmd)
except FileNotFoundError:
return False
return 'PostgreSQL custom database dump' in output.decode()
def get_map_paths() -> List[Path]:
"""Find all pbf/dump files in the map directory."""
map_directory = get_map_directory()
paths = walk(map_directory)
def is_valid(path: Path) -> bool:
if path.is_file():
if str(path).endswith('.osm.pbf') and is_pbf_file(path):
return True
elif path.suffix == '.dump' and is_dump_file(path):
return True
return False
return list(filter(is_valid, paths))
def get_or_create_map_file(pbf_path: Path, session: Session) -> MapFile:
@ -60,43 +76,47 @@ def get_or_create_map_file(pbf_path: Path, session: Session) -> MapFile:
return map_file
async def import_pbfs(pbfs: List[str]):
async def import_files(files: List[str]):
if IMPORT_EVENT.is_set():
logger.warning('Map import already running...')
return
logger.warning(f'Importing: {", ".join(pbfs)}')
# Import dumps, then pbfs.
import_order = ('.dump', '.pbf')
files = [get_media_directory() / i for i in files]
files = sorted(files, key=lambda i: import_order.index(i.suffix))
logger.warning(f'Importing: {", ".join(map(str, files))}')
any_success = False
try:
IMPORT_EVENT.set()
for pbf in pbfs:
pbf = get_media_directory() / pbf
if not pbf.is_file():
logger.fatal(f'PBF file does not exist! {pbf}')
for path in files:
if not path.is_file():
logger.fatal(f'Map file does not exist! {path}')
continue
with get_db_session() as session:
map_file = session.query(MapFile).filter_by(path=pbf).one_or_none()
map_file = session.query(MapFile).filter_by(path=path).one_or_none()
if map_file and map_file.imported:
# Don't import a map file twice.
logger.debug(f'{pbf} is already imported')
logger.debug(f'{path} is already imported')
continue
success = False
try:
IMPORTING['pbf'] = str(pbf)
await import_pbf(pbf)
IMPORTING['path'] = str(path)
await import_file(path)
success = True
any_success = True
except Exception as e:
logger.warning('Failed to run import_pbf', exc_info=e)
logger.warning('Failed to run import', exc_info=e)
finally:
IMPORTING['pbf'] = None
IMPORTING['path'] = None
if success:
with get_db_session(commit=True) as session:
map_file = get_or_create_map_file(pbf, session)
map_file = get_or_create_map_file(path, session)
map_file.imported = True
finally:
if any_success:
@ -108,9 +128,22 @@ async def import_pbfs(pbfs: List[str]):
BASH = which('bash', '/bin/bash', warn=True)
async def import_pbf(pbf: Path):
"""Run the osm2pgsql binary to import a PBF map file."""
cmd = f'{BASH} {PROJECT_DIR}/scripts/import_map.sh {pbf.absolute()}'
async def import_file(path: Path):
"""Run the map import script on the provided path.
Supports *.osm.pbf and *.dump files."""
if str(path).endswith('.osm.pbf'):
if not is_pbf_file(path):
logger.warning(f'Could not import non-pbf file: {path}')
raise ValueError('Invalid PBF file')
elif path.suffix == '.dump':
if not is_dump_file(path):
logger.warning(f'Could not import non-dump file: {path}')
raise ValueError('Invalid dump file')
else:
raise ValueError(f'Cannot import unknown file! {path}')
cmd = f'{BASH} {PROJECT_DIR}/scripts/import_map.sh {path.absolute()}'
logger.debug(f'Running import script: {cmd}')
start = now()
proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)
@ -118,26 +151,26 @@ async def import_pbf(pbf: Path):
stdout, stderr = await proc.communicate()
elapsed = now() - start
success = 'Successful' if proc.returncode == 0 else 'Unsuccessful'
logger.info(f'{success} import of {pbf.name} took {timedelta_to_timestamp(elapsed)}')
logger.info(f'{success} import of {path.name} took {timedelta_to_timestamp(elapsed)}')
if proc.returncode != 0:
# Log all lines. Truncate long lines.
for line in stdout.decode().splitlines():
logger.debug(line[:500])
for line in stderr.decode().splitlines():
logger.error(line[:500])
raise ValueError(f'Importing PBF failed with return code {proc.returncode}')
raise ValueError(f'Importing map file failed with return code {proc.returncode}')
@optional_session
def get_pbf_import_status(session: Session = None):
pbf_paths: List[Path] = get_pbf_paths()
pbfs = []
for path in pbf_paths:
def get_import_status(session: Session = None) -> List[MapFile]:
paths = get_map_paths()
map_paths = []
for path in paths:
map_file = get_or_create_map_file(path, session)
pbfs.append(map_file)
map_paths.append(map_file)
session.commit()
return pbfs
return map_paths
MOD_TILE_CACHE_DIR = Path('/var/lib/mod_tile/ajt')

View file

@ -1,15 +1,21 @@
from pathlib import Path
from sqlalchemy import Column, Integer, Boolean, BigInteger
from wrolpi import dates
from wrolpi.common import Base, ModelHelper
from wrolpi.media_path import MediaPathType
RPI4_BYTES_PER_SECOND = 3963
RPI4_PBF_BYTES_PER_SECOND = 3963
def seconds_to_import(size: int) -> int:
def seconds_to_import(path: Path, size: int) -> int:
"""Attempt to predict how long it will take an RPi4 to import a given PBF file."""
return max(int(size // RPI4_BYTES_PER_SECOND), 0)
if str(path).endswith('.osm.pbf'):
return max(int(size // RPI4_PBF_BYTES_PER_SECOND), 0)
# Can't calculate an unknown file.
return 0
class MapFile(Base, ModelHelper):
@ -20,11 +26,14 @@ class MapFile(Base, ModelHelper):
imported = Column(Boolean, default=False, nullable=False)
size = Column(BigInteger)
def __repr__(self):
return f'<MapFile {self.path} imported={self.imported}>'
def __json__(self):
return {
'id': self.id,
'path': self.path,
'imported': self.imported,
'size': self.size,
'time_to_import': dates.seconds_to_timestamp(seconds_to_import(self.size)),
'time_to_import': dates.seconds_to_timestamp(seconds_to_import(self.path.path, self.size)),
}

View file

@ -4,4 +4,4 @@ from typing import List
@dataclass
class ImportPost:
pbfs: List[str]
files: List[str]

View file

@ -6,54 +6,77 @@ import mock
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
def test_pbf_status_and_import(test_client, test_session, test_directory):
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
def test_status_and_import(test_client, test_session, test_directory):
"""PBF files can be imported, and the status of the import can be monitored."""
(test_directory / 'map/pbf').mkdir(parents=True)
(test_directory / 'map/dump').mkdir()
pbf1 = test_directory / 'map/pbf/some-country.osm.pbf'
pbf2 = test_directory / 'map/pbf/other-country.osm.pbf'
pbf1.touch()
pbf2.touch()
dump1 = test_directory / 'map/dump/dumpy.dump'
dump1.touch()
def check_status(response_json, expected_):
for pbf, expected_ in zip_longest(response_json['pbfs'], expected_):
for pbf, expected_ in zip_longest(response_json['files'], expected_):
assert pbf['imported'] == expected_['imported']
assert pbf['path'].endswith(expected_['path'])
assert response.json['importing'] is None
# The PBF files can be found, but have not yet been imported.
request, response = test_client.get('/api/map/pbf')
request, response = test_client.get('/api/map/files')
assert response.status_code == HTTPStatus.OK
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': False, 'path': 'map/pbf/some-country.osm.pbf'},
]
check_status(response.json, expected)
with mock.patch('modules.map.lib.import_pbf') as mock_import_pbf:
body = {'pbfs': [str(pbf1), ]}
with mock.patch('modules.map.lib.import_file') as mock_import_file:
body = {'files': [str(pbf1), ], }
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Only one import is requested.
mock_import_pbf.assert_called_once()
# Only one PBF import is requested.
mock_import_file.assert_called_once()
request, response = test_client.get('/api/map/pbf')
request, response = test_client.get('/api/map/files')
assert response.status_code == HTTPStatus.OK
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
check_status(response.json, expected)
with mock.patch('modules.map.lib.import_pbf') as mock_import_pbf:
body = {'pbfs': [str(pbf1), str(pbf2)]}
with mock.patch('modules.map.lib.import_file') as mock_import_file:
body = {'files': [str(pbf1), str(pbf2)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Both imports are requested, but only one is performed.
mock_import_pbf.assert_called_once()
# Both PBF imports are requested, but only one is performed.
mock_import_file.assert_called_once()
request, response = test_client.get('/api/map/pbf')
request, response = test_client.get('/api/map/files')
assert response.status_code == HTTPStatus.OK
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
{'imported': True, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
check_status(response.json, expected)
with mock.patch('modules.map.lib.import_file') as mock_import_file:
body = {'files': [str(dump1), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Dump is imported.
mock_import_file.assert_called_once()
request, response = test_client.get('/api/map/files')
assert response.status_code == HTTPStatus.OK
expected = [
{'imported': True, 'path': 'map/dump/dumpy.dump'},
{'imported': True, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
@ -62,8 +85,18 @@ def test_pbf_status_and_import(test_client, test_session, test_directory):
def test_empty_import(test_client, test_session, test_directory):
"""Some files must be requested."""
with mock.patch('modules.map.lib.import_pbf') as mock_import_pbf:
with mock.patch('modules.map.lib.import_file') as mock_import_file:
body = {'pbfs': []}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.BAD_REQUEST
mock_import_pbf.assert_not_called()
mock_import_file.assert_not_called()
body = {'dumps': []}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.BAD_REQUEST
mock_import_file.assert_not_called()
body = {'pbfs': [], 'dumps': []}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.BAD_REQUEST
mock_import_file.assert_not_called()

View file

@ -1,16 +1,18 @@
from pathlib import Path
import pytest
from modules.map import models
@pytest.mark.parametrize('size,expected', [
(0, 0),
(-1, 0),
(17737381, 4475),
(63434267, 16006),
(87745484, 22141),
(136372996, 34411),
(116318111, 29351),
@pytest.mark.parametrize('path,size,expected', [
('file.osm.pbf', 0, 0),
('file.osm.pbf', -1, 0),
('file.osm.pbf', 17737381, 4475),
('file.osm.pbf', 63434267, 16006),
('file.osm.pbf', 87745484, 22141),
('file.osm.pbf', 136372996, 34411),
('file.osm.pbf', 116318111, 29351),
])
def test_seconds_to_import_rpi4(size, expected):
assert models.seconds_to_import(size) == expected
def test_seconds_to_import_rpi4(path, size, expected):
assert models.seconds_to_import(Path(path), size) == expected

View file

@ -11,15 +11,24 @@ if [ ! -f /opt/openstreetmap-carto/openstreetmap-carto.style ]; then
fi
if [ "${1}" == "" ]; then
echo "Missing PBF file argument."
echo "Missing file argument."
exit 3
fi
if [ ! -f "${1}" ]; then
echo "PBF file does not exist! ${1}"
echo "File does not exist! ${1}"
exit 4
fi
osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \
-S /opt/openstreetmap-carto/openstreetmap-carto.style "${1}"
if [[ ${1} == *.osm.pbf ]]; then
# Import a PBF file.
osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \
-S /opt/openstreetmap-carto/openstreetmap-carto.style "${1}"
elif [[ ${1} == *.dump ]]; then
# Import a Postgresql dump.
pg_restore -j4 --no-owner -d gis "${1}"
else
echo "Cannot import unknown map file"
exit 5
fi

View file

@ -85,7 +85,7 @@ fi
set -e
cp /opt/wrolpi/etc/ubuntu20.04/rendred.conf /usr/local/etc/renderd.conf
cp /opt/wrolpi/etc/ubuntu20.04/renderd.conf /usr/local/etc/renderd.conf
# Enable mod_tile.
cp /opt/wrolpi/etc/ubuntu20.04/mod_tile.conf /etc/apache2/conf-available/mod_tile.conf

View file

@ -55,9 +55,7 @@ def utc_now() -> datetime:
def now(tz: pytz.timezone = None) -> datetime:
"""
Get the current DateTime in the provided timezone. Timezone aware.
"""
"""Get the current DateTime in the provided timezone. Timezone aware."""
if TEST_DATETIME:
return TEST_DATETIME
tz = tz or DEFAULT_TIMEZONE