mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 06:56:18 -05:00
Reworking map import to handle postgres data dumps.
This commit is contained in:
parent
be4c3a5021
commit
5178f72b66
11 changed files with 194 additions and 109 deletions
|
|
@ -513,12 +513,12 @@ export async function setThrottle(on) {
|
|||
}
|
||||
|
||||
export async function getMapImportStatus() {
|
||||
let response = await apiGet(`${API_URI}/map/pbf`);
|
||||
let response = await apiGet(`${API_URI}/map/files`);
|
||||
return await response.json();
|
||||
}
|
||||
|
||||
export async function importPbfs(pbfs) {
|
||||
let body = {'pbfs': pbfs};
|
||||
export async function importMapFiles(paths) {
|
||||
let body = {'files': paths};
|
||||
let response = await apiPost(`${API_URI}/map/import`, body);
|
||||
if (response.status === 204) {
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import React from "react";
|
||||
import {HelpPopup, humanFileSize, minutesToTimestamp, PageContainer, TabLinks, WROLModeMessage} from "./Common";
|
||||
import {Route} from "react-router-dom";
|
||||
import {getMapImportStatus, importPbfs} from "../api";
|
||||
import {getMapImportStatus, importMapFiles} from "../api";
|
||||
import Table from "semantic-ui-react/dist/commonjs/collections/Table";
|
||||
import {Button, Checkbox, Icon, Loader, Placeholder} from "semantic-ui-react";
|
||||
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
|
||||
|
|
@ -10,7 +10,7 @@ class ManageMap extends React.Component {
|
|||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {
|
||||
pbfs: null,
|
||||
files: null,
|
||||
importing: null,
|
||||
import_running: false,
|
||||
selectedPaths: [],
|
||||
|
|
@ -30,7 +30,7 @@ class ManageMap extends React.Component {
|
|||
fetchImportStatus = async () => {
|
||||
const importStatus = await getMapImportStatus();
|
||||
this.setState({
|
||||
pbfs: importStatus['pbfs'],
|
||||
files: importStatus['files'],
|
||||
importing: importStatus['importing'],
|
||||
import_running: importStatus['import_running'],
|
||||
dockerized: importStatus['dockerized'],
|
||||
|
|
@ -41,16 +41,16 @@ class ManageMap extends React.Component {
|
|||
e.preventDefault();
|
||||
|
||||
this.setState({'ready': false});
|
||||
const {pbfs, selectedPaths} = this.state;
|
||||
const {files, selectedPaths} = this.state;
|
||||
if (!selectedPaths || selectedPaths.length === 0) {
|
||||
// No PBFs are selected, import all.
|
||||
// No files are selected, import all.
|
||||
let paths = [];
|
||||
for (let i = 0; i < pbfs.length; i++) {
|
||||
paths = paths.concat([pbfs[i].path]);
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
paths = paths.concat([files[i].path]);
|
||||
}
|
||||
await importPbfs(paths);
|
||||
await importMapFiles(paths);
|
||||
} else {
|
||||
await importPbfs(selectedPaths);
|
||||
await importMapFiles(selectedPaths);
|
||||
}
|
||||
await this.fetchImportStatus();
|
||||
}
|
||||
|
|
@ -97,7 +97,7 @@ class ManageMap extends React.Component {
|
|||
}
|
||||
|
||||
render() {
|
||||
const {pbfs, selectedPaths, import_running, importing, dockerized} = this.state;
|
||||
const {files, selectedPaths, import_running, importing, dockerized} = this.state;
|
||||
|
||||
let dockerWarning;
|
||||
if (dockerized === true) {
|
||||
|
|
@ -154,7 +154,7 @@ class ManageMap extends React.Component {
|
|||
<Loader size='large' active={import_running} inline='centered'>Importing: {importing}</Loader>
|
||||
);
|
||||
|
||||
let disabled = dockerized || !pbfs || pbfs.length === 0 || import_running;
|
||||
let disabled = dockerized || !files || files.length === 0 || import_running;
|
||||
let importButton = <Button
|
||||
primary
|
||||
disabled={disabled}
|
||||
|
|
@ -164,7 +164,7 @@ class ManageMap extends React.Component {
|
|||
</Button>;
|
||||
|
||||
let rows;
|
||||
if (!pbfs) {
|
||||
if (!files) {
|
||||
// Fetch request is not complete.
|
||||
rows = <Table.Row>
|
||||
<Table.Cell/>
|
||||
|
|
@ -175,12 +175,12 @@ class ManageMap extends React.Component {
|
|||
</Placeholder>
|
||||
</Table.Cell>
|
||||
</Table.Row>;
|
||||
} else if (pbfs.length === 0) {
|
||||
} else if (files.length === 0) {
|
||||
rows = <Table.Row>
|
||||
<Table.Cell/><Table.Cell colSpan={5}>No PBF map files were found in <b>map/pbf</b></Table.Cell>
|
||||
</Table.Row>;
|
||||
} else {
|
||||
rows = pbfs.map(i => this.tableRow(i, disabled));
|
||||
rows = files.map(i => this.tableRow(i, disabled));
|
||||
}
|
||||
|
||||
let spaceHelpPopup = <HelpPopup
|
||||
|
|
|
|||
|
|
@ -17,36 +17,37 @@ bp = get_blueprint('Map', '/api/map')
|
|||
|
||||
@bp.post('/import')
|
||||
@openapi.definition(
|
||||
summary='Import PBF map files',
|
||||
summary='Import PBF/dump map files',
|
||||
body=schema.ImportPost,
|
||||
)
|
||||
@validate(schema.ImportPost)
|
||||
@wrol_mode_check
|
||||
async def import_pbfs(request: Request, body: schema.ImportPost):
|
||||
async def import_pbfs(_: Request, body: schema.ImportPost):
|
||||
if lib.IMPORT_EVENT.is_set():
|
||||
return response.json({'error': 'Map import already running'}, HTTPStatus.CONFLICT)
|
||||
|
||||
pbfs = [i for i in body.pbfs if i]
|
||||
if not pbfs:
|
||||
raise ValidationError('No PBF files were provided')
|
||||
paths = [i for i in body.files if i]
|
||||
if not paths:
|
||||
raise ValidationError('No PBF or dump files were provided')
|
||||
|
||||
coro = lib.import_files(paths)
|
||||
if PYTEST:
|
||||
await lib.import_pbfs(pbfs)
|
||||
await coro
|
||||
else:
|
||||
asyncio.create_task(lib.import_pbfs(pbfs))
|
||||
asyncio.create_task(coro)
|
||||
return response.empty()
|
||||
|
||||
|
||||
@bp.get('/pbf')
|
||||
@openapi.description('Find any PBF map files, get their import status')
|
||||
def pbf(request: Request):
|
||||
pbfs = lib.get_pbf_import_status()
|
||||
pbfs = sorted(pbfs, key=lambda i: str(i.path))
|
||||
importing = lib.IMPORTING.get('pbf')
|
||||
@bp.get('/files')
|
||||
@openapi.description('Find any map files, get their import status')
|
||||
def pbf(_: Request):
|
||||
paths = lib.get_import_status()
|
||||
paths = sorted(paths, key=lambda i: str(i.path))
|
||||
importing = lib.IMPORTING.get('path')
|
||||
if importing:
|
||||
importing = Path(importing).relative_to(get_media_directory())
|
||||
body = dict(
|
||||
pbfs=pbfs,
|
||||
files=paths,
|
||||
importing=importing,
|
||||
import_running=lib.IMPORT_EVENT.is_set(),
|
||||
dockerized=DOCKERIZED,
|
||||
|
|
|
|||
|
|
@ -17,18 +17,14 @@ logger = logger.getChild(__name__)
|
|||
|
||||
IMPORT_EVENT = Event()
|
||||
IMPORTING = Manager().dict()
|
||||
IMPORTING['pbf'] = None
|
||||
IMPORTING['path'] = None
|
||||
|
||||
|
||||
def get_map_directory() -> Path:
|
||||
return get_media_directory() / 'map'
|
||||
|
||||
|
||||
def get_pbf_directory() -> Path:
|
||||
pbf_directory = get_map_directory() / 'pbf'
|
||||
if not pbf_directory.is_dir():
|
||||
pbf_directory.mkdir(parents=True)
|
||||
return pbf_directory
|
||||
map_directory = get_media_directory() / 'map'
|
||||
if not map_directory.is_dir():
|
||||
map_directory.mkdir()
|
||||
return map_directory
|
||||
|
||||
|
||||
def is_pbf_file(pbf: Path) -> bool:
|
||||
|
|
@ -42,11 +38,31 @@ def is_pbf_file(pbf: Path) -> bool:
|
|||
return 'OpenStreetMap Protocolbuffer Binary Format' in output.decode()
|
||||
|
||||
|
||||
def get_pbf_paths() -> List[Path]:
|
||||
"""Find all PBF files in the map/pbf directory"""
|
||||
pbf_directory = get_pbf_directory()
|
||||
paths = walk(pbf_directory)
|
||||
return list(filter(lambda i: i.is_file() and str(i).endswith('.osm.pbf') and is_pbf_file(i), paths))
|
||||
def is_dump_file(path: Path) -> bool:
|
||||
"""Uses file command to check type of a file. Returns True if a file is a Postgresql dump file."""
|
||||
cmd = ('/usr/bin/file', path)
|
||||
try:
|
||||
output = subprocess.check_output(cmd)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
return 'PostgreSQL custom database dump' in output.decode()
|
||||
|
||||
|
||||
def get_map_paths() -> List[Path]:
|
||||
"""Find all pbf/dump files in the map directory."""
|
||||
map_directory = get_map_directory()
|
||||
paths = walk(map_directory)
|
||||
|
||||
def is_valid(path: Path) -> bool:
|
||||
if path.is_file():
|
||||
if str(path).endswith('.osm.pbf') and is_pbf_file(path):
|
||||
return True
|
||||
elif path.suffix == '.dump' and is_dump_file(path):
|
||||
return True
|
||||
return False
|
||||
|
||||
return list(filter(is_valid, paths))
|
||||
|
||||
|
||||
def get_or_create_map_file(pbf_path: Path, session: Session) -> MapFile:
|
||||
|
|
@ -60,43 +76,47 @@ def get_or_create_map_file(pbf_path: Path, session: Session) -> MapFile:
|
|||
return map_file
|
||||
|
||||
|
||||
async def import_pbfs(pbfs: List[str]):
|
||||
async def import_files(files: List[str]):
|
||||
if IMPORT_EVENT.is_set():
|
||||
logger.warning('Map import already running...')
|
||||
return
|
||||
|
||||
logger.warning(f'Importing: {", ".join(pbfs)}')
|
||||
# Import dumps, then pbfs.
|
||||
import_order = ('.dump', '.pbf')
|
||||
files = [get_media_directory() / i for i in files]
|
||||
files = sorted(files, key=lambda i: import_order.index(i.suffix))
|
||||
|
||||
logger.warning(f'Importing: {", ".join(map(str, files))}')
|
||||
|
||||
any_success = False
|
||||
try:
|
||||
IMPORT_EVENT.set()
|
||||
for pbf in pbfs:
|
||||
pbf = get_media_directory() / pbf
|
||||
if not pbf.is_file():
|
||||
logger.fatal(f'PBF file does not exist! {pbf}')
|
||||
for path in files:
|
||||
if not path.is_file():
|
||||
logger.fatal(f'Map file does not exist! {path}')
|
||||
continue
|
||||
|
||||
with get_db_session() as session:
|
||||
map_file = session.query(MapFile).filter_by(path=pbf).one_or_none()
|
||||
map_file = session.query(MapFile).filter_by(path=path).one_or_none()
|
||||
if map_file and map_file.imported:
|
||||
# Don't import a map file twice.
|
||||
logger.debug(f'{pbf} is already imported')
|
||||
logger.debug(f'{path} is already imported')
|
||||
continue
|
||||
|
||||
success = False
|
||||
try:
|
||||
IMPORTING['pbf'] = str(pbf)
|
||||
await import_pbf(pbf)
|
||||
IMPORTING['path'] = str(path)
|
||||
await import_file(path)
|
||||
success = True
|
||||
any_success = True
|
||||
except Exception as e:
|
||||
logger.warning('Failed to run import_pbf', exc_info=e)
|
||||
logger.warning('Failed to run import', exc_info=e)
|
||||
finally:
|
||||
IMPORTING['pbf'] = None
|
||||
IMPORTING['path'] = None
|
||||
|
||||
if success:
|
||||
with get_db_session(commit=True) as session:
|
||||
map_file = get_or_create_map_file(pbf, session)
|
||||
map_file = get_or_create_map_file(path, session)
|
||||
map_file.imported = True
|
||||
finally:
|
||||
if any_success:
|
||||
|
|
@ -108,9 +128,22 @@ async def import_pbfs(pbfs: List[str]):
|
|||
BASH = which('bash', '/bin/bash', warn=True)
|
||||
|
||||
|
||||
async def import_pbf(pbf: Path):
|
||||
"""Run the osm2pgsql binary to import a PBF map file."""
|
||||
cmd = f'{BASH} {PROJECT_DIR}/scripts/import_map.sh {pbf.absolute()}'
|
||||
async def import_file(path: Path):
|
||||
"""Run the map import script on the provided path.
|
||||
|
||||
Supports *.osm.pbf and *.dump files."""
|
||||
if str(path).endswith('.osm.pbf'):
|
||||
if not is_pbf_file(path):
|
||||
logger.warning(f'Could not import non-pbf file: {path}')
|
||||
raise ValueError('Invalid PBF file')
|
||||
elif path.suffix == '.dump':
|
||||
if not is_dump_file(path):
|
||||
logger.warning(f'Could not import non-dump file: {path}')
|
||||
raise ValueError('Invalid dump file')
|
||||
else:
|
||||
raise ValueError(f'Cannot import unknown file! {path}')
|
||||
|
||||
cmd = f'{BASH} {PROJECT_DIR}/scripts/import_map.sh {path.absolute()}'
|
||||
logger.debug(f'Running import script: {cmd}')
|
||||
start = now()
|
||||
proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)
|
||||
|
|
@ -118,26 +151,26 @@ async def import_pbf(pbf: Path):
|
|||
stdout, stderr = await proc.communicate()
|
||||
elapsed = now() - start
|
||||
success = 'Successful' if proc.returncode == 0 else 'Unsuccessful'
|
||||
logger.info(f'{success} import of {pbf.name} took {timedelta_to_timestamp(elapsed)}')
|
||||
logger.info(f'{success} import of {path.name} took {timedelta_to_timestamp(elapsed)}')
|
||||
if proc.returncode != 0:
|
||||
# Log all lines. Truncate long lines.
|
||||
for line in stdout.decode().splitlines():
|
||||
logger.debug(line[:500])
|
||||
for line in stderr.decode().splitlines():
|
||||
logger.error(line[:500])
|
||||
raise ValueError(f'Importing PBF failed with return code {proc.returncode}')
|
||||
raise ValueError(f'Importing map file failed with return code {proc.returncode}')
|
||||
|
||||
|
||||
@optional_session
|
||||
def get_pbf_import_status(session: Session = None):
|
||||
pbf_paths: List[Path] = get_pbf_paths()
|
||||
|
||||
pbfs = []
|
||||
for path in pbf_paths:
|
||||
def get_import_status(session: Session = None) -> List[MapFile]:
|
||||
paths = get_map_paths()
|
||||
map_paths = []
|
||||
for path in paths:
|
||||
map_file = get_or_create_map_file(path, session)
|
||||
pbfs.append(map_file)
|
||||
map_paths.append(map_file)
|
||||
|
||||
session.commit()
|
||||
return pbfs
|
||||
return map_paths
|
||||
|
||||
|
||||
MOD_TILE_CACHE_DIR = Path('/var/lib/mod_tile/ajt')
|
||||
|
|
|
|||
|
|
@ -1,15 +1,21 @@
|
|||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import Column, Integer, Boolean, BigInteger
|
||||
|
||||
from wrolpi import dates
|
||||
from wrolpi.common import Base, ModelHelper
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
||||
RPI4_BYTES_PER_SECOND = 3963
|
||||
RPI4_PBF_BYTES_PER_SECOND = 3963
|
||||
|
||||
|
||||
def seconds_to_import(size: int) -> int:
|
||||
def seconds_to_import(path: Path, size: int) -> int:
|
||||
"""Attempt to predict how long it will take an RPi4 to import a given PBF file."""
|
||||
return max(int(size // RPI4_BYTES_PER_SECOND), 0)
|
||||
if str(path).endswith('.osm.pbf'):
|
||||
return max(int(size // RPI4_PBF_BYTES_PER_SECOND), 0)
|
||||
|
||||
# Can't calculate an unknown file.
|
||||
return 0
|
||||
|
||||
|
||||
class MapFile(Base, ModelHelper):
|
||||
|
|
@ -20,11 +26,14 @@ class MapFile(Base, ModelHelper):
|
|||
imported = Column(Boolean, default=False, nullable=False)
|
||||
size = Column(BigInteger)
|
||||
|
||||
def __repr__(self):
|
||||
return f'<MapFile {self.path} imported={self.imported}>'
|
||||
|
||||
def __json__(self):
|
||||
return {
|
||||
'id': self.id,
|
||||
'path': self.path,
|
||||
'imported': self.imported,
|
||||
'size': self.size,
|
||||
'time_to_import': dates.seconds_to_timestamp(seconds_to_import(self.size)),
|
||||
'time_to_import': dates.seconds_to_timestamp(seconds_to_import(self.path.path, self.size)),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,4 +4,4 @@ from typing import List
|
|||
|
||||
@dataclass
|
||||
class ImportPost:
|
||||
pbfs: List[str]
|
||||
files: List[str]
|
||||
|
|
|
|||
|
|
@ -6,54 +6,77 @@ import mock
|
|||
|
||||
|
||||
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
|
||||
def test_pbf_status_and_import(test_client, test_session, test_directory):
|
||||
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
|
||||
def test_status_and_import(test_client, test_session, test_directory):
|
||||
"""PBF files can be imported, and the status of the import can be monitored."""
|
||||
(test_directory / 'map/pbf').mkdir(parents=True)
|
||||
(test_directory / 'map/dump').mkdir()
|
||||
pbf1 = test_directory / 'map/pbf/some-country.osm.pbf'
|
||||
pbf2 = test_directory / 'map/pbf/other-country.osm.pbf'
|
||||
pbf1.touch()
|
||||
pbf2.touch()
|
||||
dump1 = test_directory / 'map/dump/dumpy.dump'
|
||||
dump1.touch()
|
||||
|
||||
def check_status(response_json, expected_):
|
||||
for pbf, expected_ in zip_longest(response_json['pbfs'], expected_):
|
||||
for pbf, expected_ in zip_longest(response_json['files'], expected_):
|
||||
assert pbf['imported'] == expected_['imported']
|
||||
assert pbf['path'].endswith(expected_['path'])
|
||||
assert response.json['importing'] is None
|
||||
|
||||
# The PBF files can be found, but have not yet been imported.
|
||||
request, response = test_client.get('/api/map/pbf')
|
||||
request, response = test_client.get('/api/map/files')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
expected = [
|
||||
{'imported': False, 'path': 'map/dump/dumpy.dump'},
|
||||
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
|
||||
{'imported': False, 'path': 'map/pbf/some-country.osm.pbf'},
|
||||
]
|
||||
check_status(response.json, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.import_pbf') as mock_import_pbf:
|
||||
body = {'pbfs': [str(pbf1), ]}
|
||||
with mock.patch('modules.map.lib.import_file') as mock_import_file:
|
||||
body = {'files': [str(pbf1), ], }
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
# Only one import is requested.
|
||||
mock_import_pbf.assert_called_once()
|
||||
# Only one PBF import is requested.
|
||||
mock_import_file.assert_called_once()
|
||||
|
||||
request, response = test_client.get('/api/map/pbf')
|
||||
request, response = test_client.get('/api/map/files')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
expected = [
|
||||
{'imported': False, 'path': 'map/dump/dumpy.dump'},
|
||||
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
|
||||
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
|
||||
]
|
||||
check_status(response.json, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.import_pbf') as mock_import_pbf:
|
||||
body = {'pbfs': [str(pbf1), str(pbf2)]}
|
||||
with mock.patch('modules.map.lib.import_file') as mock_import_file:
|
||||
body = {'files': [str(pbf1), str(pbf2)]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
# Both imports are requested, but only one is performed.
|
||||
mock_import_pbf.assert_called_once()
|
||||
# Both PBF imports are requested, but only one is performed.
|
||||
mock_import_file.assert_called_once()
|
||||
|
||||
request, response = test_client.get('/api/map/pbf')
|
||||
request, response = test_client.get('/api/map/files')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
expected = [
|
||||
{'imported': False, 'path': 'map/dump/dumpy.dump'},
|
||||
{'imported': True, 'path': 'map/pbf/other-country.osm.pbf'},
|
||||
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
|
||||
]
|
||||
check_status(response.json, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.import_file') as mock_import_file:
|
||||
body = {'files': [str(dump1), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
# Dump is imported.
|
||||
mock_import_file.assert_called_once()
|
||||
|
||||
request, response = test_client.get('/api/map/files')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
expected = [
|
||||
{'imported': True, 'path': 'map/dump/dumpy.dump'},
|
||||
{'imported': True, 'path': 'map/pbf/other-country.osm.pbf'},
|
||||
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
|
||||
]
|
||||
|
|
@ -62,8 +85,18 @@ def test_pbf_status_and_import(test_client, test_session, test_directory):
|
|||
|
||||
def test_empty_import(test_client, test_session, test_directory):
|
||||
"""Some files must be requested."""
|
||||
with mock.patch('modules.map.lib.import_pbf') as mock_import_pbf:
|
||||
with mock.patch('modules.map.lib.import_file') as mock_import_file:
|
||||
body = {'pbfs': []}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
mock_import_pbf.assert_not_called()
|
||||
mock_import_file.assert_not_called()
|
||||
|
||||
body = {'dumps': []}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
mock_import_file.assert_not_called()
|
||||
|
||||
body = {'pbfs': [], 'dumps': []}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
mock_import_file.assert_not_called()
|
||||
|
|
|
|||
|
|
@ -1,16 +1,18 @@
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from modules.map import models
|
||||
|
||||
|
||||
@pytest.mark.parametrize('size,expected', [
|
||||
(0, 0),
|
||||
(-1, 0),
|
||||
(17737381, 4475),
|
||||
(63434267, 16006),
|
||||
(87745484, 22141),
|
||||
(136372996, 34411),
|
||||
(116318111, 29351),
|
||||
@pytest.mark.parametrize('path,size,expected', [
|
||||
('file.osm.pbf', 0, 0),
|
||||
('file.osm.pbf', -1, 0),
|
||||
('file.osm.pbf', 17737381, 4475),
|
||||
('file.osm.pbf', 63434267, 16006),
|
||||
('file.osm.pbf', 87745484, 22141),
|
||||
('file.osm.pbf', 136372996, 34411),
|
||||
('file.osm.pbf', 116318111, 29351),
|
||||
])
|
||||
def test_seconds_to_import_rpi4(size, expected):
|
||||
assert models.seconds_to_import(size) == expected
|
||||
def test_seconds_to_import_rpi4(path, size, expected):
|
||||
assert models.seconds_to_import(Path(path), size) == expected
|
||||
|
|
|
|||
|
|
@ -11,15 +11,24 @@ if [ ! -f /opt/openstreetmap-carto/openstreetmap-carto.style ]; then
|
|||
fi
|
||||
|
||||
if [ "${1}" == "" ]; then
|
||||
echo "Missing PBF file argument."
|
||||
echo "Missing file argument."
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if [ ! -f "${1}" ]; then
|
||||
echo "PBF file does not exist! ${1}"
|
||||
echo "File does not exist! ${1}"
|
||||
exit 4
|
||||
fi
|
||||
|
||||
osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \
|
||||
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \
|
||||
-S /opt/openstreetmap-carto/openstreetmap-carto.style "${1}"
|
||||
if [[ ${1} == *.osm.pbf ]]; then
|
||||
# Import a PBF file.
|
||||
osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \
|
||||
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \
|
||||
-S /opt/openstreetmap-carto/openstreetmap-carto.style "${1}"
|
||||
elif [[ ${1} == *.dump ]]; then
|
||||
# Import a Postgresql dump.
|
||||
pg_restore -j4 --no-owner -d gis "${1}"
|
||||
else
|
||||
echo "Cannot import unknown map file"
|
||||
exit 5
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ fi
|
|||
|
||||
set -e
|
||||
|
||||
cp /opt/wrolpi/etc/ubuntu20.04/rendred.conf /usr/local/etc/renderd.conf
|
||||
cp /opt/wrolpi/etc/ubuntu20.04/renderd.conf /usr/local/etc/renderd.conf
|
||||
# Enable mod_tile.
|
||||
cp /opt/wrolpi/etc/ubuntu20.04/mod_tile.conf /etc/apache2/conf-available/mod_tile.conf
|
||||
|
||||
|
|
|
|||
|
|
@ -55,9 +55,7 @@ def utc_now() -> datetime:
|
|||
|
||||
|
||||
def now(tz: pytz.timezone = None) -> datetime:
|
||||
"""
|
||||
Get the current DateTime in the provided timezone. Timezone aware.
|
||||
"""
|
||||
"""Get the current DateTime in the provided timezone. Timezone aware."""
|
||||
if TEST_DATETIME:
|
||||
return TEST_DATETIME
|
||||
tz = tz or DEFAULT_TIMEZONE
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue