mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
Creating new /api/archive/upload which accepts a file from the SingleFile browser extension and converts it to an Archive.
This commit is contained in:
parent
01338e4d89
commit
531ec3d65e
62 changed files with 1000 additions and 562 deletions
|
|
@ -1,5 +1,5 @@
|
|||
import asyncio
|
||||
import subprocess
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
|
|
@ -8,9 +8,9 @@ from sqlalchemy.orm import Session
|
|||
from modules.map.models import MapFile
|
||||
from wrolpi import flags
|
||||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.cmd import SUDO_BIN
|
||||
from wrolpi.cmd import SUDO_BIN, run_command
|
||||
from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config
|
||||
from wrolpi.dates import now, timedelta_to_timestamp, seconds_to_timestamp
|
||||
from wrolpi.dates import timedelta_to_timestamp, seconds_to_timestamp
|
||||
from wrolpi.db import optional_session, get_db_session
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.vars import PROJECT_DIR, IS_RPI5
|
||||
|
|
@ -37,7 +37,7 @@ def is_pbf_file(pbf: Path) -> bool:
|
|||
|
||||
|
||||
def is_dump_file(path: Path) -> bool:
|
||||
"""Uses file command to check type of a file. Returns True if a file is a Postgresql dump file."""
|
||||
"""Uses file command to check the type of file. Returns True if a file is a Postgresql dump file."""
|
||||
cmd = ('/usr/bin/file', path)
|
||||
try:
|
||||
output = subprocess.check_output(cmd)
|
||||
|
|
@ -188,24 +188,22 @@ async def run_import_command(*paths: Path) -> int:
|
|||
|
||||
paths = ' '.join(str(i) for i in paths)
|
||||
# Run with sudo so renderd can be restarted.
|
||||
cmd = f'{SUDO_BIN} {PROJECT_DIR}/scripts/import_map.sh {paths}'
|
||||
cmd = (SUDO_BIN, f'{PROJECT_DIR}/scripts/import_map.sh', paths)
|
||||
import_logger.warning(f'Running map import command: {cmd}')
|
||||
start = now()
|
||||
proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)
|
||||
result = await run_command(cmd, debug=True)
|
||||
|
||||
stdout, stderr = await proc.communicate()
|
||||
elapsed = now() - start
|
||||
success = 'Successful' if proc.returncode == 0 else 'Unsuccessful'
|
||||
import_logger.info(f'{success} import of {repr(paths)} took {timedelta_to_timestamp(elapsed)}')
|
||||
if proc.returncode != 0:
|
||||
success = 'Successful' if result.return_code == 0 else 'Unsuccessful'
|
||||
import_logger.info(
|
||||
f'{success} import of {repr(paths)} took {timedelta_to_timestamp(timedelta(seconds=result.elapsed))}')
|
||||
if result.return_code != 0:
|
||||
# Log all lines. Truncate long lines.
|
||||
for line in stdout.decode().splitlines():
|
||||
for line in result.stdout.decode().splitlines():
|
||||
import_logger.debug(line[:500])
|
||||
for line in stderr.decode().splitlines():
|
||||
for line in result.stderr.decode().splitlines():
|
||||
import_logger.error(line[:500])
|
||||
raise ValueError(f'Importing map file failed with return code {proc.returncode}')
|
||||
raise ValueError(f'Importing map file failed with return code {result.return_code}')
|
||||
|
||||
return int(elapsed.total_seconds())
|
||||
return result.elapsed
|
||||
|
||||
|
||||
@optional_session
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ def assert_map_file_status(test_client, expected, status_code=HTTPStatus.OK):
|
|||
|
||||
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
|
||||
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
|
||||
def test_status_and_import(test_client, test_session, make_files_structure, mock_create_subprocess_shell):
|
||||
def test_status_and_import(test_client, test_session, make_files_structure, mock_run_command):
|
||||
"""PBF files can be imported, and the status of the import can be monitored."""
|
||||
pbf1, pbf2, dump1 = make_files_structure([
|
||||
'map/pbf/some-country.osm.pbf',
|
||||
|
|
@ -32,11 +32,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
|
|||
]
|
||||
assert_map_file_status(test_client, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
body = {'files': [str(pbf1), ], }
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
body = {'files': [str(pbf1), ], }
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
expected = [
|
||||
{'imported': False, 'path': 'map/dump/dumpy.dump'},
|
||||
|
|
@ -45,11 +43,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
|
|||
]
|
||||
assert_map_file_status(test_client, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
body = {'files': [str(pbf1), str(pbf2)]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
body = {'files': [str(pbf1), str(pbf2)]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
expected = [
|
||||
{'imported': False, 'path': 'map/dump/dumpy.dump'},
|
||||
|
|
@ -58,11 +54,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
|
|||
]
|
||||
assert_map_file_status(test_client, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
body = {'files': [str(dump1), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
body = {'files': [str(dump1), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
expected = [
|
||||
{'imported': True, 'path': 'map/dump/dumpy.dump'},
|
||||
|
|
@ -74,7 +68,7 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
|
|||
|
||||
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
|
||||
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
|
||||
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_create_subprocess_shell):
|
||||
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_run_command):
|
||||
"""Multiple PBFs can be imported. Importing a second time overwrites the previous imports."""
|
||||
pbf1, pbf2, pbf3, dump = make_files_structure([
|
||||
'map/pbf/country1.osm.pbf',
|
||||
|
|
@ -83,11 +77,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
|
|||
'map/dumpy.dump',
|
||||
])
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
body = {'files': [str(pbf2), str(pbf1)]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
body = {'files': [str(pbf2), str(pbf1)]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Two PBFs can be merged and imported.
|
||||
expected = [
|
||||
|
|
@ -98,11 +90,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
|
|||
]
|
||||
assert_map_file_status(test_client, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
body = {'files': [str(pbf3), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
body = {'files': [str(pbf3), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# New imports overwrite old imports.
|
||||
expected = [
|
||||
|
|
@ -113,12 +103,10 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
|
|||
]
|
||||
assert_map_file_status(test_client, expected)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
|
||||
# Can't mix PBF and dump.
|
||||
body = {'files': [str(dump), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
# Can't mix PBF and dump.
|
||||
body = {'files': [str(dump), ]}
|
||||
request, response = test_client.post('/api/map/import', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Dump does not overwrite.
|
||||
expected = [
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ def test_get_map_paths(test_directory, make_files_structure):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_import_command(test_directory, mock_create_subprocess_shell):
|
||||
async def test_run_import_command(test_directory, mock_run_command, run_command_bad_result):
|
||||
"""
|
||||
Import map files. Files to import are checked for validity before importing. Any errors returned by
|
||||
asyncio.create_subprocess_shell are caught and reported.
|
||||
|
|
@ -38,22 +38,14 @@ async def test_run_import_command(test_directory, mock_create_subprocess_shell):
|
|||
dump_file = test_directory / 'dumpy.dump'
|
||||
dump_file.touch()
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \
|
||||
mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
|
||||
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
|
||||
# Run the import, it succeeds.
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(
|
||||
communicate_return=(b'out', b'error')
|
||||
)
|
||||
mock_is_pbf_file.return_value = True
|
||||
await lib.run_import_command(pbf_file)
|
||||
|
||||
with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \
|
||||
mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
|
||||
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
|
||||
# Run the import, it fails.
|
||||
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(
|
||||
communicate_return=(b'out', b'error'),
|
||||
return_code=1,
|
||||
)
|
||||
mock_run_command(run_command_bad_result)
|
||||
mock_is_pbf_file.return_value = True
|
||||
with pytest.raises(ValueError):
|
||||
await lib.run_import_command(pbf_file)
|
||||
|
|
@ -102,7 +94,7 @@ def test_seconds_to_import_rpi5(size, expected):
|
|||
assert lib.seconds_to_import(size, True) == expected
|
||||
|
||||
|
||||
def test_get_custom_map_directory(test_directory, test_wrolpi_config):
|
||||
def test_get_custom_map_directory(async_client, test_directory, test_wrolpi_config):
|
||||
"""Custom directory can be used for map directory."""
|
||||
# Default location.
|
||||
assert lib.get_map_directory() == (test_directory / 'map')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue