Creating new /api/archive/upload which accepts a file from the SingleFile browser extension and converts it to an Archive.

This commit is contained in:
Roland 2025-01-26 12:44:16 -07:00
parent 01338e4d89
commit 531ec3d65e
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
62 changed files with 1000 additions and 562 deletions

View file

@ -1,5 +1,5 @@
import asyncio
import subprocess
from datetime import timedelta
from pathlib import Path
from typing import List
@ -8,9 +8,9 @@ from sqlalchemy.orm import Session
from modules.map.models import MapFile
from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.cmd import SUDO_BIN
from wrolpi.cmd import SUDO_BIN, run_command
from wrolpi.common import get_media_directory, walk, logger, get_wrolpi_config
from wrolpi.dates import now, timedelta_to_timestamp, seconds_to_timestamp
from wrolpi.dates import timedelta_to_timestamp, seconds_to_timestamp
from wrolpi.db import optional_session, get_db_session
from wrolpi.events import Events
from wrolpi.vars import PROJECT_DIR, IS_RPI5
@ -37,7 +37,7 @@ def is_pbf_file(pbf: Path) -> bool:
def is_dump_file(path: Path) -> bool:
"""Uses file command to check type of a file. Returns True if a file is a Postgresql dump file."""
"""Uses file command to check the type of file. Returns True if a file is a Postgresql dump file."""
cmd = ('/usr/bin/file', path)
try:
output = subprocess.check_output(cmd)
@ -188,24 +188,22 @@ async def run_import_command(*paths: Path) -> int:
paths = ' '.join(str(i) for i in paths)
# Run with sudo so renderd can be restarted.
cmd = f'{SUDO_BIN} {PROJECT_DIR}/scripts/import_map.sh {paths}'
cmd = (SUDO_BIN, f'{PROJECT_DIR}/scripts/import_map.sh', paths)
import_logger.warning(f'Running map import command: {cmd}')
start = now()
proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)
result = await run_command(cmd, debug=True)
stdout, stderr = await proc.communicate()
elapsed = now() - start
success = 'Successful' if proc.returncode == 0 else 'Unsuccessful'
import_logger.info(f'{success} import of {repr(paths)} took {timedelta_to_timestamp(elapsed)}')
if proc.returncode != 0:
success = 'Successful' if result.return_code == 0 else 'Unsuccessful'
import_logger.info(
f'{success} import of {repr(paths)} took {timedelta_to_timestamp(timedelta(seconds=result.elapsed))}')
if result.return_code != 0:
# Log all lines. Truncate long lines.
for line in stdout.decode().splitlines():
for line in result.stdout.decode().splitlines():
import_logger.debug(line[:500])
for line in stderr.decode().splitlines():
for line in result.stderr.decode().splitlines():
import_logger.error(line[:500])
raise ValueError(f'Importing map file failed with return code {proc.returncode}')
raise ValueError(f'Importing map file failed with return code {result.return_code}')
return int(elapsed.total_seconds())
return result.elapsed
@optional_session

View file

@ -16,7 +16,7 @@ def assert_map_file_status(test_client, expected, status_code=HTTPStatus.OK):
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
def test_status_and_import(test_client, test_session, make_files_structure, mock_create_subprocess_shell):
def test_status_and_import(test_client, test_session, make_files_structure, mock_run_command):
"""PBF files can be imported, and the status of the import can be monitored."""
pbf1, pbf2, dump1 = make_files_structure([
'map/pbf/some-country.osm.pbf',
@ -32,11 +32,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf1), ], }
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf1), ], }
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
@ -45,11 +43,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf1), str(pbf2)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf1), str(pbf2)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
expected = [
{'imported': False, 'path': 'map/dump/dumpy.dump'},
@ -58,11 +54,9 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(dump1), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(dump1), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
expected = [
{'imported': True, 'path': 'map/dump/dumpy.dump'},
@ -74,7 +68,7 @@ def test_status_and_import(test_client, test_session, make_files_structure, mock
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
@mock.patch('modules.map.lib.is_dump_file', lambda i: True)
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_create_subprocess_shell):
def test_multiple_import(test_client, test_session, test_directory, make_files_structure, mock_run_command):
"""Multiple PBFs can be imported. Importing a second time overwrites the previous imports."""
pbf1, pbf2, pbf3, dump = make_files_structure([
'map/pbf/country1.osm.pbf',
@ -83,11 +77,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
'map/dumpy.dump',
])
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf2), str(pbf1)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf2), str(pbf1)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Two PBFs can be merged and imported.
expected = [
@ -98,11 +90,9 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
body = {'files': [str(pbf3), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
body = {'files': [str(pbf3), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# New imports overwrite old imports.
expected = [
@ -113,12 +103,10 @@ def test_multiple_import(test_client, test_session, test_directory, make_files_s
]
assert_map_file_status(test_client, expected)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio:
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(return_code=0)
# Can't mix PBF and dump.
body = {'files': [str(dump), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Can't mix PBF and dump.
body = {'files': [str(dump), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Dump does not overwrite.
expected = [

View file

@ -22,7 +22,7 @@ def test_get_map_paths(test_directory, make_files_structure):
@pytest.mark.asyncio
async def test_run_import_command(test_directory, mock_create_subprocess_shell):
async def test_run_import_command(test_directory, mock_run_command, run_command_bad_result):
"""
Import map files. Files to import are checked for validity before importing. Any errors returned by
asyncio.create_subprocess_shell are caught and reported.
@ -38,22 +38,14 @@ async def test_run_import_command(test_directory, mock_create_subprocess_shell):
dump_file = test_directory / 'dumpy.dump'
dump_file.touch()
with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \
mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
# Run the import, it succeeds.
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(
communicate_return=(b'out', b'error')
)
mock_is_pbf_file.return_value = True
await lib.run_import_command(pbf_file)
with mock.patch('modules.map.lib.asyncio') as mock_asyncio, \
mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
with mock.patch('modules.map.lib.is_pbf_file') as mock_is_pbf_file:
# Run the import, it fails.
mock_asyncio.create_subprocess_shell = mock_create_subprocess_shell(
communicate_return=(b'out', b'error'),
return_code=1,
)
mock_run_command(run_command_bad_result)
mock_is_pbf_file.return_value = True
with pytest.raises(ValueError):
await lib.run_import_command(pbf_file)
@ -102,7 +94,7 @@ def test_seconds_to_import_rpi5(size, expected):
assert lib.seconds_to_import(size, True) == expected
def test_get_custom_map_directory(test_directory, test_wrolpi_config):
def test_get_custom_map_directory(async_client, test_directory, test_wrolpi_config):
"""Custom directory can be used for map directory."""
# Default location.
assert lib.get_map_directory() == (test_directory / 'map')