Calculating time-to-import for map pbf files in the backend. These times were created using a test RPi4. Encoding URL of archive files to prevent error which blocked displaying of archives.

This commit is contained in:
Roland 2022-03-12 09:50:16 -07:00
parent 6a22090e54
commit c815aa68c6
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
10 changed files with 46 additions and 59 deletions

View file

@ -3,7 +3,6 @@ import {Card, Confirm, Container, Icon, Image, Placeholder} from "semantic-ui-re
import {
CardGroupCentered,
ClearButton,
ExternalLink,
PageContainer,
Paginator,
SearchInput,
@ -20,44 +19,6 @@ import Table from "semantic-ui-react/dist/commonjs/collections/Table";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useArchives, useDomains} from "../hooks/customHooks";
function FailedArchiveCard({archive, syncArchive, deleteArchive}) {
let syncIcon = (
<Button onClick={syncArchive}>
<Icon name='sync' size='big'/>
</Button>
);
let trashIcon = (
<Button onClick={deleteArchive}>
<Icon name='trash' size='big'/>
</Button>
);
let externalIcon = (
<a href={archive.url} target='_blank' rel='noopener noreferrer'>
<Icon name='sign-out' size='big'/>
</a>
);
return (
<Card>
<Card.Content>
<Card.Header>
{archive.url}
</Card.Header>
<Card.Description>
<p>Failed!</p>
{syncIcon}
{trashIcon}
{externalIcon}
</Card.Description>
</Card.Content>
</Card>
);
}
function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) {
const [syncOpen, setSyncOpen] = useState(false);
const [deleteOpen, setDeleteOpen] = useState(false);
@ -71,14 +32,10 @@ function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) {
await deleteArchive(archive.id);
}
let imageSrc = archive.screenshot_path ? `/media/${archive.screenshot_path}` : null;
let singlefileUrl = archive.singlefile_path ? `/media/${archive.singlefile_path}` : null;
let imageSrc = archive.screenshot_path ? `/media/${encodeURIComponent(archive.screenshot_path)}` : null;
let singlefileUrl = archive.singlefile_path ? `/media/${encodeURIComponent(archive.singlefile_path)}` : null;
if (archive.status === 'failed') {
return <FailedArchiveCard archive={archive} syncArchive={localSyncArchive} deleteArchive={localDeleteArchive}/>;
}
let readabilityUrl = archive.readability_path ? `/media/${archive.readability_path}` : null;
let readabilityUrl = archive.readability_path ? `/media/${encodeURIComponent(archive.readability_path)}` : null;
let readabilityIcon = <Button icon disabled><Icon name='book' size='large'/></Button>;
if (readabilityUrl) {
readabilityIcon = (
@ -130,15 +87,15 @@ function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) {
return (
<Card>
<ExternalLink to={singlefileUrl} className='no-link-underscore card-link'>
<a href={singlefileUrl} target='_blank' className='no-link-underscore card-link'>
<Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}}/>
</ExternalLink>
</a>
<Card.Content>
<Card.Header>
<Container textAlign='left'>
<ExternalLink to={singlefileUrl} className='no-link-underscore card-link'>
<a href={singlefileUrl} target='_blank' className='no-link-underscore card-link'>
<p>{textEllipsis(archive.title || archive.url, 100)}</p>
</ExternalLink>
</a>
</Container>
</Card.Header>
<a onClick={() => setDomain(domain)} className="no-link-underscore card-link">

View file

@ -70,12 +70,11 @@ class ManageMap extends React.Component {
tableRow = (pbf, disabled) => {
let ref = React.createRef();
const {size, path, imported} = pbf;
const raspberryPi4Estimation = 3000000;
const {size, path, imported, time_to_import} = pbf;
let sizeCells = (<>
<Table.Cell>{humanFileSize(size)}</Table.Cell>
<Table.Cell>{humanFileSize(size * 25)}</Table.Cell>
<Table.Cell>{minutesToTimestamp(size / raspberryPi4Estimation)}</Table.Cell>
<Table.Cell>{time_to_import}</Table.Cell>
</>);
return <Table.Row key={path}>
<Table.Cell collapsing>

View file

@ -113,12 +113,14 @@ async def import_pbf(pbf: Path):
stdout, stderr = await proc.communicate()
elapsed = now() - start
logger.info(f'Import of {pbf} took {timedelta_to_timestamp(elapsed)}')
success = 'Successful' if proc.returncode == 0 else 'Unsuccessful'
logger.info(f'{success} import of {pbf.name} took {timedelta_to_timestamp(elapsed)}')
if proc.returncode != 0:
# Log all lines. Truncate long lines.
for line in stdout.decode().splitlines():
logger.info(line)
logger.debug(line[:500])
for line in stderr.decode().splitlines():
logger.error(line)
logger.error(line[:500])
raise ValueError(f'Importing PBF failed with return code {proc.returncode}')

View file

@ -1,8 +1,16 @@
from sqlalchemy import Column, Integer, Boolean, BigInteger
from wrolpi import dates
from wrolpi.common import Base, ModelHelper
from wrolpi.media_path import MediaPathType
RPI4_BYTES_PER_SECOND = 3963
def seconds_to_import(size: int) -> int:
"""Attempt to predict how long it will take an RPi4 to import a given PBF file."""
return max(int(size // RPI4_BYTES_PER_SECOND), 0)
class MapFile(Base, ModelHelper):
__tablename__ = 'map_file'
@ -18,4 +26,5 @@ class MapFile(Base, ModelHelper):
'path': self.path,
'imported': self.imported,
'size': self.size,
'time_to_import': dates.seconds_to_timestamp(seconds_to_import(self.size)),
}

View file

@ -0,0 +1,16 @@
import pytest
from modules.map import models
@pytest.mark.parametrize('size,expected', [
(0, 0),
(-1, 0),
(17737381, 4475),
(63434267, 16006),
(87745484, 22141),
(136372996, 34411),
(116318111, 29351),
])
def test_seconds_to_import_rpi4(size, expected):
assert models.seconds_to_import(size) == expected

View file

@ -713,7 +713,7 @@ def walk(path: Path) -> Generator[Path, None, None]:
# These characters are invalid in Windows or Linux.
INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*]')
INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*%]')
def escape_file_name(name: str) -> str:

View file

@ -1,6 +1,7 @@
import logging
from datetime import datetime, date, timezone, timedelta
from enum import Enum
from typing import Union
import pytz
from sqlalchemy import types
@ -97,8 +98,9 @@ def from_timestamp(timestamp: float) -> datetime:
return local_timezone(datetime.fromtimestamp(timestamp))
def seconds_to_timestamp(seconds: int) -> str:
def seconds_to_timestamp(seconds: Union[int, float]) -> str:
"""Convert an integer into a timestamp string."""
seconds = int(seconds)
weeks = seconds // Seconds.week
seconds = seconds % Seconds.week
days = seconds // Seconds.day
@ -125,7 +127,7 @@ def timedelta_to_timestamp(delta: timedelta) -> str:
>>> timedelta_to_timestamp(timedelta(days=10))
'1w 3d 00:00:00'
"""
return seconds_to_timestamp(int(delta.total_seconds()))
return seconds_to_timestamp(delta.total_seconds())
def recursive_replace_tz(obj, tz=pytz.utc):

View file

@ -291,6 +291,7 @@ def test_zig_zag(low, high, expected):
('foo?', 'foo'),
('foo*', 'foo'),
('foo&', 'foo&'),
('foo%', 'foo'),
]
)
def test_escape_file_name(name, expected):

View file

@ -74,6 +74,7 @@ class TestDates(TestAPI):
(timedelta(weeks=1, days=1, seconds=7), '1w 1d 00:00:07'),
(timedelta(weeks=16, days=3, hours=17, minutes=46, seconds=39), '16w 3d 17:46:39'),
(timedelta(days=10), '1w 3d 00:00:00'),
(timedelta(days=10, microseconds=1000), '1w 3d 00:00:00'),
])
def test_timedelta_to_timestamp(td, expected):
assert timedelta_to_timestamp(td) == expected