mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Calculating time-to-import for map pbf files in the backend. These times were created using a test RPi4. Encoding URL of archive files to prevent error which blocked displaying of archives.
This commit is contained in:
parent
6a22090e54
commit
c815aa68c6
10 changed files with 46 additions and 59 deletions
|
|
@ -3,7 +3,6 @@ import {Card, Confirm, Container, Icon, Image, Placeholder} from "semantic-ui-re
|
|||
import {
|
||||
CardGroupCentered,
|
||||
ClearButton,
|
||||
ExternalLink,
|
||||
PageContainer,
|
||||
Paginator,
|
||||
SearchInput,
|
||||
|
|
@ -20,44 +19,6 @@ import Table from "semantic-ui-react/dist/commonjs/collections/Table";
|
|||
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
|
||||
import {useArchives, useDomains} from "../hooks/customHooks";
|
||||
|
||||
|
||||
function FailedArchiveCard({archive, syncArchive, deleteArchive}) {
|
||||
|
||||
let syncIcon = (
|
||||
<Button onClick={syncArchive}>
|
||||
<Icon name='sync' size='big'/>
|
||||
</Button>
|
||||
);
|
||||
|
||||
let trashIcon = (
|
||||
<Button onClick={deleteArchive}>
|
||||
<Icon name='trash' size='big'/>
|
||||
</Button>
|
||||
);
|
||||
|
||||
let externalIcon = (
|
||||
<a href={archive.url} target='_blank' rel='noopener noreferrer'>
|
||||
<Icon name='sign-out' size='big'/>
|
||||
</a>
|
||||
);
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<Card.Content>
|
||||
<Card.Header>
|
||||
{archive.url}
|
||||
</Card.Header>
|
||||
<Card.Description>
|
||||
<p>Failed!</p>
|
||||
{syncIcon}
|
||||
{trashIcon}
|
||||
{externalIcon}
|
||||
</Card.Description>
|
||||
</Card.Content>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) {
|
||||
const [syncOpen, setSyncOpen] = useState(false);
|
||||
const [deleteOpen, setDeleteOpen] = useState(false);
|
||||
|
|
@ -71,14 +32,10 @@ function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) {
|
|||
await deleteArchive(archive.id);
|
||||
}
|
||||
|
||||
let imageSrc = archive.screenshot_path ? `/media/${archive.screenshot_path}` : null;
|
||||
let singlefileUrl = archive.singlefile_path ? `/media/${archive.singlefile_path}` : null;
|
||||
let imageSrc = archive.screenshot_path ? `/media/${encodeURIComponent(archive.screenshot_path)}` : null;
|
||||
let singlefileUrl = archive.singlefile_path ? `/media/${encodeURIComponent(archive.singlefile_path)}` : null;
|
||||
|
||||
if (archive.status === 'failed') {
|
||||
return <FailedArchiveCard archive={archive} syncArchive={localSyncArchive} deleteArchive={localDeleteArchive}/>;
|
||||
}
|
||||
|
||||
let readabilityUrl = archive.readability_path ? `/media/${archive.readability_path}` : null;
|
||||
let readabilityUrl = archive.readability_path ? `/media/${encodeURIComponent(archive.readability_path)}` : null;
|
||||
let readabilityIcon = <Button icon disabled><Icon name='book' size='large'/></Button>;
|
||||
if (readabilityUrl) {
|
||||
readabilityIcon = (
|
||||
|
|
@ -130,15 +87,15 @@ function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) {
|
|||
|
||||
return (
|
||||
<Card>
|
||||
<ExternalLink to={singlefileUrl} className='no-link-underscore card-link'>
|
||||
<a href={singlefileUrl} target='_blank' className='no-link-underscore card-link'>
|
||||
<Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}}/>
|
||||
</ExternalLink>
|
||||
</a>
|
||||
<Card.Content>
|
||||
<Card.Header>
|
||||
<Container textAlign='left'>
|
||||
<ExternalLink to={singlefileUrl} className='no-link-underscore card-link'>
|
||||
<a href={singlefileUrl} target='_blank' className='no-link-underscore card-link'>
|
||||
<p>{textEllipsis(archive.title || archive.url, 100)}</p>
|
||||
</ExternalLink>
|
||||
</a>
|
||||
</Container>
|
||||
</Card.Header>
|
||||
<a onClick={() => setDomain(domain)} className="no-link-underscore card-link">
|
||||
|
|
|
|||
|
|
@ -70,12 +70,11 @@ class ManageMap extends React.Component {
|
|||
|
||||
tableRow = (pbf, disabled) => {
|
||||
let ref = React.createRef();
|
||||
const {size, path, imported} = pbf;
|
||||
const raspberryPi4Estimation = 3000000;
|
||||
const {size, path, imported, time_to_import} = pbf;
|
||||
let sizeCells = (<>
|
||||
<Table.Cell>{humanFileSize(size)}</Table.Cell>
|
||||
<Table.Cell>{humanFileSize(size * 25)}</Table.Cell>
|
||||
<Table.Cell>{minutesToTimestamp(size / raspberryPi4Estimation)}</Table.Cell>
|
||||
<Table.Cell>{time_to_import}</Table.Cell>
|
||||
</>);
|
||||
return <Table.Row key={path}>
|
||||
<Table.Cell collapsing>
|
||||
|
|
|
|||
|
|
@ -113,12 +113,14 @@ async def import_pbf(pbf: Path):
|
|||
|
||||
stdout, stderr = await proc.communicate()
|
||||
elapsed = now() - start
|
||||
logger.info(f'Import of {pbf} took {timedelta_to_timestamp(elapsed)}')
|
||||
success = 'Successful' if proc.returncode == 0 else 'Unsuccessful'
|
||||
logger.info(f'{success} import of {pbf.name} took {timedelta_to_timestamp(elapsed)}')
|
||||
if proc.returncode != 0:
|
||||
# Log all lines. Truncate long lines.
|
||||
for line in stdout.decode().splitlines():
|
||||
logger.info(line)
|
||||
logger.debug(line[:500])
|
||||
for line in stderr.decode().splitlines():
|
||||
logger.error(line)
|
||||
logger.error(line[:500])
|
||||
raise ValueError(f'Importing PBF failed with return code {proc.returncode}')
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,16 @@
|
|||
from sqlalchemy import Column, Integer, Boolean, BigInteger
|
||||
|
||||
from wrolpi import dates
|
||||
from wrolpi.common import Base, ModelHelper
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
||||
RPI4_BYTES_PER_SECOND = 3963
|
||||
|
||||
|
||||
def seconds_to_import(size: int) -> int:
|
||||
"""Attempt to predict how long it will take an RPi4 to import a given PBF file."""
|
||||
return max(int(size // RPI4_BYTES_PER_SECOND), 0)
|
||||
|
||||
|
||||
class MapFile(Base, ModelHelper):
|
||||
__tablename__ = 'map_file'
|
||||
|
|
@ -18,4 +26,5 @@ class MapFile(Base, ModelHelper):
|
|||
'path': self.path,
|
||||
'imported': self.imported,
|
||||
'size': self.size,
|
||||
'time_to_import': dates.seconds_to_timestamp(seconds_to_import(self.size)),
|
||||
}
|
||||
|
|
|
|||
16
modules/map/test/test_models.py
Normal file
16
modules/map/test/test_models.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
import pytest
|
||||
|
||||
from modules.map import models
|
||||
|
||||
|
||||
@pytest.mark.parametrize('size,expected', [
|
||||
(0, 0),
|
||||
(-1, 0),
|
||||
(17737381, 4475),
|
||||
(63434267, 16006),
|
||||
(87745484, 22141),
|
||||
(136372996, 34411),
|
||||
(116318111, 29351),
|
||||
])
|
||||
def test_seconds_to_import_rpi4(size, expected):
|
||||
assert models.seconds_to_import(size) == expected
|
||||
|
|
@ -713,7 +713,7 @@ def walk(path: Path) -> Generator[Path, None, None]:
|
|||
|
||||
|
||||
# These characters are invalid in Windows or Linux.
|
||||
INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*]')
|
||||
INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*%]')
|
||||
|
||||
|
||||
def escape_file_name(name: str) -> str:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import logging
|
||||
from datetime import datetime, date, timezone, timedelta
|
||||
from enum import Enum
|
||||
from typing import Union
|
||||
|
||||
import pytz
|
||||
from sqlalchemy import types
|
||||
|
|
@ -97,8 +98,9 @@ def from_timestamp(timestamp: float) -> datetime:
|
|||
return local_timezone(datetime.fromtimestamp(timestamp))
|
||||
|
||||
|
||||
def seconds_to_timestamp(seconds: int) -> str:
|
||||
def seconds_to_timestamp(seconds: Union[int, float]) -> str:
|
||||
"""Convert an integer into a timestamp string."""
|
||||
seconds = int(seconds)
|
||||
weeks = seconds // Seconds.week
|
||||
seconds = seconds % Seconds.week
|
||||
days = seconds // Seconds.day
|
||||
|
|
@ -125,7 +127,7 @@ def timedelta_to_timestamp(delta: timedelta) -> str:
|
|||
>>> timedelta_to_timestamp(timedelta(days=10))
|
||||
'1w 3d 00:00:00'
|
||||
"""
|
||||
return seconds_to_timestamp(int(delta.total_seconds()))
|
||||
return seconds_to_timestamp(delta.total_seconds())
|
||||
|
||||
|
||||
def recursive_replace_tz(obj, tz=pytz.utc):
|
||||
|
|
|
|||
|
|
@ -291,6 +291,7 @@ def test_zig_zag(low, high, expected):
|
|||
('foo?', 'foo'),
|
||||
('foo*', 'foo'),
|
||||
('foo&', 'foo&'),
|
||||
('foo%', 'foo'),
|
||||
]
|
||||
)
|
||||
def test_escape_file_name(name, expected):
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ class TestDates(TestAPI):
|
|||
(timedelta(weeks=1, days=1, seconds=7), '1w 1d 00:00:07'),
|
||||
(timedelta(weeks=16, days=3, hours=17, minutes=46, seconds=39), '16w 3d 17:46:39'),
|
||||
(timedelta(days=10), '1w 3d 00:00:00'),
|
||||
(timedelta(days=10, microseconds=1000), '1w 3d 00:00:00'),
|
||||
])
|
||||
def test_timedelta_to_timestamp(td, expected):
|
||||
assert timedelta_to_timestamp(td) == expected
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue