diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index c36de3fb..d0508326 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -3,7 +3,6 @@ import {Card, Confirm, Container, Icon, Image, Placeholder} from "semantic-ui-re import { CardGroupCentered, ClearButton, - ExternalLink, PageContainer, Paginator, SearchInput, @@ -20,44 +19,6 @@ import Table from "semantic-ui-react/dist/commonjs/collections/Table"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; import {useArchives, useDomains} from "../hooks/customHooks"; - -function FailedArchiveCard({archive, syncArchive, deleteArchive}) { - - let syncIcon = ( - - ); - - let trashIcon = ( - - ); - - let externalIcon = ( - - - - ); - - return ( - - - - {archive.url} - - -

Failed!

- {syncIcon} - {trashIcon} - {externalIcon} -
-
-
- ); -} - function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) { const [syncOpen, setSyncOpen] = useState(false); const [deleteOpen, setDeleteOpen] = useState(false); @@ -71,14 +32,10 @@ function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) { await deleteArchive(archive.id); } - let imageSrc = archive.screenshot_path ? `/media/${archive.screenshot_path}` : null; - let singlefileUrl = archive.singlefile_path ? `/media/${archive.singlefile_path}` : null; + let imageSrc = archive.screenshot_path ? `/media/${encodeURIComponent(archive.screenshot_path)}` : null; + let singlefileUrl = archive.singlefile_path ? `/media/${encodeURIComponent(archive.singlefile_path)}` : null; - if (archive.status === 'failed') { - return ; - } - - let readabilityUrl = archive.readability_path ? `/media/${archive.readability_path}` : null; + let readabilityUrl = archive.readability_path ? `/media/${encodeURIComponent(archive.readability_path)}` : null; let readabilityIcon = ; if (readabilityUrl) { readabilityIcon = ( @@ -130,15 +87,15 @@ function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) { return ( - + - + - +

{textEllipsis(archive.title || archive.url, 100)}

-
+
setDomain(domain)} className="no-link-underscore card-link"> diff --git a/app/src/components/Map.js b/app/src/components/Map.js index ccf1738f..38b6f36c 100644 --- a/app/src/components/Map.js +++ b/app/src/components/Map.js @@ -70,12 +70,11 @@ class ManageMap extends React.Component { tableRow = (pbf, disabled) => { let ref = React.createRef(); - const {size, path, imported} = pbf; - const raspberryPi4Estimation = 3000000; + const {size, path, imported, time_to_import} = pbf; let sizeCells = (<> {humanFileSize(size)} {humanFileSize(size * 25)} - {minutesToTimestamp(size / raspberryPi4Estimation)} + {time_to_import} ); return diff --git a/modules/map/lib.py b/modules/map/lib.py index 4529a306..f9105129 100644 --- a/modules/map/lib.py +++ b/modules/map/lib.py @@ -113,12 +113,14 @@ async def import_pbf(pbf: Path): stdout, stderr = await proc.communicate() elapsed = now() - start - logger.info(f'Import of {pbf} took {timedelta_to_timestamp(elapsed)}') + success = 'Successful' if proc.returncode == 0 else 'Unsuccessful' + logger.info(f'{success} import of {pbf.name} took {timedelta_to_timestamp(elapsed)}') if proc.returncode != 0: + # Log all lines. Truncate long lines. for line in stdout.decode().splitlines(): - logger.info(line) + logger.debug(line[:500]) for line in stderr.decode().splitlines(): - logger.error(line) + logger.error(line[:500]) raise ValueError(f'Importing PBF failed with return code {proc.returncode}') diff --git a/modules/map/models.py b/modules/map/models.py index bcc59551..59243c07 100644 --- a/modules/map/models.py +++ b/modules/map/models.py @@ -1,8 +1,16 @@ from sqlalchemy import Column, Integer, Boolean, BigInteger +from wrolpi import dates from wrolpi.common import Base, ModelHelper from wrolpi.media_path import MediaPathType +RPI4_BYTES_PER_SECOND = 3963 + + +def seconds_to_import(size: int) -> int: + """Attempt to predict how long it will take an RPi4 to import a given PBF file.""" + return max(int(size // RPI4_BYTES_PER_SECOND), 0) + class MapFile(Base, ModelHelper): __tablename__ = 'map_file' @@ -18,4 +26,5 @@ class MapFile(Base, ModelHelper): 'path': self.path, 'imported': self.imported, 'size': self.size, + 'time_to_import': dates.seconds_to_timestamp(seconds_to_import(self.size)), } diff --git a/modules/map/test/test_lib.py b/modules/map/test/test_lib.py deleted file mode 100644 index e69de29b..00000000 diff --git a/modules/map/test/test_models.py b/modules/map/test/test_models.py new file mode 100644 index 00000000..f981a5bf --- /dev/null +++ b/modules/map/test/test_models.py @@ -0,0 +1,16 @@ +import pytest + +from modules.map import models + + +@pytest.mark.parametrize('size,expected', [ + (0, 0), + (-1, 0), + (17737381, 4475), + (63434267, 16006), + (87745484, 22141), + (136372996, 34411), + (116318111, 29351), +]) +def test_seconds_to_import_rpi4(size, expected): + assert models.seconds_to_import(size) == expected diff --git a/wrolpi/common.py b/wrolpi/common.py index a23858bb..03a4bde6 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -713,7 +713,7 @@ def walk(path: Path) -> Generator[Path, None, None]: # These characters are invalid in Windows or Linux. -INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*]') +INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*%]') def escape_file_name(name: str) -> str: diff --git a/wrolpi/dates.py b/wrolpi/dates.py index 522e6804..1ca71ac8 100644 --- a/wrolpi/dates.py +++ b/wrolpi/dates.py @@ -1,6 +1,7 @@ import logging from datetime import datetime, date, timezone, timedelta from enum import Enum +from typing import Union import pytz from sqlalchemy import types @@ -97,8 +98,9 @@ def from_timestamp(timestamp: float) -> datetime: return local_timezone(datetime.fromtimestamp(timestamp)) -def seconds_to_timestamp(seconds: int) -> str: +def seconds_to_timestamp(seconds: Union[int, float]) -> str: """Convert an integer into a timestamp string.""" + seconds = int(seconds) weeks = seconds // Seconds.week seconds = seconds % Seconds.week days = seconds // Seconds.day @@ -125,7 +127,7 @@ def timedelta_to_timestamp(delta: timedelta) -> str: >>> timedelta_to_timestamp(timedelta(days=10)) '1w 3d 00:00:00' """ - return seconds_to_timestamp(int(delta.total_seconds())) + return seconds_to_timestamp(delta.total_seconds()) def recursive_replace_tz(obj, tz=pytz.utc): diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index f602a253..f95e447e 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -291,6 +291,7 @@ def test_zig_zag(low, high, expected): ('foo?', 'foo'), ('foo*', 'foo'), ('foo&', 'foo&'), + ('foo%', 'foo'), ] ) def test_escape_file_name(name, expected): diff --git a/wrolpi/test/test_dates.py b/wrolpi/test/test_dates.py index e467e6a8..64fd21b6 100644 --- a/wrolpi/test/test_dates.py +++ b/wrolpi/test/test_dates.py @@ -74,6 +74,7 @@ class TestDates(TestAPI): (timedelta(weeks=1, days=1, seconds=7), '1w 1d 00:00:07'), (timedelta(weeks=16, days=3, hours=17, minutes=46, seconds=39), '16w 3d 17:46:39'), (timedelta(days=10), '1w 3d 00:00:00'), + (timedelta(days=10, microseconds=1000), '1w 3d 00:00:00'), ]) def test_timedelta_to_timestamp(td, expected): assert timedelta_to_timestamp(td) == expected