Merge pull request #110 from lrnselfreliance/feature/map

Feature/map
This commit is contained in:
lrnselfreliance 2022-03-12 10:30:40 -07:00 • committed by GitHub
commit 446c73c3c2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
43 changed files with 1901 additions and 345 deletions

View file

@ -5,9 +5,7 @@ version: 2.1
# Define a job to be invoked later in a workflow.
# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
jobs:
say-hello:
# Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub.
# See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor
api-tests-eight:
docker:
- image: cimg/python:3.8
- image: cimg/postgres:12.9
@ -15,8 +13,6 @@ jobs:
POSTGRES_USER: postgres
POSTGRES_DB: wrolpi
POSTGRES_PASSWORD: "wrolpi"
# Add steps to the job
# See: https://circleci.com/docs/2.0/configuration-reference/#steps
steps:
- checkout
- run: sudo apt-get update
@ -29,14 +25,50 @@ jobs:
pip install -r requirements.txt
- run:
command: './venv/bin/pytest -svv wrolpi modules'
- save_cache:
key: deps9-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
paths:
- "venv"
api-tests-nine:
docker:
- image: cimg/python:3.9
- image: cimg/postgres:13.5
environment:
POSTGRES_USER: postgres
POSTGRES_DB: wrolpi
POSTGRES_PASSWORD: "wrolpi"
steps:
- checkout
- run: sudo apt-get update
- run: sudo apt-get install -y ffmpeg texlive-latex-base texlive-latex-extra
- run:
name: Install Requirements
command: |
python3 -m venv venv
. venv/bin/activate
pip install -r requirements.txt
- run:
command: './venv/bin/pytest -svv wrolpi modules'
api-tests-ten:
docker:
- image: cimg/python:3.10
- image: cimg/postgres:14.1
environment:
POSTGRES_USER: postgres
POSTGRES_DB: wrolpi
POSTGRES_PASSWORD: "wrolpi"
steps:
- checkout
- run: sudo apt-get update
- run: sudo apt-get install -y ffmpeg texlive-latex-base texlive-latex-extra
- run:
name: Install Requirements
command: |
python3 -m venv venv
. venv/bin/activate
pip install -r requirements.txt
- run:
command: './venv/bin/pytest -svv wrolpi modules'
# Invoke jobs via workflows
# See: https://circleci.com/docs/2.0/configuration-reference/#workflows
workflows:
say-hello-workflow:
wrolpi-api-tests:
jobs:
- say-hello
- api-tests-eight
- api-tests-nine
- api-tests-ten

4
.gitignore vendored
View file

@ -105,9 +105,10 @@ venv.bak/
# intellij
.idea/
# example videos
# example files
test/videos
test/archive
test/map
# docker configs
test/config
@ -119,6 +120,7 @@ docker/archive/package-lock.json
# open street map files
*.pbf
mapnik.xml
# docker override
docker-compose.override.yml

View file

@ -21,7 +21,28 @@
<hr/>
<h2>Can't start the services?</h2>
<p>You can view the media directory by <a href="/media">clicking here</a></p>
<p>You can view the media directory by <a id="media_link" href="/media">navigating to /media</a></p>
<p>The map may be available <a id="map_link" href="#">by navigating to port 8084</a></p>
<script>
// Get the current hostname (probably an IP) link to my :8084 above.
let hostname = window.location.hostname;
let media_a = document.getElementById('media_link');
if (media_a) {
let media_url = `http://${hostname}/media`;
media_a.setAttribute('href', media_url);
media_a.textContent = `by navigating to ${media_url}`;
}
let map_a = document.getElementById('map_link');
if (map_a) {
let map_url = `http://${hostname}:8084`;
map_a.setAttribute('href', map_url);
map_a.textContent = `by navigating to ${map_url}`;
}
</script>
</body>
</html>

View file

@ -28,7 +28,7 @@ network so that any person with a laptop/tablet/phone can connect and use the da
- [x] Food Inventory management
- [x] Wi-Fi Hotspot
- [x] Universal search
- [ ] Map viewer & downloader
- [x] Map viewer
- [ ] eBook viewer
- [ ] Wiki viewer & downloader
- [ ] Synchronizer and/or duplicator
@ -48,13 +48,18 @@ You can try out WROLPi by running the docker containers.
* `git clone git@github.com:lrnselfreliance/wrolpi.git`
3. Change directory into the code base
* `cd wrolpi`
4. Start the database.
4. Build the containers
* `docker-compose build --parallel`
5. Create volumes
* `docker volume create --name=openstreetmap-data`
* `docker volume create --name=openstreetmap-rendered-tiles`
6. Start the database.
* `docker-compose up -d db`
5. Initialize the database
7. Initialize the database
* `docker-compose run --rm api db upgrade`
6. Start the docker containers
8. Start the docker containers
* `docker-compose up`
7. Navigate to WROLPi
9. Browse to WROLPi
* http://0.0.0.0:8080
# RaspberryPi Install

View file

@ -0,0 +1,40 @@
"""Create MapFile table.
Revision ID: 2fb39c1861c7
Revises: 36cae041ad31
Create Date: 2022-03-03 19:13:39.997580
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '2fb39c1861c7'
down_revision = '36cae041ad31'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('''CREATE TABLE map_file (
id SERIAL PRIMARY KEY,
path TEXT NOT NULL,
imported BOOLEAN DEFAULT FALSE,
size BIGINT
)''')
if not DOCKERIZED:
session.execute('ALTER TABLE public.map_file OWNER TO wrolpi')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('DROP TABLE IF EXISTS map_file')

View file

@ -3,7 +3,6 @@
"version": "0.1.0",
"private": true,
"dependencies": {
"dot-prop": "^5.1.1",
"lodash": "^4.17.19",
"npm": "^6.14.7",
"query-string": "^6.13.0",
@ -16,8 +15,7 @@
"react-semantic-toasts": "^0.6.5",
"react-timezone-select": "0.8.4",
"semantic-ui-offline": "^2.4.1-1",
"semantic-ui-react": "^0.88.2",
"yargs-parser": "^18.1.2"
"semantic-ui-react": "^0.88.2"
},
"scripts": {
"start": "react-scripts start",

View file

@ -15,6 +15,7 @@ import {Saver} from "./components/Upload";
import {MoreButton, PageContainer, SearchInput} from "./components/Common";
import {FilesRoute} from "./components/Files";
import {useSearch, useVersion} from "./hooks/customHooks";
import {MapRoute} from "./components/Map";
function Welcome() {
const {searchStr, setSearchStr, archives, videos} = useSearch();
@ -88,6 +89,7 @@ export default function App() {
<Route path="/apps" component={AppsRoute}/>
<Route path="/inventory" component={InventoryRoute}/>
<Route path='/archive' component={ArchiveRoute}/>
<Route path='/map' component={MapRoute}/>
<Route path='/files' component={FilesRoute}/>
<Route component={PageNotFound}/>
</Switch>

View file

@ -388,7 +388,6 @@ export async function getDownloaders() {
let response = await apiGet(`${API_URI}/downloaders`);
return await response.json();
} catch (e) {
console.log('getDownloaders', e);
return {};
}
}
@ -512,3 +511,23 @@ export async function setThrottle(on) {
}
}
}
export async function getMapImportStatus() {
let response = await apiGet(`${API_URI}/map/pbf`);
return await response.json();
}
export async function importPbfs(pbfs) {
let body = {'pbfs': pbfs};
let response = await apiPost(`${API_URI}/map/import`, body);
if (response.status === 204) {
return null;
} else {
toast({
type: 'error',
title: 'Error!',
description: 'Could not start import! See server logs.',
time: 5000,
});
}
}

View file

@ -208,7 +208,7 @@ class DownloadRow extends React.Component {
}
return (
<Table.Row positive={positive}>
<Table.Cell>{textEllipsis(url, 50)}</Table.Cell>
<Table.Cell><a href={url} target='_blank'>{textEllipsis(url, 50)}</a></Table.Cell>
<Table.Cell>{secondsToFrequency(frequency)}</Table.Cell>
<Table.Cell>{last_successful_download ? secondsToDate(last_successful_download) : null}</Table.Cell>
<Table.Cell>{secondsToDate(next_download)}</Table.Cell>
@ -305,7 +305,7 @@ class StoppableRow extends React.Component {
return (
<Table.Row positive={positive} negative={negative} warning={warning}>
<Table.Cell>{textEllipsis(url, 50)}</Table.Cell>
<Table.Cell><a href={url} target='_blank'>{textEllipsis(url, 50)}</a></Table.Cell>
<Table.Cell>{last_successful_download ? secondsToDate(last_successful_download) : null}</Table.Cell>
<Table.Cell>{status}</Table.Cell>
{buttonCell}
@ -328,6 +328,11 @@ class Downloads extends React.Component {
async componentDidMount() {
await this.fetchDownloads();
this.intervalId = setInterval(this.fetchDownloads, 1000 * 10);
}
componentWillUnmount() {
clearInterval(this.intervalId);
}
closeStop = () => {

View file

@ -3,7 +3,6 @@ import {Card, Confirm, Container, Icon, Image, Placeholder} from "semantic-ui-re
import {
CardGroupCentered,
ClearButton,
ExternalLink,
PageContainer,
Paginator,
SearchInput,
@ -20,44 +19,6 @@ import Table from "semantic-ui-react/dist/commonjs/collections/Table";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useArchives, useDomains} from "../hooks/customHooks";
function FailedArchiveCard({archive, syncArchive, deleteArchive}) {
let syncIcon = (
<Button onClick={syncArchive}>
<Icon name='sync' size='big'/>
</Button>
);
let trashIcon = (
<Button onClick={deleteArchive}>
<Icon name='trash' size='big'/>
</Button>
);
let externalIcon = (
<a href={archive.url} target='_blank' rel='noopener noreferrer'>
<Icon name='sign-out' size='big'/>
</a>
);
return (
<Card>
<Card.Content>
<Card.Header>
{archive.url}
</Card.Header>
<Card.Description>
<p>Failed!</p>
{syncIcon}
{trashIcon}
{externalIcon}
</Card.Description>
</Card.Content>
</Card>
);
}
function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) {
const [syncOpen, setSyncOpen] = useState(false);
const [deleteOpen, setDeleteOpen] = useState(false);
@ -71,14 +32,10 @@ function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) {
await deleteArchive(archive.id);
}
let imageSrc = archive.screenshot_path ? `/media/${archive.screenshot_path}` : null;
let singlefileUrl = archive.singlefile_path ? `/media/${archive.singlefile_path}` : null;
let imageSrc = archive.screenshot_path ? `/media/${encodeURIComponent(archive.screenshot_path)}` : null;
let singlefileUrl = archive.singlefile_path ? `/media/${encodeURIComponent(archive.singlefile_path)}` : null;
if (archive.status === 'failed') {
return <FailedArchiveCard archive={archive} syncArchive={localSyncArchive} deleteArchive={localDeleteArchive}/>;
}
let readabilityUrl = archive.readability_path ? `/media/${archive.readability_path}` : null;
let readabilityUrl = archive.readability_path ? `/media/${encodeURIComponent(archive.readability_path)}` : null;
let readabilityIcon = <Button icon disabled><Icon name='book' size='large'/></Button>;
if (readabilityUrl) {
readabilityIcon = (
@ -130,15 +87,15 @@ function ArchiveCard({archive, syncArchive, deleteArchive, setDomain}) {
return (
<Card>
<ExternalLink to={singlefileUrl} className='no-link-underscore card-link'>
<a href={singlefileUrl} target='_blank' className='no-link-underscore card-link'>
<Image src={imageSrc} wrapped style={{position: 'relative', width: '100%'}}/>
</ExternalLink>
</a>
<Card.Content>
<Card.Header>
<Container textAlign='left'>
<ExternalLink to={singlefileUrl} className='no-link-underscore card-link'>
<a href={singlefileUrl} target='_blank' className='no-link-underscore card-link'>
<p>{textEllipsis(archive.title || archive.url, 100)}</p>
</ExternalLink>
</a>
</Container>
</Card.Header>
<a onClick={() => setDomain(domain)} className="no-link-underscore card-link">

View file

@ -1,10 +1,22 @@
import React, {useEffect, useState} from "react";
import {Card, Checkbox, Container, Form, Icon, Image, Input, Menu, Pagination, Responsive} from 'semantic-ui-react';
import {
Card,
Checkbox,
Container,
Form,
Icon,
Image,
Input,
Menu,
Pagination,
Popup,
Responsive
} from 'semantic-ui-react';
import {Link, NavLink} from "react-router-dom";
import Button from "semantic-ui-react/dist/commonjs/elements/Button";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {getConfig, getSettings} from "../api";
import {getSettings} from "../api";
import {useDownloaders, useHotspot, useThrottle} from "../hooks/customHooks";
export const API_URI = `http://${window.location.host}/api`;
@ -256,15 +268,13 @@ export function secondsToTimestamp(seconds) {
}
export function humanFileSize(bytes, si = false, dp = 1) {
const thresh = si ? 1000 : 1024;
const thresh = 1024;
if (Math.abs(bytes) < thresh) {
return bytes + ' B';
}
const units = si
? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
: ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
const units = ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
let u = -1;
const r = 10 ** dp;
@ -685,3 +695,23 @@ export function emptyToNull(obj) {
}
return obj;
}
export function HelpPopup({icon, size, content, position}) {
return <Popup
content={content}
size={size || null}
position={position || 'left center'}
trigger={
<Icon circular link name={icon || 'question'} size='small'
style={{marginLeft: '0.25em', marginRight: '0.25em'}}
/>}
/>
}
export function minutesToTimestamp(minutes) {
minutes = Math.round(minutes);
const hours = String(Math.floor(minutes / 60)).padStart(1, '0');
minutes = minutes - (hours * 60);
minutes = String(minutes).padStart(2, '0');
return `${hours}:${minutes}`
}

249
app/src/components/Map.js Normal file
View file

@ -0,0 +1,249 @@
import React from "react";
import {HelpPopup, humanFileSize, minutesToTimestamp, PageContainer, TabLinks, WROLModeMessage} from "./Common";
import {Route} from "react-router-dom";
import {getMapImportStatus, importPbfs} from "../api";
import Table from "semantic-ui-react/dist/commonjs/collections/Table";
import {Button, Checkbox, Icon, Loader, Placeholder} from "semantic-ui-react";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
class ManageMap extends React.Component {
constructor(props) {
super(props);
this.state = {
pbfs: null,
importing: null,
import_running: false,
selectedPaths: [],
dockerized: null,
}
}
async componentDidMount() {
await this.fetchImportStatus();
this.intervalId = setInterval(this.fetchImportStatus, 1000 * 30);
}
componentWillUnmount() {
clearInterval(this.intervalId);
}
fetchImportStatus = async () => {
const importStatus = await getMapImportStatus();
this.setState({
pbfs: importStatus['pbfs'],
importing: importStatus['importing'],
import_running: importStatus['import_running'],
dockerized: importStatus['dockerized'],
});
}
import = async (e) => {
e.preventDefault();
this.setState({'ready': false});
const {pbfs, selectedPaths} = this.state;
if (!selectedPaths || selectedPaths.length === 0) {
// No PBFs are selected, import all.
let paths = [];
for (let i = 0; i < pbfs.length; i++) {
paths = paths.concat([pbfs[i].path]);
}
await importPbfs(paths);
} else {
await importPbfs(selectedPaths);
}
await this.fetchImportStatus();
}
handleCheckbox = (checked, pbf) => {
let {selectedPaths} = this.state;
if (checked === true) {
selectedPaths = selectedPaths.concat([pbf.path]);
} else {
const index = selectedPaths.indexOf(pbf.path);
if (index > -1) {
selectedPaths.splice(index, 1);
}
}
this.setState({selectedPaths});
}
tableRow = (pbf, disabled) => {
let ref = React.createRef();
const {size, path, imported, time_to_import} = pbf;
let sizeCells = (<>
<Table.Cell>{humanFileSize(size)}</Table.Cell>
<Table.Cell>{humanFileSize(size * 25)}</Table.Cell>
<Table.Cell>{time_to_import}</Table.Cell>
</>);
return <Table.Row key={path}>
<Table.Cell collapsing>
<Checkbox
disabled={disabled}
ref={ref}
onChange={(e, data) => this.handleCheckbox(data.checked, pbf)}
/>
</Table.Cell>
<Table.Cell>
<a href={`/media/${path}`}>{path}</a>
</Table.Cell>
<Table.Cell>
{path === this.state.importing ?
<Loader active inline size='mini'/> :
imported ? 'yes' : 'no'}
</Table.Cell>
{size !== null ? sizeCells : <Table.Cell colSpan={3}/>}
</Table.Row>
}
render() {
const {pbfs, selectedPaths, import_running, importing, dockerized} = this.state;
let dockerWarning;
if (dockerized === true) {
dockerWarning = <Message negative icon>
<Icon name='hand point right'/>
<Message.Content>
<Message.Header>
Maps are not fully supported in a Docker container!
</Message.Header>
<p><b>Only one PBF can be imported and displayed in the docker container.</b></p>
<p>To import a map file, run the following docker-compose commands:</p>
<pre> docker-compose stop map</pre>
<pre> docker-compose rm map</pre>
<pre> docker-compose run --rm -v /absolute/path/to/map.osm.pbf:/data.osm.pbf
-v openstreetmap-data:/var/lib/postgresql/12/main map import
</pre>
<p>Be sure to change <b>/absolute/path/to/map.osm.pbf</b>!</p>
<p>After you have imported a new PBF file, you need to clear the rendered tile cache:</p>
<pre> docker volume rm openstreetmap-rendered-tiles</pre>
<pre> docker volume create openstreetmap-rendered-tiles</pre>
<p>Start your map container:</p>
<pre> docker-compose up -d map</pre>
</Message.Content>
</Message>
}
let downloadMessage = <Message info icon>
<Icon name='question'/>
<Message.Content>
<Message.Header>
Where do I get map files?
</Message.Header>
<p>You can download map files from&nbsp;
<a href='https://download.geofabrik.de/'>https://download.geofabrik.de/</a>
</p>
<p><b>Download only the areas you need</b>. Large regions like all of Asia, or the entire
planet are most likely <b>too large</b> and won't render quickly. I recommend only
importing files less than 1GB on a Raspberry Pi.</p>
<p>Only <b>*.osm.pbf</b> files are supported!</p>
<p>Place downloaded map files into <b>map/pbf</b> so they can be imported here.</p>
</Message.Content>
</Message>
let importingLoader = (
<Loader size='large' active={import_running} inline='centered'>Importing: {importing}</Loader>
);
let disabled = dockerized || !pbfs || pbfs.length === 0 || import_running;
let importButton = <Button
primary
disabled={disabled}
onClick={this.import}
>
{selectedPaths.length > 0 ? 'Import Selected' : 'Import All'}
</Button>;
let rows;
if (!pbfs) {
// Fetch request is not complete.
rows = <Table.Row>
<Table.Cell/>
<Table.Cell colSpan={6}>
<Placeholder>
<Placeholder.Line/>
<Placeholder.Line/>
</Placeholder>
</Table.Cell>
</Table.Row>;
} else if (pbfs.length === 0) {
rows = <Table.Row>
<Table.Cell/><Table.Cell colSpan={5}>No PBF map files were found in <b>map/pbf</b></Table.Cell>
</Table.Row>;
} else {
rows = pbfs.map(i => this.tableRow(i, disabled));
}
let spaceHelpPopup = <HelpPopup
content='Upon importing, a PBF file will consume more disk space than the original file.'/>;
let timeHelpPopup = <HelpPopup content='Estimated for a Raspberry Pi 4'/>;
return (
<PageContainer>
<WROLModeMessage content='Cannot modify Map'/>
{dockerWarning}
{downloadMessage}
{importingLoader}
<Table>
<Table.Header>
<Table.Row>
<Table.HeaderCell/>
<Table.HeaderCell>PBF File</Table.HeaderCell>
<Table.HeaderCell>Imported</Table.HeaderCell>
<Table.HeaderCell>Size</Table.HeaderCell>
<Table.HeaderCell>Space Required {spaceHelpPopup}</Table.HeaderCell>
<Table.HeaderCell>Time to Import {timeHelpPopup}</Table.HeaderCell>
</Table.Row>
</Table.Header>
<Table.Body>
{rows}
</Table.Body>
<Table.Footer>
<Table.Row>
<Table.HeaderCell/>
<Table.HeaderCell colSpan={5}>
{importButton}
</Table.HeaderCell>
</Table.Row>
</Table.Footer>
</Table>
</PageContainer>
);
}
}
function MapApp() {
return (
<iframe
title='map'
src={`http://${window.location.hostname}:8084/`}
style={{
position: 'fixed',
height: '100%',
width: '100%',
border: 'none',
}}/>
)
}
export function MapRoute() {
const links = [
{text: 'Map', to: '/map', exact: true, key: 'map'},
{text: 'Manage', to: '/map/manage', exact: true, key: 'manage'},
];
return <div style={{marginTop: '2em'}}>
<TabLinks links={links}/>
<Route path='/map' exact component={MapApp}/>
<Route path='/map/manage' exact component={ManageMap}/>
</div>
}

View file

@ -6,9 +6,10 @@ const responsiveWidth = 500;
const links = [
{text: 'Videos', to: '/videos', key: 'videos'},
{text: 'Inventory', to: '/inventory', key: 'inventory'},
{text: 'Archive', to: '/archive', key: 'archive'},
{text: 'Map', to: '/map', key: 'map'},
{text: 'Files', to: '/files', key: 'files'},
{text: 'Inventory', to: '/inventory', key: 'inventory'},
{
key: 'apps', text: 'Apps', links: [
{to: '/apps/otp', text: 'One Time Pad', exact: true},

View file

@ -2,6 +2,7 @@
version: '3'
services:
db:
# https://github.com/docker-library/docs/tree/master/postgres
image: postgres:12
environment:
- POSTGRES_DB=wrolpi
@ -21,7 +22,6 @@ services:
- './main.py:/opt/wrolpi/main.py'
- './wrolpi:/opt/wrolpi/wrolpi'
- './modules:/opt/wrolpi/modules'
- './config:/opt/wrolpi/config'
- './test:/opt/wrolpi/test'
- './test:/media/wrolpi'
- './alembic:/opt/wrolpi/alembic'
@ -60,6 +60,20 @@ services:
test: [ 'CMD-SHELL', 'curl http://127.0.0.1:3000' ]
user: '${UID-1000}:${GID-1000}'
map:
# https://github.com/Overv/openstreetmap-tile-server
image: overv/openstreetmap-tile-server:v1.6.0
volumes:
- 'openstreetmap-data:/var/lib/postgresql/12/main'
- 'openstreetmap-rendered-tiles:/var/lib/mod_tile'
- './modules/map/leaflet.html:/var/www/html/index.html'
- './modules/map/leaflet.js:/var/www/html/leaflet.js'
- './modules/map/leaflet.css:/var/www/html/leaflet.css'
ports:
- '8084:80'
command: 'run'
shm_size: 1g # Increase shared size to render large map tiles
web:
depends_on:
- app
@ -73,3 +87,9 @@ services:
- './test:/opt/media'
healthcheck:
test: [ 'CMD-SHELL', 'curl http://127.0.0.1:80' ]
volumes:
openstreetmap-data:
external: true
openstreetmap-rendered-tiles:
external: true

View file

@ -4,9 +4,10 @@ WORKDIR /opt/wrolpi
# Update dependencies for the services install.
RUN apt update
# ffmpeg for Videos and Latex for OTP PDF.
# ffmpeg for Videos and Latex for OTP PDF. osm2pgsql to import map files.
RUN apt-get install -y ffmpeg texlive-latex-base texlive-latex-extra
RUN pdflatex --version
RUN ffmpeg -version
# Install WROLPi
COPY main.py /opt/wrolpi/

View file

@ -9,11 +9,8 @@ COPY app/public /app/public
COPY app/src /app/src
COPY app/package.json /app/
RUN rm -rf /app/node_modules /app/package-lock.json
# Increase network timeout because a Raspberry Pi is slow.
RUN yarn --network-timeout 100000
# Create the production build
RUN yarn global add serve
RUN npm install
RUN npm install -g serve
# Run the production build
CMD ["serve", "-s", "build"]

View file

@ -30,116 +30,23 @@ while getopts ":hb:" option; do
esac
done
if (! grep 'Raspberry Pi' /proc/cpuinfo >/dev/null); then
echo 'This script can only be run on a Raspberry Pi!' && exit 2
fi
set -x
set -e
# Check that WROLPi directory exists, and contains wrolpi.
[ -d /opt/wrolpi ] && [ ! -d /opt/wrolpi/wrolpi ] && echo "/opt/wrolpi exists but does not contain wrolpi!" && exit 2
[ -d /opt/wrolpi ] && [ ! -d /opt/wrolpi/wrolpi ] && echo "/opt/wrolpi exists but does not contain wrolpi!" && exit 3
# Install yarn repos
curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --dearmor | sudo tee /usr/share/keyrings/yarnkey.gpg >/dev/null
echo "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main" |
tee /etc/apt/sources.list.d/yarn.list
# Install npm repos.
curl -fsSL https://deb.nodesource.com/setup_14.x | bash -
# Update if we haven't updated in the last day.
[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update
# Install dependencies
apt install -y git apt-transport-https ca-certificates curl gnupg-agent gcc libpq-dev software-properties-common \
postgresql-12 nginx-full nginx-doc python3.8-minimal python3.8-dev python3.8-doc python3.8-venv \
ffmpeg hostapd nodejs yarn texlive-latex-base texlive-latex-extra chromium-browser chromium-chromedriver \
cpufrequtils network-manager
# Install Archiving tools.
npm install -g "gildas-lormeau/SingleFile#master"
npm install -g 'git+https://github.com/pirate/readability-extractor'
# Get the latest WROLPi code
# Get the latest WROLPi code. Use the branch requested.
apt install -y git
git --version
git clone https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi || :
(cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git reset --hard origin/"${BRANCH}") || exit 3
(cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git reset --hard origin/"${BRANCH}") || exit 4
# Setup the virtual environment that main.py expects
pip3 --version || (
# If no pip, install pip
curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py &&
python3 /tmp/get-pip.py
)
/opt/wrolpi/scripts/ubuntu_20.04_install.sh 2>&1 | tee /opt/wrolpi/install.log
# Install python requirements files
pip3 install -r /opt/wrolpi/requirements.txt
# Build React app
[ ! -f /usr/local/bin/serve ] && npm -g install serve
cd /opt/wrolpi/app
yarn --network-timeout 10000
yarn run build
yarn global add serve
# Install the WROLPi nginx config over the default nginx config.
cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf
cp /opt/wrolpi/50x.html /var/www/50x.html
/usr/sbin/nginx -s reload
# Create the WROLPi user
grep wrolpi /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
chown -R wrolpi:wrolpi /opt/wrolpi
# Give WROLPi group a few privileged commands via sudo without password.
cat > /etc/sudoers.d/90-wrolpi << 'EOF'
%wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set
EOF
chmod 660 /etc/sudoers.d/90-wrolpi
# Verify this new file is valid.
visudo -c -f /etc/sudoers.d/90-wrolpi
# Create the media directory. This should be mounted by the maintainer.
[ -d /media/wrolpi ] || mkdir /media/wrolpi
chown wrolpi:wrolpi /media/wrolpi
# Install the systemd services
cp /opt/wrolpi/systemd/wrolpi-api.service /etc/systemd/system/
cp /opt/wrolpi/systemd/wrolpi-app.service /etc/systemd/system/
cp /opt/wrolpi/systemd/wrolpi.target /etc/systemd/system/
/usr/bin/systemctl daemon-reload
systemctl enable wrolpi-api.service
systemctl enable wrolpi-app.service
# Stop the services so the user has to start them again. We don't want to run outdated services when updating.
systemctl stop wrolpi-api.service
systemctl stop wrolpi-app.service
# Configure Postgresql. Do this after the API is stopped.
sudo -u postgres psql -c '\l' | grep wrolpi || (
sudo -u postgres createuser wrolpi &&
sudo -u postgres psql -c "alter user postgres password 'wrolpi'" &&
sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'" &&
sudo -u postgres createdb -O wrolpi wrolpi
)
# Initialize/upgrade the WROLPi database.
(cd /opt/wrolpi && /usr/bin/python3 /opt/wrolpi/main.py db upgrade)
set +x
ip=$(hostname -i | cut -d' ' -f1)
if [[ $ip == *":"* ]]; then
# Don't suggest the ipv6 address.
ip=$(hostname -i | cut -d' ' -f2)
fi
echo "
WROLPi has successfully been installed!
Mount your external hard drive to /media/wrolpi if you have one.
Start the WROLPi services using:
# sudo systemctl start wrolpi.target
then navigate to: http://${ip}
Or, join to the Wifi hotspot:
SSID: WROLPi
Password: wrolpi hotspot
When on the hotspot, WROLPi is accessible at http://192.168.0.1
"
echo "Install end $(date '+%Y-%m-%d %H:%M:%S')" >>/opt/wrolpi/install.log
exit $?

View file

@ -59,7 +59,7 @@ async def main(loop):
# DB Parser for running Alembic migrations
db_parser = sub_commands.add_parser('db')
db_parser.add_argument('command', help=f'Supported commands: upgrade, downgrade')
db_parser.add_argument('command', help='Supported commands: upgrade, downgrade')
args = parser.parse_args()

View file

@ -1,41 +1,54 @@
import asyncio
from http import HTTPStatus
from pathlib import Path
from urllib.parse import urlparse
from sanic import Blueprint, response
from sanic import Request, response
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from modules.map.schema import PBFPostRequest, PBFPostResponse
from wrolpi.common import create_websocket_feed
from modules.map import lib, schema
from wrolpi.common import wrol_mode_check, get_media_directory
from wrolpi.errors import ValidationError
from wrolpi.root_api import get_blueprint, json_response
from wrolpi.vars import PYTEST, DOCKERIZED
NAME = 'map'
api_bp = Blueprint('Map', url_prefix='/map')
download_queue, download_event = create_websocket_feed('pbf_progress', '/feeds/pbf_progress', api_bp)
bp = get_blueprint('Map', '/api/map')
@api_bp.route('/pbf', methods=['POST'])
@openapi.description('Queue a PBF file for download and processing')
@openapi.response(HTTPStatus.OK, PBFPostResponse)
@validate(PBFPostRequest)
def pbf_post(request, data: dict):
pbf_url = data.get('pbf_url')
parsed = urlparse(pbf_url)
if not parsed.scheme or not parsed.netloc or not parsed.path:
return response.json({'error': 'Invalid PBF url'}, HTTPStatus.BAD_REQUEST)
@bp.post('/import')
@openapi.definition(
summary='Import PBF map files',
body=schema.ImportPost,
)
@validate(schema.ImportPost)
@wrol_mode_check
async def import_pbfs(request: Request, body: schema.ImportPost):
if lib.IMPORT_EVENT.is_set():
return response.json({'error': 'Map import already running'}, HTTPStatus.CONFLICT)
# Get the size of the PBF file. Also, check that it is accessible.
try:
size, filename = get_http_file_info(pbf_url)
except LookupError:
# Couldn't get the size of the file, probably doesn't exist
return response.json({'error': 'Failed to get file size. Does it exists?', 'url': pbf_url},
HTTPStatus.NOT_FOUND)
pbfs = [i for i in body.pbfs if i]
if not pbfs:
raise ValidationError('No PBF files were provided')
destination = Path('/tmp') / filename
coro = download_file(pbf_url, size, destination)
asyncio.ensure_future(coro)
return response.json({'success': 'File download started'})
if PYTEST:
await lib.import_pbfs(pbfs)
else:
asyncio.create_task(lib.import_pbfs(pbfs))
return response.empty()
@bp.get('/pbf')
@openapi.description('Find any PBF map files, get their import status')
def pbf(request: Request):
pbfs = lib.get_pbf_import_status()
pbfs = sorted(pbfs, key=lambda i: str(i.path))
importing = lib.IMPORTING.get('pbf')
if importing:
importing = Path(importing).relative_to(get_media_directory())
body = dict(
pbfs=pbfs,
importing=importing,
import_running=lib.IMPORT_EVENT.is_set(),
dockerized=DOCKERIZED,
)
return json_response(body, HTTPStatus.OK)

View file

@ -1,3 +0,0 @@
from pathlib import Path
MY_DIR: Path = Path(__file__).parent

644
modules/map/leaflet.css Normal file
View file

@ -0,0 +1,644 @@
/* required styles */
/* @preserve
* Leaflet 1.7.1, a JS library for interactive maps. http://leafletjs.com
* (c) 2010-2019 Vladimir Agafonkin, (c) 2010-2011 CloudMade
*/
.leaflet-pane,
.leaflet-tile,
.leaflet-marker-icon,
.leaflet-marker-shadow,
.leaflet-tile-container,
.leaflet-pane > svg,
.leaflet-pane > canvas,
.leaflet-zoom-box,
.leaflet-image-layer,
.leaflet-layer {
position: absolute;
left: 0;
top: 0;
}
.leaflet-container {
overflow: hidden;
}
.leaflet-tile,
.leaflet-marker-icon,
.leaflet-marker-shadow {
-webkit-user-select: none;
-moz-user-select: none;
user-select: none;
-webkit-user-drag: none;
}
/* Prevents IE11 from highlighting tiles in blue */
.leaflet-tile::selection {
background: transparent;
}
/* Safari renders non-retina tile on retina better with this, but Chrome is worse */
.leaflet-safari .leaflet-tile {
image-rendering: -webkit-optimize-contrast;
}
/* hack that prevents hw layers "stretching" when loading new tiles */
.leaflet-safari .leaflet-tile-container {
width: 1600px;
height: 1600px;
-webkit-transform-origin: 0 0;
}
.leaflet-marker-icon,
.leaflet-marker-shadow {
display: block;
}
/* .leaflet-container svg: reset svg max-width decleration shipped in Joomla! (joomla.org) 3.x */
/* .leaflet-container img: map is broken in FF if you have max-width: 100% on tiles */
.leaflet-container .leaflet-overlay-pane svg,
.leaflet-container .leaflet-marker-pane img,
.leaflet-container .leaflet-shadow-pane img,
.leaflet-container .leaflet-tile-pane img,
.leaflet-container img.leaflet-image-layer,
.leaflet-container .leaflet-tile {
max-width: none !important;
max-height: none !important;
}
.leaflet-container.leaflet-touch-zoom {
-ms-touch-action: pan-x pan-y;
touch-action: pan-x pan-y;
}
.leaflet-container.leaflet-touch-drag {
-ms-touch-action: pinch-zoom;
/* Fallback for FF which doesn't support pinch-zoom */
touch-action: none;
touch-action: pinch-zoom;
}
.leaflet-container.leaflet-touch-drag.leaflet-touch-zoom {
-ms-touch-action: none;
touch-action: none;
}
.leaflet-container {
-webkit-tap-highlight-color: transparent;
}
.leaflet-container a {
-webkit-tap-highlight-color: rgba(51, 181, 229, 0.4);
}
.leaflet-tile {
filter: inherit;
visibility: hidden;
}
.leaflet-tile-loaded {
visibility: inherit;
}
.leaflet-zoom-box {
width: 0;
height: 0;
-moz-box-sizing: border-box;
box-sizing: border-box;
z-index: 800;
}
/* workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=888319 */
.leaflet-overlay-pane svg {
-moz-user-select: none;
}
.leaflet-pane { z-index: 400; }
.leaflet-tile-pane { z-index: 200; }
.leaflet-overlay-pane { z-index: 400; }
.leaflet-shadow-pane { z-index: 500; }
.leaflet-marker-pane { z-index: 600; }
.leaflet-tooltip-pane { z-index: 650; }
.leaflet-popup-pane { z-index: 700; }
.leaflet-map-pane canvas { z-index: 100; }
.leaflet-map-pane svg { z-index: 200; }
.leaflet-vml-shape {
width: 1px;
height: 1px;
}
.lvml {
behavior: url(#default#VML);
display: inline-block;
position: absolute;
}
/* control positioning */
.leaflet-control {
position: relative;
z-index: 800;
pointer-events: visiblePainted; /* IE 9-10 doesn't have auto */
pointer-events: auto;
}
.leaflet-top,
.leaflet-bottom {
position: absolute;
z-index: 1000;
pointer-events: none;
}
.leaflet-top {
top: 0;
}
.leaflet-right {
right: 0;
}
.leaflet-bottom {
bottom: 0;
}
.leaflet-left {
left: 0;
}
.leaflet-control {
float: left;
clear: both;
}
.leaflet-right .leaflet-control {
float: right;
}
.leaflet-top .leaflet-control {
margin-top: 10px;
}
.leaflet-bottom .leaflet-control {
margin-bottom: 10px;
}
.leaflet-left .leaflet-control {
margin-left: 10px;
}
.leaflet-right .leaflet-control {
margin-right: 10px;
}
/* zoom and fade animations */
.leaflet-fade-anim .leaflet-tile {
will-change: opacity;
}
.leaflet-fade-anim .leaflet-popup {
opacity: 0;
-webkit-transition: opacity 0.2s linear;
-moz-transition: opacity 0.2s linear;
transition: opacity 0.2s linear;
}
.leaflet-fade-anim .leaflet-map-pane .leaflet-popup {
opacity: 1;
}
.leaflet-zoom-animated {
-webkit-transform-origin: 0 0;
-ms-transform-origin: 0 0;
transform-origin: 0 0;
}
.leaflet-zoom-anim .leaflet-zoom-animated {
will-change: transform;
}
.leaflet-zoom-anim .leaflet-zoom-animated {
-webkit-transition: -webkit-transform 0.25s cubic-bezier(0,0,0.25,1);
-moz-transition: -moz-transform 0.25s cubic-bezier(0,0,0.25,1);
transition: transform 0.25s cubic-bezier(0,0,0.25,1);
}
.leaflet-zoom-anim .leaflet-tile,
.leaflet-pan-anim .leaflet-tile {
-webkit-transition: none;
-moz-transition: none;
transition: none;
}
.leaflet-zoom-anim .leaflet-zoom-hide {
visibility: hidden;
}
/* cursors */
.leaflet-interactive {
cursor: pointer;
}
.leaflet-grab {
cursor: -webkit-grab;
cursor: -moz-grab;
cursor: grab;
}
.leaflet-crosshair,
.leaflet-crosshair .leaflet-interactive {
cursor: crosshair;
}
.leaflet-popup-pane,
.leaflet-control {
cursor: auto;
}
.leaflet-dragging .leaflet-grab,
.leaflet-dragging .leaflet-grab .leaflet-interactive,
.leaflet-dragging .leaflet-marker-draggable {
cursor: move;
cursor: -webkit-grabbing;
cursor: -moz-grabbing;
cursor: grabbing;
}
/* marker & overlays interactivity */
.leaflet-marker-icon,
.leaflet-marker-shadow,
.leaflet-image-layer,
.leaflet-pane > svg path,
.leaflet-tile-container {
pointer-events: none;
}
.leaflet-marker-icon.leaflet-interactive,
.leaflet-image-layer.leaflet-interactive,
.leaflet-pane > svg path.leaflet-interactive,
svg.leaflet-image-layer.leaflet-interactive path {
pointer-events: visiblePainted; /* IE 9-10 doesn't have auto */
pointer-events: auto;
}
/* visual tweaks */
.leaflet-container {
background: #ddd;
outline: 0;
}
.leaflet-container a {
color: #0078A8;
}
.leaflet-container a.leaflet-active {
outline: 2px solid orange;
}
.leaflet-zoom-box {
border: 2px dotted #38f;
background: rgba(255,255,255,0.5);
}
/* general typography */
.leaflet-container {
font: 12px/1.5 "Helvetica Neue", Arial, Helvetica, sans-serif;
}
/* general toolbar styles */
.leaflet-bar {
box-shadow: 0 1px 5px rgba(0,0,0,0.65);
border-radius: 4px;
}
.leaflet-bar a,
.leaflet-bar a:hover {
background-color: #fff;
border-bottom: 1px solid #ccc;
width: 26px;
height: 26px;
line-height: 26px;
display: block;
text-align: center;
text-decoration: none;
color: black;
}
.leaflet-bar a,
.leaflet-control-layers-toggle {
background-position: 50% 50%;
background-repeat: no-repeat;
display: block;
}
.leaflet-bar a:hover {
background-color: #f4f4f4;
}
.leaflet-bar a:first-child {
border-top-left-radius: 4px;
border-top-right-radius: 4px;
}
.leaflet-bar a:last-child {
border-bottom-left-radius: 4px;
border-bottom-right-radius: 4px;
border-bottom: none;
}
.leaflet-bar a.leaflet-disabled {
cursor: default;
background-color: #f4f4f4;
color: #bbb;
}
.leaflet-touch .leaflet-bar a {
width: 30px;
height: 30px;
line-height: 30px;
}
.leaflet-touch .leaflet-bar a:first-child {
border-top-left-radius: 2px;
border-top-right-radius: 2px;
}
.leaflet-touch .leaflet-bar a:last-child {
border-bottom-left-radius: 2px;
border-bottom-right-radius: 2px;
}
/* zoom control */
.leaflet-control-zoom-in,
.leaflet-control-zoom-out {
font: bold 18px 'Lucida Console', Monaco, monospace;
text-indent: 1px;
}
.leaflet-touch .leaflet-control-zoom-in, .leaflet-touch .leaflet-control-zoom-out {
font-size: 22px;
}
/* layers control */
.leaflet-control-layers {
box-shadow: 0 1px 5px rgba(0,0,0,0.4);
background: #fff;
border-radius: 5px;
}
.leaflet-control-layers-toggle {
background-image: url(images/layers.png);
width: 36px;
height: 36px;
}
.leaflet-retina .leaflet-control-layers-toggle {
background-image: url(images/layers-2x.png);
background-size: 26px 26px;
}
.leaflet-touch .leaflet-control-layers-toggle {
width: 44px;
height: 44px;
}
.leaflet-control-layers .leaflet-control-layers-list,
.leaflet-control-layers-expanded .leaflet-control-layers-toggle {
display: none;
}
.leaflet-control-layers-expanded .leaflet-control-layers-list {
display: block;
position: relative;
}
.leaflet-control-layers-expanded {
padding: 6px 10px 6px 6px;
color: #333;
background: #fff;
}
.leaflet-control-layers-scrollbar {
overflow-y: scroll;
overflow-x: hidden;
padding-right: 5px;
}
.leaflet-control-layers-selector {
margin-top: 2px;
position: relative;
top: 1px;
}
.leaflet-control-layers label {
display: block;
}
.leaflet-control-layers-separator {
height: 0;
border-top: 1px solid #ddd;
margin: 5px -10px 5px -6px;
}
/* Default icon URLs */
.leaflet-default-icon-path {
background-image: url(images/marker-icon.png);
}
/* attribution and scale controls */
.leaflet-container .leaflet-control-attribution {
background: #fff;
background: rgba(255, 255, 255, 0.7);
margin: 0;
}
.leaflet-control-attribution,
.leaflet-control-scale-line {
padding: 0 5px;
color: #333;
}
.leaflet-control-attribution a {
text-decoration: none;
}
.leaflet-control-attribution a:hover {
text-decoration: underline;
}
.leaflet-container .leaflet-control-attribution,
.leaflet-container .leaflet-control-scale {
font-size: 11px;
}
.leaflet-left .leaflet-control-scale {
margin-left: 5px;
}
.leaflet-bottom .leaflet-control-scale {
margin-bottom: 5px;
}
.leaflet-control-scale-line {
border: 2px solid #777;
border-top: none;
line-height: 1.1;
padding: 2px 5px 1px;
font-size: 11px;
white-space: nowrap;
overflow: hidden;
-moz-box-sizing: border-box;
box-sizing: border-box;
background: #fff;
background: rgba(255, 255, 255, 0.5);
}
.leaflet-control-scale-line:not(:first-child) {
border-top: 2px solid #777;
border-bottom: none;
margin-top: -2px;
}
.leaflet-control-scale-line:not(:first-child):not(:last-child) {
border-bottom: 2px solid #777;
}
.leaflet-touch .leaflet-control-attribution,
.leaflet-touch .leaflet-control-layers,
.leaflet-touch .leaflet-bar {
box-shadow: none;
}
.leaflet-touch .leaflet-control-layers,
.leaflet-touch .leaflet-bar {
border: 2px solid rgba(0,0,0,0.2);
background-clip: padding-box;
}
/* popup */
.leaflet-popup {
position: absolute;
text-align: center;
margin-bottom: 20px;
}
.leaflet-popup-content-wrapper {
padding: 1px;
text-align: left;
border-radius: 12px;
}
.leaflet-popup-content {
margin: 13px 19px;
line-height: 1.4;
}
.leaflet-popup-content p {
margin: 18px 0;
}
.leaflet-popup-tip-container {
width: 40px;
height: 20px;
position: absolute;
left: 50%;
margin-left: -20px;
overflow: hidden;
pointer-events: none;
}
.leaflet-popup-tip {
width: 17px;
height: 17px;
padding: 1px;
margin: -10px auto 0;
-webkit-transform: rotate(45deg);
-moz-transform: rotate(45deg);
-ms-transform: rotate(45deg);
transform: rotate(45deg);
}
.leaflet-popup-content-wrapper,
.leaflet-popup-tip {
background: white;
color: #333;
box-shadow: 0 3px 14px rgba(0,0,0,0.4);
}
.leaflet-container a.leaflet-popup-close-button {
position: absolute;
top: 0;
right: 0;
padding: 4px 4px 0 0;
border: none;
text-align: center;
width: 18px;
height: 14px;
font: 16px/14px Tahoma, Verdana, sans-serif;
color: #c3c3c3;
text-decoration: none;
font-weight: bold;
background: transparent;
}
.leaflet-container a.leaflet-popup-close-button:hover {
color: #999;
}
.leaflet-popup-scrolled {
overflow: auto;
border-bottom: 1px solid #ddd;
border-top: 1px solid #ddd;
}
.leaflet-oldie .leaflet-popup-content-wrapper {
-ms-zoom: 1;
}
.leaflet-oldie .leaflet-popup-tip {
width: 24px;
margin: 0 auto;
-ms-filter: "progid:DXImageTransform.Microsoft.Matrix(M11=0.70710678, M12=0.70710678, M21=-0.70710678, M22=0.70710678)";
filter: progid:DXImageTransform.Microsoft.Matrix(M11=0.70710678, M12=0.70710678, M21=-0.70710678, M22=0.70710678);
}
.leaflet-oldie .leaflet-popup-tip-container {
margin-top: -1px;
}
.leaflet-oldie .leaflet-control-zoom,
.leaflet-oldie .leaflet-control-layers,
.leaflet-oldie .leaflet-popup-content-wrapper,
.leaflet-oldie .leaflet-popup-tip {
border: 1px solid #999;
}
/* div icon */
.leaflet-div-icon {
background: #fff;
border: 1px solid #666;
}
/* Tooltip */
/* Base styles for the element that has a tooltip */
.leaflet-tooltip {
position: absolute;
padding: 6px;
background-color: #fff;
border: 1px solid #fff;
border-radius: 3px;
color: #222;
white-space: nowrap;
-webkit-user-select: none;
-moz-user-select: none;
-ms-user-select: none;
user-select: none;
pointer-events: none;
box-shadow: 0 1px 3px rgba(0,0,0,0.4);
}
.leaflet-tooltip.leaflet-clickable {
cursor: pointer;
pointer-events: auto;
}
.leaflet-tooltip-top:before,
.leaflet-tooltip-bottom:before,
.leaflet-tooltip-left:before,
.leaflet-tooltip-right:before {
position: absolute;
pointer-events: none;
border: 6px solid transparent;
background: transparent;
content: "";
}
/* Directions */
.leaflet-tooltip-bottom {
margin-top: 6px;
}
.leaflet-tooltip-top {
margin-top: -6px;
}
.leaflet-tooltip-bottom:before,
.leaflet-tooltip-top:before {
left: 50%;
margin-left: -6px;
}
.leaflet-tooltip-top:before {
bottom: 0;
margin-bottom: -12px;
border-top-color: #fff;
}
.leaflet-tooltip-bottom:before {
top: 0;
margin-top: -12px;
margin-left: -6px;
border-bottom-color: #fff;
}
.leaflet-tooltip-left {
margin-left: -6px;
}
.leaflet-tooltip-right {
margin-left: 6px;
}
.leaflet-tooltip-left:before,
.leaflet-tooltip-right:before {
top: 50%;
margin-top: -6px;
}
.leaflet-tooltip-left:before {
right: 0;
margin-right: -12px;
border-left-color: #fff;
}
.leaflet-tooltip-right:before {
left: 0;
margin-left: -12px;
border-right-color: #fff;
}

39
modules/map/leaflet.html Normal file
View file

@ -0,0 +1,39 @@
<!DOCTYPE HTML>
<html lang="en">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<script src="/leaflet.js"></script>
<link rel="stylesheet" href="/leaflet.css"/>
<title>Leaflet Map</title>
<style>
html, body {
height: 100%;
padding: 0;
margin: 0;
}
#map {
/* configure the size of the map */
width: 100%;
height: 100%;
}
</style>
</head>
<body>
<div id="map"></div>
<script>
// initialize Leaflet
var map = L.map('map').setView({lon: 0, lat: 0}, 2);
// add the OpenStreetMap tiles
L.tileLayer(`/tile/{z}/{x}/{y}.png`, {
maxZoom: 19,
attribution: '&copy; <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
}).addTo(map);
// show the scale bar on the lower left corner
L.control.scale({imperial: true, metric: true}).addTo(map);
</script>
</body>
</html>

6
modules/map/leaflet.js Normal file

File diff suppressed because one or more lines are too long

152
modules/map/lib.py Normal file
View file

@ -0,0 +1,152 @@
import asyncio
import subprocess
from multiprocessing import Event, Manager
from pathlib import Path
from typing import List
from sqlalchemy.orm import Session
from modules.map.models import MapFile
from wrolpi.common import get_media_directory, walk, logger, wrol_mode_check
from wrolpi.dates import now, timedelta_to_timestamp
from wrolpi.db import optional_session, get_db_session
from wrolpi.vars import PYTEST, PROJECT_DIR
logger = logger.getChild(__name__)
IMPORT_EVENT = Event()
IMPORTING = Manager().dict()
IMPORTING['pbf'] = None
def get_map_directory() -> Path:
return get_media_directory() / 'map'
def get_pbf_directory() -> Path:
pbf_directory = get_map_directory() / 'pbf'
if not pbf_directory.is_dir():
pbf_directory.mkdir(parents=True)
return pbf_directory
def is_pbf_file(pbf: Path) -> bool:
"""Uses file command to check type of a file. Returns True if a file is an OpenStreetMap PBF file."""
cmd = ('/usr/bin/file', pbf)
try:
output = subprocess.check_output(cmd)
except FileNotFoundError:
return False
return 'OpenStreetMap Protocolbuffer Binary Format' in output.decode()
def get_pbf_paths() -> List[Path]:
"""Find all PBF files in the map/pbf directory"""
pbf_directory = get_pbf_directory()
paths = walk(pbf_directory)
return list(filter(lambda i: i.is_file() and str(i).endswith('.osm.pbf') and is_pbf_file(i), paths))
def get_or_create_map_file(pbf_path: Path, session: Session) -> MapFile:
"""Finds the MapFile row in the DB, or creates one."""
map_file: MapFile = session.query(MapFile).filter_by(path=str(pbf_path)).one_or_none()
if map_file:
return map_file
map_file = MapFile(path=pbf_path, size=pbf_path.stat().st_size)
session.add(map_file)
return map_file
async def import_pbfs(pbfs: List[str]):
if IMPORT_EVENT.is_set():
logger.warning('Map import already running...')
return
logger.warning(f'Importing: {", ".join(pbfs)}')
any_success = False
try:
IMPORT_EVENT.set()
for pbf in pbfs:
pbf = get_media_directory() / pbf
if not pbf.is_file():
logger.fatal(f'PBF file does not exist! {pbf}')
continue
with get_db_session() as session:
map_file = session.query(MapFile).filter_by(path=pbf).one_or_none()
if map_file and map_file.imported:
# Don't import a map file twice.
logger.debug(f'{pbf} is already imported')
continue
success = False
try:
IMPORTING['pbf'] = str(pbf)
await import_pbf(pbf)
success = True
any_success = True
except Exception as e:
logger.warning('Failed to run import_pbf', exc_info=e)
finally:
IMPORTING['pbf'] = None
if success:
with get_db_session(commit=True) as session:
map_file = get_or_create_map_file(pbf, session)
map_file.imported = True
finally:
if any_success:
# A map was imported, remove the tile cache files.
clear_mod_tile()
IMPORT_EVENT.clear()
async def import_pbf(pbf: Path):
"""Run the osm2pgsql binary to import a PBF map file."""
cmd = f'/bin/bash {PROJECT_DIR}/scripts/import_map.sh {pbf.absolute()}'
logger.debug(f'Running import script: {cmd}')
start = now()
proc = await asyncio.create_subprocess_shell(cmd, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)
stdout, stderr = await proc.communicate()
elapsed = now() - start
success = 'Successful' if proc.returncode == 0 else 'Unsuccessful'
logger.info(f'{success} import of {pbf.name} took {timedelta_to_timestamp(elapsed)}')
if proc.returncode != 0:
# Log all lines. Truncate long lines.
for line in stdout.decode().splitlines():
logger.debug(line[:500])
for line in stderr.decode().splitlines():
logger.error(line[:500])
raise ValueError(f'Importing PBF failed with return code {proc.returncode}')
@optional_session
def get_pbf_import_status(session: Session = None):
pbf_paths: List[Path] = get_pbf_paths()
pbfs = []
for path in pbf_paths:
map_file = get_or_create_map_file(path, session)
pbfs.append(map_file)
session.commit()
return pbfs
MOD_TILE_CACHE_DIR = Path('/var/lib/mod_tile/ajt')
@wrol_mode_check
def clear_mod_tile():
"""Remove all cached map tile files"""
if PYTEST:
return
logger.warning('Clearing map tile cache files')
if MOD_TILE_CACHE_DIR.is_dir():
cmd = ('rm', '-r', MOD_TILE_CACHE_DIR)
subprocess.check_call(cmd)

30
modules/map/models.py Normal file
View file

@ -0,0 +1,30 @@
from sqlalchemy import Column, Integer, Boolean, BigInteger
from wrolpi import dates
from wrolpi.common import Base, ModelHelper
from wrolpi.media_path import MediaPathType
RPI4_BYTES_PER_SECOND = 3963
def seconds_to_import(size: int) -> int:
"""Attempt to predict how long it will take an RPi4 to import a given PBF file."""
return max(int(size // RPI4_BYTES_PER_SECOND), 0)
class MapFile(Base, ModelHelper):
__tablename__ = 'map_file'
id = Column(Integer, primary_key=True)
path = Column(MediaPathType, unique=True, nullable=False)
imported = Column(Boolean, default=False, nullable=False)
size = Column(BigInteger)
def __json__(self):
return {
'id': self.id,
'path': self.path,
'imported': self.imported,
'size': self.size,
'time_to_import': dates.seconds_to_timestamp(seconds_to_import(self.size)),
}

View file

@ -1,11 +1,7 @@
from dataclasses import dataclass
from typing import List
@dataclass
class PBFPostRequest:
pbf_url: str
@dataclass
class PBFPostResponse:
success: str
class ImportPost:
pbfs: List[str]

View file

@ -0,0 +1,69 @@
import json
from http import HTTPStatus
from itertools import zip_longest
import mock
@mock.patch('modules.map.lib.is_pbf_file', lambda i: True)
def test_pbf_status_and_import(test_client, test_session, test_directory):
"""PBF files can be imported, and the status of the import can be monitored."""
(test_directory / 'map/pbf').mkdir(parents=True)
pbf1 = test_directory / 'map/pbf/some-country.osm.pbf'
pbf2 = test_directory / 'map/pbf/other-country.osm.pbf'
pbf1.touch()
pbf2.touch()
def check_status(response_json, expected_):
for pbf, expected_ in zip_longest(response_json['pbfs'], expected_):
assert pbf['imported'] == expected_['imported']
assert pbf['path'].endswith(expected_['path'])
assert response.json['importing'] is None
# The PBF files can be found, but have not yet been imported.
request, response = test_client.get('/api/map/pbf')
assert response.status_code == HTTPStatus.OK
expected = [
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': False, 'path': 'map/pbf/some-country.osm.pbf'},
]
check_status(response.json, expected)
with mock.patch('modules.map.lib.import_pbf') as mock_import_pbf:
body = {'pbfs': [str(pbf1), ]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Only one import is requested.
mock_import_pbf.assert_called_once()
request, response = test_client.get('/api/map/pbf')
assert response.status_code == HTTPStatus.OK
expected = [
{'imported': False, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
check_status(response.json, expected)
with mock.patch('modules.map.lib.import_pbf') as mock_import_pbf:
body = {'pbfs': [str(pbf1), str(pbf2)]}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT
# Both imports are requested, but only one is performed.
mock_import_pbf.assert_called_once()
request, response = test_client.get('/api/map/pbf')
assert response.status_code == HTTPStatus.OK
expected = [
{'imported': True, 'path': 'map/pbf/other-country.osm.pbf'},
{'imported': True, 'path': 'map/pbf/some-country.osm.pbf'},
]
check_status(response.json, expected)
def test_empty_import(test_client, test_session, test_directory):
"""Some files must be requested."""
with mock.patch('modules.map.lib.import_pbf') as mock_import_pbf:
body = {'pbfs': []}
request, response = test_client.post('/api/map/import', content=json.dumps(body))
assert response.status_code == HTTPStatus.BAD_REQUEST
mock_import_pbf.assert_not_called()

View file

@ -0,0 +1,16 @@
import pytest
from modules.map import models
@pytest.mark.parametrize('size,expected', [
(0, 0),
(-1, 0),
(17737381, 4475),
(63434267, 16006),
(87745484, 22141),
(136372996, 34411),
(116318111, 29351),
])
def test_seconds_to_import_rpi4(size, expected):
assert models.seconds_to_import(size) == expected

View file

@ -1,86 +0,0 @@
-- Initial video schema.
-- depends:
CREATE TABLE public.channel
(
id integer NOT NULL,
name text,
link text,
idempotency text,
url text,
match_regex text,
directory text,
info_json json,
info_date date,
skip_download_videos text[],
generate_posters boolean DEFAULT true,
calculate_duration boolean DEFAULT true,
download_frequency integer,
next_download date
);
CREATE SEQUENCE public.channel_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE public.channel_id_seq OWNED BY public.channel.id;
CREATE TABLE public.video
(
id integer NOT NULL,
description_path text,
ext text,
poster_path text,
source_id text,
title text,
video_path text,
caption_path text,
idempotency text,
info_json_path text,
channel_id integer,
caption text,
textsearch tsvector GENERATED ALWAYS AS (to_tsvector('english'::regconfig,
((COALESCE(title, ''::text) || ' '::text) ||
COALESCE(caption, ''::text)))) STORED,
upload_date timestamp without time zone,
duration integer,
favorite timestamp without time zone,
size bigint,
viewed timestamp without time zone,
validated_poster boolean DEFAULT false
);
CREATE SEQUENCE public.video_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE public.video_id_seq OWNED BY public.video.id;
ALTER TABLE ONLY public.channel
ALTER COLUMN id SET DEFAULT nextval('public.channel_id_seq'::regclass);
ALTER TABLE ONLY public.video
ALTER COLUMN id SET DEFAULT nextval('public.video_id_seq'::regclass);
ALTER TABLE ONLY public.channel
ADD CONSTRAINT channel_pkey PRIMARY KEY (id);
CREATE INDEX textsearch_idx ON public.video USING gin (textsearch);
CREATE INDEX video_duration_idx ON public.video USING btree (duration);
CREATE INDEX video_favorite_idx ON public.video USING btree (favorite);
CREATE INDEX video_size_idx ON public.video USING btree (size);
CREATE INDEX video_upload_date_idx ON public.video USING btree (upload_date);
CREATE INDEX video_viewed_idx ON public.video USING btree (viewed);

View file

@ -66,6 +66,7 @@ http {
autoindex_exact_size off;
alias /media/wrolpi;
}
}
}

25
scripts/import_map.sh Executable file
View file

@ -0,0 +1,25 @@
#!/bin/bash
if [ ! -f /opt/openstreetmap-carto/openstreetmap-carto.lua ]; then
echo "openstreetmap-carto.lua file is missing. Has install been completed?"
exit 1
fi
if [ ! -f /opt/openstreetmap-carto/openstreetmap-carto.style ]; then
echo "openstreetmap-carto.style file is missing. Has install been completed?"
exit 2
fi
if [ "${1}" == "" ]; then
echo "Missing PBF file argument."
exit 3
fi
if [ ! -f "${1}" ]; then
echo "PBF file does not exist! ${1}"
exit 4
fi
osm2pgsql -d gis --append --slim -G --hstore --tag-transform-script \
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \
-S /opt/openstreetmap-carto/openstreetmap-carto.style "${1}"

107
scripts/ubuntu_20.04_install.sh Executable file
View file

@ -0,0 +1,107 @@
#!/bin/bash
set -x
set -e
echo "Ubuntu 20.04 install start: $(date '+%Y-%m-%d %H:%M:%S')"
# Install npm repos.
curl -fsSL https://deb.nodesource.com/setup_14.x | bash -
# Update if we haven't updated in the last day.
[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update
# Install dependencies
apt install -y apt-transport-https ca-certificates curl gnupg-agent gcc libpq-dev software-properties-common \
postgresql-12 nginx-full nginx-doc python3.8-minimal python3.8-dev python3.8-doc python3.8-venv \
ffmpeg hostapd nodejs texlive-latex-base texlive-latex-extra chromium-browser chromium-chromedriver \
cpufrequtils network-manager
# Install Archiving tools.
npm install -g "gildas-lormeau/SingleFile#master"
npm install -g 'git+https://github.com/pirate/readability-extractor'
# Setup the virtual environment that main.py expects
pip3 --version || (
# If no pip, install pip
curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py &&
python3 /tmp/get-pip.py
)
# Install python requirements files
pip3 install -r /opt/wrolpi/requirements.txt
# Build React app
[[ ! -f /usr/local/bin/serve || ! -f /usr/bin/serve ]] && npm -g install serve
cd /opt/wrolpi/app || exit 5
npm install || npm install || npm install || npm install # try install multiple times
npm run build
# Install the WROLPi nginx config over the default nginx config.
cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf
cp /opt/wrolpi/50x.html /var/www/50x.html
/usr/sbin/nginx -s reload
# Create the WROLPi user
grep wrolpi /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
chown -R wrolpi:wrolpi /opt/wrolpi
# Give WROLPi group a few privileged commands via sudo without password.
cat >/etc/sudoers.d/90-wrolpi <<'EOF'
%wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set
EOF
chmod 660 /etc/sudoers.d/90-wrolpi
# Verify this new file is valid.
visudo -c -f /etc/sudoers.d/90-wrolpi
# Create the media directory. This should be mounted by the maintainer.
[ -d /media/wrolpi ] || mkdir /media/wrolpi
chown wrolpi:wrolpi /media/wrolpi
# Install the systemd services
cp /opt/wrolpi/systemd/wrolpi-api.service /etc/systemd/system/
cp /opt/wrolpi/systemd/wrolpi-app.service /etc/systemd/system/
cp /opt/wrolpi/systemd/wrolpi.target /etc/systemd/system/
/usr/bin/systemctl daemon-reload
systemctl enable wrolpi-api.service
systemctl enable wrolpi-app.service
# Stop the services so the user has to start them again. We don't want to run outdated services when updating.
systemctl stop wrolpi-api.service
systemctl stop wrolpi-app.service
# Configure Postgresql. Do this after the API is stopped.
sudo -u postgres psql -c '\l' | grep wrolpi || (
sudo -u postgres createuser wrolpi &&
sudo -u postgres psql -c "alter user postgres password 'wrolpi'" &&
sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'" &&
sudo -u postgres createdb -E UTF8 -O wrolpi wrolpi
)
# Initialize/upgrade the WROLPi database.
(cd /opt/wrolpi && /usr/bin/python3 /opt/wrolpi/main.py db upgrade)
# Run the map installation script.
/opt/wrolpi/scripts/ubuntu_20.04_install_map.sh
set +x
ip=$(hostname -i | cut -d' ' -f1)
if [[ $ip == *":"* ]]; then
# Don't suggest the ipv6 address.
ip=$(hostname -i | cut -d' ' -f2)
fi
echo "
WROLPi has successfully been installed!
Mount your external hard drive to /media/wrolpi if you have one.
Start the WROLPi services using:
# sudo systemctl start wrolpi.target
then navigate to: http://${ip}
Or, join to the Wifi hotspot:
SSID: WROLPi
Password: wrolpi hotspot
When on the hotspot, WROLPi is accessible at http://192.168.0.1
"

View file

@ -0,0 +1,224 @@
#! /usr/bin/bash
# This script installs OpenStreetMap tile server.
# This server uses Apache2, renderd and mod_tile to render and display map tiles.
set -x
set -e
# Update if we haven't updated in the last day.
[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update
# Install map dependencies.
apt install -y libboost-all-dev git tar unzip wget bzip2 build-essential autoconf libtool libxml2-dev libgeos-dev \
libgeos++-dev libpq-dev libbz2-dev libproj-dev munin-node munin protobuf-c-compiler libfreetype6-dev libtiff5-dev \
libicu-dev libgdal-dev libcairo2-dev libcairomm-1.0-dev libagg-dev liblua5.2-dev ttf-unifont lua5.1 liblua5.1-0-dev \
postgis postgresql-12-postgis-3 postgresql-12-postgis-3-scripts osm2pgsql gdal-bin libmapnik-dev mapnik-utils \
python3-mapnik apache2 apache2-dev libcurl4-gnutls-dev libiniparser-dev libmemcached-dev librados-dev \
fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted postgresql-contrib
/usr/bin/npm install -g carto
/usr/bin/carto -v
# Create `gis` database for map.
sudo -u postgres psql -c '\l' | grep gis || sudo -u postgres createdb -E UTF8 -O wrolpi gis
sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;" || :
sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore;" || :
sudo -u postgres psql -d gis -c "ALTER TABLE geometry_columns OWNER TO wrolpi;"
sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO wrolpi;"
# Install Ubuntu 20.04 compatible mod_tile.
git clone -b switch2osm git://github.com/lrnselfreliance/mod_tile.git /tmp/mod_tile || :
if [ ! -f /usr/lib/apache2/modules/mod_tile.so ]; then
(cd /tmp/mod_tile && ./autogen.sh && ./configure && make -j4 && make install && make install-mod_tile && ldconfig)
fi
# Install init/systemd config to start map service on startup.
cp /tmp/mod_tile/debian/renderd.init /etc/init.d/renderd
sudo chmod +x /etc/init.d/renderd
cp /tmp/mod_tile/debian/renderd.service /lib/systemd/system/
sed -ie 's/^RUNASUSER.*/RUNASUSER=wrolpi/' /etc/init.d/renderd
chmod +x /etc/init.d/renderd
[ ! -d /var/lib/mod_tile ] && mkdir /var/lib/mod_tile
chown wrolpi /var/lib/mod_tile
# Initialize gis database.
git clone git://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto || :
chown -R wrolpi:wrolpi /opt/openstreetmap-carto
cd /opt/openstreetmap-carto
if [[ ! -f /opt/openstreetmap-carto/mapnik.xml || ! -s /opt/openstreetmap-carto/mapnik.xml ]]; then
/usr/bin/carto project.mml >/opt/openstreetmap-carto/mapnik.xml
fi
# Use D.C. to initialized DB because it is so small.
wget --continue https://download.geofabrik.de/north-america/us/district-of-columbia-latest.osm.pbf \
-O /tmp/district-of-columbia-latest.osm.pbf
# Initialize indexes and global polygons. This may take multiple tries.
set +e
MAX_TRIES=3
COUNT=0
SUCCESS=false
external_data=false
dc_import=false
indexes=false
while [ "${COUNT}" -lt "${MAX_TRIES}" ]; do
[[ "${external_data}" = false ]] &&
sudo -u wrolpi /opt/openstreetmap-carto/scripts/get-external-data.py -d gis -U wrolpi &&
external_data=true
[[ "${dc_import}" = false ]] &&
# Run import in "create" mode so basic tables will be created.
sudo -u wrolpi osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script \
/opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \
-S /opt/openstreetmap-carto/openstreetmap-carto.style /tmp/district-of-columbia-latest.osm.pbf &&
dc_import=true
[[ "${indexes}" = false ]] &&
sudo -u wrolpi psql -d gis -f /opt/openstreetmap-carto/indexes.sql &&
indexes=true
if [[ "${external_data}" == true && "${dc_import}" == true && "${indexes}" == true ]]; then
SUCCESS=true
break
fi
((COUNT = COUNT + 1))
done
if [[ "${SUCCESS}" == false ]]; then
echo "Could not import map data"
exit 6
fi
set -e
cat >/usr/local/etc/renderd.conf <<'EOF'
[renderd]
num_threads=4
tile_dir=/var/lib/mod_tile
stats_file=/var/run/renderd/renderd.stats
[mapnik]
plugins_dir=/usr/lib/mapnik/3.0/input
font_dir=/usr/share/fonts/truetype
font_dir_recurse=1
[ajt]
URI=/hot/
TILEDIR=/var/lib/mod_tile
XML=/opt/openstreetmap-carto/mapnik.xml
HOST=localhost
TILESIZE=256
MAXZOOM=20
EOF
# Enable mod_tile.
cat >>/etc/apache2/conf-available/mod_tile.conf <<'EOF'
LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so
EOF
/usr/sbin/a2enconf mod_tile
# Configure Apache2 to listen on 8084.
cat >/etc/apache2/ports.conf <<'EOF'
# If you just change the port or add more ports here, you will likely also
# have to change the VirtualHost statement in
# /etc/apache2/sites-enabled/000-default.conf
Listen 8084
<IfModule ssl_module>
Listen 443
</IfModule>
<IfModule mod_gnutls.c>
Listen 443
</IfModule>
# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
EOF
cat >/etc/apache2/sites-available/000-default.conf <<'EOF'
<VirtualHost *:8084>
# The ServerName directive sets the request scheme, hostname and port that
# the server uses to identify itself. This is used when creating
# redirection URLs. In the context of virtual hosts, the ServerName
# specifies what hostname must appear in the request's Host: header to
# match this virtual host. For the default virtual host (this file) this
# value is not decisive as it is used as a last resort host regardless.
# However, you must set it for any further virtual host explicitly.
#ServerName www.example.com
ServerAdmin webmaster@localhost
LoadTileConfigFile /usr/local/etc/renderd.conf
ModTileRenderdSocketName /var/run/renderd/renderd.sock
# Timeout before giving up for a tile to be rendered
ModTileRequestTimeout 0
# Timeout before giving up for a tile to be rendered that is otherwise missing
ModTileMissingRequestTimeout 30
DocumentRoot /var/www/html
# Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
# error, crit, alert, emerg.
# It is also possible to configure the loglevel for particular
# modules, e.g.
#LogLevel info ssl:warn
ErrorLog ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined
# For most configuration files from conf-available/, which are
# enabled or disabled at a global level, it is possible to
# include a line for only one particular virtual host. For example the
# following line enables the CGI configuration for this host only
# after it has been globally disabled with "a2disconf".
#Include conf-available/serve-cgi-bin.conf
</VirtualHost>
# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
EOF
cat >/var/www/html/index.html <<'EOF'
<!DOCTYPE HTML>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<script src="/leaflet.js"></script>
<link rel="stylesheet" href="/leaflet.css" />
<style>
html, body {
height: 100%;
padding: 0;
margin: 0;
}
#map {
/* configure the size of the map */
width: 100%;
height: 100%;
}
</style>
</head>
<body>
<div id="map"></div>
<script>
// initialize Leaflet
var map = L.map('map').setView({lon: 0, lat: 0}, 2);
// add the OpenStreetMap tiles
L.tileLayer(`http://${window.location.hostname}:8084/hot/{z}/{x}/{y}.png`, {
maxZoom: 19,
attribution: '&copy; <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
}).addTo(map);
// show the scale bar on the lower left corner
L.control.scale({imperial: true, metric: true}).addTo(map);
</script>
</body>
</html>
EOF
# Copy Leaflet files to Apache's directory so they can be used offline.
cp /opt/wrolpi/modules/map/leaflet.js /var/www/html/leaflet.js
cp /opt/wrolpi/modules/map/leaflet.css /var/www/html/leaflet.css
systemctl enable renderd
systemctl stop renderd
systemctl start renderd
systemctl stop apache2
systemctl start apache2

View file

@ -4,7 +4,7 @@ Wants=wrolpi-api.service
PartOf=wrolpi.target
[Service]
ExecStart=/usr/local/bin/serve -s build -l 5000
ExecStart=/usr/bin/serve -s build -l 5000
Group=wrolpi
User=wrolpi
WorkingDirectory=/opt/wrolpi/app/

View file

@ -409,9 +409,7 @@ def remove_whitespace(s: str) -> str:
def run_after(after: callable, *args, **kwargs) -> callable:
"""
Run the `after` function sometime in the future ofter the wrapped function returns.
"""
"""Run the `after` function sometime in the future ofter the wrapped function returns."""
if not inspect.iscoroutinefunction(after):
synchronous_after = after
@ -715,7 +713,7 @@ def walk(path: Path) -> Generator[Path, None, None]:
# These characters are invalid in Windows or Linux.
INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*]')
INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*%]')
def escape_file_name(name: str) -> str:

View file

@ -19,8 +19,6 @@ from wrolpi.db import postgres_engine, get_db_args
from wrolpi.downloader import DownloadManager
from wrolpi.root_api import BLUEPRINTS, api_app
os.environ['DB_PORT'] = '54321'
def get_test_db_engine():
suffix = str(uuid1()).replace('-', '')

View file

@ -1,6 +1,7 @@
import logging
from datetime import datetime, date, timezone
from datetime import datetime, date, timezone, timedelta
from enum import Enum
from typing import Union
import pytz
from sqlalchemy import types
@ -97,6 +98,38 @@ def from_timestamp(timestamp: float) -> datetime:
return local_timezone(datetime.fromtimestamp(timestamp))
def seconds_to_timestamp(seconds: Union[int, float]) -> str:
"""Convert an integer into a timestamp string."""
seconds = int(seconds)
weeks = seconds // Seconds.week
seconds = seconds % Seconds.week
days = seconds // Seconds.day
seconds = seconds % Seconds.day
hours = seconds // Seconds.hour
seconds = seconds % Seconds.hour
minutes = seconds // Seconds.minute
seconds = seconds % Seconds.minute
timestamp = f'{hours:02}:{minutes:02}:{seconds:02}'
if weeks:
timestamp = f'{weeks}w {days}d {timestamp}'
elif days:
timestamp = f'{days}d {timestamp}'
return timestamp
def timedelta_to_timestamp(delta: timedelta) -> str:
"""Convert a datetime.timestamp into a timestamp string.
>>> timedelta_to_timestamp(timedelta(seconds=60))
'00:01:00'
>>> timedelta_to_timestamp(timedelta(weeks=1, days=1, seconds=7))
'1w 1d 00:00:07'
>>> timedelta_to_timestamp(timedelta(days=10))
'1w 3d 00:00:00'
"""
return seconds_to_timestamp(delta.total_seconds())
def recursive_replace_tz(obj, tz=pytz.utc):
"""Recursively replace the timezone of any datetimes."""
from wrolpi.common import recursive_map

View file

@ -26,11 +26,9 @@ def get_db_args(dbname: str = None):
# Deployed in docker, use the docker database.
db_args['user'] = 'postgres'
db_args['host'] = 'db'
db_args['port'] = 5432
elif PYTEST:
# Pytest is running but we're not in docker, use the exposed docker container port.
# Pytest is running, but we're not in docker, use the exposed docker container port.
db_args['user'] = 'postgres'
db_args['port'] = 5432
return db_args

View file

@ -6,9 +6,11 @@ from decimal import Decimal
from functools import wraps
from http import HTTPStatus
from pathlib import Path
from typing import Union
from pytz import UnknownTimeZoneError
from sanic import Sanic, response, Blueprint, __version__ as sanic_version
from sanic.blueprint_group import BlueprintGroup
from sanic.request import Request
from sanic.response import HTTPResponse
from sanic_ext import validate
@ -45,7 +47,7 @@ def get_blueprint(name: str, url_prefix: str) -> Blueprint:
return bp
def add_blueprint(bp: Blueprint):
def add_blueprint(bp: Union[Blueprint, BlueprintGroup]):
BLUEPRINTS.append(bp)

View file

@ -43,17 +43,6 @@ class EchoResponse:
method: str
@dataclass
class EventObject:
name: str
is_set: str
@dataclass
class EventsResponse:
events: List[EventObject]
@dataclass
class DownloadRequest:
urls: str

View file

@ -291,6 +291,7 @@ def test_zig_zag(low, high, expected):
('foo?', 'foo'),
('foo*', 'foo'),
('foo&', 'foo&'),
('foo%', 'foo'),
]
)
def test_escape_file_name(name, expected):

View file

@ -1,9 +1,10 @@
from datetime import datetime, timedelta
import pytest
from sqlalchemy import Column, Integer
from wrolpi.common import Base
from wrolpi.dates import TZDateTime, local_timezone, strftime_ms
from wrolpi.dates import TZDateTime, local_timezone, strftime_ms, timedelta_to_timestamp
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.test.common import TestAPI, wrap_test_db, skip_circleci
@ -60,3 +61,20 @@ class TestDates(TestAPI):
tt = session.query(TestTable).one()
# Datetime is incremented by an hour.
self.assertEqual(tt.dt, local_timezone(datetime(2021, 10, 5, 17, 20, 10, 346823)))
@pytest.mark.parametrize('td,expected', [
(timedelta(seconds=0), '00:00:00'),
(timedelta(seconds=1), '00:00:01'),
(timedelta(seconds=60), '00:01:00'),
(timedelta(seconds=61), '00:01:01'),
(timedelta(seconds=65), '00:01:05'),
(timedelta(days=1), '1d 00:00:00'),
(timedelta(seconds=86461), '1d 00:01:01'),
(timedelta(weeks=1, days=1, seconds=7), '1w 1d 00:00:07'),
(timedelta(weeks=16, days=3, hours=17, minutes=46, seconds=39), '16w 3d 17:46:39'),
(timedelta(days=10), '1w 3d 00:00:00'),
(timedelta(days=10, microseconds=1000), '1w 3d 00:00:00'),
])
def test_timedelta_to_timestamp(td, expected):
assert timedelta_to_timestamp(td) == expected

View file

@ -1,4 +1,4 @@
__version__ = '0.3.5-beta'
__version__ = '0.4.0-beta'
import subprocess