From 888352c984035433d6349540166629d62e6bf4bd Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 8 Mar 2022 13:26:03 -0700 Subject: [PATCH] Squashed commit of the following: commit 26bde0fb48d59a86c2fb9f53a7218d49f22d4860 Author: Roland Date: Tue Mar 8 13:25:56 2022 -0700 Using osm2pgsql append to preserve other imported maps. commit c6c52623eaf02e691b1e75dcf39c3a8334ead55a Author: Roland Date: Tue Mar 8 08:20:38 2022 -0700 Using placeholder instead of loader when table is loading. commit 44fec6490e26603c88a98f6bc70fc18686e17f09 Author: Roland Date: Mon Mar 7 20:58:34 2022 -0700 Simplifying tableRow function. commit 11b8b57f1fa59556397620f3cad9a33a0237501d Author: Roland Date: Mon Mar 7 20:53:51 2022 -0700 Removing scientific humanFileSize. commit 0a09d0a6963afebed7f388aad71bbdd3151d2e9e Author: Roland Date: Mon Mar 7 20:46:22 2022 -0700 Fixing active loader. commit 07f4deeac861351d7483ef57278eb2e0941e31fb Author: Roland Date: Mon Mar 7 20:40:44 2022 -0700 Increasing recommended map file size. commit af3ac492c81f4c8f84563a448bc272ae9334d32d Author: Roland Date: Mon Mar 7 20:35:13 2022 -0700 Reworking logic order for simplicity. Fetching downloads on interval. commit 3d9d31fcec87ed02ced7f67c34907faa5a3b54e5 Author: Roland Date: Mon Mar 7 20:24:39 2022 -0700 Handling when pbf import status are loading. commit 3798961a48e7c9eb165ca4e4051f239dbb088d0c Author: Roland Date: Mon Mar 7 19:53:50 2022 -0700 Fetching import status every 30 seconds. commit f7a8193b0955d14ead8090aa74ce3c5a681b57ec Author: Roland Date: Mon Mar 7 19:33:17 2022 -0700 Stopping renderd during installation. commit 5bf050b59e0a6e391184596f5e6f0737dea1221d Author: Roland Date: Mon Mar 7 19:26:59 2022 -0700 Removing duplciate serve. commit 892e3315eeeb5d2516996b901a32a73a66dae078 Author: Roland Date: Mon Mar 7 19:22:14 2022 -0700 Setting bash flags in sub-install script. commit d5d2acd2cd85e089c78aa85a27c5d966222a1347 Author: Roland Date: Mon Mar 7 19:20:19 2022 -0700 Moving bulk of installation script into separate script so the latest code will be used during initial installation. commit ab030d8af834a4a8772edd06df360c51e72fbb02 Author: Roland Date: Mon Mar 7 18:40:34 2022 -0700 Removing mod_tile/ajt directory instead of entire contents of mod_tile. Showing import as relative to media directory. commit c05199db4ecf56f4d096051b58689457882af12a Author: Roland Date: Mon Mar 7 18:39:59 2022 -0700 Remvoing marker from map. commit ba5df4cf62d61b55281277a6254836e973c766ae Author: Roland Date: Mon Mar 7 11:57:41 2022 -0700 Creating import_map.sh script. commit 1a16e46130e775efe375adb22917ba12e44855cd Author: Roland Date: Mon Mar 7 11:11:01 2022 -0700 Increasing ram usage during import to avoid hanging import. commit 0f5ce1a9f806f1c0878dd804b04587b6340ffd16 Author: Roland Date: Sat Mar 5 19:19:47 2022 -0700 Increasing map_file.size size. commit eb6e01512af59fb4a18ab15643c7bc61f3d48e53 Author: Roland Date: Sat Mar 5 18:55:55 2022 -0700 Copying leaflet js/css files into docker container and to Apache directory so they will be available offline. commit 5b687324e119f021673f42e8d079b83034b70d53 Author: Roland Date: Sat Mar 5 18:36:06 2022 -0700 Matching tab height on Map page. commit 8031786d672e18366e9a2eb2d7bdfcd8611d27f3 Author: Roland Date: Sat Mar 5 18:34:20 2022 -0700 Shrinking HelpPopup Icon. commit 5ea94488e43bbc5c414e61c9173a3cfa908c7104 Author: Roland Date: Sat Mar 5 18:31:30 2022 -0700 Testing if no PBFs are requested to be imported. commit c4e86be1faed78bd3af1f3487471c01a1958a433 Author: Roland Date: Sat Mar 5 18:18:48 2022 -0700 Using Icon instead of Button for HelpPopup. commit 51d94f6d85885803813544fea16a56372babaa82 Author: Roland Date: Sat Mar 5 18:07:03 2022 -0700 PBF files are sorted by the api. commit 40f3a7ff04a83db0153f64a88a37f1d56ce043aa Author: Roland Date: Fri Mar 4 18:35:51 2022 -0700 Adding help popup. commit 1da8d9fa403ec2a3b3c16c11a3937de37c895a35 Author: Roland Date: Fri Mar 4 09:41:01 2022 -0700 Moving message up. Clearing cached tiles after import. commit 3192adf95192554c194c9e0a8aeb149822462682 Author: Roland Date: Thu Mar 3 21:37:13 2022 -0700 Removing osm2pgsql from api docker because it seems to be unsupported. commit 0e10522f3cdc7edd374c14f230bb2dbd68ab1a94 Author: Roland Date: Thu Mar 3 21:36:39 2022 -0700 Disabling import button for docker. Adding download instructions. commit e1d8f55d159fb4d4ef2ece859ef7cc1059885791 Author: Roland Date: Thu Mar 3 21:14:55 2022 -0700 Map files can be imported on RPi. commit 3128530e09968430a71b27cd2e75a567f85a708b Author: Roland Date: Thu Mar 3 15:31:54 2022 -0700 Adding shm_size to increase size to render large map tiles. commit f3cf80fae0ce46f44ea9ac5033f68a32d0197ae3 Author: Roland Date: Wed Mar 2 10:39:18 2022 -0700 Can get status of map imports. commit 3b7bbce004cd55a5d89980c2c09124b72c450bc6 Author: Roland Date: Tue Mar 1 20:51:13 2022 -0700 Adding map link to 50x.html. Swapping links using hostname to make it easier on the user. commit a8b6b60fe182233f40c2f1ecd825636c3d37fd23 Author: Roland Date: Tue Mar 1 12:01:50 2022 -0700 Moving Inventory link to end. commit c5346b8dc2ff83262d6fd153130912809dfd7574 Author: Roland Date: Tue Mar 1 12:00:47 2022 -0700 Squashed commit of the following: commit 39f0b8f3fc049fffd140d551cd91d7d9dba6d416 Author: Roland Date: Tue Mar 1 12:00:13 2022 -0700 Starting map refresh function. commit 2fac9ddf07e378389ddd85ca972f33b3323d30ac Author: Roland Date: Mon Feb 28 16:43:47 2022 -0700 Updating readme with map requirements. commit 8325e1ea89537128329e313ab4b5588f8809d1c9 Author: Roland Date: Mon Feb 28 16:05:35 2022 -0700 Fixing order of map db init scripts. Creating index.html to display map. Using :8084 index file to display the map. commit 8f81539dd9d25fc26607f472c156d41ddcfc11f5 Author: Roland Date: Sun Feb 27 19:17:14 2022 -0700 Installing apache2. commit c7aaef07f8291070aeaf5e7df553e34aca0719ec Author: Roland Date: Sun Feb 27 18:41:07 2022 -0700 Always configure postgres extensions. Initialize gis with dc. commit aa725ed4b8ec46d17accebd96dd41508ea2d9caf Author: Roland Date: Sun Feb 27 15:48:20 2022 -0700 Map is displayed as iframe. commit de9bb92dcaf44996e7727adce7774f01895ddf52 Author: Roland Date: Sun Feb 27 12:16:56 2022 -0700 Running script in openstreetmap directory. commit f277388095d2221aa6407643ab655f26dc349eb7 Author: Roland Date: Sun Feb 27 12:06:41 2022 -0700 Bumping version for map. commit afd4aaeffb2aced2dc5c1a8f94038885c8287265 Author: Roland Date: Sun Feb 27 11:59:31 2022 -0700 Checking that the installation script is on an rpi. commit a799382ceac3c201fc01d41fd63e37607282831a Author: Roland Date: Sun Feb 27 11:49:21 2022 -0700 Only running mod_tile install once. commit 0b98bb19cd4d86c9630a863cab4eb46410d1c9d1 Author: Roland Date: Fri Feb 25 11:31:21 2022 -0700 Building out map install script. commit 7c799775c11df573fa6cd65451adea28c0832cb0 Author: Roland Date: Thu Feb 24 12:16:53 2022 -0700 Creating OSM installation script. commit d3b93a3ee97d1ae4a00b6a0423ed59ea8b897ccd Author: Roland Date: Tue Feb 22 15:39:04 2022 -0700 Removing unused map module. --- .gitignore | 1 + 50x.html | 23 +- README.md | 9 +- .../2fb39c1861c7_create_mapfile_table.py | 40 ++ app/package.json | 4 +- app/src/App.js | 2 + app/src/api.js | 21 +- app/src/components/Admin.js | 5 + app/src/components/Common.js | 42 +- app/src/components/Map.js | 250 +++++++ app/src/components/Nav.js | 3 +- docker-compose.yml | 21 + docker/api/Dockerfile | 3 +- docker/app/Dockerfile | 7 +- install.sh | 115 +--- main.py | 2 +- modules/map/api.py | 71 +- modules/map/common.py | 3 - modules/map/leaflet.css | 644 ++++++++++++++++++ modules/map/leaflet.html | 39 ++ modules/map/leaflet.js | 6 + modules/map/lib.py | 146 ++++ modules/map/models.py | 21 + modules/map/schema.py | 10 +- modules/map/test/test_api.py | 69 ++ modules/map/test/{__init__.py => test_lib.py} | 0 nginx.conf | 1 + scripts/import_map.sh | 25 + scripts/ubuntu_20.04_install.sh | 105 +++ scripts/ubuntu_20.04_install_map.sh | 193 ++++++ systemd/wrolpi-app.service | 2 +- wrolpi/common.py | 4 +- wrolpi/root_api.py | 4 +- wrolpi/schema.py | 11 - wrolpi/version.py | 2 +- 35 files changed, 1721 insertions(+), 183 deletions(-) create mode 100644 alembic/versions/2fb39c1861c7_create_mapfile_table.py create mode 100644 app/src/components/Map.js delete mode 100644 modules/map/common.py create mode 100644 modules/map/leaflet.css create mode 100644 modules/map/leaflet.html create mode 100644 modules/map/leaflet.js create mode 100644 modules/map/lib.py create mode 100644 modules/map/models.py create mode 100644 modules/map/test/test_api.py rename modules/map/test/{__init__.py => test_lib.py} (100%) create mode 100755 scripts/import_map.sh create mode 100755 scripts/ubuntu_20.04_install.sh create mode 100755 scripts/ubuntu_20.04_install_map.sh diff --git a/.gitignore b/.gitignore index 9dbc1483..08f8ef87 100644 --- a/.gitignore +++ b/.gitignore @@ -119,6 +119,7 @@ docker/archive/package-lock.json # open street map files *.pbf +mapnik.xml # docker override docker-compose.override.yml diff --git a/50x.html b/50x.html index 2d1cf8de..c0371e9a 100644 --- a/50x.html +++ b/50x.html @@ -21,7 +21,28 @@

Can't start the services?

-

You can view the media directory by clicking here

+

You can view the media directory by navigating to /media

+ +

The map may be available by navigating to port 8084

+ + diff --git a/README.md b/README.md index 399377a7..cb190df7 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ network so that any person with a laptop/tablet/phone can connect and use the da - [x] Food Inventory management - [x] Wi-Fi Hotspot - [x] Universal search -- [ ] Map viewer & downloader +- [x] Map viewer & downloader - [ ] eBook viewer - [ ] Wiki viewer & downloader - [ ] Synchronizer and/or duplicator @@ -51,9 +51,12 @@ You can try out WROLPi by running the docker containers. * `docker-compose up -d db` 5. Initialize the database * `docker-compose run --rm api db upgrade` -6. Start the docker containers +6. Create volumes + * `docker volume create --name=openstreetmap-data` + * `docker volume create --name=openstreetmap-rendered-tiles` +7. Start the docker containers * `docker-compose up` -7. Navigate to WROLPi +8. Navigate to WROLPi * http://0.0.0.0:8080 # RaspberryPi Install diff --git a/alembic/versions/2fb39c1861c7_create_mapfile_table.py b/alembic/versions/2fb39c1861c7_create_mapfile_table.py new file mode 100644 index 00000000..3f362f0c --- /dev/null +++ b/alembic/versions/2fb39c1861c7_create_mapfile_table.py @@ -0,0 +1,40 @@ +"""Create MapFile table. + +Revision ID: 2fb39c1861c7 +Revises: 36cae041ad31 +Create Date: 2022-03-03 19:13:39.997580 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '2fb39c1861c7' +down_revision = '36cae041ad31' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('''CREATE TABLE map_file ( + id SERIAL PRIMARY KEY, + path TEXT NOT NULL, + imported BOOLEAN DEFAULT FALSE, + size BIGINT + )''') + + if not DOCKERIZED: + session.execute('ALTER TABLE public.map_file OWNER TO wrolpi') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('DROP TABLE IF EXISTS map_file') diff --git a/app/package.json b/app/package.json index 829604d4..8b32f9aa 100644 --- a/app/package.json +++ b/app/package.json @@ -3,7 +3,6 @@ "version": "0.1.0", "private": true, "dependencies": { - "dot-prop": "^5.1.1", "lodash": "^4.17.19", "npm": "^6.14.7", "query-string": "^6.13.0", @@ -16,8 +15,7 @@ "react-semantic-toasts": "^0.6.5", "react-timezone-select": "0.8.4", "semantic-ui-offline": "^2.4.1-1", - "semantic-ui-react": "^0.88.2", - "yargs-parser": "^18.1.2" + "semantic-ui-react": "^0.88.2" }, "scripts": { "start": "react-scripts start", diff --git a/app/src/App.js b/app/src/App.js index 9c58d903..2ed6b604 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -15,6 +15,7 @@ import {Saver} from "./components/Upload"; import {MoreButton, PageContainer, SearchInput} from "./components/Common"; import {FilesRoute} from "./components/Files"; import {useSearch, useVersion} from "./hooks/customHooks"; +import {MapRoute} from "./components/Map"; function Welcome() { const {searchStr, setSearchStr, archives, videos} = useSearch(); @@ -88,6 +89,7 @@ export default function App() { + diff --git a/app/src/api.js b/app/src/api.js index ad7e52b2..bcda0b2b 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -388,7 +388,6 @@ export async function getDownloaders() { let response = await apiGet(`${API_URI}/downloaders`); return await response.json(); } catch (e) { - console.log('getDownloaders', e); return {}; } } @@ -512,3 +511,23 @@ export async function setThrottle(on) { } } } + +export async function getMapImportStatus() { + let response = await apiGet(`${API_URI}/map/pbf`); + return await response.json(); +} + +export async function importPbfs(pbfs) { + let body = {'pbfs': pbfs}; + let response = await apiPost(`${API_URI}/map/import`, body); + if (response.status === 204) { + return null; + } else { + toast({ + type: 'error', + title: 'Error!', + description: 'Could not start import! See server logs.', + time: 5000, + }); + } +} diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 0e2a1084..51bf1cec 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -328,6 +328,11 @@ class Downloads extends React.Component { async componentDidMount() { await this.fetchDownloads(); + this.intervalId = setInterval(this.fetchDownloads, 1000 * 10); + } + + componentWillUnmount() { + clearInterval(this.intervalId); } closeStop = () => { diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 84d59b86..d3e8f0ef 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -1,10 +1,22 @@ import React, {useEffect, useState} from "react"; -import {Card, Checkbox, Container, Form, Icon, Image, Input, Menu, Pagination, Responsive} from 'semantic-ui-react'; +import { + Card, + Checkbox, + Container, + Form, + Icon, + Image, + Input, + Menu, + Pagination, + Popup, + Responsive +} from 'semantic-ui-react'; import {Link, NavLink} from "react-router-dom"; import Button from "semantic-ui-react/dist/commonjs/elements/Button"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; -import {getConfig, getSettings} from "../api"; +import {getSettings} from "../api"; import {useDownloaders, useHotspot, useThrottle} from "../hooks/customHooks"; export const API_URI = `http://${window.location.host}/api`; @@ -256,15 +268,13 @@ export function secondsToTimestamp(seconds) { } export function humanFileSize(bytes, si = false, dp = 1) { - const thresh = si ? 1000 : 1024; + const thresh = 1024; if (Math.abs(bytes) < thresh) { return bytes + ' B'; } - const units = si - ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] - : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; + const units = ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; let u = -1; const r = 10 ** dp; @@ -685,3 +695,23 @@ export function emptyToNull(obj) { } return obj; } + +export function HelpPopup({icon, size, content, position}) { + return } + /> +} + +export function minutesToTimestamp(minutes) { + minutes = Math.round(minutes); + const hours = String(Math.floor(minutes / 60)).padStart(1, '0'); + minutes = minutes - (hours * 60); + minutes = String(minutes).padStart(2, '0'); + return `${hours}:${minutes}` +} diff --git a/app/src/components/Map.js b/app/src/components/Map.js new file mode 100644 index 00000000..ccf1738f --- /dev/null +++ b/app/src/components/Map.js @@ -0,0 +1,250 @@ +import React from "react"; +import {HelpPopup, humanFileSize, minutesToTimestamp, PageContainer, TabLinks, WROLModeMessage} from "./Common"; +import {Route} from "react-router-dom"; +import {getMapImportStatus, importPbfs} from "../api"; +import Table from "semantic-ui-react/dist/commonjs/collections/Table"; +import {Button, Checkbox, Icon, Loader, Placeholder} from "semantic-ui-react"; +import Message from "semantic-ui-react/dist/commonjs/collections/Message"; + +class ManageMap extends React.Component { + constructor(props) { + super(props); + this.state = { + pbfs: null, + importing: null, + import_running: false, + selectedPaths: [], + dockerized: null, + } + } + + async componentDidMount() { + await this.fetchImportStatus(); + this.intervalId = setInterval(this.fetchImportStatus, 1000 * 30); + } + + componentWillUnmount() { + clearInterval(this.intervalId); + } + + fetchImportStatus = async () => { + const importStatus = await getMapImportStatus(); + this.setState({ + pbfs: importStatus['pbfs'], + importing: importStatus['importing'], + import_running: importStatus['import_running'], + dockerized: importStatus['dockerized'], + }); + } + + import = async (e) => { + e.preventDefault(); + + this.setState({'ready': false}); + const {pbfs, selectedPaths} = this.state; + if (!selectedPaths || selectedPaths.length === 0) { + // No PBFs are selected, import all. + let paths = []; + for (let i = 0; i < pbfs.length; i++) { + paths = paths.concat([pbfs[i].path]); + } + await importPbfs(paths); + } else { + await importPbfs(selectedPaths); + } + await this.fetchImportStatus(); + } + + handleCheckbox = (checked, pbf) => { + let {selectedPaths} = this.state; + if (checked === true) { + selectedPaths = selectedPaths.concat([pbf.path]); + } else { + const index = selectedPaths.indexOf(pbf.path); + if (index > -1) { + selectedPaths.splice(index, 1); + } + } + this.setState({selectedPaths}); + } + + tableRow = (pbf, disabled) => { + let ref = React.createRef(); + const {size, path, imported} = pbf; + const raspberryPi4Estimation = 3000000; + let sizeCells = (<> + {humanFileSize(size)} + {humanFileSize(size * 25)} + {minutesToTimestamp(size / raspberryPi4Estimation)} + ); + return + + this.handleCheckbox(data.checked, pbf)} + /> + + + {path} + + + {path === this.state.importing ? + : + imported ? 'yes' : 'no'} + + {size !== null ? sizeCells : } + + } + + render() { + const {pbfs, selectedPaths, import_running, importing, dockerized} = this.state; + + let dockerWarning; + if (dockerized === true) { + dockerWarning = + + + + Maps are not fully supported in a Docker container! + + +

Only one PBF can be imported and displayed in the docker container.

+ +

To import a map file, run the following docker-compose commands:

+ 
  docker-compose stop map
+ 
  docker-compose rm map
+ 
  docker-compose run --rm -v /absolute/path/to/map.osm.pbf:/data.osm.pbf
+                        -v openstreetmap-data:/var/lib/postgresql/12/main map import
+
+ +

Be sure to change /absolute/path/to/map.osm.pbf!

+ +

After you have imported a new PBF file, you need to clear the rendered tile cache:

+ 
  docker volume rm openstreetmap-rendered-tiles
+ 
  docker volume create openstreetmap-rendered-tiles
+ +

Start your map container:

+ 
  docker-compose up -d map
+ + + } + + let downloadMessage = + + + + Where do I get map files? + + +

You can download map files from  + https://download.geofabrik.de/ +

+ +

Download only the areas you need. Large regions like all of Asia, or the entire + planet are most likely too large and won't render quickly. I recommend only + importing files less than 1GB on a Raspberry Pi.

+ +

Only *.osm.pbf files are supported!

+ +

Place downloaded map files into map/pbf so they can be imported here.

+ + + + let importingLoader = ( + Importing: {importing} + ); + + let disabled = dockerized || !pbfs || pbfs.length === 0 || import_running; + let importButton = ; + + let rows; + if (!pbfs) { + // Fetch request is not complete. + rows = + + + + + + + + ; + } else if (pbfs.length === 0) { + rows = + No PBF map files were found in map/pbf + ; + } else { + rows = pbfs.map(i => this.tableRow(i, disabled)); + } + + let spaceHelpPopup = ; + let timeHelpPopup = ; + + return ( + + + {dockerWarning} + {downloadMessage} + {importingLoader} + + + + + PBF File + Imported + Size + Space Required {spaceHelpPopup} + Time to Import {timeHelpPopup} + + + + {rows} + + + + + + {importButton} + + + +
+ + ); + } +} + +function MapApp() { + return ( +