diff --git a/etc/raspberrypios/postgresql@15-map.service.d/override.conf b/etc/raspberrypios/postgresql@15-map.service.d/override.conf new file mode 100644 index 00000000..43164528 --- /dev/null +++ b/etc/raspberrypios/postgresql@15-map.service.d/override.conf @@ -0,0 +1,3 @@ +[Service] +# WROLPi's map uses a lot of processing power. Change niceness of map database to prevent blocking of other services. +Nice=19 \ No newline at end of file diff --git a/etc/raspberrypios/renderd.service b/etc/raspberrypios/renderd.service index cada8805..f1cfc3f3 100644 --- a/etc/raspberrypios/renderd.service +++ b/etc/raspberrypios/renderd.service @@ -6,7 +6,7 @@ After=network.target auditd.service [Service] # Delay startup so WROLPi App and API have time to start. ExecStartPre=/bin/sleep 30 -# Map is intense, but not a high priority. +# Map is uses a lot of resources, but not a high priority. Nice=19 ExecStart=/usr/bin/renderd -f User=_renderd diff --git a/etc/raspberrypios/wrolpi.target b/etc/raspberrypios/wrolpi.target index 3a15c792..3176cf34 100644 --- a/etc/raspberrypios/wrolpi.target +++ b/etc/raspberrypios/wrolpi.target @@ -1,3 +1,3 @@ [Unit] Description=WROLPi services group -Wants=wrolpi-api.service wrolpi-app.service nginx.service postgresql.service wrolpi-kiwix.service renderd.service apache2.service +Wants=wrolpi-api.service wrolpi-app.service nginx.service postgresql.service wrolpi-kiwix.service renderd.service apache2.service postgresql@15-map.service diff --git a/help.sh b/help.sh index 4f8fe2ff..709e382a 100755 --- a/help.sh +++ b/help.sh @@ -200,10 +200,16 @@ fi echo # Map -if sudo -i -u wrolpi psql -l 2>/dev/null | grep gis >/dev/null; then +if netstat -ant | grep LISTEN | grep 127.0.0.1:5433 >/dev/null; then + echo "OK: Port 5433 is occupied" +else + echo "FAILED: Port 5433 is not occupied" +fi + +if sudo -iu postgres psql --port=5433 -l 2>/dev/null | grep gis >/dev/null; then echo "OK: Found map database" - if sudo -i -u wrolpi psql gis -c '\d' | grep water_polygons >/dev/null; then + if sudo -iu postgres psql --port=5433 gis -c '\d' | grep water_polygons >/dev/null; then echo "OK: Map database is initialized" else echo "FAILED: Map database is not initialized" diff --git a/repair.sh b/repair.sh index 351ac858..b8269cbb 100755 --- a/repair.sh +++ b/repair.sh @@ -28,7 +28,7 @@ systemctl stop apache2 || : # Create the WROLPi user grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)" [ -f /home/wrolpi/.pgpass ] || cat >/home/wrolpi/.pgpass <<'EOF' -127.0.0.1:5432:gis:_renderd:wrolpi +127.0.0.1:5433:gis:_renderd:wrolpi 127.0.0.1:5432:wrolpi:wrolpi:wrolpi EOF chmod 0600 /home/wrolpi/.pgpass @@ -66,6 +66,8 @@ systemctl enable wrolpi-app.service systemctl enable wrolpi-kiwix.service systemctl enable wrolpi-help.service +# Copy config files necessary for map. +cp -r /opt/wrolpi/etc/raspberrypios/postgresql@15-map.service.d /etc/systemd/system/ cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf cp /opt/wrolpi/etc/raspberrypios/renderd.service /lib/systemd/system/renderd.service # Configure Apache2 to listen on 8084. @@ -91,7 +93,7 @@ visudo -c -f /etc/sudoers.d/90-wrolpi # Configure Postgresql. Do this after the API is stopped. /opt/wrolpi/scripts/initialize_api_db.sh # wrolpi user is superuser so they can import maps. -sudo -u postgres psql -c "alter user wrolpi with superuser" +sudo -iu postgres psql -c "alter user wrolpi with superuser" # Configure renderd. if [ ! -d /opt/openstreetmap-carto ]; then @@ -110,7 +112,9 @@ cp /opt/wrolpi/etc/raspberrypios/mod_tile.conf /etc/apache2/conf-available/mod_t carto -v # Initialize the map if it has not been initialized. -sudo -u postgres psql -c '\l' | grep gis || /opt/wrolpi/scripts/initialize_map_db.sh +systemctl enable postgresql@15-map.service || : +systemctl start postgresql@15-map.service || : +sudo -iu postgres psql -c '\l' --port=5433 | grep -q gis || yes | /opt/wrolpi/scripts/initialize_map_db.sh cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf # Configure Apache2 to listen on 8084. @@ -120,12 +124,6 @@ if [[ ${rpi} == true ]]; then else cp /opt/wrolpi/etc/debian12/000-default.conf /etc/apache2/sites-available/000-default.conf fi -# Copy Leaflet files to Apache's directory so they can be used offline. -cp /opt/wrolpi/etc/raspberrypios/index.html \ - /opt/wrolpi/modules/map/leaflet.js \ - /opt/wrolpi/modules/map/leaflet.css /var/www/html/ -chmod 644 /var/www/html/* -systemctl enable renderd # Create the media directory. This should be mounted by the maintainer. [ -d /media/wrolpi ] || mkdir /media/wrolpi diff --git a/scripts/import_map.sh b/scripts/import_map.sh index fde7d71f..26422f98 100755 --- a/scripts/import_map.sh +++ b/scripts/import_map.sh @@ -20,7 +20,7 @@ fi set -x HAS_ICE_SHEET=false -sudo -u postgres psql gis -c '\d' | grep icesheet_polygons && HAS_ICE_SHEET=true +sudo -iu postgres psql --port=5433 gis -c '\d' | grep icesheet_polygons && HAS_ICE_SHEET=true if [ ${HAS_ICE_SHEET} = false ]; then echo "WROLPi: Missing icesheet_polygons. Run reset_map.sh" exit 4 @@ -69,36 +69,36 @@ if [[ ${1} == *.osm.pbf ]]; then fi # Import as postgres user, then change ownership to _renderd. - nice -n 18 sudo -u postgres osm2pgsql -d gis --create --slim -G --hstore \ + nice -n 18 sudo -iu postgres osm2pgsql -d gis --create --slim -G --hstore --port=5433 \ --tag-transform-script /opt/openstreetmap-carto/openstreetmap-carto.lua \ -C ${MAX_CACHE} \ --number-processes 3 \ -S /opt/openstreetmap-carto/openstreetmap-carto.style \ "${pbf_path}" # Assign all data to _renderd so the map can be rendered. - sudo -u postgres psql -d gis -c 'ALTER TABLE external_data OWNER TO _renderd' - sudo -u postgres psql -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd' - sudo -u postgres psql -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd' - sudo -u postgres psql -d gis -c "ALTER TABLE icesheet_outlines OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE icesheet_polygons OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE ne_100m_admin_0_boundary_lines_land OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_line OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_nodes OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_point OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_polygon OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_rels OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_roads OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_ways OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE simplified_water_polygons OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO _renderd" - sudo -u postgres psql -d gis -c "ALTER TABLE water_polygons OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE external_data OWNER TO _renderd' + sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd' + sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd' + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE icesheet_outlines OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE icesheet_polygons OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE ne_100m_admin_0_boundary_lines_land OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_line OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_nodes OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_point OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_polygon OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_rels OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_roads OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_ways OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE simplified_water_polygons OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO _renderd" + sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE water_polygons OWNER TO _renderd" elif [[ ${1} == *.dump ]]; then if [ ! -f "${1}" ]; then echo "WROLPi: File does not exist! ${1}" exit 7 fi # Import a Postgresql dump. - nice -n 18 pg_restore -j3 --no-owner -d gis -U _renderd -h 127.0.0.1 "${1}" + nice -n 18 pg_restore --port=5433 -j3 --no-owner -d gis -U _renderd -h 127.0.0.1 "${1}" else echo "WROLPi: Cannot import unknown file" exit 8 diff --git a/scripts/initialize_api_db.sh b/scripts/initialize_api_db.sh index dddc2e31..455b9aae 100755 --- a/scripts/initialize_api_db.sh +++ b/scripts/initialize_api_db.sh @@ -11,11 +11,11 @@ fi set -x # Create WROLPi API database. -sudo -u postgres psql -c '\l' | grep wrolpi || ( - sudo -u postgres createuser -s wrolpi # superuser so maps can be imported - sudo -u postgres psql -c "alter user postgres password 'wrolpi'" - sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'" - sudo -u postgres createdb -E UTF8 -O wrolpi wrolpi +sudo -iu postgres psql -c '\l' | grep wrolpi || ( + sudo -iu postgres createuser -s wrolpi # superuser so maps can be imported + sudo -iu postgres psql -c "alter user postgres password 'wrolpi'" + sudo -iu postgres psql -c "alter user wrolpi password 'wrolpi'" + sudo -iu postgres createdb -E UTF8 -O wrolpi wrolpi echo "Created wrolpi database" ) -sudo -u wrolpi /bin/bash -c 'cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 main.py db upgrade' +sudo -iu wrolpi /bin/bash -c 'cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 main.py db upgrade' diff --git a/scripts/initialize_map_db.sh b/scripts/initialize_map_db.sh index b8405372..cfec353a 100755 --- a/scripts/initialize_map_db.sh +++ b/scripts/initialize_map_db.sh @@ -4,7 +4,7 @@ source /opt/wrolpi/wrolpi/scripts/lib.sh -sudo -u postgres psql -c '\l' 2>/dev/null | grep gis >/dev/null && ( +sudo -iu postgres psql -c '\l' 2>/dev/null | grep gis >/dev/null && ( yes_or_no "Are you sure you want to initialize the map database? Data from imported maps will be lost." || exit 0 ) @@ -12,20 +12,23 @@ set -x systemctl stop renderd +# Create map database if it does not exist. +pg_createcluster 15 map --port=5433 --start + # Create gis (map) database. -sudo -u postgres psql -c '\l' | grep gis || ( - sudo -u postgres createuser _renderd - sudo -u postgres psql -c "alter user _renderd password 'wrolpi'" - sudo -u postgres createdb -E UTF8 -O _renderd gis - sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis" - sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore" +sudo -iu postgres psql -c '\l' --port=5433 | grep gis || ( + sudo -iu postgres createuser --port=5433 _renderd + sudo -iu postgres psql --port=5433 -c "alter user _renderd password 'wrolpi'" + sudo -iu postgres createdb --port=5433 -E UTF8 -O _renderd gis + sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION postgis' + sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION hstore' echo "Created gis database" ) # Restore initial dump. -zcat /opt/wrolpi-blobs/gis-map.dump.gz | sudo -u postgres pg_restore --no-owner --role=_renderd -d gis -sudo -u postgres psql -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd' -sudo -u postgres psql -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd' -sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO _renderd" +zcat /opt/wrolpi-blobs/gis-map.dump.gz | sudo -iu postgres pg_restore --port=5433 --no-owner --role=_renderd -d gis +sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd' +sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd' +sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE spatial_ref_sys OWNER TO _renderd' # Clear map tile cache files. yes | /bin/bash /opt/wrolpi/scripts/clear_map_cache.sh diff --git a/scripts/reset_api_db.sh b/scripts/reset_api_db.sh index da6646f6..1fb64bf3 100755 --- a/scripts/reset_api_db.sh +++ b/scripts/reset_api_db.sh @@ -17,7 +17,7 @@ set -x systemctl stop wrolpi-api # Delete the WROLPi API DB, if it exists. -sudo -u postgres dropdb wrolpi -sudo -u postgres dropuser wrolpi +sudo -iu postgres dropdb wrolpi +sudo -iu postgres dropuser wrolpi /bin/bash /opt/wrolpi/scripts/initialize_api_db.sh diff --git a/scripts/reset_map.sh b/scripts/reset_map.sh index e8d1cefb..6845a7e4 100755 --- a/scripts/reset_map.sh +++ b/scripts/reset_map.sh @@ -11,7 +11,7 @@ fi # Change directory to avoid "sudo" home directory warning. cd /tmp -if sudo -u postgres psql -l 2>/dev/null | grep gis >/dev/null; then +if sudo -iu postgres psql -l 2>/dev/null | grep gis >/dev/null; then yes_or_no "Are you sure you want to delete the map database and cache? This can't be undone!" || exit 0 fi @@ -27,11 +27,11 @@ set -x systemctl stop renderd -sudo -u postgres dropdb gis || : -sudo -u postgres dropuser _renderd || : +sudo -iu postgres dropdb gis || : +sudo -iu postgres dropuser _renderd || : # Reset "imported" status of any map files. -sudo -u postgres psql -d wrolpi -c "UPDATE map_file SET imported=false" +sudo -iu postgres psql -d wrolpi -c "UPDATE map_file SET imported=false" yes | /opt/wrolpi/scripts/initialize_map_db.sh diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index c6a47561..ef63d301 100755 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -16,5 +16,8 @@ npm install || npm install || npm install || npm install # try install multiple # Upgrade WROLPi Help. /opt/wrolpi/scripts/install_help_service.sh || echo "Help install failed." +# Migrate map DB if necessary. Do this before repair because it will reset map if map db is empty. +/opt/wrolpi/wrolpi/scripts/migrate_map_db.sh || echo "Map DB migration failed." + # Install any configs, restart services. /opt/wrolpi/repair.sh diff --git a/wrolpi/scripts/initialize_map_tiles.sh b/wrolpi/scripts/initialize_map_tiles.sh index f21f94f3..34c81060 100755 --- a/wrolpi/scripts/initialize_map_tiles.sh +++ b/wrolpi/scripts/initialize_map_tiles.sh @@ -31,6 +31,7 @@ while getopts ":hH:" option; do done # Try to fetch the first tile multiple times if the service is starting, or if new map was imported. +ATTEMPTS=100 for i in {1..100}; do # Try to fetch the first tile. wget "${HOST}/hot/3/0/0.png" \ diff --git a/wrolpi/scripts/migrate_map_db.sh b/wrolpi/scripts/migrate_map_db.sh new file mode 100755 index 00000000..a02c40b5 --- /dev/null +++ b/wrolpi/scripts/migrate_map_db.sh @@ -0,0 +1,55 @@ +#! /usr/bin/env bash +# Moves the `gis` map from default database on 5432 to a new database on 5433. +# This script should only need to be run once. If you are having issues with your map, try +# /opt/wrolpi/scripts/reset_map.sh + +# Re-execute this script if it wasn't called with sudo. +if [ $EUID != 0 ]; then + sudo "$0" "$@" + exit $? +fi + +sudo -iu postgres psql -c '\l' --port=5433 2>/dev/null | grep -q gis && ( + echo "Map DB already migrated." + exit 0 +) + +function cleanup() { + if [ -e /tmp/map.dump ]; then + rm -rf /tmp/map.dump + fi +} + +trap cleanup EXIT +cleanup + +systemctl stop renderd + +# Create new cluster for the map. +pg_createcluster 15 map --port=5433 --start + +# Create user permissions for map user. +sudo -iu postgres psql -c '\l' --port=5433 | grep gis || ( + sudo -iu postgres createuser --port=5433 _renderd + sudo -iu postgres psql --port=5433 -c "alter user _renderd password 'wrolpi'" + sudo -iu postgres createdb --port=5433 -E UTF8 -O _renderd gis + sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION postgis' + sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION hstore' +) + +# Override niceness of new map database. +cp -r /opt/wrolpi/etc/raspberrypios/postgresql@15-map.service.d /etc/systemd/system/ +systemctl daemon-reload +systemctl restart postgresql@15-map.service + +# Dump DB, then restore in different DB. +sudo -iu postgres pg_dump -Fd gis --port=5432 -j3 -f /tmp/map.dump && # 5432 is old DB. + sudo -iu postgres pg_restore --port=5433 -j3 -d gis /tmp/map.dump && # 5433 is new DB. + sudo -iu postgres dropdb --port=5432 gis + +# Update openstreetmap-carto to use the new database. +grep -q 'port: 5433' /opt/openstreetmap-carto/project.mml || ( + # Append port line after dbname configuration line. + sed -i '/dbname: "gis"/a \ port: 5433' /opt/openstreetmap-carto/project.mml + (cd /opt/openstreetmap-carto/ && carto project.mml >mapnik.xml) +) diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index fdda47d4..cf5b67f6 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -543,7 +543,7 @@ async def test_post_vin_number_decoder(async_client): 'region': 'North America', 'serial': None, 'transmission': None, - 'years': '1995'} + 'years': '2025,1995'} @pytest.mark.asyncio diff --git a/wrolpi/version.txt b/wrolpi/version.txt index dd6d232e..d28fffee 100644 --- a/wrolpi/version.txt +++ b/wrolpi/version.txt @@ -1 +1 @@ -0.15.0-beta \ No newline at end of file +0.15.1-beta \ No newline at end of file