mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-25 18:40:06 -04:00
Creating script which migrates map database to new database for custom niceness.
This commit is contained in:
Roland
parent
28fbc8f197
commit
7324e83626
15 changed files with 126 additions and 57 deletions
|
|
@ -0,0 +1,3 @@
|
|||
[Service]
|
||||
# WROLPi's map uses a lot of processing power. Change niceness of map database to prevent blocking of other services.
|
||||
Nice=19
|
||||
|
|
@ -6,7 +6,7 @@ After=network.target auditd.service
|
|||
[Service]
|
||||
# Delay startup so WROLPi App and API have time to start.
|
||||
ExecStartPre=/bin/sleep 30
|
||||
# Map is intense, but not a high priority.
|
||||
# Map is uses a lot of resources, but not a high priority.
|
||||
Nice=19
|
||||
ExecStart=/usr/bin/renderd -f
|
||||
User=_renderd
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
[Unit]
|
||||
Description=WROLPi services group
|
||||
Wants=wrolpi-api.service wrolpi-app.service nginx.service postgresql.service wrolpi-kiwix.service renderd.service apache2.service
|
||||
Wants=wrolpi-api.service wrolpi-app.service nginx.service postgresql.service wrolpi-kiwix.service renderd.service apache2.service postgresql@15-map.service
|
||||
|
|
|
|||
10
help.sh
10
help.sh
|
|
@ -200,10 +200,16 @@ fi
|
|||
echo
|
||||
# Map
|
||||
|
||||
if sudo -i -u wrolpi psql -l 2>/dev/null | grep gis >/dev/null; then
|
||||
if netstat -ant | grep LISTEN | grep 127.0.0.1:5433 >/dev/null; then
|
||||
echo "OK: Port 5433 is occupied"
|
||||
else
|
||||
echo "FAILED: Port 5433 is not occupied"
|
||||
fi
|
||||
|
||||
if sudo -iu postgres psql --port=5433 -l 2>/dev/null | grep gis >/dev/null; then
|
||||
echo "OK: Found map database"
|
||||
|
||||
if sudo -i -u wrolpi psql gis -c '\d' | grep water_polygons >/dev/null; then
|
||||
if sudo -iu postgres psql --port=5433 gis -c '\d' | grep water_polygons >/dev/null; then
|
||||
echo "OK: Map database is initialized"
|
||||
else
|
||||
echo "FAILED: Map database is not initialized"
|
||||
|
|
|
|||
16
repair.sh
16
repair.sh
|
|
@ -28,7 +28,7 @@ systemctl stop apache2 || :
|
|||
# Create the WROLPi user
|
||||
grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
|
||||
[ -f /home/wrolpi/.pgpass ] || cat >/home/wrolpi/.pgpass <<'EOF'
|
||||
127.0.0.1:5432:gis:_renderd:wrolpi
|
||||
127.0.0.1:5433:gis:_renderd:wrolpi
|
||||
127.0.0.1:5432:wrolpi:wrolpi:wrolpi
|
||||
EOF
|
||||
chmod 0600 /home/wrolpi/.pgpass
|
||||
|
|
@ -66,6 +66,8 @@ systemctl enable wrolpi-app.service
|
|||
systemctl enable wrolpi-kiwix.service
|
||||
systemctl enable wrolpi-help.service
|
||||
|
||||
# Copy config files necessary for map.
|
||||
cp -r /opt/wrolpi/etc/raspberrypios/postgresql@15-map.service.d /etc/systemd/system/
|
||||
cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf
|
||||
cp /opt/wrolpi/etc/raspberrypios/renderd.service /lib/systemd/system/renderd.service
|
||||
# Configure Apache2 to listen on 8084.
|
||||
|
|
@ -91,7 +93,7 @@ visudo -c -f /etc/sudoers.d/90-wrolpi
|
|||
# Configure Postgresql. Do this after the API is stopped.
|
||||
/opt/wrolpi/scripts/initialize_api_db.sh
|
||||
# wrolpi user is superuser so they can import maps.
|
||||
sudo -u postgres psql -c "alter user wrolpi with superuser"
|
||||
sudo -iu postgres psql -c "alter user wrolpi with superuser"
|
||||
|
||||
# Configure renderd.
|
||||
if [ ! -d /opt/openstreetmap-carto ]; then
|
||||
|
|
@ -110,7 +112,9 @@ cp /opt/wrolpi/etc/raspberrypios/mod_tile.conf /etc/apache2/conf-available/mod_t
|
|||
carto -v
|
||||
|
||||
# Initialize the map if it has not been initialized.
|
||||
sudo -u postgres psql -c '\l' | grep gis || /opt/wrolpi/scripts/initialize_map_db.sh
|
||||
systemctl enable postgresql@15-map.service || :
|
||||
systemctl start postgresql@15-map.service || :
|
||||
sudo -iu postgres psql -c '\l' --port=5433 | grep -q gis || yes | /opt/wrolpi/scripts/initialize_map_db.sh
|
||||
|
||||
cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf
|
||||
# Configure Apache2 to listen on 8084.
|
||||
|
|
@ -120,12 +124,6 @@ if [[ ${rpi} == true ]]; then
|
|||
else
|
||||
cp /opt/wrolpi/etc/debian12/000-default.conf /etc/apache2/sites-available/000-default.conf
|
||||
fi
|
||||
# Copy Leaflet files to Apache's directory so they can be used offline.
|
||||
cp /opt/wrolpi/etc/raspberrypios/index.html \
|
||||
/opt/wrolpi/modules/map/leaflet.js \
|
||||
/opt/wrolpi/modules/map/leaflet.css /var/www/html/
|
||||
chmod 644 /var/www/html/*
|
||||
systemctl enable renderd
|
||||
|
||||
# Create the media directory. This should be mounted by the maintainer.
|
||||
[ -d /media/wrolpi ] || mkdir /media/wrolpi
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ fi
|
|||
set -x
|
||||
|
||||
HAS_ICE_SHEET=false
|
||||
sudo -u postgres psql gis -c '\d' | grep icesheet_polygons && HAS_ICE_SHEET=true
|
||||
sudo -iu postgres psql --port=5433 gis -c '\d' | grep icesheet_polygons && HAS_ICE_SHEET=true
|
||||
if [ ${HAS_ICE_SHEET} = false ]; then
|
||||
echo "WROLPi: Missing icesheet_polygons. Run reset_map.sh"
|
||||
exit 4
|
||||
|
|
@ -69,36 +69,36 @@ if [[ ${1} == *.osm.pbf ]]; then
|
|||
fi
|
||||
|
||||
# Import as postgres user, then change ownership to _renderd.
|
||||
nice -n 18 sudo -u postgres osm2pgsql -d gis --create --slim -G --hstore \
|
||||
nice -n 18 sudo -iu postgres osm2pgsql -d gis --create --slim -G --hstore --port=5433 \
|
||||
--tag-transform-script /opt/openstreetmap-carto/openstreetmap-carto.lua \
|
||||
-C ${MAX_CACHE} \
|
||||
--number-processes 3 \
|
||||
-S /opt/openstreetmap-carto/openstreetmap-carto.style \
|
||||
"${pbf_path}"
|
||||
# Assign all data to _renderd so the map can be rendered.
|
||||
sudo -u postgres psql -d gis -c 'ALTER TABLE external_data OWNER TO _renderd'
|
||||
sudo -u postgres psql -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd'
|
||||
sudo -u postgres psql -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd'
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE icesheet_outlines OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE icesheet_polygons OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE ne_100m_admin_0_boundary_lines_land OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_line OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_nodes OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_point OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_polygon OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_rels OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_roads OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE planet_osm_ways OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE simplified_water_polygons OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO _renderd"
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE water_polygons OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE external_data OWNER TO _renderd'
|
||||
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd'
|
||||
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd'
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE icesheet_outlines OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE icesheet_polygons OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE ne_100m_admin_0_boundary_lines_land OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_line OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_nodes OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_point OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_polygon OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_rels OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_roads OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE planet_osm_ways OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE simplified_water_polygons OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO _renderd"
|
||||
sudo -iu postgres psql --port=5433 -d gis -c "ALTER TABLE water_polygons OWNER TO _renderd"
|
||||
elif [[ ${1} == *.dump ]]; then
|
||||
if [ ! -f "${1}" ]; then
|
||||
echo "WROLPi: File does not exist! ${1}"
|
||||
exit 7
|
||||
fi
|
||||
# Import a Postgresql dump.
|
||||
nice -n 18 pg_restore -j3 --no-owner -d gis -U _renderd -h 127.0.0.1 "${1}"
|
||||
nice -n 18 pg_restore --port=5433 -j3 --no-owner -d gis -U _renderd -h 127.0.0.1 "${1}"
|
||||
else
|
||||
echo "WROLPi: Cannot import unknown file"
|
||||
exit 8
|
||||
|
|
|
|||
|
|
@ -11,11 +11,11 @@ fi
|
|||
set -x
|
||||
|
||||
# Create WROLPi API database.
|
||||
sudo -u postgres psql -c '\l' | grep wrolpi || (
|
||||
sudo -u postgres createuser -s wrolpi # superuser so maps can be imported
|
||||
sudo -u postgres psql -c "alter user postgres password 'wrolpi'"
|
||||
sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'"
|
||||
sudo -u postgres createdb -E UTF8 -O wrolpi wrolpi
|
||||
sudo -iu postgres psql -c '\l' | grep wrolpi || (
|
||||
sudo -iu postgres createuser -s wrolpi # superuser so maps can be imported
|
||||
sudo -iu postgres psql -c "alter user postgres password 'wrolpi'"
|
||||
sudo -iu postgres psql -c "alter user wrolpi password 'wrolpi'"
|
||||
sudo -iu postgres createdb -E UTF8 -O wrolpi wrolpi
|
||||
echo "Created wrolpi database"
|
||||
)
|
||||
sudo -u wrolpi /bin/bash -c 'cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 main.py db upgrade'
|
||||
sudo -iu wrolpi /bin/bash -c 'cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 main.py db upgrade'
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
source /opt/wrolpi/wrolpi/scripts/lib.sh
|
||||
|
||||
sudo -u postgres psql -c '\l' 2>/dev/null | grep gis >/dev/null && (
|
||||
sudo -iu postgres psql -c '\l' 2>/dev/null | grep gis >/dev/null && (
|
||||
yes_or_no "Are you sure you want to initialize the map database? Data from imported maps will be lost." || exit 0
|
||||
)
|
||||
|
||||
|
|
@ -12,20 +12,23 @@ set -x
|
|||
|
||||
systemctl stop renderd
|
||||
|
||||
# Create map database if it does not exist.
|
||||
pg_createcluster 15 map --port=5433 --start
|
||||
|
||||
# Create gis (map) database.
|
||||
sudo -u postgres psql -c '\l' | grep gis || (
|
||||
sudo -u postgres createuser _renderd
|
||||
sudo -u postgres psql -c "alter user _renderd password 'wrolpi'"
|
||||
sudo -u postgres createdb -E UTF8 -O _renderd gis
|
||||
sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis"
|
||||
sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore"
|
||||
sudo -iu postgres psql -c '\l' --port=5433 | grep gis || (
|
||||
sudo -iu postgres createuser --port=5433 _renderd
|
||||
sudo -iu postgres psql --port=5433 -c "alter user _renderd password 'wrolpi'"
|
||||
sudo -iu postgres createdb --port=5433 -E UTF8 -O _renderd gis
|
||||
sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION postgis'
|
||||
sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION hstore'
|
||||
echo "Created gis database"
|
||||
)
|
||||
# Restore initial dump.
|
||||
zcat /opt/wrolpi-blobs/gis-map.dump.gz | sudo -u postgres pg_restore --no-owner --role=_renderd -d gis
|
||||
sudo -u postgres psql -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd'
|
||||
sudo -u postgres psql -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd'
|
||||
sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO _renderd"
|
||||
zcat /opt/wrolpi-blobs/gis-map.dump.gz | sudo -iu postgres pg_restore --port=5433 --no-owner --role=_renderd -d gis
|
||||
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd'
|
||||
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd'
|
||||
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE spatial_ref_sys OWNER TO _renderd'
|
||||
|
||||
# Clear map tile cache files.
|
||||
yes | /bin/bash /opt/wrolpi/scripts/clear_map_cache.sh
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ set -x
|
|||
systemctl stop wrolpi-api
|
||||
|
||||
# Delete the WROLPi API DB, if it exists.
|
||||
sudo -u postgres dropdb wrolpi
|
||||
sudo -u postgres dropuser wrolpi
|
||||
sudo -iu postgres dropdb wrolpi
|
||||
sudo -iu postgres dropuser wrolpi
|
||||
|
||||
/bin/bash /opt/wrolpi/scripts/initialize_api_db.sh
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ fi
|
|||
# Change directory to avoid "sudo" home directory warning.
|
||||
cd /tmp
|
||||
|
||||
if sudo -u postgres psql -l 2>/dev/null | grep gis >/dev/null; then
|
||||
if sudo -iu postgres psql -l 2>/dev/null | grep gis >/dev/null; then
|
||||
yes_or_no "Are you sure you want to delete the map database and cache? This can't be undone!" || exit 0
|
||||
fi
|
||||
|
||||
|
|
@ -27,11 +27,11 @@ set -x
|
|||
|
||||
systemctl stop renderd
|
||||
|
||||
sudo -u postgres dropdb gis || :
|
||||
sudo -u postgres dropuser _renderd || :
|
||||
sudo -iu postgres dropdb gis || :
|
||||
sudo -iu postgres dropuser _renderd || :
|
||||
|
||||
# Reset "imported" status of any map files.
|
||||
sudo -u postgres psql -d wrolpi -c "UPDATE map_file SET imported=false"
|
||||
sudo -iu postgres psql -d wrolpi -c "UPDATE map_file SET imported=false"
|
||||
|
||||
yes | /opt/wrolpi/scripts/initialize_map_db.sh
|
||||
|
||||
|
|
|
|||
|
|
@ -16,5 +16,8 @@ npm install || npm install || npm install || npm install # try install multiple
|
|||
# Upgrade WROLPi Help.
|
||||
/opt/wrolpi/scripts/install_help_service.sh || echo "Help install failed."
|
||||
|
||||
# Migrate map DB if necessary. Do this before repair because it will reset map if map db is empty.
|
||||
/opt/wrolpi/wrolpi/scripts/migrate_map_db.sh || echo "Map DB migration failed."
|
||||
|
||||
# Install any configs, restart services.
|
||||
/opt/wrolpi/repair.sh
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ while getopts ":hH:" option; do
|
|||
done
|
||||
|
||||
# Try to fetch the first tile multiple times if the service is starting, or if new map was imported.
|
||||
ATTEMPTS=100
|
||||
for i in {1..100}; do
|
||||
# Try to fetch the first tile.
|
||||
wget "${HOST}/hot/3/0/0.png" \
|
||||
|
|
|
|||
55
wrolpi/scripts/migrate_map_db.sh
Executable file
55
wrolpi/scripts/migrate_map_db.sh
Executable file
|
|
@ -0,0 +1,55 @@
|
|||
#! /usr/bin/env bash
|
||||
# Moves the `gis` map from default database on 5432 to a new database on 5433.
|
||||
# This script should only need to be run once. If you are having issues with your map, try
|
||||
# /opt/wrolpi/scripts/reset_map.sh
|
||||
|
||||
# Re-execute this script if it wasn't called with sudo.
|
||||
if [ $EUID != 0 ]; then
|
||||
sudo "$0" "$@"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
sudo -iu postgres psql -c '\l' --port=5433 2>/dev/null | grep -q gis && (
|
||||
echo "Map DB already migrated."
|
||||
exit 0
|
||||
)
|
||||
|
||||
function cleanup() {
|
||||
if [ -e /tmp/map.dump ]; then
|
||||
rm -rf /tmp/map.dump
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
cleanup
|
||||
|
||||
systemctl stop renderd
|
||||
|
||||
# Create new cluster for the map.
|
||||
pg_createcluster 15 map --port=5433 --start
|
||||
|
||||
# Create user permissions for map user.
|
||||
sudo -iu postgres psql -c '\l' --port=5433 | grep gis || (
|
||||
sudo -iu postgres createuser --port=5433 _renderd
|
||||
sudo -iu postgres psql --port=5433 -c "alter user _renderd password 'wrolpi'"
|
||||
sudo -iu postgres createdb --port=5433 -E UTF8 -O _renderd gis
|
||||
sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION postgis'
|
||||
sudo -iu postgres psql -d gis --port=5433 -c 'CREATE EXTENSION hstore'
|
||||
)
|
||||
|
||||
# Override niceness of new map database.
|
||||
cp -r /opt/wrolpi/etc/raspberrypios/postgresql@15-map.service.d /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
systemctl restart postgresql@15-map.service
|
||||
|
||||
# Dump DB, then restore in different DB.
|
||||
sudo -iu postgres pg_dump -Fd gis --port=5432 -j3 -f /tmp/map.dump && # 5432 is old DB.
|
||||
sudo -iu postgres pg_restore --port=5433 -j3 -d gis /tmp/map.dump && # 5433 is new DB.
|
||||
sudo -iu postgres dropdb --port=5432 gis
|
||||
|
||||
# Update openstreetmap-carto to use the new database.
|
||||
grep -q 'port: 5433' /opt/openstreetmap-carto/project.mml || (
|
||||
# Append port line after dbname configuration line.
|
||||
sed -i '/dbname: "gis"/a \ port: 5433' /opt/openstreetmap-carto/project.mml
|
||||
(cd /opt/openstreetmap-carto/ && carto project.mml >mapnik.xml)
|
||||
)
|
||||
|
|
@ -543,7 +543,7 @@ async def test_post_vin_number_decoder(async_client):
|
|||
'region': 'North America',
|
||||
'serial': None,
|
||||
'transmission': None,
|
||||
'years': '1995'}
|
||||
'years': '2025,1995'}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
0.15.0-beta
|
||||
0.15.1-beta
|
||||
Loading…
Add table
Add a link
Reference in a new issue