From 8e4d173791ff65f554a93c809b67692afc11210d Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 2 Jan 2025 12:02:35 -0700 Subject: [PATCH] Converting map blob to use "custom" postgres dump format so restore can use multiple threads. Renaming map db blob from gis-map.dump.gz to map-db-gis.dump. --- .gitignore | 2 +- debian-live-config/README.md | 2 +- debian-live-config/build.sh | 4 ++-- docker/osm-map-dumper/README.md | 2 +- docker/osm-map-dumper/run.sh | 3 ++- help.sh | 2 +- pi-gen/README.md | 2 +- pi-gen/build.sh | 4 ++-- pi-gen/stage2/04-wrolpi/files/.gitignore | 3 ++- scripts/initialize_map_db.sh | 2 +- scripts/install_debian_12.sh | 4 ++-- scripts/install_raspberrypios.sh | 4 ++-- scripts/reset_map.sh | 14 +++++++------- scripts/upgrade.sh | 11 +++++++++++ wrolpi/scripts/migrate_map_db.sh | 2 +- 15 files changed, 37 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 2594898a..111117b6 100644 --- a/.gitignore +++ b/.gitignore @@ -129,7 +129,7 @@ test pg_data # Directories used to build images -/debian-live-config/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz +/debian-live-config/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump /debian-live-config/build.log /debian-live-config/*.iso /pi-gen/*xz diff --git a/debian-live-config/README.md b/debian-live-config/README.md index 590cf154..1e699cc6 100644 --- a/debian-live-config/README.md +++ b/debian-live-config/README.md @@ -7,7 +7,7 @@ See https://live-team.pages.debian.net/live-manual/html/live-manual/index.en.htm ## Generate Debian 12 ISO 1. Convert a map PBF file to your initial map DB dump: - * `docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper | gzip > debian-live-config/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz` + * `docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper > debian-live-config/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump` 2. Run the build. * `sudo ./build.sh` 3. The ISO is now in the `debian-live-config` directory. diff --git a/debian-live-config/build.sh b/debian-live-config/build.sh index 16a0e689..cc45d1ff 100755 --- a/debian-live-config/build.sh +++ b/debian-live-config/build.sh @@ -12,8 +12,8 @@ if [ $EUID != 0 ]; then exit $? fi -if [ ! -f "${SCRIPT_DIR}/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz" ]; then - echo "config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz does not exist!" +if [ ! -f "${SCRIPT_DIR}/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump" ]; then + echo "config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump does not exist!" exit 1 fi diff --git a/docker/osm-map-dumper/README.md b/docker/osm-map-dumper/README.md index 693c4f3b..80cfc0b4 100644 --- a/docker/osm-map-dumper/README.md +++ b/docker/osm-map-dumper/README.md @@ -9,6 +9,6 @@ Warning: Typically adds 2GB of planetary data. Import `map.osm.pbf`, dump to `output.dump` -`# docker run --rm -v /absolute/path/to/your/map.osm.pbf:/data/region.osm.pbf $(docker build -q .) > output.dump.gz` +`# docker run --rm -v /absolute/path/to/your/map.osm.pbf:/data/region.osm.pbf $(docker build -q .) > output.dump` See: https://hub.docker.com/r/lrnselfreliance/osm-map-dumper diff --git a/docker/osm-map-dumper/run.sh b/docker/osm-map-dumper/run.sh index ab9390a7..37dbd8a1 100755 --- a/docker/osm-map-dumper/run.sh +++ b/docker/osm-map-dumper/run.sh @@ -88,7 +88,8 @@ echo >&2 "Dumping the DB" sudo -u postgres pg_dump \ --create \ --clean \ - --format tar \ + --format custom \ + --compress 9 \ --no-owner \ --no-comments \ --encoding UTF-8 \ diff --git a/help.sh b/help.sh index 709e382a..64ffebd6 100755 --- a/help.sh +++ b/help.sh @@ -230,7 +230,7 @@ else echo "FAILED: Map app did not respond" fi -check_file /opt/wrolpi-blobs/gis-map.dump.gz "Map initialization blob exists" "Map initialization blob does not exist" +check_file /opt/wrolpi-blobs/map-db-gis.dump "Map initialization blob exists" "Map initialization blob does not exist" check_file /var/www/html/leaflet.js "Leaflet.js exists" "Leaflet.js does not exist" diff --git a/pi-gen/README.md b/pi-gen/README.md index 757c5af7..f4fcb30e 100644 --- a/pi-gen/README.md +++ b/pi-gen/README.md @@ -8,7 +8,7 @@ See https://github.com/RPI-Distro/pi-gen 1. Convert a map PBF file to your initial map DB dump: - docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper | gzip > pi-gen/stage2/04-wrolpi/files/gis-map.dump.gz + docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper | gzip > pi-gen/stage2/04-wrolpi/files/map-db-gis.dump 2. Run the build (or, with a specific git branch) diff --git a/pi-gen/build.sh b/pi-gen/build.sh index ebd5a627..82e133e0 100755 --- a/pi-gen/build.sh +++ b/pi-gen/build.sh @@ -12,8 +12,8 @@ if [ $EUID != 0 ]; then exit $? fi -if [ ! -f "${SCRIPT_DIR}/stage2/04-wrolpi/files/gis-map.dump.gz" ]; then - echo "stage2/04-wrolpi/files/gis-map.dump.gz does not exist!" +if [ ! -f "${SCRIPT_DIR}/stage2/04-wrolpi/files/map-db-gis.dump" ]; then + echo "stage2/04-wrolpi/files/map-db-gis.dump does not exist!" exit 1 fi diff --git a/pi-gen/stage2/04-wrolpi/files/.gitignore b/pi-gen/stage2/04-wrolpi/files/.gitignore index 3452a300..e3d80a0a 100644 --- a/pi-gen/stage2/04-wrolpi/files/.gitignore +++ b/pi-gen/stage2/04-wrolpi/files/.gitignore @@ -1 +1,2 @@ -*.dump.gz \ No newline at end of file +*.dump.gz +*.dump diff --git a/scripts/initialize_map_db.sh b/scripts/initialize_map_db.sh index cfec353a..420f90d2 100755 --- a/scripts/initialize_map_db.sh +++ b/scripts/initialize_map_db.sh @@ -25,7 +25,7 @@ sudo -iu postgres psql -c '\l' --port=5433 | grep gis || ( echo "Created gis database" ) # Restore initial dump. -zcat /opt/wrolpi-blobs/gis-map.dump.gz | sudo -iu postgres pg_restore --port=5433 --no-owner --role=_renderd -d gis +sudo -iu postgres pg_restore --port=5433 --no-owner --role=_renderd -d gis -j3 /opt/wrolpi-blobs/map-db-gis.dump sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd' sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd' sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE spatial_ref_sys OWNER TO _renderd' diff --git a/scripts/install_debian_12.sh b/scripts/install_debian_12.sh index 5060f311..44417cc4 100755 --- a/scripts/install_debian_12.sh +++ b/scripts/install_debian_12.sh @@ -35,8 +35,8 @@ git config --global --add safe.directory /opt/openstreetmap-carto chown -R _renderd:_renderd /opt/openstreetmap-carto # Get map initialization dump. -if [[ ! -f /opt/wrolpi-blobs/gis-map.dump.gz || ! -s /opt/wrolpi-blobs/gis-map.dump.gz ]]; then - wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/gis-map.dump.gz -O /opt/wrolpi-blobs/gis-map.dump.gz +if [[ ! -f /opt/wrolpi-blobs/map-db-gis.dump || ! -s /opt/wrolpi-blobs/map-db-gis.dump ]]; then + wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O /opt/wrolpi-blobs/map-db-gis.dump fi # Install WROLPi Help. diff --git a/scripts/install_raspberrypios.sh b/scripts/install_raspberrypios.sh index 359ce96d..b8c523dd 100755 --- a/scripts/install_raspberrypios.sh +++ b/scripts/install_raspberrypios.sh @@ -35,8 +35,8 @@ git config --global --add safe.directory /opt/openstreetmap-carto chown -R _renderd:_renderd /opt/openstreetmap-carto # Get map initialization dump. -if [[ ! -f /opt/wrolpi-blobs/gis-map.dump.gz || ! -s /opt/wrolpi-blobs/gis-map.dump.gz ]]; then - wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/gis-map.dump.gz -O /opt/wrolpi-blobs/gis-map.dump.gz +if [[ ! -f /opt/wrolpi-blobs/map-db-gis.dump || ! -s /opt/wrolpi-blobs/map-db-gis.dump ]]; then + wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O /opt/wrolpi-blobs/map-db-gis.dump fi # Install WROLPi Help. diff --git a/scripts/reset_map.sh b/scripts/reset_map.sh index 6845a7e4..4c602d07 100755 --- a/scripts/reset_map.sh +++ b/scripts/reset_map.sh @@ -11,7 +11,7 @@ fi # Change directory to avoid "sudo" home directory warning. cd /tmp -if sudo -iu postgres psql -l 2>/dev/null | grep gis >/dev/null; then +if sudo -iu postgres psql --port=5433 -l 2>/dev/null | grep gis >/dev/null; then yes_or_no "Are you sure you want to delete the map database and cache? This can't be undone!" || exit 0 fi @@ -27,10 +27,10 @@ set -x systemctl stop renderd -sudo -iu postgres dropdb gis || : -sudo -iu postgres dropuser _renderd || : +sudo -iu postgres dropdb --port=5433 gis || : +sudo -iu postgres dropuser --port=5433 _renderd || : -# Reset "imported" status of any map files. +# Reset "imported" status of any map files in API DB. sudo -iu postgres psql -d wrolpi -c "UPDATE map_file SET imported=false" yes | /opt/wrolpi/scripts/initialize_map_db.sh @@ -38,15 +38,15 @@ yes | /opt/wrolpi/scripts/initialize_map_db.sh # Use curl to fetch the first few layers of map tiles so the map is ready to use. bash /opt/wrolpi/wrolpi/scripts/initialize_map_tiles.sh -# Check the hash value of a Washington DC tile from /opt/wrolpi-blobs/gis-map.dump.gz +# Check the hash value of a Washington DC tile from /opt/wrolpi-blobs/map-db-gis.dump wget "https://127.0.0.1:8084/hot/13/2345/3134.png" \ --quiet \ --no-check-certificate \ --timeout 600 \ -O /tmp/tile.png set +x -echo "b9b7884d64b555fd4f433f991a86cc72 /tmp/tile.png" > /tmp/tile.md5 -if md5sum -c /tmp/tile.md5 >/dev/null ; then +echo "58b261940abaf50229544839fc55c83d /tmp/tile.png" > /tmp/tile.md5 +if md5sum -c /tmp/tile.md5 >/dev/null 2>&1 ; then echo "Map tile was rendered correctly" else echo "Map tile was not rendered correctly!" diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index ef63d301..baeb8939 100755 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -16,6 +16,17 @@ npm install || npm install || npm install || npm install # try install multiple # Upgrade WROLPi Help. /opt/wrolpi/scripts/install_help_service.sh || echo "Help install failed." +# Get map db blob, if it is missing. +MAP_DB_BLOB=/opt/wrolpi-blobs/map-db-gis.dump +if [[ ! -f ${MAP_DB_BLOB} || ! -s ${MAP_DB_BLOB} ]]; then + echo "Downloading new map blob (1.2 GB)..." + wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O ${MAP_DB_BLOB} +fi +if [[ -f /opt/wrolpi-blobs/gis-map.dump.gz && -s ${MAP_DB_BLOB} ]]; then + # Remove old blob now that we have the new one. + rm ${MAP_DB_BLOB} +fi + # Migrate map DB if necessary. Do this before repair because it will reset map if map db is empty. /opt/wrolpi/wrolpi/scripts/migrate_map_db.sh || echo "Map DB migration failed." diff --git a/wrolpi/scripts/migrate_map_db.sh b/wrolpi/scripts/migrate_map_db.sh index a02c40b5..48c4ded8 100755 --- a/wrolpi/scripts/migrate_map_db.sh +++ b/wrolpi/scripts/migrate_map_db.sh @@ -42,7 +42,7 @@ cp -r /opt/wrolpi/etc/raspberrypios/postgresql@15-map.service.d /etc/systemd/sys systemctl daemon-reload systemctl restart postgresql@15-map.service -# Dump DB, then restore in different DB. +# Dump old DB, then restore in new DB. sudo -iu postgres pg_dump -Fd gis --port=5432 -j3 -f /tmp/map.dump && # 5432 is old DB. sudo -iu postgres pg_restore --port=5433 -j3 -d gis /tmp/map.dump && # 5433 is new DB. sudo -iu postgres dropdb --port=5432 gis