Converting map blob to use "custom" postgres dump format so restore can use multiple threads. Renaming map db blob from gis-map.dump.gz to map-db-gis.dump.

This commit is contained in:
Roland 2025-01-02 12:02:35 -07:00
parent 3285887a1b
commit 8e4d173791
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
15 changed files with 37 additions and 24 deletions

2
.gitignore vendored
View file

@ -129,7 +129,7 @@ test
pg_data
# Directories used to build images
/debian-live-config/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz
/debian-live-config/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump
/debian-live-config/build.log
/debian-live-config/*.iso
/pi-gen/*xz

View file

@ -7,7 +7,7 @@ See https://live-team.pages.debian.net/live-manual/html/live-manual/index.en.htm
## Generate Debian 12 ISO
1. Convert a map PBF file to your initial map DB dump:
* `docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper | gzip > debian-live-config/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz`
* `docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper > debian-live-config/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump`
2. Run the build.
* `sudo ./build.sh`
3. The ISO is now in the `debian-live-config` directory.

View file

@ -12,8 +12,8 @@ if [ $EUID != 0 ]; then
exit $?
fi
if [ ! -f "${SCRIPT_DIR}/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz" ]; then
echo "config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz does not exist!"
if [ ! -f "${SCRIPT_DIR}/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump" ]; then
echo "config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump does not exist!"
exit 1
fi

View file

@ -9,6 +9,6 @@ Warning: Typically adds 2GB of planetary data.
Import `map.osm.pbf`, dump to `output.dump`
`# docker run --rm -v /absolute/path/to/your/map.osm.pbf:/data/region.osm.pbf $(docker build -q .) > output.dump.gz`
`# docker run --rm -v /absolute/path/to/your/map.osm.pbf:/data/region.osm.pbf $(docker build -q .) > output.dump`
See: https://hub.docker.com/r/lrnselfreliance/osm-map-dumper

View file

@ -88,7 +88,8 @@ echo >&2 "Dumping the DB"
sudo -u postgres pg_dump \
--create \
--clean \
--format tar \
--format custom \
--compress 9 \
--no-owner \
--no-comments \
--encoding UTF-8 \

View file

@ -230,7 +230,7 @@ else
echo "FAILED: Map app did not respond"
fi
check_file /opt/wrolpi-blobs/gis-map.dump.gz "Map initialization blob exists" "Map initialization blob does not exist"
check_file /opt/wrolpi-blobs/map-db-gis.dump "Map initialization blob exists" "Map initialization blob does not exist"
check_file /var/www/html/leaflet.js "Leaflet.js exists" "Leaflet.js does not exist"

View file

@ -8,7 +8,7 @@ See https://github.com/RPI-Distro/pi-gen
1. Convert a map PBF file to your initial map DB dump:
docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper | gzip > pi-gen/stage2/04-wrolpi/files/gis-map.dump.gz
docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper | gzip > pi-gen/stage2/04-wrolpi/files/map-db-gis.dump
2. Run the build (or, with a specific git branch)

View file

@ -12,8 +12,8 @@ if [ $EUID != 0 ]; then
exit $?
fi
if [ ! -f "${SCRIPT_DIR}/stage2/04-wrolpi/files/gis-map.dump.gz" ]; then
echo "stage2/04-wrolpi/files/gis-map.dump.gz does not exist!"
if [ ! -f "${SCRIPT_DIR}/stage2/04-wrolpi/files/map-db-gis.dump" ]; then
echo "stage2/04-wrolpi/files/map-db-gis.dump does not exist!"
exit 1
fi

View file

@ -1 +1,2 @@
*.dump.gz
*.dump.gz
*.dump

View file

@ -25,7 +25,7 @@ sudo -iu postgres psql -c '\l' --port=5433 | grep gis || (
echo "Created gis database"
)
# Restore initial dump.
zcat /opt/wrolpi-blobs/gis-map.dump.gz | sudo -iu postgres pg_restore --port=5433 --no-owner --role=_renderd -d gis
sudo -iu postgres pg_restore --port=5433 --no-owner --role=_renderd -d gis -j3 /opt/wrolpi-blobs/map-db-gis.dump
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd'
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd'
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE spatial_ref_sys OWNER TO _renderd'

View file

@ -35,8 +35,8 @@ git config --global --add safe.directory /opt/openstreetmap-carto
chown -R _renderd:_renderd /opt/openstreetmap-carto
# Get map initialization dump.
if [[ ! -f /opt/wrolpi-blobs/gis-map.dump.gz || ! -s /opt/wrolpi-blobs/gis-map.dump.gz ]]; then
wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/gis-map.dump.gz -O /opt/wrolpi-blobs/gis-map.dump.gz
if [[ ! -f /opt/wrolpi-blobs/map-db-gis.dump || ! -s /opt/wrolpi-blobs/map-db-gis.dump ]]; then
wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O /opt/wrolpi-blobs/map-db-gis.dump
fi
# Install WROLPi Help.

View file

@ -35,8 +35,8 @@ git config --global --add safe.directory /opt/openstreetmap-carto
chown -R _renderd:_renderd /opt/openstreetmap-carto
# Get map initialization dump.
if [[ ! -f /opt/wrolpi-blobs/gis-map.dump.gz || ! -s /opt/wrolpi-blobs/gis-map.dump.gz ]]; then
wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/gis-map.dump.gz -O /opt/wrolpi-blobs/gis-map.dump.gz
if [[ ! -f /opt/wrolpi-blobs/map-db-gis.dump || ! -s /opt/wrolpi-blobs/map-db-gis.dump ]]; then
wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O /opt/wrolpi-blobs/map-db-gis.dump
fi
# Install WROLPi Help.

View file

@ -11,7 +11,7 @@ fi
# Change directory to avoid "sudo" home directory warning.
cd /tmp
if sudo -iu postgres psql -l 2>/dev/null | grep gis >/dev/null; then
if sudo -iu postgres psql --port=5433 -l 2>/dev/null | grep gis >/dev/null; then
yes_or_no "Are you sure you want to delete the map database and cache? This can't be undone!" || exit 0
fi
@ -27,10 +27,10 @@ set -x
systemctl stop renderd
sudo -iu postgres dropdb gis || :
sudo -iu postgres dropuser _renderd || :
sudo -iu postgres dropdb --port=5433 gis || :
sudo -iu postgres dropuser --port=5433 _renderd || :
# Reset "imported" status of any map files.
# Reset "imported" status of any map files in API DB.
sudo -iu postgres psql -d wrolpi -c "UPDATE map_file SET imported=false"
yes | /opt/wrolpi/scripts/initialize_map_db.sh
@ -38,15 +38,15 @@ yes | /opt/wrolpi/scripts/initialize_map_db.sh
# Use curl to fetch the first few layers of map tiles so the map is ready to use.
bash /opt/wrolpi/wrolpi/scripts/initialize_map_tiles.sh
# Check the hash value of a Washington DC tile from /opt/wrolpi-blobs/gis-map.dump.gz
# Check the hash value of a Washington DC tile from /opt/wrolpi-blobs/map-db-gis.dump
wget "https://127.0.0.1:8084/hot/13/2345/3134.png" \
--quiet \
--no-check-certificate \
--timeout 600 \
-O /tmp/tile.png
set +x
echo "b9b7884d64b555fd4f433f991a86cc72 /tmp/tile.png" > /tmp/tile.md5
if md5sum -c /tmp/tile.md5 >/dev/null ; then
echo "58b261940abaf50229544839fc55c83d /tmp/tile.png" > /tmp/tile.md5
if md5sum -c /tmp/tile.md5 >/dev/null 2>&1 ; then
echo "Map tile was rendered correctly"
else
echo "Map tile was not rendered correctly!"

View file

@ -16,6 +16,17 @@ npm install || npm install || npm install || npm install # try install multiple
# Upgrade WROLPi Help.
/opt/wrolpi/scripts/install_help_service.sh || echo "Help install failed."
# Get map db blob, if it is missing.
MAP_DB_BLOB=/opt/wrolpi-blobs/map-db-gis.dump
if [[ ! -f ${MAP_DB_BLOB} || ! -s ${MAP_DB_BLOB} ]]; then
echo "Downloading new map blob (1.2 GB)..."
wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O ${MAP_DB_BLOB}
fi
if [[ -f /opt/wrolpi-blobs/gis-map.dump.gz && -s ${MAP_DB_BLOB} ]]; then
# Remove old blob now that we have the new one.
rm ${MAP_DB_BLOB}
fi
# Migrate map DB if necessary. Do this before repair because it will reset map if map db is empty.
/opt/wrolpi/wrolpi/scripts/migrate_map_db.sh || echo "Map DB migration failed."

View file

@ -42,7 +42,7 @@ cp -r /opt/wrolpi/etc/raspberrypios/postgresql@15-map.service.d /etc/systemd/sys
systemctl daemon-reload
systemctl restart postgresql@15-map.service
# Dump DB, then restore in different DB.
# Dump old DB, then restore in new DB.
sudo -iu postgres pg_dump -Fd gis --port=5432 -j3 -f /tmp/map.dump && # 5432 is old DB.
sudo -iu postgres pg_restore --port=5433 -j3 -d gis /tmp/map.dump && # 5433 is new DB.
sudo -iu postgres dropdb --port=5432 gis