mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
Converting map blob to use "custom" postgres dump format so restore can use multiple threads. Renaming map db blob from gis-map.dump.gz to map-db-gis.dump.
This commit is contained in:
parent
3285887a1b
commit
8e4d173791
15 changed files with 37 additions and 24 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -129,7 +129,7 @@ test
|
|||
pg_data
|
||||
|
||||
# Directories used to build images
|
||||
/debian-live-config/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz
|
||||
/debian-live-config/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump
|
||||
/debian-live-config/build.log
|
||||
/debian-live-config/*.iso
|
||||
/pi-gen/*xz
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ See https://live-team.pages.debian.net/live-manual/html/live-manual/index.en.htm
|
|||
## Generate Debian 12 ISO
|
||||
|
||||
1. Convert a map PBF file to your initial map DB dump:
|
||||
* `docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper | gzip > debian-live-config/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz`
|
||||
* `docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper > debian-live-config/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump`
|
||||
2. Run the build.
|
||||
* `sudo ./build.sh`
|
||||
3. The ISO is now in the `debian-live-config` directory.
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ if [ $EUID != 0 ]; then
|
|||
exit $?
|
||||
fi
|
||||
|
||||
if [ ! -f "${SCRIPT_DIR}/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz" ]; then
|
||||
echo "config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz does not exist!"
|
||||
if [ ! -f "${SCRIPT_DIR}/config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump" ]; then
|
||||
echo "config/includes.chroot/opt/wrolpi-blobs/map-db-gis.dump does not exist!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,6 @@ Warning: Typically adds 2GB of planetary data.
|
|||
|
||||
Import `map.osm.pbf`, dump to `output.dump`
|
||||
|
||||
`# docker run --rm -v /absolute/path/to/your/map.osm.pbf:/data/region.osm.pbf $(docker build -q .) > output.dump.gz`
|
||||
`# docker run --rm -v /absolute/path/to/your/map.osm.pbf:/data/region.osm.pbf $(docker build -q .) > output.dump`
|
||||
|
||||
See: https://hub.docker.com/r/lrnselfreliance/osm-map-dumper
|
||||
|
|
|
|||
|
|
@ -88,7 +88,8 @@ echo >&2 "Dumping the DB"
|
|||
sudo -u postgres pg_dump \
|
||||
--create \
|
||||
--clean \
|
||||
--format tar \
|
||||
--format custom \
|
||||
--compress 9 \
|
||||
--no-owner \
|
||||
--no-comments \
|
||||
--encoding UTF-8 \
|
||||
|
|
|
|||
2
help.sh
2
help.sh
|
|
@ -230,7 +230,7 @@ else
|
|||
echo "FAILED: Map app did not respond"
|
||||
fi
|
||||
|
||||
check_file /opt/wrolpi-blobs/gis-map.dump.gz "Map initialization blob exists" "Map initialization blob does not exist"
|
||||
check_file /opt/wrolpi-blobs/map-db-gis.dump "Map initialization blob exists" "Map initialization blob does not exist"
|
||||
|
||||
check_file /var/www/html/leaflet.js "Leaflet.js exists" "Leaflet.js does not exist"
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ See https://github.com/RPI-Distro/pi-gen
|
|||
|
||||
1. Convert a map PBF file to your initial map DB dump:
|
||||
|
||||
docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper | gzip > pi-gen/stage2/04-wrolpi/files/gis-map.dump.gz
|
||||
docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper | gzip > pi-gen/stage2/04-wrolpi/files/map-db-gis.dump
|
||||
|
||||
2. Run the build (or, with a specific git branch)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ if [ $EUID != 0 ]; then
|
|||
exit $?
|
||||
fi
|
||||
|
||||
if [ ! -f "${SCRIPT_DIR}/stage2/04-wrolpi/files/gis-map.dump.gz" ]; then
|
||||
echo "stage2/04-wrolpi/files/gis-map.dump.gz does not exist!"
|
||||
if [ ! -f "${SCRIPT_DIR}/stage2/04-wrolpi/files/map-db-gis.dump" ]; then
|
||||
echo "stage2/04-wrolpi/files/map-db-gis.dump does not exist!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
|||
3
pi-gen/stage2/04-wrolpi/files/.gitignore
vendored
3
pi-gen/stage2/04-wrolpi/files/.gitignore
vendored
|
|
@ -1 +1,2 @@
|
|||
*.dump.gz
|
||||
*.dump.gz
|
||||
*.dump
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ sudo -iu postgres psql -c '\l' --port=5433 | grep gis || (
|
|||
echo "Created gis database"
|
||||
)
|
||||
# Restore initial dump.
|
||||
zcat /opt/wrolpi-blobs/gis-map.dump.gz | sudo -iu postgres pg_restore --port=5433 --no-owner --role=_renderd -d gis
|
||||
sudo -iu postgres pg_restore --port=5433 --no-owner --role=_renderd -d gis -j3 /opt/wrolpi-blobs/map-db-gis.dump
|
||||
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd'
|
||||
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd'
|
||||
sudo -iu postgres psql --port=5433 -d gis -c 'ALTER TABLE spatial_ref_sys OWNER TO _renderd'
|
||||
|
|
|
|||
|
|
@ -35,8 +35,8 @@ git config --global --add safe.directory /opt/openstreetmap-carto
|
|||
chown -R _renderd:_renderd /opt/openstreetmap-carto
|
||||
|
||||
# Get map initialization dump.
|
||||
if [[ ! -f /opt/wrolpi-blobs/gis-map.dump.gz || ! -s /opt/wrolpi-blobs/gis-map.dump.gz ]]; then
|
||||
wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/gis-map.dump.gz -O /opt/wrolpi-blobs/gis-map.dump.gz
|
||||
if [[ ! -f /opt/wrolpi-blobs/map-db-gis.dump || ! -s /opt/wrolpi-blobs/map-db-gis.dump ]]; then
|
||||
wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O /opt/wrolpi-blobs/map-db-gis.dump
|
||||
fi
|
||||
|
||||
# Install WROLPi Help.
|
||||
|
|
|
|||
|
|
@ -35,8 +35,8 @@ git config --global --add safe.directory /opt/openstreetmap-carto
|
|||
chown -R _renderd:_renderd /opt/openstreetmap-carto
|
||||
|
||||
# Get map initialization dump.
|
||||
if [[ ! -f /opt/wrolpi-blobs/gis-map.dump.gz || ! -s /opt/wrolpi-blobs/gis-map.dump.gz ]]; then
|
||||
wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/gis-map.dump.gz -O /opt/wrolpi-blobs/gis-map.dump.gz
|
||||
if [[ ! -f /opt/wrolpi-blobs/map-db-gis.dump || ! -s /opt/wrolpi-blobs/map-db-gis.dump ]]; then
|
||||
wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O /opt/wrolpi-blobs/map-db-gis.dump
|
||||
fi
|
||||
|
||||
# Install WROLPi Help.
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ fi
|
|||
# Change directory to avoid "sudo" home directory warning.
|
||||
cd /tmp
|
||||
|
||||
if sudo -iu postgres psql -l 2>/dev/null | grep gis >/dev/null; then
|
||||
if sudo -iu postgres psql --port=5433 -l 2>/dev/null | grep gis >/dev/null; then
|
||||
yes_or_no "Are you sure you want to delete the map database and cache? This can't be undone!" || exit 0
|
||||
fi
|
||||
|
||||
|
|
@ -27,10 +27,10 @@ set -x
|
|||
|
||||
systemctl stop renderd
|
||||
|
||||
sudo -iu postgres dropdb gis || :
|
||||
sudo -iu postgres dropuser _renderd || :
|
||||
sudo -iu postgres dropdb --port=5433 gis || :
|
||||
sudo -iu postgres dropuser --port=5433 _renderd || :
|
||||
|
||||
# Reset "imported" status of any map files.
|
||||
# Reset "imported" status of any map files in API DB.
|
||||
sudo -iu postgres psql -d wrolpi -c "UPDATE map_file SET imported=false"
|
||||
|
||||
yes | /opt/wrolpi/scripts/initialize_map_db.sh
|
||||
|
|
@ -38,15 +38,15 @@ yes | /opt/wrolpi/scripts/initialize_map_db.sh
|
|||
# Use curl to fetch the first few layers of map tiles so the map is ready to use.
|
||||
bash /opt/wrolpi/wrolpi/scripts/initialize_map_tiles.sh
|
||||
|
||||
# Check the hash value of a Washington DC tile from /opt/wrolpi-blobs/gis-map.dump.gz
|
||||
# Check the hash value of a Washington DC tile from /opt/wrolpi-blobs/map-db-gis.dump
|
||||
wget "https://127.0.0.1:8084/hot/13/2345/3134.png" \
|
||||
--quiet \
|
||||
--no-check-certificate \
|
||||
--timeout 600 \
|
||||
-O /tmp/tile.png
|
||||
set +x
|
||||
echo "b9b7884d64b555fd4f433f991a86cc72 /tmp/tile.png" > /tmp/tile.md5
|
||||
if md5sum -c /tmp/tile.md5 >/dev/null ; then
|
||||
echo "58b261940abaf50229544839fc55c83d /tmp/tile.png" > /tmp/tile.md5
|
||||
if md5sum -c /tmp/tile.md5 >/dev/null 2>&1 ; then
|
||||
echo "Map tile was rendered correctly"
|
||||
else
|
||||
echo "Map tile was not rendered correctly!"
|
||||
|
|
|
|||
|
|
@ -16,6 +16,17 @@ npm install || npm install || npm install || npm install # try install multiple
|
|||
# Upgrade WROLPi Help.
|
||||
/opt/wrolpi/scripts/install_help_service.sh || echo "Help install failed."
|
||||
|
||||
# Get map db blob, if it is missing.
|
||||
MAP_DB_BLOB=/opt/wrolpi-blobs/map-db-gis.dump
|
||||
if [[ ! -f ${MAP_DB_BLOB} || ! -s ${MAP_DB_BLOB} ]]; then
|
||||
echo "Downloading new map blob (1.2 GB)..."
|
||||
wget https://wrolpi.nyc3.cdn.digitaloceanspaces.com/map-db-gis.dump -O ${MAP_DB_BLOB}
|
||||
fi
|
||||
if [[ -f /opt/wrolpi-blobs/gis-map.dump.gz && -s ${MAP_DB_BLOB} ]]; then
|
||||
# Remove old blob now that we have the new one.
|
||||
rm ${MAP_DB_BLOB}
|
||||
fi
|
||||
|
||||
# Migrate map DB if necessary. Do this before repair because it will reset map if map db is empty.
|
||||
/opt/wrolpi/wrolpi/scripts/migrate_map_db.sh || echo "Map DB migration failed."
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ cp -r /opt/wrolpi/etc/raspberrypios/postgresql@15-map.service.d /etc/systemd/sys
|
|||
systemctl daemon-reload
|
||||
systemctl restart postgresql@15-map.service
|
||||
|
||||
# Dump DB, then restore in different DB.
|
||||
# Dump old DB, then restore in new DB.
|
||||
sudo -iu postgres pg_dump -Fd gis --port=5432 -j3 -f /tmp/map.dump && # 5432 is old DB.
|
||||
sudo -iu postgres pg_restore --port=5433 -j3 -d gis /tmp/map.dump && # 5433 is new DB.
|
||||
sudo -iu postgres dropdb --port=5432 gis
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue