diff --git a/alembic/versions/2bfb3b50b178_tag_names_migration.py b/alembic/versions/2bfb3b50b178_tag_names_migration.py new file mode 100644 index 00000000..eb54ddae --- /dev/null +++ b/alembic/versions/2bfb3b50b178_tag_names_migration.py @@ -0,0 +1,47 @@ +"""Tag names migration + +Revision ID: 2bfb3b50b178 +Revises: 8d0d81bc9c34 +Create Date: 2024-07-31 14:58:29.583029 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +from wrolpi.common import escape_file_name +from wrolpi.tags import Tag, get_tags_config + +# revision identifiers, used by Alembic. +revision = '2bfb3b50b178' +down_revision = '8d0d81bc9c34' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + need_commit = False + tags = session.query(Tag).all() + for tag in tags: + escaped_name = escape_file_name(tag.name) + if escaped_name != tag.name: + tag.name = escaped_name + session.flush([tag, ]) + need_commit = True + + if need_commit: + session.commit() + + config = get_tags_config() + config.initialize() + config.save_tags(session, ignore_lock=True) + + +def downgrade(): + pass diff --git a/etc/raspberrypios/renderd b/etc/raspberrypios/renderd deleted file mode 100644 index 21550dc1..00000000 --- a/etc/raspberrypios/renderd +++ /dev/null @@ -1,160 +0,0 @@ -#! /bin/sh -### BEGIN INIT INFO -# Provides: renderd -# Required-Start: $remote_fs -# Required-Stop: $remote_fs -# Should-Start: postgresql -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Mapnik rendering daemon -# Description: Mapnik rendering daemon. -### END INIT INFO - -# Do NOT "set -e" - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="Mapnik rendering daemon" -NAME=renderd -DAEMON=/usr/bin/$NAME -DAEMON_ARGS="" -PIDSOCKDIR=/run/$NAME -PIDFILE=$PIDSOCKDIR/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -RUNASUSER=_renderd - -# Exit if the package is not installed -[ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Load the VERBOSE setting and other rcS variables -. /lib/init/vars.sh - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.0-6) to ensure that this file is present. -. /lib/lsb/init-functions - -# -# Function that starts the daemon/service -# -do_start() -{ - [ -d "$PIDSOCKDIR" ] || mkdir -p $PIDSOCKDIR && chown $RUNASUSER $PIDSOCKDIR - - # Sleep because starting renderd prevents API from starting. - sleep 60 - - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --quiet --pidfile $PIDFILE --chuid $RUNASUSER --exec $DAEMON --test > /dev/null \ - || return 1 - start-stop-daemon --start --quiet --pidfile $PIDFILE --chuid $RUNASUSER --exec $DAEMON -- \ - $DAEMON_ARGS \ - || return 2 - # Add code here, if necessary, that waits for the process to be ready - # to handle requests from services started subsequently which depend - # on this one. As a last resort, sleep for some time. -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - # Wait for children to finish too if this is a daemon that forks - # and if the daemon is only ever run from this initscript. - # If the above conditions are not satisfied then add some other code - # that waits for the process to drop all resources that could be - # needed by services started subsequently. A last resort is to - # sleep for some time. - start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON - [ "$?" = 2 ] && return 2 - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - -# -# Function that sends a SIGHUP to the daemon/service -# -do_reload() { - # - # If the daemon can reload its configuration without - # restarting (for example, when it is sent a SIGHUP), - # then implement that here. - # - start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME - return 0 -} - -case "$1" in - start) - log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) log_end_msg 0 ;; - 2) log_end_msg 1 ;; - esac - ;; - stop) - log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) log_end_msg 0 ;; - 2) log_end_msg 1 ;; - esac - ;; - #reload|force-reload) - # - # If do_reload() is not implemented then leave this commented out - # and leave 'force-reload' as an alias for 'restart'. - # - #log_daemon_msg "Reloading $DESC" "$NAME" - #do_reload - #log_end_msg $? - #;; - restart|force-reload) - # - # If the "reload" option is implemented then remove the - # 'force-reload' alias - # - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - status) - status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? - ;; - *) - #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 - echo "Usage: $SCRIPTNAME {start|stop|restart|force-reload}" >&2 - exit 3 - ;; -esac - -: diff --git a/etc/raspberrypios/renderd.service b/etc/raspberrypios/renderd.service new file mode 100644 index 00000000..e2c9caad --- /dev/null +++ b/etc/raspberrypios/renderd.service @@ -0,0 +1,13 @@ +[Unit] +Description=Daemon that renders map tiles using mapnik +Documentation=man:renderd +After=network.target auditd.service + +[Service] +# Delay startup so WROLPi API has time to start. +ExecStartPre=/bin/sleep 30 +ExecStart=/usr/bin/renderd -f +User=_renderd + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/etc/raspberrypios/wrolpi-api.service b/etc/raspberrypios/wrolpi-api.service index c099e525..e3abdb31 100644 --- a/etc/raspberrypios/wrolpi-api.service +++ b/etc/raspberrypios/wrolpi-api.service @@ -5,7 +5,7 @@ PartOf=wrolpi.target [Service] Environment=PYTHONUNBUFFERED=1 -ExecStart=/opt/wrolpi/venv/bin/sanic main:api_app --host 0.0.0.0 --port 8081 -w 2 +ExecStart=/opt/wrolpi/venv/bin/sanic main:api_app --host 0.0.0.0 --port 8081 -w 3 ExecReload=/bin/kill -HUP $MAINPID Group=wrolpi User=wrolpi diff --git a/repair.sh b/repair.sh index 2b8ee30d..9aec1ff8 100755 --- a/repair.sh +++ b/repair.sh @@ -61,14 +61,13 @@ visudo -c -f /etc/sudoers.d/90-wrolpi # Install the systemd services cp /opt/wrolpi/etc/raspberrypios/wrolpi*.service /etc/systemd/system/ cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/ -/usr/bin/systemctl daemon-reload systemctl enable wrolpi-api.service systemctl enable wrolpi-app.service systemctl enable wrolpi-kiwix.service systemctl enable wrolpi-help.service cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf -cp /opt/wrolpi/etc/raspberrypios/renderd /etc/init.d/renderd +cp /opt/wrolpi/etc/raspberrypios/renderd.service /lib/systemd/system/renderd.service # Configure Apache2 to listen on 8084. cp /opt/wrolpi/etc/raspberrypios/ports.conf /etc/apache2/ports.conf # Copy Leaflet files to Apache's directory so they can be used offline. @@ -81,6 +80,7 @@ a2enmod headers systemctl enable renderd systemctl start renderd +/usr/bin/systemctl daemon-reload # WROLPi needs a few privileged commands. cp /opt/wrolpi/etc/raspberrypios/90-wrolpi /etc/sudoers.d/90-wrolpi diff --git a/scripts/install_debian_12.sh b/scripts/install_debian_12.sh index 1334266c..5060f311 100755 --- a/scripts/install_debian_12.sh +++ b/scripts/install_debian_12.sh @@ -22,7 +22,7 @@ npm install || npm install || npm install || npm install # try install multiple # Install python requirements in background job. python3 -m venv /opt/wrolpi/venv -/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt +/opt/wrolpi/venv/bin/pip3 install --upgrade -r /opt/wrolpi/requirements.txt # Create the WROLPi user grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)" diff --git a/scripts/install_help_service.sh b/scripts/install_help_service.sh index a22a2c44..9eee12f4 100755 --- a/scripts/install_help_service.sh +++ b/scripts/install_help_service.sh @@ -14,9 +14,9 @@ git clone https://github.com/wrolpi/wrolpi-help.git /opt/wrolpi-help || : git config --global --add safe.directory /opt/wrolpi-help (cd /opt/wrolpi-help && git fetch && git checkout master && git reset --hard origin/master) python3 -m venv /opt/wrolpi-help/venv -# Force re-install of mkdocs. +# Force re-install of mkdocs /opt/wrolpi-help/venv/bin/pip3 uninstall -y mkdocs -/opt/wrolpi-help/venv/bin/pip3 install -r /opt/wrolpi-help/requirements.txt +/opt/wrolpi-help/venv/bin/pip3 install --upgrade -r /opt/wrolpi-help/requirements.txt # Install the systemd service. cp /opt/wrolpi/etc/raspberrypios/wrolpi-help.service /etc/systemd/system/ diff --git a/scripts/install_raspberrypios.sh b/scripts/install_raspberrypios.sh index 9543baff..359ce96d 100755 --- a/scripts/install_raspberrypios.sh +++ b/scripts/install_raspberrypios.sh @@ -22,7 +22,7 @@ npm install || npm install || npm install || npm install # try install multiple # Install python requirements in background job. python3 -m venv /opt/wrolpi/venv -/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt +/opt/wrolpi/venv/bin/pip3 install --upgrade -r /opt/wrolpi/requirements.txt # Create the WROLPi user grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)" diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index 75633916..c6a47561 100755 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -9,7 +9,7 @@ cd /opt/wrolpi/app || exit 1 npm install || npm install || npm install || npm install # try install multiple times :( # Install any new Python requirements. -/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt +/opt/wrolpi/venv/bin/pip3 install --upgrade -r /opt/wrolpi/requirements.txt # Upgrade the WROLPi database. (cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 /opt/wrolpi/main.py db upgrade) diff --git a/wrolpi/common.py b/wrolpi/common.py index b5a4e1ae..5b29dfe4 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -374,7 +374,7 @@ class ConfigFile: path = path.with_name(name) return path - def save(self): + def save(self, ignore_lock: bool = False): """ Write this config to its file. @@ -390,8 +390,11 @@ class ConfigFile: raise ValueError(f'Refusing to save config file while testing: {config_file}') # Only one process can write to a config. - lock = api_app.shared_ctx.config_save_lock - acquired = lock.acquire(block=True, timeout=5.0) + acquired = False + lock = None + if ignore_lock is False: + lock = api_app.shared_ctx.config_save_lock + acquired = lock.acquire(block=True, timeout=5.0) try: # Config directory may not exist. @@ -439,11 +442,11 @@ class ConfigFile: return CONFIG_DIR / self.file_name - def update(self, config: dict): + def update(self, config: dict, ignore_lock: bool = False): """Update any values of this config. Save the config to its file.""" config = {k: v for k, v in config.items() if k in self._config} self._config.update(config) - self.save() + self.save(ignore_lock=ignore_lock) def dict(self) -> dict: """Get a deepcopy of this config.""" diff --git a/wrolpi/tags.py b/wrolpi/tags.py index 771c962f..287b3b79 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -187,7 +187,7 @@ class TagsConfig(ConfigFile): def tags(self, value: dict): self.update({'tags': value}) - def save_tags(self, session: Session): + def save_tags(self, session: Session, ignore_lock: bool = False): media_directory = get_media_directory() tags = dict() @@ -235,7 +235,7 @@ class TagsConfig(ConfigFile): 'tag_files': tag_files, 'tag_zims': tag_zims, 'tags': tags, - }) + }, ignore_lock=ignore_lock) TAGS_CONFIG: TagsConfig = TagsConfig()