Installing Kiwix packages and service in Raspberry Pi and Debian installer scripts.

50x.html

@@ -17,14 +17,17 @@
 <p>Or, start them individually:</p>
 <p><b>sudo systemctl start wrolpi-app</b></p>
 <p><b>sudo systemctl start wrolpi-api</b></p>
+<p><b>sudo systemctl start wrolpi-kiwix</b></p>
 
 <br/>
 <p>View the logs</p>
 <p><b>journalctl -u wrolpi-app</b></p>
 <p><b>journalctl -u wrolpi-api</b></p>
+<p><b>journalctl -u wrolpi-kiwix</b></p>
 <p>Watch the logs</p>
 <p><b>journalctl -fu wrolpi-app</b></p>
 <p><b>journalctl -fu wrolpi-api</b></p>
+<p><b>journalctl -fu wrolpi-kiwix</b></p>
 
 <hr/>
 

app/src/components/Zim.js

@@ -479,7 +479,7 @@ class ManageZim extends React.Component {
                 <PlaceholderLine/>
             </PlaceholderHeader>
         </Placeholder>;
-        if (subscriptions && Object.keys(subscriptions).length > 0) {
+        if (catalog && catalog.length > 0) {
             kiwixCatalog = <SortableTable
                 defaultSortColumn='name'
                 data={catalog}

etc/raspberrypios/wrolpi-kiwix.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=WROLPi Kiwix APP
+PartOf=wrolpi.target
+
+[Service]
+ExecStart=/opt/wrolpi/scripts/start_kiwix_serve.sh
+Group=wrolpi
+User=wrolpi
+WorkingDirectory=/opt/wrolpi/
+
+[Install]
+WantedBy=default.target

help.sh

@@ -212,6 +212,16 @@ check_file /opt/wrolpi-blobs/gis-map.dump.gz "Map initialization blob exists" "M
 
 check_file /var/www/html/leaflet.js "Leaflet.js exists" "Leaflet.js does not exist"
 
+echo
+# Kiwix
+
+check_file /media/wrolpi/zims/library.xml "The kiwix library file exists" "The kiwix library file does not exist at /media/wrolpi/zims/library.xml"
+if curl -s http://0.0.0.0:8085 2>/dev/null | grep -i kiwix >/dev/null; then
+  echo "OK: Kiwix app responded"
+else
+  echo "FAILED: Kiwix app did not respond"
+fi
+
 echo
 # Media Directory
 

modules/zim/downloader.py

@@ -108,6 +108,11 @@ class KiwixZimDownloader(Downloader):
         else:
             background_task(refresh_files([output_path, ]))
 
+        # Restart Kiwix serve, so it finds the new zim file.
+        return_code = await lib.restart_kiwix()
+        if return_code != 0:
+            logger.error(f'Failed to restart kiwix')
+
         # Location is just the generic Kiwix viewer.
         return DownloadResult(success=True, location='/zim')
 

modules/zim/lib.py

@@ -482,3 +482,19 @@ def flag_kiwix_restart():
         flags.kiwix_restart.set()
     else:
         logger.debug(f'Not setting kiwix_restart flag because this is not Dockerized')
+
+
+async def restart_kiwix():
+    if DOCKERIZED:
+        logger.warning('Unable to restart Kiwix serve because it is in a docker container')
+        return
+
+    logger.info('Restarting Kiwix serve')
+
+    cmd = f'sudo /usr/bin/systemctl restart wrolpi-kiwix.service'
+    proc = await asyncio.create_subprocess_shell(cmd)
+    stdout, stderr = await proc.communicate()
+    logger.debug(f'systemctl returned {proc.returncode}')
+    if proc.returncode != 0 and stderr:
+        logger.debug(stderr)
+    return proc.returncode

pi-gen/stage2/04-wrolpi/01-packages

@@ -20,6 +20,8 @@ gdal-bin
 git
 gnupg-agent
 hostapd
+kiwix
+kiwix-tools
 libagg-dev
 libapache2-mod-tile
 libboost-all-dev
@@ -75,3 +77,4 @@ ttf-unifont
 unzip
 vim
 wget
+zim-tools

pi-gen/stage2/04-wrolpi/03-run-chroot.sh

@@ -22,9 +22,9 @@ sed -i 's/port = 5433/port = 5432/' /etc/postgresql/13/main/postgresql.conf
 # Install Node console commands.
 single-file --version || npm i -g serve@12.0.1 single-file-cli@1.0.15 readability-extractor@0.0.6 carto@1.2.0
 
-# Put the latest WROLPi release in /opt/wrolpi.
+# Put the latest WROLPi master in /opt/wrolpi.
 rm -rf /opt/wrolpi
-git clone -b release https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi
+git clone -b master https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi
 git config --global --add safe.directory /opt/wrolpi
 
 # Install Python requirements.
@@ -37,6 +37,14 @@ cd /opt/wrolpi/app
 npm install
 npm run build
 
+# Add wrolpi database password to pi user
+cat >/home/pi/.pgpass <<'EOF'
+127.0.0.1:5432:gis:_renderd:wrolpi
+127.0.0.1:5432:wrolpi:wrolpi:wrolpi
+EOF
+chown -R pi:pi /home/pi
+chmod 0600 /home/pi/.pgpass
+
 chown -R wrolpi:wrolpi /opt/wrolpi
 
 # Configure nginx.
@@ -49,6 +57,9 @@ cat >/etc/sudoers.d/90-wrolpi <<'EOF'
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart renderd.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop renderd.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl start renderd.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-kiwix.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-kiwix.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-kiwix.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-api.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-api.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-api.service
@@ -63,10 +74,12 @@ chown wrolpi:wrolpi /media/wrolpi
 
 cp /opt/wrolpi/etc/raspberrypios/wrolpi-api.service /etc/systemd/system/
 cp /opt/wrolpi/etc/raspberrypios/wrolpi-app.service /etc/systemd/system/
+cp /opt/wrolpi/etc/raspberrypios/wrolpi-kiwix.service /etc/systemd/system/
 cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/
 
 systemctl enable wrolpi-api.service
 systemctl enable wrolpi-app.service
+systemctl enable wrolpi-kiwix.service
 
 # NetworkManager for the hotspot.
 systemctl enable NetworkManager

scripts/install_debian_11.sh

@@ -38,7 +38,7 @@ apt install -y \
 
 # Install npm.
 npm --version || curl -fsSL https://deb.nodesource.com/setup_18.x | bash -
-apt install -y nodejs
+apt-get install -y nodejs
 node -v
 npm -v
 
@@ -70,12 +70,29 @@ EOF
 chown -R wrolpi:wrolpi /home/wrolpi /opt/wrolpi
 chmod 0600 /home/wrolpi/.pgpass
 
+# Allow pi user to access database.
+cat >/home/pi/.pgpass <<'EOF'
+127.0.0.1:5432:gis:_renderd:wrolpi
+127.0.0.1:5432:wrolpi:wrolpi:wrolpi
+EOF
+chown -R pi:pi /home/pi
+chmod 0600 /home/pi/.pgpass
+
 # Give WROLPi group a few privileged commands via sudo without password.
 cat >/etc/sudoers.d/90-wrolpi <<'EOF'
 %wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart renderd.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop renderd.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl start renderd.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-kiwix.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-kiwix.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-kiwix.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-api.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-api.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-api.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-app.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-app.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-app.service
 EOF
 chmod 660 /etc/sudoers.d/90-wrolpi
 # Verify this new file is valid.
@@ -87,14 +104,17 @@ chown wrolpi:wrolpi /media/wrolpi
 
 # Install the systemd services
 cp /opt/wrolpi/etc/raspberrypios/wrolpi-api.service /etc/systemd/system/
-cp /opt/wrolpi/etc/debian11/wrolpi-app.service /etc/systemd/system/
+cp /opt/wrolpi/etc/raspberrypios/wrolpi-app.service /etc/systemd/system/
+cp /opt/wrolpi/etc/raspberrypios/wrolpi-kiwix.service /etc/systemd/system/
 cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/
 /usr/bin/systemctl daemon-reload
 systemctl enable wrolpi-api.service
 systemctl enable wrolpi-app.service
+systemctl enable wrolpi-kiwix.service
 # Stop the services so the user has to start them again.  We don't want to run outdated services when updating.
 systemctl stop wrolpi-api.service
 systemctl stop wrolpi-app.service
+systemctl stop wrolpi-kiwix.service
 
 # Configure Postgresql.  Do this after the API is stopped.
 sudo -u postgres psql -c '\l' | grep wrolpi || (
@@ -111,6 +131,8 @@ sudo -u postgres psql -c "alter user wrolpi with superuser"
 # Install map only if that script hasn't finished.
 [ -f /var/www/html/leaflet.css ] || /opt/wrolpi/scripts/install_map_debian_11.sh
 
+chown -R wrolpi:wrolpi /opt/wrolpi
+
 set +x
 
 ip=$(hostname -I | cut -d' ' -f1)

scripts/install_raspberrypios.sh

@@ -6,8 +6,8 @@ set -e
 
 # Update if we haven't updated in the last day.
 [ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update
-# Install dependencies
-apt install -y \
+# Install dependencies used in pi-gen.
+apt-get install -y \
   apt-transport-https \
   ca-certificates \
   calibre \
@@ -38,7 +38,7 @@ apt install -y \
 
 # Install npm.
 npm --version || curl -fsSL https://deb.nodesource.com/setup_18.x | bash -
-apt install -y nodejs
+apt-get install -y nodejs
 node -v
 npm -v
 
@@ -70,12 +70,31 @@ EOF
 chown -R wrolpi:wrolpi /home/wrolpi /opt/wrolpi
 chmod 0600 /home/wrolpi/.pgpass
 
+# Allow pi user to access database.
+[ -d /home/pi ] && (
+  cat >/home/pi/.pgpass <<'EOF'
+127.0.0.1:5432:gis:_renderd:wrolpi
+127.0.0.1:5432:wrolpi:wrolpi:wrolpi
+EOF
+  chown -R pi:pi /home/pi
+  chmod 0600 /home/pi/.pgpass
+)
+
 # Give WROLPi group a few privileged commands via sudo without password.
 cat >/etc/sudoers.d/90-wrolpi <<'EOF'
 %wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart renderd.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop renderd.service
 %wrolpi ALL= NOPASSWD:/usr/bin/systemctl start renderd.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-kiwix.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-kiwix.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-kiwix.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-api.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-api.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-api.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-app.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-app.service
+%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-app.service
 EOF
 chmod 660 /etc/sudoers.d/90-wrolpi
 # Verify this new file is valid.
@@ -88,13 +107,16 @@ chown wrolpi:wrolpi /media/wrolpi
 # Install the systemd services
 cp /opt/wrolpi/etc/raspberrypios/wrolpi-api.service /etc/systemd/system/
 cp /opt/wrolpi/etc/raspberrypios/wrolpi-app.service /etc/systemd/system/
+cp /opt/wrolpi/etc/raspberrypios/wrolpi-kiwix.service /etc/systemd/system/
 cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/
 /usr/bin/systemctl daemon-reload
 systemctl enable wrolpi-api.service
 systemctl enable wrolpi-app.service
+systemctl enable wrolpi-kiwix.service
 # Stop the services so the user has to start them again.  We don't want to run outdated services when updating.
 systemctl stop wrolpi-api.service
 systemctl stop wrolpi-app.service
+systemctl stop wrolpi-kiwix.service
 
 # Configure Postgresql.  Do this after the API is stopped.
 sudo -u postgres psql -c '\l' | grep wrolpi || (
@@ -111,6 +133,8 @@ sudo -u postgres psql -c "alter user wrolpi with superuser"
 # Install map only if that script hasn't finished.
 [ -f /var/www/html/leaflet.css ] || /opt/wrolpi/scripts/install_map_raspberrypios.sh
 
+chown -R wrolpi:wrolpi /opt/wrolpi
+
 set +x
 
 ip=$(hostname -i | cut -d' ' -f1)

scripts/start_kiwix_serve.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+# This file imports all *.zim files in the media directory, then starts kiwix-serve.
+
+[ ! -d /media/wrolpi ] && echo "Cannot start kiwix without media directory mounted" && exit 1
+
+LIBRARY=/media/wrolpi/zims/library.xml
+
+# Create the zim directory, if necessary.
+[ ! -d /media/wrolpi/zims ] && mkdir /media/wrolpi/zims
+# We will replace the library file.
+rm -f ${LIBRARY}
+# Find all Zim files and import them.
+find /media/wrolpi -iname '*.zim' -exec kiwix-manage ${LIBRARY} add {} \;
+
+[ ! -f ${LIBRARY} ] && echo "Could not find any Zim files to import" && exit 2
+
+# Serve kiwix on the
+kiwix-serve --library ${LIBRARY} --address 0.0.0.0 --port 8085