diff --git a/debian-live-config/README.md b/debian-live-config/README.md index ed058c77..f9f1bc24 100644 --- a/debian-live-config/README.md +++ b/debian-live-config/README.md @@ -20,7 +20,7 @@ sudo ./build.sh -b master # build from a specific branch Requires Debian / Ubuntu with `live-build` installed. The resulting ISO is written to this directory as `WROLPi-v${VERSION}-amd64.iso`. -Flash with Etcher, Rufus, Raspberry Pi Imager, or: +Flash with Raspberry Pi Imager, Rufus, or: ``` sudo dd if=WROLPi-v${VERSION}-amd64.iso of=/dev/sdX bs=4M status=progress conv=fsync @@ -50,7 +50,7 @@ is no safe, filesystem-agnostic way to graft WROLPi onto a full drive. `scripts/wrolpi-usb.sh install` only prints this procedure: 1. Back up everything on the drive to another disk. -2. Flash the whole drive with the ISO (`dd`/Etcher/Rufus — this erases it). +2. Flash the whole drive with the ISO (Raspberry Pi Imager/Rufus/`dd` — this erases it). 3. Boot once so WROLPi creates its persistence partition and sets itself up. 4. Copy your data into `/media/wrolpi/`, then let WROLPi refresh/repair to index the restored files. @@ -60,8 +60,9 @@ Afterwards, future ISO upgrades preserve your library via the ## Layout on disk -The drive carries everything WROLPi needs. Postgres data lives alongside -the user's library so a drive moved between hosts keeps its full state: +The drive carries everything WROLPi needs. The SQLite database lives +alongside the user's library so a drive moved between hosts keeps its +full state: ``` /media/wrolpi/ @@ -75,9 +76,9 @@ the user's library so a drive moved between hosts keeps its full state: controller.yaml fstab.yaml ssl/ - postgresql/ - data/ ← bind-mounted to /var/lib/postgresql - config/ ← bind-mounted to /etc/postgresql + wrolpi.db ← SQLite database (created by the + API on first start; -wal/-shm + sidecars appear while running) ``` ## Booting via Ventoy or other multiboot USBs @@ -93,16 +94,16 @@ user's Ventoy stick, not a blank target — appending a new ext4 partition to it would risk corrupting the user's other ISOs and data. Instead the script mounts a `tmpfs` at `/media/wrolpi` (sized to half of -RAM by default), and the rest of the boot is unchanged. Postgres -bind-mounts and DB initialisation all succeed, just into RAM. The user -sees a "Running ephemerally…" notice in the first-boot dialog. +RAM by default), and the rest of the boot is unchanged. The config +layout and SQLite database are created as usual, just into RAM. The +user sees a "Running ephemerally…" notice in the first-boot dialog. Everything written to the WROLPi library — downloads, archives, video metadata, the database — is lost on reboot. This mode is intended for trying WROLPi out without committing a USB stick; for any real use, -flash the ISO directly to its own drive with `dd` or Etcher. +flash the ISO directly to its own drive with Raspberry Pi Imager or `dd`. -Direct (dd/Etcher) boot is distinguished by the live medium living under +Direct (Raspberry Pi Imager/dd) boot is distinguished by the live medium living under `/dev/sd*`, `/dev/nvme*`, or `/dev/mmcblk*`; this is the only shape where `wrolpi-bootstrap.sh` will touch the partition table. diff --git a/debian-live-config/build.sh b/debian-live-config/build.sh index ec5609f4..bdb0bc55 100755 --- a/debian-live-config/build.sh +++ b/debian-live-config/build.sh @@ -10,7 +10,9 @@ # https://live-team.pages.debian.net/live-manual/html/live-manual/index.en.html SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -BUILD_DIR=/var/tmp/wrolpi-build-debian +# Overridable so a development build can run beside the release pipeline's +# build tree without colliding with it. +BUILD_DIR="${WROLPI_BUILD_DIR:-/var/tmp/wrolpi-build-debian}" Help() { echo "Build WROLPi Debian Live ISO image." @@ -20,6 +22,9 @@ Help() { echo "h Print this help." echo "b Build from this git BRANCH (default: 'release')." echo + echo "Set WROLPI_BUILD_DIR to build somewhere other than the default" + echo "(${BUILD_DIR}), e.g. beside the release pipeline's tree." + echo } BRANCH="release" @@ -41,9 +46,11 @@ if [ -z "${VERSION}" ]; then fi echo "Building WROLPi version: ${VERSION} from branch: ${BRANCH}" -# Re-execute this script if it wasn't called with sudo. +# Re-execute this script if it wasn't called with sudo. Pass BUILD_DIR +# through explicitly — a plain `sudo "$0"` drops WROLPI_BUILD_DIR from the +# environment and the build would silently land on the default path. if [ $EUID != 0 ]; then - sudo "$0" "$@" + sudo WROLPI_BUILD_DIR="${BUILD_DIR}" "$0" "$@" exit $? fi @@ -71,7 +78,7 @@ lb config \ --mode debian \ --architectures amd64 \ --linux-flavours amd64 \ - --distribution bookworm \ + --distribution trixie \ --debian-installer live \ --debian-installer-gui true \ --archive-areas "main contrib non-free non-free-firmware" \ diff --git a/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot b/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot index 4589b3e1..a18e824b 100644 --- a/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot +++ b/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot @@ -174,10 +174,11 @@ systemctl enable wrolpi-mounts.service systemctl enable wrolpi-cert-renew.timer # Bootstrap service: on a Live USB it creates the persistence partition on -# first boot; on every boot (Live or Installed) it ensures a Postgres -# cluster exists on /media/wrolpi/config/postgresql and the wrolpi-* runtime -# services are enabled. Installed deployments find no live medium and -# short-circuit the partition step. +# first boot; on every boot (Live or Installed) it ensures the config +# directory exists and the wrolpi-* runtime services are enabled. The +# SQLite database is created by the API itself on first start (it lives in +# /media/wrolpi/config/wrolpi.db). Installed deployments find no live +# medium and short-circuit the partition step. chmod +x /usr/local/sbin/wrolpi-bootstrap.sh systemctl enable wrolpi-bootstrap.service @@ -221,16 +222,6 @@ cp /opt/wrolpi/etc/raspberrypios/wrolpi-help-files.desktop /home/wrolpi/Desktop/ chmod +x /home/wrolpi/Desktop/*.desktop chown -R wrolpi:wrolpi /home/wrolpi/Desktop -# Drop the default Postgres cluster created by the postgresql-* package. -# wrolpi-bootstrap.service creates a fresh cluster on /media/wrolpi/config/postgresql -# at first boot regardless of deployment shape; the cluster files installed -# here would be masked by the bind-mount on Live and orphaned on Installed. -# Version derived from the installed binary so this stays correct if the -# package list is later bumped past postgresql-15. -PG_VERSION=$(pg_lsclusters -h | awk 'NR==1{print $1}') -[ -n "${PG_VERSION}" ] || (echo "ERROR: no Postgres cluster found to drop" && exit 1) -pg_dropcluster --stop "${PG_VERSION}" main - # Protect blob files from accidental modification or deletion. if [ -d /opt/wrolpi-blobs ] && ls /opt/wrolpi-blobs/* >/dev/null 2>&1; then chown -R root:root /opt/wrolpi-blobs diff --git a/debian-live-config/config/includes.chroot/etc/systemd/system/wrolpi-bootstrap.service b/debian-live-config/config/includes.chroot/etc/systemd/system/wrolpi-bootstrap.service index 7e329c86..1d1bba2e 100644 --- a/debian-live-config/config/includes.chroot/etc/systemd/system/wrolpi-bootstrap.service +++ b/debian-live-config/config/includes.chroot/etc/systemd/system/wrolpi-bootstrap.service @@ -1,12 +1,12 @@ [Unit] -Description=WROLPi bootstrap (persistence partition + Postgres bind-mounts + DB) +Description=WROLPi bootstrap (persistence partition + config directory) Documentation=https://github.com/lrnselfreliance/wrolpi After=local-fs.target systemd-tmpfiles-setup.service # Block lightdm (and any other display manager) until bootstrap finishes. # Otherwise XFCE autologin can land before bootstrap has written the # first-boot summary file, and the wrolpi-firstboot-notify autostart # fires too early and shows nothing. -Before=postgresql.service wrolpi-api.service wrolpi-app.service wrolpi-kiwix.service wrolpi-controller.service wrolpi-first-startup.service wrolpi-mounts.service wrolpi.target display-manager.service lightdm.service +Before=wrolpi-api.service wrolpi-app.service wrolpi-kiwix.service wrolpi-controller.service wrolpi-first-startup.service wrolpi-mounts.service wrolpi.target display-manager.service lightdm.service Wants=local-fs.target # live-boot uses one of these mount points depending on version; on # Installed none of them exist and RequiresMountsFor resolves to the diff --git a/debian-live-config/config/includes.chroot/usr/local/bin/wrolpi-firstboot-notify b/debian-live-config/config/includes.chroot/usr/local/bin/wrolpi-firstboot-notify index 8d605238..35aa747d 100755 --- a/debian-live-config/config/includes.chroot/usr/local/bin/wrolpi-firstboot-notify +++ b/debian-live-config/config/includes.chroot/usr/local/bin/wrolpi-firstboot-notify @@ -3,8 +3,8 @@ # XFCE session is up. # # wrolpi-bootstrap.service writes /tmp/wrolpi-firstboot-summary.txt when -# it does notable work (creates the persistence partition, initialises -# the Postgres cluster, runs the DB schema migration). This script — +# it does notable work (e.g. creates the persistence partition). This +# script — # autostarted via /etc/xdg/autostart/wrolpi-firstboot-notify.desktop — # reads it, shows the contents to the user, and removes the file. # diff --git a/debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh b/debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh index 22504194..8a005c64 100644 --- a/debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh +++ b/debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh @@ -7,12 +7,12 @@ # drive, formats it as ext4 (label `persistence`), writes a # persistence.conf so future boots pick it up automatically via # live-boot, bind-mounts the persistence root over /media/wrolpi, -# creates the Postgres cluster under /media/wrolpi/config/postgresql/, -# runs initialize_api_db.sh, and enables the WROLPi runtime services. +# and enables the WROLPi runtime services. The SQLite database is +# created by the API itself on first start (in /media/wrolpi/config/). # # 2. Live USB (direct), subsequent boot. live-boot's initramfs has # already applied the /media/wrolpi bind mount from persistence.conf. -# Cluster and DB already exist. Most steps no-op; we always +# The database already exists. Most steps no-op; we always # regenerate the leaf TLS cert and ensure services are enabled. # # 3. Live USB (loop / Ventoy / multiboot). The ISO is loop-mounted by a @@ -25,11 +25,8 @@ # 4. Installed. No live medium present; partition-creation branch is # skipped. /media/wrolpi is mounted via /etc/fstab on an external # drive, or is a plain directory on the root filesystem if the user -# installed without one. Postgres lives at the normal Debian paths on -# the root filesystem: the bind-mounts are for live boots only, where -# the root filesystem is ephemeral. An installed system's primary -# drive may be NTFS/exFAT, which cannot host a Postgres data -# directory at all. +# installed without one. The SQLite database lives inside +# /media/wrolpi/config/ in every shape, so it travels with the drive. set -euo pipefail @@ -173,9 +170,7 @@ if [ "$MODE" = "direct" ]; then # Mount the persistence partition + lay out the on-disk structure. # /media/wrolpi is bound to /mnt/persistence/wrolpi by either this - # script (first boot) or live-boot's initramfs (subsequent boots). The - # Postgres bind-mounts are managed by THIS script in all cases — see - # the "Postgres bind-mounts" section below. + # script (first boot) or live-boot's initramfs (subsequent boots). mkdir -p /mnt/persistence mountpoint -q /mnt/persistence || mount "$PERSIST_PART" /mnt/persistence mkdir -p /mnt/persistence/wrolpi @@ -183,10 +178,7 @@ if [ "$MODE" = "direct" ]; then # persistence.conf: live-boot reads this on boot and applies the bind # mount before systemd starts on subsequent boots. Only /media/wrolpi - # is here; Postgres bind-mounts are intentionally managed by this - # script (they live under /media/wrolpi/config/postgresql/ on the - # mounted /media/wrolpi, so once that bind is in place they are - # reachable). + # needs to persist — the SQLite database lives inside it. cat > /mnt/persistence/persistence.conf <<'EOF' /media/wrolpi bind,source=wrolpi EOF @@ -204,8 +196,8 @@ fi # which underlying disk is the user's data drive — appending a partition # would risk corrupting their Ventoy stick or whatever else they had on # that device. Run from RAM instead. Anything written to /media/wrolpi -# vanishes on reboot, but the rest of WROLPi (Postgres bind-mounts, -# config layout, services) works identically. +# vanishes on reboot, but the rest of WROLPi (config layout, database, +# services) works identically. if [ "$MODE" = "loop" ]; then if ! mountpoint -q /media/wrolpi; then @@ -221,10 +213,12 @@ if [ "$MODE" = "loop" ]; then fi fi -# --- /media/wrolpi/config + Postgres data layout ------------------------ +# --- /media/wrolpi/config layout ----------------------------------------- # Works regardless of whether /media/wrolpi is a bind (Live), a regular # mount from /etc/fstab (Installed + external drive), or a plain -# directory on root (Installed without external drive). +# directory on root (Installed without external drive). The SQLite +# database (wrolpi.db) is created inside config/ by the API on first +# start — no server or cluster setup is needed here. if [ ! -d /media/wrolpi ]; then mkdir -p /media/wrolpi @@ -234,69 +228,6 @@ chown wrolpi:wrolpi /media/wrolpi 2>/dev/null || true mkdir -p /media/wrolpi/config chown wrolpi:wrolpi /media/wrolpi/config -# --- Postgres bind-mounts (live boots ONLY, idempotent) ------------------ -# On live boots the root filesystem is an ephemeral overlay, so the -# cluster must live on the persistence partition: /var/lib/postgresql -# (data) and /etc/postgresql (config) are bind-mounted onto -# subdirectories of /media/wrolpi and travel with the stick. -# -# On INSTALLED systems Postgres stays at the normal Debian paths on the -# root filesystem. The primary drive there may be NTFS/exFAT, which -# cannot host a Postgres data directory (no POSIX ownership/0700), and -# repair.sh chowns /media/wrolpi/config for the wrolpi user. - -if [ "$MODE" != "installed" ]; then - mkdir -p /media/wrolpi/config/postgresql/data \ - /media/wrolpi/config/postgresql/config - chown postgres:postgres /media/wrolpi/config/postgresql \ - /media/wrolpi/config/postgresql/data \ - /media/wrolpi/config/postgresql/config - chmod 0700 /media/wrolpi/config/postgresql/data - - bind_if_unmounted() { - local src=$1 dest=$2 - mkdir -p "$dest" - mountpoint -q "$dest" || mount --bind "$src" "$dest" - } - - bind_if_unmounted /media/wrolpi/config/postgresql/data /var/lib/postgresql - bind_if_unmounted /media/wrolpi/config/postgresql/config /etc/postgresql -else - # Installed system that previously ran with the binds (older bootstrap): - # if a stale bind is still active from this boot, leave it alone — the - # cluster logic below would otherwise misdetect. It disappears on the - # next reboot with this version. - if mountpoint -q /var/lib/postgresql; then - log "NOTE: /var/lib/postgresql is still bind-mounted (old layout); reboot to move Postgres to the root filesystem." - fi -fi - -# --- Postgres cluster --------------------------------------------------- -# Derive version from the installed Postgres binary so this stays correct -# if the package list is later bumped past postgresql-15. - -PG_VERSION=$(ls /usr/lib/postgresql 2>/dev/null | sort -n | tail -1) -if [ -z "$PG_VERSION" ]; then - log "ERROR: no postgresql server package found under /usr/lib/postgresql" - exit 1 -fi - -if ! pg_lsclusters -h | awk '{print $1,$2}' | grep -q "^${PG_VERSION} main$"; then - progress "Setting up Postgres cluster..." - pg_createcluster "${PG_VERSION}" main - note_action "Set up PostgreSQL ${PG_VERSION} cluster at /media/wrolpi/config/postgresql/" -fi -progress "Starting Postgres..." -systemctl start "postgresql@${PG_VERSION}-main.service" - -# --- WROLPi DB schema + initial config import --------------------------- - -if ! sudo -iu postgres psql -lqt | cut -d \| -f 1 | grep -qw wrolpi; then - progress "Initializing WROLPi database (this can take a minute)..." - /opt/wrolpi/scripts/initialize_api_db.sh - note_action "Initialized the WROLPi database" -fi - # --- Certificates ------------------------------------------------------- # CA persists in /media/wrolpi/config/ssl/ (created on first run). Leaf # cert is regenerated every boot so SANs reflect the current hostname/IP. @@ -352,7 +283,7 @@ WROLPi finished setting itself up: $SUMMARY These persist between reboots: /media/wrolpi — your library (videos, archives, zims, ...) - /media/wrolpi/config — configuration + Postgres data + /media/wrolpi/config — configuration + database EOF fi chmod 0644 "$SUMMARY_FILE" diff --git a/debian-live-config/config/package-lists/wrolpi.list.chroot b/debian-live-config/config/package-lists/wrolpi.list.chroot index 51a129cc..5cfefaa3 100644 --- a/debian-live-config/config/package-lists/wrolpi.list.chroot +++ b/debian-live-config/config/package-lists/wrolpi.list.chroot @@ -11,7 +11,6 @@ cargo catdoc chromium chromium-driver -cpufrequtils curl dnsmasq-base dosfstools @@ -29,6 +28,7 @@ gcc-doc gdisk git git-doc +gnupg gnupg-agent gpsd gvfs @@ -49,14 +49,14 @@ libfreetype6-dev libgeos++-dev libgeos-dev libicu-dev -libpq-dev libproj-dev libreoffice libsqlite3-dev libssl-dev -libtiff5-dev +libtiff-dev libtool libxml2-dev +locales lxpolkit netcat-openbsd network-manager @@ -68,8 +68,6 @@ ntfs-3g parted pavucontrol pkg-config -postgresql-15 -postgresql-doc pulseaudio pulseaudio-utils pv @@ -82,7 +80,7 @@ rfkill rsync samba smartmontools -software-properties-common +sqlite3 sudo sysstat tar diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index af471e13..f3c9b705 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-bookworm +FROM python:3.13-trixie ENV DOCKER=true WORKDIR /opt/wrolpi diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 46272c30..ec79f1ea 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,5 +1,6 @@ # base image -FROM node:18-bullseye +# Node 20 matches the nodejs package shipped by Debian 13 (trixie). +FROM node:20-trixie WORKDIR /app ENV PATH /app/node_modules/.bin:$PATH diff --git a/docker/controller/Dockerfile b/docker/controller/Dockerfile index 599bd344..d1da7ceb 100644 --- a/docker/controller/Dockerfile +++ b/docker/controller/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.13-slim-trixie LABEL maintainer="WROLPi" LABEL description="WROLPi Controller Service" diff --git a/docker/help/Dockerfile b/docker/help/Dockerfile index 631f79f6..a8949676 100644 --- a/docker/help/Dockerfile +++ b/docker/help/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-bookworm +FROM python:3.13-trixie ENV DOCKER=true WORKDIR /opt/wrolpi-help diff --git a/docker/samba/Dockerfile b/docker/samba/Dockerfile index 082365b6..96fb3d9a 100644 --- a/docker/samba/Dockerfile +++ b/docker/samba/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:12-slim +FROM debian:13-slim RUN apt-get update && \ apt-get install -y --no-install-recommends samba samba-common && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/zim/Dockerfile b/docker/zim/Dockerfile index 22a52b9f..b390bac7 100644 --- a/docker/zim/Dockerfile +++ b/docker/zim/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:12 +FROM debian:13 ENV DOCKER=true RUN apt update && apt-get install -y kiwix kiwix-tools diff --git a/etc/raspberrypios/50x.html b/etc/raspberrypios/50x.html index be28d2ac..5e33cc34 100644 --- a/etc/raspberrypios/50x.html +++ b/etc/raspberrypios/50x.html @@ -60,10 +60,9 @@

WROLPi comes with official documentation from some third-party applications:

diff --git a/etc/raspberrypios/90-wrolpi b/etc/raspberrypios/90-wrolpi index 599e9ce7..19112361 100644 --- a/etc/raspberrypios/90-wrolpi +++ b/etc/raspberrypios/90-wrolpi @@ -1,4 +1,4 @@ -%wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set +%wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli %wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-kiwix.service %wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-kiwix.service %wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-kiwix.service diff --git a/help.sh b/help.sh index 7c1c46d5..31b9ecf5 100755 --- a/help.sh +++ b/help.sh @@ -79,73 +79,41 @@ check_file /opt/wrolpi/main.py "The WROLPi main script exists" "The WROLPi main check_file ${MEDIA_DIRECTORY}/config/wrolpi.yaml "The WROLPi config file exists" "The WROLPi config file does not exist" echo -# Postgresql +# Database (SQLite in the media directory) -systemctl status postgresql.service >/dev/null && - echo "OK: Postgres found" || - echo "FAILED: Postgres service does not exist" +WROLPI_DB="${MEDIA_DIRECTORY}/config/wrolpi.db" +if [ -f "${WROLPI_DB}" ]; then + echo "OK: Found wrolpi database at ${WROLPI_DB}" -if netstat -ant | grep LISTEN | grep -E '[:\.]5432\b' >/dev/null; then - echo "OK: Port 5432 is occupied" -else - echo "FAILED: Port 5432 is not occupied" -fi + if command -v sqlite3 >/dev/null 2>&1; then + # Run as the wrolpi user: a read of a WAL database needs write access to + # the -shm sidecar, which root-created files would break for the API. + wrolpi_sqlite() { + sudo -u wrolpi sqlite3 "${WROLPI_DB}" "$@" 2>/dev/null + } -[ -S /var/run/postgresql/.s.PGSQL.5432 ] && - echo "OK: Postgres is using file socket" || - echo "FAILED: Postgres is not using file socket" - -# Cluster-level checks. The umbrella postgresql.service reports active even -# when every cluster is down, so inspect the real clusters. -if command -v pg_lsclusters >/dev/null 2>&1; then - cluster_line=$(pg_lsclusters -h 2>/dev/null | head -1) - if [ -z "${cluster_line}" ]; then - echo "FAILED: No Postgres clusters exist (pg_lsclusters printed nothing)" - elif [ "$(echo "${cluster_line}" | awk '{print $4}')" = "online" ]; then - echo "OK: Postgres cluster is online (port $(echo "${cluster_line}" | awk '{print $3}'))" - else - echo "FAILED: Postgres cluster is not online: ${cluster_line}" - fi - cluster_data_dir=$(echo "${cluster_line}" | awk '{print $6}') - if [ -n "${cluster_data_dir}" ]; then - case "${cluster_data_dir}" in - /var/lib/postgresql/*) echo "OK: Postgres data directory is at ${cluster_data_dir}" ;; - *) echo "FAILED: Postgres data directory is in an unexpected location: ${cluster_data_dir}" ;; - esac - fi -fi - -# A mount over /var/lib/postgresql means the Portable bind layout (expected -# only on live boots). On an installed system this indicates the cluster is -# on the media drive — the wrong spot; see mount-issues.md Problem 8. -if findmnt -n /var/lib/postgresql >/dev/null 2>&1; then - echo "Note: /var/lib/postgresql is a mount: $(findmnt -n -o SOURCE,FSTYPE /var/lib/postgresql)" - echo " (Portable/live-boot layout; on an installed system this is WRONG)" -else - echo "OK: /var/lib/postgresql is on the root filesystem (installed layout)" -fi - -# Connect directly instead of `sudo -i -u wrolpi` so sudo/PAM do not spam the journal. -wrolpi_psql() { - PGPASSWORD=wrolpi psql -h 127.0.0.1 -p 5432 -U wrolpi wrolpi "$@" -} - -if wrolpi_psql -c 'select 1' >/dev/null 2>&1; then - echo "OK: Found wrolpi database" - - if wrolpi_psql -c '\d' 2>/dev/null | grep "file_group" >/dev/null; then - echo "OK: WROLPi database is initialized" - - if [ "$(wrolpi_psql -c 'copy (select count(*) from file_group) to stdout' 2>/dev/null)" -gt 0 ]; then - echo "OK: WROLPi database has files" + if [ "$(wrolpi_sqlite 'PRAGMA quick_check;' | head -1)" = "ok" ]; then + echo "OK: WROLPi database passes quick_check" else - echo "FAILED: WROLPi database has no files. You need to refresh your files: https://$(hostname).local/files" + echo "FAILED: WROLPi database is corrupt or unreadable (PRAGMA quick_check)" + fi + + if wrolpi_sqlite ".tables" | grep -q "file_group"; then + echo "OK: WROLPi database is initialized" + + if [ "$(wrolpi_sqlite 'select count(*) from file_group;')" -gt 0 ] 2>/dev/null; then + echo "OK: WROLPi database has files" + else + echo "FAILED: WROLPi database has no files. You need to refresh your files: https://$(hostname).local/files" + fi + else + echo "FAILED: WROLPi is not initialized" fi else - echo "FAILED: WROLPi is not initialized" + echo "Note: sqlite3 CLI not installed; skipping database integrity checks" fi else - echo "FAILED: Unable to find wrolpi database" + echo "FAILED: Unable to find wrolpi database at ${WROLPI_DB} (the API creates it on first start)" fi echo diff --git a/pi-gen/build.sh b/pi-gen/build.sh index 3b617912..955aeaa9 100755 --- a/pi-gen/build.sh +++ b/pi-gen/build.sh @@ -2,7 +2,9 @@ # https://github.com/RPI-Distro/pi-gen SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -BUILD_DIR=/var/tmp/wrolpi-build-pi-gen +# Overridable so a development build can run beside the release pipeline's +# build tree without colliding with it. +BUILD_DIR="${WROLPI_BUILD_DIR:-/var/tmp/wrolpi-build-pi-gen}" Help() { echo "Build WROLPi Raspberry Pi OS image." @@ -12,6 +14,9 @@ Help() { echo "h Print this help." echo "b Build from this git BRANCH (default: 'release')." echo + echo "Set WROLPI_BUILD_DIR to build somewhere other than the default" + echo "(${BUILD_DIR}), e.g. beside the release pipeline's tree." + echo } BRANCH="release" @@ -31,16 +36,18 @@ if [ -z "${VERSION}" ]; then fi echo "Building WROLPi version: ${VERSION} from branch: ${BRANCH}" -# Re-execute this script if it wasn't called with sudo. +# Re-execute this script if it wasn't called with sudo. Pass BUILD_DIR +# through explicitly — a plain `sudo "$0"` drops WROLPI_BUILD_DIR from the +# environment and the build would silently land on the default path. if [ $EUID != 0 ]; then - sudo "$0" "$@" + sudo WROLPI_BUILD_DIR="${BUILD_DIR}" "$0" "$@" exit $? fi CHROOTFS="${BUILD_DIR}/work/WROLPi/stage0/rootfs" cleanup() { - [ -f /var/tmp/bookworm-arm64.zip ] && rm /var/tmp/bookworm-arm64.zip + [ -f /var/tmp/arm64.zip ] && rm /var/tmp/arm64.zip # Only unmount if it is actually a mountpoint for mp in \ @@ -99,12 +106,12 @@ fi # the stale one and break the build. Strip the immutable flag off blob files # first (the chroot sets chattr +i), or rm -rf cannot remove them. chattr -i "${BUILD_DIR}"/work/*/stage*/rootfs/opt/wrolpi-blobs/* 2>/dev/null || : -rm -rf "${BUILD_DIR}" /var/tmp/pi-gen-bookworm-arm64 /var/tmp/bookworm-arm64.zip +rm -rf "${BUILD_DIR}" /var/tmp/pi-gen-arm64 /var/tmp/arm64.zip -# Get the latest pi-gen code. -wget https://github.com/RPi-Distro/pi-gen/archive/refs/heads/bookworm-arm64.zip -O /var/tmp/bookworm-arm64.zip -unzip /var/tmp/bookworm-arm64.zip -d /var/tmp -mv /var/tmp/pi-gen-bookworm-arm64 /var/tmp/wrolpi-build-pi-gen +# Get the latest pi-gen code. The `arm64` branch builds trixie (Debian 13). +wget https://github.com/RPi-Distro/pi-gen/archive/refs/heads/arm64.zip -O /var/tmp/arm64.zip +unzip /var/tmp/arm64.zip -d /var/tmp +mv /var/tmp/pi-gen-arm64 "${BUILD_DIR}" # Copy the configuration and build files into the pi-gen directory. cp "${SCRIPT_DIR}/config.txt" "${BUILD_DIR}/config.txt" @@ -112,7 +119,7 @@ echo "export WROLPI_BRANCH=${BRANCH}" >> "${BUILD_DIR}/config.txt" rsync -a "${SCRIPT_DIR}"/stage2/* "${BUILD_DIR}/stage2/" # We only need to build the Lite and Desktop images. -rm "${BUILD_DIR}"/stage*/EXPORT* +rm -f "${BUILD_DIR}"/stage*/EXPORT* #echo 'IMG_SUFFIX="-lite"' > "${BUILD_DIR}/stage2/EXPORT_IMAGE" echo 'IMG_SUFFIX="-desktop"' > "${BUILD_DIR}/stage5/EXPORT_IMAGE" diff --git a/pi-gen/config.txt b/pi-gen/config.txt index 0af6b605..410086c7 100644 --- a/pi-gen/config.txt +++ b/pi-gen/config.txt @@ -1,9 +1,11 @@ IMG_NAME=WROLPi BASE_QCOW2_SIZE=48G -RELEASE=bookworm +RELEASE=trixie LOCALE_DEFAULT=en_US.UTF-8 TARGET_HOSTNAME=wrolpi -KEYBOARD_KEYMAP=en_US +# XKB keymap name (see /usr/share/X11/xkb/symbols/); trixie's console-setup +# validates this, and "en_US" is not a valid keymap. +KEYBOARD_KEYMAP=us KEYBOARD_LAYOUT="English (US)" ENABLE_SSH=1 DEPLOY_COMPRESSION=xz diff --git a/pi-gen/stage2/04-wrolpi/01-packages-nr b/pi-gen/stage2/04-wrolpi/01-packages-nr index 70c73f08..400fb0a6 100644 --- a/pi-gen/stage2/04-wrolpi/01-packages-nr +++ b/pi-gen/stage2/04-wrolpi/01-packages-nr @@ -10,7 +10,6 @@ cargo catdoc chromium-browser chromium-chromedriver -cpufrequtils curl ethtool exfat-fuse @@ -42,10 +41,9 @@ libfreetype6-dev libgeos++-dev libgeos-dev libicu-dev -libpq-dev libproj-dev libssl-dev -libtiff5-dev +libtiff-dev libtool libxml2-dev netcat-openbsd @@ -55,8 +53,6 @@ nodejs npm ntfs-3g pkg-config -postgresql-15 -postgresql-doc pv python3-dev python3-doc @@ -66,7 +62,7 @@ python3-venv rsync samba smartmontools -software-properties-common +sqlite3 sysstat tar tcpdump diff --git a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh index 212a016f..3e37aa51 100755 --- a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh @@ -21,9 +21,6 @@ EOF # Create WROLPi user. This user will own the media directory, API, and App. useradd -md /home/wrolpi wrolpi -s "$(command -v bash)" -# Change postgresql "main" cluster port to 5432. -sed -i 's/port =.*/port = 5432/' /etc/postgresql/15/main/postgresql.conf - # Install Node console commands. npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6 diff --git a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh index 98e0798f..6c5dbd8d 100755 --- a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh @@ -3,11 +3,6 @@ set -e set -x -# All users can access the wrolpi database. -cat >/etc/skel/.pgpass <<'EOF' -127.0.0.1:5432:wrolpi:wrolpi:wrolpi -EOF -chmod 0600 /etc/skel/.pgpass cat >/etc/skel/.gitconfig <<'EOF' [safe] directory = /opt/wrolpi diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 959057f7..a6839af6 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -1,6 +1,7 @@ import asyncio import ctypes.wintypes import json +import sys import multiprocessing import os import pathlib @@ -313,22 +314,37 @@ def test_escape_file_name(name, expected): def test_truncate_object_bytes(): """ - Objects can be truncated (lists will be shortened) so they will fit in tsvector columns. + Objects can be truncated (lists will be shortened) so they will fit in the search index. + + Exact result lengths depend on the interpreter's object sizes (which changed in + Python 3.12), so assert the contract: the result fits the byte budget and is a + prefix of the input. """ - assert common.truncate_object_bytes(['foo'] * 10, 100) == ['foo'] * 5 - assert common.truncate_object_bytes(['foo'] * 1_000, 100) == ['foo'] * 5 - assert common.truncate_object_bytes(['foo'] * 1_000_000, 100) == ['foo'] * 5 - assert common.truncate_object_bytes(['foo'] * 1_000_000, 200) == ['foo'] * 14 + for count in (10, 1_000, 1_000_000): + previous_length = 0 + for maximum_bytes in (100, 200): + result = common.truncate_object_bytes(['foo'] * count, maximum_bytes) + assert sys.getsizeof(result) < maximum_bytes + # Small lists that already fit are returned whole. + assert 0 < len(result) <= count + assert result == ['foo'] * len(result) + # A larger budget must never retain fewer items. + assert len(result) >= previous_length + previous_length = len(result) assert common.truncate_object_bytes([], 200) == [] assert common.truncate_object_bytes(None, 100) is None assert common.truncate_object_bytes('', 100) == '' - assert common.truncate_object_bytes('foo' * 100, 99) == 'foofoofoofoofoofoofoofoofoofoofoofoofoof' - assert common.truncate_object_bytes('foo' * 100, 80) == 'foofoofoofoofoofoofoofoofo' - assert common.truncate_object_bytes('foo' * 100, 55) == 'foofo' - assert common.truncate_object_bytes('foo' * 100, 51) == 'f' - assert common.truncate_object_bytes('foo' * 100, 50) == '' + previous_length = 301 + for maximum_bytes in (99, 80, 55, 51, 50): + result = common.truncate_object_bytes('foo' * 100, maximum_bytes) + assert sys.getsizeof(result) < maximum_bytes + assert len(result) < 300 + assert ('foo' * 100).startswith(result) + # A smaller budget must never retain more characters. + assert len(result) <= previous_length + previous_length = len(result) assert common.truncate_object_bytes('foo' * 100, 0) == '' @@ -340,8 +356,19 @@ def test_truncate_generator_bytes(): yield 'foo' raise ValueError('Truncate should not get here') - assert list(common.truncate_generator_bytes(generator(), 80)) == ['foo', 'foo'] - assert list(common.truncate_generator_bytes(generator(), 200)) == ['foo', 'foo', 'foo', 'foo'] + # Exact chunk counts depend on the interpreter's str object size (which changed + # in Python 3.12), so assert the contract: only whole chunks are yielded, all + # chunks before the last fit within the budget, and the generator is never + # consumed past the budget (the ValueError is never raised). + previous_length = 0 + for maximum_bytes in (80, 200): + result = list(common.truncate_generator_bytes(generator(), maximum_bytes)) + assert result == ['foo'] * len(result) + assert 0 < len(result) <= 5 + assert sum(sys.getsizeof(chunk) for chunk in result[:-1]) < maximum_bytes + # A larger budget must never yield fewer chunks. + assert len(result) >= previous_length + previous_length = len(result) def test_check_media_directory(test_directory):