diff --git a/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot b/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot index 8518c4fc..afd3c2e5 100644 --- a/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot +++ b/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot @@ -132,6 +132,10 @@ mkdir /media/wrolpi [[ -f /opt/wrolpi-blobs/map-overview.pmtiles && -s /opt/wrolpi-blobs/map-overview.pmtiles ]] || \ (echo "ERROR: map-overview.pmtiles missing from /opt/wrolpi-blobs!" && exit 1) +# Ship committed blobs (e.g. the default Zim served by Kiwix when the media dir +# has no working zims) into /opt/wrolpi-blobs. +cp /opt/wrolpi/wrolpi/blobs/* /opt/wrolpi-blobs/ + # Bake in the map toolchain (pmtiles CLI + tippecanoe) so first boot does not # need internet. Shared with pi-gen and on-device upgrades via a single script # so versions/pins live in one place. It is safe in the chroot: unlike diff --git a/docker/zim/Dockerfile b/docker/zim/Dockerfile index b390bac7..426d6b95 100644 --- a/docker/zim/Dockerfile +++ b/docker/zim/Dockerfile @@ -2,6 +2,11 @@ FROM debian:13 ENV DOCKER=true RUN apt update && apt-get install -y kiwix kiwix-tools + +# Committed blobs (e.g. the default Zim served when the media directory has no +# functioning zim files). +COPY wrolpi/blobs/ /opt/wrolpi-blobs/ + COPY docker/zim/* / ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/docker/zim/entrypoint.sh b/docker/zim/entrypoint.sh index e0df9a03..2e622bb4 100755 --- a/docker/zim/entrypoint.sh +++ b/docker/zim/entrypoint.sh @@ -1,10 +1,38 @@ #! /usr/bin/env bash -rm -f /library.xml +set -u -# Find all Zim files and import them. -find /media/wrolpi -iname '*.zim' -exec /import_zim.sh {} \; +LIBRARY=/library.xml +# Default Zim baked into the image; served when no real zim is available. +DEFAULT_ZIM="${DEFAULT_ZIM:-/opt/wrolpi-blobs/default.zim}" + +rm -f "${LIBRARY}" + +# Import every *.zim under the mounted media directory. A corrupt or partial +# zim is rejected by kiwix-manage; skip it so the remaining valid zims are +# still served. We scan the whole media mount, so any configured +# `zims_destination` subdirectory is covered automatically -- no config read. +added_zim=false +while IFS= read -r -d '' zim; do + if /import_zim.sh "${zim}"; then + added_zim=true + else + echo "Skipping zim which could not be added (corrupt or incomplete?): ${zim}" >&2 + fi +done < <(find /media/wrolpi -iname '*.zim' -print0) + +# When no functioning zim was added, fall back to the default Zim so the user +# sees a helpful placeholder instead of a blank Kiwix. +if [ "${added_zim}" = false ] && [ -f "${DEFAULT_ZIM}" ]; then + echo "No usable Zim files found; serving the default WROLPi Zim: ${DEFAULT_ZIM}" >&2 + /import_zim.sh "${DEFAULT_ZIM}" || echo "Failed to add the default Zim" >&2 +fi + +# Ensure a library file always exists so kiwix-serve can start regardless. +if [ ! -f "${LIBRARY}" ]; then + printf '\n\n\n' > "${LIBRARY}" +fi # Bind all interfaces by omitting --address: kiwix-serve defaults to all # interfaces, and trixie's kiwix-tools 3.7.0 rejects the literal # "--address 0.0.0.0" ("IP address is not available on this system"). -kiwix-serve --monitorLibrary --library /library.xml --port 80 +kiwix-serve --monitorLibrary --library "${LIBRARY}" --port 80 diff --git a/modules/zim/test/test_read_config_value.py b/modules/zim/test/test_read_config_value.py new file mode 100644 index 00000000..41ce7f44 --- /dev/null +++ b/modules/zim/test/test_read_config_value.py @@ -0,0 +1,69 @@ +"""Tests for wrolpi/scripts/read_config_value.sh. + +The reader extracts a top-level scalar from wrolpi.yaml with zero dependencies +so non-Python scripts (e.g. start_kiwix_serve.sh, the Docker Zim entrypoint) can +read config values. Exercised directly via bash. +""" +import os +import pathlib +import subprocess + +import pytest + +REPO_ROOT = pathlib.Path(__file__).parents[3] +READER = REPO_ROOT / 'wrolpi' / 'scripts' / 'read_config_value.sh' + + +def _read(media: pathlib.Path, key: str, default: str = '') -> str: + env = {**os.environ, 'MEDIA_DIRECTORY': str(media)} + result = subprocess.run(['bash', str(READER), key, default], env=env, + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stderr + return result.stdout.strip('\n') + + +@pytest.fixture +def media_with_config(tmp_path): + media = tmp_path / 'media' + (media / 'config').mkdir(parents=True) + + def write(text: str): + (media / 'config' / 'wrolpi.yaml').write_text(text) + + return media, write + + +def test_reads_scalar_value(media_with_config): + media, write = media_with_config + write('zims_destination: mycustomzims\nvideos_destination: videos\n') + assert _read(media, 'zims_destination', 'zims') == 'mycustomzims' + + +def test_strips_surrounding_quotes(media_with_config): + media, write = media_with_config + write('videos_destination: "quoted videos"\n') + assert _read(media, 'videos_destination', 'x') == 'quoted videos' + + +def test_missing_key_returns_default(media_with_config): + media, write = media_with_config + write('videos_destination: videos\n') + assert _read(media, 'zims_destination', 'zims') == 'zims' + + +def test_empty_value_returns_default(media_with_config): + media, write = media_with_config + write('zims_destination:\n') + assert _read(media, 'zims_destination', 'zims') == 'zims' + + +def test_missing_config_file_returns_default(tmp_path): + # No config directory/file at all (fresh install) -> default, no error. + assert _read(tmp_path / 'media', 'zims_destination', 'zims') == 'zims' + + +def test_ignores_indented_nested_keys(media_with_config): + media, write = media_with_config + # A nested key with the same name must not be matched at the top level. + write('some_section:\n zims_destination: nested\ntop: value\n') + assert _read(media, 'zims_destination', 'zims') == 'zims' diff --git a/modules/zim/test/test_start_kiwix_serve.py b/modules/zim/test/test_start_kiwix_serve.py index 4999e8d5..c9d13d67 100644 --- a/modules/zim/test/test_start_kiwix_serve.py +++ b/modules/zim/test/test_start_kiwix_serve.py @@ -3,9 +3,12 @@ The script imports every *.zim in the Zim directory into a Kiwix library, then starts kiwix-serve. These tests exercise the script directly with fake `kiwix-manage`/`kiwix-serve` binaries on PATH so they need no real Kiwix -installation. The key invariant: a single corrupt/incomplete zim must never -prevent the remaining valid zims from being served, nor stop the server from -starting. +installation. Key invariants: + +- A single corrupt/incomplete zim must never prevent the remaining valid zims + from being served, nor stop the server from starting. +- The configured Zim directory (wrolpi.yaml `zims_destination`) is honored. +- When no functioning zim is found, the default Zim is served as a fallback. """ import os import pathlib @@ -14,7 +17,8 @@ import subprocess import pytest -SCRIPT = pathlib.Path(__file__).parents[3] / 'scripts' / 'start_kiwix_serve.sh' +REPO_ROOT = pathlib.Path(__file__).parents[3] +SCRIPT = REPO_ROOT / 'scripts' / 'start_kiwix_serve.sh' # Fake kiwix-manage: refuses any zim whose contents contain "CORRUPT" (mimicking # kiwix-manage's "Cannot add zim ..." rejection of partial/corrupt files), @@ -55,6 +59,7 @@ def kiwix_env(tmp_path): media = tmp_path / 'media' zims = media / 'zims' zims.mkdir(parents=True) + (media / 'config').mkdir() bin_dir = tmp_path / 'bin' bin_dir.mkdir() @@ -64,20 +69,34 @@ def kiwix_env(tmp_path): p.chmod(p.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) started_marker = tmp_path / 'started' + # Points at a file that does not exist by default (so the fallback is a + # no-op); the fallback test creates it. + default_zim = tmp_path / 'blobs' / 'default.zim' + + def write_config(text: str): + (media / 'config' / 'wrolpi.yaml').write_text(text) def run(): env = { **os.environ, 'MEDIA_DIRECTORY': str(media), + 'PROJECT_DIR': str(REPO_ROOT), + 'DEFAULT_ZIM': str(default_zim), 'PATH': f'{bin_dir}:{os.environ["PATH"]}', 'STARTED_MARKER': str(started_marker), } return subprocess.run(['bash', str(SCRIPT)], env=env, capture_output=True, text=True, timeout=60) - return dict(media=media, zims=zims, library=zims / 'library.xml', started_marker=started_marker, run=run) + def library_text(): + lib = zims / 'library.xml' + return lib.read_text() if lib.exists() else '' + + return dict(media=media, zims=zims, library=zims / 'library.xml', started_marker=started_marker, + default_zim=default_zim, write_config=write_config, library_text=library_text, run=run) def _write_zim(path: pathlib.Path, corrupt: bool = False): + path.parent.mkdir(parents=True, exist_ok=True) path.write_text('CORRUPT' if corrupt else 'ZIMDATA') @@ -107,7 +126,7 @@ def test_corrupt_zim_does_not_prevent_serving_valid_zims(kiwix_env): def test_only_corrupt_zim_still_starts_server(kiwix_env): - """Even when the only zim is corrupt, kiwix must still start (empty library).""" + """Even when the only zim is corrupt, kiwix must still start.""" _write_zim(kiwix_env['zims'] / 'sudan_broken.zim', corrupt=True) result = kiwix_env['run']() @@ -123,3 +142,30 @@ def test_no_zims_still_starts_server(kiwix_env): assert result.returncode == 0, result.stderr assert kiwix_env['started_marker'].exists(), 'kiwix-serve was not started' + + +def test_honors_configured_zims_destination(kiwix_env): + """A custom zims_destination in wrolpi.yaml is used instead of the default `zims`.""" + kiwix_env['write_config']('zims_destination: mycustomzims\n') + _write_zim(kiwix_env['media'] / 'mycustomzims' / 'wikipedia.zim') + + result = kiwix_env['run']() + + assert result.returncode == 0, result.stderr + assert kiwix_env['started_marker'].exists(), 'kiwix-serve was not started' + # The library lives in the configured directory and references the zim there. + library = kiwix_env['media'] / 'mycustomzims' / 'library.xml' + assert library.exists() + assert 'mycustomzims/wikipedia.zim' in library.read_text() + + +def test_falls_back_to_default_zim_when_no_usable_zim(kiwix_env): + """When no functioning zim is present, the default Zim is served.""" + # No zims in the directory; create the default Zim the script should fall back to. + _write_zim(kiwix_env['default_zim']) + + result = kiwix_env['run']() + + assert result.returncode == 0, result.stderr + assert kiwix_env['started_marker'].exists(), 'kiwix-serve was not started' + assert 'default.zim' in kiwix_env['library'].read_text() diff --git a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh index 6c5dbd8d..c1ca9a39 100755 --- a/pi-gen/stage2/04-wrolpi/04-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/04-run-chroot.sh @@ -18,6 +18,10 @@ cp /opt/wrolpi/etc/raspberrypios/*desktop /etc/skel/Desktop [[ -f /opt/wrolpi-blobs/map-overview.pmtiles && -s /opt/wrolpi-blobs/map-overview.pmtiles ]] || \ (echo "ERROR: map-overview.pmtiles missing from /opt/wrolpi-blobs!" && exit 1) +# Ship committed blobs (e.g. the default Zim served by Kiwix when the media dir +# has no working zims) into /opt/wrolpi-blobs. +cp /opt/wrolpi/wrolpi/blobs/* /opt/wrolpi-blobs/ + # Install WROLPi Help. /opt/wrolpi/scripts/install_help_service.sh diff --git a/scripts/start_kiwix_serve.sh b/scripts/start_kiwix_serve.sh index 092b048a..b837d1e9 100755 --- a/scripts/start_kiwix_serve.sh +++ b/scripts/start_kiwix_serve.sh @@ -2,12 +2,19 @@ # This file imports all *.zim files in the Zim media directory, then starts kiwix-serve. set -u -# The media directory is overridable for testing; defaults to the production mount. +# The media directory and project directory are overridable for testing; +# they default to the production locations. MEDIA_DIRECTORY="${MEDIA_DIRECTORY:-/media/wrolpi}" +PROJECT_DIR="${PROJECT_DIR:-/opt/wrolpi}" +# Default Zim shipped in /opt/wrolpi-blobs; served when no real zim is available. +DEFAULT_ZIM="${DEFAULT_ZIM:-/opt/wrolpi-blobs/default.zim}" [ ! -d "${MEDIA_DIRECTORY}" ] && echo "Cannot start kiwix without media directory mounted" && exit 1 -ZIM_DIRECTORY="${MEDIA_DIRECTORY}/zims" +# Honor the configured Zim directory (wrolpi.yaml `zims_destination`, always +# relative to the media directory), defaulting to `zims`. +ZIMS_DEST="$(MEDIA_DIRECTORY="${MEDIA_DIRECTORY}" "${PROJECT_DIR}/wrolpi/scripts/read_config_value.sh" zims_destination zims)" +ZIM_DIRECTORY="${MEDIA_DIRECTORY}/${ZIMS_DEST}" LIBRARY="${ZIM_DIRECTORY}/library.xml" # Wait up to 30 seconds for the zims directory to be mounted. @@ -44,11 +51,20 @@ done < <(find "${ZIM_DIRECTORY}" -iname '*.zim' -print0) if [ "${found_zim}" = false ]; then echo "Could not find any Zim files to import" elif [ "${added_zim}" = false ]; then - echo "Found Zim files but none could be added (all corrupt or incomplete?); starting Kiwix with an empty library" >&2 + echo "Found Zim files but none could be added (all corrupt or incomplete?)" >&2 +fi + +# When no functioning zim was added, fall back to the default Zim shipped in +# /opt/wrolpi-blobs so the user sees a helpful placeholder ("download some Zim +# files") instead of a blank Kiwix. A single corrupt zim must never take down +# the server. +if [ "${added_zim}" = false ] && [ -f "${DEFAULT_ZIM}" ]; then + echo "No usable Zim files found; serving the default WROLPi Zim: ${DEFAULT_ZIM}" >&2 + kiwix-manage "${LIBRARY}" add "${DEFAULT_ZIM}" || echo "Failed to add the default Zim" >&2 fi # Ensure a library file always exists so kiwix-serve can start even when there -# are no valid zims. A single corrupt zim must never take down the server. +# are no valid zims and the default Zim is missing. if [ ! -f "${LIBRARY}" ]; then printf '\n\n\n' > "${LIBRARY}" fi diff --git a/wrolpi/blobs/default.zim b/wrolpi/blobs/default.zim new file mode 100644 index 00000000..6d00174f Binary files /dev/null and b/wrolpi/blobs/default.zim differ diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 631638be..55650815 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -223,6 +223,7 @@ async def update_settings(_: Request, body: schema.SettingsRequest): # Remove any keys with None values, then save the config. new_config = {k: v for k, v in body.__dict__.items() if v is not None} wrolpi_config = get_wrolpi_config() + old_zims_destination = wrolpi_config.zims_destination if not new_config: raise InvalidConfig() @@ -300,6 +301,13 @@ async def update_settings(_: Request, body: schema.SettingsRequest): # Save config settings (hotspot control is now through Controller endpoints) wrolpi_config.update(new_config) + # If the Zim directory changed, restart Kiwix so it serves the new directory + # without waiting for a reboot. Local import avoids a circular import + # (modules depend on wrolpi, not the reverse). + if wrolpi_config.zims_destination != old_zims_destination: + from modules.zim.lib import restart_kiwix_handler + restart_kiwix_handler.activate_switch() + # Apply timezone to the system via Controller (native deployments only). if body.timezone is not None and not DOCKERIZED: timezone_value = new_config.get('timezone') diff --git a/wrolpi/scripts/default_zim/build_default_zim.sh b/wrolpi/scripts/default_zim/build_default_zim.sh new file mode 100755 index 00000000..2de45c60 --- /dev/null +++ b/wrolpi/scripts/default_zim/build_default_zim.sh @@ -0,0 +1,31 @@ +#! /usr/bin/env bash +# Build the default fallback Zim that Kiwix serves when a WROLPi has no +# functioning Zim files. The output is committed to wrolpi/blobs/default.zim, +# which every build copies into /opt/wrolpi-blobs. +# +# Reproducible via the openzim/zim-tools Docker image, so no host zim-tools +# installation is required. Re-run this only when src/ changes. +set -euo pipefail + +HERE="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &>/dev/null && pwd )" +BLOBS="$( cd -- "${HERE}/../../blobs" &>/dev/null && pwd )" +IMAGE="ghcr.io/openzim/zim-tools:latest" + +echo "Building default.zim from ${HERE}/src -> ${BLOBS}/default.zim ..." +rm -f "${BLOBS}/default.zim" # zimwriterfs refuses to overwrite an existing file. +docker run --rm -v "${HERE}:/work" -v "${BLOBS}:/out" -w /work "${IMAGE}" \ + zimwriterfs \ + --welcome=index.html \ + --illustration=favicon.png \ + --language=eng \ + --title="WROLPi" \ + --description="Default WROLPi Zim. Download Zim files to replace this placeholder." \ + --creator="WROLPi" \ + --publisher="WROLPi" \ + --name="wrolpi.default" \ + src /out/default.zim + +echo "Validating default.zim ..." +docker run --rm -v "${BLOBS}:/out" -w /out "${IMAGE}" zimcheck default.zim + +echo "Done: ${BLOBS}/default.zim" diff --git a/wrolpi/scripts/default_zim/src/favicon.png b/wrolpi/scripts/default_zim/src/favicon.png new file mode 100644 index 00000000..d6c8a169 Binary files /dev/null and b/wrolpi/scripts/default_zim/src/favicon.png differ diff --git a/wrolpi/scripts/default_zim/src/index.html b/wrolpi/scripts/default_zim/src/index.html new file mode 100644 index 00000000..b65d87e6 --- /dev/null +++ b/wrolpi/scripts/default_zim/src/index.html @@ -0,0 +1,60 @@ + + + + + + WROLPi — No Zim files yet + + + + +

This is the default WROLPi Zim

+ +

+ You are seeing this page because your WROLPi has no Zim files yet + (or the Zim files it found could not be read). This built-in placeholder keeps the + Kiwix viewer running until you download real content. +

+ +
+ To add encyclopedias and reference libraries, open Zim in the WROLPi + menu and download what you want — Wikipedia, Wiktionary, StackExchange, iFixit, + medical and survival references, and more. +
+ +

+ Downloaded Zim files are stored in your Zim directory (by default + zims inside your media directory). Once at least one working Zim file is + present, Kiwix will serve it here automatically and this page will disappear. +

+ + + + diff --git a/wrolpi/scripts/lib.sh b/wrolpi/scripts/lib.sh index 4630bd7b..13edc248 100755 --- a/wrolpi/scripts/lib.sh +++ b/wrolpi/scripts/lib.sh @@ -16,6 +16,11 @@ PROJECT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) PROJECT_DIR=$(dirname "${PROJECT_DIR}") PROJECT_DIR=$(dirname "${PROJECT_DIR}") +# Read a top-level scalar value from wrolpi.yaml. Usage: config_value KEY [DEFAULT] +function config_value { + "${PROJECT_DIR}/wrolpi/scripts/read_config_value.sh" "$@" +} + rpi=false debian12=false if (grep 'Raspberry Pi' /proc/cpuinfo >/dev/null); then diff --git a/wrolpi/scripts/read_config_value.sh b/wrolpi/scripts/read_config_value.sh new file mode 100755 index 00000000..57dfd60d --- /dev/null +++ b/wrolpi/scripts/read_config_value.sh @@ -0,0 +1,52 @@ +#! /usr/bin/env bash +# Read a single top-level scalar value from WROLPi's config (wrolpi.yaml). +# +# wrolpi.yaml is the source of truth for configuration, but scripts that run +# outside the Python API (e.g. start_kiwix_serve.sh, or a minimal Docker +# container) still need to read config values. This reader has zero +# dependencies -- no Python, no jq/yq -- so it runs identically everywhere. +# +# Usage: read_config_value.sh KEY [DEFAULT] +# Prints the value of the top-level scalar KEY, or DEFAULT (empty if omitted) +# when the config file or the key is missing/empty. +# +# Limitation: only top-level scalar keys are supported (sufficient for the +# *_destination keys). Nested keys, lists, and multi-line values are not +# parsed. wrolpi.yaml is written by PyYAML and is regular, so this is reliable +# for the keys it is used with. +set -u + +KEY="${1:?Usage: read_config_value.sh KEY [DEFAULT]}" +DEFAULT="${2:-}" + +MEDIA_DIRECTORY="${MEDIA_DIRECTORY:-/media/wrolpi}" +CONFIG="${MEDIA_DIRECTORY}/config/wrolpi.yaml" + +value="" +if [ -f "${CONFIG}" ]; then + value=$(awk -v key="${KEY}" ' + # Only consider unindented (top-level) "key:" lines. + /^[A-Za-z0-9_]+[[:space:]]*:/ { + idx = index($0, ":") + k = substr($0, 1, idx - 1) + gsub(/[[:space:]]+$/, "", k) + if (k == key) { + v = substr($0, idx + 1) + sub(/^[[:space:]]+/, "", v) # leading whitespace + sub(/[[:space:]]+#.*$/, "", v) # trailing inline comment + sub(/[[:space:]]+$/, "", v) # trailing whitespace + # Strip a matching pair of surrounding quotes. + if (v ~ /^".*"$/ || v ~ /^'\''.*'\''$/) + v = substr(v, 2, length(v) - 2) + print v + exit + } + } + ' "${CONFIG}") +fi + +if [ -z "${value}" ]; then + printf '%s\n' "${DEFAULT}" +else + printf '%s\n' "${value}" +fi diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 1f22c275..25b7ebdf 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -84,6 +84,26 @@ async def test_daily_download_limit_settings(test_session, async_client): assert response.status_code == HTTPStatus.BAD_REQUEST +@pytest.mark.asyncio +async def test_changing_zims_destination_restarts_kiwix(test_session, async_client): + """Changing zims_destination should activate the restart_kiwix switch so Kiwix serves the new + directory; an unrelated settings change should not.""" + from modules.zim import lib as zim_lib + + with mock.patch.object(zim_lib.restart_kiwix_handler, 'activate_switch') as mock_activate: + # An unrelated change does not restart Kiwix. + request, response = await async_client.patch('/api/settings', content=json.dumps({'log_level': 20})) + assert response.status_code == HTTPStatus.NO_CONTENT + mock_activate.assert_not_called() + + # Changing the Zim directory does. + data = {'zims_destination': 'my_zims'} + request, response = await async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.NO_CONTENT + assert get_wrolpi_config().zims_destination == 'my_zims' + mock_activate.assert_called_once() + + @pytest.mark.asyncio async def test_log_level(async_client): """Log level must be between 0 and 40."""