ai: package llama-server for all three deployments, with idle unload

install_llama_cpp.sh builds llama-server from the SHA-pinned v0.3.0
source tarball (single source of truth: on-device upgrade, Pi image,
Live ISO, and the new docker/llm container all call it; version-marker
makes re-runs no-ops).  start_llama_server.sh refuses to start unless
ai.yaml enables AI and the active model file exists — which also covers
unmounted media.  wrolpi-llm.service is installed but deliberately
never enabled at boot: the Controller starts it on demand (registered
in managed_services; the llm compose service follows the samba
pattern), and the API's perpetual idle-unload worker asks the
Controller to stop it after the configured idle minutes.  llama-server
is localhost-only natively, internal-network-only in docker, and never
proxied by Caddy.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-08-25 21:36:00 -06:00
parent 119256e529
commit 318b835faf
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
18 changed files with 406 additions and 4 deletions

View file

@ -78,6 +78,16 @@ DEFAULT_CONFIG = {
"use_https": True,
"description": "Kiwix/Zim server",
},
{
"name": "wrolpi-llm",
"systemd_name": "wrolpi-llm",
"port": 11435,
"viewable": False,
"use_https": False,
"description": "AI assistant (llama-server)",
# On-demand: started by the Controller when AI is enabled, stopped by the API when idle.
"kind": "persistent",
},
# System services
{
"name": "caddy",

View file

@ -27,7 +27,7 @@ except ImportError:
CONTAINER_PREFIX = os.environ.get("COMPOSE_PROJECT_NAME", "wrolpi")
# Containers that should not have a "View" button (non-HTTP services)
NON_VIEWABLE_CONTAINERS = {"redis", "memcached", "samba"}
NON_VIEWABLE_CONTAINERS = {"redis", "memcached", "samba", "llm"}
# Containers that should use HTTPS
HTTPS_CONTAINERS = {"web"}

View file

@ -57,7 +57,7 @@ class TestManagedServicesConfig:
def test_managed_services_count(self):
"""Should have expected number of managed services."""
services = DEFAULT_CONFIG["managed_services"]
assert len(services) == 11
assert len(services) == 12
@pytest.mark.parametrize("name,expected", [
("wrolpi-api",
@ -68,6 +68,8 @@ class TestManagedServicesConfig:
("wrolpi-app-dev", {"port": 3000, "viewable": False, "show_only_when_running": True}),
("caddy", {"port": 443}),
("wrolpi-upgrade", {"port": None, "viewable": False, "show_only_when_running": True}),
("wrolpi-llm",
{"systemd_name": "wrolpi-llm", "port": 11435, "viewable": False, "kind": "persistent"}),
])
def test_service_config(self, name, expected):
"""Should have service configured with expected values."""

View file

@ -143,6 +143,9 @@ cp /opt/wrolpi/wrolpi/blobs/* /opt/wrolpi-blobs/
# systemd.
/opt/wrolpi/wrolpi/scripts/install_map_tools.sh
# Bake in llama.cpp (local AI) so a fresh install does not compile it on first upgrade.
/opt/wrolpi/wrolpi/scripts/install_llama_cpp.sh
# Install WROLPi Help.
/opt/wrolpi/scripts/install_help_service.sh

View file

@ -42,6 +42,8 @@ services:
# Sanic 25.x wants 'spawn' but our code uses 'fork' for multiprocessing.
# This tells Sanic to accept the existing start method.
- SANIC_START_METHOD_SET=1
# llama-server runs in the llm container (native installs use 127.0.0.1).
- LLAMA_URL=http://llm:11435
volumes:
- './main.py:/opt/wrolpi/main.py'
- './wrolpi:/opt/wrolpi/wrolpi'
@ -147,6 +149,21 @@ services:
test: [ 'CMD-SHELL', 'curl http://127.0.0.1:80' ]
interval: 1m
llm: # llama-server (local AI) - started/stopped by the controller on demand; the API stops it when idle.
build:
context: .
dockerfile: docker/llm/Dockerfile
restart: "no"
depends_on:
controller:
condition: service_healthy
volumes:
- ${MEDIA_DIRECTORY:-./test}:/media/wrolpi
- './docker/llm/entrypoint.sh:/entrypoint.sh'
healthcheck:
test: [ 'CMD-SHELL', 'curl http://127.0.0.1:11435/health' ]
interval: 1m
samba: # Samba file sharing - started/stopped by the controller based on configured shares.
build:
context: .

17
docker/llm/Dockerfile Normal file
View file

@ -0,0 +1,17 @@
FROM debian:13
ENV DOCKER=true
RUN apt-get update && apt-get install -y curl ca-certificates
# Build llama-server from the pinned release (single source of truth with native installs).
COPY wrolpi/scripts/install_llama_cpp.sh /tmp/install_llama_cpp.sh
RUN /tmp/install_llama_cpp.sh && rm /tmp/install_llama_cpp.sh \
&& apt-get remove -y build-essential cmake && apt-get autoremove -y && apt-get clean
# The same startup scripts the native install uses.
COPY scripts/start_llama_server.sh /opt/wrolpi/scripts/start_llama_server.sh
COPY wrolpi/scripts/read_config_value.sh /opt/wrolpi/wrolpi/scripts/read_config_value.sh
COPY docker/llm/entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

5
docker/llm/entrypoint.sh Executable file
View file

@ -0,0 +1,5 @@
#! /usr/bin/env bash
# Bind all interfaces so the api container can reach llama-server on the internal docker network.
# The container publishes no host ports and is not proxied by Caddy.
export LLAMA_HOST=0.0.0.0
exec /opt/wrolpi/scripts/start_llama_server.sh

View file

@ -0,0 +1,17 @@
[Unit]
Description=WROLPi AI (llama-server)
PartOf=wrolpi.target
[Service]
ExecStart=/opt/wrolpi/scripts/start_llama_server.sh
Group=wrolpi
User=wrolpi
WorkingDirectory=/opt/wrolpi/
Restart=on-failure
# The lowest priority of any WROLPi service; inference must never starve the API.
Nice=15
# On-demand service: deliberately NOT enabled at boot and NOT wanted by wrolpi.target.
# The Controller starts it when the user enables AI; the API stops it when idle.
[Install]
WantedBy=default.target

View file

@ -1,2 +1,3 @@
from modules.ai.api import ai_bp # noqa
from modules.ai.downloader import ModelDownloader # noqa Instantiation registers the downloader.
from modules.ai import service # noqa Import registers the idle-unload perpetual worker.

View file

@ -8,7 +8,7 @@ import os
from typing import Tuple
from urllib.parse import urlencode
from wrolpi.common import aiohttp_get, logger
from wrolpi.common import aiohttp_get, aiohttp_post, logger
from wrolpi.vars import DOCKERIZED
logger = logger.getChild(__name__)
@ -25,3 +25,13 @@ async def controller_get(path: str, params: dict = None, timeout: int = 10) -> T
url = f'{url}?{urlencode(params)}'
async with aiohttp_get(url, timeout=timeout) as response:
return response.status, await response.json()
async def controller_post(path: str, json_: dict = None, timeout: int = 30) -> Tuple[int, dict]:
"""POST a Controller endpoint, returning (status, JSON body). Raises on connection errors."""
async with aiohttp_post(f'{CONTROLLER_URL}{path}', json_=json_ or {}, timeout=timeout) as response:
try:
body = await response.json()
except Exception:
body = {}
return response.status, body

112
modules/ai/service.py Normal file
View file

@ -0,0 +1,112 @@
"""llama-server lifecycle: on-demand start and idle unload.
The API owns the policy (this module); the Controller executes the mechanism through its
existing service start/stop endpoints — it runs as root natively and manages the `llm`
container in docker, so no sudoers changes are needed.
Idle unload is mandatory on small devices: a swapped-to-death Pi is worse than a slow first
token. The chat handler records activity; the perpetual worker stops llama-server after the
configured idle minutes."""
import asyncio
import os
import time
from modules.ai.config import get_ai_config
from modules.ai.controller_client import controller_post
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.common import aiohttp_get, logger
from wrolpi.vars import DOCKERIZED
logger = logger.getChild(__name__)
LLAMA_URL = os.environ.get('LLAMA_URL', 'http://127.0.0.1:11435')
# The Controller's name for the service: the systemd unit natively, the compose service in docker.
LLM_SERVICE_NAME = 'llm' if DOCKERIZED else 'wrolpi-llm'
# How long to wait for llama-server to answer /health after a cold start (model load included).
STARTUP_TIMEOUT_SECONDS = 90
def record_llm_activity():
"""Note that the LLM was just used; postpones the idle unload."""
try:
api_app.shared_ctx.ai_llm_last_request.value = time.time()
except AttributeError:
# Shared contexts are absent in some unit tests.
pass
def get_last_llm_activity() -> float:
try:
return api_app.shared_ctx.ai_llm_last_request.value
except AttributeError:
return 0.0
async def llama_healthy() -> bool:
try:
async with aiohttp_get(f'{LLAMA_URL}/health', timeout=5) as response:
return response.status == 200
except Exception:
return False
async def start_llm_service() -> bool:
status, body = await controller_post(f'/api/services/{LLM_SERVICE_NAME}/start')
if status != 200:
logger.error(f'Controller failed to start {LLM_SERVICE_NAME}: HTTP {status} {body}')
return False
return True
async def stop_llm_service() -> bool:
status, body = await controller_post(f'/api/services/{LLM_SERVICE_NAME}/stop')
if status != 200:
logger.error(f'Controller failed to stop {LLM_SERVICE_NAME}: HTTP {status} {body}')
return False
return True
async def ensure_llama_running(timeout: int = STARTUP_TIMEOUT_SECONDS) -> bool:
"""Start llama-server on demand and wait for it to answer /health.
Returns True when llama-server is up. Cold-start latency (service start + model load) is
expected and acceptable."""
if await llama_healthy():
record_llm_activity()
return True
logger.info(f'Asking the Controller to start {LLM_SERVICE_NAME}')
if not await start_llm_service():
return False
started = time.time()
while time.time() - started < timeout:
if await llama_healthy():
record_llm_activity()
return True
await asyncio.sleep(1)
logger.error(f'llama-server did not become healthy within {timeout}s')
return False
@perpetual_signal(sleep=60)
async def perpetual_llm_idle_unload():
"""Stop llama-server after the configured idle minutes. Only acts after it was used."""
try:
last_activity = get_last_llm_activity()
if not last_activity:
return
config = get_ai_config()
idle_seconds = config.idle_unload_minutes * 60
if time.time() - last_activity < idle_seconds:
return
if not await llama_healthy():
# Already stopped (or crashed); nothing to unload.
api_app.shared_ctx.ai_llm_last_request.value = 0.0
return
logger.info(f'Stopping {LLM_SERVICE_NAME} after {config.idle_unload_minutes} idle minutes')
if await stop_llm_service():
api_app.shared_ctx.ai_llm_last_request.value = 0.0
except Exception as e:
logger.error('LLM idle unload worker failed', exc_info=e)

View file

@ -0,0 +1,78 @@
"""Tests for the llama-server lifecycle: on-demand start and idle unload."""
import time
from unittest import mock
import pytest
from modules.ai import service
@pytest.mark.asyncio
async def test_ensure_llama_running_already_healthy(async_client):
"""A healthy llama-server is used as-is and activity is recorded."""
with mock.patch('modules.ai.service.llama_healthy', return_value=True), \
mock.patch('modules.ai.service.start_llm_service') as start:
assert await service.ensure_llama_running() is True
start.assert_not_called()
assert service.get_last_llm_activity() > 0
@pytest.mark.asyncio
async def test_ensure_llama_running_cold_start(async_client):
"""An unhealthy llama-server is started via the Controller and polled until healthy."""
health = iter([False, False, True])
async def fake_healthy():
return next(health)
with mock.patch('modules.ai.service.llama_healthy', side_effect=fake_healthy), \
mock.patch('modules.ai.service.start_llm_service', return_value=True) as start, \
mock.patch('modules.ai.service.asyncio.sleep'):
assert await service.ensure_llama_running(timeout=10) is True
start.assert_called_once()
@pytest.mark.asyncio
async def test_ensure_llama_running_start_fails(async_client):
"""A Controller start failure is reported as False, not an exception."""
with mock.patch('modules.ai.service.llama_healthy', return_value=False), \
mock.patch('modules.ai.service.start_llm_service', return_value=False):
assert await service.ensure_llama_running(timeout=1) is False
@pytest.mark.asyncio
async def test_idle_unload(async_client, test_ai_config):
"""llama-server is stopped only after the configured idle minutes."""
from wrolpi.api_utils import api_app
from modules.ai.config import get_ai_config
get_ai_config().idle_unload_minutes = 10
# Never used: nothing happens.
api_app.shared_ctx.ai_llm_last_request.value = 0.0
with mock.patch('modules.ai.service.stop_llm_service') as stop:
await service.perpetual_llm_idle_unload()
stop.assert_not_called()
# Recently used: not stopped.
api_app.shared_ctx.ai_llm_last_request.value = time.time()
with mock.patch('modules.ai.service.stop_llm_service') as stop, \
mock.patch('modules.ai.service.llama_healthy', return_value=True):
await service.perpetual_llm_idle_unload()
stop.assert_not_called()
# Idle past the threshold: stopped, and the activity marker resets.
api_app.shared_ctx.ai_llm_last_request.value = time.time() - 11 * 60
with mock.patch('modules.ai.service.stop_llm_service', return_value=True) as stop, \
mock.patch('modules.ai.service.llama_healthy', return_value=True):
await service.perpetual_llm_idle_unload()
stop.assert_called_once()
assert api_app.shared_ctx.ai_llm_last_request.value == 0.0
# Idle but already stopped: the marker resets without a stop call.
api_app.shared_ctx.ai_llm_last_request.value = time.time() - 11 * 60
with mock.patch('modules.ai.service.stop_llm_service') as stop, \
mock.patch('modules.ai.service.llama_healthy', return_value=False):
await service.perpetual_llm_idle_unload()
stop.assert_not_called()
assert api_app.shared_ctx.ai_llm_last_request.value == 0.0

View file

@ -45,6 +45,9 @@ git config --global --add safe.directory /opt/wrolpi
# CPU-heavy step on weak hardware.
/opt/wrolpi/wrolpi/scripts/install_map_tools.sh
# Bake in llama.cpp (local AI) so a fresh Pi does not compile it on first upgrade.
/opt/wrolpi/wrolpi/scripts/install_llama_cpp.sh
# Install Python requirements. Try multiple times because pypi may stop responding.
python3 -m venv /opt/wrolpi/venv
/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt ||

54
scripts/start_llama_server.sh Executable file
View file

@ -0,0 +1,54 @@
#! /usr/bin/env bash
# Start llama-server (WROLPi's local AI inference runtime) with the model selected in ai.yaml.
#
# AI is opt-in: this refuses to start unless ai.yaml enables it and the active model file exists
# in the media directory (which also covers the unmounted-media case). Started/stopped on demand
# by the Controller; the API's idle-unload worker stops it after inactivity.
set -u
MEDIA_DIRECTORY="${MEDIA_DIRECTORY:-/media/wrolpi}"
PROJECT_DIR="${PROJECT_DIR:-/opt/wrolpi}"
AI_CONFIG="${MEDIA_DIRECTORY}/config/ai.yaml"
# Docker overrides the host so the api container can reach this container.
LLAMA_HOST="${LLAMA_HOST:-127.0.0.1}"
LLAMA_PORT="${LLAMA_PORT:-11435}"
read_ai_config() {
MEDIA_DIRECTORY="${MEDIA_DIRECTORY}" WROLPI_CONFIG_FILE="${AI_CONFIG}" \
"${PROJECT_DIR}/wrolpi/scripts/read_config_value.sh" "$@"
}
[ ! -d "${MEDIA_DIRECTORY}" ] && echo "Cannot start llama-server without media directory mounted" && exit 1
[ ! -f "${AI_CONFIG}" ] && echo "Cannot start llama-server without ${AI_CONFIG}" && exit 1
ENABLED="$(read_ai_config enabled false)"
if [ "${ENABLED}" != "true" ] && [ "${ENABLED}" != "True" ]; then
echo "AI is not enabled in ${AI_CONFIG}; refusing to start"
exit 1
fi
ACTIVE_MODEL="$(read_ai_config active_model)"
[ -z "${ACTIVE_MODEL}" ] && echo "No active_model in ${AI_CONFIG}; refusing to start" && exit 1
# Defense in depth: the model must be a bare file name inside the models directory.
case "${ACTIVE_MODEL}" in
*/*|*..*)
echo "Ignoring unsafe active_model '${ACTIVE_MODEL}'" >&2
exit 1
;;
esac
MODEL="${MEDIA_DIRECTORY}/ai/models/${ACTIVE_MODEL}"
[ ! -f "${MODEL}" ] && echo "Model file does not exist: ${MODEL}" && exit 1
CONTEXT_SIZE="$(read_ai_config context_size 8192)"
case "${CONTEXT_SIZE}" in
''|*[!0-9]*) CONTEXT_SIZE=8192 ;;
esac
# --jinja enables tool-call chat templates; mmap is llama.cpp's default so cold RAM cost is low.
# Localhost only on native installs -- never exposed through Caddy.
exec llama-server \
--host "${LLAMA_HOST}" \
--port "${LLAMA_PORT}" \
--jinja \
-c "${CONTEXT_SIZE}" \
-m "${MODEL}"

View file

@ -258,6 +258,10 @@ rm -f /home/wrolpi/.pgpass
# Migrate from old Leaflet/renderd/Apache2 map stack to MapLibre GL JS + PMTiles.
/opt/wrolpi/scripts/install_protomaps.sh || echo "Protomaps migration failed."
# Install llama.cpp for the local AI assistant (version-guarded no-op once installed;
# the first build takes several minutes on a Raspberry Pi).
/opt/wrolpi/wrolpi/scripts/install_llama_cpp.sh || echo "llama.cpp install failed, continuing..."
# Download map fonts blob if missing.
MAP_FONTS_BLOB=/opt/wrolpi-blobs/map-fonts.tar.gz
if [[ ! -f ${MAP_FONTS_BLOB} || ! -s ${MAP_FONTS_BLOB} ]]; then

View file

@ -59,6 +59,9 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.download_manager_disabled.set()
app.shared_ctx.download_manager_stopped.set()
# Monotonic-ish timestamp of the last chat/LLM request; the AI idle-unload worker reads it.
app.shared_ctx.ai_llm_last_request = multiprocessing.Value(ctypes.c_double, 0.0)
app.shared_ctx.single_tasks_started = multiprocessing.Event()
app.shared_ctx.flags_initialized = multiprocessing.Event()
app.shared_ctx.perpetual_tasks_started = multiprocessing.Event()

View file

@ -0,0 +1,65 @@
#!/usr/bin/env bash
# Install llama.cpp's llama-server, WROLPi's local AI inference runtime.
#
# This is the single source of truth, called by every place that provisions a
# WROLPi so a version bump only has to be made here:
# - scripts/upgrade.sh (on-device upgrade)
# - pi-gen/stage2/04-wrolpi/03-run-chroot.sh (Raspberry Pi image build)
# - debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot (Debian Live ISO build)
# - docker/llm/Dockerfile (Docker llm container)
#
# Built from the SHA-pinned source tarball (llama.cpp ships no generic Linux
# binaries for ARM). The build takes several minutes on a Raspberry Pi; the
# version marker makes re-runs no-ops until the pinned version changes.
# Run as root: installs into /usr/local/bin and apt-installs build dependencies.
set -e
LLAMA_CPP_VERSION="v0.3.0"
# SHA256 of the source tarball -- arch-independent, one value covers every build.
LLAMA_CPP_SHA256="d94c02d86db22d68692f6bb5b3854763d5091e52142868dc7251995517c666d1"
# llama-server's version output tracks build numbers, not release tags, so the
# installed release is recorded in a marker file.
MARKER_DIR=/usr/local/share/wrolpi
MARKER="${MARKER_DIR}/llama_cpp_version"
install_llama_cpp() {
if [ -f "${MARKER}" ] && [ "$(cat "${MARKER}")" = "${LLAMA_CPP_VERSION}" ] \
&& command -v llama-server &>/dev/null; then
echo "llama.cpp ${LLAMA_CPP_VERSION} already installed"
return 0
fi
echo "Building llama.cpp ${LLAMA_CPP_VERSION}..."
apt-get install -y build-essential cmake
local build_dir
build_dir=$(mktemp -d)
curl -fsSL "https://github.com/ggml-org/llama.cpp/archive/refs/tags/${LLAMA_CPP_VERSION}.tar.gz" \
-o /tmp/llama_cpp.tar.gz
echo "${LLAMA_CPP_SHA256} /tmp/llama_cpp.tar.gz" | sha256sum -c -
tar -xz -C "${build_dir}" --strip-components=1 -f /tmp/llama_cpp.tar.gz
rm -f /tmp/llama_cpp.tar.gz
# Static-ish single binary; GGML_NATIVE=OFF so an image built in a chroot/QEMU
# does not bake in the build host's CPU features.
(cd "${build_dir}" &&
cmake -B build \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_SHARED_LIBS=OFF \
-DGGML_NATIVE=OFF \
-DLLAMA_CURL=OFF \
-DLLAMA_BUILD_TESTS=OFF \
-DLLAMA_BUILD_EXAMPLES=OFF \
-DLLAMA_BUILD_SERVER=ON &&
cmake --build build --target llama-server -j"$(nproc)")
install -m 0755 "${build_dir}/build/bin/llama-server" /usr/local/bin/llama-server
rm -rf "${build_dir}"
mkdir -p "${MARKER_DIR}"
echo "${LLAMA_CPP_VERSION}" > "${MARKER}"
llama-server --version || true
echo "llama.cpp ${LLAMA_CPP_VERSION} installed"
}
install_llama_cpp

View file

@ -20,7 +20,8 @@ KEY="${1:?Usage: read_config_value.sh KEY [DEFAULT]}"
DEFAULT="${2:-}"
MEDIA_DIRECTORY="${MEDIA_DIRECTORY:-/media/wrolpi}"
CONFIG="${MEDIA_DIRECTORY}/config/wrolpi.yaml"
# Another top-level config file (e.g. ai.yaml) may be selected with WROLPI_CONFIG_FILE.
CONFIG="${WROLPI_CONFIG_FILE:-${MEDIA_DIRECTORY}/config/wrolpi.yaml}"
value=""
if [ -f "${CONFIG}" ]; then