mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 02:48:38 -04:00
ai: package llama-server for all three deployments, with idle unload
install_llama_cpp.sh builds llama-server from the SHA-pinned v0.3.0 source tarball (single source of truth: on-device upgrade, Pi image, Live ISO, and the new docker/llm container all call it; version-marker makes re-runs no-ops). start_llama_server.sh refuses to start unless ai.yaml enables AI and the active model file exists — which also covers unmounted media. wrolpi-llm.service is installed but deliberately never enabled at boot: the Controller starts it on demand (registered in managed_services; the llm compose service follows the samba pattern), and the API's perpetual idle-unload worker asks the Controller to stop it after the configured idle minutes. llama-server is localhost-only natively, internal-network-only in docker, and never proxied by Caddy. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
119256e529
commit
318b835faf
18 changed files with 406 additions and 4 deletions
|
|
@ -78,6 +78,16 @@ DEFAULT_CONFIG = {
|
|||
"use_https": True,
|
||||
"description": "Kiwix/Zim server",
|
||||
},
|
||||
{
|
||||
"name": "wrolpi-llm",
|
||||
"systemd_name": "wrolpi-llm",
|
||||
"port": 11435,
|
||||
"viewable": False,
|
||||
"use_https": False,
|
||||
"description": "AI assistant (llama-server)",
|
||||
# On-demand: started by the Controller when AI is enabled, stopped by the API when idle.
|
||||
"kind": "persistent",
|
||||
},
|
||||
# System services
|
||||
{
|
||||
"name": "caddy",
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ except ImportError:
|
|||
CONTAINER_PREFIX = os.environ.get("COMPOSE_PROJECT_NAME", "wrolpi")
|
||||
|
||||
# Containers that should not have a "View" button (non-HTTP services)
|
||||
NON_VIEWABLE_CONTAINERS = {"redis", "memcached", "samba"}
|
||||
NON_VIEWABLE_CONTAINERS = {"redis", "memcached", "samba", "llm"}
|
||||
|
||||
# Containers that should use HTTPS
|
||||
HTTPS_CONTAINERS = {"web"}
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ class TestManagedServicesConfig:
|
|||
def test_managed_services_count(self):
|
||||
"""Should have expected number of managed services."""
|
||||
services = DEFAULT_CONFIG["managed_services"]
|
||||
assert len(services) == 11
|
||||
assert len(services) == 12
|
||||
|
||||
@pytest.mark.parametrize("name,expected", [
|
||||
("wrolpi-api",
|
||||
|
|
@ -68,6 +68,8 @@ class TestManagedServicesConfig:
|
|||
("wrolpi-app-dev", {"port": 3000, "viewable": False, "show_only_when_running": True}),
|
||||
("caddy", {"port": 443}),
|
||||
("wrolpi-upgrade", {"port": None, "viewable": False, "show_only_when_running": True}),
|
||||
("wrolpi-llm",
|
||||
{"systemd_name": "wrolpi-llm", "port": 11435, "viewable": False, "kind": "persistent"}),
|
||||
])
|
||||
def test_service_config(self, name, expected):
|
||||
"""Should have service configured with expected values."""
|
||||
|
|
|
|||
|
|
@ -143,6 +143,9 @@ cp /opt/wrolpi/wrolpi/blobs/* /opt/wrolpi-blobs/
|
|||
# systemd.
|
||||
/opt/wrolpi/wrolpi/scripts/install_map_tools.sh
|
||||
|
||||
# Bake in llama.cpp (local AI) so a fresh install does not compile it on first upgrade.
|
||||
/opt/wrolpi/wrolpi/scripts/install_llama_cpp.sh
|
||||
|
||||
# Install WROLPi Help.
|
||||
/opt/wrolpi/scripts/install_help_service.sh
|
||||
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ services:
|
|||
# Sanic 25.x wants 'spawn' but our code uses 'fork' for multiprocessing.
|
||||
# This tells Sanic to accept the existing start method.
|
||||
- SANIC_START_METHOD_SET=1
|
||||
# llama-server runs in the llm container (native installs use 127.0.0.1).
|
||||
- LLAMA_URL=http://llm:11435
|
||||
volumes:
|
||||
- './main.py:/opt/wrolpi/main.py'
|
||||
- './wrolpi:/opt/wrolpi/wrolpi'
|
||||
|
|
@ -147,6 +149,21 @@ services:
|
|||
test: [ 'CMD-SHELL', 'curl http://127.0.0.1:80' ]
|
||||
interval: 1m
|
||||
|
||||
llm: # llama-server (local AI) - started/stopped by the controller on demand; the API stops it when idle.
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/llm/Dockerfile
|
||||
restart: "no"
|
||||
depends_on:
|
||||
controller:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ${MEDIA_DIRECTORY:-./test}:/media/wrolpi
|
||||
- './docker/llm/entrypoint.sh:/entrypoint.sh'
|
||||
healthcheck:
|
||||
test: [ 'CMD-SHELL', 'curl http://127.0.0.1:11435/health' ]
|
||||
interval: 1m
|
||||
|
||||
samba: # Samba file sharing - started/stopped by the controller based on configured shares.
|
||||
build:
|
||||
context: .
|
||||
|
|
|
|||
17
docker/llm/Dockerfile
Normal file
17
docker/llm/Dockerfile
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
FROM debian:13
|
||||
ENV DOCKER=true
|
||||
|
||||
RUN apt-get update && apt-get install -y curl ca-certificates
|
||||
|
||||
# Build llama-server from the pinned release (single source of truth with native installs).
|
||||
COPY wrolpi/scripts/install_llama_cpp.sh /tmp/install_llama_cpp.sh
|
||||
RUN /tmp/install_llama_cpp.sh && rm /tmp/install_llama_cpp.sh \
|
||||
&& apt-get remove -y build-essential cmake && apt-get autoremove -y && apt-get clean
|
||||
|
||||
# The same startup scripts the native install uses.
|
||||
COPY scripts/start_llama_server.sh /opt/wrolpi/scripts/start_llama_server.sh
|
||||
COPY wrolpi/scripts/read_config_value.sh /opt/wrolpi/wrolpi/scripts/read_config_value.sh
|
||||
|
||||
COPY docker/llm/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
5
docker/llm/entrypoint.sh
Executable file
5
docker/llm/entrypoint.sh
Executable file
|
|
@ -0,0 +1,5 @@
|
|||
#! /usr/bin/env bash
|
||||
# Bind all interfaces so the api container can reach llama-server on the internal docker network.
|
||||
# The container publishes no host ports and is not proxied by Caddy.
|
||||
export LLAMA_HOST=0.0.0.0
|
||||
exec /opt/wrolpi/scripts/start_llama_server.sh
|
||||
17
etc/raspberrypios/wrolpi-llm.service
Normal file
17
etc/raspberrypios/wrolpi-llm.service
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[Unit]
|
||||
Description=WROLPi AI (llama-server)
|
||||
PartOf=wrolpi.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/opt/wrolpi/scripts/start_llama_server.sh
|
||||
Group=wrolpi
|
||||
User=wrolpi
|
||||
WorkingDirectory=/opt/wrolpi/
|
||||
Restart=on-failure
|
||||
# The lowest priority of any WROLPi service; inference must never starve the API.
|
||||
Nice=15
|
||||
|
||||
# On-demand service: deliberately NOT enabled at boot and NOT wanted by wrolpi.target.
|
||||
# The Controller starts it when the user enables AI; the API stops it when idle.
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
|
|
@ -1,2 +1,3 @@
|
|||
from modules.ai.api import ai_bp # noqa
|
||||
from modules.ai.downloader import ModelDownloader # noqa Instantiation registers the downloader.
|
||||
from modules.ai import service # noqa Import registers the idle-unload perpetual worker.
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import os
|
|||
from typing import Tuple
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from wrolpi.common import aiohttp_get, logger
|
||||
from wrolpi.common import aiohttp_get, aiohttp_post, logger
|
||||
from wrolpi.vars import DOCKERIZED
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -25,3 +25,13 @@ async def controller_get(path: str, params: dict = None, timeout: int = 10) -> T
|
|||
url = f'{url}?{urlencode(params)}'
|
||||
async with aiohttp_get(url, timeout=timeout) as response:
|
||||
return response.status, await response.json()
|
||||
|
||||
|
||||
async def controller_post(path: str, json_: dict = None, timeout: int = 30) -> Tuple[int, dict]:
|
||||
"""POST a Controller endpoint, returning (status, JSON body). Raises on connection errors."""
|
||||
async with aiohttp_post(f'{CONTROLLER_URL}{path}', json_=json_ or {}, timeout=timeout) as response:
|
||||
try:
|
||||
body = await response.json()
|
||||
except Exception:
|
||||
body = {}
|
||||
return response.status, body
|
||||
|
|
|
|||
112
modules/ai/service.py
Normal file
112
modules/ai/service.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
"""llama-server lifecycle: on-demand start and idle unload.
|
||||
|
||||
The API owns the policy (this module); the Controller executes the mechanism through its
|
||||
existing service start/stop endpoints — it runs as root natively and manages the `llm`
|
||||
container in docker, so no sudoers changes are needed.
|
||||
|
||||
Idle unload is mandatory on small devices: a swapped-to-death Pi is worse than a slow first
|
||||
token. The chat handler records activity; the perpetual worker stops llama-server after the
|
||||
configured idle minutes."""
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
|
||||
from modules.ai.config import get_ai_config
|
||||
from modules.ai.controller_client import controller_post
|
||||
from wrolpi.api_utils import api_app, perpetual_signal
|
||||
from wrolpi.common import aiohttp_get, logger
|
||||
from wrolpi.vars import DOCKERIZED
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
LLAMA_URL = os.environ.get('LLAMA_URL', 'http://127.0.0.1:11435')
|
||||
# The Controller's name for the service: the systemd unit natively, the compose service in docker.
|
||||
LLM_SERVICE_NAME = 'llm' if DOCKERIZED else 'wrolpi-llm'
|
||||
|
||||
# How long to wait for llama-server to answer /health after a cold start (model load included).
|
||||
STARTUP_TIMEOUT_SECONDS = 90
|
||||
|
||||
|
||||
def record_llm_activity():
|
||||
"""Note that the LLM was just used; postpones the idle unload."""
|
||||
try:
|
||||
api_app.shared_ctx.ai_llm_last_request.value = time.time()
|
||||
except AttributeError:
|
||||
# Shared contexts are absent in some unit tests.
|
||||
pass
|
||||
|
||||
|
||||
def get_last_llm_activity() -> float:
|
||||
try:
|
||||
return api_app.shared_ctx.ai_llm_last_request.value
|
||||
except AttributeError:
|
||||
return 0.0
|
||||
|
||||
|
||||
async def llama_healthy() -> bool:
|
||||
try:
|
||||
async with aiohttp_get(f'{LLAMA_URL}/health', timeout=5) as response:
|
||||
return response.status == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def start_llm_service() -> bool:
|
||||
status, body = await controller_post(f'/api/services/{LLM_SERVICE_NAME}/start')
|
||||
if status != 200:
|
||||
logger.error(f'Controller failed to start {LLM_SERVICE_NAME}: HTTP {status} {body}')
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def stop_llm_service() -> bool:
|
||||
status, body = await controller_post(f'/api/services/{LLM_SERVICE_NAME}/stop')
|
||||
if status != 200:
|
||||
logger.error(f'Controller failed to stop {LLM_SERVICE_NAME}: HTTP {status} {body}')
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def ensure_llama_running(timeout: int = STARTUP_TIMEOUT_SECONDS) -> bool:
|
||||
"""Start llama-server on demand and wait for it to answer /health.
|
||||
|
||||
Returns True when llama-server is up. Cold-start latency (service start + model load) is
|
||||
expected and acceptable."""
|
||||
if await llama_healthy():
|
||||
record_llm_activity()
|
||||
return True
|
||||
|
||||
logger.info(f'Asking the Controller to start {LLM_SERVICE_NAME}')
|
||||
if not await start_llm_service():
|
||||
return False
|
||||
|
||||
started = time.time()
|
||||
while time.time() - started < timeout:
|
||||
if await llama_healthy():
|
||||
record_llm_activity()
|
||||
return True
|
||||
await asyncio.sleep(1)
|
||||
logger.error(f'llama-server did not become healthy within {timeout}s')
|
||||
return False
|
||||
|
||||
|
||||
@perpetual_signal(sleep=60)
|
||||
async def perpetual_llm_idle_unload():
|
||||
"""Stop llama-server after the configured idle minutes. Only acts after it was used."""
|
||||
try:
|
||||
last_activity = get_last_llm_activity()
|
||||
if not last_activity:
|
||||
return
|
||||
config = get_ai_config()
|
||||
idle_seconds = config.idle_unload_minutes * 60
|
||||
if time.time() - last_activity < idle_seconds:
|
||||
return
|
||||
if not await llama_healthy():
|
||||
# Already stopped (or crashed); nothing to unload.
|
||||
api_app.shared_ctx.ai_llm_last_request.value = 0.0
|
||||
return
|
||||
logger.info(f'Stopping {LLM_SERVICE_NAME} after {config.idle_unload_minutes} idle minutes')
|
||||
if await stop_llm_service():
|
||||
api_app.shared_ctx.ai_llm_last_request.value = 0.0
|
||||
except Exception as e:
|
||||
logger.error('LLM idle unload worker failed', exc_info=e)
|
||||
78
modules/ai/test/test_service.py
Normal file
78
modules/ai/test/test_service.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
"""Tests for the llama-server lifecycle: on-demand start and idle unload."""
|
||||
import time
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from modules.ai import service
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_llama_running_already_healthy(async_client):
|
||||
"""A healthy llama-server is used as-is and activity is recorded."""
|
||||
with mock.patch('modules.ai.service.llama_healthy', return_value=True), \
|
||||
mock.patch('modules.ai.service.start_llm_service') as start:
|
||||
assert await service.ensure_llama_running() is True
|
||||
start.assert_not_called()
|
||||
assert service.get_last_llm_activity() > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_llama_running_cold_start(async_client):
|
||||
"""An unhealthy llama-server is started via the Controller and polled until healthy."""
|
||||
health = iter([False, False, True])
|
||||
|
||||
async def fake_healthy():
|
||||
return next(health)
|
||||
|
||||
with mock.patch('modules.ai.service.llama_healthy', side_effect=fake_healthy), \
|
||||
mock.patch('modules.ai.service.start_llm_service', return_value=True) as start, \
|
||||
mock.patch('modules.ai.service.asyncio.sleep'):
|
||||
assert await service.ensure_llama_running(timeout=10) is True
|
||||
start.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_llama_running_start_fails(async_client):
|
||||
"""A Controller start failure is reported as False, not an exception."""
|
||||
with mock.patch('modules.ai.service.llama_healthy', return_value=False), \
|
||||
mock.patch('modules.ai.service.start_llm_service', return_value=False):
|
||||
assert await service.ensure_llama_running(timeout=1) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idle_unload(async_client, test_ai_config):
|
||||
"""llama-server is stopped only after the configured idle minutes."""
|
||||
from wrolpi.api_utils import api_app
|
||||
from modules.ai.config import get_ai_config
|
||||
|
||||
get_ai_config().idle_unload_minutes = 10
|
||||
|
||||
# Never used: nothing happens.
|
||||
api_app.shared_ctx.ai_llm_last_request.value = 0.0
|
||||
with mock.patch('modules.ai.service.stop_llm_service') as stop:
|
||||
await service.perpetual_llm_idle_unload()
|
||||
stop.assert_not_called()
|
||||
|
||||
# Recently used: not stopped.
|
||||
api_app.shared_ctx.ai_llm_last_request.value = time.time()
|
||||
with mock.patch('modules.ai.service.stop_llm_service') as stop, \
|
||||
mock.patch('modules.ai.service.llama_healthy', return_value=True):
|
||||
await service.perpetual_llm_idle_unload()
|
||||
stop.assert_not_called()
|
||||
|
||||
# Idle past the threshold: stopped, and the activity marker resets.
|
||||
api_app.shared_ctx.ai_llm_last_request.value = time.time() - 11 * 60
|
||||
with mock.patch('modules.ai.service.stop_llm_service', return_value=True) as stop, \
|
||||
mock.patch('modules.ai.service.llama_healthy', return_value=True):
|
||||
await service.perpetual_llm_idle_unload()
|
||||
stop.assert_called_once()
|
||||
assert api_app.shared_ctx.ai_llm_last_request.value == 0.0
|
||||
|
||||
# Idle but already stopped: the marker resets without a stop call.
|
||||
api_app.shared_ctx.ai_llm_last_request.value = time.time() - 11 * 60
|
||||
with mock.patch('modules.ai.service.stop_llm_service') as stop, \
|
||||
mock.patch('modules.ai.service.llama_healthy', return_value=False):
|
||||
await service.perpetual_llm_idle_unload()
|
||||
stop.assert_not_called()
|
||||
assert api_app.shared_ctx.ai_llm_last_request.value == 0.0
|
||||
|
|
@ -45,6 +45,9 @@ git config --global --add safe.directory /opt/wrolpi
|
|||
# CPU-heavy step on weak hardware.
|
||||
/opt/wrolpi/wrolpi/scripts/install_map_tools.sh
|
||||
|
||||
# Bake in llama.cpp (local AI) so a fresh Pi does not compile it on first upgrade.
|
||||
/opt/wrolpi/wrolpi/scripts/install_llama_cpp.sh
|
||||
|
||||
# Install Python requirements. Try multiple times because pypi may stop responding.
|
||||
python3 -m venv /opt/wrolpi/venv
|
||||
/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt ||
|
||||
|
|
|
|||
54
scripts/start_llama_server.sh
Executable file
54
scripts/start_llama_server.sh
Executable file
|
|
@ -0,0 +1,54 @@
|
|||
#! /usr/bin/env bash
|
||||
# Start llama-server (WROLPi's local AI inference runtime) with the model selected in ai.yaml.
|
||||
#
|
||||
# AI is opt-in: this refuses to start unless ai.yaml enables it and the active model file exists
|
||||
# in the media directory (which also covers the unmounted-media case). Started/stopped on demand
|
||||
# by the Controller; the API's idle-unload worker stops it after inactivity.
|
||||
set -u
|
||||
|
||||
MEDIA_DIRECTORY="${MEDIA_DIRECTORY:-/media/wrolpi}"
|
||||
PROJECT_DIR="${PROJECT_DIR:-/opt/wrolpi}"
|
||||
AI_CONFIG="${MEDIA_DIRECTORY}/config/ai.yaml"
|
||||
# Docker overrides the host so the api container can reach this container.
|
||||
LLAMA_HOST="${LLAMA_HOST:-127.0.0.1}"
|
||||
LLAMA_PORT="${LLAMA_PORT:-11435}"
|
||||
|
||||
read_ai_config() {
|
||||
MEDIA_DIRECTORY="${MEDIA_DIRECTORY}" WROLPI_CONFIG_FILE="${AI_CONFIG}" \
|
||||
"${PROJECT_DIR}/wrolpi/scripts/read_config_value.sh" "$@"
|
||||
}
|
||||
|
||||
[ ! -d "${MEDIA_DIRECTORY}" ] && echo "Cannot start llama-server without media directory mounted" && exit 1
|
||||
[ ! -f "${AI_CONFIG}" ] && echo "Cannot start llama-server without ${AI_CONFIG}" && exit 1
|
||||
|
||||
ENABLED="$(read_ai_config enabled false)"
|
||||
if [ "${ENABLED}" != "true" ] && [ "${ENABLED}" != "True" ]; then
|
||||
echo "AI is not enabled in ${AI_CONFIG}; refusing to start"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ACTIVE_MODEL="$(read_ai_config active_model)"
|
||||
[ -z "${ACTIVE_MODEL}" ] && echo "No active_model in ${AI_CONFIG}; refusing to start" && exit 1
|
||||
# Defense in depth: the model must be a bare file name inside the models directory.
|
||||
case "${ACTIVE_MODEL}" in
|
||||
*/*|*..*)
|
||||
echo "Ignoring unsafe active_model '${ACTIVE_MODEL}'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
MODEL="${MEDIA_DIRECTORY}/ai/models/${ACTIVE_MODEL}"
|
||||
[ ! -f "${MODEL}" ] && echo "Model file does not exist: ${MODEL}" && exit 1
|
||||
|
||||
CONTEXT_SIZE="$(read_ai_config context_size 8192)"
|
||||
case "${CONTEXT_SIZE}" in
|
||||
''|*[!0-9]*) CONTEXT_SIZE=8192 ;;
|
||||
esac
|
||||
|
||||
# --jinja enables tool-call chat templates; mmap is llama.cpp's default so cold RAM cost is low.
|
||||
# Localhost only on native installs -- never exposed through Caddy.
|
||||
exec llama-server \
|
||||
--host "${LLAMA_HOST}" \
|
||||
--port "${LLAMA_PORT}" \
|
||||
--jinja \
|
||||
-c "${CONTEXT_SIZE}" \
|
||||
-m "${MODEL}"
|
||||
|
|
@ -258,6 +258,10 @@ rm -f /home/wrolpi/.pgpass
|
|||
# Migrate from old Leaflet/renderd/Apache2 map stack to MapLibre GL JS + PMTiles.
|
||||
/opt/wrolpi/scripts/install_protomaps.sh || echo "Protomaps migration failed."
|
||||
|
||||
# Install llama.cpp for the local AI assistant (version-guarded no-op once installed;
|
||||
# the first build takes several minutes on a Raspberry Pi).
|
||||
/opt/wrolpi/wrolpi/scripts/install_llama_cpp.sh || echo "llama.cpp install failed, continuing..."
|
||||
|
||||
# Download map fonts blob if missing.
|
||||
MAP_FONTS_BLOB=/opt/wrolpi-blobs/map-fonts.tar.gz
|
||||
if [[ ! -f ${MAP_FONTS_BLOB} || ! -s ${MAP_FONTS_BLOB} ]]; then
|
||||
|
|
|
|||
|
|
@ -59,6 +59,9 @@ def attach_shared_contexts(app: Sanic):
|
|||
app.shared_ctx.download_manager_disabled.set()
|
||||
app.shared_ctx.download_manager_stopped.set()
|
||||
|
||||
# Monotonic-ish timestamp of the last chat/LLM request; the AI idle-unload worker reads it.
|
||||
app.shared_ctx.ai_llm_last_request = multiprocessing.Value(ctypes.c_double, 0.0)
|
||||
|
||||
app.shared_ctx.single_tasks_started = multiprocessing.Event()
|
||||
app.shared_ctx.flags_initialized = multiprocessing.Event()
|
||||
app.shared_ctx.perpetual_tasks_started = multiprocessing.Event()
|
||||
|
|
|
|||
65
wrolpi/scripts/install_llama_cpp.sh
Executable file
65
wrolpi/scripts/install_llama_cpp.sh
Executable file
|
|
@ -0,0 +1,65 @@
|
|||
#!/usr/bin/env bash
|
||||
# Install llama.cpp's llama-server, WROLPi's local AI inference runtime.
|
||||
#
|
||||
# This is the single source of truth, called by every place that provisions a
|
||||
# WROLPi so a version bump only has to be made here:
|
||||
# - scripts/upgrade.sh (on-device upgrade)
|
||||
# - pi-gen/stage2/04-wrolpi/03-run-chroot.sh (Raspberry Pi image build)
|
||||
# - debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot (Debian Live ISO build)
|
||||
# - docker/llm/Dockerfile (Docker llm container)
|
||||
#
|
||||
# Built from the SHA-pinned source tarball (llama.cpp ships no generic Linux
|
||||
# binaries for ARM). The build takes several minutes on a Raspberry Pi; the
|
||||
# version marker makes re-runs no-ops until the pinned version changes.
|
||||
# Run as root: installs into /usr/local/bin and apt-installs build dependencies.
|
||||
set -e
|
||||
|
||||
LLAMA_CPP_VERSION="v0.3.0"
|
||||
# SHA256 of the source tarball -- arch-independent, one value covers every build.
|
||||
LLAMA_CPP_SHA256="d94c02d86db22d68692f6bb5b3854763d5091e52142868dc7251995517c666d1"
|
||||
|
||||
# llama-server's version output tracks build numbers, not release tags, so the
|
||||
# installed release is recorded in a marker file.
|
||||
MARKER_DIR=/usr/local/share/wrolpi
|
||||
MARKER="${MARKER_DIR}/llama_cpp_version"
|
||||
|
||||
install_llama_cpp() {
|
||||
if [ -f "${MARKER}" ] && [ "$(cat "${MARKER}")" = "${LLAMA_CPP_VERSION}" ] \
|
||||
&& command -v llama-server &>/dev/null; then
|
||||
echo "llama.cpp ${LLAMA_CPP_VERSION} already installed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Building llama.cpp ${LLAMA_CPP_VERSION}..."
|
||||
apt-get install -y build-essential cmake
|
||||
|
||||
local build_dir
|
||||
build_dir=$(mktemp -d)
|
||||
curl -fsSL "https://github.com/ggml-org/llama.cpp/archive/refs/tags/${LLAMA_CPP_VERSION}.tar.gz" \
|
||||
-o /tmp/llama_cpp.tar.gz
|
||||
echo "${LLAMA_CPP_SHA256} /tmp/llama_cpp.tar.gz" | sha256sum -c -
|
||||
tar -xz -C "${build_dir}" --strip-components=1 -f /tmp/llama_cpp.tar.gz
|
||||
rm -f /tmp/llama_cpp.tar.gz
|
||||
|
||||
# Static-ish single binary; GGML_NATIVE=OFF so an image built in a chroot/QEMU
|
||||
# does not bake in the build host's CPU features.
|
||||
(cd "${build_dir}" &&
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DBUILD_SHARED_LIBS=OFF \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DLLAMA_CURL=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_SERVER=ON &&
|
||||
cmake --build build --target llama-server -j"$(nproc)")
|
||||
install -m 0755 "${build_dir}/build/bin/llama-server" /usr/local/bin/llama-server
|
||||
rm -rf "${build_dir}"
|
||||
|
||||
mkdir -p "${MARKER_DIR}"
|
||||
echo "${LLAMA_CPP_VERSION}" > "${MARKER}"
|
||||
llama-server --version || true
|
||||
echo "llama.cpp ${LLAMA_CPP_VERSION} installed"
|
||||
}
|
||||
|
||||
install_llama_cpp
|
||||
|
|
@ -20,7 +20,8 @@ KEY="${1:?Usage: read_config_value.sh KEY [DEFAULT]}"
|
|||
DEFAULT="${2:-}"
|
||||
|
||||
MEDIA_DIRECTORY="${MEDIA_DIRECTORY:-/media/wrolpi}"
|
||||
CONFIG="${MEDIA_DIRECTORY}/config/wrolpi.yaml"
|
||||
# Another top-level config file (e.g. ai.yaml) may be selected with WROLPI_CONFIG_FILE.
|
||||
CONFIG="${WROLPI_CONFIG_FILE:-${MEDIA_DIRECTORY}/config/wrolpi.yaml}"
|
||||
|
||||
value=""
|
||||
if [ -f "${CONFIG}" ]; then
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue