ai: package llama-server for all three deployments, with idle unload

install_llama_cpp.sh builds llama-server from the SHA-pinned v0.3.0
source tarball (single source of truth: on-device upgrade, Pi image,
Live ISO, and the new docker/llm container all call it; version-marker
makes re-runs no-ops).  start_llama_server.sh refuses to start unless
ai.yaml enables AI and the active model file exists — which also covers
unmounted media.  wrolpi-llm.service is installed but deliberately
never enabled at boot: the Controller starts it on demand (registered
in managed_services; the llm compose service follows the samba
pattern), and the API's perpetual idle-unload worker asks the
Controller to stop it after the configured idle minutes.  llama-server
is localhost-only natively, internal-network-only in docker, and never
proxied by Caddy.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-08-25 21:36:00 -06:00
parent 256750e538
commit d9924b6b53
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
18 changed files with 406 additions and 4 deletions

View file

@ -1,2 +1,3 @@
from modules.ai.api import ai_bp # noqa
from modules.ai.downloader import ModelDownloader # noqa Instantiation registers the downloader.
from modules.ai import service # noqa Import registers the idle-unload perpetual worker.

View file

@ -8,7 +8,7 @@ import os
from typing import Tuple
from urllib.parse import urlencode
from wrolpi.common import aiohttp_get, logger
from wrolpi.common import aiohttp_get, aiohttp_post, logger
from wrolpi.vars import DOCKERIZED
logger = logger.getChild(__name__)
@ -25,3 +25,13 @@ async def controller_get(path: str, params: dict = None, timeout: int = 10) -> T
url = f'{url}?{urlencode(params)}'
async with aiohttp_get(url, timeout=timeout) as response:
return response.status, await response.json()
async def controller_post(path: str, json_: dict = None, timeout: int = 30) -> Tuple[int, dict]:
"""POST a Controller endpoint, returning (status, JSON body). Raises on connection errors."""
async with aiohttp_post(f'{CONTROLLER_URL}{path}', json_=json_ or {}, timeout=timeout) as response:
try:
body = await response.json()
except Exception:
body = {}
return response.status, body

112
modules/ai/service.py Normal file
View file

@ -0,0 +1,112 @@
"""llama-server lifecycle: on-demand start and idle unload.
The API owns the policy (this module); the Controller executes the mechanism through its
existing service start/stop endpoints — it runs as root natively and manages the `llm`
container in docker, so no sudoers changes are needed.
Idle unload is mandatory on small devices: a swapped-to-death Pi is worse than a slow first
token. The chat handler records activity; the perpetual worker stops llama-server after the
configured idle minutes."""
import asyncio
import os
import time
from modules.ai.config import get_ai_config
from modules.ai.controller_client import controller_post
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.common import aiohttp_get, logger
from wrolpi.vars import DOCKERIZED
logger = logger.getChild(__name__)
LLAMA_URL = os.environ.get('LLAMA_URL', 'http://127.0.0.1:11435')
# The Controller's name for the service: the systemd unit natively, the compose service in docker.
LLM_SERVICE_NAME = 'llm' if DOCKERIZED else 'wrolpi-llm'
# How long to wait for llama-server to answer /health after a cold start (model load included).
STARTUP_TIMEOUT_SECONDS = 90
def record_llm_activity():
"""Note that the LLM was just used; postpones the idle unload."""
try:
api_app.shared_ctx.ai_llm_last_request.value = time.time()
except AttributeError:
# Shared contexts are absent in some unit tests.
pass
def get_last_llm_activity() -> float:
try:
return api_app.shared_ctx.ai_llm_last_request.value
except AttributeError:
return 0.0
async def llama_healthy() -> bool:
try:
async with aiohttp_get(f'{LLAMA_URL}/health', timeout=5) as response:
return response.status == 200
except Exception:
return False
async def start_llm_service() -> bool:
status, body = await controller_post(f'/api/services/{LLM_SERVICE_NAME}/start')
if status != 200:
logger.error(f'Controller failed to start {LLM_SERVICE_NAME}: HTTP {status} {body}')
return False
return True
async def stop_llm_service() -> bool:
status, body = await controller_post(f'/api/services/{LLM_SERVICE_NAME}/stop')
if status != 200:
logger.error(f'Controller failed to stop {LLM_SERVICE_NAME}: HTTP {status} {body}')
return False
return True
async def ensure_llama_running(timeout: int = STARTUP_TIMEOUT_SECONDS) -> bool:
"""Start llama-server on demand and wait for it to answer /health.
Returns True when llama-server is up. Cold-start latency (service start + model load) is
expected and acceptable."""
if await llama_healthy():
record_llm_activity()
return True
logger.info(f'Asking the Controller to start {LLM_SERVICE_NAME}')
if not await start_llm_service():
return False
started = time.time()
while time.time() - started < timeout:
if await llama_healthy():
record_llm_activity()
return True
await asyncio.sleep(1)
logger.error(f'llama-server did not become healthy within {timeout}s')
return False
@perpetual_signal(sleep=60)
async def perpetual_llm_idle_unload():
"""Stop llama-server after the configured idle minutes. Only acts after it was used."""
try:
last_activity = get_last_llm_activity()
if not last_activity:
return
config = get_ai_config()
idle_seconds = config.idle_unload_minutes * 60
if time.time() - last_activity < idle_seconds:
return
if not await llama_healthy():
# Already stopped (or crashed); nothing to unload.
api_app.shared_ctx.ai_llm_last_request.value = 0.0
return
logger.info(f'Stopping {LLM_SERVICE_NAME} after {config.idle_unload_minutes} idle minutes')
if await stop_llm_service():
api_app.shared_ctx.ai_llm_last_request.value = 0.0
except Exception as e:
logger.error('LLM idle unload worker failed', exc_info=e)

View file

@ -0,0 +1,78 @@
"""Tests for the llama-server lifecycle: on-demand start and idle unload."""
import time
from unittest import mock
import pytest
from modules.ai import service
@pytest.mark.asyncio
async def test_ensure_llama_running_already_healthy(async_client):
"""A healthy llama-server is used as-is and activity is recorded."""
with mock.patch('modules.ai.service.llama_healthy', return_value=True), \
mock.patch('modules.ai.service.start_llm_service') as start:
assert await service.ensure_llama_running() is True
start.assert_not_called()
assert service.get_last_llm_activity() > 0
@pytest.mark.asyncio
async def test_ensure_llama_running_cold_start(async_client):
"""An unhealthy llama-server is started via the Controller and polled until healthy."""
health = iter([False, False, True])
async def fake_healthy():
return next(health)
with mock.patch('modules.ai.service.llama_healthy', side_effect=fake_healthy), \
mock.patch('modules.ai.service.start_llm_service', return_value=True) as start, \
mock.patch('modules.ai.service.asyncio.sleep'):
assert await service.ensure_llama_running(timeout=10) is True
start.assert_called_once()
@pytest.mark.asyncio
async def test_ensure_llama_running_start_fails(async_client):
"""A Controller start failure is reported as False, not an exception."""
with mock.patch('modules.ai.service.llama_healthy', return_value=False), \
mock.patch('modules.ai.service.start_llm_service', return_value=False):
assert await service.ensure_llama_running(timeout=1) is False
@pytest.mark.asyncio
async def test_idle_unload(async_client, test_ai_config):
"""llama-server is stopped only after the configured idle minutes."""
from wrolpi.api_utils import api_app
from modules.ai.config import get_ai_config
get_ai_config().idle_unload_minutes = 10
# Never used: nothing happens.
api_app.shared_ctx.ai_llm_last_request.value = 0.0
with mock.patch('modules.ai.service.stop_llm_service') as stop:
await service.perpetual_llm_idle_unload()
stop.assert_not_called()
# Recently used: not stopped.
api_app.shared_ctx.ai_llm_last_request.value = time.time()
with mock.patch('modules.ai.service.stop_llm_service') as stop, \
mock.patch('modules.ai.service.llama_healthy', return_value=True):
await service.perpetual_llm_idle_unload()
stop.assert_not_called()
# Idle past the threshold: stopped, and the activity marker resets.
api_app.shared_ctx.ai_llm_last_request.value = time.time() - 11 * 60
with mock.patch('modules.ai.service.stop_llm_service', return_value=True) as stop, \
mock.patch('modules.ai.service.llama_healthy', return_value=True):
await service.perpetual_llm_idle_unload()
stop.assert_called_once()
assert api_app.shared_ctx.ai_llm_last_request.value == 0.0
# Idle but already stopped: the marker resets without a stop call.
api_app.shared_ctx.ai_llm_last_request.value = time.time() - 11 * 60
with mock.patch('modules.ai.service.stop_llm_service') as stop, \
mock.patch('modules.ai.service.llama_healthy', return_value=False):
await service.perpetual_llm_idle_unload()
stop.assert_not_called()
assert api_app.shared_ctx.ai_llm_last_request.value == 0.0