mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 10:51:36 -04:00
485a9703eb
install_llama_cpp.sh builds llama-server from the SHA-pinned v0.3.0 source tarball (single source of truth: on-device upgrade, Pi image, Live ISO, and the new docker/llm container all call it; version-marker makes re-runs no-ops). start_llama_server.sh refuses to start unless ai.yaml enables AI and the active model file exists — which also covers unmounted media. wrolpi-llm.service is installed but deliberately never enabled at boot: the Controller starts it on demand (registered in managed_services; the llm compose service follows the samba pattern), and the API's perpetual idle-unload worker asks the Controller to stop it after the configured idle minutes. llama-server is localhost-only natively, internal-network-only in docker, and never proxied by Caddy. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
112 lines
3.9 KiB
Python
112 lines
3.9 KiB
Python
"""llama-server lifecycle: on-demand start and idle unload.
|
|
|
|
The API owns the policy (this module); the Controller executes the mechanism through its
|
|
existing service start/stop endpoints — it runs as root natively and manages the `llm`
|
|
container in docker, so no sudoers changes are needed.
|
|
|
|
Idle unload is mandatory on small devices: a swapped-to-death Pi is worse than a slow first
|
|
token. The chat handler records activity; the perpetual worker stops llama-server after the
|
|
configured idle minutes."""
|
|
import asyncio
|
|
import os
|
|
import time
|
|
|
|
from modules.ai.config import get_ai_config
|
|
from modules.ai.controller_client import controller_post
|
|
from wrolpi.api_utils import api_app, perpetual_signal
|
|
from wrolpi.common import aiohttp_get, logger
|
|
from wrolpi.vars import DOCKERIZED
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
LLAMA_URL = os.environ.get('LLAMA_URL', 'http://127.0.0.1:11435')
|
|
# The Controller's name for the service: the systemd unit natively, the compose service in docker.
|
|
LLM_SERVICE_NAME = 'llm' if DOCKERIZED else 'wrolpi-llm'
|
|
|
|
# How long to wait for llama-server to answer /health after a cold start (model load included).
|
|
STARTUP_TIMEOUT_SECONDS = 90
|
|
|
|
|
|
def record_llm_activity():
|
|
"""Note that the LLM was just used; postpones the idle unload."""
|
|
try:
|
|
api_app.shared_ctx.ai_llm_last_request.value = time.time()
|
|
except AttributeError:
|
|
# Shared contexts are absent in some unit tests.
|
|
pass
|
|
|
|
|
|
def get_last_llm_activity() -> float:
|
|
try:
|
|
return api_app.shared_ctx.ai_llm_last_request.value
|
|
except AttributeError:
|
|
return 0.0
|
|
|
|
|
|
async def llama_healthy() -> bool:
|
|
try:
|
|
async with aiohttp_get(f'{LLAMA_URL}/health', timeout=5) as response:
|
|
return response.status == 200
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
async def start_llm_service() -> bool:
|
|
status, body = await controller_post(f'/api/services/{LLM_SERVICE_NAME}/start')
|
|
if status != 200:
|
|
logger.error(f'Controller failed to start {LLM_SERVICE_NAME}: HTTP {status} {body}')
|
|
return False
|
|
return True
|
|
|
|
|
|
async def stop_llm_service() -> bool:
|
|
status, body = await controller_post(f'/api/services/{LLM_SERVICE_NAME}/stop')
|
|
if status != 200:
|
|
logger.error(f'Controller failed to stop {LLM_SERVICE_NAME}: HTTP {status} {body}')
|
|
return False
|
|
return True
|
|
|
|
|
|
async def ensure_llama_running(timeout: int = STARTUP_TIMEOUT_SECONDS) -> bool:
|
|
"""Start llama-server on demand and wait for it to answer /health.
|
|
|
|
Returns True when llama-server is up. Cold-start latency (service start + model load) is
|
|
expected and acceptable."""
|
|
if await llama_healthy():
|
|
record_llm_activity()
|
|
return True
|
|
|
|
logger.info(f'Asking the Controller to start {LLM_SERVICE_NAME}')
|
|
if not await start_llm_service():
|
|
return False
|
|
|
|
started = time.time()
|
|
while time.time() - started < timeout:
|
|
if await llama_healthy():
|
|
record_llm_activity()
|
|
return True
|
|
await asyncio.sleep(1)
|
|
logger.error(f'llama-server did not become healthy within {timeout}s')
|
|
return False
|
|
|
|
|
|
@perpetual_signal(sleep=60)
|
|
async def perpetual_llm_idle_unload():
|
|
"""Stop llama-server after the configured idle minutes. Only acts after it was used."""
|
|
try:
|
|
last_activity = get_last_llm_activity()
|
|
if not last_activity:
|
|
return
|
|
config = get_ai_config()
|
|
idle_seconds = config.idle_unload_minutes * 60
|
|
if time.time() - last_activity < idle_seconds:
|
|
return
|
|
if not await llama_healthy():
|
|
# Already stopped (or crashed); nothing to unload.
|
|
api_app.shared_ctx.ai_llm_last_request.value = 0.0
|
|
return
|
|
logger.info(f'Stopping {LLM_SERVICE_NAME} after {config.idle_unload_minutes} idle minutes')
|
|
if await stop_llm_service():
|
|
api_app.shared_ctx.ai_llm_last_request.value = 0.0
|
|
except Exception as e:
|
|
logger.error('LLM idle unload worker failed', exc_info=e)
|