mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
ai: package llama-server for all three deployments, with idle unload
install_llama_cpp.sh builds llama-server from the SHA-pinned v0.3.0 source tarball (single source of truth: on-device upgrade, Pi image, Live ISO, and the new docker/llm container all call it; version-marker makes re-runs no-ops). start_llama_server.sh refuses to start unless ai.yaml enables AI and the active model file exists — which also covers unmounted media. wrolpi-llm.service is installed but deliberately never enabled at boot: the Controller starts it on demand (registered in managed_services; the llm compose service follows the samba pattern), and the API's perpetual idle-unload worker asks the Controller to stop it after the configured idle minutes. llama-server is localhost-only natively, internal-network-only in docker, and never proxied by Caddy. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
256750e538
commit
d9924b6b53
18 changed files with 406 additions and 4 deletions
|
|
@ -1,2 +1,3 @@
|
|||
from modules.ai.api import ai_bp # noqa
|
||||
from modules.ai.downloader import ModelDownloader # noqa Instantiation registers the downloader.
|
||||
from modules.ai import service # noqa Import registers the idle-unload perpetual worker.
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import os
|
|||
from typing import Tuple
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from wrolpi.common import aiohttp_get, logger
|
||||
from wrolpi.common import aiohttp_get, aiohttp_post, logger
|
||||
from wrolpi.vars import DOCKERIZED
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -25,3 +25,13 @@ async def controller_get(path: str, params: dict = None, timeout: int = 10) -> T
|
|||
url = f'{url}?{urlencode(params)}'
|
||||
async with aiohttp_get(url, timeout=timeout) as response:
|
||||
return response.status, await response.json()
|
||||
|
||||
|
||||
async def controller_post(path: str, json_: dict = None, timeout: int = 30) -> Tuple[int, dict]:
|
||||
"""POST a Controller endpoint, returning (status, JSON body). Raises on connection errors."""
|
||||
async with aiohttp_post(f'{CONTROLLER_URL}{path}', json_=json_ or {}, timeout=timeout) as response:
|
||||
try:
|
||||
body = await response.json()
|
||||
except Exception:
|
||||
body = {}
|
||||
return response.status, body
|
||||
|
|
|
|||
112
modules/ai/service.py
Normal file
112
modules/ai/service.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
"""llama-server lifecycle: on-demand start and idle unload.
|
||||
|
||||
The API owns the policy (this module); the Controller executes the mechanism through its
|
||||
existing service start/stop endpoints — it runs as root natively and manages the `llm`
|
||||
container in docker, so no sudoers changes are needed.
|
||||
|
||||
Idle unload is mandatory on small devices: a swapped-to-death Pi is worse than a slow first
|
||||
token. The chat handler records activity; the perpetual worker stops llama-server after the
|
||||
configured idle minutes."""
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
|
||||
from modules.ai.config import get_ai_config
|
||||
from modules.ai.controller_client import controller_post
|
||||
from wrolpi.api_utils import api_app, perpetual_signal
|
||||
from wrolpi.common import aiohttp_get, logger
|
||||
from wrolpi.vars import DOCKERIZED
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
LLAMA_URL = os.environ.get('LLAMA_URL', 'http://127.0.0.1:11435')
|
||||
# The Controller's name for the service: the systemd unit natively, the compose service in docker.
|
||||
LLM_SERVICE_NAME = 'llm' if DOCKERIZED else 'wrolpi-llm'
|
||||
|
||||
# How long to wait for llama-server to answer /health after a cold start (model load included).
|
||||
STARTUP_TIMEOUT_SECONDS = 90
|
||||
|
||||
|
||||
def record_llm_activity():
|
||||
"""Note that the LLM was just used; postpones the idle unload."""
|
||||
try:
|
||||
api_app.shared_ctx.ai_llm_last_request.value = time.time()
|
||||
except AttributeError:
|
||||
# Shared contexts are absent in some unit tests.
|
||||
pass
|
||||
|
||||
|
||||
def get_last_llm_activity() -> float:
|
||||
try:
|
||||
return api_app.shared_ctx.ai_llm_last_request.value
|
||||
except AttributeError:
|
||||
return 0.0
|
||||
|
||||
|
||||
async def llama_healthy() -> bool:
|
||||
try:
|
||||
async with aiohttp_get(f'{LLAMA_URL}/health', timeout=5) as response:
|
||||
return response.status == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def start_llm_service() -> bool:
|
||||
status, body = await controller_post(f'/api/services/{LLM_SERVICE_NAME}/start')
|
||||
if status != 200:
|
||||
logger.error(f'Controller failed to start {LLM_SERVICE_NAME}: HTTP {status} {body}')
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def stop_llm_service() -> bool:
|
||||
status, body = await controller_post(f'/api/services/{LLM_SERVICE_NAME}/stop')
|
||||
if status != 200:
|
||||
logger.error(f'Controller failed to stop {LLM_SERVICE_NAME}: HTTP {status} {body}')
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def ensure_llama_running(timeout: int = STARTUP_TIMEOUT_SECONDS) -> bool:
|
||||
"""Start llama-server on demand and wait for it to answer /health.
|
||||
|
||||
Returns True when llama-server is up. Cold-start latency (service start + model load) is
|
||||
expected and acceptable."""
|
||||
if await llama_healthy():
|
||||
record_llm_activity()
|
||||
return True
|
||||
|
||||
logger.info(f'Asking the Controller to start {LLM_SERVICE_NAME}')
|
||||
if not await start_llm_service():
|
||||
return False
|
||||
|
||||
started = time.time()
|
||||
while time.time() - started < timeout:
|
||||
if await llama_healthy():
|
||||
record_llm_activity()
|
||||
return True
|
||||
await asyncio.sleep(1)
|
||||
logger.error(f'llama-server did not become healthy within {timeout}s')
|
||||
return False
|
||||
|
||||
|
||||
@perpetual_signal(sleep=60)
|
||||
async def perpetual_llm_idle_unload():
|
||||
"""Stop llama-server after the configured idle minutes. Only acts after it was used."""
|
||||
try:
|
||||
last_activity = get_last_llm_activity()
|
||||
if not last_activity:
|
||||
return
|
||||
config = get_ai_config()
|
||||
idle_seconds = config.idle_unload_minutes * 60
|
||||
if time.time() - last_activity < idle_seconds:
|
||||
return
|
||||
if not await llama_healthy():
|
||||
# Already stopped (or crashed); nothing to unload.
|
||||
api_app.shared_ctx.ai_llm_last_request.value = 0.0
|
||||
return
|
||||
logger.info(f'Stopping {LLM_SERVICE_NAME} after {config.idle_unload_minutes} idle minutes')
|
||||
if await stop_llm_service():
|
||||
api_app.shared_ctx.ai_llm_last_request.value = 0.0
|
||||
except Exception as e:
|
||||
logger.error('LLM idle unload worker failed', exc_info=e)
|
||||
78
modules/ai/test/test_service.py
Normal file
78
modules/ai/test/test_service.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
"""Tests for the llama-server lifecycle: on-demand start and idle unload."""
|
||||
import time
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from modules.ai import service
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_llama_running_already_healthy(async_client):
|
||||
"""A healthy llama-server is used as-is and activity is recorded."""
|
||||
with mock.patch('modules.ai.service.llama_healthy', return_value=True), \
|
||||
mock.patch('modules.ai.service.start_llm_service') as start:
|
||||
assert await service.ensure_llama_running() is True
|
||||
start.assert_not_called()
|
||||
assert service.get_last_llm_activity() > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_llama_running_cold_start(async_client):
|
||||
"""An unhealthy llama-server is started via the Controller and polled until healthy."""
|
||||
health = iter([False, False, True])
|
||||
|
||||
async def fake_healthy():
|
||||
return next(health)
|
||||
|
||||
with mock.patch('modules.ai.service.llama_healthy', side_effect=fake_healthy), \
|
||||
mock.patch('modules.ai.service.start_llm_service', return_value=True) as start, \
|
||||
mock.patch('modules.ai.service.asyncio.sleep'):
|
||||
assert await service.ensure_llama_running(timeout=10) is True
|
||||
start.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_llama_running_start_fails(async_client):
|
||||
"""A Controller start failure is reported as False, not an exception."""
|
||||
with mock.patch('modules.ai.service.llama_healthy', return_value=False), \
|
||||
mock.patch('modules.ai.service.start_llm_service', return_value=False):
|
||||
assert await service.ensure_llama_running(timeout=1) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idle_unload(async_client, test_ai_config):
|
||||
"""llama-server is stopped only after the configured idle minutes."""
|
||||
from wrolpi.api_utils import api_app
|
||||
from modules.ai.config import get_ai_config
|
||||
|
||||
get_ai_config().idle_unload_minutes = 10
|
||||
|
||||
# Never used: nothing happens.
|
||||
api_app.shared_ctx.ai_llm_last_request.value = 0.0
|
||||
with mock.patch('modules.ai.service.stop_llm_service') as stop:
|
||||
await service.perpetual_llm_idle_unload()
|
||||
stop.assert_not_called()
|
||||
|
||||
# Recently used: not stopped.
|
||||
api_app.shared_ctx.ai_llm_last_request.value = time.time()
|
||||
with mock.patch('modules.ai.service.stop_llm_service') as stop, \
|
||||
mock.patch('modules.ai.service.llama_healthy', return_value=True):
|
||||
await service.perpetual_llm_idle_unload()
|
||||
stop.assert_not_called()
|
||||
|
||||
# Idle past the threshold: stopped, and the activity marker resets.
|
||||
api_app.shared_ctx.ai_llm_last_request.value = time.time() - 11 * 60
|
||||
with mock.patch('modules.ai.service.stop_llm_service', return_value=True) as stop, \
|
||||
mock.patch('modules.ai.service.llama_healthy', return_value=True):
|
||||
await service.perpetual_llm_idle_unload()
|
||||
stop.assert_called_once()
|
||||
assert api_app.shared_ctx.ai_llm_last_request.value == 0.0
|
||||
|
||||
# Idle but already stopped: the marker resets without a stop call.
|
||||
api_app.shared_ctx.ai_llm_last_request.value = time.time() - 11 * 60
|
||||
with mock.patch('modules.ai.service.stop_llm_service') as stop, \
|
||||
mock.patch('modules.ai.service.llama_healthy', return_value=False):
|
||||
await service.perpetual_llm_idle_unload()
|
||||
stop.assert_not_called()
|
||||
assert api_app.shared_ctx.ai_llm_last_request.value == 0.0
|
||||
Loading…
Add table
Add a link
Reference in a new issue