ai: ground answers in a live library map, budget the context

Response-quality work, all aimed at giving the model enough context to answer accurately: - Every chat's system prompt now carries a live library map (today's date, counts, and the exact names of channels, domains, zims, and inventories, capped and cached 60s). Exact spellings are in context before the first token, and the list_collections round-trip — minutes on slow hardware — is usually unnecessary. - The request is budgeted against the model's real context: oldest turns are trimmed first, the current question is never dropped, and an over-budget turn gets a clear "start a new conversation" error instead of a silently truncated prompt. cache_prompt reuses the KV cache for the stable prefix, which is most of the latency on a Pi. - context_size plumbing fixed: selecting a model adopts its catalog default (the 4B was running at 8k instead of 16k), and llama-server runs --parallel 1 so the single user gets the whole context. - temperature is configurable in ai.yaml (default 0.3 — small models call tools far more reliably near 0.2-0.3 than at server defaults). - One worked tool-use example per mode is prepended to every conversation; small models imitate far better than they obey. - scripts/ai_eval.py scores the assistant against a live WROLPi on mechanical checks (finished, called the right tool first, included a WROLPi link, no external URLs) so prompt changes stop being validated by anecdote. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-10-26 10:51:36 -04:00 · 2026-09-02 16:48:19 -06:00 · 2026-09-02 16:48:19 -06:00 · ced78e181f
commit ced78e181f
parent 82e11cbaba
11 changed files with 502 additions and 7 deletions
--- a/modules/ai/api.py
+++ b/modules/ai/api.py
@ -619,6 +619,10 @@ async def manage_settings(_: Request, body: schema.AIManageSettingsRequest):
        if body.active_model and not (catalog.get_models_directory() / body.active_model).is_file():
            raise ValidationError(f'Model is not downloaded: {body.active_model}')
        config.active_model = body.active_model
+        # Each model has its own context default (the 4B runs 16k, the small tier 8k); selecting
+        # a model adopts it unless this request also sets context_size explicitly.
+        if body.context_size is None and (default := catalog.get_model_default_context(body.active_model)):
+            config.context_size = default
    if body.enabled is not None:
        config.enabled = body.enabled
    if body.idle_unload_minutes is not None:
--- a/modules/ai/catalog.py
+++ b/modules/ai/catalog.py
@ -7,6 +7,7 @@ committed WROLPi public key.  The release process refreshes AI_MODELS from the p
 import json
 import tempfile
 from pathlib import Path
+from typing import Optional

 import psutil

@ -106,3 +107,21 @@ def recommend_tier(total_ram_bytes: int = None) -> str:
    if gb < 24:
        return 'medium'
    return 'large'
+
+
+def get_model_default_context(name: str) -> Optional[int]:
+    """The bundled catalog's default context size for a model, if known."""
+    for model in AI_MODELS:
+        if model['name'] == name:
+            return model.get('default_context')
+    return None
+
+
+def get_effective_context_size() -> int:
+    """The context llama-server runs with: the explicit ai.yaml value, else the active model's
+    catalog default, else the conservative small-tier default.  start_llama_server.sh reads the
+    explicit value; manage_settings writes the model default there on model selection so the
+    script and this function agree."""
+    from modules.ai.config import get_ai_config
+    config = get_ai_config()
+    return config.context_size or get_model_default_context(config.active_model) or 8_192
--- a/modules/ai/chat.py
+++ b/modules/ai/chat.py
@ -28,8 +28,10 @@ from wrolpi.common import aiohttp_post, logger
 from wrolpi.errors import APIError, ValidationError

 from . import agent, service
+from .catalog import get_effective_context_size
 from .config import get_ai_config
-from .modes import MODE_SUGGESTIONS, MODES
+from .context import get_library_context
+from .modes import MODE_EXAMPLES, MODE_SUGGESTIONS, MODES

 chat_bp = Blueprint('AIChat', url_prefix='/api/chat')

@ -38,6 +40,26 @@ logger = logger.getChild(__name__)
 # The whole conversation turn (all tool calls and answers) must finish within this.
 CHAT_TIMEOUT_SECONDS = 600
 FALLBACK_MESSAGE = "I couldn't complete that request.  Try rephrasing, or a more specific question."
+# Tokens kept free for the model's reply when budgeting the prompt.
+REPLY_RESERVE_TOKENS = 768
+CONTEXT_FULL_MESSAGE = ('This conversation no longer fits the model\'s context.'
+                        '  Start a new conversation and ask a more specific question.')
+
+
+def estimate_tokens(messages: List[dict], tools: List[dict]) -> int:
+    """A deliberate overestimate (~4 chars/token) of what one completion request costs."""
+    return (len(json.dumps(messages)) + len(json.dumps(tools, default=str))) // 4
+
+
+def trim_history(history: List[dict], fixed: List[dict], tools: List[dict], budget: int) -> List[dict]:
+    """Drop the oldest conversation turns until fixed + history fits the budget.
+
+    `fixed` (system prompt, library map, few-shot example, and the current user message) is never
+    trimmed; old turns are the least valuable tokens in the request."""
+    history = list(history)
+    while history and estimate_tokens(fixed + history, tools) > budget:
+        history.pop(0)
+    return history


 class AIDisabled(APIError):
@ -67,7 +89,16 @@ async def llama_chat_stream(messages: List[dict], tools: List[dict]) -> AsyncGen
    Yields {'type': 'token', 'content': str} for answer fragments, then exactly one
    {'type': 'stop', 'tool_calls': [...]} where tool_calls is empty for a final answer.
    Patched in tests."""
-    body = dict(messages=messages, stream=True)
+    config = get_ai_config()
+    body = dict(
+        messages=messages,
+        stream=True,
+        # Low temperature: tool calls from small models are far more reliable near 0.2-0.3.
+        temperature=config.temperature,
+        # Reuse the KV cache for the stable prefix (system prompt + example); on slow CPUs
+        # re-processing that prefix is most of the latency.
+        cache_prompt=True,
+    )
    if tools:
        body['tools'] = tools

@ -142,8 +173,25 @@ async def post_chat(request: Request, body: ChatRequest):
            await _send_event(response, 'error', dict(message='The AI service could not be started.'))
            return

-        messages = [dict(role='system', content=mode['system_prompt'])]
-        messages += [dict(role=m.get('role', 'user'), content=m.get('content', '')) for m in body.messages]
+        # System prompt + the live library map, so exact names, counts, and today's date are in
+        # context before the first token; then the mode's worked example, then the conversation.
+        system_content = mode['system_prompt']
+        if library_context := get_library_context():
+            system_content += f'\n\n{library_context}'
+        fixed = [dict(role='system', content=system_content)]
+        fixed += MODE_EXAMPLES.get(body.mode, [])
+
+        history = [dict(role=m.get('role', 'user'), content=m.get('content', '')) for m in body.messages]
+        # The current user message is never trimmed away.
+        current, history = history[-1], history[:-1]
+        fixed.append(current)
+
+        budget = get_effective_context_size() - REPLY_RESERVE_TOKENS
+        history = trim_history(history, fixed, tools, budget)
+        if estimate_tokens(fixed + history, tools) > budget:
+            await _send_event(response, 'error', dict(message=CONTEXT_FULL_MESSAGE))
+            return
+        messages = fixed[:-1] + history + [current]

        final_answer = []
        for _ in range(max_tool_calls + 1):
@ -151,6 +199,11 @@ async def post_chat(request: Request, body: ChatRequest):
                await _send_event(response, 'error', dict(message=FALLBACK_MESSAGE))
                return

+            # Tool results grow the request; refuse to overflow the model's context mid-loop.
+            if estimate_tokens(messages, tools) > budget:
+                await _send_event(response, 'error', dict(message=CONTEXT_FULL_MESSAGE))
+                return
+
            service.record_llm_activity()
            # Always send something before the model call: on slow CPUs the first token can be
            # minutes away, and a silent stream looks dead to the UI and to proxies.
--- a/modules/ai/config.py
+++ b/modules/ai/config.py
@ -17,6 +17,7 @@ class AIConfigValidator:
    idle_unload_minutes: int = None
    context_size: Optional[int] = None
    max_tool_calls: int = None
+    temperature: float = None


 class AIConfig(ConfigFile):
@ -28,6 +29,7 @@ class AIConfig(ConfigFile):
        idle_unload_minutes=15,
        context_size=None,  # None uses the active model's default from the catalog.
        max_tool_calls=6,
+        temperature=0.3,  # Low: tool-calling small models are far more reliable near 0.2-0.3.
    )
    validator = AIConfigValidator

@ -76,6 +78,15 @@ class AIConfig(ConfigFile):
    def max_tool_calls(self, value: int):
        self.update({'max_tool_calls': int(value)})

+    @property
+    def temperature(self) -> float:
+        # .get(): configs imported before this field existed lack the key.
+        return self._config.get('temperature', 0.3)
+
+    @temperature.setter
+    def temperature(self, value: float):
+        self.update({'temperature': float(value)})
+

 AI_CONFIG: AIConfig = AIConfig()

--- a/modules/ai/context.py
+++ b/modules/ai/context.py
@ -0,0 +1,85 @@
+"""The library map: a compact, live overview of this WROLPi injected into every chat.
+
+Small models answer from what is in context.  Without this block the model must spend a tool
+round-trip (minutes on slow hardware) just to learn what channels exist — and it still misread
+spellings.  With it, exact names, counts, and today's date are in front of the model before the
+first token.  Kept to a few hundred tokens: names are capped and the block is cached briefly."""
+import time
+from typing import List, Optional
+
+from sqlalchemy import func
+
+from wrolpi.common import logger
+from wrolpi.dates import now
+from wrolpi.db import get_db_session
+
+logger = logger.getChild(__name__)
+
+# At most this many names per category; a huge library lists the rest as a count.
+MAX_NAMES = 30
+CACHE_SECONDS = 60
+
+_cache = dict(text=None, expires=0.0)
+
+
+def _capped_names(names: List[str], label: str) -> str:
+    names = [i for i in names if i]
+    if not names:
+        return f'{label}: none'
+    listed = ', '.join(sorted(names)[:MAX_NAMES])
+    extra = f' (and {len(names) - MAX_NAMES} more)' if len(names) > MAX_NAMES else ''
+    return f'{label} ({len(names)}): {listed}{extra}'
+
+
+def build_library_context() -> str:
+    """Build the library overview.  Every query here must stay cheap (counts and name lists)."""
+    from modules.archive.models import Archive
+    from modules.docs.models import Doc
+    from modules.videos.models import Video
+    from modules.zim.models import Zim
+    from wrolpi.collections.models import Collection
+
+    lines = [f'Today is {now().strftime("%A, %B %d, %Y")}.', "The user's library contains:"]
+    with get_db_session() as session:
+        counts = {model.__name__: session.query(func.count(model.id)).scalar() or 0
+                  for model in (Video, Archive, Doc)}
+        lines.append(f'- {counts["Video"]} videos, {counts["Archive"]} archived web pages,'
+                     f' {counts["Doc"]} documents/ebooks.')
+
+        collections = session.query(Collection.kind, Collection.name).all()
+        channels = [name for kind, name in collections if kind == 'channel']
+        domains = [name for kind, name in collections if kind == 'domain']
+        lines.append(f'- {_capped_names(channels, "Video channels")}')
+        lines.append(f'- {_capped_names(domains, "Archived websites")}')
+
+        zims = session.query(Zim).all()
+        zim_titles = []
+        for zim in zims:
+            try:
+                zim_titles.append(zim.zim_metadata.title)
+            except Exception:
+                zim_titles.append(zim.path.name if zim.path else None)
+        lines.append(f'- {_capped_names(zim_titles, "Zim encyclopedias")}')
+
+    try:
+        from modules.inventory.common import get_inventory_configs
+        inventories = [i.get('name') for i in get_inventory_configs().all_inventories()]
+        lines.append(f'- {_capped_names(inventories, "Inventories")}')
+    except Exception as e:
+        logger.debug('Library context could not list inventories', exc_info=e)
+
+    lines.append('Use the exact names above; user spellings may differ.')
+    return '\n'.join(lines)
+
+
+def get_library_context() -> Optional[str]:
+    """The cached library overview; None when it cannot be built (chat proceeds without it)."""
+    if _cache['text'] is not None and time.time() < _cache['expires']:
+        return _cache['text']
+    try:
+        text = build_library_context()
+    except Exception as e:
+        logger.warning('Could not build the library context', exc_info=e)
+        return _cache['text']
+    _cache.update(text=text, expires=time.time() + CACHE_SECONDS)
+    return text
--- a/modules/ai/modes.py
+++ b/modules/ai/modes.py
@ -82,6 +82,62 @@ MODES = {
    ),
 }

+
+def _example(*messages) -> list:
+    """A compact worked exchange (user -> tool call -> tool result -> answer) prepended to every
+    conversation.  Small models imitate far better than they obey; one example per mode is what
+    stops them answering from memory or quitting after the first tool call."""
+    return list(messages)
+
+
+def _tool_call(call_id: str, name: str, arguments: str) -> dict:
+    return dict(role='assistant', content=None, tool_calls=[
+        dict(id=call_id, type='function', function=dict(name=name, arguments=arguments))])
+
+
+def _tool_result(call_id: str, content: str) -> dict:
+    return dict(role='tool', tool_call_id=call_id, content=content)
+
+
+MODE_EXAMPLES = {
+    'help': _example(
+        dict(role='user', content='How do I tag a video?'),
+        _tool_call('ex_help_1', 'search_help', '{"search_str": "tag video"}'),
+        _tool_result('ex_help_1',
+                     '{"results": [{"slug": "system/tags", "title": "Tags", "link": "/system/tags/",'
+                     ' "snippet": "…Tags group related content across the library…"}], "total": 1}'),
+        _tool_call('ex_help_2', 'get_help_doc', '{"slug": "system/tags"}'),
+        _tool_result('ex_help_2',
+                     '{"content": "# Tags\\nOpen any video and press the tag button to add a tag.",'
+                     ' "next_offset": null, "total_chars": 62}'),
+        dict(role='assistant', content='Open any video and press the tag button to add a tag.'
+                                       '  See the Tags help page: /system/tags/'),
+    ),
+    'research': _example(
+        dict(role='user', content='Summarize the bakehouse channel'),
+        _tool_call('ex_res_1', 'list_collections', '{"kind": "channel"}'),
+        _tool_result('ex_res_1',
+                     '{"results": [{"id": 7, "name": "Bakehouse", "kind": "channel",'
+                     ' "link": "/videos/channel/7/video"}], "total": 1}'),
+        _tool_call('ex_res_2', 'search_videos', '{"channel_id": 7}'),
+        _tool_result('ex_res_2',
+                     '{"results": [{"id": 12, "kind": "video", "title": "Sourdough Basics",'
+                     ' "link": "/videos/12", "channel": "Bakehouse", "duration": 900}], "total": 1}'),
+        dict(role='assistant', content='The Bakehouse channel (/videos/channel/7/video) has one video:'
+                                       ' Sourdough Basics (/videos/12), a 15 minute introduction to'
+                                       ' sourdough baking.'),
+    ),
+    'system': _example(
+        dict(role='user', content='Is everything running?'),
+        _tool_call('ex_sys_1', 'get_system_status', '{}'),
+        _tool_result('ex_sys_1',
+                     '{"version": "0.28", "wrol_mode": false, "services": [{"name": "wrolpi-api",'
+                     ' "status": "running"}, {"name": "wrolpi-kiwix", "status": "failed"}], "errors": []}'),
+        dict(role='assistant', content='Almost: wrolpi-kiwix has failed; everything else is running.'
+                                       '  I can read its logs to find out why.'),
+    ),
+}
+
 # Suggestion chips shown by the Chat tab; small models do much better when the opening message
 # matches what the mode's prompt was tuned for.
 MODE_SUGGESTIONS = {
--- a/modules/ai/test/test_chat_api.py
+++ b/modules/ai/test/test_chat_api.py
@ -100,10 +100,15 @@ async def test_chat_tool_call_then_answer(async_client, ai_enabled, video_factor
    assert dict(events)['tool_result'] == dict(tool='search_videos', success=True)
    assert dict(events)['done'] == dict(content='Found one video.')

-    # The second llama call saw the system prompt, the user message, and the tool result.
+    # The second llama call saw the system prompt (with the live library map), the mode's
+    # worked example, the user message, and the tool result.
    second_call_messages = calls['messages'][1]
    assert second_call_messages[0]['role'] == 'system'
    assert 'librarian' in second_call_messages[0]['content']
+    assert 'Today is' in second_call_messages[0]['content']
+    assert "library contains" in second_call_messages[0]['content']
+    example = [m for m in second_call_messages if m.get('content') and 'Bakehouse' in str(m.get('content'))]
+    assert example, 'the research few-shot example is in context'
    assert second_call_messages[-1]['role'] == 'tool'
    assert 'pressure canning' in second_call_messages[-1]['content']

@ -152,8 +157,9 @@ async def test_chat_invalid_tool_args(async_client, ai_enabled):
    assert results[0]['success'] is False  # invalid JSON arguments
    assert results[1]['success'] is False  # list_disks is not a help-mode tool
    assert events[-1] == ('done', dict(content='Sorry.'))
-    # The model saw instructive errors.
-    tool_messages = [m for m in calls['messages'][-1] if m['role'] == 'tool']
+    # The model saw instructive errors (skip the few-shot example's tool messages).
+    tool_messages = [m for m in calls['messages'][-1]
+                     if m['role'] == 'tool' and not str(m.get('tool_call_id', '')).startswith('ex_')]
    assert 'not valid JSON' in tool_messages[0]['content']
    assert 'Unknown tool' in tool_messages[1]['content']

@ -187,3 +193,58 @@ async def test_chat_modes_endpoint(async_client):
    modes = {m['name']: m for m in response.json['modes']}
    assert set(modes) == {'help', 'research', 'system'}
    assert modes['research']['suggestions']
+
+
+def test_trim_history():
+    """Old turns are dropped first; fixed messages and the current turn are never trimmed."""
+    from modules.ai import chat
+    fixed = [dict(role='system', content='s' * 400), dict(role='user', content='current')]
+    history = [dict(role='user', content=f'old {i} ' + 'x' * 400) for i in range(10)]
+
+    # A huge budget keeps everything.
+    assert chat.trim_history(history, fixed, [], budget=10_000) == history
+
+    # A small budget drops the OLDEST turns.
+    trimmed = chat.trim_history(history, fixed, [], budget=400)
+    assert len(trimmed) < len(history)
+    assert trimmed == history[len(history) - len(trimmed):]
+
+    # A budget the fixed messages alone exceed empties the history without crashing.
+    assert chat.trim_history(history, fixed, [], budget=1) == []
+
+
+@pytest.mark.asyncio
+async def test_chat_context_full(async_client, ai_enabled):
+    """A conversation that cannot fit the model context gets a clear error, not a bad answer."""
+    from modules.ai import chat
+    get_ai_config().context_size = 1  # far below the reply reserve
+
+    patch, calls = _llama_script([dict(type='token', content='hi'), dict(type='stop', tool_calls=[])])
+    body = dict(mode='help', messages=[dict(role='user', content='hello')])
+    with patch:
+        request, response = await async_client.post('/api/chat/', content=json.dumps(body))
+    events = _parse_sse(response.text)
+    assert events[-1][0] == 'error'
+    assert 'no longer fits' in events[-1][1]['message']
+    assert calls['count'] == 0  # the model was never called
+
+
+@pytest.mark.asyncio
+async def test_chat_history_trimmed(async_client, ai_enabled):
+    """Old turns are dropped to fit the context; the current question always survives."""
+    get_ai_config().context_size = 4_096
+
+    patch, calls = _llama_script([dict(type='token', content='ok'), dict(type='stop', tool_calls=[])])
+    old_turns = []
+    for i in range(6):
+        old_turns += [dict(role='user', content=f'old question {i} ' + 'pad ' * 500),
+                      dict(role='assistant', content=f'old answer {i} ' + 'pad ' * 500)]
+    body = dict(mode='help', messages=old_turns + [dict(role='user', content='the current question')])
+    with patch:
+        request, response = await async_client.post('/api/chat/', content=json.dumps(body))
+    assert _parse_sse(response.text)[-1][0] == 'done'
+
+    sent = calls['messages'][0]
+    contents = [str(m.get('content')) for m in sent]
+    assert any('the current question' in c for c in contents)
+    assert not any('old question 0' in c for c in contents)  # oldest turn was trimmed
--- a/modules/ai/test/test_config_catalog.py
+++ b/modules/ai/test/test_config_catalog.py
@ -20,6 +20,7 @@ def test_ai_config_defaults_and_update(test_ai_config):
    assert config.idle_unload_minutes == 15
    assert config.context_size is None
    assert config.max_tool_calls == 6
+    assert config.temperature == 0.3

    config.enabled = True
    config.active_model = 'Qwen3-1.7B-Q4_K_M.gguf'
@ -161,3 +162,33 @@ async def test_ai_manage_endpoints(async_client, test_directory, test_ai_config)
    request, response = await async_client.post('/api/ai/manage/settings',
                                                content=json.dumps(dict(idle_unload_minutes=0)))
    assert response.status_code == HTTPStatus.BAD_REQUEST
+
+
+def test_effective_context_size(test_ai_config):
+    """Explicit ai.yaml value wins; else the active model's catalog default; else the small default."""
+    config = get_ai_config()
+    assert catalog.get_effective_context_size() == 8_192  # no model, no override
+
+    config.active_model = 'Qwen3-4B-Instruct-2507-Q4_K_M.gguf'
+    assert catalog.get_effective_context_size() == 16_384  # the 4B's catalog default
+
+    config.context_size = 4_096
+    assert catalog.get_effective_context_size() == 4_096  # explicit override wins
+
+
+@pytest.mark.asyncio
+async def test_manage_settings_adopts_model_context(async_client, test_directory, test_ai_config):
+    """Selecting a model writes its catalog context default unless the request sets one."""
+    models_dir = test_directory / 'ai/models'
+    models_dir.mkdir(parents=True)
+    (models_dir / 'Qwen3-4B-Instruct-2507-Q4_K_M.gguf').write_bytes(b'GGUF')
+
+    content = dict(active_model='Qwen3-4B-Instruct-2507-Q4_K_M.gguf')
+    request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(content))
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['context_size'] == 16_384
+
+    # An explicit context_size in the same request is respected.
+    content = dict(active_model='Qwen3-4B-Instruct-2507-Q4_K_M.gguf', context_size=4_096)
+    request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(content))
+    assert response.json['context_size'] == 4_096
--- a/modules/ai/test/test_context.py
+++ b/modules/ai/test/test_context.py
@ -0,0 +1,60 @@
+"""Tests for the library map injected into every chat."""
+import time
+from unittest import mock
+
+import pytest
+
+from modules.ai import context
+
+
+@pytest.mark.asyncio
+async def test_build_library_context(async_client, test_session, video_factory, archive_factory,
+                                     test_zim, food_inventory_factory):
+    """The map carries today's date, counts, and exact names."""
+    from wrolpi.collections.models import Collection
+    test_session.add(Collection(name='Talking Sasquach', kind='channel'))
+    video_factory(title='a video')
+    archive_factory(domain='example.com', title='a page', contents='contents')
+    food_inventory_factory(name='Pantry')
+    test_session.commit()
+
+    text = context.build_library_context()
+    assert 'Today is' in text
+    assert '1 videos, 1 archived web pages' in text
+    assert 'Talking Sasquach' in text
+    assert 'example.com' in text
+    assert 'WROLPi example ZIM' in text
+    assert 'Pantry' in text
+    assert 'user spellings may differ' in text
+
+
+@pytest.mark.asyncio
+async def test_library_context_caps_names(async_client, test_session, archive_factory):
+    """A huge library lists at most MAX_NAMES names per category, plus a count of the rest."""
+    for i in range(context.MAX_NAMES + 5):
+        archive_factory(domain=f'site{i:02}.com', title=f'page {i}', contents='x')
+    test_session.commit()
+
+    text = context.build_library_context()
+    assert f'Archived websites ({context.MAX_NAMES + 5}):' in text
+    assert 'and 5 more' in text
+    listed = [i for i in range(context.MAX_NAMES + 5) if f'site{i:02}.com' in text]
+    assert len(listed) == context.MAX_NAMES
+
+
+@pytest.mark.asyncio
+async def test_library_context_cache_and_degradation(async_client, test_session):
+    """The map is cached briefly; a build failure serves the stale copy rather than crashing chat."""
+    context._cache.update(text=None, expires=0.0)
+    first = context.get_library_context()
+    assert first and 'Today is' in first
+
+    with mock.patch('modules.ai.context.build_library_context', side_effect=RuntimeError('db down')) as build:
+        # Cached: the builder is not called.
+        assert context.get_library_context() == first
+        build.assert_not_called()
+
+        # Expired + failing builder: the stale copy is served.
+        context._cache['expires'] = time.time() - 1
+        assert context.get_library_context() == first
+        build.assert_called_once()
--- a/scripts/ai_eval.py
+++ b/scripts/ai_eval.py
@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Evaluate the AI assistant against a live WROLPi.
+
+Prompt and mode changes have so far been validated by anecdote; this scores every answer on
+mechanical properties that do not need a judge:
+
+  - the stream finished with `done` (no error, no timeout)
+  - at least one tool was called, and the FIRST tool was one the question expects
+  - the answer contains a relative WROLPi link when one is expected
+  - the answer contains no external URLs (the modes forbid them)
+
+Run it before and after any prompt/tool change:
+
+    python3 scripts/ai_eval.py --base https://localhost:8443 [--mode research] [--timeout 600]
+
+Exit code 1 when any question errors out; the score table is the real output.  Requires only
+`requests` (in WROLPi's requirements).  The suite reads the library map's world: questions are
+library-agnostic (browse/status/help) so they work on any WROLPi with some content."""
+import argparse
+import json
+import re
+import sys
+import time
+
+import requests
+
+requests.packages.urllib3.disable_warnings()  # noqa  Self-signed certs.
+
+# (mode, question, expected first tools (any of), expect a WROLPi link in the answer)
+QUESTIONS = [
+    ('help', 'How do I download videos?', {'search_help'}, True),
+    ('help', 'What are Tags?', {'search_help'}, True),
+    ('help', 'How do I add my own files to WROLPi?', {'search_help'}, True),
+    ('research', 'What are my three newest archives?', {'search_archives'}, True),
+    ('research', 'What video channels do I have?', {'list_collections', 'search_videos'}, False),
+    ('research', 'Find videos about cooking', {'search_videos', 'search_all'}, False),
+    ('research', 'Summarize my newest videos', {'search_videos'}, True),
+    ('system', 'Is everything running?', {'get_system_status', 'list_services'}, False),
+    ('system', 'How full are my drives?', {'get_system_status', 'list_disks'}, False),
+    ('system', 'Why would downloads fail?', {'get_system_status', 'search_help'}, False),
+]
+
+EXTERNAL_URL = re.compile(r'https?://(?!localhost|127\.0\.0\.1)')
+RELATIVE_LINK = re.compile(r'(^|[\s(\[])/(videos|archives|docs|epub|media|download|system|modules)[/\w]')
+
+
+def run_question(base: str, mode: str, question: str, timeout: int) -> dict:
+    started = time.time()
+    tools, answer, error = [], None, None
+    try:
+        response = requests.post(f'{base}/api/chat/', verify=False, stream=True, timeout=timeout,
+                                 json=dict(mode=mode, messages=[dict(role='user', content=question)]))
+        if response.status_code != 200:
+            return dict(error=f'HTTP {response.status_code}', tools=tools, elapsed=time.time() - started)
+        event = None
+        for raw_line in response.iter_lines(decode_unicode=True):
+            if raw_line.startswith('event:'):
+                event = raw_line[6:].strip()
+            elif raw_line.startswith('data:'):
+                data = json.loads(raw_line[5:].strip())
+                if event == 'tool_call':
+                    tools.append(data.get('tool'))
+                elif event == 'done':
+                    answer = data.get('content')
+                elif event == 'error':
+                    error = data.get('message')
+    except Exception as e:
+        error = str(e)[:120]
+    return dict(answer=answer, error=error, tools=tools, elapsed=time.time() - started)
+
+
+def score(result: dict, expected_tools: set, expect_link: bool) -> dict:
+    answer = result.get('answer') or ''
+    checks = {
+        'done': result.get('error') is None and bool(answer),
+        'used_tool': bool(result['tools']),
+        'right_tool': bool(result['tools']) and result['tools'][0] in expected_tools,
+        'no_external_urls': not EXTERNAL_URL.search(answer),
+    }
+    if expect_link:
+        checks['has_link'] = bool(RELATIVE_LINK.search(answer))
+    return checks
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--base', default='https://localhost:8443')
+    parser.add_argument('--mode', help='Only run one mode')
+    parser.add_argument('--timeout', type=int, default=600)
+    args = parser.parse_args()
+
+    questions = [q for q in QUESTIONS if not args.mode or q[0] == args.mode]
+    passed = total = errors = 0
+    for mode, question, expected_tools, expect_link in questions:
+        result = run_question(args.base, mode, question, args.timeout)
+        checks = score(result, expected_tools, expect_link)
+        passed += sum(checks.values())
+        total += len(checks)
+        if result.get('error'):
+            errors += 1
+        flags = ' '.join(f'{name}={"Y" if ok else "N"}' for name, ok in checks.items())
+        print(f'[{mode:8}] {result["elapsed"]:6.1f}s  {flags}  {question}')
+        if not all(checks.values()):
+            print(f'           tools={result["tools"]}')
+            detail = result.get('error') or (result.get('answer') or '')[:200]
+            print(f'           {detail}')
+
+    print(f'\nScore: {passed}/{total} checks passed across {len(questions)} questions; {errors} hard errors.')
+    return 1 if errors else 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
--- a/scripts/start_llama_server.sh
+++ b/scripts/start_llama_server.sh
@ -46,9 +46,11 @@ esac

 # --jinja enables tool-call chat templates; mmap is llama.cpp's default so cold RAM cost is low.
 # Localhost only on native installs -- never exposed through Caddy.
+# --parallel 1: WROLPi is single-user; one slot gets the whole context.
 exec llama-server \
  --host "${LLAMA_HOST}" \
  --port "${LLAMA_PORT}" \
  --jinja \
+  --parallel 1 \
  -c "${CONTEXT_SIZE}" \
  -m "${MODEL}"