ai: ground answers in a live library map, budget the context

Response-quality work, all aimed at giving the model enough context to
answer accurately:

- Every chat's system prompt now carries a live library map (today's
  date, counts, and the exact names of channels, domains, zims, and
  inventories, capped and cached 60s).  Exact spellings are in context
  before the first token, and the list_collections round-trip — minutes
  on slow hardware — is usually unnecessary.
- The request is budgeted against the model's real context: oldest
  turns are trimmed first, the current question is never dropped, and
  an over-budget turn gets a clear "start a new conversation" error
  instead of a silently truncated prompt.  cache_prompt reuses the KV
  cache for the stable prefix, which is most of the latency on a Pi.
- context_size plumbing fixed: selecting a model adopts its catalog
  default (the 4B was running at 8k instead of 16k), and llama-server
  runs --parallel 1 so the single user gets the whole context.
- temperature is configurable in ai.yaml (default 0.3 — small models
  call tools far more reliably near 0.2-0.3 than at server defaults).
- One worked tool-use example per mode is prepended to every
  conversation; small models imitate far better than they obey.
- scripts/ai_eval.py scores the assistant against a live WROLPi on
  mechanical checks (finished, called the right tool first, included a
  WROLPi link, no external URLs) so prompt changes stop being validated
  by anecdote.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-09-02 16:48:19 -06:00
parent 82e11cbaba
commit ced78e181f
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
11 changed files with 502 additions and 7 deletions

View file

@ -100,10 +100,15 @@ async def test_chat_tool_call_then_answer(async_client, ai_enabled, video_factor
assert dict(events)['tool_result'] == dict(tool='search_videos', success=True)
assert dict(events)['done'] == dict(content='Found one video.')
# The second llama call saw the system prompt, the user message, and the tool result.
# The second llama call saw the system prompt (with the live library map), the mode's
# worked example, the user message, and the tool result.
second_call_messages = calls['messages'][1]
assert second_call_messages[0]['role'] == 'system'
assert 'librarian' in second_call_messages[0]['content']
assert 'Today is' in second_call_messages[0]['content']
assert "library contains" in second_call_messages[0]['content']
example = [m for m in second_call_messages if m.get('content') and 'Bakehouse' in str(m.get('content'))]
assert example, 'the research few-shot example is in context'
assert second_call_messages[-1]['role'] == 'tool'
assert 'pressure canning' in second_call_messages[-1]['content']
@ -152,8 +157,9 @@ async def test_chat_invalid_tool_args(async_client, ai_enabled):
assert results[0]['success'] is False # invalid JSON arguments
assert results[1]['success'] is False # list_disks is not a help-mode tool
assert events[-1] == ('done', dict(content='Sorry.'))
# The model saw instructive errors.
tool_messages = [m for m in calls['messages'][-1] if m['role'] == 'tool']
# The model saw instructive errors (skip the few-shot example's tool messages).
tool_messages = [m for m in calls['messages'][-1]
if m['role'] == 'tool' and not str(m.get('tool_call_id', '')).startswith('ex_')]
assert 'not valid JSON' in tool_messages[0]['content']
assert 'Unknown tool' in tool_messages[1]['content']
@ -187,3 +193,58 @@ async def test_chat_modes_endpoint(async_client):
modes = {m['name']: m for m in response.json['modes']}
assert set(modes) == {'help', 'research', 'system'}
assert modes['research']['suggestions']
def test_trim_history():
"""Old turns are dropped first; fixed messages and the current turn are never trimmed."""
from modules.ai import chat
fixed = [dict(role='system', content='s' * 400), dict(role='user', content='current')]
history = [dict(role='user', content=f'old {i} ' + 'x' * 400) for i in range(10)]
# A huge budget keeps everything.
assert chat.trim_history(history, fixed, [], budget=10_000) == history
# A small budget drops the OLDEST turns.
trimmed = chat.trim_history(history, fixed, [], budget=400)
assert len(trimmed) < len(history)
assert trimmed == history[len(history) - len(trimmed):]
# A budget the fixed messages alone exceed empties the history without crashing.
assert chat.trim_history(history, fixed, [], budget=1) == []
@pytest.mark.asyncio
async def test_chat_context_full(async_client, ai_enabled):
"""A conversation that cannot fit the model context gets a clear error, not a bad answer."""
from modules.ai import chat
get_ai_config().context_size = 1 # far below the reply reserve
patch, calls = _llama_script([dict(type='token', content='hi'), dict(type='stop', tool_calls=[])])
body = dict(mode='help', messages=[dict(role='user', content='hello')])
with patch:
request, response = await async_client.post('/api/chat/', content=json.dumps(body))
events = _parse_sse(response.text)
assert events[-1][0] == 'error'
assert 'no longer fits' in events[-1][1]['message']
assert calls['count'] == 0 # the model was never called
@pytest.mark.asyncio
async def test_chat_history_trimmed(async_client, ai_enabled):
"""Old turns are dropped to fit the context; the current question always survives."""
get_ai_config().context_size = 4_096
patch, calls = _llama_script([dict(type='token', content='ok'), dict(type='stop', tool_calls=[])])
old_turns = []
for i in range(6):
old_turns += [dict(role='user', content=f'old question {i} ' + 'pad ' * 500),
dict(role='assistant', content=f'old answer {i} ' + 'pad ' * 500)]
body = dict(mode='help', messages=old_turns + [dict(role='user', content='the current question')])
with patch:
request, response = await async_client.post('/api/chat/', content=json.dumps(body))
assert _parse_sse(response.text)[-1][0] == 'done'
sent = calls['messages'][0]
contents = [str(m.get('content')) for m in sent]
assert any('the current question' in c for c in contents)
assert not any('old question 0' in c for c in contents) # oldest turn was trimmed

View file

@ -20,6 +20,7 @@ def test_ai_config_defaults_and_update(test_ai_config):
assert config.idle_unload_minutes == 15
assert config.context_size is None
assert config.max_tool_calls == 6
assert config.temperature == 0.3
config.enabled = True
config.active_model = 'Qwen3-1.7B-Q4_K_M.gguf'
@ -161,3 +162,33 @@ async def test_ai_manage_endpoints(async_client, test_directory, test_ai_config)
request, response = await async_client.post('/api/ai/manage/settings',
content=json.dumps(dict(idle_unload_minutes=0)))
assert response.status_code == HTTPStatus.BAD_REQUEST
def test_effective_context_size(test_ai_config):
"""Explicit ai.yaml value wins; else the active model's catalog default; else the small default."""
config = get_ai_config()
assert catalog.get_effective_context_size() == 8_192 # no model, no override
config.active_model = 'Qwen3-4B-Instruct-2507-Q4_K_M.gguf'
assert catalog.get_effective_context_size() == 16_384 # the 4B's catalog default
config.context_size = 4_096
assert catalog.get_effective_context_size() == 4_096 # explicit override wins
@pytest.mark.asyncio
async def test_manage_settings_adopts_model_context(async_client, test_directory, test_ai_config):
"""Selecting a model writes its catalog context default unless the request sets one."""
models_dir = test_directory / 'ai/models'
models_dir.mkdir(parents=True)
(models_dir / 'Qwen3-4B-Instruct-2507-Q4_K_M.gguf').write_bytes(b'GGUF')
content = dict(active_model='Qwen3-4B-Instruct-2507-Q4_K_M.gguf')
request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['context_size'] == 16_384
# An explicit context_size in the same request is respected.
content = dict(active_model='Qwen3-4B-Instruct-2507-Q4_K_M.gguf', context_size=4_096)
request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(content))
assert response.json['context_size'] == 4_096

View file

@ -0,0 +1,60 @@
"""Tests for the library map injected into every chat."""
import time
from unittest import mock
import pytest
from modules.ai import context
@pytest.mark.asyncio
async def test_build_library_context(async_client, test_session, video_factory, archive_factory,
test_zim, food_inventory_factory):
"""The map carries today's date, counts, and exact names."""
from wrolpi.collections.models import Collection
test_session.add(Collection(name='Talking Sasquach', kind='channel'))
video_factory(title='a video')
archive_factory(domain='example.com', title='a page', contents='contents')
food_inventory_factory(name='Pantry')
test_session.commit()
text = context.build_library_context()
assert 'Today is' in text
assert '1 videos, 1 archived web pages' in text
assert 'Talking Sasquach' in text
assert 'example.com' in text
assert 'WROLPi example ZIM' in text
assert 'Pantry' in text
assert 'user spellings may differ' in text
@pytest.mark.asyncio
async def test_library_context_caps_names(async_client, test_session, archive_factory):
"""A huge library lists at most MAX_NAMES names per category, plus a count of the rest."""
for i in range(context.MAX_NAMES + 5):
archive_factory(domain=f'site{i:02}.com', title=f'page {i}', contents='x')
test_session.commit()
text = context.build_library_context()
assert f'Archived websites ({context.MAX_NAMES + 5}):' in text
assert 'and 5 more' in text
listed = [i for i in range(context.MAX_NAMES + 5) if f'site{i:02}.com' in text]
assert len(listed) == context.MAX_NAMES
@pytest.mark.asyncio
async def test_library_context_cache_and_degradation(async_client, test_session):
"""The map is cached briefly; a build failure serves the stale copy rather than crashing chat."""
context._cache.update(text=None, expires=0.0)
first = context.get_library_context()
assert first and 'Today is' in first
with mock.patch('modules.ai.context.build_library_context', side_effect=RuntimeError('db down')) as build:
# Cached: the builder is not called.
assert context.get_library_context() == first
build.assert_not_called()
# Expired + failing builder: the stale copy is served.
context._cache['expires'] = time.time() - 1
assert context.get_library_context() == first
build.assert_called_once()