diff --git a/modules/ai/api.py b/modules/ai/api.py index cd080a6c..6092373f 100644 --- a/modules/ai/api.py +++ b/modules/ai/api.py @@ -619,6 +619,10 @@ async def manage_settings(_: Request, body: schema.AIManageSettingsRequest): if body.active_model and not (catalog.get_models_directory() / body.active_model).is_file(): raise ValidationError(f'Model is not downloaded: {body.active_model}') config.active_model = body.active_model + # Each model has its own context default (the 4B runs 16k, the small tier 8k); selecting + # a model adopts it unless this request also sets context_size explicitly. + if body.context_size is None and (default := catalog.get_model_default_context(body.active_model)): + config.context_size = default if body.enabled is not None: config.enabled = body.enabled if body.idle_unload_minutes is not None: diff --git a/modules/ai/catalog.py b/modules/ai/catalog.py index 1b32682c..165df81a 100644 --- a/modules/ai/catalog.py +++ b/modules/ai/catalog.py @@ -7,6 +7,7 @@ committed WROLPi public key. The release process refreshes AI_MODELS from the p import json import tempfile from pathlib import Path +from typing import Optional import psutil @@ -106,3 +107,21 @@ def recommend_tier(total_ram_bytes: int = None) -> str: if gb < 24: return 'medium' return 'large' + + +def get_model_default_context(name: str) -> Optional[int]: + """The bundled catalog's default context size for a model, if known.""" + for model in AI_MODELS: + if model['name'] == name: + return model.get('default_context') + return None + + +def get_effective_context_size() -> int: + """The context llama-server runs with: the explicit ai.yaml value, else the active model's + catalog default, else the conservative small-tier default. start_llama_server.sh reads the + explicit value; manage_settings writes the model default there on model selection so the + script and this function agree.""" + from modules.ai.config import get_ai_config + config = get_ai_config() + return config.context_size or get_model_default_context(config.active_model) or 8_192 diff --git a/modules/ai/chat.py b/modules/ai/chat.py index 0407a032..3be44c07 100644 --- a/modules/ai/chat.py +++ b/modules/ai/chat.py @@ -28,8 +28,10 @@ from wrolpi.common import aiohttp_post, logger from wrolpi.errors import APIError, ValidationError from . import agent, service +from .catalog import get_effective_context_size from .config import get_ai_config -from .modes import MODE_SUGGESTIONS, MODES +from .context import get_library_context +from .modes import MODE_EXAMPLES, MODE_SUGGESTIONS, MODES chat_bp = Blueprint('AIChat', url_prefix='/api/chat') @@ -38,6 +40,26 @@ logger = logger.getChild(__name__) # The whole conversation turn (all tool calls and answers) must finish within this. CHAT_TIMEOUT_SECONDS = 600 FALLBACK_MESSAGE = "I couldn't complete that request. Try rephrasing, or a more specific question." +# Tokens kept free for the model's reply when budgeting the prompt. +REPLY_RESERVE_TOKENS = 768 +CONTEXT_FULL_MESSAGE = ('This conversation no longer fits the model\'s context.' + ' Start a new conversation and ask a more specific question.') + + +def estimate_tokens(messages: List[dict], tools: List[dict]) -> int: + """A deliberate overestimate (~4 chars/token) of what one completion request costs.""" + return (len(json.dumps(messages)) + len(json.dumps(tools, default=str))) // 4 + + +def trim_history(history: List[dict], fixed: List[dict], tools: List[dict], budget: int) -> List[dict]: + """Drop the oldest conversation turns until fixed + history fits the budget. + + `fixed` (system prompt, library map, few-shot example, and the current user message) is never + trimmed; old turns are the least valuable tokens in the request.""" + history = list(history) + while history and estimate_tokens(fixed + history, tools) > budget: + history.pop(0) + return history class AIDisabled(APIError): @@ -67,7 +89,16 @@ async def llama_chat_stream(messages: List[dict], tools: List[dict]) -> AsyncGen Yields {'type': 'token', 'content': str} for answer fragments, then exactly one {'type': 'stop', 'tool_calls': [...]} where tool_calls is empty for a final answer. Patched in tests.""" - body = dict(messages=messages, stream=True) + config = get_ai_config() + body = dict( + messages=messages, + stream=True, + # Low temperature: tool calls from small models are far more reliable near 0.2-0.3. + temperature=config.temperature, + # Reuse the KV cache for the stable prefix (system prompt + example); on slow CPUs + # re-processing that prefix is most of the latency. + cache_prompt=True, + ) if tools: body['tools'] = tools @@ -142,8 +173,25 @@ async def post_chat(request: Request, body: ChatRequest): await _send_event(response, 'error', dict(message='The AI service could not be started.')) return - messages = [dict(role='system', content=mode['system_prompt'])] - messages += [dict(role=m.get('role', 'user'), content=m.get('content', '')) for m in body.messages] + # System prompt + the live library map, so exact names, counts, and today's date are in + # context before the first token; then the mode's worked example, then the conversation. + system_content = mode['system_prompt'] + if library_context := get_library_context(): + system_content += f'\n\n{library_context}' + fixed = [dict(role='system', content=system_content)] + fixed += MODE_EXAMPLES.get(body.mode, []) + + history = [dict(role=m.get('role', 'user'), content=m.get('content', '')) for m in body.messages] + # The current user message is never trimmed away. + current, history = history[-1], history[:-1] + fixed.append(current) + + budget = get_effective_context_size() - REPLY_RESERVE_TOKENS + history = trim_history(history, fixed, tools, budget) + if estimate_tokens(fixed + history, tools) > budget: + await _send_event(response, 'error', dict(message=CONTEXT_FULL_MESSAGE)) + return + messages = fixed[:-1] + history + [current] final_answer = [] for _ in range(max_tool_calls + 1): @@ -151,6 +199,11 @@ async def post_chat(request: Request, body: ChatRequest): await _send_event(response, 'error', dict(message=FALLBACK_MESSAGE)) return + # Tool results grow the request; refuse to overflow the model's context mid-loop. + if estimate_tokens(messages, tools) > budget: + await _send_event(response, 'error', dict(message=CONTEXT_FULL_MESSAGE)) + return + service.record_llm_activity() # Always send something before the model call: on slow CPUs the first token can be # minutes away, and a silent stream looks dead to the UI and to proxies. diff --git a/modules/ai/config.py b/modules/ai/config.py index 29569d36..3ea10db4 100644 --- a/modules/ai/config.py +++ b/modules/ai/config.py @@ -17,6 +17,7 @@ class AIConfigValidator: idle_unload_minutes: int = None context_size: Optional[int] = None max_tool_calls: int = None + temperature: float = None class AIConfig(ConfigFile): @@ -28,6 +29,7 @@ class AIConfig(ConfigFile): idle_unload_minutes=15, context_size=None, # None uses the active model's default from the catalog. max_tool_calls=6, + temperature=0.3, # Low: tool-calling small models are far more reliable near 0.2-0.3. ) validator = AIConfigValidator @@ -76,6 +78,15 @@ class AIConfig(ConfigFile): def max_tool_calls(self, value: int): self.update({'max_tool_calls': int(value)}) + @property + def temperature(self) -> float: + # .get(): configs imported before this field existed lack the key. + return self._config.get('temperature', 0.3) + + @temperature.setter + def temperature(self, value: float): + self.update({'temperature': float(value)}) + AI_CONFIG: AIConfig = AIConfig() diff --git a/modules/ai/context.py b/modules/ai/context.py new file mode 100644 index 00000000..90ccfbd7 --- /dev/null +++ b/modules/ai/context.py @@ -0,0 +1,85 @@ +"""The library map: a compact, live overview of this WROLPi injected into every chat. + +Small models answer from what is in context. Without this block the model must spend a tool +round-trip (minutes on slow hardware) just to learn what channels exist — and it still misread +spellings. With it, exact names, counts, and today's date are in front of the model before the +first token. Kept to a few hundred tokens: names are capped and the block is cached briefly.""" +import time +from typing import List, Optional + +from sqlalchemy import func + +from wrolpi.common import logger +from wrolpi.dates import now +from wrolpi.db import get_db_session + +logger = logger.getChild(__name__) + +# At most this many names per category; a huge library lists the rest as a count. +MAX_NAMES = 30 +CACHE_SECONDS = 60 + +_cache = dict(text=None, expires=0.0) + + +def _capped_names(names: List[str], label: str) -> str: + names = [i for i in names if i] + if not names: + return f'{label}: none' + listed = ', '.join(sorted(names)[:MAX_NAMES]) + extra = f' (and {len(names) - MAX_NAMES} more)' if len(names) > MAX_NAMES else '' + return f'{label} ({len(names)}): {listed}{extra}' + + +def build_library_context() -> str: + """Build the library overview. Every query here must stay cheap (counts and name lists).""" + from modules.archive.models import Archive + from modules.docs.models import Doc + from modules.videos.models import Video + from modules.zim.models import Zim + from wrolpi.collections.models import Collection + + lines = [f'Today is {now().strftime("%A, %B %d, %Y")}.', "The user's library contains:"] + with get_db_session() as session: + counts = {model.__name__: session.query(func.count(model.id)).scalar() or 0 + for model in (Video, Archive, Doc)} + lines.append(f'- {counts["Video"]} videos, {counts["Archive"]} archived web pages,' + f' {counts["Doc"]} documents/ebooks.') + + collections = session.query(Collection.kind, Collection.name).all() + channels = [name for kind, name in collections if kind == 'channel'] + domains = [name for kind, name in collections if kind == 'domain'] + lines.append(f'- {_capped_names(channels, "Video channels")}') + lines.append(f'- {_capped_names(domains, "Archived websites")}') + + zims = session.query(Zim).all() + zim_titles = [] + for zim in zims: + try: + zim_titles.append(zim.zim_metadata.title) + except Exception: + zim_titles.append(zim.path.name if zim.path else None) + lines.append(f'- {_capped_names(zim_titles, "Zim encyclopedias")}') + + try: + from modules.inventory.common import get_inventory_configs + inventories = [i.get('name') for i in get_inventory_configs().all_inventories()] + lines.append(f'- {_capped_names(inventories, "Inventories")}') + except Exception as e: + logger.debug('Library context could not list inventories', exc_info=e) + + lines.append('Use the exact names above; user spellings may differ.') + return '\n'.join(lines) + + +def get_library_context() -> Optional[str]: + """The cached library overview; None when it cannot be built (chat proceeds without it).""" + if _cache['text'] is not None and time.time() < _cache['expires']: + return _cache['text'] + try: + text = build_library_context() + except Exception as e: + logger.warning('Could not build the library context', exc_info=e) + return _cache['text'] + _cache.update(text=text, expires=time.time() + CACHE_SECONDS) + return text diff --git a/modules/ai/modes.py b/modules/ai/modes.py index 823ae3c5..47be6d9e 100644 --- a/modules/ai/modes.py +++ b/modules/ai/modes.py @@ -82,6 +82,62 @@ MODES = { ), } + +def _example(*messages) -> list: + """A compact worked exchange (user -> tool call -> tool result -> answer) prepended to every + conversation. Small models imitate far better than they obey; one example per mode is what + stops them answering from memory or quitting after the first tool call.""" + return list(messages) + + +def _tool_call(call_id: str, name: str, arguments: str) -> dict: + return dict(role='assistant', content=None, tool_calls=[ + dict(id=call_id, type='function', function=dict(name=name, arguments=arguments))]) + + +def _tool_result(call_id: str, content: str) -> dict: + return dict(role='tool', tool_call_id=call_id, content=content) + + +MODE_EXAMPLES = { + 'help': _example( + dict(role='user', content='How do I tag a video?'), + _tool_call('ex_help_1', 'search_help', '{"search_str": "tag video"}'), + _tool_result('ex_help_1', + '{"results": [{"slug": "system/tags", "title": "Tags", "link": "/system/tags/",' + ' "snippet": "…Tags group related content across the library…"}], "total": 1}'), + _tool_call('ex_help_2', 'get_help_doc', '{"slug": "system/tags"}'), + _tool_result('ex_help_2', + '{"content": "# Tags\\nOpen any video and press the tag button to add a tag.",' + ' "next_offset": null, "total_chars": 62}'), + dict(role='assistant', content='Open any video and press the tag button to add a tag.' + ' See the Tags help page: /system/tags/'), + ), + 'research': _example( + dict(role='user', content='Summarize the bakehouse channel'), + _tool_call('ex_res_1', 'list_collections', '{"kind": "channel"}'), + _tool_result('ex_res_1', + '{"results": [{"id": 7, "name": "Bakehouse", "kind": "channel",' + ' "link": "/videos/channel/7/video"}], "total": 1}'), + _tool_call('ex_res_2', 'search_videos', '{"channel_id": 7}'), + _tool_result('ex_res_2', + '{"results": [{"id": 12, "kind": "video", "title": "Sourdough Basics",' + ' "link": "/videos/12", "channel": "Bakehouse", "duration": 900}], "total": 1}'), + dict(role='assistant', content='The Bakehouse channel (/videos/channel/7/video) has one video:' + ' Sourdough Basics (/videos/12), a 15 minute introduction to' + ' sourdough baking.'), + ), + 'system': _example( + dict(role='user', content='Is everything running?'), + _tool_call('ex_sys_1', 'get_system_status', '{}'), + _tool_result('ex_sys_1', + '{"version": "0.28", "wrol_mode": false, "services": [{"name": "wrolpi-api",' + ' "status": "running"}, {"name": "wrolpi-kiwix", "status": "failed"}], "errors": []}'), + dict(role='assistant', content='Almost: wrolpi-kiwix has failed; everything else is running.' + ' I can read its logs to find out why.'), + ), +} + # Suggestion chips shown by the Chat tab; small models do much better when the opening message # matches what the mode's prompt was tuned for. MODE_SUGGESTIONS = { diff --git a/modules/ai/test/test_chat_api.py b/modules/ai/test/test_chat_api.py index 60525e3a..a59ac901 100644 --- a/modules/ai/test/test_chat_api.py +++ b/modules/ai/test/test_chat_api.py @@ -100,10 +100,15 @@ async def test_chat_tool_call_then_answer(async_client, ai_enabled, video_factor assert dict(events)['tool_result'] == dict(tool='search_videos', success=True) assert dict(events)['done'] == dict(content='Found one video.') - # The second llama call saw the system prompt, the user message, and the tool result. + # The second llama call saw the system prompt (with the live library map), the mode's + # worked example, the user message, and the tool result. second_call_messages = calls['messages'][1] assert second_call_messages[0]['role'] == 'system' assert 'librarian' in second_call_messages[0]['content'] + assert 'Today is' in second_call_messages[0]['content'] + assert "library contains" in second_call_messages[0]['content'] + example = [m for m in second_call_messages if m.get('content') and 'Bakehouse' in str(m.get('content'))] + assert example, 'the research few-shot example is in context' assert second_call_messages[-1]['role'] == 'tool' assert 'pressure canning' in second_call_messages[-1]['content'] @@ -152,8 +157,9 @@ async def test_chat_invalid_tool_args(async_client, ai_enabled): assert results[0]['success'] is False # invalid JSON arguments assert results[1]['success'] is False # list_disks is not a help-mode tool assert events[-1] == ('done', dict(content='Sorry.')) - # The model saw instructive errors. - tool_messages = [m for m in calls['messages'][-1] if m['role'] == 'tool'] + # The model saw instructive errors (skip the few-shot example's tool messages). + tool_messages = [m for m in calls['messages'][-1] + if m['role'] == 'tool' and not str(m.get('tool_call_id', '')).startswith('ex_')] assert 'not valid JSON' in tool_messages[0]['content'] assert 'Unknown tool' in tool_messages[1]['content'] @@ -187,3 +193,58 @@ async def test_chat_modes_endpoint(async_client): modes = {m['name']: m for m in response.json['modes']} assert set(modes) == {'help', 'research', 'system'} assert modes['research']['suggestions'] + + +def test_trim_history(): + """Old turns are dropped first; fixed messages and the current turn are never trimmed.""" + from modules.ai import chat + fixed = [dict(role='system', content='s' * 400), dict(role='user', content='current')] + history = [dict(role='user', content=f'old {i} ' + 'x' * 400) for i in range(10)] + + # A huge budget keeps everything. + assert chat.trim_history(history, fixed, [], budget=10_000) == history + + # A small budget drops the OLDEST turns. + trimmed = chat.trim_history(history, fixed, [], budget=400) + assert len(trimmed) < len(history) + assert trimmed == history[len(history) - len(trimmed):] + + # A budget the fixed messages alone exceed empties the history without crashing. + assert chat.trim_history(history, fixed, [], budget=1) == [] + + +@pytest.mark.asyncio +async def test_chat_context_full(async_client, ai_enabled): + """A conversation that cannot fit the model context gets a clear error, not a bad answer.""" + from modules.ai import chat + get_ai_config().context_size = 1 # far below the reply reserve + + patch, calls = _llama_script([dict(type='token', content='hi'), dict(type='stop', tool_calls=[])]) + body = dict(mode='help', messages=[dict(role='user', content='hello')]) + with patch: + request, response = await async_client.post('/api/chat/', content=json.dumps(body)) + events = _parse_sse(response.text) + assert events[-1][0] == 'error' + assert 'no longer fits' in events[-1][1]['message'] + assert calls['count'] == 0 # the model was never called + + +@pytest.mark.asyncio +async def test_chat_history_trimmed(async_client, ai_enabled): + """Old turns are dropped to fit the context; the current question always survives.""" + get_ai_config().context_size = 4_096 + + patch, calls = _llama_script([dict(type='token', content='ok'), dict(type='stop', tool_calls=[])]) + old_turns = [] + for i in range(6): + old_turns += [dict(role='user', content=f'old question {i} ' + 'pad ' * 500), + dict(role='assistant', content=f'old answer {i} ' + 'pad ' * 500)] + body = dict(mode='help', messages=old_turns + [dict(role='user', content='the current question')]) + with patch: + request, response = await async_client.post('/api/chat/', content=json.dumps(body)) + assert _parse_sse(response.text)[-1][0] == 'done' + + sent = calls['messages'][0] + contents = [str(m.get('content')) for m in sent] + assert any('the current question' in c for c in contents) + assert not any('old question 0' in c for c in contents) # oldest turn was trimmed diff --git a/modules/ai/test/test_config_catalog.py b/modules/ai/test/test_config_catalog.py index 6fca689a..e5c78631 100644 --- a/modules/ai/test/test_config_catalog.py +++ b/modules/ai/test/test_config_catalog.py @@ -20,6 +20,7 @@ def test_ai_config_defaults_and_update(test_ai_config): assert config.idle_unload_minutes == 15 assert config.context_size is None assert config.max_tool_calls == 6 + assert config.temperature == 0.3 config.enabled = True config.active_model = 'Qwen3-1.7B-Q4_K_M.gguf' @@ -161,3 +162,33 @@ async def test_ai_manage_endpoints(async_client, test_directory, test_ai_config) request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(dict(idle_unload_minutes=0))) assert response.status_code == HTTPStatus.BAD_REQUEST + + +def test_effective_context_size(test_ai_config): + """Explicit ai.yaml value wins; else the active model's catalog default; else the small default.""" + config = get_ai_config() + assert catalog.get_effective_context_size() == 8_192 # no model, no override + + config.active_model = 'Qwen3-4B-Instruct-2507-Q4_K_M.gguf' + assert catalog.get_effective_context_size() == 16_384 # the 4B's catalog default + + config.context_size = 4_096 + assert catalog.get_effective_context_size() == 4_096 # explicit override wins + + +@pytest.mark.asyncio +async def test_manage_settings_adopts_model_context(async_client, test_directory, test_ai_config): + """Selecting a model writes its catalog context default unless the request sets one.""" + models_dir = test_directory / 'ai/models' + models_dir.mkdir(parents=True) + (models_dir / 'Qwen3-4B-Instruct-2507-Q4_K_M.gguf').write_bytes(b'GGUF') + + content = dict(active_model='Qwen3-4B-Instruct-2507-Q4_K_M.gguf') + request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(content)) + assert response.status_code == HTTPStatus.OK + assert response.json['context_size'] == 16_384 + + # An explicit context_size in the same request is respected. + content = dict(active_model='Qwen3-4B-Instruct-2507-Q4_K_M.gguf', context_size=4_096) + request, response = await async_client.post('/api/ai/manage/settings', content=json.dumps(content)) + assert response.json['context_size'] == 4_096 diff --git a/modules/ai/test/test_context.py b/modules/ai/test/test_context.py new file mode 100644 index 00000000..cf3a5fc4 --- /dev/null +++ b/modules/ai/test/test_context.py @@ -0,0 +1,60 @@ +"""Tests for the library map injected into every chat.""" +import time +from unittest import mock + +import pytest + +from modules.ai import context + + +@pytest.mark.asyncio +async def test_build_library_context(async_client, test_session, video_factory, archive_factory, + test_zim, food_inventory_factory): + """The map carries today's date, counts, and exact names.""" + from wrolpi.collections.models import Collection + test_session.add(Collection(name='Talking Sasquach', kind='channel')) + video_factory(title='a video') + archive_factory(domain='example.com', title='a page', contents='contents') + food_inventory_factory(name='Pantry') + test_session.commit() + + text = context.build_library_context() + assert 'Today is' in text + assert '1 videos, 1 archived web pages' in text + assert 'Talking Sasquach' in text + assert 'example.com' in text + assert 'WROLPi example ZIM' in text + assert 'Pantry' in text + assert 'user spellings may differ' in text + + +@pytest.mark.asyncio +async def test_library_context_caps_names(async_client, test_session, archive_factory): + """A huge library lists at most MAX_NAMES names per category, plus a count of the rest.""" + for i in range(context.MAX_NAMES + 5): + archive_factory(domain=f'site{i:02}.com', title=f'page {i}', contents='x') + test_session.commit() + + text = context.build_library_context() + assert f'Archived websites ({context.MAX_NAMES + 5}):' in text + assert 'and 5 more' in text + listed = [i for i in range(context.MAX_NAMES + 5) if f'site{i:02}.com' in text] + assert len(listed) == context.MAX_NAMES + + +@pytest.mark.asyncio +async def test_library_context_cache_and_degradation(async_client, test_session): + """The map is cached briefly; a build failure serves the stale copy rather than crashing chat.""" + context._cache.update(text=None, expires=0.0) + first = context.get_library_context() + assert first and 'Today is' in first + + with mock.patch('modules.ai.context.build_library_context', side_effect=RuntimeError('db down')) as build: + # Cached: the builder is not called. + assert context.get_library_context() == first + build.assert_not_called() + + # Expired + failing builder: the stale copy is served. + context._cache['expires'] = time.time() - 1 + assert context.get_library_context() == first + build.assert_called_once() diff --git a/scripts/ai_eval.py b/scripts/ai_eval.py new file mode 100755 index 00000000..7a2a9222 --- /dev/null +++ b/scripts/ai_eval.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Evaluate the AI assistant against a live WROLPi. + +Prompt and mode changes have so far been validated by anecdote; this scores every answer on +mechanical properties that do not need a judge: + + - the stream finished with `done` (no error, no timeout) + - at least one tool was called, and the FIRST tool was one the question expects + - the answer contains a relative WROLPi link when one is expected + - the answer contains no external URLs (the modes forbid them) + +Run it before and after any prompt/tool change: + + python3 scripts/ai_eval.py --base https://localhost:8443 [--mode research] [--timeout 600] + +Exit code 1 when any question errors out; the score table is the real output. Requires only +`requests` (in WROLPi's requirements). The suite reads the library map's world: questions are +library-agnostic (browse/status/help) so they work on any WROLPi with some content.""" +import argparse +import json +import re +import sys +import time + +import requests + +requests.packages.urllib3.disable_warnings() # noqa Self-signed certs. + +# (mode, question, expected first tools (any of), expect a WROLPi link in the answer) +QUESTIONS = [ + ('help', 'How do I download videos?', {'search_help'}, True), + ('help', 'What are Tags?', {'search_help'}, True), + ('help', 'How do I add my own files to WROLPi?', {'search_help'}, True), + ('research', 'What are my three newest archives?', {'search_archives'}, True), + ('research', 'What video channels do I have?', {'list_collections', 'search_videos'}, False), + ('research', 'Find videos about cooking', {'search_videos', 'search_all'}, False), + ('research', 'Summarize my newest videos', {'search_videos'}, True), + ('system', 'Is everything running?', {'get_system_status', 'list_services'}, False), + ('system', 'How full are my drives?', {'get_system_status', 'list_disks'}, False), + ('system', 'Why would downloads fail?', {'get_system_status', 'search_help'}, False), +] + +EXTERNAL_URL = re.compile(r'https?://(?!localhost|127\.0\.0\.1)') +RELATIVE_LINK = re.compile(r'(^|[\s(\[])/(videos|archives|docs|epub|media|download|system|modules)[/\w]') + + +def run_question(base: str, mode: str, question: str, timeout: int) -> dict: + started = time.time() + tools, answer, error = [], None, None + try: + response = requests.post(f'{base}/api/chat/', verify=False, stream=True, timeout=timeout, + json=dict(mode=mode, messages=[dict(role='user', content=question)])) + if response.status_code != 200: + return dict(error=f'HTTP {response.status_code}', tools=tools, elapsed=time.time() - started) + event = None + for raw_line in response.iter_lines(decode_unicode=True): + if raw_line.startswith('event:'): + event = raw_line[6:].strip() + elif raw_line.startswith('data:'): + data = json.loads(raw_line[5:].strip()) + if event == 'tool_call': + tools.append(data.get('tool')) + elif event == 'done': + answer = data.get('content') + elif event == 'error': + error = data.get('message') + except Exception as e: + error = str(e)[:120] + return dict(answer=answer, error=error, tools=tools, elapsed=time.time() - started) + + +def score(result: dict, expected_tools: set, expect_link: bool) -> dict: + answer = result.get('answer') or '' + checks = { + 'done': result.get('error') is None and bool(answer), + 'used_tool': bool(result['tools']), + 'right_tool': bool(result['tools']) and result['tools'][0] in expected_tools, + 'no_external_urls': not EXTERNAL_URL.search(answer), + } + if expect_link: + checks['has_link'] = bool(RELATIVE_LINK.search(answer)) + return checks + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base', default='https://localhost:8443') + parser.add_argument('--mode', help='Only run one mode') + parser.add_argument('--timeout', type=int, default=600) + args = parser.parse_args() + + questions = [q for q in QUESTIONS if not args.mode or q[0] == args.mode] + passed = total = errors = 0 + for mode, question, expected_tools, expect_link in questions: + result = run_question(args.base, mode, question, args.timeout) + checks = score(result, expected_tools, expect_link) + passed += sum(checks.values()) + total += len(checks) + if result.get('error'): + errors += 1 + flags = ' '.join(f'{name}={"Y" if ok else "N"}' for name, ok in checks.items()) + print(f'[{mode:8}] {result["elapsed"]:6.1f}s {flags} {question}') + if not all(checks.values()): + print(f' tools={result["tools"]}') + detail = result.get('error') or (result.get('answer') or '')[:200] + print(f' {detail}') + + print(f'\nScore: {passed}/{total} checks passed across {len(questions)} questions; {errors} hard errors.') + return 1 if errors else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/start_llama_server.sh b/scripts/start_llama_server.sh index b1e786ba..97736ed3 100755 --- a/scripts/start_llama_server.sh +++ b/scripts/start_llama_server.sh @@ -46,9 +46,11 @@ esac # --jinja enables tool-call chat templates; mmap is llama.cpp's default so cold RAM cost is low. # Localhost only on native installs -- never exposed through Caddy. +# --parallel 1: WROLPi is single-user; one slot gets the whole context. exec llama-server \ --host "${LLAMA_HOST}" \ --port "${LLAMA_PORT}" \ --jinja \ + --parallel 1 \ -c "${CONTEXT_SIZE}" \ -m "${MODEL}"