mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
ai: ground answers in a live library map, budget the context
Response-quality work, all aimed at giving the model enough context to answer accurately: - Every chat's system prompt now carries a live library map (today's date, counts, and the exact names of channels, domains, zims, and inventories, capped and cached 60s). Exact spellings are in context before the first token, and the list_collections round-trip — minutes on slow hardware — is usually unnecessary. - The request is budgeted against the model's real context: oldest turns are trimmed first, the current question is never dropped, and an over-budget turn gets a clear "start a new conversation" error instead of a silently truncated prompt. cache_prompt reuses the KV cache for the stable prefix, which is most of the latency on a Pi. - context_size plumbing fixed: selecting a model adopts its catalog default (the 4B was running at 8k instead of 16k), and llama-server runs --parallel 1 so the single user gets the whole context. - temperature is configurable in ai.yaml (default 0.3 — small models call tools far more reliably near 0.2-0.3 than at server defaults). - One worked tool-use example per mode is prepended to every conversation; small models imitate far better than they obey. - scripts/ai_eval.py scores the assistant against a live WROLPi on mechanical checks (finished, called the right tool first, included a WROLPi link, no external URLs) so prompt changes stop being validated by anecdote. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
82e11cbaba
commit
ced78e181f
11 changed files with 502 additions and 7 deletions
|
|
@ -28,8 +28,10 @@ from wrolpi.common import aiohttp_post, logger
|
|||
from wrolpi.errors import APIError, ValidationError
|
||||
|
||||
from . import agent, service
|
||||
from .catalog import get_effective_context_size
|
||||
from .config import get_ai_config
|
||||
from .modes import MODE_SUGGESTIONS, MODES
|
||||
from .context import get_library_context
|
||||
from .modes import MODE_EXAMPLES, MODE_SUGGESTIONS, MODES
|
||||
|
||||
chat_bp = Blueprint('AIChat', url_prefix='/api/chat')
|
||||
|
||||
|
|
@ -38,6 +40,26 @@ logger = logger.getChild(__name__)
|
|||
# The whole conversation turn (all tool calls and answers) must finish within this.
|
||||
CHAT_TIMEOUT_SECONDS = 600
|
||||
FALLBACK_MESSAGE = "I couldn't complete that request. Try rephrasing, or a more specific question."
|
||||
# Tokens kept free for the model's reply when budgeting the prompt.
|
||||
REPLY_RESERVE_TOKENS = 768
|
||||
CONTEXT_FULL_MESSAGE = ('This conversation no longer fits the model\'s context.'
|
||||
' Start a new conversation and ask a more specific question.')
|
||||
|
||||
|
||||
def estimate_tokens(messages: List[dict], tools: List[dict]) -> int:
|
||||
"""A deliberate overestimate (~4 chars/token) of what one completion request costs."""
|
||||
return (len(json.dumps(messages)) + len(json.dumps(tools, default=str))) // 4
|
||||
|
||||
|
||||
def trim_history(history: List[dict], fixed: List[dict], tools: List[dict], budget: int) -> List[dict]:
|
||||
"""Drop the oldest conversation turns until fixed + history fits the budget.
|
||||
|
||||
`fixed` (system prompt, library map, few-shot example, and the current user message) is never
|
||||
trimmed; old turns are the least valuable tokens in the request."""
|
||||
history = list(history)
|
||||
while history and estimate_tokens(fixed + history, tools) > budget:
|
||||
history.pop(0)
|
||||
return history
|
||||
|
||||
|
||||
class AIDisabled(APIError):
|
||||
|
|
@ -67,7 +89,16 @@ async def llama_chat_stream(messages: List[dict], tools: List[dict]) -> AsyncGen
|
|||
Yields {'type': 'token', 'content': str} for answer fragments, then exactly one
|
||||
{'type': 'stop', 'tool_calls': [...]} where tool_calls is empty for a final answer.
|
||||
Patched in tests."""
|
||||
body = dict(messages=messages, stream=True)
|
||||
config = get_ai_config()
|
||||
body = dict(
|
||||
messages=messages,
|
||||
stream=True,
|
||||
# Low temperature: tool calls from small models are far more reliable near 0.2-0.3.
|
||||
temperature=config.temperature,
|
||||
# Reuse the KV cache for the stable prefix (system prompt + example); on slow CPUs
|
||||
# re-processing that prefix is most of the latency.
|
||||
cache_prompt=True,
|
||||
)
|
||||
if tools:
|
||||
body['tools'] = tools
|
||||
|
||||
|
|
@ -142,8 +173,25 @@ async def post_chat(request: Request, body: ChatRequest):
|
|||
await _send_event(response, 'error', dict(message='The AI service could not be started.'))
|
||||
return
|
||||
|
||||
messages = [dict(role='system', content=mode['system_prompt'])]
|
||||
messages += [dict(role=m.get('role', 'user'), content=m.get('content', '')) for m in body.messages]
|
||||
# System prompt + the live library map, so exact names, counts, and today's date are in
|
||||
# context before the first token; then the mode's worked example, then the conversation.
|
||||
system_content = mode['system_prompt']
|
||||
if library_context := get_library_context():
|
||||
system_content += f'\n\n{library_context}'
|
||||
fixed = [dict(role='system', content=system_content)]
|
||||
fixed += MODE_EXAMPLES.get(body.mode, [])
|
||||
|
||||
history = [dict(role=m.get('role', 'user'), content=m.get('content', '')) for m in body.messages]
|
||||
# The current user message is never trimmed away.
|
||||
current, history = history[-1], history[:-1]
|
||||
fixed.append(current)
|
||||
|
||||
budget = get_effective_context_size() - REPLY_RESERVE_TOKENS
|
||||
history = trim_history(history, fixed, tools, budget)
|
||||
if estimate_tokens(fixed + history, tools) > budget:
|
||||
await _send_event(response, 'error', dict(message=CONTEXT_FULL_MESSAGE))
|
||||
return
|
||||
messages = fixed[:-1] + history + [current]
|
||||
|
||||
final_answer = []
|
||||
for _ in range(max_tool_calls + 1):
|
||||
|
|
@ -151,6 +199,11 @@ async def post_chat(request: Request, body: ChatRequest):
|
|||
await _send_event(response, 'error', dict(message=FALLBACK_MESSAGE))
|
||||
return
|
||||
|
||||
# Tool results grow the request; refuse to overflow the model's context mid-loop.
|
||||
if estimate_tokens(messages, tools) > budget:
|
||||
await _send_event(response, 'error', dict(message=CONTEXT_FULL_MESSAGE))
|
||||
return
|
||||
|
||||
service.record_llm_activity()
|
||||
# Always send something before the model call: on slow CPUs the first token can be
|
||||
# minutes away, and a silent stream looks dead to the UI and to proxies.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue