diff --git a/modules/ai/chat.py b/modules/ai/chat.py index 43166e08..0407a032 100644 --- a/modules/ai/chat.py +++ b/modules/ai/chat.py @@ -152,6 +152,9 @@ async def post_chat(request: Request, body: ChatRequest): return service.record_llm_activity() + # Always send something before the model call: on slow CPUs the first token can be + # minutes away, and a silent stream looks dead to the UI and to proxies. + await _send_event(response, 'status', dict(message='Thinking…')) tool_calls = [] final_answer = [] async for event in llama_chat_stream(messages, tools): diff --git a/modules/ai/test/test_chat_api.py b/modules/ai/test/test_chat_api.py index cdb8b13f..60525e3a 100644 --- a/modules/ai/test/test_chat_api.py +++ b/modules/ai/test/test_chat_api.py @@ -92,7 +92,8 @@ async def test_chat_tool_call_then_answer(async_client, ai_enabled, video_factor assert response.status_code == HTTPStatus.OK events = _parse_sse(response.text) - names = [e for e, _ in events] + # A status event precedes every model call so slow CPUs never leave the stream silent. + names = [e for e, _ in events if e != 'status'] assert names == ['tool_call', 'tool_result', 'token', 'token', 'done'] tool_call = dict(events)['tool_call'] assert tool_call == dict(tool='search_videos', args=dict(search_str='canning')) @@ -116,7 +117,7 @@ async def test_chat_plain_answer(async_client, ai_enabled): body = dict(mode='help', messages=[dict(role='user', content='hi')]) with patch: request, response = await async_client.post('/api/chat/', content=json.dumps(body)) - events = _parse_sse(response.text) + events = [e for e in _parse_sse(response.text) if e[0] != 'status'] assert events == [('token', dict(content='Hello!')), ('done', dict(content='Hello!'))] diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 5fd2d82b..b411a657 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -51,6 +51,9 @@ api_app.config.FALLBACK_ERROR_FORMAT = 'json' # Caddy is the single proxy in front of the API; trust its X-Forwarded-For so request.remote_addr # is the browser's address (used by the speed test). api_app.config.PROXIES_COUNT = 1 +# The AI chat streams for as long as CHAT_TIMEOUT_SECONDS (its own hard ceiling); Sanic's 60s +# default killed the stream before a slow CPU produced its first token. +api_app.config.RESPONSE_TIMEOUT = 660 api_bp = Blueprint('RootAPI', url_prefix='/api')