From d612b03577ec64215c0e6bdb6570e7a8cb566aef Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 27 Aug 2026 12:44:37 -0600 Subject: [PATCH] =?UTF-8?q?ai:=20ground=20the=20librarian=20=E2=80=94=20co?= =?UTF-8?q?llection=20links,=20retry=20hints,=20no=20source=20URLs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live testing surfaced three ways small models went wrong summarizing a library: they gave up after one empty search (misspelled channel names found nothing), so zero-result searches now carry a retry hint and the Research prompt requires list_collections before channel/site questions and search_videos(channel_id) before claiming a channel is summarized; they presented the familiar YouTube source URL instead of the WROLPi link, so listings omit the source URL (archive detail keeps it); and they invented links for channels, so channel collections now carry their real /videos/channel page link. Search descriptions state that search_str is optional for browsing the newest items. Co-Authored-By: Claude Fable 5 --- modules/ai/api.py | 23 +++++++++++++---------- modules/ai/lib.py | 19 ++++++++++++++----- modules/ai/modes.py | 11 ++++++++++- modules/ai/test/test_lib.py | 4 +++- 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/modules/ai/api.py b/modules/ai/api.py index d5e6c56f..cd080a6c 100644 --- a/modules/ai/api.py +++ b/modules/ai/api.py @@ -52,7 +52,7 @@ def _offset(request: Request) -> int: @openapi.definition( summary='Search all WROLPi content', description='Search everything in the library at once: videos, archived web pages, ebooks, PDFs, and other' - ' files. Returns lean results with a WROLPi link for each item. Prefer the type-specific search' + ' files. search_str is optional: omit it to browse the newest items. Prefer the type-specific search' ' (videos/archives/docs) when the user asks about one kind of content. When total is large,' ' narrow the query instead of paging.', body=schema.AISearchRequest, @@ -69,15 +69,15 @@ async def search_all(_: Request, body: schema.AISearchRequest): tag_names=body.tag_names, headline=True, ) - return json_response(lib.format_file_groups(file_groups, total)) + return json_response(lib.format_file_groups(file_groups, total, searched=bool(body.search_str))) @ai_bp.post('/videos/search') @openapi.definition( summary='Search videos', description='Search downloaded videos (and audio) by title and captions. Returns lean results with a' - ' WROLPi link and a captions_link for each video. When total is large, narrow the query' - ' instead of paging.', + ' WROLPi link and a captions_link for each video. search_str is optional: omit it to browse the' + ' newest videos, or pass channel_id (from list_collections) to browse one channel.', body=schema.AIVideoSearchRequest, ) @openapi.operation('search_videos') @@ -92,7 +92,7 @@ async def search_videos(_: Request, body: schema.AIVideoSearchRequest): tag_names=body.tag_names, headline=True, ) - return json_response(lib.format_file_groups(file_groups, total)) + return json_response(lib.format_file_groups(file_groups, total, searched=bool(body.search_str))) @ai_bp.get('/videos/') @@ -152,8 +152,8 @@ async def get_video_comments(request: Request, file_group_id: int): @openapi.definition( summary='Search archived web pages', description='Search archived web pages (saved with SingleFile) by title and content. Returns lean results' - ' with a WROLPi link and a text_link to read each page. Filter with domain (e.g. "example.com")' - ' when the user asks about one site.', + ' with a WROLPi link and a text_link to read each page. search_str is optional: omit it to browse' + ' the newest pages. Filter with domain (the exact name from list_collections) for one site.', body=schema.AIArchiveSearchRequest, ) @openapi.operation('search_archives') @@ -169,7 +169,7 @@ async def search_archives(_: Request, body: schema.AIArchiveSearchRequest): body.tag_names, headline=True, ) - return json_response(lib.format_file_groups(file_groups, total)) + return json_response(lib.format_file_groups(file_groups, total, searched=bool(body.search_str))) @ai_bp.get('/archives/') @@ -183,7 +183,7 @@ async def search_archives(_: Request, body: schema.AIArchiveSearchRequest): async def get_archive(request: Request, file_group_id: int): archive = archive_lib.get_archive_by_file_group_id(request.ctx.session, file_group_id, skip_viewed=True) result = lib.format_file_group(archive.file_group.__json__(), - description_length=lib.DETAIL_DESCRIPTION_LENGTH) + description_length=lib.DETAIL_DESCRIPTION_LENGTH, include_url=True) history = [] for snapshot in archive.history[:10]: fg = snapshot.file_group.__json__() @@ -239,7 +239,7 @@ async def search_docs(_: Request, body: schema.AIDocSearchRequest): order_by='rank' if body.search_str else 'published_datetime', tag_names=body.tag_names, ) - return json_response(lib.format_file_groups(file_groups, total)) + return json_response(lib.format_file_groups(file_groups, total, searched=bool(body.search_str))) @ai_bp.get('/docs/') @@ -347,11 +347,14 @@ async def list_collections(request: Request): collections = search_collections(request.ctx.session, kind=kind) results = [] for collection in collections: + # Channels have a browsable page; a real link stops the model inventing one. + link = f"/videos/channel/{collection['id']}/video" if collection.get('kind') == 'channel' else None results.append({k: v for k, v in dict( id=collection.get('id'), name=collection.get('name'), kind=collection.get('kind'), directory=collection.get('directory'), + link=link, ).items() if v is not None}) return json_response(dict(results=results, total=len(results))) diff --git a/modules/ai/lib.py b/modules/ai/lib.py index b780b333..dbd648fd 100644 --- a/modules/ai/lib.py +++ b/modules/ai/lib.py @@ -78,10 +78,13 @@ def _truncate(text: Optional[str], length: int) -> Optional[str]: return text -def format_file_group(fg: dict, description_length: int = LISTING_DESCRIPTION_LENGTH) -> dict: +def format_file_group(fg: dict, description_length: int = LISTING_DESCRIPTION_LENGTH, + include_url: bool = False) -> dict: """Format a FileGroup JSON dict into the lean, flat shape the model consumes. - Empty fields are omitted so results stay compact on small contexts. + Empty fields are omitted so results stay compact on small contexts. The source URL is + omitted unless requested (archive detail): given both, small models present the familiar + external URL instead of the WROLPi link. """ kind = file_group_kind(fg) result = dict( @@ -94,7 +97,7 @@ def format_file_group(fg: dict, description_length: int = LISTING_DESCRIPTION_LE published=fg.get('published_datetime'), author=fg.get('author'), tags=fg.get('tags') or None, - url=fg.get('url'), + url=fg.get('url') if include_url else None, ) # FTS headline (content match context), if the search requested headlines. @@ -127,9 +130,15 @@ def format_file_group(fg: dict, description_length: int = LISTING_DESCRIPTION_LE return {k: v for k, v in result.items() if v is not None} -def format_file_groups(file_groups: list, total: int) -> dict: +def format_file_groups(file_groups: list, total: int, searched: bool = False) -> dict: """Format search results: lean items plus the total so the model narrows instead of paging.""" - return dict(results=[format_file_group(i) for i in file_groups], total=total) + result = dict(results=[format_file_group(i) for i in file_groups], total=total) + if searched and total == 0: + # Small models give up on one empty search; steer the retry. + result['hint'] = ('No matches. Names may be spelled differently: try fewer or different' + ' words, list_collections for exact channel/domain names, or omit' + ' search_str to browse the newest items.') + return result def paginate_text(text: Optional[str], offset: int = 0) -> dict: diff --git a/modules/ai/modes.py b/modules/ai/modes.py index 7c6529a0..823ae3c5 100644 --- a/modules/ai/modes.py +++ b/modules/ai/modes.py @@ -30,9 +30,18 @@ MODES = { f'{COMMON_PROMPT}' ' Your job is to be the librarian: find and summarize content from the user\'s library.' ' Search first, then read captions, page text, or entries when the user wants detail.' + ' When the user asks about a channel, website, or their whole library, call' + ' list_collections first to learn the exact names (spellings often differ).' + ' list_collections only gives names — it never tells you what content exists. You have' + ' not summarized a channel until you have called search_videos with its channel_id (and' + ' NO search_str) and read the resulting titles.' + ' To browse everything of one kind (e.g. "summarize my archives"), call the search tool' + ' WITHOUT search_str — that lists the newest items.' + ' When a search finds nothing, do not give up: retry with fewer or different words, or' + ' browse with list_collections.' ' ALWAYS include the WROLPi link for every item you mention, exactly as the tools return it.' ' Never give external URLs. When a search returns a large total, narrow the query instead' - ' of paging. If nothing matches, say so.' + ' of paging.' ), tools=( 'search_all', diff --git a/modules/ai/test/test_lib.py b/modules/ai/test/test_lib.py index 00826954..fb74f687 100644 --- a/modules/ai/test/test_lib.py +++ b/modules/ai/test/test_lib.py @@ -70,7 +70,9 @@ def test_format_file_group_archive(): assert result['kind'] == 'archive' assert result['link'] == '/archives/8' assert result['text_link'] == '/api/ai/archives/8/text' - assert result['url'] == 'https://example.com/a' + # Listings omit the source URL so small models present the WROLPi link instead. + assert 'url' not in result + assert lib.format_file_group(fg, include_url=True)['url'] == 'https://example.com/a' def test_format_file_groups_total():