ai: ground the librarian — collection links, retry hints, no source URLs

Live testing surfaced three ways small models went wrong summarizing a
library: they gave up after one empty search (misspelled channel names
found nothing), so zero-result searches now carry a retry hint and the
Research prompt requires list_collections before channel/site questions
and search_videos(channel_id) before claiming a channel is summarized;
they presented the familiar YouTube source URL instead of the WROLPi
link, so listings omit the source URL (archive detail keeps it); and
they invented links for channels, so channel collections now carry
their real /videos/channel page link.  Search descriptions state that
search_str is optional for browsing the newest items.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

modules/ai/api.py

@@ -52,7 +52,7 @@ def _offset(request: Request) -> int:
 @openapi.definition(
     summary='Search all WROLPi content',
     description='Search everything in the library at once: videos, archived web pages, ebooks, PDFs, and other'
-                ' files. Returns lean results with a WROLPi link for each item. Prefer the type-specific search'
+                ' files. search_str is optional: omit it to browse the newest items. Prefer the type-specific search'
                 ' (videos/archives/docs) when the user asks about one kind of content. When total is large,'
                 ' narrow the query instead of paging.',
     body=schema.AISearchRequest,
@@ -69,15 +69,15 @@ async def search_all(_: Request, body: schema.AISearchRequest):
         tag_names=body.tag_names,
         headline=True,
     )
-    return json_response(lib.format_file_groups(file_groups, total))
+    return json_response(lib.format_file_groups(file_groups, total, searched=bool(body.search_str)))
 
 
 @ai_bp.post('/videos/search')
 @openapi.definition(
     summary='Search videos',
     description='Search downloaded videos (and audio) by title and captions. Returns lean results with a'
-                ' WROLPi link and a captions_link for each video. When total is large, narrow the query'
-                ' instead of paging.',
+                ' WROLPi link and a captions_link for each video. search_str is optional: omit it to browse the'
+                ' newest videos, or pass channel_id (from list_collections) to browse one channel.',
     body=schema.AIVideoSearchRequest,
 )
 @openapi.operation('search_videos')
@@ -92,7 +92,7 @@ async def search_videos(_: Request, body: schema.AIVideoSearchRequest):
         tag_names=body.tag_names,
         headline=True,
     )
-    return json_response(lib.format_file_groups(file_groups, total))
+    return json_response(lib.format_file_groups(file_groups, total, searched=bool(body.search_str)))
 
 
 @ai_bp.get('/videos/<file_group_id:int>')
@@ -152,8 +152,8 @@ async def get_video_comments(request: Request, file_group_id: int):
 @openapi.definition(
     summary='Search archived web pages',
     description='Search archived web pages (saved with SingleFile) by title and content. Returns lean results'
-                ' with a WROLPi link and a text_link to read each page. Filter with domain (e.g. "example.com")'
-                ' when the user asks about one site.',
+                ' with a WROLPi link and a text_link to read each page. search_str is optional: omit it to browse'
+                ' the newest pages. Filter with domain (the exact name from list_collections) for one site.',
     body=schema.AIArchiveSearchRequest,
 )
 @openapi.operation('search_archives')
@@ -169,7 +169,7 @@ async def search_archives(_: Request, body: schema.AIArchiveSearchRequest):
         body.tag_names,
         headline=True,
     )
-    return json_response(lib.format_file_groups(file_groups, total))
+    return json_response(lib.format_file_groups(file_groups, total, searched=bool(body.search_str)))
 
 
 @ai_bp.get('/archives/<file_group_id:int>')
@@ -183,7 +183,7 @@ async def search_archives(_: Request, body: schema.AIArchiveSearchRequest):
 async def get_archive(request: Request, file_group_id: int):
     archive = archive_lib.get_archive_by_file_group_id(request.ctx.session, file_group_id, skip_viewed=True)
     result = lib.format_file_group(archive.file_group.__json__(),
-                                   description_length=lib.DETAIL_DESCRIPTION_LENGTH)
+                                   description_length=lib.DETAIL_DESCRIPTION_LENGTH, include_url=True)
     history = []
     for snapshot in archive.history[:10]:
         fg = snapshot.file_group.__json__()
@@ -239,7 +239,7 @@ async def search_docs(_: Request, body: schema.AIDocSearchRequest):
         order_by='rank' if body.search_str else 'published_datetime',
         tag_names=body.tag_names,
     )
-    return json_response(lib.format_file_groups(file_groups, total))
+    return json_response(lib.format_file_groups(file_groups, total, searched=bool(body.search_str)))
 
 
 @ai_bp.get('/docs/<file_group_id:int>')
@@ -347,11 +347,14 @@ async def list_collections(request: Request):
     collections = search_collections(request.ctx.session, kind=kind)
     results = []
     for collection in collections:
+        # Channels have a browsable page; a real link stops the model inventing one.
+        link = f"/videos/channel/{collection['id']}/video" if collection.get('kind') == 'channel' else None
         results.append({k: v for k, v in dict(
             id=collection.get('id'),
             name=collection.get('name'),
             kind=collection.get('kind'),
             directory=collection.get('directory'),
+            link=link,
         ).items() if v is not None})
     return json_response(dict(results=results, total=len(results)))
 

modules/ai/lib.py

@@ -78,10 +78,13 @@ def _truncate(text: Optional[str], length: int) -> Optional[str]:
     return text
 
 
-def format_file_group(fg: dict, description_length: int = LISTING_DESCRIPTION_LENGTH) -> dict:
+def format_file_group(fg: dict, description_length: int = LISTING_DESCRIPTION_LENGTH,
+                      include_url: bool = False) -> dict:
     """Format a FileGroup JSON dict into the lean, flat shape the model consumes.
 
-    Empty fields are omitted so results stay compact on small contexts.
+    Empty fields are omitted so results stay compact on small contexts.  The source URL is
+    omitted unless requested (archive detail): given both, small models present the familiar
+    external URL instead of the WROLPi link.
     """
     kind = file_group_kind(fg)
     result = dict(
@@ -94,7 +97,7 @@ def format_file_group(fg: dict, description_length: int = LISTING_DESCRIPTION_LE
         published=fg.get('published_datetime'),
         author=fg.get('author'),
         tags=fg.get('tags') or None,
-        url=fg.get('url'),
+        url=fg.get('url') if include_url else None,
     )
 
     # FTS headline (content match context), if the search requested headlines.
@@ -127,9 +130,15 @@ def format_file_group(fg: dict, description_length: int = LISTING_DESCRIPTION_LE
     return {k: v for k, v in result.items() if v is not None}
 
 
-def format_file_groups(file_groups: list, total: int) -> dict:
+def format_file_groups(file_groups: list, total: int, searched: bool = False) -> dict:
     """Format search results: lean items plus the total so the model narrows instead of paging."""
-    return dict(results=[format_file_group(i) for i in file_groups], total=total)
+    result = dict(results=[format_file_group(i) for i in file_groups], total=total)
+    if searched and total == 0:
+        # Small models give up on one empty search; steer the retry.
+        result['hint'] = ('No matches. Names may be spelled differently: try fewer or different'
+                          ' words, list_collections for exact channel/domain names, or omit'
+                          ' search_str to browse the newest items.')
+    return result
 
 
 def paginate_text(text: Optional[str], offset: int = 0) -> dict:

modules/ai/modes.py

@@ -30,9 +30,18 @@ MODES = {
             f'{COMMON_PROMPT}'
             ' Your job is to be the librarian: find and summarize content from the user\'s library.'
             ' Search first, then read captions, page text, or entries when the user wants detail.'
+            ' When the user asks about a channel, website, or their whole library, call'
+            ' list_collections first to learn the exact names (spellings often differ).'
+            ' list_collections only gives names — it never tells you what content exists.  You have'
+            ' not summarized a channel until you have called search_videos with its channel_id (and'
+            ' NO search_str) and read the resulting titles.'
+            ' To browse everything of one kind (e.g. "summarize my archives"), call the search tool'
+            ' WITHOUT search_str — that lists the newest items.'
+            ' When a search finds nothing, do not give up: retry with fewer or different words, or'
+            ' browse with list_collections.'
             ' ALWAYS include the WROLPi link for every item you mention, exactly as the tools return it.'
             ' Never give external URLs.  When a search returns a large total, narrow the query instead'
-            ' of paging.  If nothing matches, say so.'
+            ' of paging.'
         ),
         tools=(
             'search_all',

modules/ai/test/test_lib.py

@@ -70,7 +70,9 @@ def test_format_file_group_archive():
     assert result['kind'] == 'archive'
     assert result['link'] == '/archives/8'
     assert result['text_link'] == '/api/ai/archives/8/text'
-    assert result['url'] == 'https://example.com/a'
+    # Listings omit the source URL so small models present the WROLPi link instead.
+    assert 'url' not in result
+    assert lib.format_file_group(fg, include_url=True)['url'] == 'https://example.com/a'
 
 
 def test_format_file_groups_total():