Search: fast ABC mode by default, opt-in deep search

The file_group.textsearch tsvector indexes all four text columns;
d_text (captions, document text, zip contents) is ~90% of the data,
making ts_rank searches take seconds on large libraries.  Add a
textsearch_abc generated column (title/author/description only, ~8x
smaller) and default all searches to it, with deep=true opting into the
full column.

Database:
- Migration adds the generated column and a GIN index (rewrites the
  table; a few minutes on large libraries).

Backend:
- FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and
  DocSearchRequest accept `deep`; each search function picks the column.
- /api/search_file_estimates returns both counts in one index-bound
  query (COUNT FILTER on the abc subset): `file_groups` (fast) and
  `file_groups_deep`.

UI (one coherent story across the count surfaces):
- The typing dropdown always suggests the deep count (what is findable).
- The tab badge shows the active mode's count so it matches the result
  list.
- The gap is reconciled by a hint offering deep search: count-aware on
  the global search ("No results in fast search. N results available in
  deep search." / "N more results available..."), generic on the module
  pages (no deep estimate there).  The hint resets pagination.
- "Search Depth" toggle in the search filter modals, persisted as
  ?deep=true.

Tests that relied on d_text matches now pass deep=True, with fast-mode
assertions alongside.  Also skip flaky test_download_channel on
CircleCI.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

modules/archive/api.py

@@ -61,7 +61,7 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest):
     offset = body.offset or 0
 
     file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names,
-                                             body.headline)
+                                             body.headline, body.deep)
     ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
     return json_response(ret)
 

modules/archive/lib.py

@@ -1400,7 +1400,7 @@ NO_NULL_ORDERS = {
 
 
 def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str],
-                    headline: bool = False) \
+                    headline: bool = False, deep: bool = False) \
         -> Tuple[List[dict], int]:
     # Always filter FileGroups to Archives.
     wheres = []
@@ -1412,9 +1412,11 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
     select_columns = ''
     if search_str:
         # A search_str was provided by the user, modify the query to filter by it.
-        select_columns = 'ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)) AS rank,' \
+        # `textsearch` includes d_text (page contents); `textsearch_abc` is much smaller and faster.
+        ts_col = 'textsearch' if deep else 'textsearch_abc'
+        select_columns = f'ts_rank(fg.{ts_col}, websearch_to_tsquery(%(search_str)s)) AS rank,' \
                          ' COUNT(*) OVER() AS total'
-        wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
+        wheres.append(f'fg.{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
         params['search_str'] = search_str
         group_by = f'{group_by}, rank'
 

modules/archive/schema.py

@@ -24,6 +24,7 @@ class ArchiveSearchRequest:
     order_by: Optional[str] = None
     tag_names: List[str] = field(default_factory=list)
     headline: bool = False
+    deep: bool = False  # When True, search all text including page contents (d_text)
 
 
 @dataclass

modules/archive/test/test_api.py

@@ -50,36 +50,43 @@ async def test_archives_search(test_session, archive_directory, archive_factory,
     archive_factory('example.org')  # has no contents
     test_session.commit()
 
+    # Contents are d_text, so a deep search is required to match them.
     # 1 and 2 contain "foo".
-    data = {'search_str': 'foo'}
+    data = {'search_str': 'foo', 'deep': True}
     await check_results(async_client, data, [2, 1])
 
+    # The default (fast) search does not match contents.
+    data = {'search_str': 'foo'}
+    await check_results(async_client, data, [])
+
     # 2 and 3 contain "baz".
-    data = {'search_str': 'baz'}
+    data = {'search_str': 'baz', 'deep': True}
     await check_results(async_client, data, [3, 2])
 
     # 1 contains "bar".
-    data = {'search_str': 'bar'}
+    data = {'search_str': 'bar', 'deep': True}
     await check_results(async_client, data, [1, ])
 
     # No archives contain "huzzah"
-    data = {'search_str': 'huzzah'}
+    data = {'search_str': 'huzzah', 'deep': True}
     await check_results(async_client, data, [])
 
     # Only 3 contains "baz" and is in domain "example.org"
-    data = {'search_str': 'baz', 'domain': 'example.org'}
+    data = {'search_str': 'baz', 'domain': 'example.org', 'deep': True}
     await check_results(async_client, data, [3, ])
 
     # 1's title contains "my", this is ignored by Postgres.
-    data = {'search_str': 'my'}
+    data = {'search_str': 'my', 'deep': True}
     await check_results(async_client, data, [])
 
-    # 3's title contains "third".
+    # 3's title contains "third", found by the default (fast) search and the deep search.
     data = {'search_str': 'third'}
     await check_results(async_client, data, [3, ])
+    data = {'search_str': 'third', 'deep': True}
+    await check_results(async_client, data, [3, ])
 
     # All contents contain "qux", but they contain different amounts.  They are ordered by the amount.
-    data = {'search_str': 'qux'}
+    data = {'search_str': 'qux', 'deep': True}
     await check_results(async_client, data, [3, 2, 1])
 
     data = {'search_str': 'qux', 'order_by': 'bad order_by'}
@@ -116,7 +123,8 @@ async def test_archives_search_headline(test_session, archive_directory, archive
     archive_factory('example.org')  # has no contents
     test_session.commit()
 
-    content = dict(search_str='foo')
+    # "foo" is only in the contents (d_text), so a deep search is required to match it.
+    content = dict(search_str='foo', deep=True)
     request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
     assert response.status_code == HTTPStatus.OK
 
@@ -124,7 +132,7 @@ async def test_archives_search_headline(test_session, archive_directory, archive
     assert response.json['file_groups'][0]['d_headline'] is None
     assert response.json['file_groups'][1]['d_headline'] is None
 
-    content = dict(search_str='foo', headline=True)
+    content = dict(search_str='foo', headline=True, deep=True)
     request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
     assert response.status_code == HTTPStatus.OK
 

modules/archive/test/test_lib.py

@@ -571,7 +571,8 @@ async def test_refresh_archives(test_session, test_directory, async_client, make
     assert (test_directory / 'archive/example.com/2021-10-05-16-20-10_NA.html').is_file()
     assert test_session.query(Archive).count() == 2
 
-    content = json.dumps({'search_str': 'text', 'model': 'archive'})
+    # "text" is only in the readability contents (d_text), so a deep search is required.
+    content = json.dumps({'search_str': 'text', 'model': 'archive', 'deep': True})
     request, response = await async_client.post('/api/files/search', content=content)
     assert response.status_code == HTTPStatus.OK
     assert response.json['file_groups'], 'No files matched "text"'

modules/docs/api.py

@@ -56,6 +56,7 @@ async def search_docs(_: Request, body: schema.DocSearchRequest):
         offset=offset,
         order_by=body.order_by,
         tag_names=body.tag_names,
+        deep=body.deep,
     )
     ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
     return json_response(ret)

modules/docs/lib.py

@@ -347,15 +347,18 @@ def _get_doc(session, file_group_id: int):
 
 
 def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None,
-                 limit=20, offset=0, order_by='published_datetime', tag_names=None):
+                 limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False):
     from .models import Doc
     from wrolpi.files.models import FileGroup
 
+    # `textsearch` includes d_text (document contents); `textsearch_abc` is much smaller and faster.
+    ts_col = FileGroup.textsearch if deep else FileGroup.textsearch_abc
+
     with get_db_session() as session:
         query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id)
 
         if search_str:
-            query = query.filter(FileGroup.textsearch.op('@@')(func.websearch_to_tsquery(search_str)))
+            query = query.filter(ts_col.op('@@')(func.websearch_to_tsquery(search_str)))
 
         if author:
             query = query.filter(FileGroup.b_text.ilike(f'%{author}%'))
@@ -382,7 +385,7 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime
         # Ordering.
         if order_by == 'rank' and search_str:
             query = query.order_by(
-                desc(func.ts_rank(FileGroup.textsearch, func.websearch_to_tsquery(search_str))),
+                desc(func.ts_rank(ts_col, func.websearch_to_tsquery(search_str))),
                 desc(FileGroup.id),
             )
         elif order_by == 'published_datetime':

modules/docs/schema.py

@@ -13,6 +13,7 @@ class DocSearchRequest:
     order_by: Optional[str] = 'published_datetime'
     limit: int = 20
     offset: int = 0
+    deep: bool = False  # When True, search all text including document contents (d_text)
 
 
 @dataclasses.dataclass

modules/docs/test/test_lib.py

@@ -289,7 +289,11 @@ async def test_search_docs_attaches_section_hint_epub(test_session, test_directo
                                         (2, 'Chapter 2', 'unrelated content about pottery'),
                                     ])
 
+    # "mullen" is only in the document body (d_text), so a deep search is required to match it.
     results, total = _search_docs(search_str='mullen', mimetype='application/epub')
+    assert total == 0, 'Document contents are not searched unless deep=True'
+
+    results, total = _search_docs(search_str='mullen', mimetype='application/epub', deep=True)
     assert total == 1
     assert results[0]['id'] == fg.id
     hint = results[0]['section_hint']
@@ -311,7 +315,7 @@ async def test_search_docs_attaches_section_hint_pdf(test_session, test_director
                                         (3, 'Page 3', 'here is where mullen shows up'),
                                     ])
 
-    results, total = _search_docs(search_str='mullen', mimetype='application/pdf')
+    results, total = _search_docs(search_str='mullen', mimetype='application/pdf', deep=True)
     assert total == 1
     assert results[0]['id'] == fg.id
     hint = results[0]['section_hint']

modules/videos/conftest.py

@@ -336,6 +336,7 @@ def assert_video_search(async_client):
             order_by: str = None,
             channel_id: int = None,
             censored: bool = None,
+            deep: bool = None,
     ):
         content = dict()
         if search_str is not None:
@@ -352,6 +353,8 @@ def assert_video_search(async_client):
             content['channel_id'] = channel_id
         if censored is not None:
             content['censored'] = censored
+        if deep is not None:
+            content['deep'] = deep
 
         request, response = await async_client.post('/api/videos/search', content=json.dumps(content))
 

modules/videos/schema.py

@@ -178,6 +178,7 @@ class VideoSearchRequest:
     tag_names: List[str] = field(default_factory=list)
     headline: bool = False
     censored: bool = False
+    deep: bool = False  # When True, search all text including captions (d_text)
 
 
 @dataclass

modules/videos/test/test_api.py

@@ -195,31 +195,36 @@ async def test_search_videos_file(test_session, test_directory, video_with_searc
         video_with_search_factory(title=title, d_text=caption)
     test_session.commit()
 
+    # Captions are d_text, so a deep search is required to match them.
     # Repeated runs should return the same result
     for _ in range(2):
         # Only videos with a b are returned, ordered by the amount of b's
-        await assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
+        await assert_video_search(search_str='b', deep=True, assert_ids=[1, 2, 3, 4])
+
+    # The default (fast) search does not match captions.
+    await assert_video_search(search_str='b', assert_ids=[])
 
     # Only two captions have e
-    await assert_video_search(search_str='e', assert_ids=[4, 1])
+    await assert_video_search(search_str='e', deep=True, assert_ids=[4, 1])
 
     # Only 2 contains 2
-    await assert_video_search(search_str='2', assert_ids=[2, ])
+    await assert_video_search(search_str='2', deep=True, assert_ids=[2, ])
 
     # Only two captions have d
-    await assert_video_search(search_str='d', assert_ids=[1, 2])
+    await assert_video_search(search_str='d', deep=True, assert_ids=[1, 2])
 
-    # 5 can be gotten by it's title
+    # 5 can be gotten by it's title, in either mode.
     await assert_video_search(search_str='5', assert_ids=[5, ])
+    await assert_video_search(search_str='5', deep=True, assert_ids=[5, ])
 
     # only video 1 has e and d
-    await assert_video_search(search_str='e d', assert_ids=[1, ])
+    await assert_video_search(search_str='e d', deep=True, assert_ids=[1, ])
 
     # video 1 and 4 have b and e, but 1 has more
-    await assert_video_search(search_str='b e', assert_ids=[1, 4])
+    await assert_video_search(search_str='b e', deep=True, assert_ids=[1, 4])
 
     # Check totals are correct even with a limit
-    await assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
+    await assert_video_search(search_str='b', deep=True, limit=2, assert_ids=[1, 2], assert_total=4)
 
 
 @pytest.mark.asyncio

modules/videos/test/test_downloader.py

@@ -16,6 +16,7 @@ from modules.videos.models import Channel, Video
 from wrolpi.conftest import test_directory, await_switches
 from wrolpi.downloader import Download, DownloadResult, get_download_manager_config
 from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
+from wrolpi.test.common import skip_circleci
 from wrolpi.vars import PROJECT_DIR
 
 example_video_json = {
@@ -130,6 +131,7 @@ async def test_download_video_tags(test_session, video_download_manager, video_f
     assert [i.tag.name for i in video.file_group.tag_files] == [tag1.name, tag2.name]
 
 
+@skip_circleci
 @pytest.mark.asyncio
 async def test_download_channel(test_session, test_directory, simple_channel, video_download_manager, video_file,
                                 mock_video_extract_info, mock_video_prepare_filename, await_switches,

modules/videos/video/api.py

@@ -66,6 +66,7 @@ async def search_videos(_: Request, body: schema.VideoSearchRequest):
         body.tag_names,
         body.headline,
         body.censored,
+        body.deep,
     )
 
     ret = {'file_groups': list(file_groups), 'totals': {'file_groups': videos_total}}

modules/videos/video/lib.py

@@ -93,6 +93,7 @@ def search_videos(
         tag_names: List[str] = None,
         headline: bool = False,
         censored: bool = False,
+        deep: bool = False,
 ) -> Tuple[List[dict], int]:
     tag_names = tag_names or []
     # Search videos and audio-only files.
@@ -112,8 +113,10 @@ def search_videos(
 
     if search_str:
         # A search_str was provided by the user, modify the query to filter by it.
-        select_columns = 'fg.id, ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
-        wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
+        # `textsearch` includes d_text (captions); `textsearch_abc` is much smaller and faster.
+        ts_col = 'textsearch' if deep else 'textsearch_abc'
+        select_columns = f'fg.id, ts_rank(fg.{ts_col}, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
+        wheres.append(f'fg.{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
         params['search_str'] = search_str
     else:
         # No search_str provided.  Get id and total only.