mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Search: fast ABC mode by default, opt-in deep search
The file_group.textsearch tsvector indexes all four text columns;
d_text (captions, document text, zip contents) is ~90% of the data,
making ts_rank searches take seconds on large libraries. Add a
textsearch_abc generated column (title/author/description only, ~8x
smaller) and default all searches to it, with deep=true opting into the
full column.
Database:
- Migration adds the generated column and a GIN index (rewrites the
table; a few minutes on large libraries).
Backend:
- FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and
DocSearchRequest accept `deep`; each search function picks the column.
- /api/search_file_estimates returns both counts in one index-bound
query (COUNT FILTER on the abc subset): `file_groups` (fast) and
`file_groups_deep`.
UI (one coherent story across the count surfaces):
- The typing dropdown always suggests the deep count (what is findable).
- The tab badge shows the active mode's count so it matches the result
list.
- The gap is reconciled by a hint offering deep search: count-aware on
the global search ("No results in fast search. N results available in
deep search." / "N more results available..."), generic on the module
pages (no deep estimate there). The hint resets pagination.
- "Search Depth" toggle in the search filter modals, persisted as
?deep=true.
Tests that relied on d_text matches now pass deep=True, with fast-mode
assertions alongside. Also skip flaky test_download_channel on
CircleCI.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
1522022330
commit
73fafcd645
33 changed files with 385 additions and 77 deletions
|
|
@ -56,6 +56,7 @@ async def search_docs(_: Request, body: schema.DocSearchRequest):
|
|||
offset=offset,
|
||||
order_by=body.order_by,
|
||||
tag_names=body.tag_names,
|
||||
deep=body.deep,
|
||||
)
|
||||
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
|
||||
return json_response(ret)
|
||||
|
|
|
|||
|
|
@ -347,15 +347,18 @@ def _get_doc(session, file_group_id: int):
|
|||
|
||||
|
||||
def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None,
|
||||
limit=20, offset=0, order_by='published_datetime', tag_names=None):
|
||||
limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False):
|
||||
from .models import Doc
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
# `textsearch` includes d_text (document contents); `textsearch_abc` is much smaller and faster.
|
||||
ts_col = FileGroup.textsearch if deep else FileGroup.textsearch_abc
|
||||
|
||||
with get_db_session() as session:
|
||||
query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id)
|
||||
|
||||
if search_str:
|
||||
query = query.filter(FileGroup.textsearch.op('@@')(func.websearch_to_tsquery(search_str)))
|
||||
query = query.filter(ts_col.op('@@')(func.websearch_to_tsquery(search_str)))
|
||||
|
||||
if author:
|
||||
query = query.filter(FileGroup.b_text.ilike(f'%{author}%'))
|
||||
|
|
@ -382,7 +385,7 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime
|
|||
# Ordering.
|
||||
if order_by == 'rank' and search_str:
|
||||
query = query.order_by(
|
||||
desc(func.ts_rank(FileGroup.textsearch, func.websearch_to_tsquery(search_str))),
|
||||
desc(func.ts_rank(ts_col, func.websearch_to_tsquery(search_str))),
|
||||
desc(FileGroup.id),
|
||||
)
|
||||
elif order_by == 'published_datetime':
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ class DocSearchRequest:
|
|||
order_by: Optional[str] = 'published_datetime'
|
||||
limit: int = 20
|
||||
offset: int = 0
|
||||
deep: bool = False # When True, search all text including document contents (d_text)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
|
|
|
|||
|
|
@ -289,7 +289,11 @@ async def test_search_docs_attaches_section_hint_epub(test_session, test_directo
|
|||
(2, 'Chapter 2', 'unrelated content about pottery'),
|
||||
])
|
||||
|
||||
# "mullen" is only in the document body (d_text), so a deep search is required to match it.
|
||||
results, total = _search_docs(search_str='mullen', mimetype='application/epub')
|
||||
assert total == 0, 'Document contents are not searched unless deep=True'
|
||||
|
||||
results, total = _search_docs(search_str='mullen', mimetype='application/epub', deep=True)
|
||||
assert total == 1
|
||||
assert results[0]['id'] == fg.id
|
||||
hint = results[0]['section_hint']
|
||||
|
|
@ -311,7 +315,7 @@ async def test_search_docs_attaches_section_hint_pdf(test_session, test_director
|
|||
(3, 'Page 3', 'here is where mullen shows up'),
|
||||
])
|
||||
|
||||
results, total = _search_docs(search_str='mullen', mimetype='application/pdf')
|
||||
results, total = _search_docs(search_str='mullen', mimetype='application/pdf', deep=True)
|
||||
assert total == 1
|
||||
assert results[0]['id'] == fg.id
|
||||
hint = results[0]['section_hint']
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue