mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 10:51:36 -04:00
73fafcd645
The file_group.textsearch tsvector indexes all four text columns;
d_text (captions, document text, zip contents) is ~90% of the data,
making ts_rank searches take seconds on large libraries. Add a
textsearch_abc generated column (title/author/description only, ~8x
smaller) and default all searches to it, with deep=true opting into the
full column.
Database:
- Migration adds the generated column and a GIN index (rewrites the
table; a few minutes on large libraries).
Backend:
- FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and
DocSearchRequest accept `deep`; each search function picks the column.
- /api/search_file_estimates returns both counts in one index-bound
query (COUNT FILTER on the abc subset): `file_groups` (fast) and
`file_groups_deep`.
UI (one coherent story across the count surfaces):
- The typing dropdown always suggests the deep count (what is findable).
- The tab badge shows the active mode's count so it matches the result
list.
- The gap is reconciled by a hint offering deep search: count-aware on
the global search ("No results in fast search. N results available in
deep search." / "N more results available..."), generic on the module
pages (no deep estimate there). The hint resets pagination.
- "Search Depth" toggle in the search filter modals, persisted as
?deep=true.
Tests that relied on d_text matches now pass deep=True, with fast-mode
assertions alongside. Also skip flaky test_download_channel on
CircleCI.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
60 lines
1.1 KiB
Python
60 lines
1.1 KiB
Python
from dataclasses import dataclass, field
|
|
from typing import List, Optional
|
|
|
|
|
|
@dataclass
|
|
class ArchiveDict:
|
|
id: int
|
|
url_id: int
|
|
domain_id: int
|
|
singlefile_path: str
|
|
readability_path: str
|
|
readability_json_path: str
|
|
readability_txt_path: str
|
|
screenshot_path: str
|
|
title: str
|
|
|
|
|
|
@dataclass
|
|
class ArchiveSearchRequest:
|
|
search_str: Optional[str] = None
|
|
domain: Optional[str] = None
|
|
offset: Optional[int] = None
|
|
limit: Optional[int] = None
|
|
order_by: Optional[str] = None
|
|
tag_names: List[str] = field(default_factory=list)
|
|
headline: bool = False
|
|
deep: bool = False # When True, search all text including page contents (d_text)
|
|
|
|
|
|
@dataclass
|
|
class ArchiveSearchResponse:
|
|
file_groups: List[ArchiveDict]
|
|
totals: dict
|
|
|
|
|
|
@dataclass
|
|
class ArchiveFileFormatRequest:
|
|
archive_file_format: str
|
|
|
|
|
|
@dataclass
|
|
class ArchiveStatistics:
|
|
archives: int
|
|
week: int
|
|
month: int
|
|
year: int
|
|
sum_size: int
|
|
max_size: int
|
|
|
|
|
|
@dataclass
|
|
class DomainStatistics:
|
|
domains: int
|
|
tagged_domains: int
|
|
|
|
|
|
@dataclass
|
|
class ArchiveStatisticsResponse:
|
|
archives: ArchiveStatistics
|
|
domains: DomainStatistics
|