diff --git a/alembic/versions/e5f7a9c1d3b6_file_group_textsearch_abc.py b/alembic/versions/e5f7a9c1d3b6_file_group_textsearch_abc.py
new file mode 100644
index 00000000..16207c0d
--- /dev/null
+++ b/alembic/versions/e5f7a9c1d3b6_file_group_textsearch_abc.py
@@ -0,0 +1,43 @@
+"""file_group_textsearch_abc
+
+Add the `textsearch_abc` generated tsvector column to `file_group`. It indexes only a_text (title),
+b_text (author), and c_text (description) — excluding d_text (captions/document body), which is ~90% of all
+text data. Searches default to this much smaller column; the existing `textsearch` column remains for
+opt-in "deep" search.
+
+Adding a STORED generated column rewrites the table; expect a few minutes on large libraries.
+
+Revision ID: e5f7a9c1d3b6
+Revises: d4e6f8a0c2b5
+Create Date: 2026-07-03
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'e5f7a9c1d3b6'
+down_revision = 'd4e6f8a0c2b5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ conn = op.get_bind()
+ conn.execute(sa.text('''
+ ALTER TABLE file_group
+ ADD COLUMN textsearch_abc tsvector
+ GENERATED ALWAYS AS (
+ setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") ||
+ setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") ||
+ setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char")
+ ) STORED
+ '''))
+ conn.execute(sa.text(
+ 'CREATE INDEX IF NOT EXISTS file_group_textsearch_abc_idx ON file_group USING GIN(textsearch_abc)'))
+
+
+def downgrade():
+ conn = op.get_bind()
+ conn.execute(sa.text('DROP INDEX IF EXISTS file_group_textsearch_abc_idx'))
+ conn.execute(sa.text('ALTER TABLE file_group DROP COLUMN IF EXISTS textsearch_abc'))
diff --git a/app/src/DashboardPage.js b/app/src/DashboardPage.js
index 10b0ea93..440b4d31 100644
--- a/app/src/DashboardPage.js
+++ b/app/src/DashboardPage.js
@@ -469,7 +469,7 @@ export function DashboardPage() {
}
return
{getSearchResultsInput(inputProps)}
-
;
};
diff --git a/app/src/api.js b/app/src/api.js
index 74efc88d..3d8a3e4b 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -187,7 +187,7 @@ export async function tagChannelInfo(channelId, tagName) {
});
}
-export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline, censored) {
+export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline, censored, deep) {
// Build a search query to retrieve a list of videos from the API
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
@@ -208,6 +208,9 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by
if (censored) {
body['censored'] = true;
}
+ if (deep) {
+ body['deep'] = true;
+ }
console.debug('searching videos', body);
@@ -697,7 +700,7 @@ export async function saveCatalog(items) {
}
-export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline) {
+export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline, deep) {
// Build a search query to retrieve a list of videos from the API
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
@@ -717,6 +720,9 @@ export async function searchArchives(offset, limit, domain, searchStr, order, ta
if (headline) {
body['headline'] = true;
}
+ if (deep) {
+ body['deep'] = true;
+ }
console.debug('searching archives', body);
const response = await apiPost(`${ARCHIVES_API}/search`, body);
@@ -1212,7 +1218,7 @@ export async function deleteDownload(downloadId) {
}
}
-export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag, order, suffix, path) {
+export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag, order, suffix, path, deep) {
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit), any_tag: anyTag};
if (suffix) {
body['suffix'] = suffix;
@@ -1244,6 +1250,9 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta
if (order) {
body['order'] = order;
}
+ if (deep) {
+ body['deep'] = true;
+ }
console.info('searching files', body);
const response = await apiPost(`${API_URI}/files/search`, body);
@@ -1988,6 +1997,7 @@ export async function searchEstimateFiles(search_str, tagNames, mimetypes, month
return {
fileGroups: content.file_groups,
+ fileGroupsDeep: content.file_groups_deep,
}
} else {
console.error('Failed to get file search estimates!');
@@ -2220,7 +2230,7 @@ export async function getDoc(fileGroupId) {
}
-export async function searchDocs(offset, limit, searchStr, order, tagNames, author, subject, mimetype) {
+export async function searchDocs(offset, limit, searchStr, order, tagNames, author, subject, mimetype, deep) {
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
let body = {offset, limit};
@@ -2242,6 +2252,9 @@ export async function searchDocs(offset, limit, searchStr, order, tagNames, auth
if (mimetype) {
body['mimetype'] = mimetype;
}
+ if (deep) {
+ body['deep'] = true;
+ }
console.debug('searching docs', body);
const response = await apiPost(`${DOCS_API}/search`, body);
diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js
index 67b59837..9416c1b4 100644
--- a/app/src/components/Archive.js
+++ b/app/src/components/Archive.js
@@ -77,7 +77,7 @@ import {
useSearchDomain,
useSearchOrder
} from "../hooks/customHooks";
-import {FileCards, FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
+import {DeepSearchHint, FileCards, FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import _ from "lodash";
import {ThemeContext} from "../contexts/contexts";
@@ -1259,8 +1259,10 @@ function ArchivesPage() {
inputRef={searchInputRef}
viewButton={viewButton}
sorts={archiveOrders}
+ showDeep={true}
/>
{body}
+
{paginator}
{body}
+
{paginator}
{
@@ -582,6 +586,7 @@ export function SearchFilterModal(
setDraftMonths(seededMonths);
setDraftRange(seededRange);
setDraftCensored(censored || false);
+ setDraftDeep(deep || false);
}
}, [open]);
@@ -622,6 +627,9 @@ export function SearchFilterModal(
if (showCensored && draftCensored !== censored) {
params.censored = draftCensored ? 'true' : null;
}
+ if (showDeep && draftDeep !== deep) {
+ params.deep = draftDeep ? 'true' : null;
+ }
if (Object.keys(params).length > 0) {
params.o = 0;
updateQuery(params);
@@ -641,6 +649,7 @@ export function SearchFilterModal(
setDraftMonths([]);
setDraftRange(emptyRange);
setDraftCensored(false);
+ setDraftDeep(false);
}
// Sort section (operates on the draft).
@@ -754,6 +763,18 @@ export function SearchFilterModal(
}
+ {showDeep &&
+
+
+
+
+
+
+ }
+
{showDates &&
@@ -783,6 +804,7 @@ export function SearchFilterButton(
showTags = true,
showLimit = true,
showCensored = false,
+ showDeep = false,
limitOptions = [12, 24, 48, 96],
size = 'medium',
content = 'Filter',
@@ -794,6 +816,7 @@ export function SearchFilterButton(
const {dateRange, months} = useSearchDate();
const {activeTags, anyTag} = useSearch();
const {censored} = useSearchCensored();
+ const {deep} = useSearchDeep();
let count = 0;
if (fileFilterOptions && filter) {
@@ -811,6 +834,9 @@ export function SearchFilterButton(
if (showCensored && censored) {
count += 1;
}
+ if (showDeep && deep) {
+ count += 1;
+ }
const active = count > 0;
return <>
@@ -829,6 +855,7 @@ export function SearchFilterButton(
showTags={showTags}
showLimit={showLimit}
showCensored={showCensored}
+ showDeep={showDeep}
limitOptions={limitOptions}
/>
@@ -849,6 +876,7 @@ export function SearchControlBar(
fileFilterOptions = null,
showDates = false,
showCensored = false,
+ showDeep = false,
},
) {
const [localSearchStr, setLocalSearchStr] = useState(searchStr || '');
@@ -866,17 +894,39 @@ export function SearchControlBar(
{viewButton}
{searchInput}
+ showCensored={showCensored} showDeep={showDeep}/>
}
+// Offers deep search when a fast search found nothing. Used by the module pages (Videos/Archives/Docs),
+// which have no deep estimate; FilesSearchView has a count-aware variant.
+export function DeepSearchHint({searchStr, files}) {
+ const {deep} = useSearchDeep();
+ const {updateQuery} = useContext(QueryContext);
+
+ if (!searchStr || deep || !files || files.length > 0) {
+ return null;
+ }
+
+ return
+ Not finding what you need?
+
+
+}
+
export function FilesSearchView({
showView = true,
emptySearch = false,
model,
+ deepEstimate = null,
}) {
- const {searchFiles, pages} = useSearchFiles(24, emptySearch, model);
+ const {searchFiles, total, searchStr, pages} = useSearchFiles(24, emptySearch, model);
+ const {deep} = useSearchDeep();
+ const {updateQuery} = useContext(QueryContext);
const {body, paginator, viewButton} = FilesView(
{
@@ -891,9 +941,38 @@ export function FilesSearchView({
},
);
+ // The default search only matches titles/authors/descriptions; offer deep search when it would find
+ // more. `deepEstimate` comes from the search estimates; without it, fall back to a generic hint on
+ // zero results.
+ let deepHint = null;
+ if (Boolean(searchStr) && !deep && searchFiles) {
+ const moreCount = deepEstimate !== null && deepEstimate !== undefined && total !== null
+ ? deepEstimate - total : null;
+ // Enable deep search and reset pagination in a single update (like the filter modal does), so the
+ // user starts at page 1 of the deep results.
+ const deepButton = ;
+ if (searchFiles.length === 0 && (moreCount === null || moreCount > 0)) {
+ deepHint =
+ {moreCount > 0
+ ? `No results in fast search. ${moreCount} result${moreCount === 1 ? '' : 's'} available in deep search.`
+ : 'Not finding what you need?'}
+ {deepButton}
+ ;
+ } else if (searchFiles.length > 0 && moreCount > 0) {
+ deepHint =
+ {`${moreCount} more result${moreCount === 1 ? '' : 's'} available in deep search.`}
+ {deepButton}
+ ;
+ }
+ }
+
return <>
{showView &&
{viewButton}
}
{body}
+ {deepHint}
{paginator}
}
diff --git a/app/src/components/Search.js b/app/src/components/Search.js
index addbc6ac..ab3d2de4 100644
--- a/app/src/components/Search.js
+++ b/app/src/components/Search.js
@@ -1,7 +1,7 @@
import React, {useState} from "react";
import {Link, Route, Routes, useLocation, useNavigate} from "react-router";
import {FilesSearchView, SearchViewButton} from "./Files";
-import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchFilter} from "../hooks/customHooks";
+import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchDeep, useSearchFilter} from "../hooks/customHooks";
import {ShortcutHint} from "./ShortcutHint";
import {ZimSearchView} from "./Zim";
import {MapSearchView} from "./MapSearchView";
@@ -159,6 +159,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
sendGeneralReqeust(async () => await searchSuggestions(searchStr));
sendMapRequest(async () => await searchEstimateMap(searchStr));
}
+ // The files estimate always uses deep search server-side, so users know results are possible.
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag));
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
sendOtherRequest(async () => await searchEstimateOthers(tagNames));
@@ -191,7 +192,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
React.useEffect(() => {
if (!_.isEmpty(filesData)) {
setSuggestions((prevState) => {
- return {...prevState, fileGroups: filesData.fileGroups}
+ return {...prevState, fileGroups: filesData.fileGroups, fileGroupsDeep: filesData.fileGroupsDeep}
});
}
}, [setSuggestions, filesData]);
@@ -232,6 +233,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext);
const {dateRange, months, setDates, clearDate} = useSearchDate();
const {suggestions, loading} = useSuggestions(searchStr, searchTags, filter, anyTag);
+ const {deep} = useSearchDeep();
const {getLocationStr} = React.useContext(QueryContext);
// The results that will be displayed by .
@@ -251,19 +253,25 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
let results = {};
+ // The dropdown always suggests the deep count so users know what is findable. The search results
+ // page shows the active mode's count, and offers deep search when it found less than promised.
+ const dropdownEstimate = newSuggestions.fileGroupsDeep ?? newSuggestions.fileGroups;
+ // The tab badge shows the count matching the active search mode, so it agrees with the result list.
+ const fileGroupsEstimate = deep ? newSuggestions.fileGroupsDeep : newSuggestions.fileGroups;
+
// Suggested results are ordered.
- if (newSuggestions.fileGroups > 0) {
+ if (dropdownEstimate > 0) {
results.fileGroups = {
name: 'Files', results: [
{
- title: newSuggestions.fileGroups.toString(),
+ title: dropdownEstimate.toString(),
type: 'files',
// Add search query onto current location.
location: getLocationStr({q: searchStr}, '/search'),
}
]
};
- } else if (newSuggestions.fileGroups === 0) {
+ } else if (dropdownEstimate === 0) {
// Tell the user there are no files.
results.fileGroups = {name: 'Files', results: noResults};
}
@@ -389,7 +397,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
setSuggestionsResults(results);
setSuggestionsSums({
- fileGroups: newSuggestions.fileGroups,
+ fileGroups: fileGroupsEstimate,
+ fileGroupsDeep: newSuggestions.fileGroupsDeep,
zims: zimSum,
otherSum: otherSum,
mapPlaces: newSuggestions.mapPlaces || 0,
@@ -419,7 +428,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
normalizeSuggestionsResults(suggestions);
}
- }, [JSON.stringify(suggestions)]);
+ }, [JSON.stringify(suggestions), deep]);
// Clear timer on unmount.
React.useEffect(() => {
@@ -495,7 +504,8 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
return : null}/>
- }/>
+ }/>
}/>
}/>
}/>
diff --git a/app/src/components/SearchFilterModal.test.js b/app/src/components/SearchFilterModal.test.js
index 5e919dfb..679e4daf 100644
--- a/app/src/components/SearchFilterModal.test.js
+++ b/app/src/components/SearchFilterModal.test.js
@@ -146,6 +146,36 @@ describe('SearchFilterButton', () => {
expect(screen.getByText('1')).toBeInTheDocument();
});
+ it('counts the deep filter and applies it on close when showDeep is set', () => {
+ const spy = jest.fn();
+ renderButton({sorts: videoOrders, showDeep: true}, {}, spy);
+
+ // No deep badge initially.
+ expect(screen.queryByText('1')).toBeNull();
+
+ fireEvent.click(screen.getByText('Filter'));
+ // The Search Depth section/toggle is shown.
+ expect(screen.getByText('Search Depth')).toBeInTheDocument();
+ fireEvent.mouseUp(screen.getByTestId('toggle'));
+ // Draft only — nothing applied yet.
+ expect(spy).not.toHaveBeenCalled();
+
+ fireEvent.click(screen.getByText('Done'));
+ expect(spy).toHaveBeenCalledTimes(1);
+ expect(spy.mock.calls[0][0]).toMatchObject({deep: 'true'});
+ });
+
+ it('does not show the Search Depth section unless showDeep is set', () => {
+ renderButton({sorts: videoOrders});
+ fireEvent.click(screen.getByText('Filter'));
+ expect(screen.queryByText('Search Depth')).toBeNull();
+ });
+
+ it('shows a count badge when the deep filter is active in the URL', () => {
+ renderButton({sorts: videoOrders, showDeep: true}, {deep: 'true'});
+ expect(screen.getByText('1')).toBeInTheDocument();
+ });
+
it('shows the File Type section when fileFilterOptions are provided', () => {
renderButton({
sorts: videoOrders,
diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js
index 50c75aed..b3e434d1 100644
--- a/app/src/components/Videos.js
+++ b/app/src/components/Videos.js
@@ -45,7 +45,7 @@ import {
TableCell
} from "semantic-ui-react";
import {useChannel, useSearchOrder, useSearchVideos, useVideo, useVideoStatistics} from "../hooks/customHooks";
-import {FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
+import {DeepSearchHint, FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {Button, Card, Header, Loader, Modal, Placeholder, Popup, Segment, Statistic} from "./Theme";
@@ -241,8 +241,10 @@ export function VideosPage() {
viewButton={viewButton}
sorts={videoOrders}
showCensored={true}
+ showDeep={true}
/>
{body}
+
{paginator}
{
const activeTags = searchParams.getAll('tag');
const {view} = useSearchView();
const headline = view === 'headline';
+ const {deep} = useSearchDeep();
const [archives, setArchives] = useState(null);
const [totalPages, setTotalPages] = useState(0);
@@ -519,7 +520,8 @@ export const useSearchArchives = (defaultLimit) => {
setArchives(null);
setTotalPages(0);
try {
- let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags, headline);
+ let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags, headline,
+ deep);
setArchives(archives_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@@ -536,7 +538,7 @@ export const useSearchArchives = (defaultLimit) => {
useEffect(() => {
localSearchArchives();
- }, [searchStr, limit, domain, order, offset, JSON.stringify(activeTags), headline]);
+ }, [searchStr, limit, domain, order, offset, JSON.stringify(activeTags), headline, deep]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@@ -580,6 +582,7 @@ export const useSearchDocs = (defaultLimit) => {
const author = searchParams.get('author') || '';
const subject = searchParams.get('subject') || '';
const filter = searchParams.get('filter') || '';
+ const {deep} = useSearchDeep();
const [docs, setDocs] = useState(null);
const [totalPages, setTotalPages] = useState(0);
@@ -589,7 +592,8 @@ export const useSearchDocs = (defaultLimit) => {
setTotalPages(0);
try {
const mimetype = docFilterToMimetype(filter);
- let [docs_, total] = await searchDocs(offset, limit, searchStr, order, activeTags, author, subject, mimetype);
+ let [docs_, total] = await searchDocs(offset, limit, searchStr, order, activeTags, author, subject,
+ mimetype, deep);
setDocs(docs_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@@ -600,7 +604,7 @@ export const useSearchDocs = (defaultLimit) => {
useEffect(() => {
localSearchDocs();
- }, [searchStr, limit, order, offset, JSON.stringify(activeTags), author, subject, filter]);
+ }, [searchStr, limit, order, offset, JSON.stringify(activeTags), author, subject, filter, deep]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@@ -639,6 +643,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
const headline = view === 'headline';
const anyTag = searchParams.get('anyTag') === 'true';
const censored = searchParams.get('censored') === 'true';
+ const {deep} = useSearchDeep();
const [videos, setVideos] = useState(null);
const [totalPages, setTotalPages] = useState(0);
@@ -649,7 +654,8 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
setLoading(true);
setTotalPages(0);
try {
- let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline, censored);
+ let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline,
+ censored, deep);
setVideos(videos_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@@ -662,7 +668,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
useEffect(() => {
localSearchVideos();
- }, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags), headline, censored]);
+ }, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags), headline, censored, deep]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@@ -912,8 +918,10 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
dateRange,
} = useSearch(defaultLimit, emptySearch, model);
const {view} = useSearchView();
+ const {deep} = useSearchDeep();
const [searchFiles, setSearchFiles] = useState(null);
+ const [total, setTotal] = useState(null);
const [loading, setLoading] = useState(false);
const headline = view === 'headline';
@@ -932,12 +940,14 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
setLoading(true);
console.log('localSearchFiles', fromDate, toDate, months);
try {
- let [file_groups, total] = await filesSearch(
+ let [file_groups, total_] = await filesSearch(
pages.offset, pages.limit, searchStr, mimetypes, model || model_, activeTags, headline,
- months, fromDate, toDate, anyTag);
+ months, fromDate, toDate, anyTag, null, null, null, deep);
setSearchFiles(file_groups);
- pages.setTotal(total);
+ setTotal(total_);
+ pages.setTotal(total_);
} catch (e) {
+ setTotal(null);
pages.setTotal(0);
console.error(e);
toast({
@@ -974,10 +984,12 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
JSON.stringify(months),
JSON.stringify(dateRange),
anyTag,
+ deep,
]);
return {
searchFiles,
+ total,
searchStr,
filter,
setSearchStr,
@@ -1655,6 +1667,14 @@ export const useSearchCensored = () => {
return {censored, setCensored}
}
+export const useSearchDeep = () => {
+ // Search all text including captions/document contents (?deep=true). Slower, but more thorough.
+ const [value, setValue] = useOneQuery('deep');
+ const deep = value === 'true';
+ const setDeep = (newValue) => setValue(newValue ? 'true' : undefined);
+ return {deep, setDeep}
+}
+
export const useSearchDate = () => {
// Filters files using
// ?month=1&month=2&month=3
diff --git a/modules/archive/api.py b/modules/archive/api.py
index 506ecaea..ebaa9725 100644
--- a/modules/archive/api.py
+++ b/modules/archive/api.py
@@ -61,7 +61,7 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest):
offset = body.offset or 0
file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names,
- body.headline)
+ body.headline, body.deep)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index b122456a..778e1de2 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -1400,7 +1400,7 @@ NO_NULL_ORDERS = {
def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str],
- headline: bool = False) \
+ headline: bool = False, deep: bool = False) \
-> Tuple[List[dict], int]:
# Always filter FileGroups to Archives.
wheres = []
@@ -1412,9 +1412,11 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
select_columns = ''
if search_str:
# A search_str was provided by the user, modify the query to filter by it.
- select_columns = 'ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)) AS rank,' \
+ # `textsearch` includes d_text (page contents); `textsearch_abc` is much smaller and faster.
+ ts_col = 'textsearch' if deep else 'textsearch_abc'
+ select_columns = f'ts_rank(fg.{ts_col}, websearch_to_tsquery(%(search_str)s)) AS rank,' \
' COUNT(*) OVER() AS total'
- wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
+ wheres.append(f'fg.{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
params['search_str'] = search_str
group_by = f'{group_by}, rank'
diff --git a/modules/archive/schema.py b/modules/archive/schema.py
index 7e9b5cd0..b963a545 100644
--- a/modules/archive/schema.py
+++ b/modules/archive/schema.py
@@ -24,6 +24,7 @@ class ArchiveSearchRequest:
order_by: Optional[str] = None
tag_names: List[str] = field(default_factory=list)
headline: bool = False
+ deep: bool = False # When True, search all text including page contents (d_text)
@dataclass
diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py
index 05f434c5..c92df3d1 100644
--- a/modules/archive/test/test_api.py
+++ b/modules/archive/test/test_api.py
@@ -50,36 +50,43 @@ async def test_archives_search(test_session, archive_directory, archive_factory,
archive_factory('example.org') # has no contents
test_session.commit()
+ # Contents are d_text, so a deep search is required to match them.
# 1 and 2 contain "foo".
- data = {'search_str': 'foo'}
+ data = {'search_str': 'foo', 'deep': True}
await check_results(async_client, data, [2, 1])
+ # The default (fast) search does not match contents.
+ data = {'search_str': 'foo'}
+ await check_results(async_client, data, [])
+
# 2 and 3 contain "baz".
- data = {'search_str': 'baz'}
+ data = {'search_str': 'baz', 'deep': True}
await check_results(async_client, data, [3, 2])
# 1 contains "bar".
- data = {'search_str': 'bar'}
+ data = {'search_str': 'bar', 'deep': True}
await check_results(async_client, data, [1, ])
# No archives contain "huzzah"
- data = {'search_str': 'huzzah'}
+ data = {'search_str': 'huzzah', 'deep': True}
await check_results(async_client, data, [])
# Only 3 contains "baz" and is in domain "example.org"
- data = {'search_str': 'baz', 'domain': 'example.org'}
+ data = {'search_str': 'baz', 'domain': 'example.org', 'deep': True}
await check_results(async_client, data, [3, ])
# 1's title contains "my", this is ignored by Postgres.
- data = {'search_str': 'my'}
+ data = {'search_str': 'my', 'deep': True}
await check_results(async_client, data, [])
- # 3's title contains "third".
+ # 3's title contains "third", found by the default (fast) search and the deep search.
data = {'search_str': 'third'}
await check_results(async_client, data, [3, ])
+ data = {'search_str': 'third', 'deep': True}
+ await check_results(async_client, data, [3, ])
# All contents contain "qux", but they contain different amounts. They are ordered by the amount.
- data = {'search_str': 'qux'}
+ data = {'search_str': 'qux', 'deep': True}
await check_results(async_client, data, [3, 2, 1])
data = {'search_str': 'qux', 'order_by': 'bad order_by'}
@@ -116,7 +123,8 @@ async def test_archives_search_headline(test_session, archive_directory, archive
archive_factory('example.org') # has no contents
test_session.commit()
- content = dict(search_str='foo')
+ # "foo" is only in the contents (d_text), so a deep search is required to match it.
+ content = dict(search_str='foo', deep=True)
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
@@ -124,7 +132,7 @@ async def test_archives_search_headline(test_session, archive_directory, archive
assert response.json['file_groups'][0]['d_headline'] is None
assert response.json['file_groups'][1]['d_headline'] is None
- content = dict(search_str='foo', headline=True)
+ content = dict(search_str='foo', headline=True, deep=True)
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index 5b99e37b..625782d0 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -571,7 +571,8 @@ async def test_refresh_archives(test_session, test_directory, async_client, make
assert (test_directory / 'archive/example.com/2021-10-05-16-20-10_NA.html').is_file()
assert test_session.query(Archive).count() == 2
- content = json.dumps({'search_str': 'text', 'model': 'archive'})
+ # "text" is only in the readability contents (d_text), so a deep search is required.
+ content = json.dumps({'search_str': 'text', 'model': 'archive', 'deep': True})
request, response = await async_client.post('/api/files/search', content=content)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], 'No files matched "text"'
diff --git a/modules/docs/api.py b/modules/docs/api.py
index c69c1cb4..2aeeded3 100644
--- a/modules/docs/api.py
+++ b/modules/docs/api.py
@@ -56,6 +56,7 @@ async def search_docs(_: Request, body: schema.DocSearchRequest):
offset=offset,
order_by=body.order_by,
tag_names=body.tag_names,
+ deep=body.deep,
)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)
diff --git a/modules/docs/lib.py b/modules/docs/lib.py
index 708b3d70..79060638 100644
--- a/modules/docs/lib.py
+++ b/modules/docs/lib.py
@@ -347,15 +347,18 @@ def _get_doc(session, file_group_id: int):
def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None,
- limit=20, offset=0, order_by='published_datetime', tag_names=None):
+ limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False):
from .models import Doc
from wrolpi.files.models import FileGroup
+ # `textsearch` includes d_text (document contents); `textsearch_abc` is much smaller and faster.
+ ts_col = FileGroup.textsearch if deep else FileGroup.textsearch_abc
+
with get_db_session() as session:
query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id)
if search_str:
- query = query.filter(FileGroup.textsearch.op('@@')(func.websearch_to_tsquery(search_str)))
+ query = query.filter(ts_col.op('@@')(func.websearch_to_tsquery(search_str)))
if author:
query = query.filter(FileGroup.b_text.ilike(f'%{author}%'))
@@ -382,7 +385,7 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime
# Ordering.
if order_by == 'rank' and search_str:
query = query.order_by(
- desc(func.ts_rank(FileGroup.textsearch, func.websearch_to_tsquery(search_str))),
+ desc(func.ts_rank(ts_col, func.websearch_to_tsquery(search_str))),
desc(FileGroup.id),
)
elif order_by == 'published_datetime':
diff --git a/modules/docs/schema.py b/modules/docs/schema.py
index a2ad7c8a..c95a09d9 100644
--- a/modules/docs/schema.py
+++ b/modules/docs/schema.py
@@ -13,6 +13,7 @@ class DocSearchRequest:
order_by: Optional[str] = 'published_datetime'
limit: int = 20
offset: int = 0
+ deep: bool = False # When True, search all text including document contents (d_text)
@dataclasses.dataclass
diff --git a/modules/docs/test/test_lib.py b/modules/docs/test/test_lib.py
index c6dc661a..2db26844 100644
--- a/modules/docs/test/test_lib.py
+++ b/modules/docs/test/test_lib.py
@@ -289,7 +289,11 @@ async def test_search_docs_attaches_section_hint_epub(test_session, test_directo
(2, 'Chapter 2', 'unrelated content about pottery'),
])
+ # "mullen" is only in the document body (d_text), so a deep search is required to match it.
results, total = _search_docs(search_str='mullen', mimetype='application/epub')
+ assert total == 0, 'Document contents are not searched unless deep=True'
+
+ results, total = _search_docs(search_str='mullen', mimetype='application/epub', deep=True)
assert total == 1
assert results[0]['id'] == fg.id
hint = results[0]['section_hint']
@@ -311,7 +315,7 @@ async def test_search_docs_attaches_section_hint_pdf(test_session, test_director
(3, 'Page 3', 'here is where mullen shows up'),
])
- results, total = _search_docs(search_str='mullen', mimetype='application/pdf')
+ results, total = _search_docs(search_str='mullen', mimetype='application/pdf', deep=True)
assert total == 1
assert results[0]['id'] == fg.id
hint = results[0]['section_hint']
diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py
index 7241b255..5f42e5e8 100644
--- a/modules/videos/conftest.py
+++ b/modules/videos/conftest.py
@@ -336,6 +336,7 @@ def assert_video_search(async_client):
order_by: str = None,
channel_id: int = None,
censored: bool = None,
+ deep: bool = None,
):
content = dict()
if search_str is not None:
@@ -352,6 +353,8 @@ def assert_video_search(async_client):
content['channel_id'] = channel_id
if censored is not None:
content['censored'] = censored
+ if deep is not None:
+ content['deep'] = deep
request, response = await async_client.post('/api/videos/search', content=json.dumps(content))
diff --git a/modules/videos/schema.py b/modules/videos/schema.py
index 7da8ae3e..711de62e 100644
--- a/modules/videos/schema.py
+++ b/modules/videos/schema.py
@@ -178,6 +178,7 @@ class VideoSearchRequest:
tag_names: List[str] = field(default_factory=list)
headline: bool = False
censored: bool = False
+ deep: bool = False # When True, search all text including captions (d_text)
@dataclass
diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py
index 6f714996..2a496c0b 100644
--- a/modules/videos/test/test_api.py
+++ b/modules/videos/test/test_api.py
@@ -195,31 +195,36 @@ async def test_search_videos_file(test_session, test_directory, video_with_searc
video_with_search_factory(title=title, d_text=caption)
test_session.commit()
+ # Captions are d_text, so a deep search is required to match them.
# Repeated runs should return the same result
for _ in range(2):
# Only videos with a b are returned, ordered by the amount of b's
- await assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
+ await assert_video_search(search_str='b', deep=True, assert_ids=[1, 2, 3, 4])
+
+ # The default (fast) search does not match captions.
+ await assert_video_search(search_str='b', assert_ids=[])
# Only two captions have e
- await assert_video_search(search_str='e', assert_ids=[4, 1])
+ await assert_video_search(search_str='e', deep=True, assert_ids=[4, 1])
# Only 2 contains 2
- await assert_video_search(search_str='2', assert_ids=[2, ])
+ await assert_video_search(search_str='2', deep=True, assert_ids=[2, ])
# Only two captions have d
- await assert_video_search(search_str='d', assert_ids=[1, 2])
+ await assert_video_search(search_str='d', deep=True, assert_ids=[1, 2])
- # 5 can be gotten by it's title
+ # 5 can be gotten by it's title, in either mode.
await assert_video_search(search_str='5', assert_ids=[5, ])
+ await assert_video_search(search_str='5', deep=True, assert_ids=[5, ])
# only video 1 has e and d
- await assert_video_search(search_str='e d', assert_ids=[1, ])
+ await assert_video_search(search_str='e d', deep=True, assert_ids=[1, ])
# video 1 and 4 have b and e, but 1 has more
- await assert_video_search(search_str='b e', assert_ids=[1, 4])
+ await assert_video_search(search_str='b e', deep=True, assert_ids=[1, 4])
# Check totals are correct even with a limit
- await assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
+ await assert_video_search(search_str='b', deep=True, limit=2, assert_ids=[1, 2], assert_total=4)
@pytest.mark.asyncio
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index 83dd5421..375ee632 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -16,6 +16,7 @@ from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory, await_switches
from wrolpi.downloader import Download, DownloadResult, get_download_manager_config
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
+from wrolpi.test.common import skip_circleci
from wrolpi.vars import PROJECT_DIR
example_video_json = {
@@ -130,6 +131,7 @@ async def test_download_video_tags(test_session, video_download_manager, video_f
assert [i.tag.name for i in video.file_group.tag_files] == [tag1.name, tag2.name]
+@skip_circleci
@pytest.mark.asyncio
async def test_download_channel(test_session, test_directory, simple_channel, video_download_manager, video_file,
mock_video_extract_info, mock_video_prepare_filename, await_switches,
diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py
index 4f9f3dd9..4a4b96b6 100644
--- a/modules/videos/video/api.py
+++ b/modules/videos/video/api.py
@@ -66,6 +66,7 @@ async def search_videos(_: Request, body: schema.VideoSearchRequest):
body.tag_names,
body.headline,
body.censored,
+ body.deep,
)
ret = {'file_groups': list(file_groups), 'totals': {'file_groups': videos_total}}
diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py
index 9de2065c..5e30f463 100644
--- a/modules/videos/video/lib.py
+++ b/modules/videos/video/lib.py
@@ -93,6 +93,7 @@ def search_videos(
tag_names: List[str] = None,
headline: bool = False,
censored: bool = False,
+ deep: bool = False,
) -> Tuple[List[dict], int]:
tag_names = tag_names or []
# Search videos and audio-only files.
@@ -112,8 +113,10 @@ def search_videos(
if search_str:
# A search_str was provided by the user, modify the query to filter by it.
- select_columns = 'fg.id, ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
- wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
+ # `textsearch` includes d_text (captions); `textsearch_abc` is much smaller and faster.
+ ts_col = 'textsearch' if deep else 'textsearch_abc'
+ select_columns = f'fg.id, ts_rank(fg.{ts_col}, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
+ wheres.append(f'fg.{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
params['search_str'] = search_str
else:
# No search_str provided. Get id and total only.
diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py
index 9463ec3e..2a8875f4 100644
--- a/wrolpi/files/api.py
+++ b/wrolpi/files/api.py
@@ -165,7 +165,7 @@ async def post_search_files(_: Request, body: schema.FilesSearchRequest):
with timer('Searching all files', 'info', logger__=logger):
file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model,
body.tag_names, body.headline, body.months, body.from_year, body.to_year,
- body.any_tag, body.order, body.url, body.suffix, body.path)
+ body.any_tag, body.order, body.url, body.suffix, body.path, body.deep)
return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total)))
diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py
index c461eb6a..b2fa0c12 100644
--- a/wrolpi/files/lib.py
+++ b/wrolpi/files/lib.py
@@ -1039,7 +1039,7 @@ async def search_directories_by_name(session: Session, name: str, excluded: List
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
from_year: int = None, to_year: int = None, any_tag: bool = False, order: str = None,
- url: str = None, suffix: str = None, path: str = None) -> \
+ url: str = None, suffix: str = None, path: str = None, deep: bool = False) -> \
Tuple[List[dict], int]:
"""Search the FileGroup table.
@@ -1061,6 +1061,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
@param url: Filter by URL using case-insensitive partial match (ILIKE).
@param suffix: Only return files whose primary file has this suffix (e.g. ".bin"), case-insensitive.
@param path: Filter by primary_path using case-insensitive partial match (ILIKE).
+ @param deep: Search all text including captions/body (d_text); slower but more thorough.
"""
params = dict(offset=offset, limit=limit, search_str=search_str, url_search_str=f'%{search_str}%')
wheres = []
@@ -1069,9 +1070,10 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
joins = []
if search_str:
- # Search by textsearch column.
- wheres.append('textsearch @@ websearch_to_tsquery(%(search_str)s)')
- selects.append('ts_rank(textsearch, websearch_to_tsquery(%(search_str)s))')
+ # `textsearch` includes d_text (captions/body); `textsearch_abc` is much smaller and faster.
+ ts_col = 'textsearch' if deep else 'textsearch_abc'
+ wheres.append(f'{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
+ selects.append(f'ts_rank({ts_col}, websearch_to_tsquery(%(search_str)s))')
order_by = '2 DESC'
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
@@ -1479,9 +1481,13 @@ def mimetypes_to_sql_wheres(wheres: List[str], params: dict, mimetypes: List[str
async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str],
months: List[int] = None, from_year: int = None, to_year: int = None,
- any_tag: bool = False):
+ any_tag: bool = False) -> dict:
"""
- Return FileGroup count of what will be returned if the search is actually performed.
+ Return FileGroup counts of what will be returned if the search is actually performed.
+
+ Returns both the fast (`textsearch_abc`) and deep (`textsearch`) counts so the UI can show the count
+ matching the current search mode, and offer deep search when it has more results. Both are cheap
+ index-bound COUNTs (no ts_rank).
"""
wheres = []
joins = list()
@@ -1490,8 +1496,13 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
having = ''
if search_str:
- # Search by textsearch, and by matching url.
+ # `textsearch` matches are a superset of `textsearch_abc`, so filter by the deep column and count
+ # the fast subset with FILTER in the same pass.
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
+ selects = ('COUNT(*) FILTER (WHERE fg.textsearch_abc @@ websearch_to_tsquery(%(search_str)s)) AS estimate,'
+ ' COUNT(*) AS deep_estimate')
+ else:
+ selects = 'COUNT(*) AS estimate, COUNT(*) AS deep_estimate'
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
@@ -1501,7 +1512,7 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
joins = "\n".join(joins)
wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
stmt = f'''
- SELECT COUNT(*) OVER() AS estimate
+ SELECT {selects}
FROM file_group fg
{joins}
{wheres}
@@ -1514,8 +1525,8 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
curs.execute(stmt, params)
result = curs.fetchone()
if result:
- return int(result['estimate'])
- return 0
+ return dict(file_groups=int(result['estimate']), file_groups_deep=int(result['deep_estimate']))
+ return dict(file_groups=0, file_groups_deep=0)
MOVE_CHUNK_SIZE = 100
diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py
index 63e59b10..f48f4c34 100644
--- a/wrolpi/files/models.py
+++ b/wrolpi/files/models.py
@@ -79,6 +79,7 @@ class FileGroup(ModelHelper, Base):
Index('file_group_published_datetime_idx', 'published_datetime'),
Index('file_group_published_modified_datetime_idx', 'published_modified_datetime'),
Index('file_group_size_ix', 'size'),
+ Index('file_group_textsearch_abc_idx', 'textsearch_abc'),
Index('file_group_textsearch_idx', 'textsearch'),
Index('file_group_url_idx', 'url'),
Index('file_group_viewed_idx', 'viewed'),
@@ -126,6 +127,13 @@ class FileGroup(ModelHelper, Base):
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") ||
setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char")
''')))
+ # Excludes d_text (captions/body); much smaller than `textsearch`, used for the default fast search.
+ textsearch_abc = deferred(
+ Column(tsvector, Computed('''
+ setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") ||
+ setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") ||
+ setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char")
+ ''')))
def __repr__(self):
m = f'model={self.model}' if self.model else f'mimetype={self.mimetype}'
diff --git a/wrolpi/files/schema.py b/wrolpi/files/schema.py
index 2f085e46..0eac4945 100644
--- a/wrolpi/files/schema.py
+++ b/wrolpi/files/schema.py
@@ -44,6 +44,7 @@ class FilesSearchRequest:
url: Optional[str] = None # Filter by URL (partial match)
suffix: Optional[str] = None # Filter by primary file suffix (e.g. ".bin"), case-insensitive exact match
path: Optional[str] = None # Filter by primary_path (case-insensitive partial match)
+ deep: bool = False # When True, search all text including captions/body (d_text)
def __post_init__(self):
if self.any_tag and self.tag_names:
diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py
index eb7b10bf..ae999fe6 100644
--- a/wrolpi/files/test/test_api.py
+++ b/wrolpi/files/test/test_api.py
@@ -315,6 +315,51 @@ async def test_files_search(test_session, async_client, make_files_structure, as
await assert_files_search('nothing', [])
+@pytest.mark.asyncio
+async def test_files_search_deep(test_session, async_client, make_files_structure):
+ """Search defaults to the fast ABC column (title/author/description); `deep` opts into d_text
+ (captions/file contents)."""
+ guide, lecture = make_files_structure([
+ 'gardening guide.txt',
+ 'lecture.txt',
+ ])
+ guide.write_text('planting tips')
+ # "gardening" only appears in this file's contents (d_text), not its name.
+ lecture.write_text('a lecture about gardening')
+
+ request, response = await async_client.post('/api/files/refresh')
+ assert response.status_code == HTTPStatus.NO_CONTENT
+
+ async def search(**kwargs):
+ request, response = await async_client.post('/api/files/search', json=kwargs)
+ assert response.status_code == HTTPStatus.OK
+ return response.json['file_groups']
+
+ # Default (fast) search only matches the title.
+ results = await search(search_str='gardening')
+ assert [i['primary_path'] for i in results] == ['gardening guide.txt']
+
+ # Deep search also matches file contents; the title match ranks higher (weight A vs D).
+ results = await search(search_str='gardening', deep=True)
+ assert [i['primary_path'] for i in results] == ['gardening guide.txt', 'lecture.txt']
+
+ # Headlines work in both modes.
+ results = await search(search_str='gardening', headline=True)
+ assert [i['primary_path'] for i in results] == ['gardening guide.txt']
+ assert 'gardening' in results[0]['title_headline']
+ results = await search(search_str='gardening', deep=True, headline=True)
+ assert [i['primary_path'] for i in results] == ['gardening guide.txt', 'lecture.txt']
+ assert 'gardening' in results[1]['d_headline']
+
+ # The estimate endpoint returns both counts: `file_groups` matches the default fast search, and
+ # `file_groups_deep` shows what deep search would find, so the UI can offer deep search.
+ request, response = await async_client.post('/api/search_file_estimates',
+ json=dict(search_str='gardening'))
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['file_groups'] == 1
+ assert response.json['file_groups_deep'] == 2
+
+
@pytest.mark.asyncio
async def test_files_search_by_suffix(test_session, async_client, make_files_structure):
"""Files can be filtered by their primary file's suffix (used by the firmware flasher to find .bin files)."""
@@ -367,7 +412,8 @@ async def test_files_search_attaches_doc_section_hint(async_client, test_session
bug where only the docs-specific endpoint (/api/docs/search) attached hints."""
await refresh_files()
- body = dict(search_str='chapter')
+ # "chapter" only appears in the EPUB body (d_text), so deep search is required to match it.
+ body = dict(search_str='chapter', deep=True)
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py
index 52d9748a..a2da0071 100644
--- a/wrolpi/files/test/test_lib.py
+++ b/wrolpi/files/test/test_lib.py
@@ -580,18 +580,20 @@ async def test_files_indexer(async_client, test_session, make_files_structure, t
# Video are indexed by the modeler, not by an indexer.
assert video_file.mimetype == 'video/mp4' and video_file.indexer == indexers.DefaultIndexer
- # File are indexed by their titles and contents.
+ # File are indexed by their titles and contents. Contents (d_text) require a deep search.
files, total = lib.search_files('file', 10, 0)
assert total == 5, 'All files contain "file" in their file name. The associated video file is hidden.'
files, total = lib.search_files('image', 10, 0)
assert total == 1 and files[0]['title'] == 'an image file.jpeg', 'The image file title contains "image".'
- files, total = lib.search_files('contents', 10, 0)
+ files, total = lib.search_files('contents', 10, 0, deep=True)
assert total == 1 and files[0]['title'] == 'a text file.txt', 'The text file contains "contents".'
+ files, total = lib.search_files('contents', 10, 0)
+ assert total == 0, 'File contents are not searched unless deep=True.'
files, total = lib.search_files('video', 10, 0)
assert total == 1 and {i['title'] for i in files} == {'a video file'}, 'The video file contains "video".'
- files, total = lib.search_files('yawn', 10, 0)
+ files, total = lib.search_files('yawn', 10, 0, deep=True)
assert total == 1 and files[0]['title'] == 'a video file', 'The video file captions contain "yawn".'
- files, total = lib.search_files('bunny', 10, 0)
+ files, total = lib.search_files('bunny', 10, 0, deep=True)
assert total == 1 and {i['title'] for i in files} == {'a zip file.zip'}, \
'The zip file contains a file with "bunny" in the title.'
@@ -603,7 +605,7 @@ async def test_files_indexer(async_client, test_session, make_files_structure, t
text_path.write_text('new text contents')
await refresh_files()
await await_background_tasks()
- files, total = lib.search_files('new', 10, 0)
+ files, total = lib.search_files('new', 10, 0, deep=True)
assert total == 1
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 506bfd86..cb5cdcc0 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -768,7 +768,7 @@ async def post_search_suggestions(request: Request, body: schema.SearchSuggestio
)
async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimateRequest):
"""Used by the Global search to suggest FileGroup count to the user."""
- file_groups = await search_file_suggestion_count(
+ counts = await search_file_suggestion_count(
body.search_str,
body.tag_names,
body.mimetypes,
@@ -779,7 +779,8 @@ async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimate
)
ret = dict(
- file_groups=file_groups,
+ file_groups=counts['file_groups'],
+ file_groups_deep=counts['file_groups_deep'],
)
return json_response(ret)
diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py
index a608ff75..1f22c275 100644
--- a/wrolpi/test/test_root_api.py
+++ b/wrolpi/test/test_root_api.py
@@ -666,7 +666,8 @@ async def test_search_file_estimates(test_session, async_client, archive_factory
request, response = await async_client.post('/api/search_file_estimates', json=body)
assert response.status_code == HTTPStatus.OK
- assert response.json['file_groups'] == expected_file_groups or 0
+ # The deep count includes contents/captions matches; the fast (ABC) count is in `file_groups`.
+ assert response.json['file_groups_deep'] == expected_file_groups or 0
await assert_results(dict(search_str='foo'), 1)
@@ -727,7 +728,8 @@ async def test_search_file_estimates_any_tag(test_session, async_client, archive
request, response = await async_client.post('/api/search_file_estimates', json=body)
assert response.status_code == HTTPStatus.OK
- assert response.json['file_groups'] == expected_file_groups or 0
+ # The deep count includes contents/captions matches; the fast (ABC) count is in `file_groups`.
+ assert response.json['file_groups_deep'] == expected_file_groups or 0
await assert_results(dict(search_str='bunny'), 2)
await assert_results(dict(search_str='bunny', any_tag=True), 1)