mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Search: fast ABC mode by default, opt-in deep search
The file_group.textsearch tsvector indexes all four text columns;
d_text (captions, document text, zip contents) is ~90% of the data,
making ts_rank searches take seconds on large libraries. Add a
textsearch_abc generated column (title/author/description only, ~8x
smaller) and default all searches to it, with deep=true opting into the
full column.
Database:
- Migration adds the generated column and a GIN index (rewrites the
table; a few minutes on large libraries).
Backend:
- FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and
DocSearchRequest accept `deep`; each search function picks the column.
- /api/search_file_estimates returns both counts in one index-bound
query (COUNT FILTER on the abc subset): `file_groups` (fast) and
`file_groups_deep`.
UI (one coherent story across the count surfaces):
- The typing dropdown always suggests the deep count (what is findable).
- The tab badge shows the active mode's count so it matches the result
list.
- The gap is reconciled by a hint offering deep search: count-aware on
the global search ("No results in fast search. N results available in
deep search." / "N more results available..."), generic on the module
pages (no deep estimate there). The hint resets pagination.
- "Search Depth" toggle in the search filter modals, persisted as
?deep=true.
Tests that relied on d_text matches now pass deep=True, with fast-mode
assertions alongside. Also skip flaky test_download_channel on
CircleCI.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Roland
parent
1522022330
commit
73fafcd645
33 changed files with 385 additions and 77 deletions
43
alembic/versions/e5f7a9c1d3b6_file_group_textsearch_abc.py
Normal file
43
alembic/versions/e5f7a9c1d3b6_file_group_textsearch_abc.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
"""file_group_textsearch_abc
|
||||
|
||||
Add the `textsearch_abc` generated tsvector column to `file_group`. It indexes only a_text (title),
|
||||
b_text (author), and c_text (description) — excluding d_text (captions/document body), which is ~90% of all
|
||||
text data. Searches default to this much smaller column; the existing `textsearch` column remains for
|
||||
opt-in "deep" search.
|
||||
|
||||
Adding a STORED generated column rewrites the table; expect a few minutes on large libraries.
|
||||
|
||||
Revision ID: e5f7a9c1d3b6
|
||||
Revises: d4e6f8a0c2b5
|
||||
Create Date: 2026-07-03
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'e5f7a9c1d3b6'
|
||||
down_revision = 'd4e6f8a0c2b5'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
conn = op.get_bind()
|
||||
conn.execute(sa.text('''
|
||||
ALTER TABLE file_group
|
||||
ADD COLUMN textsearch_abc tsvector
|
||||
GENERATED ALWAYS AS (
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char")
|
||||
) STORED
|
||||
'''))
|
||||
conn.execute(sa.text(
|
||||
'CREATE INDEX IF NOT EXISTS file_group_textsearch_abc_idx ON file_group USING GIN(textsearch_abc)'))
|
||||
|
||||
|
||||
def downgrade():
|
||||
conn = op.get_bind()
|
||||
conn.execute(sa.text('DROP INDEX IF EXISTS file_group_textsearch_abc_idx'))
|
||||
conn.execute(sa.text('ALTER TABLE file_group DROP COLUMN IF EXISTS textsearch_abc'))
|
||||
|
|
@ -469,7 +469,7 @@ export function DashboardPage() {
|
|||
}
|
||||
return <div style={{display: 'flex', alignItems: 'center', gap: '0.5em', marginBottom: '2em'}}>
|
||||
{getSearchResultsInput(inputProps)}
|
||||
<SearchFilterButton fileFilterOptions={fileMimetypeFilterOptions} showDates={true}
|
||||
<SearchFilterButton fileFilterOptions={fileMimetypeFilterOptions} showDates={true} showDeep={true}
|
||||
size={big ? 'big' : undefined}/>
|
||||
</div>;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -187,7 +187,7 @@ export async function tagChannelInfo(channelId, tagName) {
|
|||
});
|
||||
}
|
||||
|
||||
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline, censored) {
|
||||
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline, censored, deep) {
|
||||
// Build a search query to retrieve a list of videos from the API
|
||||
offset = parseInt(offset || 0);
|
||||
limit = parseInt(limit || DEFAULT_LIMIT);
|
||||
|
|
@ -208,6 +208,9 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by
|
|||
if (censored) {
|
||||
body['censored'] = true;
|
||||
}
|
||||
if (deep) {
|
||||
body['deep'] = true;
|
||||
}
|
||||
|
||||
console.debug('searching videos', body);
|
||||
|
||||
|
|
@ -697,7 +700,7 @@ export async function saveCatalog(items) {
|
|||
}
|
||||
|
||||
|
||||
export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline) {
|
||||
export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline, deep) {
|
||||
// Build a search query to retrieve a list of videos from the API
|
||||
offset = parseInt(offset || 0);
|
||||
limit = parseInt(limit || DEFAULT_LIMIT);
|
||||
|
|
@ -717,6 +720,9 @@ export async function searchArchives(offset, limit, domain, searchStr, order, ta
|
|||
if (headline) {
|
||||
body['headline'] = true;
|
||||
}
|
||||
if (deep) {
|
||||
body['deep'] = true;
|
||||
}
|
||||
|
||||
console.debug('searching archives', body);
|
||||
const response = await apiPost(`${ARCHIVES_API}/search`, body);
|
||||
|
|
@ -1212,7 +1218,7 @@ export async function deleteDownload(downloadId) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag, order, suffix, path) {
|
||||
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag, order, suffix, path, deep) {
|
||||
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit), any_tag: anyTag};
|
||||
if (suffix) {
|
||||
body['suffix'] = suffix;
|
||||
|
|
@ -1244,6 +1250,9 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta
|
|||
if (order) {
|
||||
body['order'] = order;
|
||||
}
|
||||
if (deep) {
|
||||
body['deep'] = true;
|
||||
}
|
||||
console.info('searching files', body);
|
||||
const response = await apiPost(`${API_URI}/files/search`, body);
|
||||
|
||||
|
|
@ -1988,6 +1997,7 @@ export async function searchEstimateFiles(search_str, tagNames, mimetypes, month
|
|||
|
||||
return {
|
||||
fileGroups: content.file_groups,
|
||||
fileGroupsDeep: content.file_groups_deep,
|
||||
}
|
||||
} else {
|
||||
console.error('Failed to get file search estimates!');
|
||||
|
|
@ -2220,7 +2230,7 @@ export async function getDoc(fileGroupId) {
|
|||
}
|
||||
|
||||
|
||||
export async function searchDocs(offset, limit, searchStr, order, tagNames, author, subject, mimetype) {
|
||||
export async function searchDocs(offset, limit, searchStr, order, tagNames, author, subject, mimetype, deep) {
|
||||
offset = parseInt(offset || 0);
|
||||
limit = parseInt(limit || DEFAULT_LIMIT);
|
||||
let body = {offset, limit};
|
||||
|
|
@ -2242,6 +2252,9 @@ export async function searchDocs(offset, limit, searchStr, order, tagNames, auth
|
|||
if (mimetype) {
|
||||
body['mimetype'] = mimetype;
|
||||
}
|
||||
if (deep) {
|
||||
body['deep'] = true;
|
||||
}
|
||||
|
||||
console.debug('searching docs', body);
|
||||
const response = await apiPost(`${DOCS_API}/search`, body);
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ import {
|
|||
useSearchDomain,
|
||||
useSearchOrder
|
||||
} from "../hooks/customHooks";
|
||||
import {FileCards, FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
|
||||
import {DeepSearchHint, FileCards, FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
|
||||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
import _ from "lodash";
|
||||
import {ThemeContext} from "../contexts/contexts";
|
||||
|
|
@ -1259,8 +1259,10 @@ function ArchivesPage() {
|
|||
inputRef={searchInputRef}
|
||||
viewButton={viewButton}
|
||||
sorts={archiveOrders}
|
||||
showDeep={true}
|
||||
/>
|
||||
{body}
|
||||
<DeepSearchHint searchStr={searchStr} files={archives}/>
|
||||
{paginator}
|
||||
<TaggedDeleteConfirmModal
|
||||
open={taggedFileGroups !== null}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import {
|
|||
import {Button, darkTheme, Header, Icon, Segment, Statistic, Tab} from "./Theme";
|
||||
import {BulkTagModal} from "./BulkTagModal";
|
||||
import {TaggedDeleteConfirmModal} from "./TaggedDeleteConfirmModal";
|
||||
import {docMimetypeFilterOptions, FilesView, SearchControlBar} from "./Files";
|
||||
import {DeepSearchHint, docMimetypeFilterOptions, FilesView, SearchControlBar} from "./Files";
|
||||
import {useAuthors, useDoc, useOneQuery, useSearchDocs, useSubjects} from "../hooks/customHooks";
|
||||
import {TagsSelector} from "../Tags";
|
||||
import {toast} from "react-semantic-toasts-2";
|
||||
|
|
@ -138,8 +138,10 @@ function DocsPage() {
|
|||
viewButton={viewButton}
|
||||
sorts={docOrders}
|
||||
fileFilterOptions={docMimetypeFilterOptions}
|
||||
showDeep={true}
|
||||
/>
|
||||
{body}
|
||||
<DeepSearchHint searchStr={searchStr} files={docs}/>
|
||||
{paginator}
|
||||
<TaggedDeleteConfirmModal
|
||||
open={taggedFileGroups !== null}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ import {
|
|||
useSearchDate,
|
||||
useSearchFiles,
|
||||
useSearchCensored,
|
||||
useSearchDeep,
|
||||
useSearchFilter,
|
||||
useSearchOrder,
|
||||
useSearchView,
|
||||
|
|
@ -544,6 +545,7 @@ export function SearchFilterModal(
|
|||
showTags = true,
|
||||
showLimit = true,
|
||||
showCensored = false,
|
||||
showDeep = false,
|
||||
limitOptions = [12, 24, 48, 96],
|
||||
},
|
||||
) {
|
||||
|
|
@ -553,6 +555,7 @@ export function SearchFilterModal(
|
|||
const {dateRange, months} = useSearchDate();
|
||||
const {activeTags, anyTag} = useSearch();
|
||||
const {censored} = useSearchCensored();
|
||||
const {deep} = useSearchDeep();
|
||||
const {limit} = usePages(DEFAULT_SEARCH_LIMIT);
|
||||
const {updateQuery} = useContext(QueryContext);
|
||||
|
||||
|
|
@ -570,6 +573,7 @@ export function SearchFilterModal(
|
|||
const [draftMonths, setDraftMonths] = useState([]);
|
||||
const [draftRange, setDraftRange] = useState(emptyRange);
|
||||
const [draftCensored, setDraftCensored] = useState(false);
|
||||
const [draftDeep, setDraftDeep] = useState(false);
|
||||
|
||||
// Seed the drafts from the URL each time the modal is opened.
|
||||
React.useEffect(() => {
|
||||
|
|
@ -582,6 +586,7 @@ export function SearchFilterModal(
|
|||
setDraftMonths(seededMonths);
|
||||
setDraftRange(seededRange);
|
||||
setDraftCensored(censored || false);
|
||||
setDraftDeep(deep || false);
|
||||
}
|
||||
}, [open]);
|
||||
|
||||
|
|
@ -622,6 +627,9 @@ export function SearchFilterModal(
|
|||
if (showCensored && draftCensored !== censored) {
|
||||
params.censored = draftCensored ? 'true' : null;
|
||||
}
|
||||
if (showDeep && draftDeep !== deep) {
|
||||
params.deep = draftDeep ? 'true' : null;
|
||||
}
|
||||
if (Object.keys(params).length > 0) {
|
||||
params.o = 0;
|
||||
updateQuery(params);
|
||||
|
|
@ -641,6 +649,7 @@ export function SearchFilterModal(
|
|||
setDraftMonths([]);
|
||||
setDraftRange(emptyRange);
|
||||
setDraftCensored(false);
|
||||
setDraftDeep(false);
|
||||
}
|
||||
|
||||
// Sort section (operates on the draft).
|
||||
|
|
@ -754,6 +763,18 @@ export function SearchFilterModal(
|
|||
</Grid.Column>
|
||||
</Grid.Row>}
|
||||
|
||||
{showDeep &&
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<SearchFilterSection header='Search Depth'>
|
||||
<Toggle label='Deep search (captions, document text; slower)'
|
||||
checked={draftDeep}
|
||||
onChange={setDraftDeep}
|
||||
/>
|
||||
</SearchFilterSection>
|
||||
</Grid.Column>
|
||||
</Grid.Row>}
|
||||
|
||||
{showDates &&
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
|
|
@ -783,6 +804,7 @@ export function SearchFilterButton(
|
|||
showTags = true,
|
||||
showLimit = true,
|
||||
showCensored = false,
|
||||
showDeep = false,
|
||||
limitOptions = [12, 24, 48, 96],
|
||||
size = 'medium',
|
||||
content = 'Filter',
|
||||
|
|
@ -794,6 +816,7 @@ export function SearchFilterButton(
|
|||
const {dateRange, months} = useSearchDate();
|
||||
const {activeTags, anyTag} = useSearch();
|
||||
const {censored} = useSearchCensored();
|
||||
const {deep} = useSearchDeep();
|
||||
|
||||
let count = 0;
|
||||
if (fileFilterOptions && filter) {
|
||||
|
|
@ -811,6 +834,9 @@ export function SearchFilterButton(
|
|||
if (showCensored && censored) {
|
||||
count += 1;
|
||||
}
|
||||
if (showDeep && deep) {
|
||||
count += 1;
|
||||
}
|
||||
const active = count > 0;
|
||||
|
||||
return <>
|
||||
|
|
@ -829,6 +855,7 @@ export function SearchFilterButton(
|
|||
showTags={showTags}
|
||||
showLimit={showLimit}
|
||||
showCensored={showCensored}
|
||||
showDeep={showDeep}
|
||||
limitOptions={limitOptions}
|
||||
/>
|
||||
</>
|
||||
|
|
@ -849,6 +876,7 @@ export function SearchControlBar(
|
|||
fileFilterOptions = null,
|
||||
showDates = false,
|
||||
showCensored = false,
|
||||
showDeep = false,
|
||||
},
|
||||
) {
|
||||
const [localSearchStr, setLocalSearchStr] = useState(searchStr || '');
|
||||
|
|
@ -866,17 +894,39 @@ export function SearchControlBar(
|
|||
{viewButton}
|
||||
<div style={{flexGrow: 1, minWidth: 0}}>{searchInput}</div>
|
||||
<SearchFilterButton sorts={sorts} fileFilterOptions={fileFilterOptions} showDates={showDates}
|
||||
showCensored={showCensored}/>
|
||||
showCensored={showCensored} showDeep={showDeep}/>
|
||||
</div>
|
||||
}
|
||||
|
||||
// Offers deep search when a fast search found nothing. Used by the module pages (Videos/Archives/Docs),
|
||||
// which have no deep estimate; FilesSearchView has a count-aware variant.
|
||||
export function DeepSearchHint({searchStr, files}) {
|
||||
const {deep} = useSearchDeep();
|
||||
const {updateQuery} = useContext(QueryContext);
|
||||
|
||||
if (!searchStr || deep || !files || files.length > 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return <Segment>
|
||||
Not finding what you need?
|
||||
<Button primary
|
||||
onClick={() => updateQuery({deep: 'true', o: 0})}
|
||||
style={{marginLeft: '1em'}}
|
||||
>Try deep search</Button>
|
||||
</Segment>
|
||||
}
|
||||
|
||||
export function FilesSearchView({
|
||||
showView = true,
|
||||
emptySearch = false,
|
||||
model,
|
||||
deepEstimate = null,
|
||||
}) {
|
||||
|
||||
const {searchFiles, pages} = useSearchFiles(24, emptySearch, model);
|
||||
const {searchFiles, total, searchStr, pages} = useSearchFiles(24, emptySearch, model);
|
||||
const {deep} = useSearchDeep();
|
||||
const {updateQuery} = useContext(QueryContext);
|
||||
|
||||
const {body, paginator, viewButton} = FilesView(
|
||||
{
|
||||
|
|
@ -891,9 +941,38 @@ export function FilesSearchView({
|
|||
},
|
||||
);
|
||||
|
||||
// The default search only matches titles/authors/descriptions; offer deep search when it would find
|
||||
// more. `deepEstimate` comes from the search estimates; without it, fall back to a generic hint on
|
||||
// zero results.
|
||||
let deepHint = null;
|
||||
if (Boolean(searchStr) && !deep && searchFiles) {
|
||||
const moreCount = deepEstimate !== null && deepEstimate !== undefined && total !== null
|
||||
? deepEstimate - total : null;
|
||||
// Enable deep search and reset pagination in a single update (like the filter modal does), so the
|
||||
// user starts at page 1 of the deep results.
|
||||
const deepButton = <Button primary
|
||||
onClick={() => updateQuery({deep: 'true', o: 0})}
|
||||
style={{marginLeft: '1em'}}
|
||||
>Try deep search</Button>;
|
||||
if (searchFiles.length === 0 && (moreCount === null || moreCount > 0)) {
|
||||
deepHint = <Segment>
|
||||
{moreCount > 0
|
||||
? `No results in fast search. ${moreCount} result${moreCount === 1 ? '' : 's'} available in deep search.`
|
||||
: 'Not finding what you need?'}
|
||||
{deepButton}
|
||||
</Segment>;
|
||||
} else if (searchFiles.length > 0 && moreCount > 0) {
|
||||
deepHint = <Segment>
|
||||
{`${moreCount} more result${moreCount === 1 ? '' : 's'} available in deep search.`}
|
||||
{deepButton}
|
||||
</Segment>;
|
||||
}
|
||||
}
|
||||
|
||||
return <>
|
||||
{showView && <div style={{marginBottom: '1em'}}>{viewButton}</div>}
|
||||
{body}
|
||||
{deepHint}
|
||||
{paginator}
|
||||
</>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import React, {useState} from "react";
|
||||
import {Link, Route, Routes, useLocation, useNavigate} from "react-router";
|
||||
import {FilesSearchView, SearchViewButton} from "./Files";
|
||||
import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchFilter} from "../hooks/customHooks";
|
||||
import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchDeep, useSearchFilter} from "../hooks/customHooks";
|
||||
import {ShortcutHint} from "./ShortcutHint";
|
||||
import {ZimSearchView} from "./Zim";
|
||||
import {MapSearchView} from "./MapSearchView";
|
||||
|
|
@ -159,6 +159,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
|
|||
sendGeneralReqeust(async () => await searchSuggestions(searchStr));
|
||||
sendMapRequest(async () => await searchEstimateMap(searchStr));
|
||||
}
|
||||
// The files estimate always uses deep search server-side, so users know results are possible.
|
||||
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag));
|
||||
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
|
||||
sendOtherRequest(async () => await searchEstimateOthers(tagNames));
|
||||
|
|
@ -191,7 +192,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
|
|||
React.useEffect(() => {
|
||||
if (!_.isEmpty(filesData)) {
|
||||
setSuggestions((prevState) => {
|
||||
return {...prevState, fileGroups: filesData.fileGroups}
|
||||
return {...prevState, fileGroups: filesData.fileGroups, fileGroupsDeep: filesData.fileGroupsDeep}
|
||||
});
|
||||
}
|
||||
}, [setSuggestions, filesData]);
|
||||
|
|
@ -232,6 +233,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext);
|
||||
const {dateRange, months, setDates, clearDate} = useSearchDate();
|
||||
const {suggestions, loading} = useSuggestions(searchStr, searchTags, filter, anyTag);
|
||||
const {deep} = useSearchDeep();
|
||||
const {getLocationStr} = React.useContext(QueryContext);
|
||||
|
||||
// The results that will be displayed by <Search>.
|
||||
|
|
@ -251,19 +253,25 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
|
||||
let results = {};
|
||||
|
||||
// The dropdown always suggests the deep count so users know what is findable. The search results
|
||||
// page shows the active mode's count, and offers deep search when it found less than promised.
|
||||
const dropdownEstimate = newSuggestions.fileGroupsDeep ?? newSuggestions.fileGroups;
|
||||
// The tab badge shows the count matching the active search mode, so it agrees with the result list.
|
||||
const fileGroupsEstimate = deep ? newSuggestions.fileGroupsDeep : newSuggestions.fileGroups;
|
||||
|
||||
// Suggested results are ordered.
|
||||
if (newSuggestions.fileGroups > 0) {
|
||||
if (dropdownEstimate > 0) {
|
||||
results.fileGroups = {
|
||||
name: 'Files', results: [
|
||||
{
|
||||
title: newSuggestions.fileGroups.toString(),
|
||||
title: dropdownEstimate.toString(),
|
||||
type: 'files',
|
||||
// Add search query onto current location.
|
||||
location: getLocationStr({q: searchStr}, '/search'),
|
||||
}
|
||||
]
|
||||
};
|
||||
} else if (newSuggestions.fileGroups === 0) {
|
||||
} else if (dropdownEstimate === 0) {
|
||||
// Tell the user there are no files.
|
||||
results.fileGroups = {name: 'Files', results: noResults};
|
||||
}
|
||||
|
|
@ -389,7 +397,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
|
||||
setSuggestionsResults(results);
|
||||
setSuggestionsSums({
|
||||
fileGroups: newSuggestions.fileGroups,
|
||||
fileGroups: fileGroupsEstimate,
|
||||
fileGroupsDeep: newSuggestions.fileGroupsDeep,
|
||||
zims: zimSum,
|
||||
otherSum: otherSum,
|
||||
mapPlaces: newSuggestions.mapPlaces || 0,
|
||||
|
|
@ -419,7 +428,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
|
||||
normalizeSuggestionsResults(suggestions);
|
||||
}
|
||||
}, [JSON.stringify(suggestions)]);
|
||||
}, [JSON.stringify(suggestions), deep]);
|
||||
|
||||
// Clear timer on unmount.
|
||||
React.useEffect(() => {
|
||||
|
|
@ -495,7 +504,8 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
|
|||
return <React.Fragment>
|
||||
<TabLinks links={links} right={isFilesTab ? <SearchViewButton headlines/> : null}/>
|
||||
<Routes>
|
||||
<Route path='/*' element={<FilesSearchView showView={false}/>}/>
|
||||
<Route path='/*' element={<FilesSearchView showView={false}
|
||||
deepEstimate={suggestionsSums?.fileGroupsDeep}/>}/>
|
||||
<Route path='/zim' exact element={<ZimSearchView suggestions={suggestions} loading={loading}/>}/>
|
||||
<Route path='/map' exact element={<MapSearchView/>}/>
|
||||
<Route path='/other' exact element={<OtherSearchView loading={loading}/>}/>
|
||||
|
|
|
|||
|
|
@ -146,6 +146,36 @@ describe('SearchFilterButton', () => {
|
|||
expect(screen.getByText('1')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('counts the deep filter and applies it on close when showDeep is set', () => {
|
||||
const spy = jest.fn();
|
||||
renderButton({sorts: videoOrders, showDeep: true}, {}, spy);
|
||||
|
||||
// No deep badge initially.
|
||||
expect(screen.queryByText('1')).toBeNull();
|
||||
|
||||
fireEvent.click(screen.getByText('Filter'));
|
||||
// The Search Depth section/toggle is shown.
|
||||
expect(screen.getByText('Search Depth')).toBeInTheDocument();
|
||||
fireEvent.mouseUp(screen.getByTestId('toggle'));
|
||||
// Draft only — nothing applied yet.
|
||||
expect(spy).not.toHaveBeenCalled();
|
||||
|
||||
fireEvent.click(screen.getByText('Done'));
|
||||
expect(spy).toHaveBeenCalledTimes(1);
|
||||
expect(spy.mock.calls[0][0]).toMatchObject({deep: 'true'});
|
||||
});
|
||||
|
||||
it('does not show the Search Depth section unless showDeep is set', () => {
|
||||
renderButton({sorts: videoOrders});
|
||||
fireEvent.click(screen.getByText('Filter'));
|
||||
expect(screen.queryByText('Search Depth')).toBeNull();
|
||||
});
|
||||
|
||||
it('shows a count badge when the deep filter is active in the URL', () => {
|
||||
renderButton({sorts: videoOrders, showDeep: true}, {deep: 'true'});
|
||||
expect(screen.getByText('1')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows the File Type section when fileFilterOptions are provided', () => {
|
||||
renderButton({
|
||||
sorts: videoOrders,
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ import {
|
|||
TableCell
|
||||
} from "semantic-ui-react";
|
||||
import {useChannel, useSearchOrder, useSearchVideos, useVideo, useVideoStatistics} from "../hooks/customHooks";
|
||||
import {FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
|
||||
import {DeepSearchHint, FileRowTagIcon, FilesView, SearchControlBar} from "./Files";
|
||||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
|
||||
import {Button, Card, Header, Loader, Modal, Placeholder, Popup, Segment, Statistic} from "./Theme";
|
||||
|
|
@ -241,8 +241,10 @@ export function VideosPage() {
|
|||
viewButton={viewButton}
|
||||
sorts={videoOrders}
|
||||
showCensored={true}
|
||||
showDeep={true}
|
||||
/>
|
||||
{body}
|
||||
<DeepSearchHint searchStr={searchStr} files={videos}/>
|
||||
{paginator}
|
||||
<TaggedDeleteConfirmModal
|
||||
open={taggedFileGroups !== null}
|
||||
|
|
|
|||
|
|
@ -511,6 +511,7 @@ export const useSearchArchives = (defaultLimit) => {
|
|||
const activeTags = searchParams.getAll('tag');
|
||||
const {view} = useSearchView();
|
||||
const headline = view === 'headline';
|
||||
const {deep} = useSearchDeep();
|
||||
|
||||
const [archives, setArchives] = useState(null);
|
||||
const [totalPages, setTotalPages] = useState(0);
|
||||
|
|
@ -519,7 +520,8 @@ export const useSearchArchives = (defaultLimit) => {
|
|||
setArchives(null);
|
||||
setTotalPages(0);
|
||||
try {
|
||||
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags, headline);
|
||||
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags, headline,
|
||||
deep);
|
||||
setArchives(archives_);
|
||||
setTotalPages(calculateTotalPages(total, limit));
|
||||
} catch (e) {
|
||||
|
|
@ -536,7 +538,7 @@ export const useSearchArchives = (defaultLimit) => {
|
|||
|
||||
useEffect(() => {
|
||||
localSearchArchives();
|
||||
}, [searchStr, limit, domain, order, offset, JSON.stringify(activeTags), headline]);
|
||||
}, [searchStr, limit, domain, order, offset, JSON.stringify(activeTags), headline, deep]);
|
||||
|
||||
const setSearchStr = (value) => {
|
||||
updateQuery({q: value, o: 0, order: undefined});
|
||||
|
|
@ -580,6 +582,7 @@ export const useSearchDocs = (defaultLimit) => {
|
|||
const author = searchParams.get('author') || '';
|
||||
const subject = searchParams.get('subject') || '';
|
||||
const filter = searchParams.get('filter') || '';
|
||||
const {deep} = useSearchDeep();
|
||||
|
||||
const [docs, setDocs] = useState(null);
|
||||
const [totalPages, setTotalPages] = useState(0);
|
||||
|
|
@ -589,7 +592,8 @@ export const useSearchDocs = (defaultLimit) => {
|
|||
setTotalPages(0);
|
||||
try {
|
||||
const mimetype = docFilterToMimetype(filter);
|
||||
let [docs_, total] = await searchDocs(offset, limit, searchStr, order, activeTags, author, subject, mimetype);
|
||||
let [docs_, total] = await searchDocs(offset, limit, searchStr, order, activeTags, author, subject,
|
||||
mimetype, deep);
|
||||
setDocs(docs_);
|
||||
setTotalPages(calculateTotalPages(total, limit));
|
||||
} catch (e) {
|
||||
|
|
@ -600,7 +604,7 @@ export const useSearchDocs = (defaultLimit) => {
|
|||
|
||||
useEffect(() => {
|
||||
localSearchDocs();
|
||||
}, [searchStr, limit, order, offset, JSON.stringify(activeTags), author, subject, filter]);
|
||||
}, [searchStr, limit, order, offset, JSON.stringify(activeTags), author, subject, filter, deep]);
|
||||
|
||||
const setSearchStr = (value) => {
|
||||
updateQuery({q: value, o: 0, order: undefined});
|
||||
|
|
@ -639,6 +643,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
|
|||
const headline = view === 'headline';
|
||||
const anyTag = searchParams.get('anyTag') === 'true';
|
||||
const censored = searchParams.get('censored') === 'true';
|
||||
const {deep} = useSearchDeep();
|
||||
|
||||
const [videos, setVideos] = useState(null);
|
||||
const [totalPages, setTotalPages] = useState(0);
|
||||
|
|
@ -649,7 +654,8 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
|
|||
setLoading(true);
|
||||
setTotalPages(0);
|
||||
try {
|
||||
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline, censored);
|
||||
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline,
|
||||
censored, deep);
|
||||
setVideos(videos_);
|
||||
setTotalPages(calculateTotalPages(total, limit));
|
||||
} catch (e) {
|
||||
|
|
@ -662,7 +668,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
|
|||
|
||||
useEffect(() => {
|
||||
localSearchVideos();
|
||||
}, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags), headline, censored]);
|
||||
}, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags), headline, censored, deep]);
|
||||
|
||||
const setSearchStr = (value) => {
|
||||
updateQuery({q: value, o: 0, order: undefined});
|
||||
|
|
@ -912,8 +918,10 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
|
|||
dateRange,
|
||||
} = useSearch(defaultLimit, emptySearch, model);
|
||||
const {view} = useSearchView();
|
||||
const {deep} = useSearchDeep();
|
||||
|
||||
const [searchFiles, setSearchFiles] = useState(null);
|
||||
const [total, setTotal] = useState(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const headline = view === 'headline';
|
||||
|
||||
|
|
@ -932,12 +940,14 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
|
|||
setLoading(true);
|
||||
console.log('localSearchFiles', fromDate, toDate, months);
|
||||
try {
|
||||
let [file_groups, total] = await filesSearch(
|
||||
let [file_groups, total_] = await filesSearch(
|
||||
pages.offset, pages.limit, searchStr, mimetypes, model || model_, activeTags, headline,
|
||||
months, fromDate, toDate, anyTag);
|
||||
months, fromDate, toDate, anyTag, null, null, null, deep);
|
||||
setSearchFiles(file_groups);
|
||||
pages.setTotal(total);
|
||||
setTotal(total_);
|
||||
pages.setTotal(total_);
|
||||
} catch (e) {
|
||||
setTotal(null);
|
||||
pages.setTotal(0);
|
||||
console.error(e);
|
||||
toast({
|
||||
|
|
@ -974,10 +984,12 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
|
|||
JSON.stringify(months),
|
||||
JSON.stringify(dateRange),
|
||||
anyTag,
|
||||
deep,
|
||||
]);
|
||||
|
||||
return {
|
||||
searchFiles,
|
||||
total,
|
||||
searchStr,
|
||||
filter,
|
||||
setSearchStr,
|
||||
|
|
@ -1655,6 +1667,14 @@ export const useSearchCensored = () => {
|
|||
return {censored, setCensored}
|
||||
}
|
||||
|
||||
export const useSearchDeep = () => {
|
||||
// Search all text including captions/document contents (?deep=true). Slower, but more thorough.
|
||||
const [value, setValue] = useOneQuery('deep');
|
||||
const deep = value === 'true';
|
||||
const setDeep = (newValue) => setValue(newValue ? 'true' : undefined);
|
||||
return {deep, setDeep}
|
||||
}
|
||||
|
||||
export const useSearchDate = () => {
|
||||
// Filters files using
|
||||
// ?month=1&month=2&month=3
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest):
|
|||
offset = body.offset or 0
|
||||
|
||||
file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names,
|
||||
body.headline)
|
||||
body.headline, body.deep)
|
||||
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
|
||||
return json_response(ret)
|
||||
|
||||
|
|
|
|||
|
|
@ -1400,7 +1400,7 @@ NO_NULL_ORDERS = {
|
|||
|
||||
|
||||
def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str],
|
||||
headline: bool = False) \
|
||||
headline: bool = False, deep: bool = False) \
|
||||
-> Tuple[List[dict], int]:
|
||||
# Always filter FileGroups to Archives.
|
||||
wheres = []
|
||||
|
|
@ -1412,9 +1412,11 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
|
|||
select_columns = ''
|
||||
if search_str:
|
||||
# A search_str was provided by the user, modify the query to filter by it.
|
||||
select_columns = 'ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)) AS rank,' \
|
||||
# `textsearch` includes d_text (page contents); `textsearch_abc` is much smaller and faster.
|
||||
ts_col = 'textsearch' if deep else 'textsearch_abc'
|
||||
select_columns = f'ts_rank(fg.{ts_col}, websearch_to_tsquery(%(search_str)s)) AS rank,' \
|
||||
' COUNT(*) OVER() AS total'
|
||||
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
|
||||
wheres.append(f'fg.{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
|
||||
params['search_str'] = search_str
|
||||
group_by = f'{group_by}, rank'
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ class ArchiveSearchRequest:
|
|||
order_by: Optional[str] = None
|
||||
tag_names: List[str] = field(default_factory=list)
|
||||
headline: bool = False
|
||||
deep: bool = False # When True, search all text including page contents (d_text)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -50,36 +50,43 @@ async def test_archives_search(test_session, archive_directory, archive_factory,
|
|||
archive_factory('example.org') # has no contents
|
||||
test_session.commit()
|
||||
|
||||
# Contents are d_text, so a deep search is required to match them.
|
||||
# 1 and 2 contain "foo".
|
||||
data = {'search_str': 'foo'}
|
||||
data = {'search_str': 'foo', 'deep': True}
|
||||
await check_results(async_client, data, [2, 1])
|
||||
|
||||
# The default (fast) search does not match contents.
|
||||
data = {'search_str': 'foo'}
|
||||
await check_results(async_client, data, [])
|
||||
|
||||
# 2 and 3 contain "baz".
|
||||
data = {'search_str': 'baz'}
|
||||
data = {'search_str': 'baz', 'deep': True}
|
||||
await check_results(async_client, data, [3, 2])
|
||||
|
||||
# 1 contains "bar".
|
||||
data = {'search_str': 'bar'}
|
||||
data = {'search_str': 'bar', 'deep': True}
|
||||
await check_results(async_client, data, [1, ])
|
||||
|
||||
# No archives contain "huzzah"
|
||||
data = {'search_str': 'huzzah'}
|
||||
data = {'search_str': 'huzzah', 'deep': True}
|
||||
await check_results(async_client, data, [])
|
||||
|
||||
# Only 3 contains "baz" and is in domain "example.org"
|
||||
data = {'search_str': 'baz', 'domain': 'example.org'}
|
||||
data = {'search_str': 'baz', 'domain': 'example.org', 'deep': True}
|
||||
await check_results(async_client, data, [3, ])
|
||||
|
||||
# 1's title contains "my", this is ignored by Postgres.
|
||||
data = {'search_str': 'my'}
|
||||
data = {'search_str': 'my', 'deep': True}
|
||||
await check_results(async_client, data, [])
|
||||
|
||||
# 3's title contains "third".
|
||||
# 3's title contains "third", found by the default (fast) search and the deep search.
|
||||
data = {'search_str': 'third'}
|
||||
await check_results(async_client, data, [3, ])
|
||||
data = {'search_str': 'third', 'deep': True}
|
||||
await check_results(async_client, data, [3, ])
|
||||
|
||||
# All contents contain "qux", but they contain different amounts. They are ordered by the amount.
|
||||
data = {'search_str': 'qux'}
|
||||
data = {'search_str': 'qux', 'deep': True}
|
||||
await check_results(async_client, data, [3, 2, 1])
|
||||
|
||||
data = {'search_str': 'qux', 'order_by': 'bad order_by'}
|
||||
|
|
@ -116,7 +123,8 @@ async def test_archives_search_headline(test_session, archive_directory, archive
|
|||
archive_factory('example.org') # has no contents
|
||||
test_session.commit()
|
||||
|
||||
content = dict(search_str='foo')
|
||||
# "foo" is only in the contents (d_text), so a deep search is required to match it.
|
||||
content = dict(search_str='foo', deep=True)
|
||||
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
|
|
@ -124,7 +132,7 @@ async def test_archives_search_headline(test_session, archive_directory, archive
|
|||
assert response.json['file_groups'][0]['d_headline'] is None
|
||||
assert response.json['file_groups'][1]['d_headline'] is None
|
||||
|
||||
content = dict(search_str='foo', headline=True)
|
||||
content = dict(search_str='foo', headline=True, deep=True)
|
||||
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
|
|
|
|||
|
|
@ -571,7 +571,8 @@ async def test_refresh_archives(test_session, test_directory, async_client, make
|
|||
assert (test_directory / 'archive/example.com/2021-10-05-16-20-10_NA.html').is_file()
|
||||
assert test_session.query(Archive).count() == 2
|
||||
|
||||
content = json.dumps({'search_str': 'text', 'model': 'archive'})
|
||||
# "text" is only in the readability contents (d_text), so a deep search is required.
|
||||
content = json.dumps({'search_str': 'text', 'model': 'archive', 'deep': True})
|
||||
request, response = await async_client.post('/api/files/search', content=content)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['file_groups'], 'No files matched "text"'
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ async def search_docs(_: Request, body: schema.DocSearchRequest):
|
|||
offset=offset,
|
||||
order_by=body.order_by,
|
||||
tag_names=body.tag_names,
|
||||
deep=body.deep,
|
||||
)
|
||||
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
|
||||
return json_response(ret)
|
||||
|
|
|
|||
|
|
@ -347,15 +347,18 @@ def _get_doc(session, file_group_id: int):
|
|||
|
||||
|
||||
def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None,
|
||||
limit=20, offset=0, order_by='published_datetime', tag_names=None):
|
||||
limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False):
|
||||
from .models import Doc
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
# `textsearch` includes d_text (document contents); `textsearch_abc` is much smaller and faster.
|
||||
ts_col = FileGroup.textsearch if deep else FileGroup.textsearch_abc
|
||||
|
||||
with get_db_session() as session:
|
||||
query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id)
|
||||
|
||||
if search_str:
|
||||
query = query.filter(FileGroup.textsearch.op('@@')(func.websearch_to_tsquery(search_str)))
|
||||
query = query.filter(ts_col.op('@@')(func.websearch_to_tsquery(search_str)))
|
||||
|
||||
if author:
|
||||
query = query.filter(FileGroup.b_text.ilike(f'%{author}%'))
|
||||
|
|
@ -382,7 +385,7 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime
|
|||
# Ordering.
|
||||
if order_by == 'rank' and search_str:
|
||||
query = query.order_by(
|
||||
desc(func.ts_rank(FileGroup.textsearch, func.websearch_to_tsquery(search_str))),
|
||||
desc(func.ts_rank(ts_col, func.websearch_to_tsquery(search_str))),
|
||||
desc(FileGroup.id),
|
||||
)
|
||||
elif order_by == 'published_datetime':
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ class DocSearchRequest:
|
|||
order_by: Optional[str] = 'published_datetime'
|
||||
limit: int = 20
|
||||
offset: int = 0
|
||||
deep: bool = False # When True, search all text including document contents (d_text)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
|
|
|
|||
|
|
@ -289,7 +289,11 @@ async def test_search_docs_attaches_section_hint_epub(test_session, test_directo
|
|||
(2, 'Chapter 2', 'unrelated content about pottery'),
|
||||
])
|
||||
|
||||
# "mullen" is only in the document body (d_text), so a deep search is required to match it.
|
||||
results, total = _search_docs(search_str='mullen', mimetype='application/epub')
|
||||
assert total == 0, 'Document contents are not searched unless deep=True'
|
||||
|
||||
results, total = _search_docs(search_str='mullen', mimetype='application/epub', deep=True)
|
||||
assert total == 1
|
||||
assert results[0]['id'] == fg.id
|
||||
hint = results[0]['section_hint']
|
||||
|
|
@ -311,7 +315,7 @@ async def test_search_docs_attaches_section_hint_pdf(test_session, test_director
|
|||
(3, 'Page 3', 'here is where mullen shows up'),
|
||||
])
|
||||
|
||||
results, total = _search_docs(search_str='mullen', mimetype='application/pdf')
|
||||
results, total = _search_docs(search_str='mullen', mimetype='application/pdf', deep=True)
|
||||
assert total == 1
|
||||
assert results[0]['id'] == fg.id
|
||||
hint = results[0]['section_hint']
|
||||
|
|
|
|||
|
|
@ -336,6 +336,7 @@ def assert_video_search(async_client):
|
|||
order_by: str = None,
|
||||
channel_id: int = None,
|
||||
censored: bool = None,
|
||||
deep: bool = None,
|
||||
):
|
||||
content = dict()
|
||||
if search_str is not None:
|
||||
|
|
@ -352,6 +353,8 @@ def assert_video_search(async_client):
|
|||
content['channel_id'] = channel_id
|
||||
if censored is not None:
|
||||
content['censored'] = censored
|
||||
if deep is not None:
|
||||
content['deep'] = deep
|
||||
|
||||
request, response = await async_client.post('/api/videos/search', content=json.dumps(content))
|
||||
|
||||
|
|
|
|||
|
|
@ -178,6 +178,7 @@ class VideoSearchRequest:
|
|||
tag_names: List[str] = field(default_factory=list)
|
||||
headline: bool = False
|
||||
censored: bool = False
|
||||
deep: bool = False # When True, search all text including captions (d_text)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -195,31 +195,36 @@ async def test_search_videos_file(test_session, test_directory, video_with_searc
|
|||
video_with_search_factory(title=title, d_text=caption)
|
||||
test_session.commit()
|
||||
|
||||
# Captions are d_text, so a deep search is required to match them.
|
||||
# Repeated runs should return the same result
|
||||
for _ in range(2):
|
||||
# Only videos with a b are returned, ordered by the amount of b's
|
||||
await assert_video_search(search_str='b', assert_ids=[1, 2, 3, 4])
|
||||
await assert_video_search(search_str='b', deep=True, assert_ids=[1, 2, 3, 4])
|
||||
|
||||
# The default (fast) search does not match captions.
|
||||
await assert_video_search(search_str='b', assert_ids=[])
|
||||
|
||||
# Only two captions have e
|
||||
await assert_video_search(search_str='e', assert_ids=[4, 1])
|
||||
await assert_video_search(search_str='e', deep=True, assert_ids=[4, 1])
|
||||
|
||||
# Only 2 contains 2
|
||||
await assert_video_search(search_str='2', assert_ids=[2, ])
|
||||
await assert_video_search(search_str='2', deep=True, assert_ids=[2, ])
|
||||
|
||||
# Only two captions have d
|
||||
await assert_video_search(search_str='d', assert_ids=[1, 2])
|
||||
await assert_video_search(search_str='d', deep=True, assert_ids=[1, 2])
|
||||
|
||||
# 5 can be gotten by it's title
|
||||
# 5 can be gotten by it's title, in either mode.
|
||||
await assert_video_search(search_str='5', assert_ids=[5, ])
|
||||
await assert_video_search(search_str='5', deep=True, assert_ids=[5, ])
|
||||
|
||||
# only video 1 has e and d
|
||||
await assert_video_search(search_str='e d', assert_ids=[1, ])
|
||||
await assert_video_search(search_str='e d', deep=True, assert_ids=[1, ])
|
||||
|
||||
# video 1 and 4 have b and e, but 1 has more
|
||||
await assert_video_search(search_str='b e', assert_ids=[1, 4])
|
||||
await assert_video_search(search_str='b e', deep=True, assert_ids=[1, 4])
|
||||
|
||||
# Check totals are correct even with a limit
|
||||
await assert_video_search(search_str='b', limit=2, assert_ids=[1, 2], assert_total=4)
|
||||
await assert_video_search(search_str='b', deep=True, limit=2, assert_ids=[1, 2], assert_total=4)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from modules.videos.models import Channel, Video
|
|||
from wrolpi.conftest import test_directory, await_switches
|
||||
from wrolpi.downloader import Download, DownloadResult, get_download_manager_config
|
||||
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
|
||||
from wrolpi.test.common import skip_circleci
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
example_video_json = {
|
||||
|
|
@ -130,6 +131,7 @@ async def test_download_video_tags(test_session, video_download_manager, video_f
|
|||
assert [i.tag.name for i in video.file_group.tag_files] == [tag1.name, tag2.name]
|
||||
|
||||
|
||||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_channel(test_session, test_directory, simple_channel, video_download_manager, video_file,
|
||||
mock_video_extract_info, mock_video_prepare_filename, await_switches,
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ async def search_videos(_: Request, body: schema.VideoSearchRequest):
|
|||
body.tag_names,
|
||||
body.headline,
|
||||
body.censored,
|
||||
body.deep,
|
||||
)
|
||||
|
||||
ret = {'file_groups': list(file_groups), 'totals': {'file_groups': videos_total}}
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ def search_videos(
|
|||
tag_names: List[str] = None,
|
||||
headline: bool = False,
|
||||
censored: bool = False,
|
||||
deep: bool = False,
|
||||
) -> Tuple[List[dict], int]:
|
||||
tag_names = tag_names or []
|
||||
# Search videos and audio-only files.
|
||||
|
|
@ -112,8 +113,10 @@ def search_videos(
|
|||
|
||||
if search_str:
|
||||
# A search_str was provided by the user, modify the query to filter by it.
|
||||
select_columns = 'fg.id, ts_rank(fg.textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
|
||||
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
|
||||
# `textsearch` includes d_text (captions); `textsearch_abc` is much smaller and faster.
|
||||
ts_col = 'textsearch' if deep else 'textsearch_abc'
|
||||
select_columns = f'fg.id, ts_rank(fg.{ts_col}, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
|
||||
wheres.append(f'fg.{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
|
||||
params['search_str'] = search_str
|
||||
else:
|
||||
# No search_str provided. Get id and total only.
|
||||
|
|
|
|||
|
|
@ -165,7 +165,7 @@ async def post_search_files(_: Request, body: schema.FilesSearchRequest):
|
|||
with timer('Searching all files', 'info', logger__=logger):
|
||||
file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model,
|
||||
body.tag_names, body.headline, body.months, body.from_year, body.to_year,
|
||||
body.any_tag, body.order, body.url, body.suffix, body.path)
|
||||
body.any_tag, body.order, body.url, body.suffix, body.path, body.deep)
|
||||
return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total)))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1039,7 +1039,7 @@ async def search_directories_by_name(session: Session, name: str, excluded: List
|
|||
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
|
||||
tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
|
||||
from_year: int = None, to_year: int = None, any_tag: bool = False, order: str = None,
|
||||
url: str = None, suffix: str = None, path: str = None) -> \
|
||||
url: str = None, suffix: str = None, path: str = None, deep: bool = False) -> \
|
||||
Tuple[List[dict], int]:
|
||||
"""Search the FileGroup table.
|
||||
|
||||
|
|
@ -1061,6 +1061,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
|
|||
@param url: Filter by URL using case-insensitive partial match (ILIKE).
|
||||
@param suffix: Only return files whose primary file has this suffix (e.g. ".bin"), case-insensitive.
|
||||
@param path: Filter by primary_path using case-insensitive partial match (ILIKE).
|
||||
@param deep: Search all text including captions/body (d_text); slower but more thorough.
|
||||
"""
|
||||
params = dict(offset=offset, limit=limit, search_str=search_str, url_search_str=f'%{search_str}%')
|
||||
wheres = []
|
||||
|
|
@ -1069,9 +1070,10 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
|
|||
joins = []
|
||||
|
||||
if search_str:
|
||||
# Search by textsearch column.
|
||||
wheres.append('textsearch @@ websearch_to_tsquery(%(search_str)s)')
|
||||
selects.append('ts_rank(textsearch, websearch_to_tsquery(%(search_str)s))')
|
||||
# `textsearch` includes d_text (captions/body); `textsearch_abc` is much smaller and faster.
|
||||
ts_col = 'textsearch' if deep else 'textsearch_abc'
|
||||
wheres.append(f'{ts_col} @@ websearch_to_tsquery(%(search_str)s)')
|
||||
selects.append(f'ts_rank({ts_col}, websearch_to_tsquery(%(search_str)s))')
|
||||
order_by = '2 DESC'
|
||||
|
||||
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
|
||||
|
|
@ -1479,9 +1481,13 @@ def mimetypes_to_sql_wheres(wheres: List[str], params: dict, mimetypes: List[str
|
|||
|
||||
async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str],
|
||||
months: List[int] = None, from_year: int = None, to_year: int = None,
|
||||
any_tag: bool = False):
|
||||
any_tag: bool = False) -> dict:
|
||||
"""
|
||||
Return FileGroup count of what will be returned if the search is actually performed.
|
||||
Return FileGroup counts of what will be returned if the search is actually performed.
|
||||
|
||||
Returns both the fast (`textsearch_abc`) and deep (`textsearch`) counts so the UI can show the count
|
||||
matching the current search mode, and offer deep search when it has more results. Both are cheap
|
||||
index-bound COUNTs (no ts_rank).
|
||||
"""
|
||||
wheres = []
|
||||
joins = list()
|
||||
|
|
@ -1490,8 +1496,13 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
|
|||
having = ''
|
||||
|
||||
if search_str:
|
||||
# Search by textsearch, and by matching url.
|
||||
# `textsearch` matches are a superset of `textsearch_abc`, so filter by the deep column and count
|
||||
# the fast subset with FILTER in the same pass.
|
||||
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
|
||||
selects = ('COUNT(*) FILTER (WHERE fg.textsearch_abc @@ websearch_to_tsquery(%(search_str)s)) AS estimate,'
|
||||
' COUNT(*) AS deep_estimate')
|
||||
else:
|
||||
selects = 'COUNT(*) AS estimate, COUNT(*) AS deep_estimate'
|
||||
|
||||
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
|
||||
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
|
||||
|
|
@ -1501,7 +1512,7 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
|
|||
joins = "\n".join(joins)
|
||||
wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
|
||||
stmt = f'''
|
||||
SELECT COUNT(*) OVER() AS estimate
|
||||
SELECT {selects}
|
||||
FROM file_group fg
|
||||
{joins}
|
||||
{wheres}
|
||||
|
|
@ -1514,8 +1525,8 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
|
|||
curs.execute(stmt, params)
|
||||
result = curs.fetchone()
|
||||
if result:
|
||||
return int(result['estimate'])
|
||||
return 0
|
||||
return dict(file_groups=int(result['estimate']), file_groups_deep=int(result['deep_estimate']))
|
||||
return dict(file_groups=0, file_groups_deep=0)
|
||||
|
||||
|
||||
MOVE_CHUNK_SIZE = 100
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ class FileGroup(ModelHelper, Base):
|
|||
Index('file_group_published_datetime_idx', 'published_datetime'),
|
||||
Index('file_group_published_modified_datetime_idx', 'published_modified_datetime'),
|
||||
Index('file_group_size_ix', 'size'),
|
||||
Index('file_group_textsearch_abc_idx', 'textsearch_abc'),
|
||||
Index('file_group_textsearch_idx', 'textsearch'),
|
||||
Index('file_group_url_idx', 'url'),
|
||||
Index('file_group_viewed_idx', 'viewed'),
|
||||
|
|
@ -126,6 +127,13 @@ class FileGroup(ModelHelper, Base):
|
|||
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char")
|
||||
''')))
|
||||
# Excludes d_text (captions/body); much smaller than `textsearch`, used for the default fast search.
|
||||
textsearch_abc = deferred(
|
||||
Column(tsvector, Computed('''
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") ||
|
||||
setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char")
|
||||
''')))
|
||||
|
||||
def __repr__(self):
|
||||
m = f'model={self.model}' if self.model else f'mimetype={self.mimetype}'
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ class FilesSearchRequest:
|
|||
url: Optional[str] = None # Filter by URL (partial match)
|
||||
suffix: Optional[str] = None # Filter by primary file suffix (e.g. ".bin"), case-insensitive exact match
|
||||
path: Optional[str] = None # Filter by primary_path (case-insensitive partial match)
|
||||
deep: bool = False # When True, search all text including captions/body (d_text)
|
||||
|
||||
def __post_init__(self):
|
||||
if self.any_tag and self.tag_names:
|
||||
|
|
|
|||
|
|
@ -315,6 +315,51 @@ async def test_files_search(test_session, async_client, make_files_structure, as
|
|||
await assert_files_search('nothing', [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_files_search_deep(test_session, async_client, make_files_structure):
|
||||
"""Search defaults to the fast ABC column (title/author/description); `deep` opts into d_text
|
||||
(captions/file contents)."""
|
||||
guide, lecture = make_files_structure([
|
||||
'gardening guide.txt',
|
||||
'lecture.txt',
|
||||
])
|
||||
guide.write_text('planting tips')
|
||||
# "gardening" only appears in this file's contents (d_text), not its name.
|
||||
lecture.write_text('a lecture about gardening')
|
||||
|
||||
request, response = await async_client.post('/api/files/refresh')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
async def search(**kwargs):
|
||||
request, response = await async_client.post('/api/files/search', json=kwargs)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
return response.json['file_groups']
|
||||
|
||||
# Default (fast) search only matches the title.
|
||||
results = await search(search_str='gardening')
|
||||
assert [i['primary_path'] for i in results] == ['gardening guide.txt']
|
||||
|
||||
# Deep search also matches file contents; the title match ranks higher (weight A vs D).
|
||||
results = await search(search_str='gardening', deep=True)
|
||||
assert [i['primary_path'] for i in results] == ['gardening guide.txt', 'lecture.txt']
|
||||
|
||||
# Headlines work in both modes.
|
||||
results = await search(search_str='gardening', headline=True)
|
||||
assert [i['primary_path'] for i in results] == ['gardening guide.txt']
|
||||
assert '<b>gardening</b>' in results[0]['title_headline']
|
||||
results = await search(search_str='gardening', deep=True, headline=True)
|
||||
assert [i['primary_path'] for i in results] == ['gardening guide.txt', 'lecture.txt']
|
||||
assert '<b>gardening</b>' in results[1]['d_headline']
|
||||
|
||||
# The estimate endpoint returns both counts: `file_groups` matches the default fast search, and
|
||||
# `file_groups_deep` shows what deep search would find, so the UI can offer deep search.
|
||||
request, response = await async_client.post('/api/search_file_estimates',
|
||||
json=dict(search_str='gardening'))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['file_groups'] == 1
|
||||
assert response.json['file_groups_deep'] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_files_search_by_suffix(test_session, async_client, make_files_structure):
|
||||
"""Files can be filtered by their primary file's suffix (used by the firmware flasher to find .bin files)."""
|
||||
|
|
@ -367,7 +412,8 @@ async def test_files_search_attaches_doc_section_hint(async_client, test_session
|
|||
bug where only the docs-specific endpoint (/api/docs/search) attached hints."""
|
||||
await refresh_files()
|
||||
|
||||
body = dict(search_str='chapter')
|
||||
# "chapter" only appears in the EPUB body (d_text), so deep search is required to match it.
|
||||
body = dict(search_str='chapter', deep=True)
|
||||
request, response = await async_client.post('/api/files/search', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
|
|
|
|||
|
|
@ -580,18 +580,20 @@ async def test_files_indexer(async_client, test_session, make_files_structure, t
|
|||
# Video are indexed by the modeler, not by an indexer.
|
||||
assert video_file.mimetype == 'video/mp4' and video_file.indexer == indexers.DefaultIndexer
|
||||
|
||||
# File are indexed by their titles and contents.
|
||||
# File are indexed by their titles and contents. Contents (d_text) require a deep search.
|
||||
files, total = lib.search_files('file', 10, 0)
|
||||
assert total == 5, 'All files contain "file" in their file name. The associated video file is hidden.'
|
||||
files, total = lib.search_files('image', 10, 0)
|
||||
assert total == 1 and files[0]['title'] == 'an image file.jpeg', 'The image file title contains "image".'
|
||||
files, total = lib.search_files('contents', 10, 0)
|
||||
files, total = lib.search_files('contents', 10, 0, deep=True)
|
||||
assert total == 1 and files[0]['title'] == 'a text file.txt', 'The text file contains "contents".'
|
||||
files, total = lib.search_files('contents', 10, 0)
|
||||
assert total == 0, 'File contents are not searched unless deep=True.'
|
||||
files, total = lib.search_files('video', 10, 0)
|
||||
assert total == 1 and {i['title'] for i in files} == {'a video file'}, 'The video file contains "video".'
|
||||
files, total = lib.search_files('yawn', 10, 0)
|
||||
files, total = lib.search_files('yawn', 10, 0, deep=True)
|
||||
assert total == 1 and files[0]['title'] == 'a video file', 'The video file captions contain "yawn".'
|
||||
files, total = lib.search_files('bunny', 10, 0)
|
||||
files, total = lib.search_files('bunny', 10, 0, deep=True)
|
||||
assert total == 1 and {i['title'] for i in files} == {'a zip file.zip'}, \
|
||||
'The zip file contains a file with "bunny" in the title.'
|
||||
|
||||
|
|
@ -603,7 +605,7 @@ async def test_files_indexer(async_client, test_session, make_files_structure, t
|
|||
text_path.write_text('new text contents')
|
||||
await refresh_files()
|
||||
await await_background_tasks()
|
||||
files, total = lib.search_files('new', 10, 0)
|
||||
files, total = lib.search_files('new', 10, 0, deep=True)
|
||||
assert total == 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -768,7 +768,7 @@ async def post_search_suggestions(request: Request, body: schema.SearchSuggestio
|
|||
)
|
||||
async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimateRequest):
|
||||
"""Used by the Global search to suggest FileGroup count to the user."""
|
||||
file_groups = await search_file_suggestion_count(
|
||||
counts = await search_file_suggestion_count(
|
||||
body.search_str,
|
||||
body.tag_names,
|
||||
body.mimetypes,
|
||||
|
|
@ -779,7 +779,8 @@ async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimate
|
|||
)
|
||||
|
||||
ret = dict(
|
||||
file_groups=file_groups,
|
||||
file_groups=counts['file_groups'],
|
||||
file_groups_deep=counts['file_groups_deep'],
|
||||
)
|
||||
return json_response(ret)
|
||||
|
||||
|
|
|
|||
|
|
@ -666,7 +666,8 @@ async def test_search_file_estimates(test_session, async_client, archive_factory
|
|||
|
||||
request, response = await async_client.post('/api/search_file_estimates', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['file_groups'] == expected_file_groups or 0
|
||||
# The deep count includes contents/captions matches; the fast (ABC) count is in `file_groups`.
|
||||
assert response.json['file_groups_deep'] == expected_file_groups or 0
|
||||
|
||||
await assert_results(dict(search_str='foo'), 1)
|
||||
|
||||
|
|
@ -727,7 +728,8 @@ async def test_search_file_estimates_any_tag(test_session, async_client, archive
|
|||
|
||||
request, response = await async_client.post('/api/search_file_estimates', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['file_groups'] == expected_file_groups or 0
|
||||
# The deep count includes contents/captions matches; the fast (ABC) count is in `file_groups`.
|
||||
assert response.json['file_groups_deep'] == expected_file_groups or 0
|
||||
|
||||
await assert_results(dict(search_str='bunny'), 2)
|
||||
await assert_results(dict(search_str='bunny', any_tag=True), 1)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue