diff --git a/app/src/api.js b/app/src/api.js index 93d2dbbf..f44c20e8 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -1,5 +1,6 @@ import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, emptyToNull, OTP_API, VIDEOS_API} from "./components/Common"; import {toast} from "react-semantic-toasts-2"; +import {computeHeadingLevel} from "@testing-library/react"; function timeoutPromise(ms, promise) { // Create a timeout wrapper around a promise. If the timeout is reached, throw an error. Otherwise, return @@ -103,7 +104,7 @@ export async function getChannel(id) { return (await response.json())['channel']; } -export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames) { +export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline) { // Build a search query to retrieve a list of videos from the API offset = parseInt(offset || 0); limit = parseInt(limit || DEFAULT_LIMIT); @@ -118,6 +119,9 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by if (tagNames) { body['tag_names'] = tagNames; } + if (headline) { + body['headline'] = true; + } console.debug('searching videos', body); @@ -350,7 +354,7 @@ export async function deleteArchives(archiveIds) { } } -export async function searchArchives(offset, limit, domain, searchStr, order, tagNames) { +export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline) { // Build a search query to retrieve a list of videos from the API offset = parseInt(offset || 0); limit = parseInt(limit || DEFAULT_LIMIT); @@ -367,6 +371,9 @@ export async function searchArchives(offset, limit, domain, searchStr, order, ta if (tagNames) { body['tag_names'] = tagNames; } + if (headline) { + body['headline'] = true; + } console.debug('searching archives', body); let response = await apiPost(`${ARCHIVES_API}/search`, body); @@ -479,7 +486,7 @@ export async function deleteDownload(downloadId) { } } -export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames) { +export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline) { const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)}; if (mimetypes) { body['mimetypes'] = mimetypes; @@ -490,6 +497,9 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta if (tagNames) { body['tag_names'] = tagNames; } + if (headline) { + body['headline'] = true; + } console.debug('searching files', body); const response = await apiPost(`${API_URI}/files/search`, body); diff --git a/app/src/components/Headline.js b/app/src/components/Headline.js index 6948beae..3ba88fa7 100644 --- a/app/src/components/Headline.js +++ b/app/src/components/Headline.js @@ -76,25 +76,24 @@ export const Headlines = ({results}) => { return No results! } - let items = []; + let headlines = []; for (let i = 0; i < results.length; i++) { const result = results[i]; const {model, data, video} = result; if (model === 'video' && !_.isEmpty(video)) { - let video_url = `/videos/video/${video.id}`; - const channel = video.channel ? video.channel : null; - if (channel) { - video_url = `/videos/channel/${channel.id}/video/${video.id}`; - } - items = [...items, ]; + const {channel} = video; + const video_url = channel + ? `/videos/channel/${channel.id}/video/${video.id}` + : `/videos/video/${video.id}`; + headlines = [...headlines, ]; } else if (model === 'archive' && !_.isEmpty(data)) { - items = [...items, ]; + headlines = [...headlines, ]; } else { - items = [...items, ]; + headlines = [...headlines, ]; } } return <> - {items} + {headlines} } \ No newline at end of file diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index e7fdd7d1..0b2360b4 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -172,6 +172,8 @@ export const useSearchArchives = (defaultLimit) => { const searchStr = searchParams.get('q') || ''; const order = searchParams.get('order'); const activeTags = searchParams.getAll('tag'); + const {view} = useSearchView(); + const headline = view === 'headline'; const [archives, setArchives] = useState(null); const [totalPages, setTotalPages] = useState(0); @@ -180,7 +182,7 @@ export const useSearchArchives = (defaultLimit) => { setArchives(null); setTotalPages(0); try { - let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags); + let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags, headline); setArchives(archives_); setTotalPages(calculateTotalPages(total, limit)); } catch (e) { @@ -197,7 +199,7 @@ export const useSearchArchives = (defaultLimit) => { useEffect(() => { localSearchArchives(); - }, [searchStr, limit, domain, order, activePage, JSON.stringify(activeTags)]); + }, [searchStr, limit, domain, order, activePage, JSON.stringify(activeTags), headline]); const setSearchStr = (value) => { updateQuery({q: value, o: 0, order: undefined}); @@ -230,6 +232,8 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => { const searchStr = searchParams.get('q') || ''; const order = searchParams.get('order') || order_by; const activeTags = searchParams.getAll('tag'); + const {view} = useSearchView(); + const headline = view === 'headline'; const [videos, setVideos] = useState(); const [totalPages, setTotalPages] = useState(0); @@ -238,7 +242,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => { setVideos(null); setTotalPages(0); try { - let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags); + let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline); setVideos(videos_); setTotalPages(calculateTotalPages(total, limit)); } catch (e) { @@ -255,7 +259,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => { useEffect(() => { localSearchVideos(); - }, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags)]); + }, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags), headline]); const setSearchStr = (value) => { updateQuery({q: value, o: 0, order: undefined}); @@ -404,10 +408,12 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => const filter = searchParams.get('filter'); const model_ = searchParams.get('model'); const activeTags = searchParams.getAll('tag'); + const {view} = useSearchView(); const [searchFiles, setSearchFiles] = useState(null); const [totalPages, setTotalPages] = useState(0); const [activePage, setActivePage] = useState(calculatePage(offset, limit)); + const headline = view === 'headline'; const localSearchFiles = async () => { if (!emptySearch && !searchStr && !activeTags) { @@ -418,7 +424,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => setTotalPages(0); try { let [file_groups, total] = await filesSearch( - offset, limit, searchStr, mimetypes, model || model_, activeTags); + offset, limit, searchStr, mimetypes, model || model_, activeTags, headline); setSearchFiles(file_groups); setTotalPages(calculateTotalPages(total, limit)); } catch (e) { @@ -434,7 +440,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => useEffect(() => { localSearchFiles(); - }, [searchStr, limit, offset, activePage, filter, model, model_, JSON.stringify(activeTags)]); + }, [searchStr, limit, offset, activePage, filter, model, model_, JSON.stringify(activeTags), headline]); const setPage = (i) => { i = parseInt(i); diff --git a/modules/archive/api.py b/modules/archive/api.py index 45071684..ea7e2068 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -64,6 +64,7 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest): limit = archive_limit_limiter(body.limit) offset = body.offset or 0 - file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names) + file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names, + body.headline) ret = dict(file_groups=file_groups, totals=dict(file_groups=total)) return json_response(ret) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 2137b96b..d5afe6ba 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -375,7 +375,8 @@ ARCHIVE_ORDERS = { } -def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str]) \ +def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str], + headline: bool = False) \ -> Tuple[List[dict], int]: # Always filter FileGroups to Archives. wheres = ["fg.model = 'archive'"] @@ -405,6 +406,15 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order params.update(params_) joins.append(join_) + if headline: + headline = ''', + ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline", + ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline", + ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline", + ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"''' + else: + headline = '' + if domain: params['domain'] = domain wheres.append('a.domain_id = (select id from domains where domains.domain = %(domain)s)') @@ -416,11 +426,8 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order stmt = f''' SELECT fg.id -- always get `file_group.id` for `handle_file_group_search_results` - {select_columns}, - ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline", - ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline", - ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline", - ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline" + {select_columns} + {headline} FROM file_group fg LEFT JOIN archive a ON a.file_group_id = fg.id {join} diff --git a/modules/archive/schema.py b/modules/archive/schema.py index f6c9ffbc..dbcb33f5 100644 --- a/modules/archive/schema.py +++ b/modules/archive/schema.py @@ -31,6 +31,7 @@ class ArchiveSearchRequest: limit: Optional[int] = None order_by: Optional[str] = None tag_names: List[str] = field(default_factory=list) + headline: bool = False @dataclass diff --git a/modules/videos/schema.py b/modules/videos/schema.py index 9774b43a..32d307b4 100644 --- a/modules/videos/schema.py +++ b/modules/videos/schema.py @@ -145,6 +145,7 @@ class VideoSearchRequest: order_by: Optional[str] = DEFAULT_VIDEO_ORDER channel_id: Optional[int] = None tag_names: List[str] = field(default_factory=list) + headline: bool = False @dataclass diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py index 8469747f..34781118 100644 --- a/modules/videos/video/api.py +++ b/modules/videos/video/api.py @@ -45,6 +45,7 @@ async def search(_: Request, body: schema.VideoSearchRequest): body.channel_id, body.order_by, body.tag_names, + body.headline, ) ret = {'file_groups': list(file_groups), 'totals': {'file_groups': videos_total}} diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index b738f2e1..2b314f23 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -77,6 +77,7 @@ def search_videos( channel_id: int = None, order: str = None, tag_names: List[str] = None, + headline: bool = False, ) -> Tuple[List[dict], int]: tag_names = tag_names or [] # Only search videos. @@ -106,6 +107,15 @@ def search_videos( params.update(params_) joins.append(join_) + if headline: + headline = ''', + ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline", + ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline", + ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline", + ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"''' + else: + headline = '' + # Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the # whitelist. order_by = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER] @@ -127,11 +137,8 @@ def search_videos( join = '\n'.join(joins) stmt = f''' SELECT - {select_columns}, - ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline", - ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline", - ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline", - ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline" + {select_columns} + {headline} FROM file_group fg {join} {where} diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index b600aef2..75ecd072 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -87,7 +87,7 @@ async def refresh_progress(request: Request): @validate(schema.FilesSearchRequest) async def post_search_files(_: Request, body: schema.FilesSearchRequest): file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model, - body.tag_names) + body.tag_names, body.headline) return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total))) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 1a90f3e5..44421081 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -540,7 +540,7 @@ async def apply_indexers(): def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None, - tags: List[str] = None) -> \ + tags: List[str] = None, headlines: bool = False) -> \ Tuple[List[dict], int]: """Search the FileGroup table. @@ -553,6 +553,8 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] offset: Offset the query. mimetypes: Only return files that match these mimetypes. model: Only return files that match this model. + tags: A list of tag names. + headlines: Includes Postgresql headlines if True. """ params = dict(offset=offset, limit=limit) wheres = [] @@ -588,15 +590,21 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] params.update(params_) joins.append(join_) + if headlines: + headlines = ''', + ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline", + ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline", + ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline", + ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"''' + else: + headlines = '' + wheres = '\n AND '.join(wheres) selects = f"{', '.join(selects)}, " if selects else "" join = '\n'.join(joins) stmt = f''' - SELECT fg.id, {selects} COUNT(*) OVER() AS total, - ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline", - ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline", - ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline", - ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline" + SELECT fg.id, {selects} COUNT(*) OVER() AS total + {headlines} FROM file_group fg {join} {f"WHERE {wheres}" if wheres else ""} diff --git a/wrolpi/files/schema.py b/wrolpi/files/schema.py index db281f77..f05e8f62 100644 --- a/wrolpi/files/schema.py +++ b/wrolpi/files/schema.py @@ -25,6 +25,7 @@ class FilesSearchRequest: mimetypes: List[str] = field(default_factory=list) model: Optional[str] = None tag_names: List[str] = field(default_factory=list) + headline: bool = False @dataclass