rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 02:48:38 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Only fetching headlines if requested to speed up RPi responses.g

Browse source
This commit is contained in:
Roland 2023-04-12 18:51:26 -06:00
parent 8faac3f3a1
commit bba6a341b7
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
12 changed files with 80 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

16
app/src/api.js
View file

@ -1,5 +1,6 @@
import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, emptyToNull, OTP_API, VIDEOS_API} from "./components/Common";
import {toast} from "react-semantic-toasts-2";
import {computeHeadingLevel} from "@testing-library/react";
function timeoutPromise(ms, promise) {
// Create a timeout wrapper around a promise. If the timeout is reached, throw an error. Otherwise, return
@ -103,7 +104,7 @@ export async function getChannel(id) {
return (await response.json())['channel'];
}
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames) {
export async function searchVideos(offset, limit, channelId, searchStr, order_by, tagNames, headline) {
// Build a search query to retrieve a list of videos from the API
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
@ -118,6 +119,9 @@ export async function searchVideos(offset, limit, channelId, searchStr, order_by
if (tagNames) {
body['tag_names'] = tagNames;
}
if (headline) {
body['headline'] = true;
}
console.debug('searching videos', body);
@ -350,7 +354,7 @@ export async function deleteArchives(archiveIds) {
}
}
export async function searchArchives(offset, limit, domain, searchStr, order, tagNames) {
export async function searchArchives(offset, limit, domain, searchStr, order, tagNames, headline) {
// Build a search query to retrieve a list of videos from the API
offset = parseInt(offset || 0);
limit = parseInt(limit || DEFAULT_LIMIT);
@ -367,6 +371,9 @@ export async function searchArchives(offset, limit, domain, searchStr, order, ta
if (tagNames) {
body['tag_names'] = tagNames;
}
if (headline) {
body['headline'] = true;
}
console.debug('searching archives', body);
let response = await apiPost(`${ARCHIVES_API}/search`, body);
@ -479,7 +486,7 @@ export async function deleteDownload(downloadId) {
}
}
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames) {
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline) {
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)};
if (mimetypes) {
body['mimetypes'] = mimetypes;
@ -490,6 +497,9 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta
if (tagNames) {
body['tag_names'] = tagNames;
}
if (headline) {
body['headline'] = true;
}
console.debug('searching files', body);
const response = await apiPost(`${API_URI}/files/search`, body);

19
app/src/components/Headline.js
View file

@ -76,25 +76,24 @@ export const Headlines = ({results}) => {
return <Segment>No results!</Segment>
}
let items = [];
let headlines = [];
for (let i = 0; i < results.length; i++) {
const result = results[i];
const {model, data, video} = result;
if (model === 'video' && !_.isEmpty(video)) {
let video_url = `/videos/video/${video.id}`;
const channel = video.channel ? video.channel : null;
if (channel) {
video_url = `/videos/channel/${channel.id}/video/${video.id}`;
}
items = [...items, <FileHeadline key={result['key']} file={result} to={video_url}/>];
const {channel} = video;
const video_url = channel
? `/videos/channel/${channel.id}/video/${video.id}`
: `/videos/video/${video.id}`;
headlines = [...headlines, <FileHeadline key={result['key']} file={result} to={video_url}/>];
} else if (model === 'archive' && !_.isEmpty(data)) {
items = [...items, <FileHeadline key={result['key']} file={result} to={`/archive/${data.id}`}/>];
headlines = [...headlines, <FileHeadline key={result['key']} file={result} to={`/archive/${data.id}`}/>];
} else {
items = [...items, <FileHeadline key={result['key']} file={result}/>];
headlines = [...headlines, <FileHeadline key={result['key']} file={result}/>];
}
}
return <>
{items}
{headlines}
</>
}

18
app/src/hooks/customHooks.js
View file

@ -172,6 +172,8 @@ export const useSearchArchives = (defaultLimit) => {
const searchStr = searchParams.get('q') || '';
const order = searchParams.get('order');
const activeTags = searchParams.getAll('tag');
const {view} = useSearchView();
const headline = view === 'headline';
const [archives, setArchives] = useState(null);
const [totalPages, setTotalPages] = useState(0);
@ -180,7 +182,7 @@ export const useSearchArchives = (defaultLimit) => {
setArchives(null);
setTotalPages(0);
try {
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags);
let [archives_, total] = await searchArchives(offset, limit, domain, searchStr, order, activeTags, headline);
setArchives(archives_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -197,7 +199,7 @@ export const useSearchArchives = (defaultLimit) => {
useEffect(() => {
localSearchArchives();
}, [searchStr, limit, domain, order, activePage, JSON.stringify(activeTags)]);
}, [searchStr, limit, domain, order, activePage, JSON.stringify(activeTags), headline]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@ -230,6 +232,8 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
const searchStr = searchParams.get('q') || '';
const order = searchParams.get('order') || order_by;
const activeTags = searchParams.getAll('tag');
const {view} = useSearchView();
const headline = view === 'headline';
const [videos, setVideos] = useState();
const [totalPages, setTotalPages] = useState(0);
@ -238,7 +242,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
setVideos(null);
setTotalPages(0);
try {
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags);
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline);
setVideos(videos_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -255,7 +259,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
useEffect(() => {
localSearchVideos();
}, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags)]);
}, [searchStr, limit, channelId, offset, order_by, JSON.stringify(activeTags), headline]);
const setSearchStr = (value) => {
updateQuery({q: value, o: 0, order: undefined});
@ -404,10 +408,12 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
const filter = searchParams.get('filter');
const model_ = searchParams.get('model');
const activeTags = searchParams.getAll('tag');
const {view} = useSearchView();
const [searchFiles, setSearchFiles] = useState(null);
const [totalPages, setTotalPages] = useState(0);
const [activePage, setActivePage] = useState(calculatePage(offset, limit));
const headline = view === 'headline';
const localSearchFiles = async () => {
if (!emptySearch && !searchStr && !activeTags) {
@ -418,7 +424,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
setTotalPages(0);
try {
let [file_groups, total] = await filesSearch(
offset, limit, searchStr, mimetypes, model || model_, activeTags);
offset, limit, searchStr, mimetypes, model || model_, activeTags, headline);
setSearchFiles(file_groups);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
@ -434,7 +440,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
useEffect(() => {
localSearchFiles();
}, [searchStr, limit, offset, activePage, filter, model, model_, JSON.stringify(activeTags)]);
}, [searchStr, limit, offset, activePage, filter, model, model_, JSON.stringify(activeTags), headline]);
const setPage = (i) => {
i = parseInt(i);

3
modules/archive/api.py
View file

@ -64,6 +64,7 @@ async def search_archives(_: Request, body: schema.ArchiveSearchRequest):
limit = archive_limit_limiter(body.limit)
offset = body.offset or 0
file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names)
file_groups, total = lib.search_archives(search_str, domain, limit, offset, body.order_by, body.tag_names,
body.headline)
ret = dict(file_groups=file_groups, totals=dict(file_groups=total))
return json_response(ret)

19
modules/archive/lib.py
View file

@ -375,7 +375,8 @@ ARCHIVE_ORDERS = {
}
def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str]) \
def search_archives(search_str: str, domain: str, limit: int, offset: int, order: str, tag_names: List[str],
headline: bool = False) \
-> Tuple[List[dict], int]:
# Always filter FileGroups to Archives.
wheres = ["fg.model = 'archive'"]
@ -405,6 +406,15 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
params.update(params_)
joins.append(join_)
if headline:
headline = ''',
ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline",
ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline",
ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline",
ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"'''
else:
headline = ''
if domain:
params['domain'] = domain
wheres.append('a.domain_id = (select id from domains where domains.domain = %(domain)s)')
@ -416,11 +426,8 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
stmt = f'''
SELECT
fg.id -- always get `file_group.id` for `handle_file_group_search_results`
{select_columns},
ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline",
ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline",
ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline",
ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"
{select_columns}
{headline}
FROM file_group fg
LEFT JOIN archive a ON a.file_group_id = fg.id
{join}

1
modules/archive/schema.py
View file

@ -31,6 +31,7 @@ class ArchiveSearchRequest:
limit: Optional[int] = None
order_by: Optional[str] = None
tag_names: List[str] = field(default_factory=list)
headline: bool = False
@dataclass

1
modules/videos/schema.py
View file

@ -145,6 +145,7 @@ class VideoSearchRequest:
order_by: Optional[str] = DEFAULT_VIDEO_ORDER
channel_id: Optional[int] = None
tag_names: List[str] = field(default_factory=list)
headline: bool = False
@dataclass

1
modules/videos/video/api.py
View file

@ -45,6 +45,7 @@ async def search(_: Request, body: schema.VideoSearchRequest):
body.channel_id,
body.order_by,
body.tag_names,
body.headline,
)
ret = {'file_groups': list(file_groups), 'totals': {'file_groups': videos_total}}

17
modules/videos/video/lib.py
View file

@ -77,6 +77,7 @@ def search_videos(
channel_id: int = None,
order: str = None,
tag_names: List[str] = None,
headline: bool = False,
) -> Tuple[List[dict], int]:
tag_names = tag_names or []
# Only search videos.
@ -106,6 +107,15 @@ def search_videos(
params.update(params_)
joins.append(join_)
if headline:
headline = ''',
ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline",
ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline",
ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline",
ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"'''
else:
headline = ''
# Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the
# whitelist.
order_by = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER]
@ -127,11 +137,8 @@ def search_videos(
join = '\n'.join(joins)
stmt = f'''
SELECT
{select_columns},
ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline",
ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline",
ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline",
ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"
{select_columns}
{headline}
FROM file_group fg
{join}
{where}

2
wrolpi/files/api.py
View file

@ -87,7 +87,7 @@ async def refresh_progress(request: Request):
@validate(schema.FilesSearchRequest)
async def post_search_files(_: Request, body: schema.FilesSearchRequest):
file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model,
body.tag_names)
body.tag_names, body.headline)
return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total)))

20
wrolpi/files/lib.py
View file

@ -540,7 +540,7 @@ async def apply_indexers():
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
tags: List[str] = None) -> \
tags: List[str] = None, headlines: bool = False) -> \
Tuple[List[dict], int]:
"""Search the FileGroup table.
@ -553,6 +553,8 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
offset: Offset the query.
mimetypes: Only return files that match these mimetypes.
model: Only return files that match this model.
tags: A list of tag names.
headlines: Includes Postgresql headlines if True.
"""
params = dict(offset=offset, limit=limit)
wheres = []
@ -588,15 +590,21 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
params.update(params_)
joins.append(join_)
if headlines:
headlines = ''',
ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline",
ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline",
ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline",
ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"'''
else:
headlines = ''
wheres = '\n AND '.join(wheres)
selects = f"{', '.join(selects)}, " if selects else ""
join = '\n'.join(joins)
stmt = f'''
SELECT fg.id, {selects} COUNT(*) OVER() AS total,
ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline",
ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline",
ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline",
ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"
SELECT fg.id, {selects} COUNT(*) OVER() AS total
{headlines}
FROM file_group fg
{join}
{f"WHERE {wheres}" if wheres else ""}

1
wrolpi/files/schema.py
View file

@ -25,6 +25,7 @@ class FilesSearchRequest:
mimetypes: List[str] = field(default_factory=list)
model: Optional[str] = None
tag_names: List[str] = field(default_factory=list)
headline: bool = False
@dataclass