Splitting FileGroups from search suggestions to speed up channel/domain response time.

This commit is contained in:
Roland 2024-04-16 09:39:29 -06:00
parent 65f94b4807
commit 969a065a6c
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
6 changed files with 102 additions and 52 deletions

View file

@ -1061,7 +1061,20 @@ export async function fetchDecoded(vinNumber) {
}
}
export async function searchSuggestions(search_str, tagNames, mimetypes, months, dateRange) {
export async function searchSuggestions(search_str) {
const body = {search_str};
const response = await apiPost(`${API_URI}/search_suggestions`, body);
if (response.ok) {
const content = await response.json();
return {
channels: content.channels,
domains: content.domains,
}
}
}
export async function searchEstimateFiles(search_str, tagNames, mimetypes, months, dateRange) {
months = months ? months.map(i => parseInt(i)) : [];
const body = {search_str, tag_names: tagNames, mimetypes, months};
@ -1069,17 +1082,12 @@ export async function searchSuggestions(search_str, tagNames, mimetypes, months,
body['from_year'] = dateRange[0];
body['to_year'] = dateRange[1];
}
const response = await apiPost(`${API_URI}/search_suggestions`, body);
const response = await apiPost(`${API_URI}/search_file_estimates`, body);
if (response.ok) {
const content = await response.json();
const fileGroups = content.file_groups;
const channels = content.channels;
const domains = content.domains;
return {
fileGroups,
channels,
domains,
fileGroups: content.file_groups,
}
}
}

View file

@ -182,7 +182,7 @@ export function DateSelectorButton({
let newFromDate;
let newToDate;
let newMonths;
if (dateRange && dateRange[0] < dateRange[1]) {
if (dateRange && dateRange[0] <= dateRange[1]) {
newFromDate = dateRange[0];
newToDate = dateRange[1];
}

View file

@ -3,7 +3,7 @@ import {Route, Routes, useNavigate} from "react-router-dom";
import {FilesSearchView} from "./Files";
import {useLatestRequest, usePages, useSearchDate, useSearchFilter} from "../hooks/customHooks";
import {ZimSearchView} from "./Zim";
import {searchEstimateZims, searchSuggestions} from "../api";
import {searchEstimateFiles, searchEstimateZims, searchSuggestions} from "../api";
import {filterToMimetypes, fuzzyMatch, normalizeEstimate, SearchResultsInput, TabLinks} from "./Common";
import _ from "lodash";
import {TagsContext} from "../Tags";
@ -102,20 +102,23 @@ export function useSuggestions(searchStr, tagNames, filter) {
}
const [suggestions, setSuggestions] = React.useState(defaultSuggestions);
const {dateRange, months} = useSearchDate();
// fileGroups/channels/domains.
const {data, sendRequest, loading} = useLatestRequest(500);
// channels/domains.
const {data: generalData, sendRequest: sendGeneralReqeust, loading: generalLoading} = useLatestRequest(500);
const {data: filesData, sendRequest: sendFilesRequest, loading: filesLoading} = useLatestRequest(500);
// Zims are slow, so they are separate.
const {data: zimData, sendRequest: sendZimRequest, loading: zimLoading} = useLatestRequest(500);
React.useEffect(() => {
if ((searchStr && searchStr.length > 0) || (tagNames && tagNames.length > 0)) {
setSuggestions(defaultSuggestions);
const mimetypes = filterToMimetypes(filter);
sendRequest(async () => await searchSuggestions(searchStr, tagNames, mimetypes, months, dateRange));
sendGeneralReqeust(async () => await searchSuggestions(searchStr));
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange));
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
}
}, [
searchStr,
sendRequest,
sendGeneralReqeust,
sendZimRequest,
JSON.stringify(tagNames),
JSON.stringify(months),
@ -123,15 +126,24 @@ export function useSuggestions(searchStr, tagNames, filter) {
]);
React.useEffect(() => {
if (!_.isEmpty(data)) {
if (!_.isEmpty(generalData)) {
setSuggestions({
...suggestions,
channels: data.channels,
fileGroups: data.fileGroups,
domains: data.domains,
channels: generalData.channels,
domains: generalData.domains,
});
}
}, [JSON.stringify(data)]);
}, [JSON.stringify(generalData)]);
React.useEffect(() => {
if (!_.isEmpty(filesData)) {
console.log(filesData);
setSuggestions({
...suggestions,
fileGroups: filesData.fileGroups,
});
}
}, [JSON.stringify(filesData)]);
React.useEffect(() => {
if (!_.isEmpty(zimData)) {
@ -139,7 +151,7 @@ export function useSuggestions(searchStr, tagNames, filter) {
}
}, [JSON.stringify(zimData)]);
return {suggestions, loading: loading || zimLoading}
return {suggestions, loading: generalLoading || zimLoading || filesLoading}
}

View file

@ -538,12 +538,28 @@ async def post_shutdown(_: Request):
@api_bp.post('/search_suggestions')
@validate(json=schema.SearchSuggestionsRequest)
async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequest):
async def post_search_suggestions(_: Request, body: schema.SearchFileEstimateRequest):
"""Used by the Global search to suggest related Channels/Domains/etc. to the user."""
from modules.videos.channel.lib import search_channels_by_name
from modules.archive.lib import search_domains_by_name
file_coro = search_file_suggestion_count(
channels, domains = await asyncio.gather(
search_channels_by_name(body.search_str),
search_domains_by_name(body.search_str),
)
ret = dict(
channels=channels,
domains=domains,
)
return json_response(ret)
@api_bp.post('/search_file_estimates')
@validate(json=schema.SearchFileEstimateRequest)
async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimateRequest):
"""Used by the Global search to suggest FileGroup count to the user."""
file_groups = await search_file_suggestion_count(
body.search_str,
body.tag_names,
body.mimetypes,
@ -551,15 +567,9 @@ async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequ
body.from_year,
body.to_year,
)
channels_coro = search_channels_by_name(body.search_str)
domains_coro = search_domains_by_name(body.search_str)
file_groups, channels, domains = await asyncio.gather(file_coro, channels_coro, domains_coro)
ret = dict(
file_groups=file_groups,
channels=channels,
domains=domains,
)
return json_response(ret)

View file

@ -145,6 +145,11 @@ class VINDecoderResponse:
@dataclass
class SearchSuggestionsRequest:
search_str: Optional[str] = None
@dataclass
class SearchFileEstimateRequest:
search_str: Optional[str] = None
mimetypes: List[str] = field(default_factory=lambda: list())
tag_names: List[str] = field(default_factory=lambda: list())
months: Optional[List[int]] = None

View file

@ -468,29 +468,24 @@ async def test_post_vin_number_decoder(test_async_client):
@pytest.mark.asyncio
async def test_search_suggestions(test_session, test_async_client, channel_factory, archive_factory, tag_factory,
video_factory):
async def test_search_suggestions(test_session, test_async_client, channel_factory, archive_factory):
# WARNING results are cached, this test uses unique queries to avoid conflicts.
channel_factory(name='Foo')
channel_factory(name='Fool')
channel_factory(name='Bar')
tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory()
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
archive_factory(domain='foo.com')
archive_factory(domain='bar.com')
test_session.commit()
async def assert_results(body: dict, expected_channels=None, expected_domains=None, expected_file_groups=None):
async def assert_results(body: dict, expected_channels=None, expected_domains=None):
expected_channels = expected_channels or []
expected_domains = expected_domains or []
expected_file_groups = expected_file_groups or 0
request, response = await test_async_client.post('/api/search_suggestions', json=body)
assert response.status_code == HTTPStatus.OK
if expected_channels:
assert response.json['channels'] == expected_channels
assert response.json['domains'] == expected_domains
assert response.json['file_groups'] == expected_file_groups or 0
await assert_results(
dict(search_str='foo'),
@ -500,7 +495,6 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
'name': 'Fool',
'url': 'https://example.com/Fool'}],
[{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}],
1,
)
# Channel name "Fool" is matched because spaces are stripped in addition to only
@ -512,40 +506,61 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
'name': 'Fool',
'url': 'https://example.com/Fool'}],
[],
0,
)
await assert_results(
dict(search_str='bar'),
[{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar'}],
[{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}],
1,
)
@pytest.mark.asyncio
async def test_search_file_estimates(test_session, test_async_client, archive_factory, tag_factory, video_factory):
# WARNING results are cached, this test uses unique queries to avoid conflicts.
tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory()
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
test_session.commit()
async def assert_results(body: dict, expected_file_groups=None):
expected_file_groups = expected_file_groups or 0
request, response = await test_async_client.post('/api/search_file_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == expected_file_groups or 0
await assert_results(dict(search_str='foo'), 1)
# Channel name "Fool" is matched because spaces are stripped in addition to only
await assert_results(dict(search_str='foo l'), 0)
await assert_results(dict(search_str='bar'), 1)
# "foo" archive and video both contain bunny.
await assert_results(dict(search_str='bunny'), expected_file_groups=2)
await assert_results(dict(search_str='bunny'), 2)
# Filtering with mimetypes removes archive result.
await assert_results(dict(search_str='bunny', mimetypes=['video']), expected_file_groups=1)
await assert_results(dict(search_str='bunny', mimetypes=['video']), 1)
# tag1 has been used twice
await assert_results(dict(tag_names=[tag1.name, ]), expected_file_groups=2)
await assert_results(dict(tag_names=[tag1.name, ]), 2)
# archive2 was tagged with tag2, but only video contains bunny.
await assert_results(dict(search_str='bunny', tag_names=[tag2.name, ]), expected_file_groups=1)
await assert_results(dict(search_str='bunny', tag_names=[tag2.name, ]), 1)
# tag2 has been used twice
await assert_results(dict(tag_names=[tag2.name, ]), expected_file_groups=2)
await assert_results(dict(tag_names=[tag2.name, ]), 2)
# tag2 has been used twice, but filter by video mimetype.
await assert_results(dict(tag_names=[tag2.name], mimetypes=['video']), expected_file_groups=1)
await assert_results(dict(tag_names=[tag2.name], mimetypes=['video']), 1)
# Can use all filters simultaneously.
await assert_results(dict(search_str='bunny', tag_names=[tag2.name], mimetypes=['video']),
expected_file_groups=1)
await assert_results(dict(search_str='bunny', tag_names=[tag2.name], mimetypes=['video']), 1)
# Only the video has been tagged with both tag1 and tag2.
await assert_results(dict(tag_names=[tag1.name, tag2.name]), expected_file_groups=1)
await assert_results(dict(tag_names=[tag1.name, tag2.name]), 1)
# tag3 was never used
await assert_results(dict(tag_names=[tag3.name]))
@ -556,9 +571,9 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
await assert_results(dict(mimetypes=['application/pdf']))
# Can filter by published datetime.
await assert_results(dict(months=[1, ]), expected_file_groups=2)
await assert_results(dict(months=[2, ]), expected_file_groups=0)
await assert_results(dict(from_year=2000, to_year=2000), expected_file_groups=2)
await assert_results(dict(months=[1, ]), 2)
await assert_results(dict(months=[2, ]), 0)
await assert_results(dict(from_year=2000, to_year=2000), 2)
def test_recursive_errors():