mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Splitting FileGroups from search suggestions to speed up channel/domain response time.
This commit is contained in:
parent
65f94b4807
commit
969a065a6c
6 changed files with 102 additions and 52 deletions
|
|
@ -1061,7 +1061,20 @@ export async function fetchDecoded(vinNumber) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function searchSuggestions(search_str, tagNames, mimetypes, months, dateRange) {
|
||||
export async function searchSuggestions(search_str) {
|
||||
const body = {search_str};
|
||||
const response = await apiPost(`${API_URI}/search_suggestions`, body);
|
||||
if (response.ok) {
|
||||
const content = await response.json();
|
||||
|
||||
return {
|
||||
channels: content.channels,
|
||||
domains: content.domains,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function searchEstimateFiles(search_str, tagNames, mimetypes, months, dateRange) {
|
||||
months = months ? months.map(i => parseInt(i)) : [];
|
||||
|
||||
const body = {search_str, tag_names: tagNames, mimetypes, months};
|
||||
|
|
@ -1069,17 +1082,12 @@ export async function searchSuggestions(search_str, tagNames, mimetypes, months,
|
|||
body['from_year'] = dateRange[0];
|
||||
body['to_year'] = dateRange[1];
|
||||
}
|
||||
const response = await apiPost(`${API_URI}/search_suggestions`, body);
|
||||
const response = await apiPost(`${API_URI}/search_file_estimates`, body);
|
||||
if (response.ok) {
|
||||
const content = await response.json();
|
||||
|
||||
const fileGroups = content.file_groups;
|
||||
const channels = content.channels;
|
||||
const domains = content.domains;
|
||||
return {
|
||||
fileGroups,
|
||||
channels,
|
||||
domains,
|
||||
fileGroups: content.file_groups,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -182,7 +182,7 @@ export function DateSelectorButton({
|
|||
let newFromDate;
|
||||
let newToDate;
|
||||
let newMonths;
|
||||
if (dateRange && dateRange[0] < dateRange[1]) {
|
||||
if (dateRange && dateRange[0] <= dateRange[1]) {
|
||||
newFromDate = dateRange[0];
|
||||
newToDate = dateRange[1];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import {Route, Routes, useNavigate} from "react-router-dom";
|
|||
import {FilesSearchView} from "./Files";
|
||||
import {useLatestRequest, usePages, useSearchDate, useSearchFilter} from "../hooks/customHooks";
|
||||
import {ZimSearchView} from "./Zim";
|
||||
import {searchEstimateZims, searchSuggestions} from "../api";
|
||||
import {searchEstimateFiles, searchEstimateZims, searchSuggestions} from "../api";
|
||||
import {filterToMimetypes, fuzzyMatch, normalizeEstimate, SearchResultsInput, TabLinks} from "./Common";
|
||||
import _ from "lodash";
|
||||
import {TagsContext} from "../Tags";
|
||||
|
|
@ -102,20 +102,23 @@ export function useSuggestions(searchStr, tagNames, filter) {
|
|||
}
|
||||
const [suggestions, setSuggestions] = React.useState(defaultSuggestions);
|
||||
const {dateRange, months} = useSearchDate();
|
||||
// fileGroups/channels/domains.
|
||||
const {data, sendRequest, loading} = useLatestRequest(500);
|
||||
// channels/domains.
|
||||
const {data: generalData, sendRequest: sendGeneralReqeust, loading: generalLoading} = useLatestRequest(500);
|
||||
const {data: filesData, sendRequest: sendFilesRequest, loading: filesLoading} = useLatestRequest(500);
|
||||
// Zims are slow, so they are separate.
|
||||
const {data: zimData, sendRequest: sendZimRequest, loading: zimLoading} = useLatestRequest(500);
|
||||
|
||||
React.useEffect(() => {
|
||||
if ((searchStr && searchStr.length > 0) || (tagNames && tagNames.length > 0)) {
|
||||
setSuggestions(defaultSuggestions);
|
||||
const mimetypes = filterToMimetypes(filter);
|
||||
sendRequest(async () => await searchSuggestions(searchStr, tagNames, mimetypes, months, dateRange));
|
||||
sendGeneralReqeust(async () => await searchSuggestions(searchStr));
|
||||
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange));
|
||||
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
|
||||
}
|
||||
}, [
|
||||
searchStr,
|
||||
sendRequest,
|
||||
sendGeneralReqeust,
|
||||
sendZimRequest,
|
||||
JSON.stringify(tagNames),
|
||||
JSON.stringify(months),
|
||||
|
|
@ -123,15 +126,24 @@ export function useSuggestions(searchStr, tagNames, filter) {
|
|||
]);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!_.isEmpty(data)) {
|
||||
if (!_.isEmpty(generalData)) {
|
||||
setSuggestions({
|
||||
...suggestions,
|
||||
channels: data.channels,
|
||||
fileGroups: data.fileGroups,
|
||||
domains: data.domains,
|
||||
channels: generalData.channels,
|
||||
domains: generalData.domains,
|
||||
});
|
||||
}
|
||||
}, [JSON.stringify(data)]);
|
||||
}, [JSON.stringify(generalData)]);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!_.isEmpty(filesData)) {
|
||||
console.log(filesData);
|
||||
setSuggestions({
|
||||
...suggestions,
|
||||
fileGroups: filesData.fileGroups,
|
||||
});
|
||||
}
|
||||
}, [JSON.stringify(filesData)]);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!_.isEmpty(zimData)) {
|
||||
|
|
@ -139,7 +151,7 @@ export function useSuggestions(searchStr, tagNames, filter) {
|
|||
}
|
||||
}, [JSON.stringify(zimData)]);
|
||||
|
||||
return {suggestions, loading: loading || zimLoading}
|
||||
return {suggestions, loading: generalLoading || zimLoading || filesLoading}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -538,12 +538,28 @@ async def post_shutdown(_: Request):
|
|||
|
||||
@api_bp.post('/search_suggestions')
|
||||
@validate(json=schema.SearchSuggestionsRequest)
|
||||
async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequest):
|
||||
async def post_search_suggestions(_: Request, body: schema.SearchFileEstimateRequest):
|
||||
"""Used by the Global search to suggest related Channels/Domains/etc. to the user."""
|
||||
from modules.videos.channel.lib import search_channels_by_name
|
||||
from modules.archive.lib import search_domains_by_name
|
||||
|
||||
file_coro = search_file_suggestion_count(
|
||||
channels, domains = await asyncio.gather(
|
||||
search_channels_by_name(body.search_str),
|
||||
search_domains_by_name(body.search_str),
|
||||
)
|
||||
|
||||
ret = dict(
|
||||
channels=channels,
|
||||
domains=domains,
|
||||
)
|
||||
return json_response(ret)
|
||||
|
||||
|
||||
@api_bp.post('/search_file_estimates')
|
||||
@validate(json=schema.SearchFileEstimateRequest)
|
||||
async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimateRequest):
|
||||
"""Used by the Global search to suggest FileGroup count to the user."""
|
||||
file_groups = await search_file_suggestion_count(
|
||||
body.search_str,
|
||||
body.tag_names,
|
||||
body.mimetypes,
|
||||
|
|
@ -551,15 +567,9 @@ async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequ
|
|||
body.from_year,
|
||||
body.to_year,
|
||||
)
|
||||
channels_coro = search_channels_by_name(body.search_str)
|
||||
domains_coro = search_domains_by_name(body.search_str)
|
||||
|
||||
file_groups, channels, domains = await asyncio.gather(file_coro, channels_coro, domains_coro)
|
||||
|
||||
ret = dict(
|
||||
file_groups=file_groups,
|
||||
channels=channels,
|
||||
domains=domains,
|
||||
)
|
||||
return json_response(ret)
|
||||
|
||||
|
|
|
|||
|
|
@ -145,6 +145,11 @@ class VINDecoderResponse:
|
|||
@dataclass
|
||||
class SearchSuggestionsRequest:
|
||||
search_str: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchFileEstimateRequest:
|
||||
search_str: Optional[str] = None
|
||||
mimetypes: List[str] = field(default_factory=lambda: list())
|
||||
tag_names: List[str] = field(default_factory=lambda: list())
|
||||
months: Optional[List[int]] = None
|
||||
|
|
|
|||
|
|
@ -468,29 +468,24 @@ async def test_post_vin_number_decoder(test_async_client):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_suggestions(test_session, test_async_client, channel_factory, archive_factory, tag_factory,
|
||||
video_factory):
|
||||
async def test_search_suggestions(test_session, test_async_client, channel_factory, archive_factory):
|
||||
# WARNING results are cached, this test uses unique queries to avoid conflicts.
|
||||
channel_factory(name='Foo')
|
||||
channel_factory(name='Fool')
|
||||
channel_factory(name='Bar')
|
||||
tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory()
|
||||
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
|
||||
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
|
||||
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
|
||||
archive_factory(domain='foo.com')
|
||||
archive_factory(domain='bar.com')
|
||||
test_session.commit()
|
||||
|
||||
async def assert_results(body: dict, expected_channels=None, expected_domains=None, expected_file_groups=None):
|
||||
async def assert_results(body: dict, expected_channels=None, expected_domains=None):
|
||||
expected_channels = expected_channels or []
|
||||
expected_domains = expected_domains or []
|
||||
expected_file_groups = expected_file_groups or 0
|
||||
|
||||
request, response = await test_async_client.post('/api/search_suggestions', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
if expected_channels:
|
||||
assert response.json['channels'] == expected_channels
|
||||
assert response.json['domains'] == expected_domains
|
||||
assert response.json['file_groups'] == expected_file_groups or 0
|
||||
|
||||
await assert_results(
|
||||
dict(search_str='foo'),
|
||||
|
|
@ -500,7 +495,6 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
|
|||
'name': 'Fool',
|
||||
'url': 'https://example.com/Fool'}],
|
||||
[{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}],
|
||||
1,
|
||||
)
|
||||
|
||||
# Channel name "Fool" is matched because spaces are stripped in addition to only
|
||||
|
|
@ -512,40 +506,61 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
|
|||
'name': 'Fool',
|
||||
'url': 'https://example.com/Fool'}],
|
||||
[],
|
||||
0,
|
||||
)
|
||||
|
||||
await assert_results(
|
||||
dict(search_str='bar'),
|
||||
[{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar'}],
|
||||
[{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}],
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_file_estimates(test_session, test_async_client, archive_factory, tag_factory, video_factory):
|
||||
# WARNING results are cached, this test uses unique queries to avoid conflicts.
|
||||
tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory()
|
||||
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
|
||||
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
|
||||
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
|
||||
test_session.commit()
|
||||
|
||||
async def assert_results(body: dict, expected_file_groups=None):
|
||||
expected_file_groups = expected_file_groups or 0
|
||||
|
||||
request, response = await test_async_client.post('/api/search_file_estimates', json=body)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['file_groups'] == expected_file_groups or 0
|
||||
|
||||
await assert_results(dict(search_str='foo'), 1)
|
||||
|
||||
# Channel name "Fool" is matched because spaces are stripped in addition to only
|
||||
await assert_results(dict(search_str='foo l'), 0)
|
||||
|
||||
await assert_results(dict(search_str='bar'), 1)
|
||||
|
||||
# "foo" archive and video both contain bunny.
|
||||
await assert_results(dict(search_str='bunny'), expected_file_groups=2)
|
||||
await assert_results(dict(search_str='bunny'), 2)
|
||||
|
||||
# Filtering with mimetypes removes archive result.
|
||||
await assert_results(dict(search_str='bunny', mimetypes=['video']), expected_file_groups=1)
|
||||
await assert_results(dict(search_str='bunny', mimetypes=['video']), 1)
|
||||
|
||||
# tag1 has been used twice
|
||||
await assert_results(dict(tag_names=[tag1.name, ]), expected_file_groups=2)
|
||||
await assert_results(dict(tag_names=[tag1.name, ]), 2)
|
||||
|
||||
# archive2 was tagged with tag2, but only video contains bunny.
|
||||
await assert_results(dict(search_str='bunny', tag_names=[tag2.name, ]), expected_file_groups=1)
|
||||
await assert_results(dict(search_str='bunny', tag_names=[tag2.name, ]), 1)
|
||||
|
||||
# tag2 has been used twice
|
||||
await assert_results(dict(tag_names=[tag2.name, ]), expected_file_groups=2)
|
||||
await assert_results(dict(tag_names=[tag2.name, ]), 2)
|
||||
|
||||
# tag2 has been used twice, but filter by video mimetype.
|
||||
await assert_results(dict(tag_names=[tag2.name], mimetypes=['video']), expected_file_groups=1)
|
||||
await assert_results(dict(tag_names=[tag2.name], mimetypes=['video']), 1)
|
||||
|
||||
# Can use all filters simultaneously.
|
||||
await assert_results(dict(search_str='bunny', tag_names=[tag2.name], mimetypes=['video']),
|
||||
expected_file_groups=1)
|
||||
await assert_results(dict(search_str='bunny', tag_names=[tag2.name], mimetypes=['video']), 1)
|
||||
|
||||
# Only the video has been tagged with both tag1 and tag2.
|
||||
await assert_results(dict(tag_names=[tag1.name, tag2.name]), expected_file_groups=1)
|
||||
await assert_results(dict(tag_names=[tag1.name, tag2.name]), 1)
|
||||
|
||||
# tag3 was never used
|
||||
await assert_results(dict(tag_names=[tag3.name]))
|
||||
|
|
@ -556,9 +571,9 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
|
|||
await assert_results(dict(mimetypes=['application/pdf']))
|
||||
|
||||
# Can filter by published datetime.
|
||||
await assert_results(dict(months=[1, ]), expected_file_groups=2)
|
||||
await assert_results(dict(months=[2, ]), expected_file_groups=0)
|
||||
await assert_results(dict(from_year=2000, to_year=2000), expected_file_groups=2)
|
||||
await assert_results(dict(months=[1, ]), 2)
|
||||
await assert_results(dict(months=[2, ]), 0)
|
||||
await assert_results(dict(from_year=2000, to_year=2000), 2)
|
||||
|
||||
|
||||
def test_recursive_errors():
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue