rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 10:51:36 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Fetching search estimate with tags separate from suggestions.

Browse source
This commit is contained in:
Roland 2023-11-26 20:49:36 -07:00
parent ff62f9e21b
commit d87bc7b314
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
9 changed files with 258 additions and 164 deletions
Download patch file Download diff file Expand all files Collapse all files

15
app/src/api.js
View file

@ -1022,6 +1022,21 @@ export async function searchSuggestions(search_str) {
}
}
export async function searchEstimate(search_str, tagNames) {
const body = {search_str: search_str, tag_names: tagNames};
const response = await apiPost(`${API_URI}/search_estimates`, body);
if (response.ok) {
const content = await response.json();
const fileGroups = content.file_groups;
const zimsEstimates = content.zims_estimates;
return {
fileGroups,
zimsEstimates,
}
}
}
export async function getOutdatedZims() {
const response = await apiGet(`${API_URI}/zim/outdated`);
if (response.status === 200) {

4
app/src/components/Common.js
View file

@ -70,7 +70,7 @@ export function Paginator({activePage, onPageChange, totalPages, showFirstAndLas
onPageChange={handlePageChange}
size={size}
siblingRange={2}
totalPages={totalPages}
totalPages={totalPages || 1}
firstItem={showFirstAndLast ? undefined : null}
lastItem={showFirstAndLast ? undefined : null}
/>
@ -82,7 +82,7 @@ export function Paginator({activePage, onPageChange, totalPages, showFirstAndLas
onPageChange={handlePageChange}
size={size}
siblingRange={5}
totalPages={totalPages}
totalPages={totalPages || 1}
firstItem={showFirstAndLast ? undefined : null}
lastItem={showFirstAndLast ? undefined : null}
/>

326
app/src/components/Search.js
View file

@ -1,4 +1,4 @@
import React, {useContext, useEffect, useState} from "react";
import React, {useState} from "react";
import {createSearchParams, Route, Routes, useNavigate} from "react-router-dom";
import {FilesSearchView} from "./Files";
import {useLatestRequest, usePages, useQuery} from "../hooks/customHooks";
@ -7,7 +7,7 @@ import {searchEstimate, searchSuggestions} from "../api";
import {fuzzyMatch, normalizeEstimate, TabLinks} from "./Common";
import _ from "lodash";
import {TagsContext} from "../Tags";
import {Header as SHeader} from "semantic-ui-react";
import {Header as SHeader, Label} from "semantic-ui-react";
export const SearchSuggestionsContext = React.createContext({
suggestionsResults: {},
@ -18,6 +18,15 @@ export const SearchSuggestionsContext = React.createContext({
loading: false,
});
export const SearchSuggestionsProvider = (props) => {
const value = useSearchSuggestions();
return <SearchSuggestionsContext.Provider value={value}>
{props.children}
</SearchSuggestionsContext.Provider>
}
const SUGGESTED_APPS = [
{location: '/more/otp', title: 'One Time Pad', description: 'Encrypt and Decrypt messages'},
{location: '/inventory', title: 'Inventory', description: 'Track and organize your food storage'},
@ -29,142 +38,6 @@ const SUGGESTED_APPS = [
{location: '/help', title: 'Help', description: 'Help documents for WROLPi'},
];
export function useSearchSuggestions() {
const navigate = useNavigate();
const {searchParams} = useQuery();
const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext);
const searchStr = searchParams.get('q');
const {data, sendRequest, loading} = useLatestRequest(500);
const [suggestionsResults, setSuggestionsResults] = useState({});
const [suggestions, setSuggestions] = useState({});
const [suggestionsSums, setSuggestionsSums] = useState({});
const normalizeSuggestionsResults = (newSuggestions) => {
// Convert the suggestions from the Backend to what the Semantic <Search> expects.
const lowerSearchStr = searchStr.toLowerCase();
let results = {};
// Suggested results are ordered.
if (newSuggestions.fileGroups > 0) {
results.fileGroups = {
name: 'Files', results: [
{
title: newSuggestions.fileGroups.toString(),
type: 'files',
location: `/search?q=${encodeURIComponent(searchStr)}`
}
]
};
}
const zimSum = newSuggestions.zimsEstimates.reduce((i, j) => i + j['estimate'], 0);
if (zimSum > 0) {
results.zimsSum = {
name: 'Zims', results: [
{title: zimSum.toString(), type: 'zims', location: `/search/zim?q=${encodeURIComponent(searchStr)}`}
],
}
}
if (newSuggestions.channels && newSuggestions.channels.length > 0) {
results.channels = {
name: 'Channels', results: newSuggestions.channels.map(i => {
return {type: 'channel', title: i['name'], id: i['id'], location: `/videos/channel/${i.id}/video`}
})
}
}
if (newSuggestions.domains && newSuggestions.domains.length > 0) {
results.domains = {
name: 'Domains', results: newSuggestions.domains.map(i => {
return {
type: 'domain',
title: i.domain,
id: i.id,
domain: i.domain,
location: `/archive?domain=${i.domain}`
}
})
}
}
const matchingTags = fuzzyMatchTagsByName(searchStr);
if (matchingTags && matchingTags.length > 0) {
results.tags = {
name: 'Tags', results: matchingTags.map(i => {
return {type: 'tag', title: i.name, location: `/search?tag=${encodeURIComponent(i.name)}`}
})
}
}
const matchingApps = SUGGESTED_APPS.filter(i =>
i.title.toLowerCase().includes(lowerSearchStr)
|| fuzzyMatch(i.title.toLowerCase(), lowerSearchStr));
if (matchingApps && matchingApps.length > 0) {
results.apps = {name: 'Apps', results: matchingApps};
}
setSuggestionsResults(results);
setSuggestionsSums({
fileGroups: newSuggestions.fileGroups,
zims: zimSum,
channels: newSuggestions.channels.length,
domains: newSuggestions.domains.length,
tags: matchingTags.length,
apps: matchingApps.length,
})
}
React.useEffect(() => {
if (data) {
setSuggestions(data);
normalizeSuggestionsResults(data);
}
}, [JSON.stringify(data)]);
React.useEffect(() => {
if (!searchStr || searchStr.length === 0) {
console.debug('Not getting suggestions because there is no search.');
return;
}
setSuggestions({});
setSuggestionsSums({});
setSuggestionsResults({});
// Use the useLatestRequest to handle user typing.
sendRequest(async () => await searchSuggestions(searchStr));
}, [searchStr, sendRequest]);
const handleResultSelect = ({result}) => navigate(result.location);
const resultRenderer = ({type, title, description}) => {
if (type === 'tag') {
return <SingleTag name={title}/>;
}
// No specific renderer, use the generic.
if (description) {
return <>
<SHeader as='h4'>{title}</SHeader>
{description}
</>
}
return <span>{title}</span>
};
return {suggestions, suggestionsResults, suggestionsSums, searchStr, handleResultSelect, resultRenderer, loading}
}
export const SearchSuggestionsProvider = (props) => {
const value = useSearchSuggestions();
return <SearchSuggestionsContext.Provider value={value}>
{props.children}
</SearchSuggestionsContext.Provider>
}
export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false, model) => {
const navigate = useNavigate();
@ -224,15 +97,178 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false
}
}
export function useSuggestions(searchStr) {
const {data, sendRequest, loading} = useLatestRequest(500);
React.useEffect(() => {
if (!searchStr || searchStr.length === 0) {
return;
}
// Use the `useLatestRequest` to handle user typing.
sendRequest(async () => await searchSuggestions(searchStr));
}, [searchStr, sendRequest]);
return {suggestions: data, loading}
}
export function useSearchSuggestions() {
const navigate = useNavigate();
const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext);
const {searchStr} = useSearch();
const {suggestions, loading} = useSuggestions(searchStr);
// The results that will be displayed by <Search>.
const [suggestionsResults, setSuggestionsResults] = useState({});
// The results summarized.
const [suggestionsSums, setSuggestionsSums] = useState({});
const normalizeSuggestionsResults = (newSuggestions) => {
// Convert the suggestions from the Backend to what the Semantic <Search> expects.
const lowerSearchStr = searchStr.toLowerCase();
let results = {};
// Suggested results are ordered.
if (newSuggestions.fileGroups > 0) {
results.fileGroups = {
name: 'Files', results: [
{
title: newSuggestions.fileGroups.toString(),
type: 'files',
location: `/search?q=${encodeURIComponent(searchStr)}`
}
]
};
}
const zimSum = newSuggestions.zimsEstimates.reduce((i, j) => i + j, 0);
if (zimSum > 0) {
results.zimsSum = {
name: 'Zims', results: [
{title: zimSum.toString(), type: 'zims', location: `/search/zim?q=${encodeURIComponent(searchStr)}`}
],
}
}
if (newSuggestions.channels && newSuggestions.channels.length > 0) {
results.channels = {
name: 'Channels', results: newSuggestions.channels.map(i => {
return {type: 'channel', title: i['name'], id: i['id'], location: `/videos/channel/${i.id}/video`}
})
}
}
if (newSuggestions.domains && newSuggestions.domains.length > 0) {
results.domains = {
name: 'Domains', results: newSuggestions.domains.map(i => {
return {
type: 'domain',
title: i.domain,
id: i.id,
domain: i.domain,
location: `/archive?domain=${i.domain}`
}
})
}
}
const matchingTags = fuzzyMatchTagsByName(searchStr);
if (matchingTags && matchingTags.length > 0) {
results.tags = {
name: 'Tags', results: matchingTags.map(i => {
return {type: 'tag', title: i.name, location: `/search?tag=${encodeURIComponent(i.name)}`}
})
}
}
const matchingApps = SUGGESTED_APPS.filter(i =>
i.title.toLowerCase().includes(lowerSearchStr)
|| fuzzyMatch(i.title.toLowerCase(), lowerSearchStr));
if (matchingApps && matchingApps.length > 0) {
results.apps = {name: 'Apps', results: matchingApps};
}
setSuggestionsResults(results);
setSuggestionsSums({
fileGroups: newSuggestions.fileGroups,
zims: zimSum,
channels: newSuggestions.channels.length,
domains: newSuggestions.domains.length,
tags: matchingTags.length,
apps: matchingApps.length,
});
}
React.useEffect(() => {
setSuggestionsSums({});
setSuggestionsResults({});
if (!_.isEmpty(suggestions)) {
normalizeSuggestionsResults(suggestions);
}
}, [JSON.stringify(suggestions)]);
// User clicked on a result in the dropdown.
const handleResultSelect = ({result}) => navigate(result.location);
const resultRenderer = ({type, title, description}) => {
if (type === 'tag') {
return <SingleTag name={title}/>;
}
if (description) {
return <>
<SHeader as='h4'>{title}</SHeader>
{description}
</>
}
// No specific renderer, use the generic.
return <span>{title}</span>
};
return {suggestions, suggestionsResults, suggestionsSums, searchStr, handleResultSelect, resultRenderer, loading}
}
export function useSearchEstimate(searchStr, tagNames) {
const {data, sendRequest, loading} = useLatestRequest(250);
// The estimates from the response.
const [estimates, setEstimates] = React.useState({});
// The estimates summarized.
const [estimatesSums, setEstimatesSums] = React.useState({});
React.useEffect(() => {
// Handle results from `sendRequest`.
if (_.isEmpty(data)) {
return;
}
setEstimates(data);
setEstimatesSums({
fileGroups: data.fileGroups,
zims: data.zimsEstimates.reduce((i, j) => i + j['estimate'], 0),
});
}, [JSON.stringify(data)]);
React.useEffect(() => {
// Use the `useLatestRequest` to handle user typing.
sendRequest(async () => await searchEstimate(searchStr, tagNames));
}, [searchStr, JSON.stringify(tagNames)], sendRequest);
return {estimates, estimatesSums, loading}
}
export function SearchView() {
const {suggestionsSums, suggestions} = useContext(SearchSuggestionsContext);
const {searchStr, activeTags} = useSearch();
const {estimates, estimatesSums} = useSearchEstimate(searchStr, activeTags);
let filesTabName = 'Files';
let zimsTabName = 'Zims';
if (!_.isEmpty(suggestionsSums)) {
filesTabName = `Files (${normalizeEstimate(suggestionsSums.fileGroups)})`;
zimsTabName = `Zims (${normalizeEstimate(suggestionsSums.zims)})`;
let filesTabName = <span>Files <Label>?</Label></span>;
let zimsTabName = <span>Zims<Label>?</Label></span>;
if (!_.isEmpty(estimatesSums)) {
filesTabName = <span>Files <Label>{normalizeEstimate(estimatesSums.fileGroups)}</Label></span>;
zimsTabName = <span>Zims <Label>{normalizeEstimate(estimatesSums.zims)}</Label></span>;
}
const links = [
@ -244,7 +280,7 @@ export function SearchView() {
<TabLinks links={links}/>
<Routes>
<Route path='/*' element={<FilesSearchView/>}/>
<Route path='/zim' exact element={<ZimSearchView estimates={suggestions}/>}/>
<Route path='/zim' exact element={<ZimSearchView estimates={estimates}/>}/>
</Routes>
</React.Fragment>
}

8
modules/zim/lib.py
View file

@ -5,6 +5,7 @@ import re
from datetime import datetime
from typing import List, Tuple, Dict
import cachetools.func
from libzim import Entry
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound # noqa
@ -208,6 +209,13 @@ def headline_zim(search_str: str, zim_id: int, tag_names: List[str] = None, offs
return results
@cachetools.func.mru_cache(maxsize=1_000)
def get_estimates(search_str: str) -> List[int]:
zims = Zims.get_all()
estimates = [i.estimate(search_str) for i in zims]
return estimates
@optional_session
def get_zim(zim_id: int, session: Session = None) -> Zim:
"""Return the Zim record of the provided path.

2
requirements.txt
View file

@ -8,7 +8,7 @@ beautifulsoup4==4.12.2
cachetools==5.3.0
ebooklib==0.18
feedparser==6.0.10
libzim==3.1.0
libzim==3.3.0.post0
mock==5.0.2
psutil==5.9.5
psycopg2-binary==2.9.6

40
wrolpi/root_api.py
View file

@ -507,21 +507,43 @@ async def post_shutdown(_: Request):
@api_bp.post('/search_suggestions')
@validate(schema.SearchSuggestionsRequest)
@validate(json=schema.SearchSuggestionsRequest)
async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequest):
"""Used by the Global search to suggest links to the user."""
"""Used by the Global search to suggest related Channels/Domains/etc. to the user.
@note: Does not handle Tags, unlike search_estimates"""
from modules.videos.channel.lib import search_channels_by_name
from modules.archive.lib import search_domains_by_name
from modules.zim import lib as zim_lib
file_groups, channels, domains = await asyncio.gather(
estimate_search(body.search_str, []),
search_channels_by_name(body.search_str),
search_domains_by_name(body.search_str),
)
# Get estimates using libzim.
zims_estimates = zim_lib.get_estimates(body.search_str)
ret = dict(
file_groups=file_groups,
zims_estimates=zims_estimates,
channels=channels,
domains=domains,
)
return json_response(ret)
@api_bp.post('/search_estimates')
@validate(json=schema.SearchEstimateRequest)
async def post_search_estimates(_: Request, body: schema.SearchEstimateRequest):
"""Get an estimated count of FileGroups/Zims which may or may not have been tagged."""
from modules.zim.models import Zims
if not body.search_str and not body.tag_names:
return response.empty(HTTPStatus.BAD_REQUEST)
file_groups, channels, domains = await asyncio.gather(
estimate_search(body.search_str, body.tag_names),
search_channels_by_name(body.search_str),
search_domains_by_name(body.search_str),
)
file_groups = await estimate_search(body.search_str, body.tag_names)
if body.tag_names:
# Get actual count of entries tagged with the tag names.
@ -544,9 +566,7 @@ async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequ
ret = dict(
file_groups=file_groups,
zims_estimates=zims_estimates,
channels=channels,
domains=domains,
zims_estimates=zims_estimates
)
return json_response(ret)

5
wrolpi/schema.py
View file

@ -138,12 +138,11 @@ class VINDecoderResponse:
@dataclass
class SearchEstimateRequest:
class SearchSuggestionsRequest:
search_str: Optional[str] = None
tag_names: List[str] = field(default_factory=lambda: list())
@dataclass
class SearchSuggestionsRequest:
class SearchEstimateRequest:
search_str: Optional[str] = None
tag_names: List[str] = field(default_factory=lambda: list())

20
wrolpi/test/test_root_api.py
View file

@ -470,6 +470,7 @@ async def test_post_vin_number_decoder(test_async_client):
@pytest.mark.asyncio
async def test_search_suggestions(test_session, test_async_client, channel_factory, archive_factory, tag_factory,
zim_factory):
# WARNING results are cached, this test uses unique queries to avoid conflicts.
channel_factory(name='Foo')
channel_factory(name='Fool')
channel_factory(name='Bar')
@ -489,7 +490,7 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
assert response.json['domains'] == [{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}]
assert response.json['file_groups'] == 1
assert 'zims_estimates' in response.json and len(response.json['zims_estimates']) == 1
assert response.json['zims_estimates'][0]['estimate'] == 0
assert response.json['zims_estimates'][0] == 0
body = dict(search_str='bar')
request, response = await test_async_client.post('/api/search_suggestions', json=body)
@ -497,7 +498,7 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
assert response.json['channels'] == [{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar'}]
assert response.json['domains'] == [{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}]
assert response.json['file_groups'] == 1
assert response.json['zims_estimates'][0]['estimate'] == 0
assert response.json['zims_estimates'][0] == 0
body = dict(search_str='one')
request, response = await test_async_client.post('/api/search_suggestions', json=body)
@ -505,6 +506,21 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
assert response.json['channels'] == []
assert response.json['domains'] == []
assert response.json['file_groups'] == 0
assert response.json['zims_estimates'][0] == 2
@pytest.mark.asyncio
async def test_search_estimates(test_session, test_async_client, archive_factory, zim_factory):
archive_factory(domain='foo.com', contents='contents of foo')
archive_factory(domain='bar.com', contents='contents of bar')
zim_factory('test zim')
test_session.commit()
body = dict(search_str='one')
request, response = await test_async_client.post('/api/search_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == 0
assert len(response.json['zims_estimates']) == 1
assert response.json['zims_estimates'][0]['estimate'] == 2

2
wrolpi/version.txt
View file

@ -1 +1 @@
0.12.2-beta
0.12.3-beta