diff --git a/app/src/api.js b/app/src/api.js
index 216c45a7..3b4d9628 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -1022,6 +1022,21 @@ export async function searchSuggestions(search_str) {
}
}
+export async function searchEstimate(search_str, tagNames) {
+ const body = {search_str: search_str, tag_names: tagNames};
+ const response = await apiPost(`${API_URI}/search_estimates`, body);
+ if (response.ok) {
+ const content = await response.json();
+
+ const fileGroups = content.file_groups;
+ const zimsEstimates = content.zims_estimates;
+ return {
+ fileGroups,
+ zimsEstimates,
+ }
+ }
+}
+
export async function getOutdatedZims() {
const response = await apiGet(`${API_URI}/zim/outdated`);
if (response.status === 200) {
diff --git a/app/src/components/Common.js b/app/src/components/Common.js
index 86539253..db313488 100644
--- a/app/src/components/Common.js
+++ b/app/src/components/Common.js
@@ -70,7 +70,7 @@ export function Paginator({activePage, onPageChange, totalPages, showFirstAndLas
onPageChange={handlePageChange}
size={size}
siblingRange={2}
- totalPages={totalPages}
+ totalPages={totalPages || 1}
firstItem={showFirstAndLast ? undefined : null}
lastItem={showFirstAndLast ? undefined : null}
/>
@@ -82,7 +82,7 @@ export function Paginator({activePage, onPageChange, totalPages, showFirstAndLas
onPageChange={handlePageChange}
size={size}
siblingRange={5}
- totalPages={totalPages}
+ totalPages={totalPages || 1}
firstItem={showFirstAndLast ? undefined : null}
lastItem={showFirstAndLast ? undefined : null}
/>
diff --git a/app/src/components/Search.js b/app/src/components/Search.js
index c9040700..4b33e81e 100644
--- a/app/src/components/Search.js
+++ b/app/src/components/Search.js
@@ -1,4 +1,4 @@
-import React, {useContext, useEffect, useState} from "react";
+import React, {useState} from "react";
import {createSearchParams, Route, Routes, useNavigate} from "react-router-dom";
import {FilesSearchView} from "./Files";
import {useLatestRequest, usePages, useQuery} from "../hooks/customHooks";
@@ -7,7 +7,7 @@ import {searchEstimate, searchSuggestions} from "../api";
import {fuzzyMatch, normalizeEstimate, TabLinks} from "./Common";
import _ from "lodash";
import {TagsContext} from "../Tags";
-import {Header as SHeader} from "semantic-ui-react";
+import {Header as SHeader, Label} from "semantic-ui-react";
export const SearchSuggestionsContext = React.createContext({
suggestionsResults: {},
@@ -18,6 +18,15 @@ export const SearchSuggestionsContext = React.createContext({
loading: false,
});
+
+export const SearchSuggestionsProvider = (props) => {
+ const value = useSearchSuggestions();
+
+ return
+ {props.children}
+
+}
+
const SUGGESTED_APPS = [
{location: '/more/otp', title: 'One Time Pad', description: 'Encrypt and Decrypt messages'},
{location: '/inventory', title: 'Inventory', description: 'Track and organize your food storage'},
@@ -29,142 +38,6 @@ const SUGGESTED_APPS = [
{location: '/help', title: 'Help', description: 'Help documents for WROLPi'},
];
-
-export function useSearchSuggestions() {
- const navigate = useNavigate();
- const {searchParams} = useQuery();
- const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext);
-
- const searchStr = searchParams.get('q');
-
- const {data, sendRequest, loading} = useLatestRequest(500);
- const [suggestionsResults, setSuggestionsResults] = useState({});
- const [suggestions, setSuggestions] = useState({});
- const [suggestionsSums, setSuggestionsSums] = useState({});
-
- const normalizeSuggestionsResults = (newSuggestions) => {
- // Convert the suggestions from the Backend to what the Semantic expects.
- const lowerSearchStr = searchStr.toLowerCase();
-
- let results = {};
-
- // Suggested results are ordered.
- if (newSuggestions.fileGroups > 0) {
- results.fileGroups = {
- name: 'Files', results: [
- {
- title: newSuggestions.fileGroups.toString(),
- type: 'files',
- location: `/search?q=${encodeURIComponent(searchStr)}`
- }
- ]
- };
- }
-
- const zimSum = newSuggestions.zimsEstimates.reduce((i, j) => i + j['estimate'], 0);
- if (zimSum > 0) {
- results.zimsSum = {
- name: 'Zims', results: [
- {title: zimSum.toString(), type: 'zims', location: `/search/zim?q=${encodeURIComponent(searchStr)}`}
- ],
- }
- }
- if (newSuggestions.channels && newSuggestions.channels.length > 0) {
- results.channels = {
- name: 'Channels', results: newSuggestions.channels.map(i => {
- return {type: 'channel', title: i['name'], id: i['id'], location: `/videos/channel/${i.id}/video`}
- })
- }
- }
- if (newSuggestions.domains && newSuggestions.domains.length > 0) {
- results.domains = {
- name: 'Domains', results: newSuggestions.domains.map(i => {
- return {
- type: 'domain',
- title: i.domain,
- id: i.id,
- domain: i.domain,
- location: `/archive?domain=${i.domain}`
- }
- })
- }
- }
-
- const matchingTags = fuzzyMatchTagsByName(searchStr);
- if (matchingTags && matchingTags.length > 0) {
- results.tags = {
- name: 'Tags', results: matchingTags.map(i => {
- return {type: 'tag', title: i.name, location: `/search?tag=${encodeURIComponent(i.name)}`}
- })
- }
- }
-
- const matchingApps = SUGGESTED_APPS.filter(i =>
- i.title.toLowerCase().includes(lowerSearchStr)
- || fuzzyMatch(i.title.toLowerCase(), lowerSearchStr));
- if (matchingApps && matchingApps.length > 0) {
- results.apps = {name: 'Apps', results: matchingApps};
- }
-
- setSuggestionsResults(results);
- setSuggestionsSums({
- fileGroups: newSuggestions.fileGroups,
- zims: zimSum,
- channels: newSuggestions.channels.length,
- domains: newSuggestions.domains.length,
- tags: matchingTags.length,
- apps: matchingApps.length,
- })
- }
-
- React.useEffect(() => {
- if (data) {
- setSuggestions(data);
- normalizeSuggestionsResults(data);
- }
- }, [JSON.stringify(data)]);
-
- React.useEffect(() => {
- if (!searchStr || searchStr.length === 0) {
- console.debug('Not getting suggestions because there is no search.');
- return;
- }
- setSuggestions({});
- setSuggestionsSums({});
- setSuggestionsResults({});
-
- // Use the useLatestRequest to handle user typing.
- sendRequest(async () => await searchSuggestions(searchStr));
- }, [searchStr, sendRequest]);
-
- const handleResultSelect = ({result}) => navigate(result.location);
-
- const resultRenderer = ({type, title, description}) => {
- if (type === 'tag') {
- return ;
- }
-
- // No specific renderer, use the generic.
- if (description) {
- return <>
- {title}
- {description}
-
- }
- return {title}
- };
-
- return {suggestions, suggestionsResults, suggestionsSums, searchStr, handleResultSelect, resultRenderer, loading}
-}
-
-export const SearchSuggestionsProvider = (props) => {
- const value = useSearchSuggestions();
-
- return
- {props.children}
-
-}
-
export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false, model) => {
const navigate = useNavigate();
@@ -224,15 +97,178 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false
}
}
+export function useSuggestions(searchStr) {
+ const {data, sendRequest, loading} = useLatestRequest(500);
+
+ React.useEffect(() => {
+ if (!searchStr || searchStr.length === 0) {
+ return;
+ }
+
+ // Use the `useLatestRequest` to handle user typing.
+ sendRequest(async () => await searchSuggestions(searchStr));
+ }, [searchStr, sendRequest]);
+
+ return {suggestions: data, loading}
+}
+
+
+export function useSearchSuggestions() {
+ const navigate = useNavigate();
+ const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext);
+ const {searchStr} = useSearch();
+ const {suggestions, loading} = useSuggestions(searchStr);
+
+ // The results that will be displayed by .
+ const [suggestionsResults, setSuggestionsResults] = useState({});
+ // The results summarized.
+ const [suggestionsSums, setSuggestionsSums] = useState({});
+
+ const normalizeSuggestionsResults = (newSuggestions) => {
+ // Convert the suggestions from the Backend to what the Semantic expects.
+ const lowerSearchStr = searchStr.toLowerCase();
+
+ let results = {};
+
+ // Suggested results are ordered.
+ if (newSuggestions.fileGroups > 0) {
+ results.fileGroups = {
+ name: 'Files', results: [
+ {
+ title: newSuggestions.fileGroups.toString(),
+ type: 'files',
+ location: `/search?q=${encodeURIComponent(searchStr)}`
+ }
+ ]
+ };
+ }
+
+ const zimSum = newSuggestions.zimsEstimates.reduce((i, j) => i + j, 0);
+ if (zimSum > 0) {
+ results.zimsSum = {
+ name: 'Zims', results: [
+ {title: zimSum.toString(), type: 'zims', location: `/search/zim?q=${encodeURIComponent(searchStr)}`}
+ ],
+ }
+ }
+ if (newSuggestions.channels && newSuggestions.channels.length > 0) {
+ results.channels = {
+ name: 'Channels', results: newSuggestions.channels.map(i => {
+ return {type: 'channel', title: i['name'], id: i['id'], location: `/videos/channel/${i.id}/video`}
+ })
+ }
+ }
+ if (newSuggestions.domains && newSuggestions.domains.length > 0) {
+ results.domains = {
+ name: 'Domains', results: newSuggestions.domains.map(i => {
+ return {
+ type: 'domain',
+ title: i.domain,
+ id: i.id,
+ domain: i.domain,
+ location: `/archive?domain=${i.domain}`
+ }
+ })
+ }
+ }
+
+ const matchingTags = fuzzyMatchTagsByName(searchStr);
+ if (matchingTags && matchingTags.length > 0) {
+ results.tags = {
+ name: 'Tags', results: matchingTags.map(i => {
+ return {type: 'tag', title: i.name, location: `/search?tag=${encodeURIComponent(i.name)}`}
+ })
+ }
+ }
+
+ const matchingApps = SUGGESTED_APPS.filter(i =>
+ i.title.toLowerCase().includes(lowerSearchStr)
+ || fuzzyMatch(i.title.toLowerCase(), lowerSearchStr));
+ if (matchingApps && matchingApps.length > 0) {
+ results.apps = {name: 'Apps', results: matchingApps};
+ }
+
+ setSuggestionsResults(results);
+ setSuggestionsSums({
+ fileGroups: newSuggestions.fileGroups,
+ zims: zimSum,
+ channels: newSuggestions.channels.length,
+ domains: newSuggestions.domains.length,
+ tags: matchingTags.length,
+ apps: matchingApps.length,
+ });
+ }
+
+ React.useEffect(() => {
+ setSuggestionsSums({});
+ setSuggestionsResults({});
+
+ if (!_.isEmpty(suggestions)) {
+ normalizeSuggestionsResults(suggestions);
+ }
+ }, [JSON.stringify(suggestions)]);
+
+ // User clicked on a result in the dropdown.
+ const handleResultSelect = ({result}) => navigate(result.location);
+
+ const resultRenderer = ({type, title, description}) => {
+ if (type === 'tag') {
+ return ;
+ }
+
+ if (description) {
+ return <>
+ {title}
+ {description}
+
+ }
+ // No specific renderer, use the generic.
+ return {title}
+ };
+
+ return {suggestions, suggestionsResults, suggestionsSums, searchStr, handleResultSelect, resultRenderer, loading}
+}
+
+
+export function useSearchEstimate(searchStr, tagNames) {
+ const {data, sendRequest, loading} = useLatestRequest(250);
+
+ // The estimates from the response.
+ const [estimates, setEstimates] = React.useState({});
+ // The estimates summarized.
+ const [estimatesSums, setEstimatesSums] = React.useState({});
+
+ React.useEffect(() => {
+ // Handle results from `sendRequest`.
+ if (_.isEmpty(data)) {
+ return;
+ }
+
+ setEstimates(data);
+ setEstimatesSums({
+ fileGroups: data.fileGroups,
+ zims: data.zimsEstimates.reduce((i, j) => i + j['estimate'], 0),
+ });
+ }, [JSON.stringify(data)]);
+
+ React.useEffect(() => {
+ // Use the `useLatestRequest` to handle user typing.
+ sendRequest(async () => await searchEstimate(searchStr, tagNames));
+ }, [searchStr, JSON.stringify(tagNames)], sendRequest);
+
+ return {estimates, estimatesSums, loading}
+}
+
export function SearchView() {
- const {suggestionsSums, suggestions} = useContext(SearchSuggestionsContext);
+ const {searchStr, activeTags} = useSearch();
+ const {estimates, estimatesSums} = useSearchEstimate(searchStr, activeTags);
- let filesTabName = 'Files';
- let zimsTabName = 'Zims';
- if (!_.isEmpty(suggestionsSums)) {
- filesTabName = `Files (${normalizeEstimate(suggestionsSums.fileGroups)})`;
- zimsTabName = `Zims (${normalizeEstimate(suggestionsSums.zims)})`;
+ let filesTabName = Files ;
+ let zimsTabName = Zims;
+ if (!_.isEmpty(estimatesSums)) {
+ filesTabName = Files ;
+ zimsTabName = Zims ;
}
const links = [
@@ -244,7 +280,7 @@ export function SearchView() {
}/>
- }/>
+ }/>
}
diff --git a/modules/zim/lib.py b/modules/zim/lib.py
index efa377cb..09e518b6 100644
--- a/modules/zim/lib.py
+++ b/modules/zim/lib.py
@@ -5,6 +5,7 @@ import re
from datetime import datetime
from typing import List, Tuple, Dict
+import cachetools.func
from libzim import Entry
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound # noqa
@@ -208,6 +209,13 @@ def headline_zim(search_str: str, zim_id: int, tag_names: List[str] = None, offs
return results
+@cachetools.func.mru_cache(maxsize=1_000)
+def get_estimates(search_str: str) -> List[int]:
+ zims = Zims.get_all()
+ estimates = [i.estimate(search_str) for i in zims]
+ return estimates
+
+
@optional_session
def get_zim(zim_id: int, session: Session = None) -> Zim:
"""Return the Zim record of the provided path.
diff --git a/requirements.txt b/requirements.txt
index eadaf584..676c5779 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,7 +8,7 @@ beautifulsoup4==4.12.2
cachetools==5.3.0
ebooklib==0.18
feedparser==6.0.10
-libzim==3.1.0
+libzim==3.3.0.post0
mock==5.0.2
psutil==5.9.5
psycopg2-binary==2.9.6
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 3771fb45..66d5be17 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -507,21 +507,43 @@ async def post_shutdown(_: Request):
@api_bp.post('/search_suggestions')
-@validate(schema.SearchSuggestionsRequest)
+@validate(json=schema.SearchSuggestionsRequest)
async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequest):
- """Used by the Global search to suggest links to the user."""
+ """Used by the Global search to suggest related Channels/Domains/etc. to the user.
+
+ @note: Does not handle Tags, unlike search_estimates"""
from modules.videos.channel.lib import search_channels_by_name
from modules.archive.lib import search_domains_by_name
+ from modules.zim import lib as zim_lib
+
+ file_groups, channels, domains = await asyncio.gather(
+ estimate_search(body.search_str, []),
+ search_channels_by_name(body.search_str),
+ search_domains_by_name(body.search_str),
+ )
+
+ # Get estimates using libzim.
+ zims_estimates = zim_lib.get_estimates(body.search_str)
+
+ ret = dict(
+ file_groups=file_groups,
+ zims_estimates=zims_estimates,
+ channels=channels,
+ domains=domains,
+ )
+ return json_response(ret)
+
+
+@api_bp.post('/search_estimates')
+@validate(json=schema.SearchEstimateRequest)
+async def post_search_estimates(_: Request, body: schema.SearchEstimateRequest):
+ """Get an estimated count of FileGroups/Zims which may or may not have been tagged."""
from modules.zim.models import Zims
if not body.search_str and not body.tag_names:
return response.empty(HTTPStatus.BAD_REQUEST)
- file_groups, channels, domains = await asyncio.gather(
- estimate_search(body.search_str, body.tag_names),
- search_channels_by_name(body.search_str),
- search_domains_by_name(body.search_str),
- )
+ file_groups = await estimate_search(body.search_str, body.tag_names)
if body.tag_names:
# Get actual count of entries tagged with the tag names.
@@ -544,9 +566,7 @@ async def post_search_suggestions(_: Request, body: schema.SearchSuggestionsRequ
ret = dict(
file_groups=file_groups,
- zims_estimates=zims_estimates,
- channels=channels,
- domains=domains,
+ zims_estimates=zims_estimates
)
return json_response(ret)
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index 0655f384..99aab86f 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -138,12 +138,11 @@ class VINDecoderResponse:
@dataclass
-class SearchEstimateRequest:
+class SearchSuggestionsRequest:
search_str: Optional[str] = None
- tag_names: List[str] = field(default_factory=lambda: list())
@dataclass
-class SearchSuggestionsRequest:
+class SearchEstimateRequest:
search_str: Optional[str] = None
tag_names: List[str] = field(default_factory=lambda: list())
diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py
index f96fc46a..2abb484b 100644
--- a/wrolpi/test/test_root_api.py
+++ b/wrolpi/test/test_root_api.py
@@ -470,6 +470,7 @@ async def test_post_vin_number_decoder(test_async_client):
@pytest.mark.asyncio
async def test_search_suggestions(test_session, test_async_client, channel_factory, archive_factory, tag_factory,
zim_factory):
+ # WARNING results are cached, this test uses unique queries to avoid conflicts.
channel_factory(name='Foo')
channel_factory(name='Fool')
channel_factory(name='Bar')
@@ -489,7 +490,7 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
assert response.json['domains'] == [{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}]
assert response.json['file_groups'] == 1
assert 'zims_estimates' in response.json and len(response.json['zims_estimates']) == 1
- assert response.json['zims_estimates'][0]['estimate'] == 0
+ assert response.json['zims_estimates'][0] == 0
body = dict(search_str='bar')
request, response = await test_async_client.post('/api/search_suggestions', json=body)
@@ -497,7 +498,7 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
assert response.json['channels'] == [{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar'}]
assert response.json['domains'] == [{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}]
assert response.json['file_groups'] == 1
- assert response.json['zims_estimates'][0]['estimate'] == 0
+ assert response.json['zims_estimates'][0] == 0
body = dict(search_str='one')
request, response = await test_async_client.post('/api/search_suggestions', json=body)
@@ -505,6 +506,21 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
assert response.json['channels'] == []
assert response.json['domains'] == []
assert response.json['file_groups'] == 0
+ assert response.json['zims_estimates'][0] == 2
+
+
+@pytest.mark.asyncio
+async def test_search_estimates(test_session, test_async_client, archive_factory, zim_factory):
+ archive_factory(domain='foo.com', contents='contents of foo')
+ archive_factory(domain='bar.com', contents='contents of bar')
+ zim_factory('test zim')
+ test_session.commit()
+
+ body = dict(search_str='one')
+ request, response = await test_async_client.post('/api/search_estimates', json=body)
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['file_groups'] == 0
+ assert len(response.json['zims_estimates']) == 1
assert response.json['zims_estimates'][0]['estimate'] == 2
diff --git a/wrolpi/version.txt b/wrolpi/version.txt
index 39e8174c..c2191180 100644
--- a/wrolpi/version.txt
+++ b/wrolpi/version.txt
@@ -1 +1 @@
-0.12.2-beta
\ No newline at end of file
+0.12.3-beta
\ No newline at end of file