Only files that have been tagged will be returned when new 'Any' tag button is used to filter file search results.

This commit is contained in:
Roland 2024-07-22 16:36:57 -06:00
parent f3439c2bd5
commit 53d2402d32
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
31 changed files with 303 additions and 146 deletions

View file

@ -161,7 +161,7 @@ export function DashboardPage() {
const navigate = useNavigate();
// The search the user submitted.
const {searchStr, setSearchStr, activeTags, isEmpty} = useSearch();
const {searchStr, setSearchStr, activeTags, isEmpty, anyTag} = useSearch();
// The search that the user is typing.
const [localSearchStr, setLocalSearchStr] = React.useState(searchStr);
const {
@ -174,7 +174,7 @@ export function DashboardPage() {
setSearchStr: setSuggestionSearchStr,
setSearchTags,
months, dateRange, setDates, clearDate,
} = useSearchSuggestions(searchStr, activeTags);
} = useSearchSuggestions(searchStr, activeTags, anyTag);
React.useEffect(() => {
setSuggestionSearchStr(localSearchStr);

View file

@ -310,11 +310,12 @@ function EditTagsModal() {
export function AddTagsButton({
hideEdit,
active,
showAny = false,
selectedTagNames = [],
onToggle = _.noop,
anyTag = false,
onAdd = _.noop,
onRemove = _.noop,
onChange = _.noop, // Expects to send: (tagNames, anyTag)
limit = null,
}) {
// A button which displays a modal in which the user can add or remove tags.
@ -324,6 +325,8 @@ export function AddTagsButton({
const [loading, setLoading] = React.useState(false);
const [localTags, setLocalTags] = React.useState(selectedTagNames);
const active = anyTag || (selectedTagNames && selectedTagNames.length > 0);
const handleOpen = (e) => {
if (e) {
e.preventDefault();
@ -339,8 +342,8 @@ export function AddTagsButton({
return;
}
setLocalTags(newTags);
onToggle(newTags);
onAdd(name);
onChange(newTags, null);
} finally {
setLoading(false);
}
@ -350,30 +353,30 @@ export function AddTagsButton({
setLoading(true);
try {
const newTags = localTags.filter(i => i !== name);
setLocalTags(newTags);
onToggle(newTags);
setLocalTags(newTags)
onRemove(name);
onChange(newTags, null);
} finally {
setLoading(false);
}
}
const clearLocalTags = () => {
if (!localTags || (localTags && localTags.length === 0)) {
if (!anyTag && (!localTags || (localTags && localTags.length === 0))) {
console.debug('No tags to clear');
return
}
setLoading(true);
try {
for (let i = 0; i < localTags.length; i++) {
onRemove(localTags[i]);
}
setLocalTags([]);
onToggle([]);
} finally {
setLoading(false);
}
console.debug('Clearing selected tags');
onChange([], null);
setLocalTags([]);
setOpen(false);
}
const localOnAnyTag = () => {
console.debug('Setting any tag');
onChange([], true);
setOpen(false);
}
const selectedTagsGroup = <TagsGroup tagNames={localTags} onClick={removeTag}/>;
@ -382,7 +385,11 @@ export function AddTagsButton({
const emptySelectedTags = limit === 1 ? 'Add only one tag below' : 'Add one or more tags below';
return <>
<Button icon={active ? 'tags' : 'tag'} onClick={handleOpen} primary={!!active}/>
<Button
icon={active ? 'tags' : 'tag'}
color={active ? 'violet' : undefined}
onClick={handleOpen}
/>
<Modal closeIcon
open={open}
onOpen={(e) => handleOpen(e)}
@ -405,6 +412,7 @@ export function AddTagsButton({
</Grid.Column>
<Grid.Column width={8}>
<Button onClick={() => setOpen(false)} floated='right'>Close</Button>
{showAny && <Button color='violet' onClick={localOnAnyTag} floated='right'>Any</Button>}
<Button floated='right' secondary onClick={() => clearLocalTags()}>Clear</Button>
</Grid.Column>
</Grid.Row>
@ -418,12 +426,13 @@ export const taggedImageLabel = {corner: 'left', icon: 'tag', color: 'green'};
export const TagsSelector = ({
hideEdit = false,
active,
showAny = false,
hideGroup = false,
selectedTagNames = [],
onToggle = _.noop,
anyTag = false,
onAdd = _.noop,
onRemove = _.noop,
onChange = _.noop,
limit = null,
}) => {
// Provides a button to add tags to a list. Displays the tags of that list.
@ -436,11 +445,12 @@ export const TagsSelector = ({
const button = <AddTagsButton
hideEdit={hideEdit}
active={active}
showAny={showAny}
selectedTagNames={selectedTagNames}
onToggle={onToggle}
onAdd={onAdd}
onRemove={onRemove}
onChange={onChange}
anyTag={anyTag}
limit={limit}
/>;

View file

@ -627,8 +627,8 @@ export async function deleteDownload(downloadId) {
}
}
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear) {
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)};
export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag) {
const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit), any_tag: anyTag};
if (mimetypes) {
body['mimetypes'] = mimetypes;
}
@ -1183,10 +1183,10 @@ export async function searchSuggestions(search_str) {
}
}
export async function searchEstimateFiles(search_str, tagNames, mimetypes, months, dateRange) {
export async function searchEstimateFiles(search_str, tagNames, mimetypes, months, dateRange, anyTag) {
months = months ? months.map(i => parseInt(i)) : [];
const body = {search_str, tag_names: tagNames, mimetypes, months};
const body = {search_str, tag_names: tagNames, mimetypes, months, any_tag: anyTag};
if (dateRange) {
body['from_year'] = dateRange[0];
body['to_year'] = dateRange[1];
@ -1198,6 +1198,8 @@ export async function searchEstimateFiles(search_str, tagNames, mimetypes, month
return {
fileGroups: content.file_groups,
}
} else {
console.error('Failed to get file search estimates!');
}
}

View file

@ -263,7 +263,11 @@ function ArchivePage() {
</Segment>
<Segment>
<TagsSelector selectedTagNames={archiveFile['tags']} onAdd={localAddTag} onRemove={localRemoveTag}/>
<TagsSelector
selectedTagNames={archiveFile['tags']}
onAdd={localAddTag}
onRemove={localRemoveTag}
/>
</Segment>
<Tab menu={tabMenu} panes={tabPanes}/>
@ -497,14 +501,16 @@ function ArchivesPage() {
</div>;
const {body, paginator, selectButton, viewButton, limitDropdown, tagQuerySelector} = FilesView(
archives,
activePage,
totalPages,
selectElm,
selectedArchives,
onSelect,
setPage,
!!searchStr,
{
files: archives,
activePage: activePage,
totalPages: totalPages,
selectElem: selectElm,
selectedKeys: selectedArchives,
onSelect: onSelect,
setPage: setPage,
headlines: !!searchStr
},
);

View file

@ -488,14 +488,6 @@ export function ChannelDownloadForm({channel_id, afterSave, closeModal, download
}
};
const handleAddTag = (tagName) => {
setTagNames([...tagNames, tagName]);
}
const handleUntag = (tagName) => {
setTagNames(tagNames.filter(i => i !== tagName));
}
const deleteDownloadButton = <APIButton
color='red'
floated='left'
@ -571,7 +563,7 @@ export function ChannelDownloadForm({channel_id, afterSave, closeModal, download
</Grid.Row>
<Grid.Row columns={1}>
<Grid.Column>
<TagsSelector selectedTagNames={tagNames} onAdd={handleAddTag} onRemove={handleUntag}/>
<TagsSelector selectedTagNames={tagNames} onChange={(i, j) => setTagNames(i)}/>
</Grid.Column>
</Grid.Row>
<Grid.Row columns={1}>

View file

@ -77,7 +77,7 @@ class Downloader extends React.Component {
}
}
handleSelectedTags = (tagNames) => {
handleTagsChange = (tagNames, newAnyTag) => {
this.setState({tagNames});
}
@ -119,7 +119,7 @@ class Downloader extends React.Component {
let tagsSelector;
if (withTags) {
tagsSelector = <TagsSelector selectedTagNames={tagNames} onToggle={this.handleSelectedTags}/>;
tagsSelector = <TagsSelector selectedTagNames={tagNames} onChange={this.handleTagsChange}/>;
}
const advancedAccordion = <Accordion>

View file

@ -32,7 +32,6 @@ import {
useTitle
} from "./Common";
import {
useAllQuery,
useFilesProgressInterval,
usePages,
useSearchFiles,
@ -66,6 +65,7 @@ import {refreshFiles} from "../api";
import {useSubscribeEventName} from "../Events";
import {TagsSelector} from "../Tags";
import {Headlines} from "./Headline";
import {useSearch} from "./Search";
function EbookCard({file}) {
const {s} = useContext(ThemeContext);
@ -311,15 +311,18 @@ export function SearchLimitDropdown({limits = []}) {
}
export function FilesView(
files,
activePage,
totalPages,
selectElem,
selectedKeys,
onSelect,
setPage,
headlines = false,
limitOptions = [12, 24, 48, 96],
{
files,
activePage,
totalPages,
selectElem,
selectedKeys,
onSelect,
setPage,
headlines = false,
limitOptions = [12, 24, 48, 96],
showAnyTag = false,
},
) {
const {view} = useSearchView();
@ -360,7 +363,7 @@ export function FilesView(
const viewButton = <SearchViewButton headlines={headlines}/>
const limitDropdown = <SearchLimitDropdown limits={limitOptions}/>;
const tagQuerySelector = <TagsQuerySelector/>;
const tagQuerySelector = <TagsQuerySelector showAny={showAnyTag}/>;
return {
body,
@ -432,22 +435,28 @@ export function SearchFilter({filters = [], modalHeader, size = 'medium'}) {
return <></>
}
export function TagsQuerySelector({onChange}) {
// Creates modal that the User can use to manipulate the tag query.
const [activeTags, setActiveTags] = useAllQuery('tag');
export function TagsQuerySelector({onChange, showAny = true}) {
// Creates modal that the User can use to manipulate the tag query params.
const {activeTags, anyTag, setTags, setAnyTag} = useSearch();
const localOnChange = (tagNames) => {
setActiveTags(tagNames);
const localOnChange = (tagNames, newAnyTag) => {
if (newAnyTag) {
setAnyTag(true);
} else {
setTags(tagNames);
}
if (onChange) {
onChange(tagNames);
onChange(tagNames, newAnyTag);
}
}
return <TagsSelector hideGroup={true}
hideEdit={true}
active={activeTags && activeTags.length > 0}
showAny={showAny}
active={anyTag || activeTags && activeTags.length > 0}
selectedTagNames={activeTags}
onToggle={localOnChange}
anyTag={anyTag}
onChange={localOnChange}
style={{marginLeft: '0.3em', marginTop: '0.3em'}}
/>
}
@ -479,14 +488,17 @@ export function FilesSearchView({
const {searchFiles, pages} = useSearchFiles(24, emptySearch, model);
const {body, paginator, selectButton, viewButton, limitDropdown, tagQuerySelector} = FilesView(
searchFiles,
pages.activePage,
pages.totalPages,
null,
null,
null,
pages.setPage,
true,
{
files: searchFiles,
activePage: pages.activePage,
totalPages: pages.totalPages,
selectElem: null,
selectedKeys: null,
onSelect: null,
setPage: pages.setPage,
headlines: true,
showAnyTag: true,
},
);
return <>

View file

@ -82,7 +82,7 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false
}
const setTags = (tags) => {
updateQuery({tag: tags});
updateQuery({tag: tags, anyTag: null});
}
const addTag = (name) => {
@ -95,24 +95,26 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false
setTags(newTags);
}
const anyTag = searchParams.get('anyTag') === 'true';
const setAnyTag = (value) => {
updateQuery({tag: [], anyTag: value ? 'true' : null});
}
return {
activeTags,
addTag,
activeTags, addTag, removeTag, anyTag, setAnyTag, setTags,
filter,
model: model_,
pages,
removeTag,
searchParams,
searchStr,
setSearchStr,
clearSearch,
setTags,
months, dateRange, setDateRange, setDates,
anySearch, isEmpty,
}
}
export function useSuggestions(searchStr, tagNames, filter) {
export function useSuggestions(searchStr, tagNames, filter, anyTag) {
const defaultSuggestions = {
fileGroups: [],
channels: [],
@ -135,7 +137,7 @@ export function useSuggestions(searchStr, tagNames, filter) {
// We can't search Channels/Domains without some string to filter by.
sendGeneralReqeust(async () => await searchSuggestions(searchStr));
}
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange));
sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag));
sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
}
}, [
@ -146,6 +148,7 @@ export function useSuggestions(searchStr, tagNames, filter) {
JSON.stringify(months),
JSON.stringify(dateRange),
filter,
anyTag,
]);
React.useEffect(() => {
@ -180,14 +183,14 @@ export function useSuggestions(searchStr, tagNames, filter) {
}
export function useSearchSuggestions(defaultSearchStr, defaultTagNames) {
export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag) {
const navigate = useNavigate();
const {filter} = useSearchFilter();
const [searchStr, setSearchStr] = React.useState(defaultSearchStr || '');
const [searchTags, setSearchTags] = React.useState(defaultTagNames);
const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext);
const {dateRange, months, setDates, clearDate} = useSearchDate();
const {suggestions, loading} = useSuggestions(searchStr, searchTags, filter);
const {suggestions, loading} = useSuggestions(searchStr, searchTags, filter, anyTag);
const {getLocationStr} = React.useContext(QueryContext);
// The results that will be displayed by <Search>.
@ -239,6 +242,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames) {
return {type: 'channel', title: i['name'], id: i['id'], location: `/videos/channel/${i.id}/video`}
})
}
} else if (newSuggestions.channels && newSuggestions.channels.length === 0) {
results.channels = {name: 'Channels', results: noResults};
}
if (newSuggestions.domains && newSuggestions.domains.length > 0) {
results.domains = {
@ -339,7 +344,7 @@ export function SearchView({suggestions, suggestionsSums, loading}) {
const zimsTabName = <span>Zims <Label>{normalizeEstimate(suggestionsSums?.zims)}</Label></span>;
const links = [
{text: filesTabName, to: '/search', key: 'filesSearch', end: true},
{text: filesTabName, to: '/search', key: 'filesSearch_', end: true},
{text: zimsTabName, to: '/search/zim', key: 'zimsSearch'},
];

View file

@ -31,6 +31,7 @@ import {
CardDescription,
CardHeader,
Container,
Dimmer,
Image,
PlaceholderHeader,
PlaceholderLine,
@ -74,7 +75,7 @@ function VideosPage() {
searchOrder = defaultSearchOrder;
}
const {searchStr, setSearchStr, videos, activePage, setPage, totalPages, fetchVideos} =
const {searchStr, setSearchStr, videos, activePage, setPage, totalPages, fetchVideos, loading} =
useSearchVideos(null, channelId, searchOrder);
const {channel} = useChannel(channelId);
@ -165,14 +166,17 @@ function VideosPage() {
</div>;
const {body, paginator, selectButton, viewButton, limitDropdown, tagQuerySelector} = FilesView(
videos,
activePage,
totalPages,
selectElm,
selectedVideos,
onSelect,
setPage,
!!searchStr,
{
files: videos,
activePage: activePage,
totalPages: totalPages,
selectElem: selectElm,
selectedKeys: selectedVideos,
onSelect: onSelect,
setPage: setPage,
headlines: !!searchStr,
loading: loading,
},
);
const [localSearchStr, setLocalSearchStr] = React.useState(searchStr || '');

View file

@ -320,7 +320,7 @@ export const ZimSearchView = ({suggestions, loading}) => {
}
return <>
<TagsQuerySelector onChange={setTags}/>
<TagsQuerySelector onChange={(i, j) => setTags(i)}/>
<Accordion>
{body}
</Accordion>
@ -533,7 +533,7 @@ class ManageZim extends React.Component {
}
}
function ZimViewer({src = ZIM_VIEWER_URI, style=null}) {
function ZimViewer({src = ZIM_VIEWER_URI, style = null}) {
const fallback = <Segment>
<Header as='h3'>Failed to fetch Zim service.</Header>
<p>You may need to give permission to access the page: <a href={src}>{src}</a></p>

View file

@ -324,20 +324,25 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
const activeTags = searchParams.getAll('tag');
const {view} = useSearchView();
const headline = view === 'headline';
const anyTag = searchParams.get('anyTag') === 'true';
const [videos, setVideos] = useState(null);
const [totalPages, setTotalPages] = useState(0);
const [loading, setLoading] = useState(false);
const localSearchVideos = async () => {
setVideos(null);
setLoading(true);
setTotalPages(0);
try {
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline);
let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline, anyTag);
setVideos(videos_);
setTotalPages(calculateTotalPages(total, limit));
} catch (e) {
console.error(e);
setVideos(undefined);// Could not get Videos, display error.
} finally {
setLoading(false);
}
}
@ -368,6 +373,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => {
setOrderBy,
fetchVideos: localSearchVideos,
activeTags,
loading,
}
}
@ -481,7 +487,7 @@ export const useChannels = () => {
export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => {
const {
activeTags,
activeTags, anyTag,
pages,
searchStr,
filter,
@ -493,6 +499,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
const {view} = useSearchView();
const [searchFiles, setSearchFiles] = useState(null);
const [loading, setLoading] = useState(false);
const headline = view === 'headline';
const localSearchFiles = async () => {
@ -507,11 +514,12 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
fromDate = dateRange[0];
toDate = dateRange[1];
}
setLoading(true);
console.log('localSearchFiles', fromDate, toDate, months);
try {
let [file_groups, total] = await filesSearch(
pages.offset, pages.limit, searchStr, mimetypes, model || model_, activeTags, headline,
months, fromDate, toDate);
months, fromDate, toDate, anyTag);
setSearchFiles(file_groups);
pages.setTotal(total);
} catch (e) {
@ -523,6 +531,8 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
description: 'Could not get files',
time: 5000,
});
} finally {
setLoading(false);
}
}
@ -533,6 +543,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
useEffect(() => {
if (searchStr || (activeTags && activeTags.length > 0)) {
setLoading(true);
debouncedLocalSearchFiles();
}
// Handle when this is unmounted.
@ -547,6 +558,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
headline,
JSON.stringify(months),
JSON.stringify(dateRange),
anyTag,
]);
return {
@ -555,7 +567,8 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
filter,
setSearchStr,
pages,
activeTags
activeTags,
loading,
};
}
@ -1106,15 +1119,7 @@ export const useUploadFile = () => {
doUpload()
}, [JSON.stringify(files)]);
const handleAddTag = (tagName) => {
setTagNames([...tagNames, tagName]);
}
const handleUntag = (tagName) => {
setTagNames(tagNames.filter(i => i !== tagName));
}
const tagsSelector = <TagsSelector selectedTagNames={tagNames} onAdd={handleAddTag} onRemove={handleUntag}/>;
const tagsSelector = <TagsSelector selectedTagNames={tagNames} onChange={(i,) => setTagNames(i)}/>;
return {
destination,

View file

@ -4,6 +4,7 @@ apache2-doc
apt-transport-https
aria2
autoconf
bind9-dnsutils
build-essential
bzip2
ca-certificates

16
help.sh
View file

@ -341,4 +341,18 @@ if curl -k -s https://0.0.0.0:8086/ | grep MkDocs 2>/dev/null >/dev/null; then
echo "OK: Help service is running"
else
echo "FAILED: Help service is not running"
fi
fi
echo
# Internet
if ping -c1 1.1.1.1 -W 5 &>/dev/null; then
echo "OK: Can ping 1.1.1.1"
else
echo "FAILED: Cannot ping 1.1.1.1. Is internet working?"
fi
if ping -c1 wrolpi.org -W 5 &>/dev/null; then
echo "OK: Can ping wrolpi.org"
else
echo "FAILED: Cannot ping wrolpi.org Is internet working?"
fi

View file

@ -482,7 +482,7 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
if order in NO_NULL_ORDERS:
wheres.append(NO_NULL_ORDERS[order])
wheres, params = tag_append_sub_select_where(tag_names, wheres, params)
wheres, params = tag_append_sub_select_where(wheres, params, tag_names)
if search_str and headline:
headline = ''',

View file

@ -101,7 +101,7 @@ def search_videos(
# No search_str provided. Get id and total only.
select_columns = 'fg.id, COUNT(*) OVER() AS total'
wheres, params = tag_append_sub_select_where(tag_names, wheres, params)
wheres, params = tag_append_sub_select_where(wheres, params, tag_names)
if search_str and headline:
headline = ''',

View file

@ -99,3 +99,4 @@ class OutdatedZims:
class SearchEstimateRequest:
search_str: Optional[str] = None
tag_names: List[str] = field(default_factory=lambda: list())
any_tag: bool = False

View file

@ -4,6 +4,7 @@ apache2-doc
apt-transport-https
aria2
autoconf
bind9-dnsutils
build-essential
bzip2
ca-certificates
@ -83,9 +84,9 @@ python3-full
python3-mapnik
python3-pip
python3-venv
smartmontools
renderd
rsync
smartmontools
software-properties-common
tar
tcpdump

View file

@ -1067,15 +1067,17 @@ async def aiohttp_post(url: str, json_, timeout: int = None) -> Tuple[Dict, int]
async def aiohttp_get(url: str, timeout: int = None, headers: dict = None) -> Tuple[bytes, int]:
"""Perform an async aiohttp GET request. Return the contents."""
headers = headers or DEFAULT_HTTP_HEADERS
async with aiohttp_session(timeout) as session:
async with session.get(url, headers=headers) as response:
return await response.content.read(), response.status
async def aiohttp_head(url: str, timeout: int = None) -> Tuple[ClientResponse, int]:
async def aiohttp_head(url: str, timeout: int = None, headers: dict = None) -> Tuple[ClientResponse, int]:
"""Perform an async aiohttp HEAD request. Return the contents."""
headers = headers or DEFAULT_HTTP_HEADERS
async with aiohttp_session(timeout) as session:
async with session.head(url) as response:
async with session.head(url, headers=headers) as response:
return response, response.status

View file

@ -189,7 +189,7 @@ class TZDateTime(types.TypeDecorator):
return value
def months_selector_to_where(months: List[int], wheres: list, params: dict):
def months_selector_to_where(wheres: list, params: dict, months: List[int]):
"""Filter FileGroup results by the month that the file was published. Returns wheres/params unchanged if months
is empty.
@ -208,7 +208,7 @@ def months_selector_to_where(months: List[int], wheres: list, params: dict):
return wheres, params
def date_range_to_where(from_year: int, to_year: int, wheres: list, params: dict):
def date_range_to_where(wheres: list, params: dict, from_year: int, to_year: int):
"""Filter FileGroup results by the year that the file was published.
@param from_year: The earliest year the file was published

View file

@ -1096,6 +1096,10 @@ async def signal_download_download(download_id: int, download_url: str):
worker_logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e)
result = DownloadResult(success=False, error=str(traceback.format_exc()))
# If download has error, and not Internet is available, tell the user!
if result.error and not flags.have_internet.is_set():
result.error = f'{result.error}\n\nNo internet available!'
error_len = len(result.error) if result.error else 0
worker_logger.debug(
f'Got success={result.success} from {downloader} download_id={download.id} with {error_len=}')

View file

@ -101,7 +101,8 @@ async def refresh_progress(request: Request):
async def post_search_files(_: Request, body: schema.FilesSearchRequest):
with timer('Searching all files', 'info', logger__=logger):
file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model,
body.tag_names, body.headline, body.months, body.from_year, body.to_year)
body.tag_names, body.headline, body.months, body.from_year, body.to_year,
body.any_tag)
return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total)))

View file

@ -711,13 +711,16 @@ async def search_directories_by_name(name: str, excluded: List[str] = None, limi
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
tag_names: List[str] = None, headline: bool = False, months: List[int] = None,
from_year: int = None, to_year: int = None) -> \
from_year: int = None, to_year: int = None, any_tag: bool = False) -> \
Tuple[List[dict], int]:
"""Search the FileGroup table.
Order the returned Files by their rank if `search_str` is provided. Return all files if
`search_str` is empty.
@param any_tag: The file must have some tag.
@param to_year: The file must be published on or before this year.
@param from_year: The file must be published on or after this year.
@param search_str: Search the ts_vector of the file. Returns all files if this is empty.
@param limit: Return only this many files.
@param offset: Offset the query.
@ -739,17 +742,14 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
selects.append('ts_rank(textsearch, websearch_to_tsquery(%(search_str)s))')
order_by = '2 DESC'
mimetype_wheres, params = mimetypes_to_sql_wheres(mimetypes, params)
if mimetype_wheres:
wheres.append(mimetype_wheres)
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
if model:
params['model'] = model
wheres.append('model = %(model)s')
wheres, params = tag_append_sub_select_where(tag_names, wheres, params)
wheres, params = months_selector_to_where(months, wheres, params)
wheres, params = date_range_to_where(from_year, to_year, wheres, params)
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
wheres, params = months_selector_to_where(wheres, params, months)
wheres, params = date_range_to_where(wheres, params, from_year, to_year)
if search_str and headline:
headline = ''',
@ -1094,22 +1094,23 @@ def get_refresh_progress() -> RefreshProgress:
return progress
def mimetypes_to_sql_wheres(mimetypes: List[str], params: dict):
def mimetypes_to_sql_wheres(wheres: List[str], params: dict, mimetypes: List[str]) -> Tuple[List[str], dict]:
if mimetypes:
wheres = list()
local_wheres = list()
for idx, mimetype in enumerate(mimetypes):
key = f'mimetype{idx}'
if len(mimetype.split('/')) == 1:
params[key] = f'{mimetype}/%'
else:
params[key] = f'{mimetype}%'
wheres.append(f'mimetype LIKE %({key})s')
return f'({" OR ".join(wheres)})', params
return '', params
local_wheres.append(f'mimetype LIKE %({key})s')
wheres.append(f'({" OR ".join(local_wheres)})')
return wheres, params
async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str],
months: List[int] = None, from_year: int = None, to_year: int = None):
months: List[int] = None, from_year: int = None, to_year: int = None,
any_tag: bool = False):
"""
Return FileGroup count of what will be returned if the search is actually performed.
"""
@ -1123,13 +1124,10 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi
# Search by textsearch, and by matching url.
wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)')
wheres, params = tag_append_sub_select_where(tag_names, wheres, params)
mimetype_wheres, params = mimetypes_to_sql_wheres(mimetypes, params)
if mimetype_wheres:
wheres.append(mimetype_wheres)
wheres, params = months_selector_to_where(months, wheres, params)
wheres, params = date_range_to_where(from_year, to_year, wheres, params)
wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag)
wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes)
wheres, params = months_selector_to_where(wheres, params, months)
wheres, params = date_range_to_where(wheres, params, from_year, to_year)
joins = "\n".join(joins)
wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else ''
@ -1386,6 +1384,7 @@ async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_nam
session.add(file_group)
session.flush([file_group, ])
file_group.do_index()
file_group.indexed = True
for tag_name in (tag_names or []):
if tag_name not in file_group.tag_names:

View file

@ -278,6 +278,7 @@ class FileGroup(ModelHelper, Base):
return file_group
@staticmethod
def find_by_id(id_: int, session: Session = None) -> 'FileGroup':
fg = session.query(FileGroup).filter(FileGroup.id == id_).one_or_none()
if not fg:

View file

@ -1,6 +1,8 @@
from dataclasses import dataclass, field
from typing import List, Optional
from wrolpi.errors import ValidationError
@dataclass
class FilesRequest:
@ -29,6 +31,11 @@ class FilesSearchRequest:
months: Optional[List[int]] = None
from_year: Optional[int] = None
to_year: Optional[int] = None
any_tag: bool = False
def __post_init__(self):
if self.any_tag and self.tag_names:
raise ValidationError('Cannot use both tag_names and any_tag.')
@dataclass

View file

@ -185,6 +185,48 @@ def test_files_search(test_session, test_client, make_files_structure, assert_fi
assert_files_search('nothing', [])
@pytest.mark.asyncio
async def test_files_search_any_tag(test_async_client, test_session, make_files_structure, tag_factory):
one, two = tag_factory(), tag_factory()
files = [
'foo.txt',
'foo bar.txt',
'bar.txt',
]
make_files_structure(files)
await lib.refresh_files()
bar, foobar, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
assert bar.primary_path.name == 'bar.txt' \
and foo.primary_path.name == 'foo.txt' \
and foobar.primary_path.name == 'foo bar.txt'
foo.add_tag(one)
foobar.add_tag(two)
test_session.commit()
# Only `foo` is tagged with `one`
body = dict(search_str='foo', tag_names=['one'])
request, response = await test_async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo.txt'}
# Both `foo bar` and `foo` are tagged.
body = dict(search_str='foo', any_tag=True)
request, response = await test_async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo.txt', 'foo bar.txt'}
# `bar` is not tagged.
body = dict(search_str='bar', any_tag=True)
request, response = await test_async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo bar.txt'}
# Cannot search for any_tag, and tag names.
body = dict(search_str='foo', tag_names=['one'], any_tag=True)
request, response = await test_async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.BAD_REQUEST
def test_refresh_files_list(test_session, test_client, make_files_structure, test_directory, video_bytes):
"""The user can request to refresh specific files."""
make_files_structure({

View file

@ -77,7 +77,7 @@ async def test_extract_cover(test_session, test_directory, example_epub):
assert ebook.cover_path and ebook.cover_path.stat().st_size == 297099
def test_search(test_session, test_client, example_epub):
def test_search_ebooks(test_session, test_client, example_epub):
"""Ebooks are handled in File search results."""
request, response = test_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT

View file

@ -717,7 +717,7 @@ async def test_refresh_directories(test_session, test_directory, assert_director
assert_directories({'foo', 'baz'})
def test_file_group_merge(test_session, test_directory, make_files_structure, tag_factory, video_bytes, srt_file3):
def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory, video_bytes, srt_file3):
"""A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be
migrated."""
vid, srt = make_files_structure({
@ -736,6 +736,9 @@ def test_file_group_merge(test_session, test_directory, make_files_structure, ta
srt_file_created_at = srt_tag_file.created_at
test_session.commit()
assert vid_group.mimetype == 'video/mp4'
assert srt_group.mimetype == 'text/srt'
# Both FileGroups are merged.
vid = FileGroup.from_paths(test_session, vid, srt)
test_session.commit()
@ -749,6 +752,7 @@ def test_file_group_merge(test_session, test_directory, make_files_structure, ta
assert [i for i in vid.tag_files if i.tag.name == 'two'][0].created_at == srt_file_created_at
# Size is combined
assert vid.size > len(video_bytes)
assert vid.mimetype == 'video/mp4'
@pytest.mark.asyncio

View file

@ -583,6 +583,7 @@ async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimate
body.months,
body.from_year,
body.to_year,
body.any_tag,
)
ret = dict(

View file

@ -179,3 +179,8 @@ class SearchFileEstimateRequest:
months: Optional[List[int]] = None
from_year: Optional[int] = None
to_year: Optional[int] = None
any_tag: bool = False
def __post_init__(self):
if self.any_tag and self.tag_names:
raise ValidationError('Cannot search for any tag, and list of tags.')

View file

@ -12,7 +12,7 @@ from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_direc
register_refresh_cleanup, get_relative_to_media_directory
from wrolpi.dates import TZDateTime
from wrolpi.db import optional_session, get_db_curs
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, ValidationError
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
@ -442,12 +442,20 @@ def tag_names_to_file_group_sub_select(tag_names: List[str], params: dict) -> Tu
return sub_select, params
def tag_append_sub_select_where(tag_names: List[str], wheres: List[str], params: dict) -> Tuple[List[str], dict]:
"""Modify provided `wheres` and `params` to filter by `tag_names`, if any."""
if not tag_names:
def tag_append_sub_select_where(wheres: List[str], params: dict, tag_names: List[str], any_tag: bool = False) \
-> Tuple[List[str], dict]:
"""Modify provided `wheres` and `params` to filter by `tag_names`, if any. If no tag names are provided, but
`any_tag` is True, then files will be filtered to only those that have at least one tag."""
if any_tag and tag_names:
raise RuntimeError('Cannot search for any tag, and list of tags.')
if not tag_names and not any_tag:
return wheres, params
tags_sub_select, params = tag_names_to_file_group_sub_select(tag_names, params)
if any_tag:
tags_sub_select = 'SELECT file_group_id FROM tag_file'
else:
tags_sub_select, params = tag_names_to_file_group_sub_select(tag_names, params)
wheres.append(f'fg.id = ANY({tags_sub_select})')
return wheres, params

View file

@ -560,7 +560,11 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
@pytest.mark.asyncio
async def test_search_file_estimates(test_session, test_async_client, archive_factory, tag_factory, video_factory):
# WARNING results are cached, this test uses unique queries to avoid conflicts.
"""Can quickly get count of possible files when searching."""
body_ = dict(search_str='can search with no files')
request_, response_ = await test_async_client.post('/api/search_file_estimates', json=body_)
assert response_.status_code == HTTPStatus.OK
tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory()
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
@ -619,6 +623,32 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa
await assert_results(dict(from_year=2000, to_year=2000), 2)
@pytest.mark.asyncio
async def test_search_file_estimates_any_tag(test_session, test_async_client, archive_factory, tag_factory,
video_factory):
"""File estimates can be filtered by any_tag."""
tag1 = tag_factory()
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
archive_factory(domain='bar.com', contents='contents of bar bunny')
test_session.commit()
async def assert_results(body: dict, expected_file_groups=None):
expected_file_groups = expected_file_groups or 0
request, response = await test_async_client.post('/api/search_file_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == expected_file_groups or 0
await assert_results(dict(search_str='bunny'), 2)
await assert_results(dict(search_str='bunny', any_tag=True), 1)
await assert_results(dict(search_str='bar'), 1)
await assert_results(dict(search_str='bar', any_tag=True), 0)
body_ = dict(search_str='bunny', tag_names=[tag1.name, ], any_tag=True)
request_, response_ = await test_async_client.post('/api/search_file_estimates', json=body_)
assert response_.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_external_error(test_async_client):
"""A non-SanicException error is handled as an Internal Server Error."""