diff --git a/app/src/DashboardPage.js b/app/src/DashboardPage.js index 6857e6c1..33c327f9 100644 --- a/app/src/DashboardPage.js +++ b/app/src/DashboardPage.js @@ -161,7 +161,7 @@ export function DashboardPage() { const navigate = useNavigate(); // The search the user submitted. - const {searchStr, setSearchStr, activeTags, isEmpty} = useSearch(); + const {searchStr, setSearchStr, activeTags, isEmpty, anyTag} = useSearch(); // The search that the user is typing. const [localSearchStr, setLocalSearchStr] = React.useState(searchStr); const { @@ -174,7 +174,7 @@ export function DashboardPage() { setSearchStr: setSuggestionSearchStr, setSearchTags, months, dateRange, setDates, clearDate, - } = useSearchSuggestions(searchStr, activeTags); + } = useSearchSuggestions(searchStr, activeTags, anyTag); React.useEffect(() => { setSuggestionSearchStr(localSearchStr); diff --git a/app/src/Tags.js b/app/src/Tags.js index 9a6eaeea..582a1bc0 100644 --- a/app/src/Tags.js +++ b/app/src/Tags.js @@ -310,11 +310,12 @@ function EditTagsModal() { export function AddTagsButton({ hideEdit, - active, + showAny = false, selectedTagNames = [], - onToggle = _.noop, + anyTag = false, onAdd = _.noop, onRemove = _.noop, + onChange = _.noop, // Expects to send: (tagNames, anyTag) limit = null, }) { // A button which displays a modal in which the user can add or remove tags. @@ -324,6 +325,8 @@ export function AddTagsButton({ const [loading, setLoading] = React.useState(false); const [localTags, setLocalTags] = React.useState(selectedTagNames); + const active = anyTag || (selectedTagNames && selectedTagNames.length > 0); + const handleOpen = (e) => { if (e) { e.preventDefault(); @@ -339,8 +342,8 @@ export function AddTagsButton({ return; } setLocalTags(newTags); - onToggle(newTags); onAdd(name); + onChange(newTags, null); } finally { setLoading(false); } @@ -350,30 +353,30 @@ export function AddTagsButton({ setLoading(true); try { const newTags = localTags.filter(i => i !== name); - setLocalTags(newTags); - onToggle(newTags); + setLocalTags(newTags) onRemove(name); + onChange(newTags, null); } finally { setLoading(false); } } const clearLocalTags = () => { - if (!localTags || (localTags && localTags.length === 0)) { + if (!anyTag && (!localTags || (localTags && localTags.length === 0))) { console.debug('No tags to clear'); return } - setLoading(true); - try { - for (let i = 0; i < localTags.length; i++) { - onRemove(localTags[i]); - } - setLocalTags([]); - onToggle([]); - } finally { - setLoading(false); - } + console.debug('Clearing selected tags'); + onChange([], null); + setLocalTags([]); + setOpen(false); + } + + const localOnAnyTag = () => { + console.debug('Setting any tag'); + onChange([], true); + setOpen(false); } const selectedTagsGroup = ; @@ -382,7 +385,11 @@ export function AddTagsButton({ const emptySelectedTags = limit === 1 ? 'Add only one tag below' : 'Add one or more tags below'; return <> - + {showAny && } @@ -418,12 +426,13 @@ export const taggedImageLabel = {corner: 'left', icon: 'tag', color: 'green'}; export const TagsSelector = ({ hideEdit = false, - active, + showAny = false, hideGroup = false, selectedTagNames = [], - onToggle = _.noop, + anyTag = false, onAdd = _.noop, onRemove = _.noop, + onChange = _.noop, limit = null, }) => { // Provides a button to add tags to a list. Displays the tags of that list. @@ -436,11 +445,12 @@ export const TagsSelector = ({ const button = ; diff --git a/app/src/api.js b/app/src/api.js index e879468a..9a9d3e2b 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -627,8 +627,8 @@ export async function deleteDownload(downloadId) { } } -export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear) { - const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)}; +export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag) { + const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit), any_tag: anyTag}; if (mimetypes) { body['mimetypes'] = mimetypes; } @@ -1183,10 +1183,10 @@ export async function searchSuggestions(search_str) { } } -export async function searchEstimateFiles(search_str, tagNames, mimetypes, months, dateRange) { +export async function searchEstimateFiles(search_str, tagNames, mimetypes, months, dateRange, anyTag) { months = months ? months.map(i => parseInt(i)) : []; - const body = {search_str, tag_names: tagNames, mimetypes, months}; + const body = {search_str, tag_names: tagNames, mimetypes, months, any_tag: anyTag}; if (dateRange) { body['from_year'] = dateRange[0]; body['to_year'] = dateRange[1]; @@ -1198,6 +1198,8 @@ export async function searchEstimateFiles(search_str, tagNames, mimetypes, month return { fileGroups: content.file_groups, } + } else { + console.error('Failed to get file search estimates!'); } } diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index 93c0a3fb..97ca7d54 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -263,7 +263,11 @@ function ArchivePage() { - + @@ -497,14 +501,16 @@ function ArchivesPage() { ; const {body, paginator, selectButton, viewButton, limitDropdown, tagQuerySelector} = FilesView( - archives, - activePage, - totalPages, - selectElm, - selectedArchives, - onSelect, - setPage, - !!searchStr, + { + files: archives, + activePage: activePage, + totalPages: totalPages, + selectElem: selectElm, + selectedKeys: selectedArchives, + onSelect: onSelect, + setPage: setPage, + headlines: !!searchStr + }, ); diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index 3cc046e9..93fa4516 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -488,14 +488,6 @@ export function ChannelDownloadForm({channel_id, afterSave, closeModal, download } }; - const handleAddTag = (tagName) => { - setTagNames([...tagNames, tagName]); - } - - const handleUntag = (tagName) => { - setTagNames(tagNames.filter(i => i !== tagName)); - } - const deleteDownloadButton = - + setTagNames(i)}/> diff --git a/app/src/components/Download.js b/app/src/components/Download.js index fa8b82e4..73197413 100644 --- a/app/src/components/Download.js +++ b/app/src/components/Download.js @@ -77,7 +77,7 @@ class Downloader extends React.Component { } } - handleSelectedTags = (tagNames) => { + handleTagsChange = (tagNames, newAnyTag) => { this.setState({tagNames}); } @@ -119,7 +119,7 @@ class Downloader extends React.Component { let tagsSelector; if (withTags) { - tagsSelector = ; + tagsSelector = ; } const advancedAccordion = diff --git a/app/src/components/Files.js b/app/src/components/Files.js index c04422a7..39a4c126 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -32,7 +32,6 @@ import { useTitle } from "./Common"; import { - useAllQuery, useFilesProgressInterval, usePages, useSearchFiles, @@ -66,6 +65,7 @@ import {refreshFiles} from "../api"; import {useSubscribeEventName} from "../Events"; import {TagsSelector} from "../Tags"; import {Headlines} from "./Headline"; +import {useSearch} from "./Search"; function EbookCard({file}) { const {s} = useContext(ThemeContext); @@ -311,15 +311,18 @@ export function SearchLimitDropdown({limits = []}) { } export function FilesView( - files, - activePage, - totalPages, - selectElem, - selectedKeys, - onSelect, - setPage, - headlines = false, - limitOptions = [12, 24, 48, 96], + { + files, + activePage, + totalPages, + selectElem, + selectedKeys, + onSelect, + setPage, + headlines = false, + limitOptions = [12, 24, 48, 96], + showAnyTag = false, + }, ) { const {view} = useSearchView(); @@ -360,7 +363,7 @@ export function FilesView( const viewButton = const limitDropdown = ; - const tagQuerySelector = ; + const tagQuerySelector = ; return { body, @@ -432,22 +435,28 @@ export function SearchFilter({filters = [], modalHeader, size = 'medium'}) { return <> } -export function TagsQuerySelector({onChange}) { - // Creates modal that the User can use to manipulate the tag query. - const [activeTags, setActiveTags] = useAllQuery('tag'); +export function TagsQuerySelector({onChange, showAny = true}) { + // Creates modal that the User can use to manipulate the tag query params. + const {activeTags, anyTag, setTags, setAnyTag} = useSearch(); - const localOnChange = (tagNames) => { - setActiveTags(tagNames); + const localOnChange = (tagNames, newAnyTag) => { + if (newAnyTag) { + setAnyTag(true); + } else { + setTags(tagNames); + } if (onChange) { - onChange(tagNames); + onChange(tagNames, newAnyTag); } } return 0} + showAny={showAny} + active={anyTag || activeTags && activeTags.length > 0} selectedTagNames={activeTags} - onToggle={localOnChange} + anyTag={anyTag} + onChange={localOnChange} style={{marginLeft: '0.3em', marginTop: '0.3em'}} /> } @@ -479,14 +488,17 @@ export function FilesSearchView({ const {searchFiles, pages} = useSearchFiles(24, emptySearch, model); const {body, paginator, selectButton, viewButton, limitDropdown, tagQuerySelector} = FilesView( - searchFiles, - pages.activePage, - pages.totalPages, - null, - null, - null, - pages.setPage, - true, + { + files: searchFiles, + activePage: pages.activePage, + totalPages: pages.totalPages, + selectElem: null, + selectedKeys: null, + onSelect: null, + setPage: pages.setPage, + headlines: true, + showAnyTag: true, + }, ); return <> diff --git a/app/src/components/Search.js b/app/src/components/Search.js index b0e0dd35..83e05942 100644 --- a/app/src/components/Search.js +++ b/app/src/components/Search.js @@ -82,7 +82,7 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false } const setTags = (tags) => { - updateQuery({tag: tags}); + updateQuery({tag: tags, anyTag: null}); } const addTag = (name) => { @@ -95,24 +95,26 @@ export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false setTags(newTags); } + const anyTag = searchParams.get('anyTag') === 'true'; + const setAnyTag = (value) => { + updateQuery({tag: [], anyTag: value ? 'true' : null}); + } + return { - activeTags, - addTag, + activeTags, addTag, removeTag, anyTag, setAnyTag, setTags, filter, model: model_, pages, - removeTag, searchParams, searchStr, setSearchStr, clearSearch, - setTags, months, dateRange, setDateRange, setDates, anySearch, isEmpty, } } -export function useSuggestions(searchStr, tagNames, filter) { +export function useSuggestions(searchStr, tagNames, filter, anyTag) { const defaultSuggestions = { fileGroups: [], channels: [], @@ -135,7 +137,7 @@ export function useSuggestions(searchStr, tagNames, filter) { // We can't search Channels/Domains without some string to filter by. sendGeneralReqeust(async () => await searchSuggestions(searchStr)); } - sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange)); + sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag)); sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames)); } }, [ @@ -146,6 +148,7 @@ export function useSuggestions(searchStr, tagNames, filter) { JSON.stringify(months), JSON.stringify(dateRange), filter, + anyTag, ]); React.useEffect(() => { @@ -180,14 +183,14 @@ export function useSuggestions(searchStr, tagNames, filter) { } -export function useSearchSuggestions(defaultSearchStr, defaultTagNames) { +export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag) { const navigate = useNavigate(); const {filter} = useSearchFilter(); const [searchStr, setSearchStr] = React.useState(defaultSearchStr || ''); const [searchTags, setSearchTags] = React.useState(defaultTagNames); const {SingleTag, fuzzyMatchTagsByName} = React.useContext(TagsContext); const {dateRange, months, setDates, clearDate} = useSearchDate(); - const {suggestions, loading} = useSuggestions(searchStr, searchTags, filter); + const {suggestions, loading} = useSuggestions(searchStr, searchTags, filter, anyTag); const {getLocationStr} = React.useContext(QueryContext); // The results that will be displayed by . @@ -239,6 +242,8 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames) { return {type: 'channel', title: i['name'], id: i['id'], location: `/videos/channel/${i.id}/video`} }) } + } else if (newSuggestions.channels && newSuggestions.channels.length === 0) { + results.channels = {name: 'Channels', results: noResults}; } if (newSuggestions.domains && newSuggestions.domains.length > 0) { results.domains = { @@ -339,7 +344,7 @@ export function SearchView({suggestions, suggestionsSums, loading}) { const zimsTabName = Zims ; const links = [ - {text: filesTabName, to: '/search', key: 'filesSearch', end: true}, + {text: filesTabName, to: '/search', key: 'filesSearch_', end: true}, {text: zimsTabName, to: '/search/zim', key: 'zimsSearch'}, ]; diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 8b74a14c..14e1955f 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -31,6 +31,7 @@ import { CardDescription, CardHeader, Container, + Dimmer, Image, PlaceholderHeader, PlaceholderLine, @@ -74,7 +75,7 @@ function VideosPage() { searchOrder = defaultSearchOrder; } - const {searchStr, setSearchStr, videos, activePage, setPage, totalPages, fetchVideos} = + const {searchStr, setSearchStr, videos, activePage, setPage, totalPages, fetchVideos, loading} = useSearchVideos(null, channelId, searchOrder); const {channel} = useChannel(channelId); @@ -165,14 +166,17 @@ function VideosPage() { ; const {body, paginator, selectButton, viewButton, limitDropdown, tagQuerySelector} = FilesView( - videos, - activePage, - totalPages, - selectElm, - selectedVideos, - onSelect, - setPage, - !!searchStr, + { + files: videos, + activePage: activePage, + totalPages: totalPages, + selectElem: selectElm, + selectedKeys: selectedVideos, + onSelect: onSelect, + setPage: setPage, + headlines: !!searchStr, + loading: loading, + }, ); const [localSearchStr, setLocalSearchStr] = React.useState(searchStr || ''); diff --git a/app/src/components/Zim.js b/app/src/components/Zim.js index d8f1e01f..c89f5252 100644 --- a/app/src/components/Zim.js +++ b/app/src/components/Zim.js @@ -320,7 +320,7 @@ export const ZimSearchView = ({suggestions, loading}) => { } return <> - + setTags(i)}/> {body} @@ -533,7 +533,7 @@ class ManageZim extends React.Component { } } -function ZimViewer({src = ZIM_VIEWER_URI, style=null}) { +function ZimViewer({src = ZIM_VIEWER_URI, style = null}) { const fallback =
Failed to fetch Zim service.

You may need to give permission to access the page: {src}

diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index c306695c..fa3a52be 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -324,20 +324,25 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => { const activeTags = searchParams.getAll('tag'); const {view} = useSearchView(); const headline = view === 'headline'; + const anyTag = searchParams.get('anyTag') === 'true'; const [videos, setVideos] = useState(null); const [totalPages, setTotalPages] = useState(0); + const [loading, setLoading] = useState(false); const localSearchVideos = async () => { setVideos(null); + setLoading(true); setTotalPages(0); try { - let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline); + let [videos_, total] = await searchVideos(offset, limit, channelId, searchStr, order, activeTags, headline, anyTag); setVideos(videos_); setTotalPages(calculateTotalPages(total, limit)); } catch (e) { console.error(e); setVideos(undefined);// Could not get Videos, display error. + } finally { + setLoading(false); } } @@ -368,6 +373,7 @@ export const useSearchVideos = (defaultLimit, channelId, order_by) => { setOrderBy, fetchVideos: localSearchVideos, activeTags, + loading, } } @@ -481,7 +487,7 @@ export const useChannels = () => { export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => { const { - activeTags, + activeTags, anyTag, pages, searchStr, filter, @@ -493,6 +499,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => const {view} = useSearchView(); const [searchFiles, setSearchFiles] = useState(null); + const [loading, setLoading] = useState(false); const headline = view === 'headline'; const localSearchFiles = async () => { @@ -507,11 +514,12 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => fromDate = dateRange[0]; toDate = dateRange[1]; } + setLoading(true); console.log('localSearchFiles', fromDate, toDate, months); try { let [file_groups, total] = await filesSearch( pages.offset, pages.limit, searchStr, mimetypes, model || model_, activeTags, headline, - months, fromDate, toDate); + months, fromDate, toDate, anyTag); setSearchFiles(file_groups); pages.setTotal(total); } catch (e) { @@ -523,6 +531,8 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => description: 'Could not get files', time: 5000, }); + } finally { + setLoading(false); } } @@ -533,6 +543,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => useEffect(() => { if (searchStr || (activeTags && activeTags.length > 0)) { + setLoading(true); debouncedLocalSearchFiles(); } // Handle when this is unmounted. @@ -547,6 +558,7 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => headline, JSON.stringify(months), JSON.stringify(dateRange), + anyTag, ]); return { @@ -555,7 +567,8 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => filter, setSearchStr, pages, - activeTags + activeTags, + loading, }; } @@ -1106,15 +1119,7 @@ export const useUploadFile = () => { doUpload() }, [JSON.stringify(files)]); - const handleAddTag = (tagName) => { - setTagNames([...tagNames, tagName]); - } - - const handleUntag = (tagName) => { - setTagNames(tagNames.filter(i => i !== tagName)); - } - - const tagsSelector = ; + const tagsSelector = setTagNames(i)}/>; return { destination, diff --git a/debian-live-config/config/package-lists/wrolpi.list.chroot b/debian-live-config/config/package-lists/wrolpi.list.chroot index dac4597c..94cc05db 100644 --- a/debian-live-config/config/package-lists/wrolpi.list.chroot +++ b/debian-live-config/config/package-lists/wrolpi.list.chroot @@ -4,6 +4,7 @@ apache2-doc apt-transport-https aria2 autoconf +bind9-dnsutils build-essential bzip2 ca-certificates diff --git a/help.sh b/help.sh index 9ebf0c46..c0e582f0 100755 --- a/help.sh +++ b/help.sh @@ -341,4 +341,18 @@ if curl -k -s https://0.0.0.0:8086/ | grep MkDocs 2>/dev/null >/dev/null; then echo "OK: Help service is running" else echo "FAILED: Help service is not running" -fi \ No newline at end of file +fi + +echo +# Internet +if ping -c1 1.1.1.1 -W 5 &>/dev/null; then + echo "OK: Can ping 1.1.1.1" +else + echo "FAILED: Cannot ping 1.1.1.1. Is internet working?" +fi + +if ping -c1 wrolpi.org -W 5 &>/dev/null; then + echo "OK: Can ping wrolpi.org" +else + echo "FAILED: Cannot ping wrolpi.org Is internet working?" +fi diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 71b084ea..3131072d 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -482,7 +482,7 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order if order in NO_NULL_ORDERS: wheres.append(NO_NULL_ORDERS[order]) - wheres, params = tag_append_sub_select_where(tag_names, wheres, params) + wheres, params = tag_append_sub_select_where(wheres, params, tag_names) if search_str and headline: headline = ''', diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index cea389e4..238778d5 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -101,7 +101,7 @@ def search_videos( # No search_str provided. Get id and total only. select_columns = 'fg.id, COUNT(*) OVER() AS total' - wheres, params = tag_append_sub_select_where(tag_names, wheres, params) + wheres, params = tag_append_sub_select_where(wheres, params, tag_names) if search_str and headline: headline = ''', diff --git a/modules/zim/schema.py b/modules/zim/schema.py index a6786450..046a382b 100644 --- a/modules/zim/schema.py +++ b/modules/zim/schema.py @@ -99,3 +99,4 @@ class OutdatedZims: class SearchEstimateRequest: search_str: Optional[str] = None tag_names: List[str] = field(default_factory=lambda: list()) + any_tag: bool = False diff --git a/pi-gen/stage2/04-wrolpi/01-packages b/pi-gen/stage2/04-wrolpi/01-packages index f5268dfe..ce36a6f9 100644 --- a/pi-gen/stage2/04-wrolpi/01-packages +++ b/pi-gen/stage2/04-wrolpi/01-packages @@ -4,6 +4,7 @@ apache2-doc apt-transport-https aria2 autoconf +bind9-dnsutils build-essential bzip2 ca-certificates @@ -83,9 +84,9 @@ python3-full python3-mapnik python3-pip python3-venv -smartmontools renderd rsync +smartmontools software-properties-common tar tcpdump diff --git a/wrolpi/common.py b/wrolpi/common.py index 5d94f610..d0795944 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -1067,15 +1067,17 @@ async def aiohttp_post(url: str, json_, timeout: int = None) -> Tuple[Dict, int] async def aiohttp_get(url: str, timeout: int = None, headers: dict = None) -> Tuple[bytes, int]: """Perform an async aiohttp GET request. Return the contents.""" + headers = headers or DEFAULT_HTTP_HEADERS async with aiohttp_session(timeout) as session: async with session.get(url, headers=headers) as response: return await response.content.read(), response.status -async def aiohttp_head(url: str, timeout: int = None) -> Tuple[ClientResponse, int]: +async def aiohttp_head(url: str, timeout: int = None, headers: dict = None) -> Tuple[ClientResponse, int]: """Perform an async aiohttp HEAD request. Return the contents.""" + headers = headers or DEFAULT_HTTP_HEADERS async with aiohttp_session(timeout) as session: - async with session.head(url) as response: + async with session.head(url, headers=headers) as response: return response, response.status diff --git a/wrolpi/dates.py b/wrolpi/dates.py index 8f656dcd..ea71d3f2 100644 --- a/wrolpi/dates.py +++ b/wrolpi/dates.py @@ -189,7 +189,7 @@ class TZDateTime(types.TypeDecorator): return value -def months_selector_to_where(months: List[int], wheres: list, params: dict): +def months_selector_to_where(wheres: list, params: dict, months: List[int]): """Filter FileGroup results by the month that the file was published. Returns wheres/params unchanged if months is empty. @@ -208,7 +208,7 @@ def months_selector_to_where(months: List[int], wheres: list, params: dict): return wheres, params -def date_range_to_where(from_year: int, to_year: int, wheres: list, params: dict): +def date_range_to_where(wheres: list, params: dict, from_year: int, to_year: int): """Filter FileGroup results by the year that the file was published. @param from_year: The earliest year the file was published diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index ef20fd5c..11acfbfa 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1096,6 +1096,10 @@ async def signal_download_download(download_id: int, download_url: str): worker_logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e) result = DownloadResult(success=False, error=str(traceback.format_exc())) + # If download has error, and not Internet is available, tell the user! + if result.error and not flags.have_internet.is_set(): + result.error = f'{result.error}\n\nNo internet available!' + error_len = len(result.error) if result.error else 0 worker_logger.debug( f'Got success={result.success} from {downloader} download_id={download.id} with {error_len=}') diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index 841f81e2..d28841ce 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -101,7 +101,8 @@ async def refresh_progress(request: Request): async def post_search_files(_: Request, body: schema.FilesSearchRequest): with timer('Searching all files', 'info', logger__=logger): file_groups, total = lib.search_files(body.search_str, body.limit, body.offset, body.mimetypes, body.model, - body.tag_names, body.headline, body.months, body.from_year, body.to_year) + body.tag_names, body.headline, body.months, body.from_year, body.to_year, + body.any_tag) return json_response(dict(file_groups=file_groups, totals=dict(file_groups=total))) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 5d56bc56..4439d33e 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -711,13 +711,16 @@ async def search_directories_by_name(name: str, excluded: List[str] = None, limi def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None, tag_names: List[str] = None, headline: bool = False, months: List[int] = None, - from_year: int = None, to_year: int = None) -> \ + from_year: int = None, to_year: int = None, any_tag: bool = False) -> \ Tuple[List[dict], int]: """Search the FileGroup table. Order the returned Files by their rank if `search_str` is provided. Return all files if `search_str` is empty. + @param any_tag: The file must have some tag. + @param to_year: The file must be published on or before this year. + @param from_year: The file must be published on or after this year. @param search_str: Search the ts_vector of the file. Returns all files if this is empty. @param limit: Return only this many files. @param offset: Offset the query. @@ -739,17 +742,14 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] selects.append('ts_rank(textsearch, websearch_to_tsquery(%(search_str)s))') order_by = '2 DESC' - mimetype_wheres, params = mimetypes_to_sql_wheres(mimetypes, params) - if mimetype_wheres: - wheres.append(mimetype_wheres) - + wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes) if model: params['model'] = model wheres.append('model = %(model)s') - wheres, params = tag_append_sub_select_where(tag_names, wheres, params) - wheres, params = months_selector_to_where(months, wheres, params) - wheres, params = date_range_to_where(from_year, to_year, wheres, params) + wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag) + wheres, params = months_selector_to_where(wheres, params, months) + wheres, params = date_range_to_where(wheres, params, from_year, to_year) if search_str and headline: headline = ''', @@ -1094,22 +1094,23 @@ def get_refresh_progress() -> RefreshProgress: return progress -def mimetypes_to_sql_wheres(mimetypes: List[str], params: dict): +def mimetypes_to_sql_wheres(wheres: List[str], params: dict, mimetypes: List[str]) -> Tuple[List[str], dict]: if mimetypes: - wheres = list() + local_wheres = list() for idx, mimetype in enumerate(mimetypes): key = f'mimetype{idx}' if len(mimetype.split('/')) == 1: params[key] = f'{mimetype}/%' else: params[key] = f'{mimetype}%' - wheres.append(f'mimetype LIKE %({key})s') - return f'({" OR ".join(wheres)})', params - return '', params + local_wheres.append(f'mimetype LIKE %({key})s') + wheres.append(f'({" OR ".join(local_wheres)})') + return wheres, params async def search_file_suggestion_count(search_str: str, tag_names: List[str], mimetypes: List[str], - months: List[int] = None, from_year: int = None, to_year: int = None): + months: List[int] = None, from_year: int = None, to_year: int = None, + any_tag: bool = False): """ Return FileGroup count of what will be returned if the search is actually performed. """ @@ -1123,13 +1124,10 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi # Search by textsearch, and by matching url. wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)') - wheres, params = tag_append_sub_select_where(tag_names, wheres, params) - - mimetype_wheres, params = mimetypes_to_sql_wheres(mimetypes, params) - if mimetype_wheres: - wheres.append(mimetype_wheres) - wheres, params = months_selector_to_where(months, wheres, params) - wheres, params = date_range_to_where(from_year, to_year, wheres, params) + wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag) + wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes) + wheres, params = months_selector_to_where(wheres, params, months) + wheres, params = date_range_to_where(wheres, params, from_year, to_year) joins = "\n".join(joins) wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else '' @@ -1386,6 +1384,7 @@ async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_nam session.add(file_group) session.flush([file_group, ]) file_group.do_index() + file_group.indexed = True for tag_name in (tag_names or []): if tag_name not in file_group.tag_names: diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index cc88200b..b2cc9188 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -278,6 +278,7 @@ class FileGroup(ModelHelper, Base): return file_group + @staticmethod def find_by_id(id_: int, session: Session = None) -> 'FileGroup': fg = session.query(FileGroup).filter(FileGroup.id == id_).one_or_none() if not fg: diff --git a/wrolpi/files/schema.py b/wrolpi/files/schema.py index 41ce6773..466232f0 100644 --- a/wrolpi/files/schema.py +++ b/wrolpi/files/schema.py @@ -1,6 +1,8 @@ from dataclasses import dataclass, field from typing import List, Optional +from wrolpi.errors import ValidationError + @dataclass class FilesRequest: @@ -29,6 +31,11 @@ class FilesSearchRequest: months: Optional[List[int]] = None from_year: Optional[int] = None to_year: Optional[int] = None + any_tag: bool = False + + def __post_init__(self): + if self.any_tag and self.tag_names: + raise ValidationError('Cannot use both tag_names and any_tag.') @dataclass diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index a77c26e6..b7949b1a 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -185,6 +185,48 @@ def test_files_search(test_session, test_client, make_files_structure, assert_fi assert_files_search('nothing', []) +@pytest.mark.asyncio +async def test_files_search_any_tag(test_async_client, test_session, make_files_structure, tag_factory): + one, two = tag_factory(), tag_factory() + files = [ + 'foo.txt', + 'foo bar.txt', + 'bar.txt', + ] + make_files_structure(files) + await lib.refresh_files() + bar, foobar, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all() + assert bar.primary_path.name == 'bar.txt' \ + and foo.primary_path.name == 'foo.txt' \ + and foobar.primary_path.name == 'foo bar.txt' + foo.add_tag(one) + foobar.add_tag(two) + test_session.commit() + + # Only `foo` is tagged with `one` + body = dict(search_str='foo', tag_names=['one']) + request, response = await test_async_client.post('/api/files/search', json=body) + assert response.status_code == HTTPStatus.OK + assert {i['primary_path'] for i in response.json['file_groups']} == {'foo.txt'} + + # Both `foo bar` and `foo` are tagged. + body = dict(search_str='foo', any_tag=True) + request, response = await test_async_client.post('/api/files/search', json=body) + assert response.status_code == HTTPStatus.OK + assert {i['primary_path'] for i in response.json['file_groups']} == {'foo.txt', 'foo bar.txt'} + + # `bar` is not tagged. + body = dict(search_str='bar', any_tag=True) + request, response = await test_async_client.post('/api/files/search', json=body) + assert response.status_code == HTTPStatus.OK + assert {i['primary_path'] for i in response.json['file_groups']} == {'foo bar.txt'} + + # Cannot search for any_tag, and tag names. + body = dict(search_str='foo', tag_names=['one'], any_tag=True) + request, response = await test_async_client.post('/api/files/search', json=body) + assert response.status_code == HTTPStatus.BAD_REQUEST + + def test_refresh_files_list(test_session, test_client, make_files_structure, test_directory, video_bytes): """The user can request to refresh specific files.""" make_files_structure({ diff --git a/wrolpi/files/test/test_ebooks.py b/wrolpi/files/test/test_ebooks.py index 5933e141..b8e1058b 100644 --- a/wrolpi/files/test/test_ebooks.py +++ b/wrolpi/files/test/test_ebooks.py @@ -77,7 +77,7 @@ async def test_extract_cover(test_session, test_directory, example_epub): assert ebook.cover_path and ebook.cover_path.stat().st_size == 297099 -def test_search(test_session, test_client, example_epub): +def test_search_ebooks(test_session, test_client, example_epub): """Ebooks are handled in File search results.""" request, response = test_client.post('/api/files/refresh') assert response.status_code == HTTPStatus.NO_CONTENT diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 094f00ae..3cb38dbd 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -717,7 +717,7 @@ async def test_refresh_directories(test_session, test_directory, assert_director assert_directories({'foo', 'baz'}) -def test_file_group_merge(test_session, test_directory, make_files_structure, tag_factory, video_bytes, srt_file3): +def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory, video_bytes, srt_file3): """A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be migrated.""" vid, srt = make_files_structure({ @@ -736,6 +736,9 @@ def test_file_group_merge(test_session, test_directory, make_files_structure, ta srt_file_created_at = srt_tag_file.created_at test_session.commit() + assert vid_group.mimetype == 'video/mp4' + assert srt_group.mimetype == 'text/srt' + # Both FileGroups are merged. vid = FileGroup.from_paths(test_session, vid, srt) test_session.commit() @@ -749,6 +752,7 @@ def test_file_group_merge(test_session, test_directory, make_files_structure, ta assert [i for i in vid.tag_files if i.tag.name == 'two'][0].created_at == srt_file_created_at # Size is combined assert vid.size > len(video_bytes) + assert vid.mimetype == 'video/mp4' @pytest.mark.asyncio diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index bfed33e4..65da626e 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -583,6 +583,7 @@ async def post_search_file_estimates(_: Request, body: schema.SearchFileEstimate body.months, body.from_year, body.to_year, + body.any_tag, ) ret = dict( diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 968a900f..923b173a 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -179,3 +179,8 @@ class SearchFileEstimateRequest: months: Optional[List[int]] = None from_year: Optional[int] = None to_year: Optional[int] = None + any_tag: bool = False + + def __post_init__(self): + if self.any_tag and self.tag_names: + raise ValidationError('Cannot search for any tag, and list of tags.') diff --git a/wrolpi/tags.py b/wrolpi/tags.py index cc780e9f..832eccaf 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -12,7 +12,7 @@ from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_direc register_refresh_cleanup, get_relative_to_media_directory from wrolpi.dates import TZDateTime from wrolpi.db import optional_session, get_db_curs -from wrolpi.errors import UnknownTag, UsedTag, InvalidTag +from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, ValidationError from wrolpi.vars import PYTEST logger = logger.getChild(__name__) @@ -442,12 +442,20 @@ def tag_names_to_file_group_sub_select(tag_names: List[str], params: dict) -> Tu return sub_select, params -def tag_append_sub_select_where(tag_names: List[str], wheres: List[str], params: dict) -> Tuple[List[str], dict]: - """Modify provided `wheres` and `params` to filter by `tag_names`, if any.""" - if not tag_names: +def tag_append_sub_select_where(wheres: List[str], params: dict, tag_names: List[str], any_tag: bool = False) \ + -> Tuple[List[str], dict]: + """Modify provided `wheres` and `params` to filter by `tag_names`, if any. If no tag names are provided, but + `any_tag` is True, then files will be filtered to only those that have at least one tag.""" + if any_tag and tag_names: + raise RuntimeError('Cannot search for any tag, and list of tags.') + + if not tag_names and not any_tag: return wheres, params - tags_sub_select, params = tag_names_to_file_group_sub_select(tag_names, params) + if any_tag: + tags_sub_select = 'SELECT file_group_id FROM tag_file' + else: + tags_sub_select, params = tag_names_to_file_group_sub_select(tag_names, params) wheres.append(f'fg.id = ANY({tags_sub_select})') return wheres, params diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 889109fa..85eaba52 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -560,7 +560,11 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto @pytest.mark.asyncio async def test_search_file_estimates(test_session, test_async_client, archive_factory, tag_factory, video_factory): - # WARNING results are cached, this test uses unique queries to avoid conflicts. + """Can quickly get count of possible files when searching.""" + body_ = dict(search_str='can search with no files') + request_, response_ = await test_async_client.post('/api/search_file_estimates', json=body_) + assert response_.status_code == HTTPStatus.OK + tag1, tag2, tag3 = tag_factory(), tag_factory(), tag_factory() archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ]) archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ]) @@ -619,6 +623,32 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa await assert_results(dict(from_year=2000, to_year=2000), 2) +@pytest.mark.asyncio +async def test_search_file_estimates_any_tag(test_session, test_async_client, archive_factory, tag_factory, + video_factory): + """File estimates can be filtered by any_tag.""" + tag1 = tag_factory() + archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ]) + archive_factory(domain='bar.com', contents='contents of bar bunny') + test_session.commit() + + async def assert_results(body: dict, expected_file_groups=None): + expected_file_groups = expected_file_groups or 0 + + request, response = await test_async_client.post('/api/search_file_estimates', json=body) + assert response.status_code == HTTPStatus.OK + assert response.json['file_groups'] == expected_file_groups or 0 + + await assert_results(dict(search_str='bunny'), 2) + await assert_results(dict(search_str='bunny', any_tag=True), 1) + await assert_results(dict(search_str='bar'), 1) + await assert_results(dict(search_str='bar', any_tag=True), 0) + + body_ = dict(search_str='bunny', tag_names=[tag1.name, ], any_tag=True) + request_, response_ = await test_async_client.post('/api/search_file_estimates', json=body_) + assert response_.status_code == HTTPStatus.BAD_REQUEST + + @pytest.mark.asyncio async def test_external_error(test_async_client): """A non-SanicException error is handled as an Internal Server Error."""