diff --git a/alembic/versions/f9928a3b6bc5_file_directories.py b/alembic/versions/f9928a3b6bc5_file_directories.py new file mode 100644 index 00000000..9437ee8c --- /dev/null +++ b/alembic/versions/f9928a3b6bc5_file_directories.py @@ -0,0 +1,44 @@ +"""File Directories + +Revision ID: f9928a3b6bc5 +Revises: c9561bda3cf0 +Create Date: 2023-04-18 16:58:40.043115 + +""" +import os +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'f9928a3b6bc5' +down_revision = 'c9561bda3cf0' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute(''' + CREATE TABLE directory ( + path TEXT PRIMARY KEY, + name TEXT NOT NULL, + idempotency TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp + ) + ''') + + session.execute('CREATE INDEX directory_name_idx ON directory(name)') + + if not DOCKERIZED: + session.execute('ALTER TABLE public.directory OWNER TO wrolpi') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('DROP INDEX IF EXISTS directory_name_idx') + session.execute('DROP TABLE IF EXISTS directory') diff --git a/app/src/App.css b/app/src/App.css index 1c050b5d..e7d23c3b 100644 --- a/app/src/App.css +++ b/app/src/App.css @@ -172,6 +172,35 @@ textarea.otp { background-color: #222222; } +.modal .inverted { + border-color: #888888; +} + +.modal .header.inverted { + color: #DDDDDD !important; + background-color: #171616 !important; +} + +.modal .content.inverted { + color: #DDDDDD !important; + border-color: #888888 !important; + background-color: #171616 !important; +} + +.modal .actions.inverted { + color: #DDDDDD !important; + border-color: #888888 !important; + background-color: #171616 !important; +} + .clickable { cursor: pointer; } + +.segment.inverted { + border: 1px solid #ffffff22 !important; +} + +.inverted label { + color: #DDDDDD !important; +} \ No newline at end of file diff --git a/app/src/Dashboard.js b/app/src/Dashboard.js index 9b9e1ee3..c2112d8e 100644 --- a/app/src/Dashboard.js +++ b/app/src/Dashboard.js @@ -1,18 +1,29 @@ import {LoadStatistic, PageContainer, SearchInput, useTitle} from "./components/Common"; -import {useQuery, useSearchFiles} from "./hooks/customHooks"; +import {useQuery} from "./hooks/customHooks"; import React, {useContext, useState} from "react"; import {StatusContext} from "./contexts/contexts"; import {DownloadMenu} from "./components/Download"; import {FilesSearchView} from "./components/Files"; -import {Header, Segment, Statistic, StatisticGroup} from "./components/Theme"; +import { + Button, + Divider, + Header, + Modal, + ModalContent, + ModalHeader, + Segment, + Statistic, + StatisticGroup +} from "./components/Theme"; import {Link} from "react-router-dom"; import {BandwidthProgressCombined, CPUUsageProgress} from "./components/admin/Status"; import {ProgressPlaceholder} from "./components/Placeholder"; -import {Divider, Icon, Message} from "semantic-ui-react"; +import {GridColumn, GridRow, Icon, Message} from "semantic-ui-react"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import {refreshFiles} from "./api"; import _ from "lodash"; import {TagsDashboard} from "./Tags"; +import {Upload} from "./components/Upload"; function FlagsMessages({flags}) { if (!flags) { @@ -29,8 +40,8 @@ function FlagsMessages({flags}) { refreshing = - Refreshing - Your files are being refreshed. + Your files are being refreshed. + Click here to view the progress ; } else if (flags.indexOf('refresh_complete') === -1) { @@ -73,10 +84,51 @@ export function Dashboard() { const {status} = useContext(StatusContext); const wrol_mode = status ? status['wrol_mode'] : null; - const [downloadOpen, setDownloadOpen] = useState(false); - const onDownloadOpen = (name) => setDownloadOpen(!!name); - const downloadsDisabled = status?.flags?.indexOf('refresh_complete') === -1; - const downloads = ; + // Getters are Downloads or Uploads. + const [selectedGetter, setSelectedGetter] = useState(null); + const gettersDisabled = status?.flags?.indexOf('refresh_complete') === -1; + const handleSetGetter = (e, value) => { + if (e) { + e.preventDefault(); + } + setSelectedGetter(value); + } + + let getter = + + Or + + + handleSetGetter(e, 'downloads')}> + + Download + + + + handleSetGetter(e, 'upload')}> + + Upload + + + + + ; + let getterModal; + if (selectedGetter === 'downloads') { + getterModal = handleSetGetter(null, null)}> + Downloads + + + + ; + } else if (selectedGetter === 'upload') { + getterModal = handleSetGetter(null, null)}> + Upload + + + + ; + } // Only show dashboard parts if not searching. let body; @@ -84,10 +136,10 @@ export function Dashboard() { body = ; } else { body = <> - {!wrol_mode && downloads} - {/* Hide extras when user is starting a download */} - {!downloadOpen && } - {!downloadOpen && } + {!wrol_mode && getter} + {getterModal} + + >; } diff --git a/app/src/Tags.js b/app/src/Tags.js index cd464356..eec474ab 100644 --- a/app/src/Tags.js +++ b/app/src/Tags.js @@ -1,21 +1,14 @@ import React, {useEffect, useState} from "react"; import {deleteTag, getTags, saveTag} from "./api"; import { - Button as SButton, Confirm, Dimmer, Divider, Dropdown, Form, - FormInput, Grid, - Header, Label, LabelGroup, - Loader, - Loader as SLoader, - Modal, - Table as STable, TableBody, TableCell, TableHeader, @@ -23,7 +16,18 @@ import { TableRow, } from "semantic-ui-react"; import {contrastingColor} from "./components/Common"; -import {Segment} from "./components/Theme"; +import { + Button, + FormInput, + Header, + Loader, + Modal, + ModalActions, + ModalContent, + ModalHeader, + Segment, + Table +} from "./components/Theme"; import _ from "lodash"; import {HexColorPicker} from "react-colorful"; import {useRecurringTimeout} from "./hooks/customHooks"; @@ -135,7 +139,7 @@ function EditTagRow({tag, onDelete, onEdit}) { const {name, color, id, count} = tag; const deleteConfirm = <> - setConfirmDeleteOpen(true)} icon='trash'/> + setConfirmDeleteOpen(true)} icon='trash'/> onDelete(id, name)} /> >; - const editButton = onEdit(name, color, id)} icon='edit'/>; + const editButton = onEdit(name, color, id)} icon='edit'/>; const nameLabel = ; @@ -217,8 +221,8 @@ function EditTagsModal() { onOpen={() => setOpen(true)} onClose={localOnClose} > - Edit Tags - + Edit Tags + {tagName || 'Example Tag'} @@ -227,7 +231,7 @@ function EditTagsModal() { Tag Name} placeholder='Unique name' value={tagName} onChange={(e, {value}) => setTagName(value)} @@ -236,17 +240,17 @@ function EditTagsModal() { - Save - + - + Delete @@ -264,12 +268,12 @@ function EditTagsModal() { {content} - - + + - setOpen(true)} color='violet'> + setOpen(true)} color='violet'> Edit - + > } @@ -324,33 +328,33 @@ export function AddTagsButton({ const unusedTagsGroup = ; return <> - + handleOpen(e)} onClose={() => setOpen(false)}> - - {loading && } + + {loading && } Applied Tags - {localTags && localTags.length > 0 ? selectedTagsGroup : 'Add tags below'} + {localTags && localTags.length > 0 ? selectedTagsGroup : 'Add one or more tags below'} {unusedTags && unusedTags.length > 0 ? unusedTagsGroup : 'You have no tags'} - - + + {!hideEdit && } - setOpen(false)} floated='right'>Close + setOpen(false)} floated='right'>Close - + > } @@ -402,7 +406,7 @@ export const TagsSelector = ({ export const TagsDashboard = () => { const {tagNames, TagsLinkGroup} = React.useContext(TagsContext); - let availableTagsGroup = ; + let availableTagsGroup = ; if (tagNames && tagNames.length) { availableTagsGroup = ; } diff --git a/app/src/api.js b/app/src/api.js index f44c20e8..43467bc8 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -1,6 +1,5 @@ import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, emptyToNull, OTP_API, VIDEOS_API} from "./components/Common"; import {toast} from "react-semantic-toasts-2"; -import {computeHeadingLevel} from "@testing-library/react"; function timeoutPromise(ms, promise) { // Create a timeout wrapper around a promise. If the timeout is reached, throw an error. Otherwise, return @@ -804,3 +803,25 @@ export async function sendNotification(message, url) { console.error('Failed to share'); } } + +export async function searchDirectories(name) { + const body = {name: name || ''}; + const response = await apiPost(`${API_URI}/files/search_directories`, body); + if (response.status === 204) { + return []; + } else if (response.status === 200) { + const content = await response.json(); + return content; + } else { + toast({type: 'error', title: 'Error', description: 'Failed to search directories!', time: 5000}); + } +} + +export async function uploadFiles(files, destination) { + const data = new FormData(); + for (let i = 0; i < files.length; i++) { + data.append('file', files[i]); + } + data.append('destination', destination); + console.log(data); +} diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index eef37d0e..ebfa421b 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -9,7 +9,6 @@ import { Dropdown, Image, Input, - Modal, PlaceholderHeader, PlaceholderLine, TableCell, @@ -19,7 +18,6 @@ import { BackButton, CardLink, cardTitleWrapper, - defaultSearchOrder, encodeMediaPath, ExternalCardLink, FileIcon, @@ -39,7 +37,7 @@ import { import {addTag, deleteArchives, postDownload, removeTag} from "../api"; import {Link, Route, Routes, useNavigate, useParams} from "react-router-dom"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; -import {useArchive, useDomains, useQuery, useSearchArchives, useSearchDomain} from "../hooks/customHooks"; +import {useArchive, useDomains, useSearchArchives, useSearchDomain} from "../hooks/customHooks"; import {FileCards, FileRowTagIcon, FilesView} from "./Files"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import _ from "lodash"; @@ -304,21 +302,6 @@ export function DomainsPage() { export function SearchDomain() { // A Dropdown which allows the user to filter by Archive domains. const {domain, domains, setDomain} = useSearchDomain(); - const [open, setOpen] = useState(false); - - const handleOpen = (e) => { - if (e) { - e.preventDefault(); - } - setOpen(true); - } - - const handleClose = (e) => { - if (e) { - e.preventDefault(); - } - setOpen(false); - } const handleChange = (e, {value}) => { setDomain(value); @@ -329,15 +312,6 @@ export function SearchDomain() { return {key: i['domain'], value: i['domain'], text: i['domain']} }); return <> - handleOpen()} - onClose={() => handleClose()} - > - - domains - - } @@ -792,14 +826,14 @@ export function UnsupportedModal(header, message, icon) { {header || 'Unsupported'} - - + + {message} - - - + + + Ok - + ; return {modal, doClose: onClose, doOpen: onOpen}; @@ -851,6 +885,99 @@ export function useTitle(title) { }, [title]); } +export function DirectorySearch({onSelect, value, ...props}) { + const { + directoryName, + setDirectoryName, + directories, + channelDirectories, + domainDirectories, + loading, + } = useSearchDirectories(value); + const [results, setResults] = useState(); + + useEffect(() => { + if ( + (directories && directories.length >= 0) + || (channelDirectories && channelDirectories.length >= 0) + || (domainDirectories && domainDirectories.length >= 0)) { + setResults({ + directories: { + name: 'Directories', + results: directories.map(i => { + return {title: i['path']} + }), + }, + channel_directories: { + name: 'Channels', + results: channelDirectories.map(i => { + return {title: i['path'], description: i['name']}; + }), + }, + domain_directories: { + name: 'Domains', + results: domainDirectories.map(i => { + return {title: i['path'], description: i['domain']}; + }), + } + }); + } + }, [ + JSON.stringify(directories), + JSON.stringify(channelDirectories), + JSON.stringify(domainDirectories), + loading, + ]); + + const handleSearchChange = (e, data) => { + if (e) { + e.preventDefault(); + } + if (onSelect) { + onSelect(''); + } + setDirectoryName(data.value); + } + + const handleResultSelect = (e, data) => { + if (e) { + e.preventDefault(); + } + // title is the relative path. + if (onSelect) { + onSelect(data.result.title); + } + setDirectoryName(data.result.title); + } + + const handleClear = (e) => { + if (e) { + e.preventDefault(); + } + onSelect(''); + setDirectoryName(''); + } + + return + + + + + + + + + +} + export function DirectoryInput({disabled, error, placeholder, setInput, value, required, isDirectory}) { const {directory, directories, setDirectory, isDir} = useDirectories(value); const {settings} = useSettings(); @@ -1009,7 +1136,7 @@ export function SortButton({sorts = []}) { let sortFields; if (sorts && sorts.length) { sortFields = sorts.map((i) => { - return handleSortButton(i['value'])}>{i['text']} + return handleSortButton(i['value'])}>{i['text']} }) } @@ -1018,10 +1145,10 @@ export function SortButton({sorts = []}) { open={open} onClose={() => setOpen(false)} > - Sort By - + Sort By + {sortFields} - + toggleDesc()}/> diff --git a/app/src/components/Donate.js b/app/src/components/Donate.js index ddd7d172..fd574ee0 100644 --- a/app/src/components/Donate.js +++ b/app/src/components/Donate.js @@ -1,7 +1,7 @@ import React, {useState} from "react"; import {PageContainer, useTitle} from "./Common"; -import {Button, Header, Segment, Table} from "./Theme"; -import {Icon, Modal, TableBody, TableCell, TableRow} from "semantic-ui-react"; +import {Button, Header, Modal, ModalContent, ModalHeader, Segment, Table} from "./Theme"; +import {Icon, TableBody, TableCell, TableRow} from "semantic-ui-react"; import QRCode from "react-qr-code"; const bitcoinAddress = '1mi1ddLSd6LmkuwJd1ttZsKuED724vYjS'; @@ -26,10 +26,12 @@ function CoinQRButton({qrCodeValue, header, buttonColor}) { onClose={() => setOpen(false)} onOpen={() => setOpen(true)} > - {header} - - - + {header} + + + + + > } diff --git a/app/src/components/Download.js b/app/src/components/Download.js index c007033e..0bfc72e2 100644 --- a/app/src/components/Download.js +++ b/app/src/components/Download.js @@ -1,13 +1,14 @@ import React, {useContext, useState} from "react"; import {getDownloaders, postDownload} from "../api"; -import {DirectoryInput, frequencyOptions, HelpPopup, rssFrequencyOptions, WROLModeMessage} from "./Common"; +import {DirectorySearch, frequencyOptions, HelpPopup, rssFrequencyOptions, WROLModeMessage} from "./Common"; import Icon from "semantic-ui-react/dist/commonjs/elements/Icon"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; import {ThemeContext} from "../contexts/contexts"; -import {Accordion, Button, Form, FormField, FormGroup, FormInput, Header, Loader, Segment, TextArea} from "./Theme"; -import {AccordionContent, AccordionTitle, FormDropdown} from "semantic-ui-react"; +import {Accordion, Button, Form, FormInput, Header, Loader, TextArea} from "./Theme"; +import {AccordionContent, AccordionTitle, Form as SForm, FormDropdown} from "semantic-ui-react"; import {Link} from "react-router-dom"; import {TagsSelector} from "../Tags"; +import {toast} from "react-semantic-toasts-2"; const validUrl = /^(http|https):\/\/[^ "]+$/; @@ -24,6 +25,7 @@ class Downloader extends React.Component { tagNames: [], urls: '', valid: true, + destinationRequired: props.destinationRequired !== undefined ? props.destinationRequired : false, }; } @@ -35,6 +37,8 @@ class Downloader extends React.Component { let response = await postDownload(urls, downloader, null, null, null, destination, tagNames); if (response.status === 204) { this.setState({urls: '', pending: false, submitted: true}); + } else { + toast({type: 'error', title: 'Error', description: 'Failed to create download!'}); } } finally { this.setState({pending: false}); @@ -46,10 +50,6 @@ class Downloader extends React.Component { this.setState({[name]: value}, this.validateUrls); } - handleDestinationChange = (value) => { - this.setState({destination: value}); - } - validateUrls = async () => { // Validate that all URLs in state are valid. if (this.state.urls) { @@ -84,23 +84,31 @@ class Downloader extends React.Component { } render() { - let disabled = !this.state.urls || !this.state.valid || this.state.pending || !this.state.downloader; - const {advancedOpen, destination, submitted, tagNames} = this.state; - const {header, withDestination, withTags} = this.props; + const { + advancedOpen, + submitted, + tagNames, + urls, + valid, + pending, + downloader, + destination, + destinationRequired + } = this.state; + let disabled = !urls || !valid || pending || !downloader; + const {header, withTags, withSearchDirectory} = this.props; - let destinationField; - if (withDestination) { - destinationField = - + let directorySearch; + if (withSearchDirectory) { + directorySearch = + Destination - - - ; + this.setState({destination: i})}/> + + ; + } + if (destinationRequired) { + disabled = !destination || disabled; } let tagsSelector; @@ -109,15 +117,12 @@ class Downloader extends React.Component { } const advancedAccordion = - this.setState({advancedOpen: !advancedOpen})}> - Advanced + this.setState({advancedOpen: !advancedOpen})}> + + Advanced - - {destinationField} - {withDestination && withTags && } - {tagsSelector} - + {directorySearch} ; @@ -132,8 +137,14 @@ class Downloader extends React.Component { name='urls' onChange={this.handleInputChange} value={this.state.urls} + style={{marginBottom: '1em'}} /> - {(withDestination || withTags) && advancedAccordion} + {tagsSelector} + {pending && } + + {/* Display DirectorySearch if its required, show it in an Accordion if it's not required */} + {withSearchDirectory ? destinationRequired ? directorySearch : advancedAccordion : null} + Download {submitted && viewDownloads} @@ -195,7 +206,7 @@ class ChannelDownload extends React.Component { return - Channel / Playlist + Channel / Playlist - + {pending && } {success && } } @@ -318,7 +329,7 @@ class RSSDownload extends ChannelDownload { return - RSS Feed + RSS Feed - + {pending && } {success && } } @@ -397,7 +408,7 @@ export function DownloadMenu({onOpen, disabled}) { /> localOnOpen('archive')} style={{marginBottom: '1em'}} @@ -415,6 +426,13 @@ export function DownloadMenu({onOpen, disabled}) { onClick={() => localOnOpen('rss')} style={{marginBottom: '1em'}} /> + localOnOpen('file')} + style={{marginBottom: '1em'}} + /> >); function clearSelected() { @@ -425,25 +443,29 @@ export function DownloadMenu({onOpen, disabled}) { const downloaders = { archive: Archive>} + header={<> Archives>} downloader='archive' withTags={true}/>, video: Videos>} + header={<> Videos>} downloader='video' - withDestination={true} + withSearchDirectory={true} withTags={true}/>, video_channel: , - rss: , + file: RSS>}/>, + header={<> Files>} + downloader='file' + withSearchDirectory={true} + destinationRequired={true} + withTags={true}/>, }; body = downloader in downloaders ? downloaders[downloader] : body; return <> - Download {body} > } diff --git a/app/src/components/FilePreview.js b/app/src/components/FilePreview.js index e09d0730..e16f8264 100644 --- a/app/src/components/FilePreview.js +++ b/app/src/components/FilePreview.js @@ -1,20 +1,16 @@ import React from "react"; import _ from "lodash"; import { - Button as SButton, Grid, Header as SHeader, Image, - Modal, - ModalActions, - ModalContent, - ModalHeader } from "semantic-ui-react"; import {TagsSelector} from "../Tags"; import {Media} from "../contexts/contexts"; import {encodeMediaPath} from "./Common"; import {addTag, fetchFile, removeTag} from "../api"; import {StlViewer} from "react-stl-viewer"; +import {Button, Modal, ModalActions, ModalContent, ModalHeader} from "./Theme"; function getMediaPathURL(previewFile) { if (previewFile['primary_path']) { @@ -47,6 +43,8 @@ function getIframeModal(previewFile) { @@ -201,16 +199,16 @@ export function FilePreviewWrapper({children}) { function setModalContent(content, url, downloadURL) { let openButton; if (url) { - openButton = Open; + openButton = Open; } - const closeButton = Close; + const closeButton = Close; const tagsDisplay = ; let downloadButton; if (downloadURL) { - downloadButton = + downloadButton = Download - ; + ; } setPreviewModal( 0) { return <> handleOpen()} onClose={() => setOpen(false)} closeIcon> - {modalHeader || Filter} - + {modalHeader || Filter} + {filterFields} - - - Clear - setOpen(false)}>Close - + + + Clear + setOpen(false)}>Close + { } } - return Share with everyone - + } export const ShareButton = () => { @@ -43,15 +43,17 @@ export const ShareButton = () => { open={open} onClose={handleClose} > - Share this page - + Share this page + Another user can scan this QR code to view this page - - - + + + + + - Close - + Close + diff --git a/app/src/components/Theme.js b/app/src/components/Theme.js index fb98ace7..f131d86f 100644 --- a/app/src/components/Theme.js +++ b/app/src/components/Theme.js @@ -1,27 +1,33 @@ import React, {useContext, useEffect, useState} from 'react'; import {ThemeContext} from "../contexts/contexts"; import { - Accordion as ACCORDION, - Button as BUTTON, - Card as CARD, - Form as FORM, - FormField as FORM_FIELD, - FormGroup as FORM_GROUP, - FormInput as FORM_INPUT, - Header as HEADER, - Icon as ICON, - Loader as LOADER, - Menu as MENU, - Placeholder as PLACEHOLDER, - Popup as POPUP, - Progress as PROGRESS, - Segment as SEGMENT, - Statistic as STATISTIC, - StatisticGroup as STATISTIC_GROUP, - Tab as TAB, - Table as TABLE, - TabPane as TAB_PANE, - TextArea as TEXTAREA + Accordion as SAccordion, + Button as SButton, + Card as SCard, + Divider as SDivider, + Form as SForm, + FormField as SFormField, + FormGroup as SFormGroup, + FormInput as SFormInput, + Header as SHeader, + Icon as SIcon, + Loader as SLoader, + Menu as SMenu, + Modal as SModal, + ModalActions as SModalActions, + ModalContent as SModalContent, + ModalDescription as SModalDescription, + ModalHeader as SModalHeader, + Placeholder as SPlaceholder, + Popup as SPopup, + Progress as SProgress, + Segment as SSegment, + Statistic as SStatistic, + StatisticGroup as SStatisticGroup, + Tab as STab, + Table as STable, + TabPane as STabPane, + TextArea as STextArea } from "semantic-ui-react"; import {ColorToSemanticHexColor} from "./Common"; import _ from "lodash"; @@ -56,7 +62,7 @@ export function ThemeWrapper({children, ...props}) { console.debug('setDarkTheme'); setI({inverted: true}); setS({style: {backgroundColor: '#1B1C1D', color: '#dddddd'}}); - setT({style: {color: '#dddddd'}}); + setT({style: {color: '#eeeeee'}}); setInverted('inverted'); setTheme(darkTheme); document.body.style.background = '#171616'; @@ -161,125 +167,158 @@ const defaultGrey = (props, inverted) => { return props; } -export function Button(props) { - const {i, inverted} = useContext(ThemeContext); - props = defaultGrey({...i, ...props}, inverted); - return -} - export function Accordion(props) { const {i, inverted} = useContext(ThemeContext); props = defaultGrey({...i, ...props}, inverted); - return + return +} + +export function Button(props) { + const {i, inverted} = useContext(ThemeContext); + props = defaultGrey({...i, ...props}, inverted); + return +} + +export function Divider({...props}) { + const {i} = useContext(ThemeContext); + props = {...i, ...props}; + return } export function Header(props) { - const {i, inverted} = useContext(ThemeContext); - props = defaultGrey({...i, ...props}, inverted); - return + const {t, inverted} = useContext(ThemeContext); + props = defaultGrey({...t, ...props}, inverted); + return } export function Form(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return } export function FormField(props) { const {i} = useContext(ThemeContext); props = invertedNull({...i, ...props}); - return + return } export function FormGroup(props) { const {i} = useContext(ThemeContext); props = invertedNull({...i, ...props}); - return + return } export function FormInput(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return } export function Icon(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return } export function Loader(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return } export function Menu(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return +} + +export function Modal({...props}) { + const {inverted} = useContext(ThemeContext); + return +} + +export function ModalActions({...props}) { + const {inverted} = useContext(ThemeContext); + return +} + +export function ModalContent({...props}) { + const {inverted} = useContext(ThemeContext); + return +} + +export function ModalDescription({...props}) { + const {inverted} = useContext(ThemeContext); + return +} + +export function ModalHeader({...props}) { + const {inverted} = useContext(ThemeContext); + return + {props.children} + } export function Placeholder(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return } export function Popup(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return } export function Progress(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return } export function Segment(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return } export function Statistic(props) { const {i, inverted} = useContext(ThemeContext); props = defaultGrey({...i, ...props}, inverted); - return + return } export function StatisticGroup(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; props['style'] = {...props['style'], marginLeft: 0, marginRight: 0}; - return + return } export function Tab(props) { const {i} = useContext(ThemeContext); props = invertedNull({...i, ...props}); - return + return } export function TabPane(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return } export function Table(props) { const {i} = useContext(ThemeContext); props = {...i, ...props}; - return + return } export function TextArea(props) { const {i} = useContext(ThemeContext); props = invertedNull({...i, ...props}); - return + return } export function CardIcon({onClick, ...props}) { @@ -304,5 +343,5 @@ export function Card({color, ...props}) { const borderColor = inverted ? '#888' : '#ddd'; props['style']['boxShadow'] = `0 0 0 2px ${borderColor}, 0 5px 0 0 ${emphasisColor}, 0 0px 3px 0 #d4d4d5`; } - return + return } diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js new file mode 100644 index 00000000..71262162 --- /dev/null +++ b/app/src/components/Upload.js @@ -0,0 +1,110 @@ +import React from "react"; +import {useDropzone} from "react-dropzone"; +import {Form, Header, Icon, Progress, Segment} from "./Theme"; +import {DirectorySearch, mimetypeIconName} from "./Common"; +import {useUploadFile} from "../hooks/customHooks"; +import _ from "lodash"; +import {ThemeContext} from "../contexts/contexts"; +import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; +import {Form as SForm} from "semantic-ui-react"; + +export function Upload({disabled}) { + const {setFiles, progresses, destination, setDestination, doClear} = useUploadFile(); + const {t} = React.useContext(ThemeContext); + + const onDrop = React.useCallback(async (acceptedFiles) => { + if (acceptedFiles && acceptedFiles.length > 0) { + setFiles(acceptedFiles); + } + }, [destination]); + + const {getRootProps, getInputProps} = useDropzone({onDrop}); + + React.useEffect(() => { + if (!destination) { + // Destination was cleared, clear progresses. + doClear(); + } + }, [destination]); + + const handleDestination = (value) => { + setDestination(value); + } + + let progressBars; + if (!_.isEmpty(progresses)) { + progressBars = Object.entries(progresses).map(([name, value]) => { + const {percent, status, type} = value; + let color = 'grey'; + let statusString; + if (status === 'complete') { + color = 'green'; + statusString = 'Complete:'; + } else if (status === 'pending') { + statusString = 'Pending:'; + } else if (status === 'failed') { + color = 'red'; + statusString = 'Failed:'; + } else if (status === 'conflicting') { + color = 'orange'; + statusString = 'Already Exists:'; + } + // Only indicating when upload is pending. + let indicating = status === 'pending'; + + return + + + + + + {statusString} {name} + + + + }) + } + + return <> + + + Destination + + + + + {destination ? + + + + + { + }}> + + + + + + + + Click here, or drop files here to upload + + + + + + + + + + + : You must search for a directory to place your files + } + + + + + {progressBars} + + > +} diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js index 8f5880be..62629e4d 100644 --- a/app/src/components/admin/Downloads.js +++ b/app/src/components/admin/Downloads.js @@ -13,7 +13,6 @@ import { Button as SButton, Confirm, Loader, - Modal, PlaceholderLine, TableBody, TableCell, @@ -21,7 +20,7 @@ import { TableHeaderCell, TableRow } from "semantic-ui-react"; -import {Button, Header, Placeholder, Segment, Table} from "../Theme"; +import {Button, Header, Modal, ModalActions, ModalContent, ModalHeader, Placeholder, Segment, Table} from "../Theme"; import {useDownloads} from "../../hooks/customHooks"; import _ from "lodash"; @@ -135,13 +134,13 @@ class DownloadRow extends React.Component { open={errorModalOpen} trigger={} > - Download Error - + Download Error + {error} - - + + this.setState({errorModalOpen: false})}>Close - + ; const deleteButton = <> @@ -242,7 +241,8 @@ class StoppableRow extends React.Component { let {url, last_successful_download, status, location, error} = this.props; let {stopOpen, startOpen, errorModalOpen, deleteOpen} = this.state; - const link = location ? + // Open downloads (/download), or external links in an anchor. + const link = location && !location.startsWith('/download') ? (text) => {text} : (text) => {text}; @@ -301,13 +301,13 @@ class StoppableRow extends React.Component { open={errorModalOpen} trigger={} > - Download Error - + Download Error + {error} - - + + this.setState({errorModalOpen: false})}>Close - + ) } diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js index fc6b289d..ca570e51 100644 --- a/app/src/components/admin/Settings.js +++ b/app/src/components/admin/Settings.js @@ -1,7 +1,7 @@ import React from "react"; import {getSettings, saveSettings} from "../../api"; -import {Button, Form, FormGroup, FormInput, Header, Loader, Segment} from "../Theme"; -import {Container, Dimmer, Icon, Modal} from "semantic-ui-react"; +import {Button, Form, FormGroup, FormInput, Header, Loader, Modal, ModalContent, ModalHeader, Segment} from "../Theme"; +import {Container, Dimmer, Icon} from "semantic-ui-react"; import {ThemeContext} from "../../contexts/contexts"; import {HelpPopup, HotspotToggle, ThrottleToggle, Toggle, WROLModeMessage} from "../Common"; import QRCode from "react-qr-code"; @@ -203,12 +203,14 @@ export class Settings extends React.Component { open={this.state.qrOpen} trigger={qrButton} > - + Scan this code to join the hotspot - - - - + + + + + + diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 0b2360b4..218429b2 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -17,6 +17,7 @@ import { getVideo, getVideosStatistics, searchArchives, + searchDirectories, searchVideos, setHotspot, setThrottle, @@ -622,6 +623,39 @@ export const useDirectories = (defaultDirectory) => { return {directory, directories, setDirectory, exists, isDir, isFile}; } +export const useSearchDirectories = (value) => { + const [directoryName, setDirectoryName] = useState(value ?? ''); + const [directories, setDirectories] = useState(null); + const [channelDirectories, setChannelDirectories] = useState(null); + const [domainDirectories, setDomainDirectories] = useState(null); + const [loading, setLoading] = useState(false); + + const localSearchDirectories = async () => { + setLoading(true); + try { + const {directories: dirs, channel_directories, domain_directories} = await searchDirectories(directoryName); + setDirectories(dirs); + setChannelDirectories(channel_directories); + setDomainDirectories(domain_directories); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + localSearchDirectories(); + }, [directoryName]); + + return { + directoryName, + setDirectoryName, + directories, + loading, + channelDirectories, + domainDirectories + } +} + export const useSettings = () => { const [settings, setSettings] = useState({}); @@ -646,6 +680,12 @@ export const useSettings = () => { return {settings, fetchSettings}; } +export const useMediaDirectory = () => { + const {settings} = useSettings(); + + return settings['media_directory']; +} + export const useSettingsInterval = () => { const {settings, fetchSettings} = useSettings(); @@ -769,3 +809,114 @@ export const useSearchOrder = () => { const [sort, setSort] = useOneQuery('order'); return {sort, setSort} } + +export const useUploadFile = () => { + const [files, setFiles] = useState([]); + const [progresses, setProgresses] = useState({}); + const [destination, setDestination] = useState(''); + + const handleProgress = (name, chunk, totalChunks, status, type) => { + const percent = Math.round((100 * chunk) / totalChunks); + const newProgress = {[name]: {percent, status, type}}; + setProgresses(prevState => ({...prevState, ...newProgress})); + } + + const handleFilesChange = (newFiles) => { + let newProgresses = {}; + newFiles.map(i => { + newProgresses = {...newProgresses, [i['name']]: {percent: 0, status: 'pending'}}; + }); + setProgresses(newProgresses); + setFiles(newFiles); + } + + const uploadChunk = async (file, chunkNum, chunkSize, totalChunks, tries, maxTries) => { + if (tries > maxTries) { + console.error(`Exceeded max tries ${maxTries}`); + } + + const start = chunkNum * chunkSize; + const end = Math.min(start + chunkSize, file.size); + // The bytes that we will send. + const chunk = file.slice(start, end); + + const formData = new FormData(); + formData.append('chunkNumber', chunkNum.toString()); + formData.append('filename', file.path); + formData.append('totalChunks', totalChunks.toString()); + formData.append('destination', destination); + // Send the size that we're actually sending. + formData.append('chunkSize', chunk.size.toString()); + formData.append('chunk', chunk); + + console.debug(`file upload: tries=${tries} chunkNum=${chunkNum} totalChunks=${totalChunks} chunkSize=${chunk.size} destination=${destination}`); + + const xhr = new XMLHttpRequest(); + xhr.open('POST', '/api/files/upload', true); + xhr.onreadystatechange = async () => { + if (xhr.readyState === 4) { + if (xhr.status === 200 || xhr.status === 416) { + const data = JSON.parse(xhr.responseText); + handleProgress(file.name, chunkNum, totalChunks, 'pending', file.type); + const expectedChunk = data['expected_chunk']; + if (xhr.status === 416) { + console.log(`Server requested a different chunk ${chunkNum}`); + await uploadChunk(file, expectedChunk, chunkSize, totalChunks, tries + 1, maxTries); + } else { + console.debug(`Uploading of chunk ${chunkNum} succeeded, got request for chunk ${chunkNum}`); + // Success, reset tries. + await uploadChunk(file, expectedChunk, chunkSize, totalChunks, 0, maxTries); + } + } else if (xhr.status === 201) { + handleProgress(file.name, totalChunks, totalChunks, 'complete', file.type); + console.log(`Uploading of ${file.path} completed.`); + } else if (xhr.status === 400) { + handleProgress(file.name, totalChunks, totalChunks, 'conflicting', file.type); + const data = JSON.parse(xhr.responseText); + if (data['code'] === 41) { + console.error('File already exists. Giving up.'); + } + } else { + handleProgress(file.name, totalChunks, totalChunks, 'failed', file.type); + console.error(`Failed to upload chunk ${chunkNum}. Giving up.`); + } + } + } + await xhr.send(formData); + }; + + const doUpload = async () => { + if (!files || files.length === 0 || !destination) { + return; + } + + for (let i = 0; i < files.length; i++) { + const file = files[i]; + console.log(`Starting file upload`); + console.log(file); + + const chunkNum = 0; + const chunkSize = 10 * 1024 * 1024; // 10MB + const totalChunks = Math.ceil(file.size / chunkSize); + const tries = 0; + const maxTries = 20; + + // Start recursive function to upload the file. + await uploadChunk(file, chunkNum, chunkSize, totalChunks, tries, maxTries); + } + + // Clear form after upload. + setFiles([]); + } + + const doClear = () => { + setFiles([]); + setProgresses({}); + } + + useEffect(() => { + doUpload() + }, [JSON.stringify(files)]); + + return {files, setFiles: handleFilesChange, progresses, destination, setDestination, doClear, doUpload} +} diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index bab96877..115774ff 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -53,7 +53,6 @@ class ArchiveDownloader(Downloader, ABC): return archives -# Archive downloader is the last downloader which should be used. archive_downloader = ArchiveDownloader() diff --git a/modules/archive/lib.py b/modules/archive/lib.py index a8057f73..494beed7 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -11,6 +11,7 @@ from typing import Optional, Tuple, List from bs4 import BeautifulSoup from selenium import webdriver +from sqlalchemy import asc from sqlalchemy.orm import Session from modules.archive.models import Domain, Archive @@ -440,3 +441,13 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order results, total = handle_file_group_search_results(stmt, params) return results, total + + +@optional_session +def search_domains_by_directory(name: str, limit: int = 5, session: Session = None) -> List[Domain]: + domains = session.query(Domain) \ + .filter(Domain.directory.ilike(f'%{name}%')) \ + .order_by(asc(Domain.domain)) \ + .limit(limit) \ + .all() + return domains diff --git a/modules/inventory/common.py b/modules/inventory/common.py index c0fd9901..ca75654f 100644 --- a/modules/inventory/common.py +++ b/modules/inventory/common.py @@ -127,7 +127,7 @@ class InventoriesConfig(ConfigFile): self.update({'version': value}) -INVENTORIES_CONFIG: InventoriesConfig = InventoriesConfig(global_=True) +INVENTORIES_CONFIG: InventoriesConfig = InventoriesConfig() TEST_INVENTORIES_CONFIG: InventoriesConfig = None diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index c36c3cb4..ba119de6 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -1,6 +1,8 @@ +import pathlib from pathlib import Path from typing import List, Dict, Union +from sqlalchemy import asc from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound @@ -194,3 +196,13 @@ def download_channel(id_: int): raise InvalidDownload(f'Channel {channel.name} does not have a download!') download.renew(reset_attempts=True) session.commit() + + +@optional_session +def search_channels_by_directory(name: str, limit: int = 5, session: Session = None) -> List[Channel]: + channels = session.query(Channel) \ + .filter(Channel.directory.ilike(f'%{name}%')) \ + .order_by(asc(Channel.name)) \ + .limit(limit) \ + .all() + return channels diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index b6a7ef1b..31cfd0f4 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -208,8 +208,13 @@ class VideoDownloader(Downloader, ABC): if not channel and (local_channel_id or channel_url or destination): # Could not find channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download. logger.info(f'Using download.settings to find channel') - channel = get_channel(channel_id=local_channel_id, url=channel_url, directory=destination, - return_dict=False) + try: + channel = get_channel(channel_id=local_channel_id, url=channel_url, directory=destination, + return_dict=False) + except UnknownChannel: + # We do not need a Channel if we have a destination directory. + if not destination: + raise if channel: found_channel = 'download_settings' @@ -297,8 +302,10 @@ class VideoDownloader(Downloader, ABC): session.commit() if download.settings and (tag_names := download.settings.get('tag_names')): + existing_names = video.file_group.tag_names for name in tag_names: - video.add_tag(name) + if name not in existing_names: + video.add_tag(name) except UnrecoverableDownloadError: raise diff --git a/modules/videos/lib.py b/modules/videos/lib.py index b676a90e..b3a2d11c 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -211,7 +211,7 @@ class ChannelsConfig(ConfigFile): self.update({'favorites': value}) -CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig(global_=True) +CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig() TEST_CHANNELS_CONFIG: ChannelsConfig = None @@ -328,7 +328,7 @@ class VideoDownloaderConfig(ConfigFile): self.update({'youtube_include_dash_manifest': value}) -VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig(global_=True) +VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig() TEST_VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = None diff --git a/nginx.conf b/nginx.conf index 8af63f17..0c01f23a 100644 --- a/nginx.conf +++ b/nginx.conf @@ -5,6 +5,8 @@ events { } http { + client_max_body_size 0; + map $http_upgrade $connection_upgrade { default upgrade; '' close; diff --git a/nginx.dev.conf b/nginx.dev.conf index 4489ed50..f747b86d 100644 --- a/nginx.dev.conf +++ b/nginx.dev.conf @@ -4,6 +4,8 @@ events { http { + client_max_body_size 0; + map $http_upgrade $connection_upgrade { default upgrade; '' close; diff --git a/requirements.txt b/requirements.txt index 6f8df6b1..732e8205 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,4 +25,3 @@ srt==3.5.1 websockets==10.1 webvtt-py==0.4.6 yt-dlp==2023.2.17 - diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py index 029369d5..08b75a3f 100644 --- a/wrolpi/cmd.py +++ b/wrolpi/cmd.py @@ -51,6 +51,9 @@ NMCLI_BIN = which('nmcli', '/usr/bin/nmcli', flag=flags.nmcli_installed) CPUFREQ_INFO_BIN = which('cpufreq-info', '/usr/bin/cpufreq-info') CPUFREQ_SET_BIN = which('cpufreq-set', '/usr/bin/cpufreq-set') +# Files +WGET_BIN = which('wget', '/usr/bin/wget', flag=flags.wget_installed) + # Map BASH_BIN = which('bash', '/bin/bash', warn=True) diff --git a/wrolpi/common.py b/wrolpi/common.py index 74ef65a7..55d2b918 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -180,28 +180,28 @@ def sanitize_link(link: str) -> str: class ConfigFile: """ - This class keeps track of the contents of a config file. You can update the config by calling - .update(), or by creating a property that does so. Save your changes using .save(). + This class keeps track of the contents of a config file. You can update the config (and it's file) by + calling .update(). """ file_name: str = None default_config: dict = None width: int = None - def __init__(self, global_: bool = False): + def __init__(self): self.file_lock = Lock() + self.width = self.width or 90 if PYTEST: # Do not load a global config on import while testing. A global instance will never be used for testing. self._config = self.default_config.copy() return - - self.width = self.width or 90 self.initialize() def __repr__(self): return f'<{self.__class__.__name__} file={self.get_file()}>' def initialize(self): + """Initializes this config dict using the default config and the config file.""" config_file = self.get_file() self._config = Manager().dict() # Use the default settings to initialize the config. @@ -261,7 +261,7 @@ class ConfigFile: self._config.update(config) self.save() - def dict(self): + def dict(self) -> dict: """Get a deepcopy of this config.""" if not hasattr(self, '_config'): raise NotImplementedError('You cannot use a global config while testing!') @@ -347,7 +347,7 @@ class WROLPiConfig(ConfigFile): self.update({'wrol_mode': value}) -WROLPI_CONFIG: WROLPiConfig = WROLPiConfig(global_=True) +WROLPI_CONFIG: WROLPiConfig = WROLPiConfig() TEST_WROLPI_CONFIG: WROLPiConfig = None @@ -765,7 +765,7 @@ def walk(path: Path) -> Generator[Path, None, None]: def get_files_and_directories(directory: Path): """Walk a directory, return a tuple of all files and all directories within. (Not recursive).""" if not directory.exists(): - raise ValueError(f'Directory does not exist: {directory}') + raise FileNotFoundError(f'Directory does not exist: {directory}') if not directory.is_dir(): raise ValueError('Can only walk a directory.') diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 901fc7b3..75ee1ac7 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -12,7 +12,7 @@ import tempfile import zipfile from abc import ABC from itertools import zip_longest -from typing import List, Callable, Union, Dict +from typing import List, Callable, Union, Dict, Sequence from typing import Tuple, Set from unittest import mock from unittest.mock import MagicMock @@ -33,7 +33,7 @@ from wrolpi.dates import set_test_now from wrolpi.db import postgres_engine, get_db_args from wrolpi.downloader import DownloadManager, DownloadResult, Download, Downloader, \ downloads_manager_config_context -from wrolpi.files import lib as files_lib +from wrolpi.files.models import Directory from wrolpi.root_api import BLUEPRINTS, api_app from wrolpi.tags import Tag from wrolpi.vars import PROJECT_DIR @@ -593,3 +593,32 @@ def zip_file_factory(test_directory): return fh.read() return _ + + +@pytest.fixture +def assert_directories(test_session, test_directory): + def _(expected: Sequence[str]): + directories = test_session.query(Directory).all() + names = {str(i.path.relative_to(test_directory)) for i in directories} + assert names == set(expected) + + return _ + + +@pytest.fixture +def make_multipart_form(): + def _(forms: List[dict]): + boundary = '-----sanic\n' + new_forms = [] + for form in forms: + name, value, filename = form['name'], form['value'], form.get('filename') + if filename: + new_forms.append(f'Content-Disposition: form-data; name="{name}"; filename="{filename}"\n\n{value}\n') + else: + new_forms.append(f'Content-Disposition: form-data; name="{name}"\n\n{value}\n') + body = f'\n{boundary}'.join(new_forms) + body = f'{boundary}{body}\n{boundary}' + body = '\r\n'.join(body.split('\n')).strip() + return body + + return _ diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 8dd0acc7..56727468 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1,5 +1,6 @@ import asyncio import contextlib +import json import logging import multiprocessing import os @@ -94,6 +95,7 @@ class Download(ModelHelper, Base): # noqa last_successful_download=self.last_successful_download, location=self.location, next_download=self.next_download, + settings=self.settings, status=self.status, sub_downloader=self.sub_downloader, url=self.url, @@ -352,7 +354,6 @@ class DownloadManager: download.manager = self downloader: Downloader = download.get_downloader() - logger.warning(f'{downloader=}') if not downloader: worker_logger.warning(f'Could not find downloader for {download.downloader=}') @@ -375,7 +376,8 @@ class DownloadManager: result = DownloadResult(success=False, error=str(traceback.format_exc())) error_len = len(result.error) if result.error else 0 - worker_logger.debug(f'Got success={result.success} from {downloader} with {error_len=}') + worker_logger.debug( + f'Got success={result.success} from {downloader} download_id={download.id} with {error_len=}') with get_db_session(commit=True) as session: # Modify the download in a new session because downloads may take a long time. @@ -1064,6 +1066,7 @@ async def save_downloads_config(session: Session): frequency=download.frequency, last_successful_download=download.last_successful_download, next_download=download.next_download, + settings=download.settings, status=download.status, sub_downloader=download.sub_downloader, url=download.url, @@ -1099,7 +1102,7 @@ async def import_downloads_config(session: Session): existing.next_download = download['next_download'] existing.status = download['status'] existing.sub_downloader = download['sub_downloader'] - logger.debug(f'Updating download {existing.url} with {download}') + existing.settings = download['settings'] or dict() for download in downloads_by_url.values(): # These downloads are new, import them. @@ -1108,6 +1111,7 @@ async def import_downloads_config(session: Session): frequency=download['frequency'], last_successful_download=download['last_successful_download'], next_download=download['next_download'], + settings=download['settings'] or dict(), status=download['status'], sub_downloader=download['sub_downloader'], url=(url := download['url']), diff --git a/wrolpi/errors.py b/wrolpi/errors.py index 0569d6a3..72a726c5 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -168,6 +168,10 @@ class FileGroupIsTagged(APIError): pass +class FileUploadFailed(APIError): + pass + + error_codes = iter(range(1, 1000)) API_ERRORS = { @@ -371,4 +375,9 @@ API_ERRORS = { 'message': 'File group is tagged', 'status': HTTPStatus.CONFLICT, }, + FileUploadFailed: { + 'code': next(error_codes), + 'message': 'Failed to upload file', + 'status': HTTPStatus.BAD_REQUEST, + }, } diff --git a/wrolpi/files/__init__.py b/wrolpi/files/__init__.py index 4347d483..7332b8af 100644 --- a/wrolpi/files/__init__.py +++ b/wrolpi/files/__init__.py @@ -1,3 +1,4 @@ from . import ebooks # noqa from . import pdfs # noqa from .api import bp +from . import downloader # noqa diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index 75ecd072..9701ed26 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -1,17 +1,23 @@ +import pathlib from http import HTTPStatus +from multiprocessing import Manager +from typing import List +import sanic.request from sanic import response, Request from sanic_ext import validate from sanic_ext.extensions.openapi import openapi -from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory -from wrolpi.errors import InvalidFile +from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory, logger, background_task +from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed from wrolpi.root_api import get_blueprint, json_response from . import lib, schema from ..schema import JSONErrorResponse bp = get_blueprint('Files', '/api/files') +logger = logger.getChild(__name__) + @bp.post('/') @openapi.definition( @@ -109,6 +115,39 @@ def post_directories(_, body: schema.DirectoriesRequest): return response.json(body) +@bp.post('/search_directories') +@openapi.definition( + summary='Get all directories whose name matches the provided name.', + body=schema.DirectoriesSearchRequest, +) +@validate(schema.DirectoriesSearchRequest) +async def post_search_directories(_, body: schema.DirectoriesSearchRequest): + if len(body.name) <= 1: + return response.empty() + + from modules.videos.channel import lib as channels_lib + channel_directories = channels_lib.search_channels_by_directory(name=body.name) + channel_directories = [dict(path=i.directory, name=i.name) for i in channel_directories] + channel_paths = [i['path'] for i in channel_directories] + + from modules.archive import lib as archives_lib + domain_directories = archives_lib.search_domains_by_directory(name=body.name) + domain_directories = [dict(path=i.directory, domain=i.domain) for i in domain_directories] + domain_paths = [i['path'] for i in domain_directories] + + from wrolpi.files.models import Directory + directories: List[Directory] = await lib.search_directories( + name=body.name, + excluded=list(map(str, channel_paths + domain_paths))) + + body = { + 'directories': directories, + 'channel_directories': channel_directories, + 'domain_directories': domain_directories + } + return json_response(body) + + @bp.post('/tag') @validate(schema.TagFileGroupPost) async def post_tag_file_group(_, body: schema.TagFileGroupPost): @@ -132,3 +171,101 @@ async def post_tag_file_group(_, body: schema.TagFileGroupPost): async def post_untag_file_group(_, body: schema.TagFileGroupPost): await lib.remove_file_group_tag(body.file_group_id, body.file_group_primary_path, body.tag_name, body.tag_id) return response.empty(HTTPStatus.NO_CONTENT) + + +# { +# '/media/directory/sub-dir/the-file-name.suffix': 2, # The chunk number we will receive next. +# } +UPLOADED_FILES = Manager().dict() + + +@bp.post('/upload') +async def post_upload(request: Request): + """Accepts a multipart/form-data request to upload a single file. + + Tracks the number of chunks and will request the correct chunk of the chunks come out of order. + + Will not overwrite an existing file, unless a previous upload did not complete.""" + try: + destination = request.form['destination'][0] + except Exception as e: + logger.error(f'Failed to get upload destination', exc_info=e) + raise UnknownDirectory('Must provide destination') from e + + destination = get_media_directory() / destination + if not destination.is_dir(): + raise UnknownDirectory('Destination must be a relative directory that is already in the media directory!') + + try: + filename = str(request.form['filename'][0]) + except Exception: + raise FileUploadFailed('filename string is required') + + try: + chunk_num = int(request.form['chunkNumber'][0]) + except Exception: + raise FileUploadFailed('chunkNumber integer is required') + + try: + total_chunks = int(request.form['totalChunks'][0]) + except Exception: + raise FileUploadFailed('totalChunks integer is required') + + filename = pathlib.Path(filename.lstrip('/')) + output = destination / filename + output_str = str(output) + + # Chunks start at 0. + expected_chunk_num = UPLOADED_FILES.get(output_str, 0) + logger.debug(f'last_chunk_num is {expected_chunk_num} for {repr(output_str)} received {chunk_num=}') + + chunk_size = int(request.form['chunkSize'][0]) + # The bytes of the uploading file. + chunk: sanic.request.File = request.files['chunk'][0] + if (body_size := len(chunk.body)) != chunk_size: + raise FileUploadFailed(f'Chunk size does not match the size of the chunk! {chunk_size} != {body_size=}') + + if chunk_num == 0 and output.is_file() and output_str in UPLOADED_FILES: + # User attempted to upload this same file, but it did not finish. User has started over again. + logger.info(f'Restarting upload of {repr(output_str)}') + output.unlink() + expected_chunk_num = 0 + + if expected_chunk_num != chunk_num: + # Respond with a request for the correct chunk number. + return json_response({'expected_chunk': expected_chunk_num}, HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE) + + if chunk_num == 0 and output.is_file(): + # Do not overwrite files that already exist. + raise FileUploadFailed('File already exists!') + if output.is_dir(): + raise FileUploadFailed('File already exists as a directory!') + + if chunk_num == 0: + try: + output.touch() + except FileNotFoundError: + # Uploading a file in a directory. Use mkdir to create all parents, but remove final directory that is + # created because it will be the file. + output.mkdir(parents=True) + output.rmdir() + with output.open('wb') as fh: + fh.write(chunk.body) + else: + with output.open('ab') as fh: + # Finally append the chunk to the file. + fh.write(chunk.body) + + # Store what we expect to receive next. + expected_chunk_num += 1 + UPLOADED_FILES[output_str] = expected_chunk_num + + # Chunks start at 0. + if chunk_num == total_chunks: + # File upload is complete. + del UPLOADED_FILES[output_str] + background_task(lib.refresh_files([output])) + return response.empty(HTTPStatus.CREATED) + + # Request the next chunk. + return json_response({'expected_chunk': expected_chunk_num}, HTTPStatus.OK) diff --git a/wrolpi/files/downloader.py b/wrolpi/files/downloader.py new file mode 100644 index 00000000..85d4e02f --- /dev/null +++ b/wrolpi/files/downloader.py @@ -0,0 +1,175 @@ +import asyncio +import pathlib +import re +import traceback +from abc import ABC +from dataclasses import dataclass +from http import HTTPStatus +from typing import List +from urllib.parse import urlparse + +import aiohttp + +from wrolpi.common import get_media_directory, logger, background_task +from wrolpi.db import get_db_session +from wrolpi.downloader import Downloader, Download, DownloadResult +from wrolpi.errors import UnrecoverableDownloadError + +__all__ = ['get_download_info', 'download_file', 'FileDownloader', 'file_downloader'] + +from wrolpi.files.models import FileGroup + +from wrolpi.vars import PYTEST + +logger = logger.getChild(__name__) + + +@dataclass +class DownloadFileInfo: + name: str = None + size: int = None + type: str = None + accept_ranges: str = None + + +FILENAME_MATCHER = re.compile(r'.*filename="(.*)"') + + +async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo: + """Gets information (name, size, etc.) about a downloadable file at the provided URL.""" + timeout = aiohttp.ClientTimeout(total=timeout) if timeout is not None else None + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.head(url) as response: + info = DownloadFileInfo( + type=response.headers.get('Content-Type'), + size=int(response.headers['Content-Length']) * 8 if 'Content-Length' in response.headers else None, + accept_ranges=response.headers.get('Accept-Ranges'), + ) + + disposition = response.headers.get('Content-Disposition') + + if disposition and 'filename' in disposition: + if (match := FILENAME_MATCHER.match(disposition)) and (groups := match.groups()): + info.name = groups[0] + else: + # No Content-Disposition with filename, use the URL name. + parsed = urlparse(url) + info.name = parsed.path.split('/')[-1] + + return info + + +async def download_file(url: str, info: DownloadFileInfo, output_path: pathlib.Path): + """Uses aiohttp to download an HTTP file. + + Attempts to resume the file if `output_path` already exists. + """ + logger.debug(f'Starting download of file {repr(str(url))}') + + if output_path.is_file() and info.size == (output_path.stat().st_size * 8): + logger.warning(f'Already downloaded {repr(str(url))} to {repr(str(output_path))}') + return + + if info.accept_ranges == 'bytes' or not output_path.is_file(): + with open(output_path, 'ab') as fh: + headers = dict() + # Check the position of append, if it is 0 then we do not need to resume. + position = fh.tell() + if position: + headers['Range'] = f'bytes={position}-' + + async with aiohttp.ClientSession() as session: + async with session.get(url, headers=headers) as response: + if response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE: + logger.warning(f'Server responded with 416, file is probably already downloaded') + return + + if position and response.status != HTTPStatus.PARTIAL_CONTENT: + raise UnrecoverableDownloadError( + f'Tried to resume {repr(str(url))} but got status {response.status}') + + # May or may not be using Range. Append each chunk to the output file. + async for data in response.content.iter_any(): + fh.write(data) + + # TODO this cannot be canceled. + # Sleep to catch cancel. + await asyncio.sleep(0) + elif output_path.is_file(): + # TODO support downloading files that cannot be resumed. + raise UnrecoverableDownloadError(f'Cannot resume download {url}') + + +class FileDownloader(Downloader, ABC): + """A Downloader which can download an arbitrary HTTP file. + + Attempts to use the Content-Disposition name, will fail over to the path name if that's not possible. + """ + name = 'file' + pretty_name = 'File' + listable = False + + def __repr__(self): + return '' + + async def do_download(self, download: Download) -> DownloadResult: + if download.attempts > 3: + raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}') + + media_directory = get_media_directory() + + # We don't know where to put the file if we don't have a destination. + destination = download.settings.get('destination') + if not destination: + raise UnrecoverableDownloadError(f'Cannot download the file without a destination') + + # Create the destination only if it is within the media directory. + destination = pathlib.Path(destination) + if not str(destination.absolute()).startswith(str(media_directory)): + raise UnrecoverableDownloadError(f'Cannot download outside media directory!') + destination.mkdir(parents=True, exist_ok=True) + + info = await get_download_info(download.url) + + # TODO verify that this output_path is exclusive. + output_path = destination / info.name + + try: + await download_file(download.url, info, output_path) + background_task(save_and_tag(output_path, download.settings.get('tag_names'))) + + return DownloadResult( + success=True, + location=f'/download/{output_path.relative_to(media_directory)}' + ) + except Exception as e: + logger.error(f'Failed to download {repr(str(download.url))}', exc_info=e) + if PYTEST: + raise + + error = str(traceback.format_exc()) + return DownloadResult( + success=False, + error=error, + ) + + +async def save_and_tag(file: pathlib.Path, tag_names: List[str] = None): + """Stores the file as a FileGroup. Applies any provided Tags to the FileGroup.""" + from wrolpi.tags import Tag + + tag_names = tag_names or list() + + with get_db_session(commit=True) as session: + file_group = FileGroup.find_by_path(file) + if not file_group: + file_group = FileGroup.from_paths(session, file) + session.commit() + + if tag_names: + tags = session.query(Tag).filter(Tag.name.in_(tag_names)) + for tag in tags: + file_group.add_tag(tag) + + +file_downloader = FileDownloader() diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 6f0c63d2..0b19c11f 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -14,6 +14,7 @@ from typing import List, Tuple, Union, Dict, Generator import cachetools.func import psycopg2 +from sqlalchemy import asc from sqlalchemy.orm import Session from wrolpi import flags @@ -25,7 +26,7 @@ from wrolpi.dates import now, from_timestamp from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag from wrolpi.events import Events -from wrolpi.files.models import FileGroup +from wrolpi.files.models import FileGroup, Directory from wrolpi.lang import ISO_639_CODES from wrolpi.tags import TagFile, Tag from wrolpi.vars import PYTEST @@ -204,6 +205,7 @@ SUFFIXES = { '.readability.json', '.readability.txt', '.readability.html', + '.live_chat.json' } SUFFIXES |= {f'.{i}.srt' for i in ISO_639_CODES} SUFFIXES |= {f'.{i}.vtt' for i in ISO_639_CODES} @@ -301,6 +303,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime): It is assumed all files exist.""" idempotency = str(idempotency) + non_primary_files = set() for chunk in chunks_by_stem(files, 100): # Group all files by their shared full-stem. `chunk_by_stem` already sorted the files. grouped = group_files_by_stem(chunk, pre_sorted=True) @@ -309,17 +312,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime): for group in grouped: # The primary file is the video/SingleFile/epub, etc. primary_path_or_paths = get_primary_file(group) - if isinstance(primary_path_or_paths, list) or isinstance(primary_path_or_paths, tuple): - # Cannot find primary path, create group for each file. - for primary_path in primary_path_or_paths: - primary_path: pathlib.Path - # The primary mimetype allows modelers to find its file_groups. - mimetype = get_mimetype(primary_path) - modification_datetime = from_timestamp(primary_path.stat().st_mtime) - size = primary_path.stat().st_size - files = json.dumps(_paths_to_files_dict([primary_path])) - values[primary_path] = (modification_datetime, mimetype, size, files) - else: + if isinstance(primary_path_or_paths, pathlib.Path): # Multiple files in this group. primary_path: pathlib.Path = primary_path_or_paths # The primary mimetype allows modelers to find its file_groups. @@ -330,9 +323,21 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime): size = sum(i.stat().st_size for i in group) files = json.dumps(_paths_to_files_dict(group)) values[primary_path] = (modification_datetime, mimetype, size, files) + non_primary_files = {i for i in group if i != primary_path} + else: + # Cannot find primary path, create a `file_group` for each file. + for primary_path in primary_path_or_paths: + primary_path: pathlib.Path + # The primary mimetype allows modelers to find its file_groups. + mimetype = get_mimetype(primary_path) + modification_datetime = from_timestamp(primary_path.stat().st_mtime) + size = primary_path.stat().st_size + files = json.dumps(_paths_to_files_dict([primary_path])) + values[primary_path] = (modification_datetime, mimetype, size, files) - # `False` is the `indexed` which is false to force indexing by default. - values = [(str(primary_path), False, idempotency, *i) for primary_path, i in values.items()] + values = [(str(primary_path), + False, # `indexed` is false to force indexing by default. + idempotency, *i) for primary_path, i in values.items()] with get_db_curs(commit=True) as curs: values = mogrify(curs, values) stmt = f''' @@ -370,6 +375,12 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime): if need_index: refresh_logger.info(f'Invalidated indexes of {len(need_index)} file groups') + if non_primary_files: + # New files may have been added which change what primary paths exist. Delete any file_groups which + # use paths which are not primary. + curs.execute('DELETE FROM file_group WHERE primary_path = ANY(%s)', + (list(map(str, non_primary_files)),)) + async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetime.datetime = None): """Discover all files in the directories provided in paths, as well as all files in paths. @@ -463,12 +474,14 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr paths = paths or [get_media_directory()] directories = list(filter(lambda i: i.is_dir(), paths)) + found_directories = set() if directories: with flags.refresh_counting: while directories: directory = directories.pop() files, dirs = get_files_and_directories(directory) directories.extend(dirs) + found_directories |= set(dirs) REFRESH['counted_files'] = REFRESH.get('counted_files', 0) + len(files) # Sleep to catch cancel. await asyncio.sleep(0) @@ -476,34 +489,40 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr with flags.refresh_discovery: await refresh_discover_paths(paths, idempotency) - if send_events: - Events.send_global_refresh_discovery_completed() + if send_events: + Events.send_global_refresh_discovery_completed() # Model all files that have not been indexed. with flags.refresh_modeling: await apply_modelers() - if send_events: - Events.send_global_refresh_modeling_completed() + if send_events: + Events.send_global_refresh_modeling_completed() # Index the rest of the files that were not indexed by modelers. with flags.refresh_indexing: await apply_indexers() - if send_events: - Events.send_global_refresh_indexing_completed() + if send_events: + Events.send_global_refresh_indexing_completed() # Cleanup any outdated file data. with flags.refresh_cleanup: await apply_refresh_cleanup() - if send_events: - Events.send_global_after_refresh_completed() - if send_events: - Events.send_global_refresh_completed() + with get_db_session() as session: + parent_directories = {i[0] for i in session.query(Directory.path).filter(Directory.path.in_(paths))} + parent_directories |= set(filter(lambda i: i.is_dir(), paths)) + await upsert_directories(parent_directories, found_directories) + + if send_events: + Events.send_global_after_refresh_completed() + refresh_logger.warning('Done refreshing Files') if refreshing_all_files: # Only set refresh_complete flag if all files have been refreshed. flags.refresh_complete.set() + if send_events: + Events.send_global_refresh_completed() async def apply_indexers(): @@ -539,6 +558,50 @@ async def apply_indexers(): break +async def upsert_directories(parent_directories, directories): + """ + Insert/update/delete directories provided. Deletes all children of `parent_directories` which are not in + `directories`. + """ + idempotency = now() + directories = list(directories) + list(parent_directories) + + # Only insert directories that are children of `media_directory` and exist. + media_directory = get_media_directory() + directories = [i for i in directories if i.is_dir() and i != media_directory] + + if directories: + # Insert any directories that were created, update any directories which previously existed. + with get_db_curs(commit=True) as curs: + values = [(str(i.absolute()), i.name, idempotency) for i in directories] + values = mogrify(curs, values) + stmt = f''' + INSERT INTO directory (path, name, idempotency) VALUES {values} + ON CONFLICT (path) DO UPDATE + SET idempotency = EXCLUDED.idempotency + ''' + curs.execute(stmt) + + with get_db_curs(commit=True) as curs: + # Delete the children of any parent directory which no longer exists. + for directory in parent_directories: + stmt = f'DELETE FROM directory WHERE path LIKE %(directory)s AND idempotency != %(idempotency)s' + curs.execute(stmt, dict(directory=f'{directory.absolute()}%', idempotency=idempotency)) + + +@optional_session +async def search_directories(name: str, excluded: List[str] = None, limit: int = 20, session: Session = None) \ + -> List[Directory]: + """Find the Directories whose names contain the `name` string.""" + excluded = excluded or [] + directories = session.query(Directory) \ + .filter(Directory.name.ilike(f'%{name}%')) \ + .filter(Directory.path.notin_(excluded)) \ + .order_by(asc(Directory.name)) \ + .limit(limit).all() + return directories + + def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None, tags: List[str] = None, headline: bool = False) -> \ Tuple[List[dict], int]: diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 04bba14a..d3ff5a66 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -246,3 +246,23 @@ class FileGroup(ModelHelper, Base): def find_by_path(path, session) -> Optional['FileGroup']: file_group = session.query(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none() return file_group + + @property + def tag_names(self) -> List[str]: + return [i.tag.name for i in self.tag_files] + + +class Directory(ModelHelper, Base): + """A representation of a file directory in the media directory.""" + __tablename__ = 'directory' + + path: pathlib.Path = Column(MediaPathType, primary_key=True) + name: str = Column(String, nullable=False) + idempotency = Column(TZDateTime, default=lambda: now()) + + def __json__(self): + d = dict( + path=self.path, + name=self.name, + ) + return d diff --git a/wrolpi/files/schema.py b/wrolpi/files/schema.py index f05e8f62..d0621017 100644 --- a/wrolpi/files/schema.py +++ b/wrolpi/files/schema.py @@ -38,6 +38,11 @@ class DirectoriesRequest: search_str: Optional[str] = '' +@dataclass +class DirectoriesSearchRequest: + name: str + + @dataclass class DirectoriesResponse: directories: List[str] diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index 7ccf8102..b240610a 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -5,7 +5,6 @@ from unittest import mock import pytest from wrolpi.errors import API_ERRORS, WROLModeEnabled -from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup from wrolpi.tags import TagFile from wrolpi.test.common import assert_dict_contains @@ -179,31 +178,6 @@ def test_files_search(test_session, test_client, make_files_structure, assert_fi assert_files_search('nothing', []) -def test_directory_search(test_client, make_files_structure): - """Test that directories can be searched.""" - make_files_structure([ - 'foo/', - 'fool/', - 'not a directory', - ]) - - def assert_directories(search_str, expected): - body = dict(search_str=search_str) - request, response = test_client.post('/api/files/directories', content=json.dumps(body)) - assert response.status_code == HTTPStatus.OK - assert response.json['directories'] == expected - - # All directories are returned. - assert_directories(None, ['foo', 'fool']) - assert_directories('', ['foo', 'fool']) - # Matches both directories. - assert_directories('fo', ['foo', 'fool']) - # Matches the one directory exactly. - assert_directories('foo', ['foo']) - # Does not exist. - assert_directories('food', []) - - def test_refresh_files_list(test_session, test_client, make_files_structure, test_directory, video_bytes): """The user can request to refresh specific files.""" make_files_structure({ @@ -307,3 +281,158 @@ def test_file_group_tag(test_client): request, response = test_client.post('/api/files/tag', content=json.dumps(dict(file_group_id=1))) assert response.status_code == HTTPStatus.BAD_REQUEST assert 'tag_id' in response.json['error'] + + +def test_search_directories(test_client, test_session, make_files_structure, assert_directories): + """Directories can be searched by name.""" + make_files_structure(['foo/one.txt', 'foo/two.txt', 'bar/one.txt']) + request, response = test_client.post('/api/files/refresh') + assert response.status_code == HTTPStatus.NO_CONTENT + assert_directories({'foo', 'bar'}) + + # More than one character required. + content = dict(name='f') + request, response = test_client.post('/api/files/search_directories', content=json.dumps(content)) + assert response.status_code == HTTPStatus.NO_CONTENT + + content = dict(name='fo') + request, response = test_client.post('/api/files/search_directories', content=json.dumps(content)) + assert response.status_code == HTTPStatus.OK + assert [i['path'] for i in response.json['directories']] == ['foo', ] + + # Case is ignored. + content = dict(name='BAR') + request, response = test_client.post('/api/files/search_directories', content=json.dumps(content)) + assert response.status_code == HTTPStatus.OK + assert [i['path'] for i in response.json['directories']] == ['bar', ] + + # Searching for something that does not exist. + content = dict(name='does not exist') + request, response = test_client.post('/api/files/search_directories', content=json.dumps(content)) + assert response.status_code == HTTPStatus.OK + assert [i['path'] for i in response.json['directories']] == [] + + # Only first 20 "foo" are returned. + make_files_structure([ + 'fooo/', + 'foooo/', + 'fooooo/', + 'foooooo/', + 'fooooooo/', + 'foooooooo/', + 'fooooooooo/', + 'foooooooooo/', + 'fooooooooooo/', + 'foooooooooooo/', + 'fooooooooooooo/', + 'foooooooooooooo/', + 'fooooooooooooooo/', + 'foooooooooooooooo/', + 'fooooooooooooooooo/', + 'foooooooooooooooooo/', + 'fooooooooooooooooooo/', + 'foooooooooooooooooooo/', + 'fooooooooooooooooooooo/', + 'foooooooooooooooooooooo/', + 'fooooooooooooooooooooooo/', + ]) + test_client.post('/api/files/refresh') + content = dict(name='fo') + request, response = test_client.post('/api/files/search_directories', content=json.dumps(content)) + assert response.status_code == HTTPStatus.OK + assert [i['path'] for i in response.json['directories']] == [f'f{"o" * i}' for i in range(2, 22)] + + +def test_post_search_directories(test_session, test_client, make_files_structure): + """Directory names can be searched. This endpoint also returns Channel and Domain directories.""" + channel_dir, domain_dir, _, _ = make_files_structure([ + 'dir1/', + 'dir2/', + 'dir3/', + 'dir4/', + ]) + request, response = test_client.post('/api/files/refresh') + assert response.status_code == HTTPStatus.NO_CONTENT + + from modules.videos.models import Channel + channel = Channel(directory=channel_dir, name='Channel Name') + from modules.archive.models import Domain + domain = Domain(directory=domain_dir, domain='example.com') + test_session.add_all([channel, domain]) + test_session.commit() + + content = {'name': 'di'} + request, response = test_client.post('/api/files/search_directories', content=json.dumps(content)) + assert response.status_code == HTTPStatus.OK + # All directories contain "di", but directories filters out the Channel and Domain directories. + assert response.json['directories'] == [{'name': 'dir3', 'path': 'dir3'}, {'name': 'dir4', 'path': 'dir4'}] + assert response.json['channel_directories'] == [{'name': 'Channel Name', 'path': 'dir1'}, ] + assert response.json['domain_directories'] == [{'domain': 'example.com', 'path': 'dir2'}, ] + + +def test_post_upload_directory(test_session, test_client, test_directory, make_files_structure, make_multipart_form): + """A file can be uploaded in a directory in the destination.""" + make_files_structure(['uploads/']) + + forms = [ + dict(name='chunkNumber', value='0'), + dict(name='filename', value='/foo/bar.txt'), # notice the directory + dict(name='totalChunks', value='0'), + dict(name='destination', value='uploads'), + dict(name='chunkSize', value='3'), + dict(name='chunk', value='foo', filename='chunk') + ] + body = make_multipart_form(forms) + request, response = test_client.post('/api/files/upload', data=body, + headers={ + 'Content-Type': 'multipart/form-data; name=upload; filename="file.txt";' + ' boundary=-----sanic'}) + assert response.status_code == HTTPStatus.CREATED + + assert (test_directory / 'uploads/foo/bar.txt').is_file() + assert (test_directory / 'uploads/foo/bar.txt').read_text() == 'foo' + + assert test_session.query(FileGroup).count() == 1 + + +def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form): + """A file can be uploaded in chunks directly to the destination.""" + make_files_structure(['uploads/']) + + forms = [ + dict(name='chunkNumber', value='0'), + dict(name='filename', value='foo.txt'), + dict(name='totalChunks', value='1'), + dict(name='destination', value='uploads'), + dict(name='chunkSize', value='3'), + dict(name='chunk', value='foo', filename='chunk') + ] + body = make_multipart_form(forms) + request, response = test_client.post('/api/files/upload', data=body, + headers={ + 'Content-Type': 'multipart/form-data; name=upload; filename="file.txt";' + ' boundary=-----sanic'}) + assert response.status_code == HTTPStatus.OK + + assert (test_directory / 'uploads/foo.txt').is_file() + assert (test_directory / 'uploads/foo.txt').read_text() == 'foo' + + forms = [ + dict(name='chunkNumber', value='1'), + dict(name='filename', value='foo.txt'), + dict(name='totalChunks', value='1'), + dict(name='destination', value='uploads'), + dict(name='chunkSize', value='3'), + dict(name='chunk', value='bar', filename='chunk') + ] + body = make_multipart_form(forms) + request, response = test_client.post('/api/files/upload', data=body, + headers={ + 'Content-Type': 'multipart/form-data; name=upload; filename="file.txt";' + ' boundary=-----sanic'}) + assert response.status_code == HTTPStatus.CREATED + + assert (test_directory / 'uploads/foo.txt').is_file() + assert (test_directory / 'uploads/foo.txt').read_text() == 'foobar' + + assert test_session.query(FileGroup).count() == 1 diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index d9507bdb..b4800c1f 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -10,6 +10,7 @@ from uuid import uuid4 import mock import pytest +import sqlalchemy.exc from PIL import Image from wrolpi.common import timer @@ -158,7 +159,8 @@ async def test__upsert_files(test_session, make_files_structure, test_directory, test_session.query(FileGroup).filter_by(primary_path=video_file).one().indexed = True test_session.commit() assert_file_groups( - [{'primary_path': str(video_file), 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': True}], + [{'primary_path': str(video_file), 'idempotency': idempotency, 'modification_datetime': foo_mtime, + 'indexed': True}], assert_count=False) # Only modified files need to be re-indexed. @@ -411,6 +413,7 @@ async def test_files_indexer(test_session, make_files_structure, test_directory) ('name two', 'name two'), ('this self-reliance_split.txt', 'this self reliance self-reliance split txt'), ('-be_split!.txt', '-be split! txt'), + ('WROLPi-v0.10-aarch64-desktop.img.xz', 'WROLPi v0.10 aarch64 desktop.img WROLPi-v0.10-aarch64-desktop.img xz'), ]) def test_split_file_name_words(name, expected): assert lib.split_file_name_words(name) == expected @@ -503,6 +506,7 @@ def test_group_files_by_stem(make_files_structure, test_directory): 'foo.mp4', 'foo.txt', 'foo.info.json', + 'foo.live_chat.json', 'bar.txt', 'baz.txt', ]) @@ -511,7 +515,7 @@ def test_group_files_by_stem(make_files_structure, test_directory): assert list(lib.group_files_by_stem(files)) == [ [test_directory / 'bar.txt'], [test_directory / 'baz.txt'], - [test_directory / 'foo.info.json', test_directory / 'foo.mp4', test_directory / 'foo.txt'], + [test_directory / 'foo.info.json', test_directory / 'foo.live_chat.json', test_directory / 'foo.mp4', test_directory / 'foo.txt'], ] @@ -600,3 +604,44 @@ async def test_refresh_files_no_groups(test_session, test_directory, make_files_ txt, zip_ = test_session.query(FileGroup) assert txt.primary_path == foo_txt and txt.size == foo_txt.stat().st_size assert zip_.primary_path == foo_zip and zip_.size == foo_zip.stat().st_size + + +@pytest.mark.asyncio +async def test_refresh_directories(test_session, test_directory, assert_directories): + """ + Directories are stored when they are discovered. They are removed when they can no longer be found. + """ + foo = test_directory / 'foo' + bar = test_directory / 'bar' + baz = test_directory / 'baz' + foo.mkdir() + bar.mkdir() + baz.mkdir() + + await lib.refresh_files() + assert_directories({'foo', 'bar', 'baz'}) + + # Deleted directory is removed. + foo.rmdir() + await lib.refresh_files() + assert_directories({'bar', 'baz'}) + + bar.rmdir() + await lib.refresh_files([bar]) + assert_directories({'baz', }) + + # A new directory can be refreshed directly. + foo.mkdir() + await lib.refresh_files([foo]) + assert_directories({'foo', 'baz'}) + + +def test_file_group_primary_path_is_unique(test_session, test_directory): + """You cannot create a FileGroup that shares a path""" + path = test_directory / 'file.txt' + path.touch() + + one = FileGroup.from_paths(test_session, path) + two = FileGroup.from_paths(test_session, path) + with pytest.raises(sqlalchemy.exc.IntegrityError): + test_session.commit() diff --git a/wrolpi/flags.py b/wrolpi/flags.py index 8e1660fe..c78c49c1 100644 --- a/wrolpi/flags.py +++ b/wrolpi/flags.py @@ -129,6 +129,7 @@ ffprobe_installed = Flag('ffprobe_installed') nmcli_installed = Flag('nmcli_installed') readability_installed = Flag('readability_installed') singlefile_installed = Flag('singlefile_installed') +wget_installed = Flag('wget_installed') yt_dlp_installed = Flag('yt_dlp_installed') diff --git a/wrolpi/roland@learningselfreliance.com.gpg b/wrolpi/roland@learningselfreliance.com.gpg new file mode 100644 index 00000000..c416df52 --- /dev/null +++ b/wrolpi/roland@learningselfreliance.com.gpg @@ -0,0 +1,51 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF2ebPIBEADG0T2VsDOH85qTF3UuAwQ9MO4T9yXDyb9tKVDX1wUsLEFH5tUn +dv51vhIZ+aVpFcrA7ULasikqdCo/Fomy9ujtf4EMYXwfg1uBrtvZZTwdciQnEfhR +/YCruypeXz91SukCjJMVLARWI1EUWyq7glSvXUOgPccLA6BCGN7Dv+JoXGDBVJAo +Uw74aCEIdhcxDM1u2RSaPWeVuA5P6PBYSnw4f0F91JHI1OobMFp7SmApCzLHzEyd +9J7ZqRpMVJaz160GzvemMdeDg7hj7GSmlVgeF4qmbE4E4cxqE2tRniYtgxjh0lyx +C6E0riRt2U00Z0CvaFuvwNX0fr0vZc+p82i9lakxy/9cUYKhVvcQsh/ofmbakGlz +MxdQpRsRpaivt2KCwHKzZUHsvDxGm/aH2liiz3bV9MeoBVo5ngFaBSLlnF08tGtR +g0M7olkTgX29BJM9++QaRvS8FwN2yG/iTIX7VTIEaEBoeoA0d7g6ojZC1UUZfI/R +Uy3AS0N1+Cza99PVU4vB9nsjUmtsO1ru0WxEKmfEJ7TI3ZTuH6+cpi8z9Vf6pdnN +LxB3utC5SvkYCYl9PPlm8QHnIAiAcL7VuAUpdWqJcMuMdytEPe8N8JKU9wWfxvdB +67t+0OSM8aBouq58UM8xi/3MxYVwZ7r4Eleir1wtjountk8abhAiXHJqnQARAQAB +tChSb2xhbmQgPHJvbGFuZEBsZWFybmluZ3NlbGZyZWxpYW5jZS5jb20+iQI/BBMB +AgApBQJdnmzyAhsDBQkJZgGABwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AACgkQ +nW29Os8Rq5M35w/7BJnPZ44BpdXpohyXbbNho7ewv0ggJFeKGcrsNVyYvhZp6vQx +W1zrpbMZc0w0QR7XZ8QKqQJAIihGKTs8dZ5Gl9EZs2EueMUEWMwAAjOL1pQ6Oa/A +m1GYv6H6v8pZSJluN33T9mGmQE/v1N3ASgB6IW0syj4WOKSxIj0tQqat8O1r9wwp +GHFVZQlCmUf7RTt0K/nGXeqFk7kUOtmidKHmfUJ8VXtyW36q+uZXlpQE+1o1KtMt +tdjaHGzpKKKpLbc/8hoNGJWE58uHxy40ZG85seq5qLgmKkdW9py98ov6nhcWTYJN +LI+3sEAYuSD1UU+iRBY9wHgh1dcxy+6AOuylsXehpwtABK7asm+anFmG7vso4Z0+ +mPWR3yHFK2MhZBW37VISBL0S5dxyEioLb2PTLGtRW+3ZagHv9xyUSI5gRfGOS9Sk +gAO/dyK4J6q+IxnABdU3l/8dwtV4+Qk4hp2iEwEy1ZAULWx4EZA9NxWTK6k0rCqs +f4w2yihC2ubo49xMa1iaYP/qMctFIlNttiKOVonDI+cfH4iUXljA1DDcYCbzOMCV +Iw8gdEkzyXaxWPBxcfzTQ8H8JWrxHc4/q5j+fC3D7YzkmmxwgR1pUSmOKyCEq2E/ +UjMSaZl26u0gzlUbE1uAKv8xXxYbh7SUPDOjCPqWljQm4iZnTiLcnnlqpu25Ag0E +XZ5s8gEQAOPHYlgQP8Han72JK6Wf9Hs8u7C0SqS9+bIN13yK14M6KxZprPYCS4fO +xmukFfchIqzQrzuiTTbDQ90EuuncFiHKeofe5TXUDrZehjkRg8/Ad115lM6Eme7q +iDdtHHxfSm8OgFE9xZg8sX9fhsQHQTrW/pDULAnO4mW5AKMlq4J4r/1AEN4wq0p+ +fgkd3d5KBAdXlmV+m4F6PM5xBDBoMZP5LGTeg8p68hWEGV+EuzpmJ6ixNSQ88a/d +j2IB3AH5LYLe0CQ4QWQ2EpM9NylmC9LwnBYM7ykaMXTb3twJioL2k2s20p5CQbnd +9j6h3rqncGPs9AoGnLUqf/jz4qPObBNsFpqHgFZ98vB+3Ynzdd+5O/MffCDBNKc3 +Zd4G5ZQ9NEM1EqB+s8uNcO7lSMqUbhEVE77PMxsqt0XrQASL3IkNPXA61apmS7Sc +XDcyaFzImwE03KDJF+zxVqlknxjJcJb+gdCMW1kYstKjGQsHd/KUlITQclLihifb +4WD/9bRkPuXteddt36Q9phkd0oO+OQqOoh4yjkI9OdlOgJVR4NQEDIi6hGf2eaKh +vp07dwRiQdrrRGLUEcXLIkuju+cQ4gxbgfSO1QdEGNVyXW88fzramF7LkGtZ0fXE +N0paw/0NOeSmV6hLyTsMkDHjNsOiDjDJSNVF1q5uq/Doz8o+csxZABEBAAGJAiUE +GAECAA8FAl2ebPICGwwFCQlmAYAACgkQnW29Os8Rq5OGkg/+MnJMjiQX+qFOzFGy +Twlxdm1cKRTDHwpWE1P7R9q//jPTjIDc8MF5GTGHkr1uFAV1J8mQ0KI34F/ek09K +e75x73OOlNTtxj/q+S5kbY/4Nd6ukvlkiE6UErK3dYiiY4JX9fRnyHuPgwNOH0qr +PNRq62pUk7MpRH0vvS2Lh4HmfMt2fz9aL4pjclJzM+gP5d84Rg4FeUvBTxSi/HMk +3KXxa6vlZyD+YE6CttptrtNmvu03haIhdjwCMFvR8y6Ytr6Ks0f5Le65jbh+u0xD +FRwdwSpG6LGGx+1BmOrtPOre2bvycMPz5pC5glQ9kUGhrugcdGB4gX2QkE28Vec5 +h5Vh6GOOy11p4XPDXA2eQg3SwHorltQcT283IJm7ZyrVTnGAvFNa8zaOlSjLw+P5 +Y6b+Ek/TCeqWS5yerODVRz7GyEn+2qjL/HL+g77XnxR7hAlI2eqdeTSN+GIkHTgp +7V9SBWYbGbKvsHlofd6XJjDOs4BHlQRkpfe+7g6w2x8xQkbZBN+hceTyGMTNJPSw +uCkizNFZNZoNi11HtWMhqSRBNox2GW+moiAvQRgjVHW0W78SPpzwtpY+TAdLaXfm +J25aqkmGRFvjfLP56VxXrrKNWNx5zPCvGe2T5MoZSx70Ji4KMw4auNyDxe/l3ywb +Xd3Fca66xs8xVb/52jI0Vv4oI3U= +=y08S +-----END PGP PUBLIC KEY BLOCK----- diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 0f497c8a..c4497520 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -483,6 +483,9 @@ def json_error_handler(request: Request, exception: Exception): except KeyError: # Cause was not an APIError. logger.error(f'Could not find cause {cause}') + logger.debug(f'API returning JSON error {exception=}' + f' error={body.get("api_error") or body.get("error")}' + f' code={body["code"]}') return json_response(body, error['status']) diff --git a/wrolpi/tags.py b/wrolpi/tags.py index 17125c43..6ee0ffb2 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -148,7 +148,7 @@ class TagsConfig(ConfigFile): }) -TAGS_CONFIG: TagsConfig = TagsConfig(global_=True) +TAGS_CONFIG: TagsConfig = TagsConfig() TEST_TAGS_CONFIG: TAGS_CONFIG = None diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index e1836585..4bb2d3bc 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -9,6 +9,7 @@ from unittest.mock import MagicMock import pytest import pytz +import yaml from wrolpi.dates import Seconds from wrolpi.db import get_db_context @@ -338,8 +339,6 @@ def test_crud_download(test_client, test_session, test_download_manager, test_do @pytest.mark.asyncio async def test_downloads_config(test_session, test_client, test_download_manager, test_download_manager_config, test_downloader, assert_downloads): - from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG - # Can import with an empty config. await import_downloads_config() assert_downloads([]) @@ -354,6 +353,7 @@ async def test_downloads_config(test_session, test_client, test_download_manager test_download_manager.recurring_download('https://example.com/3', frequency=DownloadFrequency.weekly, downloader_name=test_downloader.name) download2.next_download = datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) + download2.settings = {'destination': 'some directory'} # Completed once-downloads should be ignored. download1.last_successful_download = datetime(2000, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) test_session.commit() @@ -370,12 +370,14 @@ async def test_downloads_config(test_session, test_client, test_download_manager frequency=None, next_download=datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC), downloader='test_downloader', + settings={'destination': 'some directory'}, sub_downloader=None, ), dict( url='https://example.com/3', frequency=DownloadFrequency.weekly, downloader='test_downloader', + settings=None, sub_downloader=None, ), ] @@ -386,6 +388,16 @@ async def test_downloads_config(test_session, test_client, test_download_manager test_session.query(Download).delete() test_session.commit() + # Change the destination. + with test_download_manager_config.open('rt') as fh: + config_contents = yaml.load(fh, Loader=yaml.Loader) + for idx, download in enumerate(config_contents['downloads']): + if download['url'] == 'https://example.com/2': + config_contents['downloads'][idx]['settings'] = dict(destination='a different directory') + with test_download_manager_config.open('wt') as fh: + yaml.dump(config_contents, fh) + get_download_manager_config().initialize() + # Import the saved downloads. await import_downloads_config() expected = [ @@ -394,12 +406,15 @@ async def test_downloads_config(test_session, test_client, test_download_manager frequency=None, next_download=datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC), downloader='test_downloader', + # The destination was imported from the config. + settings={'destination': 'a different directory'}, sub_downloader=None, ), dict( url='https://example.com/3', frequency=DownloadFrequency.weekly, downloader='test_downloader', + settings=dict(), sub_downloader=None, ), ]
Click here to view the progress
{statusString} {name}
{error}