Adding my public gpg key.

Moving upload destination input out of drop Segment.

Disabling uploads when downloads are disabled.

Inverting labels for checkboxes in inverted modal.

Deleting file_groups which no longer contain the exclusive primary path.

Adding debug logging for all API errors.

Checking that upload output is not a directory.

Expanding nginx request size.

Adding docs to upload endpoint.

Using dot-circle icon for more image mimetypes.

Using Form.Field to make upload destination required.

Adding border to inverted Segments.

Many files can be uploaded in chunks.

Files can be uploaded, progress is displayed.

Showing white border around share QR code.

Displaying WROLPi colors for download icons.

Creating button to clear DirectorySearch.

Excluding channel/domain directories so other directories can be returned.

Ordering search results.

Ignoring when no channel is found only if we have a destination.

Limiting directory search results.

Displaying categories when searching destination directories.

Handling when live_chat.json file exists.

Allowing refresh of directory directly.

Logging which download was successful.

Removing timeout.

Creating index for directory names.

Sleeping to catch cancel.

Arbitrary files can be downloaded using the new FileDownloader.
This commit is contained in:
Roland 2023-04-18 22:03:45 -06:00
parent 83f9f6f085
commit 387282b445
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
45 changed files with 1665 additions and 356 deletions

View file

@ -0,0 +1,44 @@
"""File Directories
Revision ID: f9928a3b6bc5
Revises: c9561bda3cf0
Create Date: 2023-04-18 16:58:40.043115
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = 'f9928a3b6bc5'
down_revision = 'c9561bda3cf0'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('''
CREATE TABLE directory (
path TEXT PRIMARY KEY,
name TEXT NOT NULL,
idempotency TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp
)
''')
session.execute('CREATE INDEX directory_name_idx ON directory(name)')
if not DOCKERIZED:
session.execute('ALTER TABLE public.directory OWNER TO wrolpi')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('DROP INDEX IF EXISTS directory_name_idx')
session.execute('DROP TABLE IF EXISTS directory')

View file

@ -172,6 +172,35 @@ textarea.otp {
background-color: #222222;
}
.modal .inverted {
border-color: #888888;
}
.modal .header.inverted {
color: #DDDDDD !important;
background-color: #171616 !important;
}
.modal .content.inverted {
color: #DDDDDD !important;
border-color: #888888 !important;
background-color: #171616 !important;
}
.modal .actions.inverted {
color: #DDDDDD !important;
border-color: #888888 !important;
background-color: #171616 !important;
}
.clickable {
cursor: pointer;
}
.segment.inverted {
border: 1px solid #ffffff22 !important;
}
.inverted label {
color: #DDDDDD !important;
}

View file

@ -1,18 +1,29 @@
import {LoadStatistic, PageContainer, SearchInput, useTitle} from "./components/Common";
import {useQuery, useSearchFiles} from "./hooks/customHooks";
import {useQuery} from "./hooks/customHooks";
import React, {useContext, useState} from "react";
import {StatusContext} from "./contexts/contexts";
import {DownloadMenu} from "./components/Download";
import {FilesSearchView} from "./components/Files";
import {Header, Segment, Statistic, StatisticGroup} from "./components/Theme";
import {
Button,
Divider,
Header,
Modal,
ModalContent,
ModalHeader,
Segment,
Statistic,
StatisticGroup
} from "./components/Theme";
import {Link} from "react-router-dom";
import {BandwidthProgressCombined, CPUUsageProgress} from "./components/admin/Status";
import {ProgressPlaceholder} from "./components/Placeholder";
import {Divider, Icon, Message} from "semantic-ui-react";
import {GridColumn, GridRow, Icon, Message} from "semantic-ui-react";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {refreshFiles} from "./api";
import _ from "lodash";
import {TagsDashboard} from "./Tags";
import {Upload} from "./components/Upload";
function FlagsMessages({flags}) {
if (!flags) {
@ -29,8 +40,8 @@ function FlagsMessages({flags}) {
refreshing = <Message icon>
<Icon name='circle notched' loading/>
<Message.Content>
<Message.Header>Refreshing</Message.Header>
Your files are being refreshed.
<Message.Header>Your files are being refreshed.</Message.Header>
<p><Link to='/files'>Click here to view the progress</Link></p>
</Message.Content>
</Message>;
} else if (flags.indexOf('refresh_complete') === -1) {
@ -73,10 +84,51 @@ export function Dashboard() {
const {status} = useContext(StatusContext);
const wrol_mode = status ? status['wrol_mode'] : null;
const [downloadOpen, setDownloadOpen] = useState(false);
const onDownloadOpen = (name) => setDownloadOpen(!!name);
const downloadsDisabled = status?.flags?.indexOf('refresh_complete') === -1;
const downloads = <Segment><DownloadMenu onOpen={onDownloadOpen} disabled={downloadsDisabled}/></Segment>;
// Getters are Downloads or Uploads.
const [selectedGetter, setSelectedGetter] = useState(null);
const gettersDisabled = status?.flags?.indexOf('refresh_complete') === -1;
const handleSetGetter = (e, value) => {
if (e) {
e.preventDefault();
}
setSelectedGetter(value);
}
let getter = <Segment>
<Grid columns={2} textAlign='center'>
<Divider vertical>Or</Divider>
<GridRow verticalAlign='middle'>
<GridColumn>
<Button color='violet' onClick={e => handleSetGetter(e, 'downloads')}>
<Icon name='download'/>
Download
</Button>
</GridColumn>
<GridColumn>
<Button onClick={e => handleSetGetter(e, 'upload')}>
<Icon name='upload'/>
Upload
</Button>
</GridColumn>
</GridRow>
</Grid>
</Segment>;
let getterModal;
if (selectedGetter === 'downloads') {
getterModal = <Modal open={true} closeIcon onClose={() => handleSetGetter(null, null)}>
<ModalHeader>Downloads</ModalHeader>
<ModalContent>
<DownloadMenu disabled={gettersDisabled}/>
</ModalContent>
</Modal>;
} else if (selectedGetter === 'upload') {
getterModal = <Modal open={true} closeIcon onClose={() => handleSetGetter(null, null)}>
<ModalHeader>Upload</ModalHeader>
<ModalContent>
<Upload disabled={gettersDisabled}/>
</ModalContent>
</Modal>;
}
// Only show dashboard parts if not searching.
let body;
@ -84,10 +136,10 @@ export function Dashboard() {
body = <FilesSearchView/>;
} else {
body = <>
{!wrol_mode && downloads}
{/* Hide extras when user is starting a download */}
{!downloadOpen && <DashboardStatus/>}
{!downloadOpen && <TagsDashboard/>}
{!wrol_mode && getter}
{getterModal}
<TagsDashboard/>
<DashboardStatus/>
</>;
}

View file

@ -1,21 +1,14 @@
import React, {useEffect, useState} from "react";
import {deleteTag, getTags, saveTag} from "./api";
import {
Button as SButton,
Confirm,
Dimmer,
Divider,
Dropdown,
Form,
FormInput,
Grid,
Header,
Label,
LabelGroup,
Loader,
Loader as SLoader,
Modal,
Table as STable,
TableBody,
TableCell,
TableHeader,
@ -23,7 +16,18 @@ import {
TableRow,
} from "semantic-ui-react";
import {contrastingColor} from "./components/Common";
import {Segment} from "./components/Theme";
import {
Button,
FormInput,
Header,
Loader,
Modal,
ModalActions,
ModalContent,
ModalHeader,
Segment,
Table
} from "./components/Theme";
import _ from "lodash";
import {HexColorPicker} from "react-colorful";
import {useRecurringTimeout} from "./hooks/customHooks";
@ -135,7 +139,7 @@ function EditTagRow({tag, onDelete, onEdit}) {
const {name, color, id, count} = tag;
const deleteConfirm = <>
<SButton color='red' onClick={() => setConfirmDeleteOpen(true)} icon='trash'/>
<Button color='red' onClick={() => setConfirmDeleteOpen(true)} icon='trash'/>
<Confirm
id={`confirm${name}`}
open={confirmDeleteOpen}
@ -145,7 +149,7 @@ function EditTagRow({tag, onDelete, onEdit}) {
onConfirm={async () => onDelete(id, name)}
/>
</>;
const editButton = <SButton primary onClick={() => onEdit(name, color, id)} icon='edit'/>;
const editButton = <Button primary onClick={() => onEdit(name, color, id)} icon='edit'/>;
const nameLabel = <LabelGroup tag>
<NameToTagLabel name={name}/>
</LabelGroup>;
@ -217,8 +221,8 @@ function EditTagsModal() {
onOpen={() => setOpen(true)}
onClose={localOnClose}
>
<Modal.Header>Edit Tags</Modal.Header>
<Modal.Content>
<ModalHeader>Edit Tags</ModalHeader>
<ModalContent>
<Label.Group tag>
<Label size='large' style={{backgroundColor: tagColor, color: textColor}}>
{tagName || 'Example Tag'}
@ -227,7 +231,7 @@ function EditTagsModal() {
<Form autoComplete='off'>
<FormInput required
label='Tag Name'
label={<b>Tag Name</b>}
placeholder='Unique name'
value={tagName}
onChange={(e, {value}) => setTagName(value)}
@ -236,17 +240,17 @@ function EditTagsModal() {
<HexColorPicker color={tagColor} onChange={setTagColor} style={{marginTop: '1em'}}/>
<SButton color='violet'
size='big'
onClick={localSaveTag}
style={{marginTop: '2em'}}
disabled={!!!tagName}
<Button color='violet'
size='big'
onClick={localSaveTag}
style={{marginTop: '2em'}}
disabled={!!!tagName}
>
Save
</SButton>
</Button>
<Divider/>
<STable striped basic='very' unstackable>
<Table striped basic='very' unstackable>
<TableHeader>
<TableRow>
<TableHeaderCell width={2}>Delete</TableHeaderCell>
@ -264,12 +268,12 @@ function EditTagsModal() {
<TableBody>
{content}
</TableBody>
</STable>
</Modal.Content>
</Table>
</ModalContent>
</Modal>
<SButton onClick={() => setOpen(true)} color='violet'>
<Button onClick={() => setOpen(true)} color='violet'>
Edit
</SButton>
</Button>
</>
}
@ -324,33 +328,33 @@ export function AddTagsButton({
const unusedTagsGroup = <TagsGroup tagNames={unusedTags} onClick={addTag}/>;
return <>
<SButton icon={active ? 'tags' : 'tag'} onClick={handleOpen} primary={!!active}/>
<Button icon={active ? 'tags' : 'tag'} onClick={handleOpen} primary={!!active}/>
<Modal closeIcon
open={open}
onOpen={(e) => handleOpen(e)}
onClose={() => setOpen(false)}>
<Modal.Content>
{loading && <Dimmer active><SLoader/></Dimmer>}
<ModalContent>
{loading && <Dimmer active><Loader/></Dimmer>}
<Header as='h4'>Applied Tags</Header>
{localTags && localTags.length > 0 ? selectedTagsGroup : 'Add tags below'}
{localTags && localTags.length > 0 ? selectedTagsGroup : 'Add one or more tags below'}
<Divider/>
{unusedTags && unusedTags.length > 0 ? unusedTagsGroup : 'You have no tags'}
</Modal.Content>
<Modal.Actions>
</ModalContent>
<ModalActions>
<Grid textAlign='left'>
<Grid.Row>
<Grid.Column width={8}>
{!hideEdit && <EditTagsModal/>}
</Grid.Column>
<Grid.Column width={8}>
<SButton onClick={() => setOpen(false)} floated='right'>Close</SButton>
<Button onClick={() => setOpen(false)} floated='right'>Close</Button>
</Grid.Column>
</Grid.Row>
</Grid>
</Modal.Actions>
</ModalActions>
</Modal>
</>
}
@ -402,7 +406,7 @@ export const TagsSelector = ({
export const TagsDashboard = () => {
const {tagNames, TagsLinkGroup} = React.useContext(TagsContext);
let availableTagsGroup = <SLoader active inline/>;
let availableTagsGroup = <Loader active inline/>;
if (tagNames && tagNames.length) {
availableTagsGroup = <TagsLinkGroup tagNames={tagNames} style={{marginTop: '0.5em'}}/>;
}

View file

@ -1,6 +1,5 @@
import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, emptyToNull, OTP_API, VIDEOS_API} from "./components/Common";
import {toast} from "react-semantic-toasts-2";
import {computeHeadingLevel} from "@testing-library/react";
function timeoutPromise(ms, promise) {
// Create a timeout wrapper around a promise. If the timeout is reached, throw an error. Otherwise, return
@ -804,3 +803,25 @@ export async function sendNotification(message, url) {
console.error('Failed to share');
}
}
export async function searchDirectories(name) {
const body = {name: name || ''};
const response = await apiPost(`${API_URI}/files/search_directories`, body);
if (response.status === 204) {
return [];
} else if (response.status === 200) {
const content = await response.json();
return content;
} else {
toast({type: 'error', title: 'Error', description: 'Failed to search directories!', time: 5000});
}
}
export async function uploadFiles(files, destination) {
const data = new FormData();
for (let i = 0; i < files.length; i++) {
data.append('file', files[i]);
}
data.append('destination', destination);
console.log(data);
}

View file

@ -9,7 +9,6 @@ import {
Dropdown,
Image,
Input,
Modal,
PlaceholderHeader,
PlaceholderLine,
TableCell,
@ -19,7 +18,6 @@ import {
BackButton,
CardLink,
cardTitleWrapper,
defaultSearchOrder,
encodeMediaPath,
ExternalCardLink,
FileIcon,
@ -39,7 +37,7 @@ import {
import {addTag, deleteArchives, postDownload, removeTag} from "../api";
import {Link, Route, Routes, useNavigate, useParams} from "react-router-dom";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useArchive, useDomains, useQuery, useSearchArchives, useSearchDomain} from "../hooks/customHooks";
import {useArchive, useDomains, useSearchArchives, useSearchDomain} from "../hooks/customHooks";
import {FileCards, FileRowTagIcon, FilesView} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import _ from "lodash";
@ -304,21 +302,6 @@ export function DomainsPage() {
export function SearchDomain() {
// A Dropdown which allows the user to filter by Archive domains.
const {domain, domains, setDomain} = useSearchDomain();
const [open, setOpen] = useState(false);
const handleOpen = (e) => {
if (e) {
e.preventDefault();
}
setOpen(true);
}
const handleClose = (e) => {
if (e) {
e.preventDefault();
}
setOpen(false);
}
const handleChange = (e, {value}) => {
setDomain(value);
@ -329,15 +312,6 @@ export function SearchDomain() {
return {key: i['domain'], value: i['domain'], text: i['domain']}
});
return <>
<Modal closeIcon
open={open}
onOpen={() => handleOpen()}
onClose={() => handleClose()}
>
<Modal.Content>
domains
</Modal.Content>
</Modal>
<Dropdown selection search clearable fluid
placeholder='Domains'
options={domainOptions}

View file

@ -1,13 +1,36 @@
import React, {useContext, useEffect, useState} from "react";
import {Button as SButton, ButtonGroup, Card, Container, IconGroup, Input, Modal, Pagination} from 'semantic-ui-react';
import {Button as SButton, ButtonGroup, Card, Container, IconGroup, Input, Pagination, Search} from 'semantic-ui-react';
import {Link, NavLink, useNavigate} from "react-router-dom";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useDirectories, useHotspot, useSearchOrder, useSettings, useThrottle} from "../hooks/customHooks";
import {
useDirectories,
useHotspot,
useSearchDirectories,
useSearchOrder,
useSettings,
useThrottle
} from "../hooks/customHooks";
import {Media, StatusContext, ThemeContext} from "../contexts/contexts";
import {Button, CardIcon, darkTheme, Form, Header, Icon, lightTheme, Menu, Popup, Statistic} from "./Theme";
import {
Button,
CardIcon,
darkTheme,
Form,
Header,
Icon,
lightTheme,
Menu,
Modal,
ModalActions,
ModalContent,
ModalDescription, ModalHeader,
Popup,
Statistic
} from "./Theme";
import {FilePreviewContext} from "./FilePreview";
import _ from "lodash";
import {killDownloads, startDownloads} from "../api";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
export const API_URI = process.env && process.env.REACT_APP_API_URI ? process.env.REACT_APP_API_URI : `${window.location.protocol}//${window.location.host}/api`;
export const VIDEOS_API = `${API_URI}/videos`;
@ -697,54 +720,65 @@ export function isZipMimetype(mimetype) {
return mimetype.startsWith('application/zip') || mimetype.startsWith('application/zlib') || mimetype.startsWith('application/x-7z-compressed') || mimetype.startsWith('application/x-bzip2') || mimetype.startsWith('application/x-xz') || mimetype.startsWith('application/gzip') || mimetype.startsWith('application/x-rar');
}
export function mimetypeIconName(mimetype, lowerPath = '') {
if (mimetype) {
if (mimetype.startsWith('text/html') || mimetype.startsWith('application/json') || mimetype.startsWith('text/yaml') || mimetype.startsWith('text/xml')) {
return 'file code';
} else if (mimetype.startsWith('application/pdf')) {
return 'file pdf';
} else if (mimetype.startsWith('text/plain')) {
return 'file text';
} else if (mimetype.startsWith('image/')) {
return 'image';
} else if (mimetype.startsWith('video/')) {
return 'film';
} else if (mimetype.startsWith('message/rfc822')) {
return 'mail';
} else if (isZipMimetype(mimetype)) {
return 'file archive';
} else if (mimetype.startsWith('application/x-iso9660-image') || mimetype.startsWith('application/x-raw-disk-image') || mimetype.startsWith('application/x-cd-image')) {
return 'dot circle';
} else if (mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook') || mimetype.startsWith('application/vnd.amazon.mobi8-ebook')) {
return 'book';
} else if (mimetype.startsWith('text/vtt') || mimetype.startsWith('text/srt')) {
return 'closed captioning';
} else if (mimetype.startsWith('application/x-openscad') || mimetype.startsWith('model/stl')) {
return 'cube';
} else if (mimetype.startsWith('application/x-dosexec') || mimetype.startsWith('application/x-msi')) {
return 'microsoft';
} else if (mimetype.startsWith('audio/')) {
return 'file audio';
} else if (mimetype.startsWith('application/vnd.openxmlformats-officedocument.wordprocessingml.document')) {
return 'file word';
} else if (mimetype.startsWith('application/x-x509-ca-cert')) {
return 'certificate';
} else if (mimetype.startsWith('application/octet-stream')) {
if (lowerPath.endsWith('.mp3')) {
return 'file audio';
} else if (lowerPath.endsWith('.stl')) {
return 'cube';
} else if (lowerPath.endsWith('.blend')) {
return 'cube';
} else if (lowerPath.endsWith('.dmg')) {
return 'apple';
} else if (lowerPath.endsWith('.azw3')) {
return 'book';
}
}
}
if (lowerPath.endsWith('.pem')) {
return 'certificate';
}
return 'file';
}
export function FileIcon({file, disabled = true, size = 'huge', ...props}) {
// Default to a grey file icon.
const {mimetype, path, primary_path} = file;
// `file` may be a file_group or a file.
const lowerPath = primary_path ? primary_path.toLocaleString() : path.toLowerCase();
props['name'] = 'file';
props['name'] = mimetypeIconName(mimetype, lowerPath);
props['color'] = mimetypeColor(mimetype);
if (mimetype) {
if (mimetype.startsWith('text/html') || mimetype.startsWith('application/json') || mimetype.startsWith('text/yaml') || mimetype.startsWith('text/xml')) {
props['name'] = 'file code';
} else if (mimetype.startsWith('application/pdf')) {
props['name'] = 'file pdf';
} else if (mimetype.startsWith('text/plain')) {
props['name'] = 'file text';
} else if (mimetype.startsWith('image/')) {
props['name'] = 'image';
} else if (mimetype.startsWith('video/')) {
props['name'] = 'film';
} else if (mimetype.startsWith('message/rfc822')) {
props['name'] = 'mail';
} else if (isZipMimetype(mimetype)) {
props['name'] = 'file archive';
} else if (mimetype.startsWith('application/x-iso9660-image')) {
props['name'] = 'dot circle';
} else if (mimetype.startsWith('application/epub') || mimetype.startsWith('application/x-mobipocket-ebook') || mimetype.startsWith('application/vnd.amazon.mobi8-ebook')) {
props['name'] = 'book';
} else if (mimetype.startsWith('text/vtt') || mimetype.startsWith('text/srt')) {
props['name'] = 'closed captioning';
} else if (mimetype.startsWith('application/x-openscad') || mimetype.startsWith('model/stl')) {
props['name'] = 'cube';
} else if (mimetype.startsWith('application/x-dosexec') || mimetype.startsWith('application/x-msi')) {
props['name'] = 'microsoft';
} else if (mimetype.startsWith('audio/')) {
props['name'] = 'file audio';
} else if (mimetype.startsWith('application/octet-stream')) {
if (lowerPath.endsWith('.mp3')) {
props['name'] = 'file audio';
} else if (lowerPath.endsWith('.stl')) {
props['name'] = 'cube';
} else if (lowerPath.endsWith('.blend')) {
props['name'] = 'cube';
} else if (lowerPath.endsWith('.dmg')) {
props['name'] = 'apple';
} else if (lowerPath.endsWith('.azw3')) {
props['name'] = 'book';
}
}
}
return <Icon disabled={disabled} size={size} {...props}/>
}
@ -792,14 +826,14 @@ export function UnsupportedModal(header, message, icon) {
<Icon name={icon || 'exclamation triangle'}/>
{header || 'Unsupported'}
</Header>
<Modal.Content>
<Modal.Description>
<ModalContent>
<ModalDescription>
{message}
</Modal.Description>
</Modal.Content>
<Modal.Actions>
</ModalDescription>
</ModalContent>
<ModalActions>
<Button basic inverted onClick={onClose}>Ok</Button>
</Modal.Actions>
</ModalActions>
</Modal>;
return {modal, doClose: onClose, doOpen: onOpen};
@ -851,6 +885,99 @@ export function useTitle(title) {
}, [title]);
}
export function DirectorySearch({onSelect, value, ...props}) {
const {
directoryName,
setDirectoryName,
directories,
channelDirectories,
domainDirectories,
loading,
} = useSearchDirectories(value);
const [results, setResults] = useState();
useEffect(() => {
if (
(directories && directories.length >= 0)
|| (channelDirectories && channelDirectories.length >= 0)
|| (domainDirectories && domainDirectories.length >= 0)) {
setResults({
directories: {
name: 'Directories',
results: directories.map(i => {
return {title: i['path']}
}),
},
channel_directories: {
name: 'Channels',
results: channelDirectories.map(i => {
return {title: i['path'], description: i['name']};
}),
},
domain_directories: {
name: 'Domains',
results: domainDirectories.map(i => {
return {title: i['path'], description: i['domain']};
}),
}
});
}
}, [
JSON.stringify(directories),
JSON.stringify(channelDirectories),
JSON.stringify(domainDirectories),
loading,
]);
const handleSearchChange = (e, data) => {
if (e) {
e.preventDefault();
}
if (onSelect) {
onSelect('');
}
setDirectoryName(data.value);
}
const handleResultSelect = (e, data) => {
if (e) {
e.preventDefault();
}
// title is the relative path.
if (onSelect) {
onSelect(data.result.title);
}
setDirectoryName(data.result.title);
}
const handleClear = (e) => {
if (e) {
e.preventDefault();
}
onSelect('');
setDirectoryName('');
}
return <Grid>
<Grid.Row>
<Grid.Column mobile={13} tablet={14} computer={15}>
<Search category
placeholder='Search directory names...'
onSearchChange={handleSearchChange}
onResultSelect={handleResultSelect}
loading={loading}
value={directoryName}
results={results}
{...props}
/>
</Grid.Column>
<Grid.Column mobile={3} tablet={2} computer={1}>
<Button secondary icon='close' onClick={handleClear}/>
</Grid.Column>
</Grid.Row>
</Grid>
}
export function DirectoryInput({disabled, error, placeholder, setInput, value, required, isDirectory}) {
const {directory, directories, setDirectory, isDir} = useDirectories(value);
const {settings} = useSettings();
@ -1009,7 +1136,7 @@ export function SortButton({sorts = []}) {
let sortFields;
if (sorts && sorts.length) {
sortFields = sorts.map((i) => {
return <SButton key={i['value']} onClick={() => handleSortButton(i['value'])}>{i['text']}</SButton>
return <Button key={i['value']} onClick={() => handleSortButton(i['value'])}>{i['text']}</Button>
})
}
@ -1018,10 +1145,10 @@ export function SortButton({sorts = []}) {
open={open}
onClose={() => setOpen(false)}
>
<Modal.Header>Sort By</Modal.Header>
<Modal.Content>
<ModalHeader>Sort By</ModalHeader>
<ModalContent>
{sortFields}
</Modal.Content>
</ModalContent>
</Modal>
<ButtonGroup icon>
<Button icon={desc ? 'sort down' : 'sort up'} onClick={() => toggleDesc()}/>

View file

@ -1,7 +1,7 @@
import React, {useState} from "react";
import {PageContainer, useTitle} from "./Common";
import {Button, Header, Segment, Table} from "./Theme";
import {Icon, Modal, TableBody, TableCell, TableRow} from "semantic-ui-react";
import {Button, Header, Modal, ModalContent, ModalHeader, Segment, Table} from "./Theme";
import {Icon, TableBody, TableCell, TableRow} from "semantic-ui-react";
import QRCode from "react-qr-code";
const bitcoinAddress = '1mi1ddLSd6LmkuwJd1ttZsKuED724vYjS';
@ -26,10 +26,12 @@ function CoinQRButton({qrCodeValue, header, buttonColor}) {
onClose={() => setOpen(false)}
onOpen={() => setOpen(true)}
>
<Modal.Header>{header}</Modal.Header>
<Modal.Content>
<QRCode value={qrCodeValue} size={300}/>
</Modal.Content>
<ModalHeader>{header}</ModalHeader>
<ModalContent>
<div style={{backgroundColor: '#FFFFFF', display: 'inline-block', padding: '1em'}}>
<QRCode value={qrCodeValue} size={300}/>
</div>
</ModalContent>
</Modal>
</>
}

View file

@ -1,13 +1,14 @@
import React, {useContext, useState} from "react";
import {getDownloaders, postDownload} from "../api";
import {DirectoryInput, frequencyOptions, HelpPopup, rssFrequencyOptions, WROLModeMessage} from "./Common";
import {DirectorySearch, frequencyOptions, HelpPopup, rssFrequencyOptions, WROLModeMessage} from "./Common";
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {ThemeContext} from "../contexts/contexts";
import {Accordion, Button, Form, FormField, FormGroup, FormInput, Header, Loader, Segment, TextArea} from "./Theme";
import {AccordionContent, AccordionTitle, FormDropdown} from "semantic-ui-react";
import {Accordion, Button, Form, FormInput, Header, Loader, TextArea} from "./Theme";
import {AccordionContent, AccordionTitle, Form as SForm, FormDropdown} from "semantic-ui-react";
import {Link} from "react-router-dom";
import {TagsSelector} from "../Tags";
import {toast} from "react-semantic-toasts-2";
const validUrl = /^(http|https):\/\/[^ "]+$/;
@ -24,6 +25,7 @@ class Downloader extends React.Component {
tagNames: [],
urls: '',
valid: true,
destinationRequired: props.destinationRequired !== undefined ? props.destinationRequired : false,
};
}
@ -35,6 +37,8 @@ class Downloader extends React.Component {
let response = await postDownload(urls, downloader, null, null, null, destination, tagNames);
if (response.status === 204) {
this.setState({urls: '', pending: false, submitted: true});
} else {
toast({type: 'error', title: 'Error', description: 'Failed to create download!'});
}
} finally {
this.setState({pending: false});
@ -46,10 +50,6 @@ class Downloader extends React.Component {
this.setState({[name]: value}, this.validateUrls);
}
handleDestinationChange = (value) => {
this.setState({destination: value});
}
validateUrls = async () => {
// Validate that all URLs in state are valid.
if (this.state.urls) {
@ -84,23 +84,31 @@ class Downloader extends React.Component {
}
render() {
let disabled = !this.state.urls || !this.state.valid || this.state.pending || !this.state.downloader;
const {advancedOpen, destination, submitted, tagNames} = this.state;
const {header, withDestination, withTags} = this.props;
const {
advancedOpen,
submitted,
tagNames,
urls,
valid,
pending,
downloader,
destination,
destinationRequired
} = this.state;
let disabled = !urls || !valid || pending || !downloader;
const {header, withTags, withSearchDirectory} = this.props;
let destinationField;
if (withDestination) {
destinationField = <FormGroup>
<FormField width={16}>
let directorySearch;
if (withSearchDirectory) {
directorySearch = <div style={{marginTop: '1em'}}>
<SForm.Field required={destinationRequired}>
<label>Destination</label>
<DirectoryInput
isDirectory={true}
value={destination}
setInput={this.handleDestinationChange}
placeholder='download/into/custom/directory'
/>
</FormField>
</FormGroup>;
<DirectorySearch onSelect={i => this.setState({destination: i})}/>
</SForm.Field>
</div>;
}
if (destinationRequired) {
disabled = !destination || disabled;
}
let tagsSelector;
@ -109,15 +117,12 @@ class Downloader extends React.Component {
}
const advancedAccordion = <Accordion>
<AccordionTitle onClick={() => this.setState({advancedOpen: !advancedOpen})}>
<Icon name='dropdown'/>Advanced
<AccordionTitle active={advancedOpen} onClick={() => this.setState({advancedOpen: !advancedOpen})}>
<Icon name='dropdown'/>
Advanced
</AccordionTitle>
<AccordionContent active={advancedOpen}>
<Segment>
{destinationField}
{withDestination && withTags && <br/>}
{tagsSelector}
</Segment>
{directorySearch}
</AccordionContent>
</Accordion>;
@ -132,8 +137,14 @@ class Downloader extends React.Component {
name='urls'
onChange={this.handleInputChange}
value={this.state.urls}
style={{marginBottom: '1em'}}
/>
{(withDestination || withTags) && advancedAccordion}
{tagsSelector}
{pending && <Loader active={pending}/>}
{/* Display DirectorySearch if its required, show it in an Accordion if it's not required */}
{withSearchDirectory ? destinationRequired ? directorySearch : advancedAccordion : null}
<Button content='Cancel' onClick={this.props.clearSelected}/>
<Button primary style={{marginTop: '1em'}} disabled={disabled}>Download</Button>
{submitted && viewDownloads}
@ -195,7 +206,7 @@ class ChannelDownload extends React.Component {
return <Form>
<WROLModeMessage content='Downloading is disabled while WROL Mode is enabled'/>
<Header as='h4'><Icon name='film'/> Channel / Playlist</Header>
<Header as='h4'><Icon name='film' color='blue'/> Channel / Playlist</Header>
<FormInput
required
label='URL'
@ -225,7 +236,7 @@ class ChannelDownload extends React.Component {
onClick={this.handleSubmit}
disabled={buttonDisabled}
/>
<Loader active={pending}/>
{pending && <Loader active={pending}/>}
{success && <Icon name='check'/>}
</Form>
}
@ -318,7 +329,7 @@ class RSSDownload extends ChannelDownload {
return <Form>
<WROLModeMessage content='Downloading is disabled while WROL Mode is enabled'/>
<Header as='h4'><Icon name='rss'/> RSS Feed</Header>
<Header as='h4'><Icon name='rss' color='orange'/> RSS Feed</Header>
<FormInput
required
label='URL'
@ -373,7 +384,7 @@ class RSSDownload extends ChannelDownload {
onClick={this.handleSubmit}
disabled={buttonDisabled}
/>
<Loader active={pending}/>
{pending && <Loader active={pending}/>}
{success && <Icon name='check'/>}
</Form>
}
@ -397,7 +408,7 @@ export function DownloadMenu({onOpen, disabled}) {
/>
<Button
color='green'
content='Archive'
content='Archives'
disabled={disabled}
onClick={() => localOnOpen('archive')}
style={{marginBottom: '1em'}}
@ -415,6 +426,13 @@ export function DownloadMenu({onOpen, disabled}) {
onClick={() => localOnOpen('rss')}
style={{marginBottom: '1em'}}
/>
<Button
color='black'
content='File'
disabled={disabled}
onClick={() => localOnOpen('file')}
style={{marginBottom: '1em'}}
/>
</>);
function clearSelected() {
@ -425,25 +443,29 @@ export function DownloadMenu({onOpen, disabled}) {
const downloaders = {
archive: <Downloader
clearSelected={clearSelected}
header={<><Icon name='file alternate'/> Archive</>}
header={<><Icon name='file alternate' color='green'/> Archives</>}
downloader='archive'
withTags={true}/>,
video: <Downloader
clearSelected={clearSelected}
header={<><Icon name='film'/> Videos</>}
header={<><Icon name='film' color='blue'/> Videos</>}
downloader='video'
withDestination={true}
withSearchDirectory={true}
withTags={true}/>,
video_channel: <ChannelDownload clearSelected={clearSelected}/>,
rss: <RSSDownload
rss: <RSSDownload clearSelected={clearSelected}/>,
file: <Downloader
clearSelected={clearSelected}
header={<><Icon name='rss square'/> RSS</>}/>,
header={<><Icon name='file'/> Files</>}
downloader='file'
withSearchDirectory={true}
destinationRequired={true}
withTags={true}/>,
};
body = downloader in downloaders ? downloaders[downloader] : body;
return <>
<Header as='h2'>Download</Header>
{body}
</>
}

View file

@ -1,20 +1,16 @@
import React from "react";
import _ from "lodash";
import {
Button as SButton,
Grid,
Header as SHeader,
Image,
Modal,
ModalActions,
ModalContent,
ModalHeader
} from "semantic-ui-react";
import {TagsSelector} from "../Tags";
import {Media} from "../contexts/contexts";
import {encodeMediaPath} from "./Common";
import {addTag, fetchFile, removeTag} from "../api";
import {StlViewer} from "react-stl-viewer";
import {Button, Modal, ModalActions, ModalContent, ModalHeader} from "./Theme";
function getMediaPathURL(previewFile) {
if (previewFile['primary_path']) {
@ -47,6 +43,8 @@ function getIframeModal(previewFile) {
<iframe title='textModal' src={url}
style={{
height: '100%', width: '100%', border: 'none', position: 'absolute', top: 0,
// Use white to avoid iframe displaying with dark-theme.
backgroundColor: '#ffffff',
}}/>
</div>
</ModalContent>
@ -201,16 +199,16 @@ export function FilePreviewWrapper({children}) {
function setModalContent(content, url, downloadURL) {
let openButton;
if (url) {
openButton = <SButton color='blue' as='a' href={url}>Open</SButton>;
openButton = <Button color='blue' as='a' href={url}>Open</Button>;
}
const closeButton = <SButton onClick={handleClose}>Close</SButton>;
const closeButton = <Button onClick={handleClose}>Close</Button>;
const tagsDisplay = <TagsSelector selectedTagNames={previewFile['tags']} onAdd={localAddTag}
onRemove={localRemoveTag}/>;
let downloadButton;
if (downloadURL) {
downloadButton = <SButton color='yellow' as='a' href={downloadURL} floated='left'>
downloadButton = <Button color='yellow' as='a' href={downloadURL} floated='left'>
Download
</SButton>;
</Button>;
}
setPreviewModal(<Modal closeIcon

View file

@ -1,6 +1,5 @@
import React, {useContext, useState} from "react";
import {
Button as SButton,
CardContent,
CardDescription,
CardHeader,
@ -11,7 +10,6 @@ import {
Form,
Image,
Label,
Modal,
PlaceholderLine,
TableCell,
TableHeaderCell,
@ -46,7 +44,19 @@ import {CardPlacholder} from "./Placeholder";
import {ArchiveCard, ArchiveRowCells} from "./Archive";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {Media, StatusContext, ThemeContext} from "../contexts/contexts";
import {Button, Card, CardIcon, Icon, Placeholder, Progress, Segment} from "./Theme";
import {
Button,
Card,
CardIcon,
Icon,
Modal,
ModalActions,
ModalContent,
ModalHeader,
Placeholder,
Progress,
Segment
} from "./Theme";
import {SelectableTable} from "./Tables";
import {VideoCard, VideoRowCells} from "./Videos";
import _ from 'lodash';
@ -395,16 +405,16 @@ export function SearchFilter({filters = [], modalHeader}) {
if (filters && filters.length > 0) {
return <>
<Modal open={open} onOpen={() => handleOpen()} onClose={() => setOpen(false)} closeIcon>
{modalHeader || <Modal.Header>Filter</Modal.Header>}
<Modal.Content>
{modalHeader || <ModalHeader>Filter</ModalHeader>}
<ModalContent>
<Form>
{filterFields}
</Form>
</Modal.Content>
<Modal.Actions>
<SButton onClick={handleClear} secondary>Clear</SButton>
<SButton onClick={() => setOpen(false)}>Close</SButton>
</Modal.Actions>
</ModalContent>
<ModalActions>
<Button onClick={handleClear} secondary>Clear</Button>
<Button onClick={() => setOpen(false)}>Close</Button>
</ModalActions>
</Modal>
<Button
icon='filter'

View file

@ -1,5 +1,5 @@
import {CardIcon, Header, Placeholder, Segment} from "./Theme";
import {PlaceholderLine, Item, Image} from "semantic-ui-react";
import {Placeholder, Segment} from "./Theme";
import {Image, PlaceholderLine} from "semantic-ui-react";
import React from "react";
import {ThemeContext} from "../contexts/contexts";
import {encodeMediaPath, FileIcon, findPosterPath, PreviewLink} from "./Common";

View file

@ -1,6 +1,6 @@
import React from "react";
import {Icon} from "./Theme";
import {Button as SButton, Header, IconGroup, Modal} from "semantic-ui-react";
import {Button, Header, Icon, Modal, ModalActions, ModalContent, ModalHeader} from "./Theme";
import {IconGroup} from "semantic-ui-react";
import QRCode from "react-qr-code";
import {sendNotification} from "../api";
@ -12,13 +12,13 @@ export const ShareEveryoneButton = ({url, size, onClick}) => {
}
}
return <SButton
return <Button
color='violet'
size={size}
onClick={handleShare}
>
Share with everyone
</SButton>
</Button>
}
export const ShareButton = () => {
@ -43,15 +43,17 @@ export const ShareButton = () => {
open={open}
onClose={handleClose}
>
<Modal.Header>Share this page</Modal.Header>
<Modal.Content>
<ModalHeader>Share this page</ModalHeader>
<ModalContent>
<Header as='h4'>Another user can scan this QR code to view this page</Header>
<QRCode value={window.location.href}/>
</Modal.Content>
<Modal.Actions>
<div style={{padding: '1em', backgroundColor: '#ffffff', display: "inline-block"}}>
<QRCode value={window.location.href}/>
</div>
</ModalContent>
<ModalActions>
<ShareEveryoneButton url={window.location.href} onClick={handleClose}/>
<SButton onClick={handleClose}>Close</SButton>
</Modal.Actions>
<Button onClick={handleClose}>Close</Button>
</ModalActions>
</Modal>
<a href='#' onClick={handleOpen}>
<IconGroup size='large'>

View file

@ -1,27 +1,33 @@
import React, {useContext, useEffect, useState} from 'react';
import {ThemeContext} from "../contexts/contexts";
import {
Accordion as ACCORDION,
Button as BUTTON,
Card as CARD,
Form as FORM,
FormField as FORM_FIELD,
FormGroup as FORM_GROUP,
FormInput as FORM_INPUT,
Header as HEADER,
Icon as ICON,
Loader as LOADER,
Menu as MENU,
Placeholder as PLACEHOLDER,
Popup as POPUP,
Progress as PROGRESS,
Segment as SEGMENT,
Statistic as STATISTIC,
StatisticGroup as STATISTIC_GROUP,
Tab as TAB,
Table as TABLE,
TabPane as TAB_PANE,
TextArea as TEXTAREA
Accordion as SAccordion,
Button as SButton,
Card as SCard,
Divider as SDivider,
Form as SForm,
FormField as SFormField,
FormGroup as SFormGroup,
FormInput as SFormInput,
Header as SHeader,
Icon as SIcon,
Loader as SLoader,
Menu as SMenu,
Modal as SModal,
ModalActions as SModalActions,
ModalContent as SModalContent,
ModalDescription as SModalDescription,
ModalHeader as SModalHeader,
Placeholder as SPlaceholder,
Popup as SPopup,
Progress as SProgress,
Segment as SSegment,
Statistic as SStatistic,
StatisticGroup as SStatisticGroup,
Tab as STab,
Table as STable,
TabPane as STabPane,
TextArea as STextArea
} from "semantic-ui-react";
import {ColorToSemanticHexColor} from "./Common";
import _ from "lodash";
@ -56,7 +62,7 @@ export function ThemeWrapper({children, ...props}) {
console.debug('setDarkTheme');
setI({inverted: true});
setS({style: {backgroundColor: '#1B1C1D', color: '#dddddd'}});
setT({style: {color: '#dddddd'}});
setT({style: {color: '#eeeeee'}});
setInverted('inverted');
setTheme(darkTheme);
document.body.style.background = '#171616';
@ -161,125 +167,158 @@ const defaultGrey = (props, inverted) => {
return props;
}
export function Button(props) {
const {i, inverted} = useContext(ThemeContext);
props = defaultGrey({...i, ...props}, inverted);
return <BUTTON {...props}/>
}
export function Accordion(props) {
const {i, inverted} = useContext(ThemeContext);
props = defaultGrey({...i, ...props}, inverted);
return <ACCORDION {...props}/>
return <SAccordion {...props}/>
}
export function Button(props) {
const {i, inverted} = useContext(ThemeContext);
props = defaultGrey({...i, ...props}, inverted);
return <SButton {...props}/>
}
export function Divider({...props}) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <SDivider {...props}/>
}
export function Header(props) {
const {i, inverted} = useContext(ThemeContext);
props = defaultGrey({...i, ...props}, inverted);
return <HEADER {...props}/>
const {t, inverted} = useContext(ThemeContext);
props = defaultGrey({...t, ...props}, inverted);
return <SHeader {...props}/>
}
export function Form(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <FORM {...props}/>
return <SForm {...props}/>
}
export function FormField(props) {
const {i} = useContext(ThemeContext);
props = invertedNull({...i, ...props});
return <FORM_FIELD {...props}/>
return <SFormField {...props}/>
}
export function FormGroup(props) {
const {i} = useContext(ThemeContext);
props = invertedNull({...i, ...props});
return <FORM_GROUP {...props}/>
return <SFormGroup {...props}/>
}
export function FormInput(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <FORM_INPUT {...props}/>
return <SFormInput {...props}/>
}
export function Icon(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <ICON {...props}/>
return <SIcon {...props}/>
}
export function Loader(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <LOADER {...props}/>
return <SLoader {...props}/>
}
export function Menu(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <MENU {...props}/>
return <SMenu {...props}/>
}
export function Modal({...props}) {
const {inverted} = useContext(ThemeContext);
return <SModal {...props} className={`${inverted}`}/>
}
export function ModalActions({...props}) {
const {inverted} = useContext(ThemeContext);
return <SModalActions {...props} className={`${inverted}`}/>
}
export function ModalContent({...props}) {
const {inverted} = useContext(ThemeContext);
return <SModalContent {...props} className={`${inverted}`}/>
}
export function ModalDescription({...props}) {
const {inverted} = useContext(ThemeContext);
return <SModalDescription {...props} className={`${inverted}`}/>
}
export function ModalHeader({...props}) {
const {inverted} = useContext(ThemeContext);
return <SModalHeader {...props} className={`${inverted}`}>
{props.children}
</SModalHeader>
}
export function Placeholder(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <PLACEHOLDER {...props}/>
return <SPlaceholder {...props}/>
}
export function Popup(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <POPUP {...props}/>
return <SPopup {...props}/>
}
export function Progress(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <PROGRESS {...props}/>
return <SProgress {...props}/>
}
export function Segment(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <SEGMENT {...props}/>
return <SSegment {...props}/>
}
export function Statistic(props) {
const {i, inverted} = useContext(ThemeContext);
props = defaultGrey({...i, ...props}, inverted);
return <STATISTIC {...props}/>
return <SStatistic {...props}/>
}
export function StatisticGroup(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
props['style'] = {...props['style'], marginLeft: 0, marginRight: 0};
return <STATISTIC_GROUP {...props}/>
return <SStatisticGroup {...props}/>
}
export function Tab(props) {
const {i} = useContext(ThemeContext);
props = invertedNull({...i, ...props});
return <TAB {...props}/>
return <STab {...props}/>
}
export function TabPane(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <TAB_PANE {...props}/>
return <STabPane {...props}/>
}
export function Table(props) {
const {i} = useContext(ThemeContext);
props = {...i, ...props};
return <TABLE {...props}/>
return <STable {...props}/>
}
export function TextArea(props) {
const {i} = useContext(ThemeContext);
props = invertedNull({...i, ...props});
return <TEXTAREA {...props}/>
return <STextArea {...props}/>
}
export function CardIcon({onClick, ...props}) {
@ -304,5 +343,5 @@ export function Card({color, ...props}) {
const borderColor = inverted ? '#888' : '#ddd';
props['style']['boxShadow'] = `0 0 0 2px ${borderColor}, 0 5px 0 0 ${emphasisColor}, 0 0px 3px 0 #d4d4d5`;
}
return <CARD {...props}/>
return <SCard {...props}/>
}

View file

@ -0,0 +1,110 @@
import React from "react";
import {useDropzone} from "react-dropzone";
import {Form, Header, Icon, Progress, Segment} from "./Theme";
import {DirectorySearch, mimetypeIconName} from "./Common";
import {useUploadFile} from "../hooks/customHooks";
import _ from "lodash";
import {ThemeContext} from "../contexts/contexts";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {Form as SForm} from "semantic-ui-react";
export function Upload({disabled}) {
const {setFiles, progresses, destination, setDestination, doClear} = useUploadFile();
const {t} = React.useContext(ThemeContext);
const onDrop = React.useCallback(async (acceptedFiles) => {
if (acceptedFiles && acceptedFiles.length > 0) {
setFiles(acceptedFiles);
}
}, [destination]);
const {getRootProps, getInputProps} = useDropzone({onDrop});
React.useEffect(() => {
if (!destination) {
// Destination was cleared, clear progresses.
doClear();
}
}, [destination]);
const handleDestination = (value) => {
setDestination(value);
}
let progressBars;
if (!_.isEmpty(progresses)) {
progressBars = Object.entries(progresses).map(([name, value]) => {
const {percent, status, type} = value;
let color = 'grey';
let statusString;
if (status === 'complete') {
color = 'green';
statusString = 'Complete:';
} else if (status === 'pending') {
statusString = 'Pending:';
} else if (status === 'failed') {
color = 'red';
statusString = 'Failed:';
} else if (status === 'conflicting') {
color = 'orange';
statusString = 'Already Exists:';
}
// Only indicating when upload is pending.
let indicating = status === 'pending';
return <Grid.Row key={name}>
<Grid.Column mobile={3} tablet={2} computer={1}>
<Icon name={mimetypeIconName(type, name.toLowerCase())} size='big'/>
</Grid.Column>
<Grid.Column mobile={13} tablet={14} computer={15}>
<Progress progress indicating={indicating} percent={percent} color={color}>
<p {...t}>{statusString} {name}</p>
</Progress>
</Grid.Column>
</Grid.Row>
})
}
return <>
<Form>
<SForm.Field required>
<label>Destination</label>
<DirectorySearch onSelect={handleDestination} disabled={disabled}/>
</SForm.Field>
</Form>
{destination ?
<Segment>
<Grid columns={1}>
<Grid.Row>
<Grid.Column style={{padding: '1em'}}>
<Form onSubmit={() => {
}}>
<div {...getRootProps()}>
<input {...getInputProps()}/>
<Grid textAlign='center'>
<Grid.Row>
<Grid.Column>
<Header icon>
<Icon name='file text'/>
Click here, or drop files here to upload
</Header>
</Grid.Column>
</Grid.Row>
</Grid>
</div>
</Form>
</Grid.Column>
</Grid.Row>
</Grid>
</Segment>
: <Header as='h3'>You must search for a directory to place your files</Header>
}
<br/>
<Grid columns={2}>
{progressBars}
</Grid>
</>
}

View file

@ -13,7 +13,6 @@ import {
Button as SButton,
Confirm,
Loader,
Modal,
PlaceholderLine,
TableBody,
TableCell,
@ -21,7 +20,7 @@ import {
TableHeaderCell,
TableRow
} from "semantic-ui-react";
import {Button, Header, Placeholder, Segment, Table} from "../Theme";
import {Button, Header, Modal, ModalActions, ModalContent, ModalHeader, Placeholder, Segment, Table} from "../Theme";
import {useDownloads} from "../../hooks/customHooks";
import _ from "lodash";
@ -135,13 +134,13 @@ class DownloadRow extends React.Component {
open={errorModalOpen}
trigger={<Button icon='exclamation circle' color='orange'/>}
>
<Modal.Header>Download Error</Modal.Header>
<Modal.Content>
<ModalHeader>Download Error</ModalHeader>
<ModalContent>
<pre style={{overflowX: 'scroll'}}>{error}</pre>
</Modal.Content>
<Modal.Actions>
</ModalContent>
<ModalActions>
<SButton onClick={() => this.setState({errorModalOpen: false})}>Close</SButton>
</Modal.Actions>
</ModalActions>
</Modal>;
const deleteButton = <>
@ -242,7 +241,8 @@ class StoppableRow extends React.Component {
let {url, last_successful_download, status, location, error} = this.props;
let {stopOpen, startOpen, errorModalOpen, deleteOpen} = this.state;
const link = location ?
// Open downloads (/download), or external links in an anchor.
const link = location && !location.startsWith('/download') ?
(text) => <Link to={location}>{text}</Link> :
(text) => <a href={url} target='_blank' rel='noopener noreferrer'>{text}</a>;
@ -301,13 +301,13 @@ class StoppableRow extends React.Component {
open={errorModalOpen}
trigger={<Button icon='exclamation circle' color='red'/>}
>
<Modal.Header>Download Error</Modal.Header>
<Modal.Content>
<ModalHeader>Download Error</ModalHeader>
<ModalContent>
<pre style={{overflowX: 'scroll'}}>{error}</pre>
</Modal.Content>
<Modal.Actions>
</ModalContent>
<ModalActions>
<SButton onClick={() => this.setState({errorModalOpen: false})}>Close</SButton>
</Modal.Actions>
</ModalActions>
</Modal>
)
}

View file

@ -1,7 +1,7 @@
import React from "react";
import {getSettings, saveSettings} from "../../api";
import {Button, Form, FormGroup, FormInput, Header, Loader, Segment} from "../Theme";
import {Container, Dimmer, Icon, Modal} from "semantic-ui-react";
import {Button, Form, FormGroup, FormInput, Header, Loader, Modal, ModalContent, ModalHeader, Segment} from "../Theme";
import {Container, Dimmer, Icon} from "semantic-ui-react";
import {ThemeContext} from "../../contexts/contexts";
import {HelpPopup, HotspotToggle, ThrottleToggle, Toggle, WROLModeMessage} from "../Common";
import QRCode from "react-qr-code";
@ -203,12 +203,14 @@ export class Settings extends React.Component {
open={this.state.qrOpen}
trigger={qrButton}
>
<Modal.Header>
<ModalHeader>
Scan this code to join the hotspot
</Modal.Header>
<Modal.Content>
<QRCode value={qrCodeValue} size={300}/>
</Modal.Content>
</ModalHeader>
<ModalContent>
<div style={{display: 'inline-block', backgroundColor: '#ffffff', padding: '1em'}}>
<QRCode value={qrCodeValue} size={300}/>
</div>
</ModalContent>
</Modal>
<br/>

View file

@ -17,6 +17,7 @@ import {
getVideo,
getVideosStatistics,
searchArchives,
searchDirectories,
searchVideos,
setHotspot,
setThrottle,
@ -622,6 +623,39 @@ export const useDirectories = (defaultDirectory) => {
return {directory, directories, setDirectory, exists, isDir, isFile};
}
export const useSearchDirectories = (value) => {
const [directoryName, setDirectoryName] = useState(value ?? '');
const [directories, setDirectories] = useState(null);
const [channelDirectories, setChannelDirectories] = useState(null);
const [domainDirectories, setDomainDirectories] = useState(null);
const [loading, setLoading] = useState(false);
const localSearchDirectories = async () => {
setLoading(true);
try {
const {directories: dirs, channel_directories, domain_directories} = await searchDirectories(directoryName);
setDirectories(dirs);
setChannelDirectories(channel_directories);
setDomainDirectories(domain_directories);
} finally {
setLoading(false);
}
};
useEffect(() => {
localSearchDirectories();
}, [directoryName]);
return {
directoryName,
setDirectoryName,
directories,
loading,
channelDirectories,
domainDirectories
}
}
export const useSettings = () => {
const [settings, setSettings] = useState({});
@ -646,6 +680,12 @@ export const useSettings = () => {
return {settings, fetchSettings};
}
export const useMediaDirectory = () => {
const {settings} = useSettings();
return settings['media_directory'];
}
export const useSettingsInterval = () => {
const {settings, fetchSettings} = useSettings();
@ -769,3 +809,114 @@ export const useSearchOrder = () => {
const [sort, setSort] = useOneQuery('order');
return {sort, setSort}
}
export const useUploadFile = () => {
const [files, setFiles] = useState([]);
const [progresses, setProgresses] = useState({});
const [destination, setDestination] = useState('');
const handleProgress = (name, chunk, totalChunks, status, type) => {
const percent = Math.round((100 * chunk) / totalChunks);
const newProgress = {[name]: {percent, status, type}};
setProgresses(prevState => ({...prevState, ...newProgress}));
}
const handleFilesChange = (newFiles) => {
let newProgresses = {};
newFiles.map(i => {
newProgresses = {...newProgresses, [i['name']]: {percent: 0, status: 'pending'}};
});
setProgresses(newProgresses);
setFiles(newFiles);
}
const uploadChunk = async (file, chunkNum, chunkSize, totalChunks, tries, maxTries) => {
if (tries > maxTries) {
console.error(`Exceeded max tries ${maxTries}`);
}
const start = chunkNum * chunkSize;
const end = Math.min(start + chunkSize, file.size);
// The bytes that we will send.
const chunk = file.slice(start, end);
const formData = new FormData();
formData.append('chunkNumber', chunkNum.toString());
formData.append('filename', file.path);
formData.append('totalChunks', totalChunks.toString());
formData.append('destination', destination);
// Send the size that we're actually sending.
formData.append('chunkSize', chunk.size.toString());
formData.append('chunk', chunk);
console.debug(`file upload: tries=${tries} chunkNum=${chunkNum} totalChunks=${totalChunks} chunkSize=${chunk.size} destination=${destination}`);
const xhr = new XMLHttpRequest();
xhr.open('POST', '/api/files/upload', true);
xhr.onreadystatechange = async () => {
if (xhr.readyState === 4) {
if (xhr.status === 200 || xhr.status === 416) {
const data = JSON.parse(xhr.responseText);
handleProgress(file.name, chunkNum, totalChunks, 'pending', file.type);
const expectedChunk = data['expected_chunk'];
if (xhr.status === 416) {
console.log(`Server requested a different chunk ${chunkNum}`);
await uploadChunk(file, expectedChunk, chunkSize, totalChunks, tries + 1, maxTries);
} else {
console.debug(`Uploading of chunk ${chunkNum} succeeded, got request for chunk ${chunkNum}`);
// Success, reset tries.
await uploadChunk(file, expectedChunk, chunkSize, totalChunks, 0, maxTries);
}
} else if (xhr.status === 201) {
handleProgress(file.name, totalChunks, totalChunks, 'complete', file.type);
console.log(`Uploading of ${file.path} completed.`);
} else if (xhr.status === 400) {
handleProgress(file.name, totalChunks, totalChunks, 'conflicting', file.type);
const data = JSON.parse(xhr.responseText);
if (data['code'] === 41) {
console.error('File already exists. Giving up.');
}
} else {
handleProgress(file.name, totalChunks, totalChunks, 'failed', file.type);
console.error(`Failed to upload chunk ${chunkNum}. Giving up.`);
}
}
}
await xhr.send(formData);
};
const doUpload = async () => {
if (!files || files.length === 0 || !destination) {
return;
}
for (let i = 0; i < files.length; i++) {
const file = files[i];
console.log(`Starting file upload`);
console.log(file);
const chunkNum = 0;
const chunkSize = 10 * 1024 * 1024; // 10MB
const totalChunks = Math.ceil(file.size / chunkSize);
const tries = 0;
const maxTries = 20;
// Start recursive function to upload the file.
await uploadChunk(file, chunkNum, chunkSize, totalChunks, tries, maxTries);
}
// Clear form after upload.
setFiles([]);
}
const doClear = () => {
setFiles([]);
setProgresses({});
}
useEffect(() => {
doUpload()
}, [JSON.stringify(files)]);
return {files, setFiles: handleFilesChange, progresses, destination, setDestination, doClear, doUpload}
}

View file

@ -53,7 +53,6 @@ class ArchiveDownloader(Downloader, ABC):
return archives
# Archive downloader is the last downloader which should be used.
archive_downloader = ArchiveDownloader()

View file

@ -11,6 +11,7 @@ from typing import Optional, Tuple, List
from bs4 import BeautifulSoup
from selenium import webdriver
from sqlalchemy import asc
from sqlalchemy.orm import Session
from modules.archive.models import Domain, Archive
@ -440,3 +441,13 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
results, total = handle_file_group_search_results(stmt, params)
return results, total
@optional_session
def search_domains_by_directory(name: str, limit: int = 5, session: Session = None) -> List[Domain]:
domains = session.query(Domain) \
.filter(Domain.directory.ilike(f'%{name}%')) \
.order_by(asc(Domain.domain)) \
.limit(limit) \
.all()
return domains

View file

@ -127,7 +127,7 @@ class InventoriesConfig(ConfigFile):
self.update({'version': value})
INVENTORIES_CONFIG: InventoriesConfig = InventoriesConfig(global_=True)
INVENTORIES_CONFIG: InventoriesConfig = InventoriesConfig()
TEST_INVENTORIES_CONFIG: InventoriesConfig = None

View file

@ -1,6 +1,8 @@
import pathlib
from pathlib import Path
from typing import List, Dict, Union
from sqlalchemy import asc
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound
@ -194,3 +196,13 @@ def download_channel(id_: int):
raise InvalidDownload(f'Channel {channel.name} does not have a download!')
download.renew(reset_attempts=True)
session.commit()
@optional_session
def search_channels_by_directory(name: str, limit: int = 5, session: Session = None) -> List[Channel]:
channels = session.query(Channel) \
.filter(Channel.directory.ilike(f'%{name}%')) \
.order_by(asc(Channel.name)) \
.limit(limit) \
.all()
return channels

View file

@ -208,8 +208,13 @@ class VideoDownloader(Downloader, ABC):
if not channel and (local_channel_id or channel_url or destination):
# Could not find channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
logger.info(f'Using download.settings to find channel')
channel = get_channel(channel_id=local_channel_id, url=channel_url, directory=destination,
return_dict=False)
try:
channel = get_channel(channel_id=local_channel_id, url=channel_url, directory=destination,
return_dict=False)
except UnknownChannel:
# We do not need a Channel if we have a destination directory.
if not destination:
raise
if channel:
found_channel = 'download_settings'
@ -297,8 +302,10 @@ class VideoDownloader(Downloader, ABC):
session.commit()
if download.settings and (tag_names := download.settings.get('tag_names')):
existing_names = video.file_group.tag_names
for name in tag_names:
video.add_tag(name)
if name not in existing_names:
video.add_tag(name)
except UnrecoverableDownloadError:
raise

View file

@ -211,7 +211,7 @@ class ChannelsConfig(ConfigFile):
self.update({'favorites': value})
CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig(global_=True)
CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig()
TEST_CHANNELS_CONFIG: ChannelsConfig = None
@ -328,7 +328,7 @@ class VideoDownloaderConfig(ConfigFile):
self.update({'youtube_include_dash_manifest': value})
VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig(global_=True)
VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig()
TEST_VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = None

View file

@ -5,6 +5,8 @@ events {
}
http {
client_max_body_size 0;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;

View file

@ -4,6 +4,8 @@ events {
http {
client_max_body_size 0;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;

View file

@ -25,4 +25,3 @@ srt==3.5.1
websockets==10.1
webvtt-py==0.4.6
yt-dlp==2023.2.17

View file

@ -51,6 +51,9 @@ NMCLI_BIN = which('nmcli', '/usr/bin/nmcli', flag=flags.nmcli_installed)
CPUFREQ_INFO_BIN = which('cpufreq-info', '/usr/bin/cpufreq-info')
CPUFREQ_SET_BIN = which('cpufreq-set', '/usr/bin/cpufreq-set')
# Files
WGET_BIN = which('wget', '/usr/bin/wget', flag=flags.wget_installed)
# Map
BASH_BIN = which('bash', '/bin/bash', warn=True)

View file

@ -180,28 +180,28 @@ def sanitize_link(link: str) -> str:
class ConfigFile:
"""
This class keeps track of the contents of a config file. You can update the config by calling
.update(), or by creating a property that does so. Save your changes using .save().
This class keeps track of the contents of a config file. You can update the config (and it's file) by
calling .update().
"""
file_name: str = None
default_config: dict = None
width: int = None
def __init__(self, global_: bool = False):
def __init__(self):
self.file_lock = Lock()
self.width = self.width or 90
if PYTEST:
# Do not load a global config on import while testing. A global instance will never be used for testing.
self._config = self.default_config.copy()
return
self.width = self.width or 90
self.initialize()
def __repr__(self):
return f'<{self.__class__.__name__} file={self.get_file()}>'
def initialize(self):
"""Initializes this config dict using the default config and the config file."""
config_file = self.get_file()
self._config = Manager().dict()
# Use the default settings to initialize the config.
@ -261,7 +261,7 @@ class ConfigFile:
self._config.update(config)
self.save()
def dict(self):
def dict(self) -> dict:
"""Get a deepcopy of this config."""
if not hasattr(self, '_config'):
raise NotImplementedError('You cannot use a global config while testing!')
@ -347,7 +347,7 @@ class WROLPiConfig(ConfigFile):
self.update({'wrol_mode': value})
WROLPI_CONFIG: WROLPiConfig = WROLPiConfig(global_=True)
WROLPI_CONFIG: WROLPiConfig = WROLPiConfig()
TEST_WROLPI_CONFIG: WROLPiConfig = None
@ -765,7 +765,7 @@ def walk(path: Path) -> Generator[Path, None, None]:
def get_files_and_directories(directory: Path):
"""Walk a directory, return a tuple of all files and all directories within. (Not recursive)."""
if not directory.exists():
raise ValueError(f'Directory does not exist: {directory}')
raise FileNotFoundError(f'Directory does not exist: {directory}')
if not directory.is_dir():
raise ValueError('Can only walk a directory.')

View file

@ -12,7 +12,7 @@ import tempfile
import zipfile
from abc import ABC
from itertools import zip_longest
from typing import List, Callable, Union, Dict
from typing import List, Callable, Union, Dict, Sequence
from typing import Tuple, Set
from unittest import mock
from unittest.mock import MagicMock
@ -33,7 +33,7 @@ from wrolpi.dates import set_test_now
from wrolpi.db import postgres_engine, get_db_args
from wrolpi.downloader import DownloadManager, DownloadResult, Download, Downloader, \
downloads_manager_config_context
from wrolpi.files import lib as files_lib
from wrolpi.files.models import Directory
from wrolpi.root_api import BLUEPRINTS, api_app
from wrolpi.tags import Tag
from wrolpi.vars import PROJECT_DIR
@ -593,3 +593,32 @@ def zip_file_factory(test_directory):
return fh.read()
return _
@pytest.fixture
def assert_directories(test_session, test_directory):
def _(expected: Sequence[str]):
directories = test_session.query(Directory).all()
names = {str(i.path.relative_to(test_directory)) for i in directories}
assert names == set(expected)
return _
@pytest.fixture
def make_multipart_form():
def _(forms: List[dict]):
boundary = '-----sanic\n'
new_forms = []
for form in forms:
name, value, filename = form['name'], form['value'], form.get('filename')
if filename:
new_forms.append(f'Content-Disposition: form-data; name="{name}"; filename="{filename}"\n\n{value}\n')
else:
new_forms.append(f'Content-Disposition: form-data; name="{name}"\n\n{value}\n')
body = f'\n{boundary}'.join(new_forms)
body = f'{boundary}{body}\n{boundary}'
body = '\r\n'.join(body.split('\n')).strip()
return body
return _

View file

@ -1,5 +1,6 @@
import asyncio
import contextlib
import json
import logging
import multiprocessing
import os
@ -94,6 +95,7 @@ class Download(ModelHelper, Base): # noqa
last_successful_download=self.last_successful_download,
location=self.location,
next_download=self.next_download,
settings=self.settings,
status=self.status,
sub_downloader=self.sub_downloader,
url=self.url,
@ -352,7 +354,6 @@ class DownloadManager:
download.manager = self
downloader: Downloader = download.get_downloader()
logger.warning(f'{downloader=}')
if not downloader:
worker_logger.warning(f'Could not find downloader for {download.downloader=}')
@ -375,7 +376,8 @@ class DownloadManager:
result = DownloadResult(success=False, error=str(traceback.format_exc()))
error_len = len(result.error) if result.error else 0
worker_logger.debug(f'Got success={result.success} from {downloader} with {error_len=}')
worker_logger.debug(
f'Got success={result.success} from {downloader} download_id={download.id} with {error_len=}')
with get_db_session(commit=True) as session:
# Modify the download in a new session because downloads may take a long time.
@ -1064,6 +1066,7 @@ async def save_downloads_config(session: Session):
frequency=download.frequency,
last_successful_download=download.last_successful_download,
next_download=download.next_download,
settings=download.settings,
status=download.status,
sub_downloader=download.sub_downloader,
url=download.url,
@ -1099,7 +1102,7 @@ async def import_downloads_config(session: Session):
existing.next_download = download['next_download']
existing.status = download['status']
existing.sub_downloader = download['sub_downloader']
logger.debug(f'Updating download {existing.url} with {download}')
existing.settings = download['settings'] or dict()
for download in downloads_by_url.values():
# These downloads are new, import them.
@ -1108,6 +1111,7 @@ async def import_downloads_config(session: Session):
frequency=download['frequency'],
last_successful_download=download['last_successful_download'],
next_download=download['next_download'],
settings=download['settings'] or dict(),
status=download['status'],
sub_downloader=download['sub_downloader'],
url=(url := download['url']),

View file

@ -168,6 +168,10 @@ class FileGroupIsTagged(APIError):
pass
class FileUploadFailed(APIError):
pass
error_codes = iter(range(1, 1000))
API_ERRORS = {
@ -371,4 +375,9 @@ API_ERRORS = {
'message': 'File group is tagged',
'status': HTTPStatus.CONFLICT,
},
FileUploadFailed: {
'code': next(error_codes),
'message': 'Failed to upload file',
'status': HTTPStatus.BAD_REQUEST,
},
}

View file

@ -1,3 +1,4 @@
from . import ebooks # noqa
from . import pdfs # noqa
from .api import bp
from . import downloader # noqa

View file

@ -1,17 +1,23 @@
import pathlib
from http import HTTPStatus
from multiprocessing import Manager
from typing import List
import sanic.request
from sanic import response, Request
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory
from wrolpi.errors import InvalidFile
from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory, logger, background_task
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed
from wrolpi.root_api import get_blueprint, json_response
from . import lib, schema
from ..schema import JSONErrorResponse
bp = get_blueprint('Files', '/api/files')
logger = logger.getChild(__name__)
@bp.post('/')
@openapi.definition(
@ -109,6 +115,39 @@ def post_directories(_, body: schema.DirectoriesRequest):
return response.json(body)
@bp.post('/search_directories')
@openapi.definition(
summary='Get all directories whose name matches the provided name.',
body=schema.DirectoriesSearchRequest,
)
@validate(schema.DirectoriesSearchRequest)
async def post_search_directories(_, body: schema.DirectoriesSearchRequest):
if len(body.name) <= 1:
return response.empty()
from modules.videos.channel import lib as channels_lib
channel_directories = channels_lib.search_channels_by_directory(name=body.name)
channel_directories = [dict(path=i.directory, name=i.name) for i in channel_directories]
channel_paths = [i['path'] for i in channel_directories]
from modules.archive import lib as archives_lib
domain_directories = archives_lib.search_domains_by_directory(name=body.name)
domain_directories = [dict(path=i.directory, domain=i.domain) for i in domain_directories]
domain_paths = [i['path'] for i in domain_directories]
from wrolpi.files.models import Directory
directories: List[Directory] = await lib.search_directories(
name=body.name,
excluded=list(map(str, channel_paths + domain_paths)))
body = {
'directories': directories,
'channel_directories': channel_directories,
'domain_directories': domain_directories
}
return json_response(body)
@bp.post('/tag')
@validate(schema.TagFileGroupPost)
async def post_tag_file_group(_, body: schema.TagFileGroupPost):
@ -132,3 +171,101 @@ async def post_tag_file_group(_, body: schema.TagFileGroupPost):
async def post_untag_file_group(_, body: schema.TagFileGroupPost):
await lib.remove_file_group_tag(body.file_group_id, body.file_group_primary_path, body.tag_name, body.tag_id)
return response.empty(HTTPStatus.NO_CONTENT)
# {
# '/media/directory/sub-dir/the-file-name.suffix': 2, # The chunk number we will receive next.
# }
UPLOADED_FILES = Manager().dict()
@bp.post('/upload')
async def post_upload(request: Request):
"""Accepts a multipart/form-data request to upload a single file.
Tracks the number of chunks and will request the correct chunk of the chunks come out of order.
Will not overwrite an existing file, unless a previous upload did not complete."""
try:
destination = request.form['destination'][0]
except Exception as e:
logger.error(f'Failed to get upload destination', exc_info=e)
raise UnknownDirectory('Must provide destination') from e
destination = get_media_directory() / destination
if not destination.is_dir():
raise UnknownDirectory('Destination must be a relative directory that is already in the media directory!')
try:
filename = str(request.form['filename'][0])
except Exception:
raise FileUploadFailed('filename string is required')
try:
chunk_num = int(request.form['chunkNumber'][0])
except Exception:
raise FileUploadFailed('chunkNumber integer is required')
try:
total_chunks = int(request.form['totalChunks'][0])
except Exception:
raise FileUploadFailed('totalChunks integer is required')
filename = pathlib.Path(filename.lstrip('/'))
output = destination / filename
output_str = str(output)
# Chunks start at 0.
expected_chunk_num = UPLOADED_FILES.get(output_str, 0)
logger.debug(f'last_chunk_num is {expected_chunk_num} for {repr(output_str)} received {chunk_num=}')
chunk_size = int(request.form['chunkSize'][0])
# The bytes of the uploading file.
chunk: sanic.request.File = request.files['chunk'][0]
if (body_size := len(chunk.body)) != chunk_size:
raise FileUploadFailed(f'Chunk size does not match the size of the chunk! {chunk_size} != {body_size=}')
if chunk_num == 0 and output.is_file() and output_str in UPLOADED_FILES:
# User attempted to upload this same file, but it did not finish. User has started over again.
logger.info(f'Restarting upload of {repr(output_str)}')
output.unlink()
expected_chunk_num = 0
if expected_chunk_num != chunk_num:
# Respond with a request for the correct chunk number.
return json_response({'expected_chunk': expected_chunk_num}, HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
if chunk_num == 0 and output.is_file():
# Do not overwrite files that already exist.
raise FileUploadFailed('File already exists!')
if output.is_dir():
raise FileUploadFailed('File already exists as a directory!')
if chunk_num == 0:
try:
output.touch()
except FileNotFoundError:
# Uploading a file in a directory. Use mkdir to create all parents, but remove final directory that is
# created because it will be the file.
output.mkdir(parents=True)
output.rmdir()
with output.open('wb') as fh:
fh.write(chunk.body)
else:
with output.open('ab') as fh:
# Finally append the chunk to the file.
fh.write(chunk.body)
# Store what we expect to receive next.
expected_chunk_num += 1
UPLOADED_FILES[output_str] = expected_chunk_num
# Chunks start at 0.
if chunk_num == total_chunks:
# File upload is complete.
del UPLOADED_FILES[output_str]
background_task(lib.refresh_files([output]))
return response.empty(HTTPStatus.CREATED)
# Request the next chunk.
return json_response({'expected_chunk': expected_chunk_num}, HTTPStatus.OK)

175
wrolpi/files/downloader.py Normal file
View file

@ -0,0 +1,175 @@
import asyncio
import pathlib
import re
import traceback
from abc import ABC
from dataclasses import dataclass
from http import HTTPStatus
from typing import List
from urllib.parse import urlparse
import aiohttp
from wrolpi.common import get_media_directory, logger, background_task
from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
__all__ = ['get_download_info', 'download_file', 'FileDownloader', 'file_downloader']
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
@dataclass
class DownloadFileInfo:
name: str = None
size: int = None
type: str = None
accept_ranges: str = None
FILENAME_MATCHER = re.compile(r'.*filename="(.*)"')
async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
"""Gets information (name, size, etc.) about a downloadable file at the provided URL."""
timeout = aiohttp.ClientTimeout(total=timeout) if timeout is not None else None
async with aiohttp.ClientSession(timeout=timeout) as session:
async with session.head(url) as response:
info = DownloadFileInfo(
type=response.headers.get('Content-Type'),
size=int(response.headers['Content-Length']) * 8 if 'Content-Length' in response.headers else None,
accept_ranges=response.headers.get('Accept-Ranges'),
)
disposition = response.headers.get('Content-Disposition')
if disposition and 'filename' in disposition:
if (match := FILENAME_MATCHER.match(disposition)) and (groups := match.groups()):
info.name = groups[0]
else:
# No Content-Disposition with filename, use the URL name.
parsed = urlparse(url)
info.name = parsed.path.split('/')[-1]
return info
async def download_file(url: str, info: DownloadFileInfo, output_path: pathlib.Path):
"""Uses aiohttp to download an HTTP file.
Attempts to resume the file if `output_path` already exists.
"""
logger.debug(f'Starting download of file {repr(str(url))}')
if output_path.is_file() and info.size == (output_path.stat().st_size * 8):
logger.warning(f'Already downloaded {repr(str(url))} to {repr(str(output_path))}')
return
if info.accept_ranges == 'bytes' or not output_path.is_file():
with open(output_path, 'ab') as fh:
headers = dict()
# Check the position of append, if it is 0 then we do not need to resume.
position = fh.tell()
if position:
headers['Range'] = f'bytes={position}-'
async with aiohttp.ClientSession() as session:
async with session.get(url, headers=headers) as response:
if response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:
logger.warning(f'Server responded with 416, file is probably already downloaded')
return
if position and response.status != HTTPStatus.PARTIAL_CONTENT:
raise UnrecoverableDownloadError(
f'Tried to resume {repr(str(url))} but got status {response.status}')
# May or may not be using Range. Append each chunk to the output file.
async for data in response.content.iter_any():
fh.write(data)
# TODO this cannot be canceled.
# Sleep to catch cancel.
await asyncio.sleep(0)
elif output_path.is_file():
# TODO support downloading files that cannot be resumed.
raise UnrecoverableDownloadError(f'Cannot resume download {url}')
class FileDownloader(Downloader, ABC):
"""A Downloader which can download an arbitrary HTTP file.
Attempts to use the Content-Disposition name, will fail over to the path name if that's not possible.
"""
name = 'file'
pretty_name = 'File'
listable = False
def __repr__(self):
return '<FileDownloader>'
async def do_download(self, download: Download) -> DownloadResult:
if download.attempts > 3:
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
media_directory = get_media_directory()
# We don't know where to put the file if we don't have a destination.
destination = download.settings.get('destination')
if not destination:
raise UnrecoverableDownloadError(f'Cannot download the file without a destination')
# Create the destination only if it is within the media directory.
destination = pathlib.Path(destination)
if not str(destination.absolute()).startswith(str(media_directory)):
raise UnrecoverableDownloadError(f'Cannot download outside media directory!')
destination.mkdir(parents=True, exist_ok=True)
info = await get_download_info(download.url)
# TODO verify that this output_path is exclusive.
output_path = destination / info.name
try:
await download_file(download.url, info, output_path)
background_task(save_and_tag(output_path, download.settings.get('tag_names')))
return DownloadResult(
success=True,
location=f'/download/{output_path.relative_to(media_directory)}'
)
except Exception as e:
logger.error(f'Failed to download {repr(str(download.url))}', exc_info=e)
if PYTEST:
raise
error = str(traceback.format_exc())
return DownloadResult(
success=False,
error=error,
)
async def save_and_tag(file: pathlib.Path, tag_names: List[str] = None):
"""Stores the file as a FileGroup. Applies any provided Tags to the FileGroup."""
from wrolpi.tags import Tag
tag_names = tag_names or list()
with get_db_session(commit=True) as session:
file_group = FileGroup.find_by_path(file)
if not file_group:
file_group = FileGroup.from_paths(session, file)
session.commit()
if tag_names:
tags = session.query(Tag).filter(Tag.name.in_(tag_names))
for tag in tags:
file_group.add_tag(tag)
file_downloader = FileDownloader()

View file

@ -14,6 +14,7 @@ from typing import List, Tuple, Union, Dict, Generator
import cachetools.func
import psycopg2
from sqlalchemy import asc
from sqlalchemy.orm import Session
from wrolpi import flags
@ -25,7 +26,7 @@ from wrolpi.dates import now, from_timestamp
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag
from wrolpi.events import Events
from wrolpi.files.models import FileGroup
from wrolpi.files.models import FileGroup, Directory
from wrolpi.lang import ISO_639_CODES
from wrolpi.tags import TagFile, Tag
from wrolpi.vars import PYTEST
@ -204,6 +205,7 @@ SUFFIXES = {
'.readability.json',
'.readability.txt',
'.readability.html',
'.live_chat.json'
}
SUFFIXES |= {f'.{i}.srt' for i in ISO_639_CODES}
SUFFIXES |= {f'.{i}.vtt' for i in ISO_639_CODES}
@ -301,6 +303,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
It is assumed all files exist."""
idempotency = str(idempotency)
non_primary_files = set()
for chunk in chunks_by_stem(files, 100):
# Group all files by their shared full-stem. `chunk_by_stem` already sorted the files.
grouped = group_files_by_stem(chunk, pre_sorted=True)
@ -309,17 +312,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
for group in grouped:
# The primary file is the video/SingleFile/epub, etc.
primary_path_or_paths = get_primary_file(group)
if isinstance(primary_path_or_paths, list) or isinstance(primary_path_or_paths, tuple):
# Cannot find primary path, create group for each file.
for primary_path in primary_path_or_paths:
primary_path: pathlib.Path
# The primary mimetype allows modelers to find its file_groups.
mimetype = get_mimetype(primary_path)
modification_datetime = from_timestamp(primary_path.stat().st_mtime)
size = primary_path.stat().st_size
files = json.dumps(_paths_to_files_dict([primary_path]))
values[primary_path] = (modification_datetime, mimetype, size, files)
else:
if isinstance(primary_path_or_paths, pathlib.Path):
# Multiple files in this group.
primary_path: pathlib.Path = primary_path_or_paths
# The primary mimetype allows modelers to find its file_groups.
@ -330,9 +323,21 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
size = sum(i.stat().st_size for i in group)
files = json.dumps(_paths_to_files_dict(group))
values[primary_path] = (modification_datetime, mimetype, size, files)
non_primary_files = {i for i in group if i != primary_path}
else:
# Cannot find primary path, create a `file_group` for each file.
for primary_path in primary_path_or_paths:
primary_path: pathlib.Path
# The primary mimetype allows modelers to find its file_groups.
mimetype = get_mimetype(primary_path)
modification_datetime = from_timestamp(primary_path.stat().st_mtime)
size = primary_path.stat().st_size
files = json.dumps(_paths_to_files_dict([primary_path]))
values[primary_path] = (modification_datetime, mimetype, size, files)
# `False` is the `indexed` which is false to force indexing by default.
values = [(str(primary_path), False, idempotency, *i) for primary_path, i in values.items()]
values = [(str(primary_path),
False, # `indexed` is false to force indexing by default.
idempotency, *i) for primary_path, i in values.items()]
with get_db_curs(commit=True) as curs:
values = mogrify(curs, values)
stmt = f'''
@ -370,6 +375,12 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
if need_index:
refresh_logger.info(f'Invalidated indexes of {len(need_index)} file groups')
if non_primary_files:
# New files may have been added which change what primary paths exist. Delete any file_groups which
# use paths which are not primary.
curs.execute('DELETE FROM file_group WHERE primary_path = ANY(%s)',
(list(map(str, non_primary_files)),))
async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetime.datetime = None):
"""Discover all files in the directories provided in paths, as well as all files in paths.
@ -463,12 +474,14 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
paths = paths or [get_media_directory()]
directories = list(filter(lambda i: i.is_dir(), paths))
found_directories = set()
if directories:
with flags.refresh_counting:
while directories:
directory = directories.pop()
files, dirs = get_files_and_directories(directory)
directories.extend(dirs)
found_directories |= set(dirs)
REFRESH['counted_files'] = REFRESH.get('counted_files', 0) + len(files)
# Sleep to catch cancel.
await asyncio.sleep(0)
@ -476,34 +489,40 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
with flags.refresh_discovery:
await refresh_discover_paths(paths, idempotency)
if send_events:
Events.send_global_refresh_discovery_completed()
if send_events:
Events.send_global_refresh_discovery_completed()
# Model all files that have not been indexed.
with flags.refresh_modeling:
await apply_modelers()
if send_events:
Events.send_global_refresh_modeling_completed()
if send_events:
Events.send_global_refresh_modeling_completed()
# Index the rest of the files that were not indexed by modelers.
with flags.refresh_indexing:
await apply_indexers()
if send_events:
Events.send_global_refresh_indexing_completed()
if send_events:
Events.send_global_refresh_indexing_completed()
# Cleanup any outdated file data.
with flags.refresh_cleanup:
await apply_refresh_cleanup()
if send_events:
Events.send_global_after_refresh_completed()
if send_events:
Events.send_global_refresh_completed()
with get_db_session() as session:
parent_directories = {i[0] for i in session.query(Directory.path).filter(Directory.path.in_(paths))}
parent_directories |= set(filter(lambda i: i.is_dir(), paths))
await upsert_directories(parent_directories, found_directories)
if send_events:
Events.send_global_after_refresh_completed()
refresh_logger.warning('Done refreshing Files')
if refreshing_all_files:
# Only set refresh_complete flag if all files have been refreshed.
flags.refresh_complete.set()
if send_events:
Events.send_global_refresh_completed()
async def apply_indexers():
@ -539,6 +558,50 @@ async def apply_indexers():
break
async def upsert_directories(parent_directories, directories):
"""
Insert/update/delete directories provided. Deletes all children of `parent_directories` which are not in
`directories`.
"""
idempotency = now()
directories = list(directories) + list(parent_directories)
# Only insert directories that are children of `media_directory` and exist.
media_directory = get_media_directory()
directories = [i for i in directories if i.is_dir() and i != media_directory]
if directories:
# Insert any directories that were created, update any directories which previously existed.
with get_db_curs(commit=True) as curs:
values = [(str(i.absolute()), i.name, idempotency) for i in directories]
values = mogrify(curs, values)
stmt = f'''
INSERT INTO directory (path, name, idempotency) VALUES {values}
ON CONFLICT (path) DO UPDATE
SET idempotency = EXCLUDED.idempotency
'''
curs.execute(stmt)
with get_db_curs(commit=True) as curs:
# Delete the children of any parent directory which no longer exists.
for directory in parent_directories:
stmt = f'DELETE FROM directory WHERE path LIKE %(directory)s AND idempotency != %(idempotency)s'
curs.execute(stmt, dict(directory=f'{directory.absolute()}%', idempotency=idempotency))
@optional_session
async def search_directories(name: str, excluded: List[str] = None, limit: int = 20, session: Session = None) \
-> List[Directory]:
"""Find the Directories whose names contain the `name` string."""
excluded = excluded or []
directories = session.query(Directory) \
.filter(Directory.name.ilike(f'%{name}%')) \
.filter(Directory.path.notin_(excluded)) \
.order_by(asc(Directory.name)) \
.limit(limit).all()
return directories
def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] = None, model: str = None,
tags: List[str] = None, headline: bool = False) -> \
Tuple[List[dict], int]:

View file

@ -246,3 +246,23 @@ class FileGroup(ModelHelper, Base):
def find_by_path(path, session) -> Optional['FileGroup']:
file_group = session.query(FileGroup).filter(FileGroup.primary_path == str(path)).one_or_none()
return file_group
@property
def tag_names(self) -> List[str]:
return [i.tag.name for i in self.tag_files]
class Directory(ModelHelper, Base):
"""A representation of a file directory in the media directory."""
__tablename__ = 'directory'
path: pathlib.Path = Column(MediaPathType, primary_key=True)
name: str = Column(String, nullable=False)
idempotency = Column(TZDateTime, default=lambda: now())
def __json__(self):
d = dict(
path=self.path,
name=self.name,
)
return d

View file

@ -38,6 +38,11 @@ class DirectoriesRequest:
search_str: Optional[str] = ''
@dataclass
class DirectoriesSearchRequest:
name: str
@dataclass
class DirectoriesResponse:
directories: List[str]

View file

@ -5,7 +5,6 @@ from unittest import mock
import pytest
from wrolpi.errors import API_ERRORS, WROLModeEnabled
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.tags import TagFile
from wrolpi.test.common import assert_dict_contains
@ -179,31 +178,6 @@ def test_files_search(test_session, test_client, make_files_structure, assert_fi
assert_files_search('nothing', [])
def test_directory_search(test_client, make_files_structure):
"""Test that directories can be searched."""
make_files_structure([
'foo/',
'fool/',
'not a directory',
])
def assert_directories(search_str, expected):
body = dict(search_str=search_str)
request, response = test_client.post('/api/files/directories', content=json.dumps(body))
assert response.status_code == HTTPStatus.OK
assert response.json['directories'] == expected
# All directories are returned.
assert_directories(None, ['foo', 'fool'])
assert_directories('', ['foo', 'fool'])
# Matches both directories.
assert_directories('fo', ['foo', 'fool'])
# Matches the one directory exactly.
assert_directories('foo', ['foo'])
# Does not exist.
assert_directories('food', [])
def test_refresh_files_list(test_session, test_client, make_files_structure, test_directory, video_bytes):
"""The user can request to refresh specific files."""
make_files_structure({
@ -307,3 +281,158 @@ def test_file_group_tag(test_client):
request, response = test_client.post('/api/files/tag', content=json.dumps(dict(file_group_id=1)))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'tag_id' in response.json['error']
def test_search_directories(test_client, test_session, make_files_structure, assert_directories):
"""Directories can be searched by name."""
make_files_structure(['foo/one.txt', 'foo/two.txt', 'bar/one.txt'])
request, response = test_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
assert_directories({'foo', 'bar'})
# More than one character required.
content = dict(name='f')
request, response = test_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
content = dict(name='fo')
request, response = test_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['directories']] == ['foo', ]
# Case is ignored.
content = dict(name='BAR')
request, response = test_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['directories']] == ['bar', ]
# Searching for something that does not exist.
content = dict(name='does not exist')
request, response = test_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['directories']] == []
# Only first 20 "foo" are returned.
make_files_structure([
'fooo/',
'foooo/',
'fooooo/',
'foooooo/',
'fooooooo/',
'foooooooo/',
'fooooooooo/',
'foooooooooo/',
'fooooooooooo/',
'foooooooooooo/',
'fooooooooooooo/',
'foooooooooooooo/',
'fooooooooooooooo/',
'foooooooooooooooo/',
'fooooooooooooooooo/',
'foooooooooooooooooo/',
'fooooooooooooooooooo/',
'foooooooooooooooooooo/',
'fooooooooooooooooooooo/',
'foooooooooooooooooooooo/',
'fooooooooooooooooooooooo/',
])
test_client.post('/api/files/refresh')
content = dict(name='fo')
request, response = test_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert [i['path'] for i in response.json['directories']] == [f'f{"o" * i}' for i in range(2, 22)]
def test_post_search_directories(test_session, test_client, make_files_structure):
"""Directory names can be searched. This endpoint also returns Channel and Domain directories."""
channel_dir, domain_dir, _, _ = make_files_structure([
'dir1/',
'dir2/',
'dir3/',
'dir4/',
])
request, response = test_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
from modules.videos.models import Channel
channel = Channel(directory=channel_dir, name='Channel Name')
from modules.archive.models import Domain
domain = Domain(directory=domain_dir, domain='example.com')
test_session.add_all([channel, domain])
test_session.commit()
content = {'name': 'di'}
request, response = test_client.post('/api/files/search_directories', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
# All directories contain "di", but directories filters out the Channel and Domain directories.
assert response.json['directories'] == [{'name': 'dir3', 'path': 'dir3'}, {'name': 'dir4', 'path': 'dir4'}]
assert response.json['channel_directories'] == [{'name': 'Channel Name', 'path': 'dir1'}, ]
assert response.json['domain_directories'] == [{'domain': 'example.com', 'path': 'dir2'}, ]
def test_post_upload_directory(test_session, test_client, test_directory, make_files_structure, make_multipart_form):
"""A file can be uploaded in a directory in the destination."""
make_files_structure(['uploads/'])
forms = [
dict(name='chunkNumber', value='0'),
dict(name='filename', value='/foo/bar.txt'), # notice the directory
dict(name='totalChunks', value='0'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value='3'),
dict(name='chunk', value='foo', filename='chunk')
]
body = make_multipart_form(forms)
request, response = test_client.post('/api/files/upload', data=body,
headers={
'Content-Type': 'multipart/form-data; name=upload; filename="file.txt";'
' boundary=-----sanic'})
assert response.status_code == HTTPStatus.CREATED
assert (test_directory / 'uploads/foo/bar.txt').is_file()
assert (test_directory / 'uploads/foo/bar.txt').read_text() == 'foo'
assert test_session.query(FileGroup).count() == 1
def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form):
"""A file can be uploaded in chunks directly to the destination."""
make_files_structure(['uploads/'])
forms = [
dict(name='chunkNumber', value='0'),
dict(name='filename', value='foo.txt'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value='3'),
dict(name='chunk', value='foo', filename='chunk')
]
body = make_multipart_form(forms)
request, response = test_client.post('/api/files/upload', data=body,
headers={
'Content-Type': 'multipart/form-data; name=upload; filename="file.txt";'
' boundary=-----sanic'})
assert response.status_code == HTTPStatus.OK
assert (test_directory / 'uploads/foo.txt').is_file()
assert (test_directory / 'uploads/foo.txt').read_text() == 'foo'
forms = [
dict(name='chunkNumber', value='1'),
dict(name='filename', value='foo.txt'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value='3'),
dict(name='chunk', value='bar', filename='chunk')
]
body = make_multipart_form(forms)
request, response = test_client.post('/api/files/upload', data=body,
headers={
'Content-Type': 'multipart/form-data; name=upload; filename="file.txt";'
' boundary=-----sanic'})
assert response.status_code == HTTPStatus.CREATED
assert (test_directory / 'uploads/foo.txt').is_file()
assert (test_directory / 'uploads/foo.txt').read_text() == 'foobar'
assert test_session.query(FileGroup).count() == 1

View file

@ -10,6 +10,7 @@ from uuid import uuid4
import mock
import pytest
import sqlalchemy.exc
from PIL import Image
from wrolpi.common import timer
@ -158,7 +159,8 @@ async def test__upsert_files(test_session, make_files_structure, test_directory,
test_session.query(FileGroup).filter_by(primary_path=video_file).one().indexed = True
test_session.commit()
assert_file_groups(
[{'primary_path': str(video_file), 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': True}],
[{'primary_path': str(video_file), 'idempotency': idempotency, 'modification_datetime': foo_mtime,
'indexed': True}],
assert_count=False)
# Only modified files need to be re-indexed.
@ -411,6 +413,7 @@ async def test_files_indexer(test_session, make_files_structure, test_directory)
('name two', 'name two'),
('this self-reliance_split.txt', 'this self reliance self-reliance split txt'),
('-be_split!.txt', '-be split! txt'),
('WROLPi-v0.10-aarch64-desktop.img.xz', 'WROLPi v0.10 aarch64 desktop.img WROLPi-v0.10-aarch64-desktop.img xz'),
])
def test_split_file_name_words(name, expected):
assert lib.split_file_name_words(name) == expected
@ -503,6 +506,7 @@ def test_group_files_by_stem(make_files_structure, test_directory):
'foo.mp4',
'foo.txt',
'foo.info.json',
'foo.live_chat.json',
'bar.txt',
'baz.txt',
])
@ -511,7 +515,7 @@ def test_group_files_by_stem(make_files_structure, test_directory):
assert list(lib.group_files_by_stem(files)) == [
[test_directory / 'bar.txt'],
[test_directory / 'baz.txt'],
[test_directory / 'foo.info.json', test_directory / 'foo.mp4', test_directory / 'foo.txt'],
[test_directory / 'foo.info.json', test_directory / 'foo.live_chat.json', test_directory / 'foo.mp4', test_directory / 'foo.txt'],
]
@ -600,3 +604,44 @@ async def test_refresh_files_no_groups(test_session, test_directory, make_files_
txt, zip_ = test_session.query(FileGroup)
assert txt.primary_path == foo_txt and txt.size == foo_txt.stat().st_size
assert zip_.primary_path == foo_zip and zip_.size == foo_zip.stat().st_size
@pytest.mark.asyncio
async def test_refresh_directories(test_session, test_directory, assert_directories):
"""
Directories are stored when they are discovered. They are removed when they can no longer be found.
"""
foo = test_directory / 'foo'
bar = test_directory / 'bar'
baz = test_directory / 'baz'
foo.mkdir()
bar.mkdir()
baz.mkdir()
await lib.refresh_files()
assert_directories({'foo', 'bar', 'baz'})
# Deleted directory is removed.
foo.rmdir()
await lib.refresh_files()
assert_directories({'bar', 'baz'})
bar.rmdir()
await lib.refresh_files([bar])
assert_directories({'baz', })
# A new directory can be refreshed directly.
foo.mkdir()
await lib.refresh_files([foo])
assert_directories({'foo', 'baz'})
def test_file_group_primary_path_is_unique(test_session, test_directory):
"""You cannot create a FileGroup that shares a path"""
path = test_directory / 'file.txt'
path.touch()
one = FileGroup.from_paths(test_session, path)
two = FileGroup.from_paths(test_session, path)
with pytest.raises(sqlalchemy.exc.IntegrityError):
test_session.commit()

View file

@ -129,6 +129,7 @@ ffprobe_installed = Flag('ffprobe_installed')
nmcli_installed = Flag('nmcli_installed')
readability_installed = Flag('readability_installed')
singlefile_installed = Flag('singlefile_installed')
wget_installed = Flag('wget_installed')
yt_dlp_installed = Flag('yt_dlp_installed')

View file

@ -0,0 +1,51 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBF2ebPIBEADG0T2VsDOH85qTF3UuAwQ9MO4T9yXDyb9tKVDX1wUsLEFH5tUn
dv51vhIZ+aVpFcrA7ULasikqdCo/Fomy9ujtf4EMYXwfg1uBrtvZZTwdciQnEfhR
/YCruypeXz91SukCjJMVLARWI1EUWyq7glSvXUOgPccLA6BCGN7Dv+JoXGDBVJAo
Uw74aCEIdhcxDM1u2RSaPWeVuA5P6PBYSnw4f0F91JHI1OobMFp7SmApCzLHzEyd
9J7ZqRpMVJaz160GzvemMdeDg7hj7GSmlVgeF4qmbE4E4cxqE2tRniYtgxjh0lyx
C6E0riRt2U00Z0CvaFuvwNX0fr0vZc+p82i9lakxy/9cUYKhVvcQsh/ofmbakGlz
MxdQpRsRpaivt2KCwHKzZUHsvDxGm/aH2liiz3bV9MeoBVo5ngFaBSLlnF08tGtR
g0M7olkTgX29BJM9++QaRvS8FwN2yG/iTIX7VTIEaEBoeoA0d7g6ojZC1UUZfI/R
Uy3AS0N1+Cza99PVU4vB9nsjUmtsO1ru0WxEKmfEJ7TI3ZTuH6+cpi8z9Vf6pdnN
LxB3utC5SvkYCYl9PPlm8QHnIAiAcL7VuAUpdWqJcMuMdytEPe8N8JKU9wWfxvdB
67t+0OSM8aBouq58UM8xi/3MxYVwZ7r4Eleir1wtjountk8abhAiXHJqnQARAQAB
tChSb2xhbmQgPHJvbGFuZEBsZWFybmluZ3NlbGZyZWxpYW5jZS5jb20+iQI/BBMB
AgApBQJdnmzyAhsDBQkJZgGABwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AACgkQ
nW29Os8Rq5M35w/7BJnPZ44BpdXpohyXbbNho7ewv0ggJFeKGcrsNVyYvhZp6vQx
W1zrpbMZc0w0QR7XZ8QKqQJAIihGKTs8dZ5Gl9EZs2EueMUEWMwAAjOL1pQ6Oa/A
m1GYv6H6v8pZSJluN33T9mGmQE/v1N3ASgB6IW0syj4WOKSxIj0tQqat8O1r9wwp
GHFVZQlCmUf7RTt0K/nGXeqFk7kUOtmidKHmfUJ8VXtyW36q+uZXlpQE+1o1KtMt
tdjaHGzpKKKpLbc/8hoNGJWE58uHxy40ZG85seq5qLgmKkdW9py98ov6nhcWTYJN
LI+3sEAYuSD1UU+iRBY9wHgh1dcxy+6AOuylsXehpwtABK7asm+anFmG7vso4Z0+
mPWR3yHFK2MhZBW37VISBL0S5dxyEioLb2PTLGtRW+3ZagHv9xyUSI5gRfGOS9Sk
gAO/dyK4J6q+IxnABdU3l/8dwtV4+Qk4hp2iEwEy1ZAULWx4EZA9NxWTK6k0rCqs
f4w2yihC2ubo49xMa1iaYP/qMctFIlNttiKOVonDI+cfH4iUXljA1DDcYCbzOMCV
Iw8gdEkzyXaxWPBxcfzTQ8H8JWrxHc4/q5j+fC3D7YzkmmxwgR1pUSmOKyCEq2E/
UjMSaZl26u0gzlUbE1uAKv8xXxYbh7SUPDOjCPqWljQm4iZnTiLcnnlqpu25Ag0E
XZ5s8gEQAOPHYlgQP8Han72JK6Wf9Hs8u7C0SqS9+bIN13yK14M6KxZprPYCS4fO
xmukFfchIqzQrzuiTTbDQ90EuuncFiHKeofe5TXUDrZehjkRg8/Ad115lM6Eme7q
iDdtHHxfSm8OgFE9xZg8sX9fhsQHQTrW/pDULAnO4mW5AKMlq4J4r/1AEN4wq0p+
fgkd3d5KBAdXlmV+m4F6PM5xBDBoMZP5LGTeg8p68hWEGV+EuzpmJ6ixNSQ88a/d
j2IB3AH5LYLe0CQ4QWQ2EpM9NylmC9LwnBYM7ykaMXTb3twJioL2k2s20p5CQbnd
9j6h3rqncGPs9AoGnLUqf/jz4qPObBNsFpqHgFZ98vB+3Ynzdd+5O/MffCDBNKc3
Zd4G5ZQ9NEM1EqB+s8uNcO7lSMqUbhEVE77PMxsqt0XrQASL3IkNPXA61apmS7Sc
XDcyaFzImwE03KDJF+zxVqlknxjJcJb+gdCMW1kYstKjGQsHd/KUlITQclLihifb
4WD/9bRkPuXteddt36Q9phkd0oO+OQqOoh4yjkI9OdlOgJVR4NQEDIi6hGf2eaKh
vp07dwRiQdrrRGLUEcXLIkuju+cQ4gxbgfSO1QdEGNVyXW88fzramF7LkGtZ0fXE
N0paw/0NOeSmV6hLyTsMkDHjNsOiDjDJSNVF1q5uq/Doz8o+csxZABEBAAGJAiUE
GAECAA8FAl2ebPICGwwFCQlmAYAACgkQnW29Os8Rq5OGkg/+MnJMjiQX+qFOzFGy
Twlxdm1cKRTDHwpWE1P7R9q//jPTjIDc8MF5GTGHkr1uFAV1J8mQ0KI34F/ek09K
e75x73OOlNTtxj/q+S5kbY/4Nd6ukvlkiE6UErK3dYiiY4JX9fRnyHuPgwNOH0qr
PNRq62pUk7MpRH0vvS2Lh4HmfMt2fz9aL4pjclJzM+gP5d84Rg4FeUvBTxSi/HMk
3KXxa6vlZyD+YE6CttptrtNmvu03haIhdjwCMFvR8y6Ytr6Ks0f5Le65jbh+u0xD
FRwdwSpG6LGGx+1BmOrtPOre2bvycMPz5pC5glQ9kUGhrugcdGB4gX2QkE28Vec5
h5Vh6GOOy11p4XPDXA2eQg3SwHorltQcT283IJm7ZyrVTnGAvFNa8zaOlSjLw+P5
Y6b+Ek/TCeqWS5yerODVRz7GyEn+2qjL/HL+g77XnxR7hAlI2eqdeTSN+GIkHTgp
7V9SBWYbGbKvsHlofd6XJjDOs4BHlQRkpfe+7g6w2x8xQkbZBN+hceTyGMTNJPSw
uCkizNFZNZoNi11HtWMhqSRBNox2GW+moiAvQRgjVHW0W78SPpzwtpY+TAdLaXfm
J25aqkmGRFvjfLP56VxXrrKNWNx5zPCvGe2T5MoZSx70Ji4KMw4auNyDxe/l3ywb
Xd3Fca66xs8xVb/52jI0Vv4oI3U=
=y08S
-----END PGP PUBLIC KEY BLOCK-----

View file

@ -483,6 +483,9 @@ def json_error_handler(request: Request, exception: Exception):
except KeyError:
# Cause was not an APIError.
logger.error(f'Could not find cause {cause}')
logger.debug(f'API returning JSON error {exception=}'
f' error={body.get("api_error") or body.get("error")}'
f' code={body["code"]}')
return json_response(body, error['status'])

View file

@ -148,7 +148,7 @@ class TagsConfig(ConfigFile):
})
TAGS_CONFIG: TagsConfig = TagsConfig(global_=True)
TAGS_CONFIG: TagsConfig = TagsConfig()
TEST_TAGS_CONFIG: TAGS_CONFIG = None

View file

@ -9,6 +9,7 @@ from unittest.mock import MagicMock
import pytest
import pytz
import yaml
from wrolpi.dates import Seconds
from wrolpi.db import get_db_context
@ -338,8 +339,6 @@ def test_crud_download(test_client, test_session, test_download_manager, test_do
@pytest.mark.asyncio
async def test_downloads_config(test_session, test_client, test_download_manager, test_download_manager_config,
test_downloader, assert_downloads):
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
# Can import with an empty config.
await import_downloads_config()
assert_downloads([])
@ -354,6 +353,7 @@ async def test_downloads_config(test_session, test_client, test_download_manager
test_download_manager.recurring_download('https://example.com/3', frequency=DownloadFrequency.weekly,
downloader_name=test_downloader.name)
download2.next_download = datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC)
download2.settings = {'destination': 'some directory'}
# Completed once-downloads should be ignored.
download1.last_successful_download = datetime(2000, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
test_session.commit()
@ -370,12 +370,14 @@ async def test_downloads_config(test_session, test_client, test_download_manager
frequency=None,
next_download=datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC),
downloader='test_downloader',
settings={'destination': 'some directory'},
sub_downloader=None,
),
dict(
url='https://example.com/3',
frequency=DownloadFrequency.weekly,
downloader='test_downloader',
settings=None,
sub_downloader=None,
),
]
@ -386,6 +388,16 @@ async def test_downloads_config(test_session, test_client, test_download_manager
test_session.query(Download).delete()
test_session.commit()
# Change the destination.
with test_download_manager_config.open('rt') as fh:
config_contents = yaml.load(fh, Loader=yaml.Loader)
for idx, download in enumerate(config_contents['downloads']):
if download['url'] == 'https://example.com/2':
config_contents['downloads'][idx]['settings'] = dict(destination='a different directory')
with test_download_manager_config.open('wt') as fh:
yaml.dump(config_contents, fh)
get_download_manager_config().initialize()
# Import the saved downloads.
await import_downloads_config()
expected = [
@ -394,12 +406,15 @@ async def test_downloads_config(test_session, test_client, test_download_manager
frequency=None,
next_download=datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC),
downloader='test_downloader',
# The destination was imported from the config.
settings={'destination': 'a different directory'},
sub_downloader=None,
),
dict(
url='https://example.com/3',
frequency=DownloadFrequency.weekly,
downloader='test_downloader',
settings=dict(),
sub_downloader=None,
),
]