From 387282b445e7801c90b8f6ecdf67e87cd08916f0 Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 18 Apr 2023 22:03:45 -0600 Subject: [PATCH] Adding my public gpg key. Moving upload destination input out of drop Segment. Disabling uploads when downloads are disabled. Inverting labels for checkboxes in inverted modal. Deleting file_groups which no longer contain the exclusive primary path. Adding debug logging for all API errors. Checking that upload output is not a directory. Expanding nginx request size. Adding docs to upload endpoint. Using dot-circle icon for more image mimetypes. Using Form.Field to make upload destination required. Adding border to inverted Segments. Many files can be uploaded in chunks. Files can be uploaded, progress is displayed. Showing white border around share QR code. Displaying WROLPi colors for download icons. Creating button to clear DirectorySearch. Excluding channel/domain directories so other directories can be returned. Ordering search results. Ignoring when no channel is found only if we have a destination. Limiting directory search results. Displaying categories when searching destination directories. Handling when live_chat.json file exists. Allowing refresh of directory directly. Logging which download was successful. Removing timeout. Creating index for directory names. Sleeping to catch cancel. Arbitrary files can be downloaded using the new FileDownloader. --- .../versions/f9928a3b6bc5_file_directories.py | 44 ++++ app/src/App.css | 29 +++ app/src/Dashboard.js | 78 +++++- app/src/Tags.js | 70 +++--- app/src/api.js | 23 +- app/src/components/Archive.js | 28 +-- app/src/components/Common.js | 237 ++++++++++++++---- app/src/components/Donate.js | 14 +- app/src/components/Download.js | 104 +++++--- app/src/components/FilePreview.js | 16 +- app/src/components/Files.js | 30 ++- app/src/components/Headline.js | 4 +- app/src/components/Share.js | 24 +- app/src/components/Theme.js | 139 ++++++---- app/src/components/Upload.js | 110 ++++++++ app/src/components/admin/Downloads.js | 26 +- app/src/components/admin/Settings.js | 16 +- app/src/hooks/customHooks.js | 151 +++++++++++ modules/archive/__init__.py | 1 - modules/archive/lib.py | 11 + modules/inventory/common.py | 2 +- modules/videos/channel/lib.py | 12 + modules/videos/downloader.py | 13 +- modules/videos/lib.py | 4 +- nginx.conf | 2 + nginx.dev.conf | 2 + requirements.txt | 1 - wrolpi/cmd.py | 3 + wrolpi/common.py | 16 +- wrolpi/conftest.py | 33 ++- wrolpi/downloader.py | 10 +- wrolpi/errors.py | 9 + wrolpi/files/__init__.py | 1 + wrolpi/files/api.py | 141 ++++++++++- wrolpi/files/downloader.py | 175 +++++++++++++ wrolpi/files/lib.py | 111 ++++++-- wrolpi/files/models.py | 20 ++ wrolpi/files/schema.py | 5 + wrolpi/files/test/test_api.py | 181 +++++++++++-- wrolpi/files/test/test_lib.py | 49 +++- wrolpi/flags.py | 1 + wrolpi/roland@learningselfreliance.com.gpg | 51 ++++ wrolpi/root_api.py | 3 + wrolpi/tags.py | 2 +- wrolpi/test/test_downloader.py | 19 +- 45 files changed, 1665 insertions(+), 356 deletions(-) create mode 100644 alembic/versions/f9928a3b6bc5_file_directories.py create mode 100644 app/src/components/Upload.js create mode 100644 wrolpi/files/downloader.py create mode 100644 wrolpi/roland@learningselfreliance.com.gpg diff --git a/alembic/versions/f9928a3b6bc5_file_directories.py b/alembic/versions/f9928a3b6bc5_file_directories.py new file mode 100644 index 00000000..9437ee8c --- /dev/null +++ b/alembic/versions/f9928a3b6bc5_file_directories.py @@ -0,0 +1,44 @@ +"""File Directories + +Revision ID: f9928a3b6bc5 +Revises: c9561bda3cf0 +Create Date: 2023-04-18 16:58:40.043115 + +""" +import os +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'f9928a3b6bc5' +down_revision = 'c9561bda3cf0' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute(''' + CREATE TABLE directory ( + path TEXT PRIMARY KEY, + name TEXT NOT NULL, + idempotency TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp + ) + ''') + + session.execute('CREATE INDEX directory_name_idx ON directory(name)') + + if not DOCKERIZED: + session.execute('ALTER TABLE public.directory OWNER TO wrolpi') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('DROP INDEX IF EXISTS directory_name_idx') + session.execute('DROP TABLE IF EXISTS directory') diff --git a/app/src/App.css b/app/src/App.css index 1c050b5d..e7d23c3b 100644 --- a/app/src/App.css +++ b/app/src/App.css @@ -172,6 +172,35 @@ textarea.otp { background-color: #222222; } +.modal .inverted { + border-color: #888888; +} + +.modal .header.inverted { + color: #DDDDDD !important; + background-color: #171616 !important; +} + +.modal .content.inverted { + color: #DDDDDD !important; + border-color: #888888 !important; + background-color: #171616 !important; +} + +.modal .actions.inverted { + color: #DDDDDD !important; + border-color: #888888 !important; + background-color: #171616 !important; +} + .clickable { cursor: pointer; } + +.segment.inverted { + border: 1px solid #ffffff22 !important; +} + +.inverted label { + color: #DDDDDD !important; +} \ No newline at end of file diff --git a/app/src/Dashboard.js b/app/src/Dashboard.js index 9b9e1ee3..c2112d8e 100644 --- a/app/src/Dashboard.js +++ b/app/src/Dashboard.js @@ -1,18 +1,29 @@ import {LoadStatistic, PageContainer, SearchInput, useTitle} from "./components/Common"; -import {useQuery, useSearchFiles} from "./hooks/customHooks"; +import {useQuery} from "./hooks/customHooks"; import React, {useContext, useState} from "react"; import {StatusContext} from "./contexts/contexts"; import {DownloadMenu} from "./components/Download"; import {FilesSearchView} from "./components/Files"; -import {Header, Segment, Statistic, StatisticGroup} from "./components/Theme"; +import { + Button, + Divider, + Header, + Modal, + ModalContent, + ModalHeader, + Segment, + Statistic, + StatisticGroup +} from "./components/Theme"; import {Link} from "react-router-dom"; import {BandwidthProgressCombined, CPUUsageProgress} from "./components/admin/Status"; import {ProgressPlaceholder} from "./components/Placeholder"; -import {Divider, Icon, Message} from "semantic-ui-react"; +import {GridColumn, GridRow, Icon, Message} from "semantic-ui-react"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import {refreshFiles} from "./api"; import _ from "lodash"; import {TagsDashboard} from "./Tags"; +import {Upload} from "./components/Upload"; function FlagsMessages({flags}) { if (!flags) { @@ -29,8 +40,8 @@ function FlagsMessages({flags}) { refreshing = - Refreshing - Your files are being refreshed. + Your files are being refreshed. +

Click here to view the progress

; } else if (flags.indexOf('refresh_complete') === -1) { @@ -73,10 +84,51 @@ export function Dashboard() { const {status} = useContext(StatusContext); const wrol_mode = status ? status['wrol_mode'] : null; - const [downloadOpen, setDownloadOpen] = useState(false); - const onDownloadOpen = (name) => setDownloadOpen(!!name); - const downloadsDisabled = status?.flags?.indexOf('refresh_complete') === -1; - const downloads = ; + // Getters are Downloads or Uploads. + const [selectedGetter, setSelectedGetter] = useState(null); + const gettersDisabled = status?.flags?.indexOf('refresh_complete') === -1; + const handleSetGetter = (e, value) => { + if (e) { + e.preventDefault(); + } + setSelectedGetter(value); + } + + let getter = + + Or + + + + + + + + + + ; + let getterModal; + if (selectedGetter === 'downloads') { + getterModal = handleSetGetter(null, null)}> + Downloads + + + + ; + } else if (selectedGetter === 'upload') { + getterModal = handleSetGetter(null, null)}> + Upload + + + + ; + } // Only show dashboard parts if not searching. let body; @@ -84,10 +136,10 @@ export function Dashboard() { body = ; } else { body = <> - {!wrol_mode && downloads} - {/* Hide extras when user is starting a download */} - {!downloadOpen && } - {!downloadOpen && } + {!wrol_mode && getter} + {getterModal} + + ; } diff --git a/app/src/Tags.js b/app/src/Tags.js index cd464356..eec474ab 100644 --- a/app/src/Tags.js +++ b/app/src/Tags.js @@ -1,21 +1,14 @@ import React, {useEffect, useState} from "react"; import {deleteTag, getTags, saveTag} from "./api"; import { - Button as SButton, Confirm, Dimmer, Divider, Dropdown, Form, - FormInput, Grid, - Header, Label, LabelGroup, - Loader, - Loader as SLoader, - Modal, - Table as STable, TableBody, TableCell, TableHeader, @@ -23,7 +16,18 @@ import { TableRow, } from "semantic-ui-react"; import {contrastingColor} from "./components/Common"; -import {Segment} from "./components/Theme"; +import { + Button, + FormInput, + Header, + Loader, + Modal, + ModalActions, + ModalContent, + ModalHeader, + Segment, + Table +} from "./components/Theme"; import _ from "lodash"; import {HexColorPicker} from "react-colorful"; import {useRecurringTimeout} from "./hooks/customHooks"; @@ -135,7 +139,7 @@ function EditTagRow({tag, onDelete, onEdit}) { const {name, color, id, count} = tag; const deleteConfirm = <> - setConfirmDeleteOpen(true)} icon='trash'/> + - + Delete @@ -264,12 +268,12 @@ function EditTagsModal() { {content} - - +
+ - setOpen(true)} color='violet'> + } @@ -324,33 +328,33 @@ export function AddTagsButton({ const unusedTagsGroup = ; return <> - + - + } @@ -402,7 +406,7 @@ export const TagsSelector = ({ export const TagsDashboard = () => { const {tagNames, TagsLinkGroup} = React.useContext(TagsContext); - let availableTagsGroup = ; + let availableTagsGroup = ; if (tagNames && tagNames.length) { availableTagsGroup = ; } diff --git a/app/src/api.js b/app/src/api.js index f44c20e8..43467bc8 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -1,6 +1,5 @@ import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, emptyToNull, OTP_API, VIDEOS_API} from "./components/Common"; import {toast} from "react-semantic-toasts-2"; -import {computeHeadingLevel} from "@testing-library/react"; function timeoutPromise(ms, promise) { // Create a timeout wrapper around a promise. If the timeout is reached, throw an error. Otherwise, return @@ -804,3 +803,25 @@ export async function sendNotification(message, url) { console.error('Failed to share'); } } + +export async function searchDirectories(name) { + const body = {name: name || ''}; + const response = await apiPost(`${API_URI}/files/search_directories`, body); + if (response.status === 204) { + return []; + } else if (response.status === 200) { + const content = await response.json(); + return content; + } else { + toast({type: 'error', title: 'Error', description: 'Failed to search directories!', time: 5000}); + } +} + +export async function uploadFiles(files, destination) { + const data = new FormData(); + for (let i = 0; i < files.length; i++) { + data.append('file', files[i]); + } + data.append('destination', destination); + console.log(data); +} diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index eef37d0e..ebfa421b 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -9,7 +9,6 @@ import { Dropdown, Image, Input, - Modal, PlaceholderHeader, PlaceholderLine, TableCell, @@ -19,7 +18,6 @@ import { BackButton, CardLink, cardTitleWrapper, - defaultSearchOrder, encodeMediaPath, ExternalCardLink, FileIcon, @@ -39,7 +37,7 @@ import { import {addTag, deleteArchives, postDownload, removeTag} from "../api"; import {Link, Route, Routes, useNavigate, useParams} from "react-router-dom"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; -import {useArchive, useDomains, useQuery, useSearchArchives, useSearchDomain} from "../hooks/customHooks"; +import {useArchive, useDomains, useSearchArchives, useSearchDomain} from "../hooks/customHooks"; import {FileCards, FileRowTagIcon, FilesView} from "./Files"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import _ from "lodash"; @@ -304,21 +302,6 @@ export function DomainsPage() { export function SearchDomain() { // A Dropdown which allows the user to filter by Archive domains. const {domain, domains, setDomain} = useSearchDomain(); - const [open, setOpen] = useState(false); - - const handleOpen = (e) => { - if (e) { - e.preventDefault(); - } - setOpen(true); - } - - const handleClose = (e) => { - if (e) { - e.preventDefault(); - } - setOpen(false); - } const handleChange = (e, {value}) => { setDomain(value); @@ -329,15 +312,6 @@ export function SearchDomain() { return {key: i['domain'], value: i['domain'], text: i['domain']} }); return <> - handleOpen()} - onClose={() => handleClose()} - > - - domains - - } @@ -792,14 +826,14 @@ export function UnsupportedModal(header, message, icon) { {header || 'Unsupported'} - - + + {message} - - - + + + - + ; return {modal, doClose: onClose, doOpen: onOpen}; @@ -851,6 +885,99 @@ export function useTitle(title) { }, [title]); } +export function DirectorySearch({onSelect, value, ...props}) { + const { + directoryName, + setDirectoryName, + directories, + channelDirectories, + domainDirectories, + loading, + } = useSearchDirectories(value); + const [results, setResults] = useState(); + + useEffect(() => { + if ( + (directories && directories.length >= 0) + || (channelDirectories && channelDirectories.length >= 0) + || (domainDirectories && domainDirectories.length >= 0)) { + setResults({ + directories: { + name: 'Directories', + results: directories.map(i => { + return {title: i['path']} + }), + }, + channel_directories: { + name: 'Channels', + results: channelDirectories.map(i => { + return {title: i['path'], description: i['name']}; + }), + }, + domain_directories: { + name: 'Domains', + results: domainDirectories.map(i => { + return {title: i['path'], description: i['domain']}; + }), + } + }); + } + }, [ + JSON.stringify(directories), + JSON.stringify(channelDirectories), + JSON.stringify(domainDirectories), + loading, + ]); + + const handleSearchChange = (e, data) => { + if (e) { + e.preventDefault(); + } + if (onSelect) { + onSelect(''); + } + setDirectoryName(data.value); + } + + const handleResultSelect = (e, data) => { + if (e) { + e.preventDefault(); + } + // title is the relative path. + if (onSelect) { + onSelect(data.result.title); + } + setDirectoryName(data.result.title); + } + + const handleClear = (e) => { + if (e) { + e.preventDefault(); + } + onSelect(''); + setDirectoryName(''); + } + + return + + + + + + }) } @@ -1018,10 +1145,10 @@ export function SortButton({sorts = []}) { open={open} onClose={() => setOpen(false)} > - Sort By - + Sort By + {sortFields} - + {submitted && viewDownloads} @@ -195,7 +206,7 @@ class ChannelDownload extends React.Component { return -
Channel / Playlist
+
Channel / Playlist
- + {pending && } {success && } } @@ -318,7 +329,7 @@ class RSSDownload extends ChannelDownload { return
-
RSS Feed
+
RSS Feed
- + {pending && } {success && } } @@ -397,7 +408,7 @@ export function DownloadMenu({onOpen, disabled}) { />