From f14a230bc663a2b965538f6abf0a8a57f82caff0 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 20 May 2022 15:15:31 -0600 Subject: [PATCH] RSSDownloader can schedule downloads for other Downloaders using sub_downloader. RSS download can be submitted in UI. Checking if entry has link. A video playlist can be downloaded. Passing sub_downloader when submitting RSS feed. Adding Archive downloader button. Playlists need more info. Removing expunge, it causes errors elsewhere. Fixing tests. Showing errors on recurring downloads. Ignoring CharacterEncodingOverride bozo error for rss feed. Changing download button order. Fixing bug where channel attributes were unavailable after session. Simplifying video format selection. Calling validate_video in Video.validate() to avoid duplicate code. Adding more frequent downloads for RSS Feeds. DownloadManager skips downloads in it's list. Renaming get_next_download to more clear calculate_next_download. Stopping download manager when server is shutting down. Not shutting down downloads during tests, not all tests use downloads. Making methods static. --- .../467235201c70_download_sub_downloader.py | 29 + app/src/App.js | 7 +- app/src/api.js | 7 +- app/src/components/Admin.js | 33 +- app/src/components/Common.js | 16 +- app/src/components/Upload.js | 287 +++++- modules/archive/__init__.py | 11 +- modules/map/leaflet.css | 908 ++++++++++-------- modules/videos/channel/lib.py | 1 + modules/videos/channel/test/test_api.py | 3 +- modules/videos/downloader.py | 181 ++-- modules/videos/lib.py | 21 +- modules/videos/models.py | 37 +- modules/videos/test/test_downloader.py | 108 ++- requirements.txt | 1 + wrolpi/conftest.py | 11 +- wrolpi/downloader.py | 201 +++- wrolpi/root_api.py | 17 +- wrolpi/schema.py | 2 + wrolpi/test/test_downloader.py | 34 +- wrolpi/test/test_root_api.py | 2 +- wrolpi/test/test_rss.py | 114 +++ wrolpi/version.py | 2 +- 23 files changed, 1413 insertions(+), 620 deletions(-) create mode 100644 alembic/versions/467235201c70_download_sub_downloader.py create mode 100644 wrolpi/test/test_rss.py diff --git a/alembic/versions/467235201c70_download_sub_downloader.py b/alembic/versions/467235201c70_download_sub_downloader.py new file mode 100644 index 00000000..c8f11c34 --- /dev/null +++ b/alembic/versions/467235201c70_download_sub_downloader.py @@ -0,0 +1,29 @@ +"""Download.sub_downloader. + +Revision ID: 467235201c70 +Revises: 1a70196c05bb +Create Date: 2022-05-20 11:51:58.064020 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '467235201c70' +down_revision = '1a70196c05bb' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + session = Session(bind=op.get_bind()) + session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS sub_downloader TEXT') + + +def downgrade(): + session = Session(bind=op.get_bind()) + session.execute('ALTER TABLE download DROP COLUMN IF EXISTS sub_downloader') diff --git a/app/src/App.js b/app/src/App.js index 30e47f40..8ab5d1d8 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -11,7 +11,7 @@ import 'react-semantic-toasts/styles/react-semantic-alert.css'; import {AppsRoute} from "./components/Apps"; import {InventoryRoute} from "./components/Inventory"; import {ArchiveRoute, ArchivesList} from "./components/Archive"; -import {Saver} from "./components/Upload"; +import {Downloads, Saver} from "./components/Upload"; import {MoreButton, PageContainer, SearchInput} from "./components/Common"; import {FilesRoute} from "./components/Files"; import {useSearch, useUp, useVersion} from "./hooks/customHooks"; @@ -23,10 +23,7 @@ function Welcome() { useUp(); let body = ( - <> -
Save your media
- - + ); if (searchStr) { diff --git a/app/src/api.js b/app/src/api.js index c24ef47f..cd4e3a90 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -445,8 +445,11 @@ export async function getArchive(archiveId) { } } -export async function postDownload(urls, downloader) { - let body = {urls: urls, downloader: downloader}; +export async function postDownload(urls, downloader, frequency, sub_downloader) { + let body = {urls: urls, downloader: downloader, frequency: frequency || null}; + if (sub_downloader) { + body['sub_downloader'] = sub_downloader; + } let response = await apiPost(`${API_URI}/download`, body); return response; } diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 090f1fe9..9898582e 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -3,7 +3,8 @@ import { Button, Checkbox, Confirm, - Container, Dimmer, + Container, + Dimmer, Divider, Form, Header, @@ -337,20 +338,46 @@ function ClearFailedDownloads({callback}) { } class DownloadRow extends React.Component { + constructor(props) { + super(props); + this.state = { + errorModalOpen: false, + } + } + render() { - let {url, frequency, last_successful_download, status, location, next_download} = this.props; + let {url, frequency, last_successful_download, status, location, next_download, error} = this.props; + const {errorModalOpen} = this.state; + let positive = false; if (status === 'pending') { positive = true; } + + const errorModal = this.setState({errorModalOpen: false})} + onOpen={() => this.setState({errorModalOpen: true})} + open={errorModalOpen} + trigger={ + + ; + return ( {textEllipsis(url, 50)} {secondsToFrequency(frequency)} {last_successful_download ? secondsToDate(last_successful_download) : null} {secondsToDate(next_download)} - {location && View} + {error && errorModal}{location && View} ); } diff --git a/app/src/components/Common.js b/app/src/components/Common.js index bc227f3f..48bc2dae 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -221,9 +221,21 @@ export const frequencyOptions = [ {key: '90days', text: '90 Days', value: 7776000}, ]; +export const rssFrequencyOptions = [ + {key: 'once', text: 'Once', value: 0}, + {key: 'hourly', text: 'Hourly', value: 3600}, + {key: '3hours', text: '3 hours', value: 10800}, + {key: '12hours', text: '12 hours', value: 43200}, + {key: 'daily', text: 'Daily', value: 86400}, + {key: 'weekly', text: 'Weekly', value: 604800}, + {key: 'biweekly', text: 'Biweekly', value: 1209600}, + {key: '30days', text: '30 Days', value: 2592000}, + {key: '90days', text: '90 Days', value: 7776000}, +]; + export function secondsToFrequency(seconds) { - for (let i = 0; i < Object.keys(frequencyOptions).length; i++) { - let d = frequencyOptions[i]; + for (let i = 0; i < Object.keys(rssFrequencyOptions).length; i++) { + let d = rssFrequencyOptions[i]; if (d.value === seconds) { return d.text; } diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js index 321a3e37..14250fc5 100644 --- a/app/src/components/Upload.js +++ b/app/src/components/Upload.js @@ -1,8 +1,9 @@ -import React from "react"; -import {Button, Dropdown, Form, Header, Segment, TextArea} from "semantic-ui-react"; +import React, {useState} from "react"; +import {Button, Form, Header, Loader, TextArea} from "semantic-ui-react"; import {getDownloaders, postDownload} from "../api"; -import {WROLModeMessage} from "./Common"; -import {Link} from "react-router-dom"; +import {frequencyOptions, rssFrequencyOptions, WROLModeMessage} from "./Common"; +import Icon from "semantic-ui-react/dist/commonjs/elements/Icon"; +import Message from "semantic-ui-react/dist/commonjs/collections/Message"; const validUrl = /^(http|https):\/\/[^ "]+$/; @@ -13,23 +14,10 @@ class Downloader extends React.Component { urls: '', valid: true, pending: false, - downloaders: [], - downloader: null, + downloader: props.downloader, }; } - async componentDidMount() { - await this.fetchDownloaders(); - } - - fetchDownloaders = async () => { - let {downloaders} = await getDownloaders(); - downloaders = downloaders.map((i) => { - return {key: i.name, text: i.pretty_name || i.name, value: i.name} - }) - this.setState({downloaders}); - } - submitDownload = async () => { let {urls, downloader} = this.state; if (urls) { @@ -64,46 +52,265 @@ class Downloader extends React.Component { } render() { - let textareaPlaceholder = 'Enter one URL per line'; let disabled = !this.state.urls || !this.state.valid || this.state.pending || !this.state.downloader; + const {header} = this.props; return (
-
Enter the URLs you wish to save offline.
+
{header}
-