diff --git a/alembic/versions/184329ed5137_downloader.py b/alembic/versions/184329ed5137_downloader.py new file mode 100644 index 00000000..c24fb200 --- /dev/null +++ b/alembic/versions/184329ed5137_downloader.py @@ -0,0 +1,51 @@ +"""Downloader + +Revision ID: 184329ed5137 +Revises: 1870bec7c81e +Create Date: 2021-11-17 19:13:58.305140 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '184329ed5137' +down_revision = '1870bec7c81e' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute(''' + CREATE TABLE download ( + id SERIAL PRIMARY KEY, + url TEXT NOT NULL, + status TEXT DEFAULT 'new', + next_download TIMESTAMPTZ, + last_successful_download TIMESTAMPTZ, + frequency INT, + attempts INT DEFAULT 0 + ) + ''') + session.execute('ALTER TABLE video ADD COLUMN url TEXT') + session.execute('ALTER TABLE channel ADD COLUMN source_id TEXT') + session.execute('CREATE INDEX channel_source_id ON channel (source_id)') + + if not DOCKERIZED: + session.execute('ALTER TABLE public.download OWNER TO wrolpi') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('DROP TABLE IF EXISTS download') + session.execute('DROP INDEX IF EXISTS channel_source_id') + session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS source_id') + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS url') diff --git a/app/package.json b/app/package.json index ef135f5b..103b9680 100644 --- a/app/package.json +++ b/app/package.json @@ -9,6 +9,7 @@ "query-string": "^6.13.0", "react": "^16.13.1", "react-dom": "^16.13.1", + "react-dropzone": "^11.4.2", "react-router-dom": "^5.2.0", "react-scripts": "^3.4.1", "react-semantic-toasts": "^0.6.5", diff --git a/app/src/App.js b/app/src/App.js index e4a37c78..75c6ce47 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -4,21 +4,23 @@ import {NavBar} from "./components/Nav"; import {Route, Switch} from "react-router-dom"; import {FavoriteVideosPreview, VideosRoute, VideoWrapper, ViewedVideosPreview} from "./components/Videos"; import Admin from "./components/Admin"; -import {Container, Header, Segment} from "semantic-ui-react"; +import {Container, Divider, Header, Segment} from "semantic-ui-react"; import 'semantic-ui-offline/semantic.min.css'; import {SemanticToastContainer} from 'react-semantic-toasts'; import 'react-semantic-toasts/styles/react-semantic-alert.css'; import {AppsRoute} from "./components/Apps"; import {InventoryRoute} from "./components/Inventory"; import {ArchiveRoute} from "./components/Archive"; +import {Saver} from "./components/Upload"; function Welcome() { return ( -
Welcome to WROLPi!
-

- Take your internet, off-grid. -

+
Save your media
+ + + + @@ -45,6 +47,7 @@ export default function App() { + diff --git a/app/src/api.js b/app/src/api.js index 2205ff48..0c207a76 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -32,7 +32,16 @@ async function apiCall(url, method, body, ms = 10000) { try { // await the response or error. - return await timeoutPromise(ms, promise); + let response = await timeoutPromise(ms, promise); + if (response.status === 403 && (await response.json())['code'] === 17) { + toast({ + type: 'error', + title: 'WROL Mode is enabled!', + description: 'This functionality is not enabled while WROL Mode is enabled!', + time: 5000, + }); + } + return response; } catch (e) { // Timeout, or some other exception. if (e.message === 'promise timeout') { @@ -165,6 +174,13 @@ export async function saveConfig(config) { return await apiPatch(`${API_URI}/settings`, config); } +export async function getDownloads() { + let response = await apiGet(`${API_URI}/download`); + let data = await response.json(); + return data; +} + + export async function validateRegex(regex) { let response = await apiPost(`${API_URI}/valid_regex`, {regex}); return (await response.json())['valid']; @@ -172,7 +188,7 @@ export async function validateRegex(regex) { export async function favoriteVideo(video_id, favorite) { let body = {favorite: favorite, video_id}; - let response = await apiPost(`${VIDEOS_API}:favorite`, body); + let response = await apiPost(`${VIDEOS_API}/favorite`, body); return (await response.json())['favorite']; } @@ -182,28 +198,22 @@ export async function getStatistics() { } export async function refresh() { - let response = await apiPost(`${VIDEOS_API}:refresh`); + let response = await apiPost(`${VIDEOS_API}/refresh`); return await response.json(); } export async function download() { - let response = await apiPost(`${VIDEOS_API}:download`); + let response = await apiPost(`${VIDEOS_API}/download`); return await response.json(); } export async function downloadChannel(link) { - let url = `${VIDEOS_API}:download/${link}`; + let url = `${VIDEOS_API}/download/${link}`; await fetch(url, {method: 'POST'}); } -export async function downloadHistory() { - let url = `${VIDEOS_API}/download_history`; - let resp = await apiGet(url); - return await resp.json(); -} - export async function refreshChannel(link) { - let url = `${VIDEOS_API}:refresh/${link}`; + let url = `${VIDEOS_API}/refresh/${link}`; await fetch(url, {method: 'POST'}); } @@ -316,7 +326,7 @@ export async function searchURLs(offset, limit, domain = null) { } export async function refreshArchives() { - return await apiPost(`${ARCHIVES_API}:refresh`); + return await apiPost(`${ARCHIVES_API}/refresh`); } export async function fetchDomains() { @@ -328,3 +338,15 @@ export async function fetchDomains() { throw Error(`Unable to fetch domains`); } } + +export async function postDownload(urls) { + let body = {'urls': urls}; + let response = await apiPost(`${API_URI}/download`, body); + return response; +} + + +export async function killDownload(download_id) { + let response = await apiPost(`${API_URI}/download/${download_id}/kill`); + return response; +} diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 6e535e68..b7035cc7 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -1,7 +1,20 @@ import React from 'react'; -import {Button, Checkbox, Container, Divider, Form, Header, Loader, Tab} from "semantic-ui-react"; -import {getConfig, saveConfig} from "../api"; +import { + Button, + Checkbox, + Confirm, + Container, + Divider, + Form, + Header, + Loader, + Placeholder, + Tab, + Table +} from "semantic-ui-react"; +import {getConfig, getDownloads, killDownload, postDownload, saveConfig} from "../api"; import TimezoneSelect from 'react-timezone-select'; +import {secondsToDate, secondsToFrequency} from "./Common"; class Settings extends React.Component { @@ -152,11 +165,217 @@ class WROLMode extends React.Component { } } +class DownloadRow extends React.Component { + + render() { + let {url, frequency, last_successful_download, status, next_download} = this.props; + let positive = false; + if (status === 'pending') { + positive = true; + } + return ( + + {url} + {secondsToFrequency(frequency)} + {last_successful_download ? secondsToDate(last_successful_download) : null} + {secondsToDate(next_download)} + {status} + + ); + } +} + +class StoppableRow extends React.Component { + constructor(props) { + super(props); + this.state = { + stopOpen: false, + startOpen: false, + }; + } + + openStop = () => { + this.setState({stopOpen: true}); + } + + closeStop = () => { + this.setState({stopOpen: false}); + } + + openStart = () => { + this.setState({startOpen: true}); + } + + closeStart = () => { + this.setState({startOpen: false}); + } + + handleStop = async (e) => { + e.preventDefault(); + await killDownload(this.props.id); + this.closeStop(); + await this.props.fetchDownloads(); + }; + + handleStart = async (e) => { + e.preventDefault(); + await postDownload(`${this.props.url}`); + this.closeStart(); + await this.props.fetchDownloads(); + }; + + render() { + let {url, last_successful_download, status} = this.props; + let {stopOpen, startOpen} = this.state; + + let buttonCell = ; + let positive = false; + let negative = false; + let warning = false; + if (status === 'pending' || status === 'new') { + positive = status === 'pending'; + buttonCell = ( + + + + + ); + } else if (status === 'failed' || status === 'deferred') { + negative = status === 'failed'; + warning = status === 'deferred'; + buttonCell = ( + + + + + ); + } + + return ( + + {url} + {last_successful_download ? secondsToDate(last_successful_download) : null} + {status} + {buttonCell} + + ); + } +} + +class Downloads extends React.Component { + constructor(props) { + super(props); + this.state = { + once_downloads: null, + recurring_downloads: null, + pending_downloads: null, + }; + } + + async componentDidMount() { + await this.fetchDownloads(); + } + + fetchDownloads = async () => { + let data = await getDownloads(); + this.setState({ + once_downloads: data.once_downloads, + recurring_downloads: data.recurring_downloads, + pending_downloads: data.pending_downloads, + }); + } + + render() { + let tablePlaceholder = ( + + + + + ); + + let stoppableHeader = ( + + + URL + Completed At + Status + Control + + + ); + + let nonStoppableHeader = ( + + + URL + Download Frequency + Last Successful Download + Next Download + Status + + + ); + + let onceTable = tablePlaceholder; + if (this.state.once_downloads !== null && this.state.once_downloads.length === 0) { + onceTable =

No downloads are scheduled to be downloaded.

+ } else if (this.state.once_downloads !== null) { + onceTable = ( + {stoppableHeader} + + {this.state.once_downloads.map((i) => )} + +
); + } + + let recurringTable = tablePlaceholder; + if (this.state.recurring_downloads !== null && this.state.recurring_downloads.length === 0) { + recurringTable =

No downloads are scheduled to be downloaded.

+ } else if (this.state.recurring_downloads !== null) { + recurringTable = ( + {nonStoppableHeader} + + {this.state.recurring_downloads.map((i) => )} + +
); + } + + return ( +
+
Downloads
+ {onceTable} + +
Recurring Downloads
+ {recurringTable} +
+ ); + } +} + class Admin extends React.Component { render() { const panes = [ + {menuItem: 'Downloads', render: () => }, {menuItem: 'Settings', render: () => }, {menuItem: 'WROL Mode', render: () => }, ]; diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index af6791d8..666e562c 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -348,8 +348,17 @@ class ArchiveAddForm extends APIForm { handleSubmit = async (e) => { e.preventDefault(); - await postArchive(this.state.inputs.url); - this.setState({inputs: {url: ''}}, this.fetchURLs); + this.setLoading(); + try { + let response = await postArchive(this.state.inputs.url); + if (response.status === 204) { + this.setState({inputs: {url: ''}, success: true}, this.fetchURLs); + } else { + this.setState({loading: false, success: undefined, error: true}); + } + } finally { + this.clearLoading(); + } } fetchURLs = async (e) => { @@ -365,10 +374,13 @@ class ArchiveAddForm extends APIForm { - Archive + Archive diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index 6cd38bf6..ab982aea 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -101,13 +101,11 @@ class ChannelPage extends APIForm { match_regex: '', generate_posters: true, calculate_duration: true, - download_frequency: 604800, + download_frequency: null, }, errors: {}, }; - this.calculateDuration = React.createRef(); - this.generatePosters = React.createRef(); this.mkdir = React.createRef(); } @@ -304,7 +302,7 @@ class ChannelPage extends APIForm { - - - - this.handleCheckbox(this.generatePosters)} - /> - - - this.handleCheckbox(this.calculateDuration)} - /> - @@ -399,24 +374,24 @@ class ChannelPage extends APIForm { {!this.props.create && - <> - - this.setState({deleteOpen: false})} - onConfirm={this.handleConfirm} - /> - - - + <> + + this.setState({deleteOpen: false})} + onConfirm={this.handleConfirm} + /> + + + }
@@ -455,13 +430,13 @@ class ChannelRow extends React.Component { {channel.video_count} - {channel.url ? secondsToDate(channel.info_date) : null} + {channel.url && channel.info_date ? secondsToDate(channel.info_date) : null} - {channel.url ? secondsToDate(channel.next_download) : null} + {channel.url && channel.next_download ? secondsToDate(channel.next_download) : null} - {channel.url ? secondsToFrequency(channel.download_frequency) : null} + {channel.url && channel.download_frequency ? secondsToFrequency(channel.download_frequency) : null} Edit @@ -486,16 +461,17 @@ class MobileChannelRow extends ChannelRow {

- Last Update: {channel.url ? secondsToDate(channel.info_date) : null} + Last Update: {channel.url && channel.info_date ? secondsToDate(channel.info_date) : null}

- Next Update: {channel.url ? secondsToDate(channel.next_download) : null} + Next Update: {channel.url && channel.next_download ? secondsToDate(channel.next_download) : null}

Videos: {channel.video_count}

- Frequency: {channel.url ? secondsToFrequency(channel.download_frequency) : null} + Frequency: {channel.url && channel.download_frequency ? + secondsToFrequency(channel.download_frequency) : null}

@@ -621,7 +597,7 @@ export class Channels extends React.Component { - +
{this.state.results.map((channel) => )} diff --git a/app/src/components/Common.js b/app/src/components/Common.js index d37c0250..925c75d7 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -87,12 +87,16 @@ export function uploadDate(d) { } export function VideoCard({video}) { - let channel = video.channel; - let channel_url = `/videos/channel/${channel.link}/video`; - + let video_url = `/videos/video/${video.id}`; let upload_date = uploadDate(video.upload_date); - let video_url = `/videos/channel/${channel.link}/video/${video.id}`; - let poster_url = video.poster_path ? `/media/${channel.directory}/${encodeURIComponent(video.poster_path)}` : null; + // A video may not have a channel. + let channel = video.channel ? video.channel : null; + let channel_url = null; + if (channel) { + channel_url = `/videos/channel/${channel.link}/video`; + video_url = `/videos/channel/${channel.link}/video/${video.id}`; + } + let poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null; return ( @@ -110,9 +114,10 @@ export function VideoCard({video}) { - - {channel.name} - + {channel && + + {channel.name} + }

{upload_date}

@@ -229,6 +234,27 @@ export function humanFileSize(bytes, si = false, dp = 1) { return bytes.toFixed(dp) + ' ' + units[u]; } +export function humanNumber(num, dp = 1) { + // Convert large numbers to a more human readable format. + // >> humanNumber(1000) + // 1.0k + // >> humanNumber(1500000) + // 1.5m + const divisor = 1000; + if (Math.abs(num) < divisor) { + return num; + } + const units = ['k', 'm', 'b']; + let i = -1; + const r = 10 ** dp; + do { + num /= divisor; + ++i; + } while (Math.round(Math.abs(num) * r) / r >= divisor && i < units.length - 1); + + return num.toFixed(dp) + units[i]; +} + export class APIForm extends React.Component { constructor(props) { @@ -317,7 +343,7 @@ export class APIForm extends React.Component { this.setState({ disabled: true, error: false, - errors: this.getEmptyErrors, + errors: this.getEmptyErrors(), loading: true, success: false, }) diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js new file mode 100644 index 00000000..de4356cd --- /dev/null +++ b/app/src/components/Upload.js @@ -0,0 +1,74 @@ +import React from "react"; +import {Button, Form, Header, Segment, TextArea} from "semantic-ui-react"; +import {postDownload} from "../api"; + +const validUrl = /^(http|https):\/\/[^ "]+$/; + +class Downloader extends React.Component { + constructor(props) { + super(props); + this.state = { + urls: '', + valid: true, + }; + } + + submitDownload = async () => { + if (this.state.urls) { + let response = await postDownload(this.state.urls); + if (response.status === 204) { + this.setState({urls: ''}); + } + } + } + + handleInputChange = async (event, {name, value}) => { + this.setState({[name]: value}, this.validateUrls); + } + + validateUrls = async () => { + // Validate that all URLs in state are valid. + if (this.state.urls) { + let urls = this.state.urls.split(/\r?\n/); + let valid = true; + urls.forEach((url) => { + if (valid && url && !validUrl.test(url)) { + valid = false; + } + }) + this.setState({valid}); + } + } + + render() { + let textareaPlaceholder = 'Enter one URL per line'; + let disabled = !this.state.urls || !this.state.valid; + + return ( +
+
Enter the URLs you wish to bring offline.
+