From 46796f783ac323fb809b0f6b9bcd4b6a7453c50b Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Mon, 20 Dec 2021 16:08:13 -0700 Subject: [PATCH] Arbitrary videos can be downloaded. (#63) * Squashed commit of the following: commit c527475fcb4d4906457be5c10f0e9c6dee7fedc1 Author: Roland Date: Tue Dec 14 22:16:36 2021 -0700 Only saving a favorite video to the config if it has a source_id. commit ae8b712ebeaef467b8d5a5b87ca0f4f276373a0d Author: Roland Date: Tue Dec 14 22:14:26 2021 -0700 Ading description and changing Save button to Download. commit 03a72be6dfdfa383915950a80e3f1ee9febf6705 Author: Roland Date: Tue Dec 14 22:09:44 2021 -0700 Removing uploader for now. commit 5d0231284bf84e459687f6ab903b06bf4e3762bd Author: Roland Date: Tue Dec 14 22:07:17 2021 -0700 Removing unique data from example info. commit 9cc721a32d4715d9d6433e51db3b35e039c78d68 Author: Roland Date: Tue Dec 14 22:02:33 2021 -0700 Updating downloader tests. commit 8e193b03349ae4613f105418cd3e8176c7101304 Author: Roland Date: Tue Dec 14 21:53:31 2021 -0700 Toasting error when WROL Mode is enabled. Disabling new downloads during WROL Mode. commit 48c4311c31db8f2d2c8dd0a65f979a5f0a49a911 Author: Roland Date: Tue Dec 14 21:45:06 2021 -0700 Raising error if download exits without a 0. Clearing kill event when starting a new download. commit 54ba3a890d59facc837410d34bf7afb40db3b685 Author: Roland Date: Tue Dec 14 21:16:46 2021 -0700 Downloads can be restarted and killed. commit bd60751262247259c3e9ceb5116e017d6314c219 Author: Roland Date: Mon Dec 13 16:28:18 2021 -0700 Downloads can be started again. Downloads are sorted on Downloads page. commit ea7f39231b92982ae0b263b76c5bd1902cc8169d Author: Roland Date: Sat Dec 11 21:24:16 2021 -0700 Building Downloads tab. commit d3321872ac3743ff6df781b07ef0fb537f1d4c98 Author: Roland Date: Sat Dec 11 20:41:46 2021 -0700 Not scheduling channels withtout a URL/frequency. commit 84f268a0d4f58c85f86b6377900492e0766f6619 Author: Roland Date: Sat Dec 11 19:56:08 2021 -0700 Channel downloads will be spread out when each channel is created. commit 26fa6beea38ef3f8d6b7bf218f18222a51a87a60 Author: Roland Date: Sat Dec 11 13:08:11 2021 -0700 Recurring downloads are renewed on startup. commit fc2ab850533f1c3ad073f4692c7dea2be43e85d6 Author: Roland Date: Fri Dec 10 17:03:57 2021 -0700 Fetching download again after do_download. commit fdadad327fc09c17a49eb63b11e632fb72cc3195 Author: Roland Date: Fri Dec 10 16:20:51 2021 -0700 Fixing error with test. Mock was not being reset. commit 070dee238e53377acd22097c63cf3794e68ed423 Author: Roland Date: Fri Dec 10 15:36:39 2021 -0700 Fixed bug in date handling (big surprise). commit e53a8992268d68620c679455adaec3a590f20602 Author: Roland Date: Wed Dec 1 16:48:36 2021 -0700 Using chunks for regular channel refresh. commit 3bd60c58d1bef0226325f321978c0434b36b3be5 Author: Roland Date: Wed Dec 1 15:22:02 2021 -0700 Reformatting. commit 4fdc325f25a32c311ca6543fe321624b2bd564b2 Author: Roland Date: Wed Dec 1 15:13:34 2021 -0700 Creating function that refreshes NO CHANNEL videos. commit d67feafb6c2994f5a4a1295920a70db369cc6816 Author: Roland Date: Wed Dec 1 14:35:54 2021 -0700 Removing unused functions. commit 6b3918e4e78fbac64920f85c4de0df873ec366a1 Author: Roland Date: Wed Dec 1 14:31:48 2021 -0700 Removing reporter and events. This will be replaced in the future with some sort of SSE. commit 54fa1b0f5e93ca25c80b780dcdc3876b700273b1 Author: Roland Date: Sun Nov 28 17:55:43 2021 -0700 Better error handling for archive button. commit c31d124e511b27f626ed2df71486d47302b396f3 Author: Roland Date: Sun Nov 28 17:29:43 2021 -0700 Displaying View Count in a human readable format. commit 0ee2bd65588fd93b475cad6144e76f1b45191b39 Author: Roland Date: Sat Nov 27 09:29:41 2021 -0700 Fixing tests. commit e64aff32bab95c311ea3747e1a21b58b99750d5e Author: Roland Date: Sat Nov 27 08:59:51 2021 -0700 Handling when a Video has no Channel in UI. commit b9549ecd5aac03c002adca7b7667f0a7189b662a Author: Roland Date: Fri Nov 26 08:35:36 2021 -0700 Downloaders can raise UnrecoverableDownloadError to signal a downlaod should not be retried. commit da88798e2584d3e84900a70e34166c8c208c64fb Author: Roland Date: Wed Nov 24 20:40:29 2021 -0700 Downloaders can refuse a download, which will then be deleted. commit b865d856121db5ec8a3660c5d3c13b9846fde688 Author: Roland Date: Wed Nov 24 19:35:43 2021 -0700 Removing relative path handling. commit 6d7757870b02ba030d78b3ef7e43f28b906eea61 Author: Roland Date: Wed Nov 24 19:28:27 2021 -0700 Using __json__ for Video and Channel models. commit 649e4d6a61608272759d1e7ee02b490bb5c20b3e Author: Roland Date: Wed Nov 24 18:23:38 2021 -0700 Fixing video links in UI. commit 9d081af12742070b8519bd752812130cb5dbe393 Author: Roland Date: Wed Nov 24 14:42:38 2021 -0700 Fixing docs. commit 1202cbe0f9ae744d7801b53a5071cded8ea39924 Author: Roland Date: Wed Nov 24 14:40:36 2021 -0700 Fixing wrolpi tests. commit 3f6af8cc1123f2f63a3fc0f21ec4d994c642cf13 Author: Roland Date: Wed Nov 24 14:13:35 2021 -0700 Fixing tests to use MediaPath. commit faec04f2a5e4c267a502c03884940a9c5bcfe387 Author: Roland Date: Fri Nov 19 10:21:20 2021 -0700 Testing when a video has no channel. commit cbf22aad7a0b72e61685e0236e843554b57c4af1 Author: Roland Date: Thu Nov 18 16:24:03 2021 -0700 Using MediaPath for Video and Channel models. Testing VideoDownloader.do_download. commit 7916ed61e43439e6cd60f23ce3fdf6c1a75bf02d Author: Roland Date: Thu Nov 18 12:43:44 2021 -0700 Channels/Videos downloaders can get gotten. commit 808ef796cec01db6e7009065f9e26d25d9500ce2 Author: Roland Date: Wed Nov 17 21:45:19 2021 -0700 Upgrading to Sanic 21.9.1. commit ea32813be28e78393ac456ee782a7245c098188f Author: Roland Date: Wed Nov 17 20:29:01 2021 -0700 Creating Downloader and DownloaderManager classes which handle downloading URLs. commit e859a77a77277ddf8b6e221ec9c9383c8ab91d66 Author: Roland Date: Wed Nov 17 12:41:39 2021 -0700 Creating UI segement which allows the download/upload of media. * Displaying all recurring downloads. * Not downloading channels with empty url. * Showing positive row for recurring downloads. * Removing default last_successful_download. * Only displaying Last Successful Download when there is one. * Refreshing videos before finding missing videos. * Using short video path name for log. * Removing download history feature in favor of new Downloads page. * Only showing info date if it exists. * Making get_or_create_channel more reliable. * Allowing null for channel download frequency. * Logging when searching for channel. * Storing channel's source_id in config. * Importing using link instead of name. * Getting channel id when importing channels. * Removing redunant parentheses from repr. * Preserving database channel values if there are not values in the config. * Only highlighting pending downloads. * Deleting old downloads periodically. * Sorting recurring by next_download/frequency. * Creating fixtures for channels, media directory and db session. * Removing unused function. Wrapping config for tests. * Deleting channel download when deleting a channel. * Testing that a channel's download is deleted when the channel is deleted. * Updating youtube-dl * Switching from youtube-dl in favor of yt-dlp. * Sorting once-downlaods by last_successful_download then id. * Adding special handling to video_url_resolver for full urls. * Consolidating downloader methods and functions. Reversing once-download sort order so the most recently downloaded are at the top. * Getting session if one is not provided. * Adding test for create_downloads. * Not passing format option when getting filename. * Linting Signed-off-by: Roland --- alembic/versions/184329ed5137_downloader.py | 51 ++ app/package.json | 1 + app/src/App.js | 13 +- app/src/api.js | 48 +- app/src/components/Admin.js | 223 +++++++- app/src/components/Archive.js | 20 +- app/src/components/Channels.js | 80 +-- app/src/components/Common.js | 44 +- app/src/components/Upload.js | 74 +++ app/src/components/VideoPlayer.js | 23 +- app/src/components/Videos.js | 30 +- conftest.py | 6 +- docker-compose.yml | 1 - main.py | 56 +- modules/archive/__init__.py | 25 + modules/archive/api.py | 2 +- modules/archive/lib.py | 21 +- modules/archive/test/test_lib.py | 2 +- modules/inventory/models.py | 6 +- modules/videos/__init__.py | 1 + modules/videos/api.py | 112 +--- modules/videos/captions.py | 12 +- modules/videos/channel/api.py | 22 +- modules/videos/channel/lib.py | 136 +++-- modules/videos/channel/test/test_api.py | 305 +++++++--- modules/videos/channel/test/test_channel.py | 21 - modules/videos/common.py | 105 ++-- modules/videos/conftest.py | 33 ++ modules/videos/downloader.py | 393 ++++++++----- modules/videos/lib.py | 169 +++--- modules/videos/models.py | 173 +++--- modules/videos/schema.py | 14 +- modules/videos/test/common.py | 22 +- modules/videos/test/test_captions.py | 50 +- modules/videos/test/test_common.py | 43 +- modules/videos/test/test_downloader.py | 448 ++++++++++----- modules/videos/test/test_lib.py | 74 +++ .../videos/test/test_video_url_resolver.py | 15 + modules/videos/video/api.py | 6 +- modules/videos/video/lib.py | 8 +- modules/videos/video/test/test_api.py | 37 +- modules/videos/video_url_resolver.py | 24 + requirements.txt | 9 +- .../wrolpi => }/big_buck_bunny_720p_1mb.mp4 | Bin wrolpi/__init__.py | 41 +- wrolpi/common.py | 164 ++---- wrolpi/conftest.py | 92 ++- wrolpi/dates.py | 5 +- wrolpi/db.py | 25 +- wrolpi/downloader.py | 527 ++++++++++++++++++ wrolpi/errors.py | 27 + wrolpi/media_path.py | 6 + wrolpi/root_api.py | 123 ++-- wrolpi/schema.py | 4 + wrolpi/test/common.py | 116 +--- wrolpi/test/test_api.py | 74 --- wrolpi/test/test_common.py | 212 ++----- wrolpi/test/test_dates.py | 36 +- wrolpi/test/test_downloader.py | 303 ++++++++++ wrolpi/test/test_media_path.py | 3 + wrolpi/test/test_root_api.py | 156 ++++++ 61 files changed, 3347 insertions(+), 1525 deletions(-) create mode 100644 alembic/versions/184329ed5137_downloader.py create mode 100644 app/src/components/Upload.js delete mode 100644 modules/videos/channel/test/test_channel.py create mode 100644 modules/videos/conftest.py create mode 100644 modules/videos/test/test_lib.py create mode 100644 modules/videos/test/test_video_url_resolver.py create mode 100644 modules/videos/video_url_resolver.py rename test/{videos/wrolpi => }/big_buck_bunny_720p_1mb.mp4 (100%) create mode 100644 wrolpi/downloader.py delete mode 100644 wrolpi/test/test_api.py create mode 100644 wrolpi/test/test_downloader.py create mode 100644 wrolpi/test/test_root_api.py diff --git a/alembic/versions/184329ed5137_downloader.py b/alembic/versions/184329ed5137_downloader.py new file mode 100644 index 00000000..c24fb200 --- /dev/null +++ b/alembic/versions/184329ed5137_downloader.py @@ -0,0 +1,51 @@ +"""Downloader + +Revision ID: 184329ed5137 +Revises: 1870bec7c81e +Create Date: 2021-11-17 19:13:58.305140 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '184329ed5137' +down_revision = '1870bec7c81e' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute(''' + CREATE TABLE download ( + id SERIAL PRIMARY KEY, + url TEXT NOT NULL, + status TEXT DEFAULT 'new', + next_download TIMESTAMPTZ, + last_successful_download TIMESTAMPTZ, + frequency INT, + attempts INT DEFAULT 0 + ) + ''') + session.execute('ALTER TABLE video ADD COLUMN url TEXT') + session.execute('ALTER TABLE channel ADD COLUMN source_id TEXT') + session.execute('CREATE INDEX channel_source_id ON channel (source_id)') + + if not DOCKERIZED: + session.execute('ALTER TABLE public.download OWNER TO wrolpi') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('DROP TABLE IF EXISTS download') + session.execute('DROP INDEX IF EXISTS channel_source_id') + session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS source_id') + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS url') diff --git a/app/package.json b/app/package.json index ef135f5b..103b9680 100644 --- a/app/package.json +++ b/app/package.json @@ -9,6 +9,7 @@ "query-string": "^6.13.0", "react": "^16.13.1", "react-dom": "^16.13.1", + "react-dropzone": "^11.4.2", "react-router-dom": "^5.2.0", "react-scripts": "^3.4.1", "react-semantic-toasts": "^0.6.5", diff --git a/app/src/App.js b/app/src/App.js index e4a37c78..75c6ce47 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -4,21 +4,23 @@ import {NavBar} from "./components/Nav"; import {Route, Switch} from "react-router-dom"; import {FavoriteVideosPreview, VideosRoute, VideoWrapper, ViewedVideosPreview} from "./components/Videos"; import Admin from "./components/Admin"; -import {Container, Header, Segment} from "semantic-ui-react"; +import {Container, Divider, Header, Segment} from "semantic-ui-react"; import 'semantic-ui-offline/semantic.min.css'; import {SemanticToastContainer} from 'react-semantic-toasts'; import 'react-semantic-toasts/styles/react-semantic-alert.css'; import {AppsRoute} from "./components/Apps"; import {InventoryRoute} from "./components/Inventory"; import {ArchiveRoute} from "./components/Archive"; +import {Saver} from "./components/Upload"; function Welcome() { return ( -
Welcome to WROLPi!
-

- Take your internet, off-grid. -

+
Save your media
+ + + + @@ -45,6 +47,7 @@ export default function App() { + diff --git a/app/src/api.js b/app/src/api.js index 2205ff48..0c207a76 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -32,7 +32,16 @@ async function apiCall(url, method, body, ms = 10000) { try { // await the response or error. - return await timeoutPromise(ms, promise); + let response = await timeoutPromise(ms, promise); + if (response.status === 403 && (await response.json())['code'] === 17) { + toast({ + type: 'error', + title: 'WROL Mode is enabled!', + description: 'This functionality is not enabled while WROL Mode is enabled!', + time: 5000, + }); + } + return response; } catch (e) { // Timeout, or some other exception. if (e.message === 'promise timeout') { @@ -165,6 +174,13 @@ export async function saveConfig(config) { return await apiPatch(`${API_URI}/settings`, config); } +export async function getDownloads() { + let response = await apiGet(`${API_URI}/download`); + let data = await response.json(); + return data; +} + + export async function validateRegex(regex) { let response = await apiPost(`${API_URI}/valid_regex`, {regex}); return (await response.json())['valid']; @@ -172,7 +188,7 @@ export async function validateRegex(regex) { export async function favoriteVideo(video_id, favorite) { let body = {favorite: favorite, video_id}; - let response = await apiPost(`${VIDEOS_API}:favorite`, body); + let response = await apiPost(`${VIDEOS_API}/favorite`, body); return (await response.json())['favorite']; } @@ -182,28 +198,22 @@ export async function getStatistics() { } export async function refresh() { - let response = await apiPost(`${VIDEOS_API}:refresh`); + let response = await apiPost(`${VIDEOS_API}/refresh`); return await response.json(); } export async function download() { - let response = await apiPost(`${VIDEOS_API}:download`); + let response = await apiPost(`${VIDEOS_API}/download`); return await response.json(); } export async function downloadChannel(link) { - let url = `${VIDEOS_API}:download/${link}`; + let url = `${VIDEOS_API}/download/${link}`; await fetch(url, {method: 'POST'}); } -export async function downloadHistory() { - let url = `${VIDEOS_API}/download_history`; - let resp = await apiGet(url); - return await resp.json(); -} - export async function refreshChannel(link) { - let url = `${VIDEOS_API}:refresh/${link}`; + let url = `${VIDEOS_API}/refresh/${link}`; await fetch(url, {method: 'POST'}); } @@ -316,7 +326,7 @@ export async function searchURLs(offset, limit, domain = null) { } export async function refreshArchives() { - return await apiPost(`${ARCHIVES_API}:refresh`); + return await apiPost(`${ARCHIVES_API}/refresh`); } export async function fetchDomains() { @@ -328,3 +338,15 @@ export async function fetchDomains() { throw Error(`Unable to fetch domains`); } } + +export async function postDownload(urls) { + let body = {'urls': urls}; + let response = await apiPost(`${API_URI}/download`, body); + return response; +} + + +export async function killDownload(download_id) { + let response = await apiPost(`${API_URI}/download/${download_id}/kill`); + return response; +} diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 6e535e68..b7035cc7 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -1,7 +1,20 @@ import React from 'react'; -import {Button, Checkbox, Container, Divider, Form, Header, Loader, Tab} from "semantic-ui-react"; -import {getConfig, saveConfig} from "../api"; +import { + Button, + Checkbox, + Confirm, + Container, + Divider, + Form, + Header, + Loader, + Placeholder, + Tab, + Table +} from "semantic-ui-react"; +import {getConfig, getDownloads, killDownload, postDownload, saveConfig} from "../api"; import TimezoneSelect from 'react-timezone-select'; +import {secondsToDate, secondsToFrequency} from "./Common"; class Settings extends React.Component { @@ -152,11 +165,217 @@ class WROLMode extends React.Component { } } +class DownloadRow extends React.Component { + + render() { + let {url, frequency, last_successful_download, status, next_download} = this.props; + let positive = false; + if (status === 'pending') { + positive = true; + } + return ( + + {url} + {secondsToFrequency(frequency)} + {last_successful_download ? secondsToDate(last_successful_download) : null} + {secondsToDate(next_download)} + {status} + + ); + } +} + +class StoppableRow extends React.Component { + constructor(props) { + super(props); + this.state = { + stopOpen: false, + startOpen: false, + }; + } + + openStop = () => { + this.setState({stopOpen: true}); + } + + closeStop = () => { + this.setState({stopOpen: false}); + } + + openStart = () => { + this.setState({startOpen: true}); + } + + closeStart = () => { + this.setState({startOpen: false}); + } + + handleStop = async (e) => { + e.preventDefault(); + await killDownload(this.props.id); + this.closeStop(); + await this.props.fetchDownloads(); + }; + + handleStart = async (e) => { + e.preventDefault(); + await postDownload(`${this.props.url}`); + this.closeStart(); + await this.props.fetchDownloads(); + }; + + render() { + let {url, last_successful_download, status} = this.props; + let {stopOpen, startOpen} = this.state; + + let buttonCell = ; + let positive = false; + let negative = false; + let warning = false; + if (status === 'pending' || status === 'new') { + positive = status === 'pending'; + buttonCell = ( + + + + + ); + } else if (status === 'failed' || status === 'deferred') { + negative = status === 'failed'; + warning = status === 'deferred'; + buttonCell = ( + + + + + ); + } + + return ( + + {url} + {last_successful_download ? secondsToDate(last_successful_download) : null} + {status} + {buttonCell} + + ); + } +} + +class Downloads extends React.Component { + constructor(props) { + super(props); + this.state = { + once_downloads: null, + recurring_downloads: null, + pending_downloads: null, + }; + } + + async componentDidMount() { + await this.fetchDownloads(); + } + + fetchDownloads = async () => { + let data = await getDownloads(); + this.setState({ + once_downloads: data.once_downloads, + recurring_downloads: data.recurring_downloads, + pending_downloads: data.pending_downloads, + }); + } + + render() { + let tablePlaceholder = ( + + + + + ); + + let stoppableHeader = ( + + + URL + Completed At + Status + Control + + + ); + + let nonStoppableHeader = ( + + + URL + Download Frequency + Last Successful Download + Next Download + Status + + + ); + + let onceTable = tablePlaceholder; + if (this.state.once_downloads !== null && this.state.once_downloads.length === 0) { + onceTable =

No downloads are scheduled to be downloaded.

+ } else if (this.state.once_downloads !== null) { + onceTable = ( + {stoppableHeader} + + {this.state.once_downloads.map((i) => )} + +
); + } + + let recurringTable = tablePlaceholder; + if (this.state.recurring_downloads !== null && this.state.recurring_downloads.length === 0) { + recurringTable =

No downloads are scheduled to be downloaded.

+ } else if (this.state.recurring_downloads !== null) { + recurringTable = ( + {nonStoppableHeader} + + {this.state.recurring_downloads.map((i) => )} + +
); + } + + return ( +
+
Downloads
+ {onceTable} + +
Recurring Downloads
+ {recurringTable} +
+ ); + } +} + class Admin extends React.Component { render() { const panes = [ + {menuItem: 'Downloads', render: () => }, {menuItem: 'Settings', render: () => }, {menuItem: 'WROL Mode', render: () => }, ]; diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index af6791d8..666e562c 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -348,8 +348,17 @@ class ArchiveAddForm extends APIForm { handleSubmit = async (e) => { e.preventDefault(); - await postArchive(this.state.inputs.url); - this.setState({inputs: {url: ''}}, this.fetchURLs); + this.setLoading(); + try { + let response = await postArchive(this.state.inputs.url); + if (response.status === 204) { + this.setState({inputs: {url: ''}, success: true}, this.fetchURLs); + } else { + this.setState({loading: false, success: undefined, error: true}); + } + } finally { + this.clearLoading(); + } } fetchURLs = async (e) => { @@ -365,10 +374,13 @@ class ArchiveAddForm extends APIForm { - Archive + Archive diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index 6cd38bf6..ab982aea 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -101,13 +101,11 @@ class ChannelPage extends APIForm { match_regex: '', generate_posters: true, calculate_duration: true, - download_frequency: 604800, + download_frequency: null, }, errors: {}, }; - this.calculateDuration = React.createRef(); - this.generatePosters = React.createRef(); this.mkdir = React.createRef(); } @@ -304,7 +302,7 @@ class ChannelPage extends APIForm { - - - - this.handleCheckbox(this.generatePosters)} - /> - - - this.handleCheckbox(this.calculateDuration)} - /> - @@ -399,24 +374,24 @@ class ChannelPage extends APIForm { {!this.props.create && - <> - - this.setState({deleteOpen: false})} - onConfirm={this.handleConfirm} - /> - - - + <> + + this.setState({deleteOpen: false})} + onConfirm={this.handleConfirm} + /> + + + }
@@ -455,13 +430,13 @@ class ChannelRow extends React.Component { {channel.video_count} - {channel.url ? secondsToDate(channel.info_date) : null} + {channel.url && channel.info_date ? secondsToDate(channel.info_date) : null} - {channel.url ? secondsToDate(channel.next_download) : null} + {channel.url && channel.next_download ? secondsToDate(channel.next_download) : null} - {channel.url ? secondsToFrequency(channel.download_frequency) : null} + {channel.url && channel.download_frequency ? secondsToFrequency(channel.download_frequency) : null} Edit @@ -486,16 +461,17 @@ class MobileChannelRow extends ChannelRow {

- Last Update: {channel.url ? secondsToDate(channel.info_date) : null} + Last Update: {channel.url && channel.info_date ? secondsToDate(channel.info_date) : null}

- Next Update: {channel.url ? secondsToDate(channel.next_download) : null} + Next Update: {channel.url && channel.next_download ? secondsToDate(channel.next_download) : null}

Videos: {channel.video_count}

- Frequency: {channel.url ? secondsToFrequency(channel.download_frequency) : null} + Frequency: {channel.url && channel.download_frequency ? + secondsToFrequency(channel.download_frequency) : null}

@@ -621,7 +597,7 @@ export class Channels extends React.Component { - +
{this.state.results.map((channel) => )} diff --git a/app/src/components/Common.js b/app/src/components/Common.js index d37c0250..925c75d7 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -87,12 +87,16 @@ export function uploadDate(d) { } export function VideoCard({video}) { - let channel = video.channel; - let channel_url = `/videos/channel/${channel.link}/video`; - + let video_url = `/videos/video/${video.id}`; let upload_date = uploadDate(video.upload_date); - let video_url = `/videos/channel/${channel.link}/video/${video.id}`; - let poster_url = video.poster_path ? `/media/${channel.directory}/${encodeURIComponent(video.poster_path)}` : null; + // A video may not have a channel. + let channel = video.channel ? video.channel : null; + let channel_url = null; + if (channel) { + channel_url = `/videos/channel/${channel.link}/video`; + video_url = `/videos/channel/${channel.link}/video/${video.id}`; + } + let poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null; return ( @@ -110,9 +114,10 @@ export function VideoCard({video}) { - - {channel.name} - + {channel && + + {channel.name} + }

{upload_date}

@@ -229,6 +234,27 @@ export function humanFileSize(bytes, si = false, dp = 1) { return bytes.toFixed(dp) + ' ' + units[u]; } +export function humanNumber(num, dp = 1) { + // Convert large numbers to a more human readable format. + // >> humanNumber(1000) + // 1.0k + // >> humanNumber(1500000) + // 1.5m + const divisor = 1000; + if (Math.abs(num) < divisor) { + return num; + } + const units = ['k', 'm', 'b']; + let i = -1; + const r = 10 ** dp; + do { + num /= divisor; + ++i; + } while (Math.round(Math.abs(num) * r) / r >= divisor && i < units.length - 1); + + return num.toFixed(dp) + units[i]; +} + export class APIForm extends React.Component { constructor(props) { @@ -317,7 +343,7 @@ export class APIForm extends React.Component { this.setState({ disabled: true, error: false, - errors: this.getEmptyErrors, + errors: this.getEmptyErrors(), loading: true, success: false, }) diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js new file mode 100644 index 00000000..de4356cd --- /dev/null +++ b/app/src/components/Upload.js @@ -0,0 +1,74 @@ +import React from "react"; +import {Button, Form, Header, Segment, TextArea} from "semantic-ui-react"; +import {postDownload} from "../api"; + +const validUrl = /^(http|https):\/\/[^ "]+$/; + +class Downloader extends React.Component { + constructor(props) { + super(props); + this.state = { + urls: '', + valid: true, + }; + } + + submitDownload = async () => { + if (this.state.urls) { + let response = await postDownload(this.state.urls); + if (response.status === 204) { + this.setState({urls: ''}); + } + } + } + + handleInputChange = async (event, {name, value}) => { + this.setState({[name]: value}, this.validateUrls); + } + + validateUrls = async () => { + // Validate that all URLs in state are valid. + if (this.state.urls) { + let urls = this.state.urls.split(/\r?\n/); + let valid = true; + urls.forEach((url) => { + if (valid && url && !validUrl.test(url)) { + valid = false; + } + }) + this.setState({valid}); + } + } + + render() { + let textareaPlaceholder = 'Enter one URL per line'; + let disabled = !this.state.urls || !this.state.valid; + + return ( +
+
Enter the URLs you wish to bring offline.
+