From 6f43f1daa003007a697cc0be79111b0cec0b62f1 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 30 Aug 2023 20:19:32 -0600 Subject: [PATCH] Creating RecursiveHTMLDownloader which recursively searches for files in the provided URLs. --- app/src/api.js | 16 ++- app/src/components/Download.js | 155 +++++++++++++++++++++++ app/src/components/admin/Downloads.js | 2 +- modules/videos/video/test/test_api.py | 12 +- wrolpi/__init__.py | 1 + wrolpi/common.py | 11 +- wrolpi/files/downloader.py | 3 + wrolpi/recursive_downloader.py | 118 +++++++++++++++++ wrolpi/root_api.py | 12 +- wrolpi/schema.py | 3 + wrolpi/tags/test_recursive_downloader.py | 55 ++++++++ wrolpi/vars.py | 4 + 12 files changed, 377 insertions(+), 15 deletions(-) create mode 100644 wrolpi/recursive_downloader.py create mode 100644 wrolpi/tags/test_recursive_downloader.py diff --git a/app/src/api.js b/app/src/api.js index e68edbe7..1fd82867 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -439,7 +439,18 @@ export async function getArchive(archiveId) { } } -export async function postDownload(urls, downloader, frequency, sub_downloader, excludedURLs, destination, tagNames) { +export async function postDownload( + urls, + downloader, + frequency, + sub_downloader, + excludedURLs, + destination, + tagNames, + depth, + suffix, + max_pages, +) { if (!downloader) { toast({ type: 'error', @@ -456,6 +467,9 @@ export async function postDownload(urls, downloader, frequency, sub_downloader, frequency: frequency || null, excluded_urls: excludedURLs, destination: destination || null, + depth: depth || null, + suffix: suffix || null, + max_pages: max_pages || null, }; if (sub_downloader) { body['sub_downloader'] = sub_downloader; diff --git a/app/src/components/Download.js b/app/src/components/Download.js index d0284442..94655c7e 100644 --- a/app/src/components/Download.js +++ b/app/src/components/Download.js @@ -9,6 +9,7 @@ import {AccordionContent, AccordionTitle, Form as SForm, FormDropdown} from "sem import {Link} from "react-router-dom"; import {TagsSelector} from "../Tags"; import {toast} from "react-semantic-toasts-2"; +import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; const validUrl = /^(http|https):\/\/[^ "]+$/; @@ -429,6 +430,147 @@ class RSSDownload extends ChannelDownload { } } +class RecursiveDownload extends Downloader { + + constructor(props) { + super(props); + this.state = { + depth: 1, + disabled: false, + max_pages: 100, + pending: false, + ready: false, + sub_downloader: null, + success: null, + suffix: '', + urls: '', + }; + } + + handleInputChange = async (event, {name, value}) => { + this.setState({[name]: value}); + } + + submitDownload = async () => { + let {urls, destination, depth, suffix, max_pages} = this.state; + if (urls) { + this.setState({pending: true, submitted: false}); + try { + let response = await postDownload( + urls, + 'recursive_html', + null, + 'file', + null, + destination, + null, + depth, + suffix, + max_pages, + ); + if (response.status === 204) { + this.setState({urls: '', pending: false, submitted: true}); + } else { + const content = await response.json(); + let error = 'Failed to create download!'; + if (content && content['error']) { + error = content['error']; + } + toast({type: 'error', title: 'Error', description: error}); + } + } finally { + this.setState({pending: false}); + } + } + } + + render() { + const {urls, depth, max_pages, suffix, destination} = this.state; + const depths = [ + {key: 1, text: 1, value: 1}, + {key: 2, text: 2, value: 2}, + {key: 3, text: 3, value: 3}, + {key: 4, text: 4, value: 4}, + ]; + const validSuffix = suffix && suffix.startsWith('.'); + const complete = urls && depth && validSuffix && max_pages && destination; + + return
+
Recursive File Downloader
+ +

Search each of the URLs for files matching the suffix (.pdf, etc.).

+ +