mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Downloader can be specified by the user.
This commit is contained in:
parent
4a169ba43c
commit
047b507119
12 changed files with 171 additions and 53 deletions
|
|
@ -339,8 +339,8 @@ export async function fetchDomains() {
|
|||
}
|
||||
}
|
||||
|
||||
export async function postDownload(urls) {
|
||||
let body = {'urls': urls};
|
||||
export async function postDownload(urls, downloader) {
|
||||
let body = {urls: urls, downloader: downloader};
|
||||
let response = await apiPost(`${API_URI}/download`, body);
|
||||
return response;
|
||||
}
|
||||
|
|
@ -350,3 +350,8 @@ export async function killDownload(download_id) {
|
|||
let response = await apiPost(`${API_URI}/download/${download_id}/kill`);
|
||||
return response;
|
||||
}
|
||||
|
||||
export async function getDownloaders() {
|
||||
let response = await apiGet(`${API_URI}/downloaders`);
|
||||
return await response.json();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -219,7 +219,8 @@ class StoppableRow extends React.Component {
|
|||
|
||||
handleStart = async (e) => {
|
||||
e.preventDefault();
|
||||
await postDownload(`${this.props.url}`);
|
||||
let downloader = this.props.downloader || null;
|
||||
await postDownload(`${this.props.url}`, downloader);
|
||||
this.closeStart();
|
||||
await this.props.fetchDownloads();
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import React from "react";
|
||||
import {Button, Form, Header, Segment, TextArea} from "semantic-ui-react";
|
||||
import {postDownload} from "../api";
|
||||
import {Button, Dropdown, Form, Header, Segment, TextArea} from "semantic-ui-react";
|
||||
import {getDownloaders, postDownload} from "../api";
|
||||
|
||||
const validUrl = /^(http|https):\/\/[^ "]+$/;
|
||||
|
||||
|
|
@ -11,14 +11,29 @@ class Downloader extends React.Component {
|
|||
urls: '',
|
||||
valid: true,
|
||||
pending: false,
|
||||
downloaders: [],
|
||||
downloader: null,
|
||||
};
|
||||
}
|
||||
|
||||
async componentDidMount() {
|
||||
await this.fetchDownloaders();
|
||||
}
|
||||
|
||||
fetchDownloaders = async () => {
|
||||
let {downloaders} = await getDownloaders();
|
||||
downloaders = downloaders.map((i) => {
|
||||
return {key: i.name, text: i.pretty_name || i.name, value: i.name}
|
||||
})
|
||||
this.setState({downloaders});
|
||||
}
|
||||
|
||||
submitDownload = async () => {
|
||||
if (this.state.urls) {
|
||||
let {urls, downloader} = this.state;
|
||||
if (urls) {
|
||||
this.setState({pending: true});
|
||||
try {
|
||||
let response = await postDownload(this.state.urls);
|
||||
let response = await postDownload(urls, downloader);
|
||||
if (response.status === 204) {
|
||||
this.setState({urls: '', pending: false});
|
||||
}
|
||||
|
|
@ -52,14 +67,25 @@ class Downloader extends React.Component {
|
|||
|
||||
return (
|
||||
<Form onSubmit={this.submitDownload}>
|
||||
<Header as='h4'>Enter the URLs you wish to bring offline.</Header>
|
||||
<TextArea
|
||||
placeholder={textareaPlaceholder}
|
||||
name='urls'
|
||||
onChange={this.handleInputChange}
|
||||
value={this.state.urls}
|
||||
error={!this.state.valid}
|
||||
/>
|
||||
<Header as='h4'>Enter the URLs you wish to save offline.</Header>
|
||||
<Form.Field>
|
||||
<TextArea
|
||||
placeholder={textareaPlaceholder}
|
||||
name='urls'
|
||||
onChange={this.handleInputChange}
|
||||
value={this.state.urls}
|
||||
error={!this.state.valid}
|
||||
/>
|
||||
</Form.Field>
|
||||
<Form.Field>
|
||||
<Dropdown
|
||||
selection
|
||||
name='downloader'
|
||||
options={this.state.downloaders}
|
||||
placeholder='Automatic'
|
||||
onChange={this.handleInputChange}
|
||||
/>
|
||||
</Form.Field>
|
||||
<Button primary style={{marginTop: '1em'}} disabled={disabled}>Download</Button>
|
||||
</Form>
|
||||
)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ logger = logger.getChild(__name__)
|
|||
|
||||
class ArchiveDownloader(Downloader, ABC):
|
||||
name = 'archive'
|
||||
pretty_name = 'Archiver'
|
||||
|
||||
def __repr__(self):
|
||||
return f'<ArchiveDownloader name={self.name}>'
|
||||
|
|
|
|||
|
|
@ -244,5 +244,6 @@ def delete_channel(link):
|
|||
|
||||
def download_channel(link: str):
|
||||
channel = get_channel(link=link, return_dict=False)
|
||||
# TODO do not download the channel if it does not already have Download record.
|
||||
with get_db_session(commit=True) as session:
|
||||
download_manager.create_download(channel.url, session)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import re
|
|||
from abc import ABC
|
||||
from typing import Tuple, List, Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
from yt_dlp import YoutubeDL
|
||||
from yt_dlp.extractor import YoutubeTabIE
|
||||
|
|
@ -48,6 +49,7 @@ class ChannelDownloader(Downloader, ABC):
|
|||
Handling downloading of all Videos in a Channel.
|
||||
"""
|
||||
name = 'video_channel'
|
||||
pretty_name = 'Video Channel'
|
||||
|
||||
def __repr__(self):
|
||||
return f'<ChannelDownloader name={self.name}>'
|
||||
|
|
@ -95,6 +97,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
Download a single video. Store the video in it's channel's directory, otherwise store it in `videos/NO CHANNEL`.
|
||||
"""
|
||||
name = 'video'
|
||||
pretty_name = 'Videos'
|
||||
|
||||
def __repr__(self):
|
||||
return f'<VideoDownloader name={self.name}>'
|
||||
|
|
@ -125,6 +128,15 @@ class VideoDownloader(Downloader, ABC):
|
|||
url = download.url
|
||||
info = download.info_json
|
||||
|
||||
if not info:
|
||||
# Info was not fetched by the DownloadManager, lets get it.
|
||||
valid, info = self.valid_url(url)
|
||||
if not valid:
|
||||
raise UnrecoverableDownloadError(f'{self} cannot download {url}')
|
||||
session = Session.object_session(download)
|
||||
download.info_json = info
|
||||
session.commit()
|
||||
|
||||
channel_name = info.get('channel')
|
||||
channel_id = info.get('channel_id')
|
||||
channel = None
|
||||
|
|
|
|||
|
|
@ -255,9 +255,6 @@ class Channel(ModelHelper, Base):
|
|||
info_json = Column(JSON)
|
||||
info_date = Column(Date)
|
||||
|
||||
# Will be filled later by directory path relative to the media directory.
|
||||
_directory = None
|
||||
|
||||
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
|
||||
|
||||
def __repr__(self):
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from wrolpi.dates import local_timezone
|
|||
from wrolpi.db import get_db_session, get_db_context
|
||||
from wrolpi.downloader import DownloadManager, Download, DownloadFrequency
|
||||
from wrolpi.test.common import wrap_test_db, TestAPI
|
||||
from wrolpi.test.test_downloader import HTTPDownloader
|
||||
|
||||
|
||||
class FakeYDL:
|
||||
|
|
@ -371,3 +372,57 @@ def test_download_info_save(test_session, video_download_manager):
|
|||
mock_extract_info.return_value = {'some': 'info'}
|
||||
video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session)
|
||||
mock_extract_info.assert_called_once()
|
||||
|
||||
|
||||
def test_selected_downloader(test_session, video_download_manager):
|
||||
"""
|
||||
A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
|
||||
"""
|
||||
# The test HTTP Downloader will claim to be able to handle any HTTP URL; will pretend to have downloaded anything.
|
||||
http_downloader = HTTPDownloader()
|
||||
mock_do_download = http_downloader.do_download = mock.MagicMock()
|
||||
mock_http_valid_url = http_downloader.valid_url = mock.MagicMock()
|
||||
mock_http_valid_url.return_value = (True, None)
|
||||
video_download_manager.register_downloader(http_downloader)
|
||||
|
||||
def check_attempts(attempts):
|
||||
assert test_session.query(Download).one().attempts == attempts
|
||||
|
||||
# DownloadManager will attempt to choose the Downloader.
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
mock_extract_info.side_effect = UnsupportedError('not this one')
|
||||
video_download_manager.create_downloads(['https://example.com'], test_session)
|
||||
mock_http_valid_url.assert_called_once()
|
||||
mock_extract_info.assert_called_once()
|
||||
mock_do_download.assert_called_once()
|
||||
|
||||
check_attempts(1)
|
||||
|
||||
mock_http_valid_url.reset_mock()
|
||||
mock_do_download.reset_mock()
|
||||
|
||||
# DownloadManager will only use the HTTPDownloader.
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
|
||||
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
|
||||
mock_extract_info.assert_not_called()
|
||||
mock_http_valid_url.assert_not_called()
|
||||
# DownloadManager called do_download.
|
||||
mock_do_download.assert_called_once()
|
||||
|
||||
check_attempts(2)
|
||||
|
||||
mock_http_valid_url.reset_mock()
|
||||
mock_do_download.reset_mock()
|
||||
|
||||
# DownloadManager will only use the HTTPDownloader, even if the download fails.
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
mock_do_download.side_effect = Exception('oh no!')
|
||||
video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
|
||||
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
|
||||
mock_extract_info.assert_not_called()
|
||||
mock_http_valid_url.assert_not_called()
|
||||
# DownloadManager called do_download.
|
||||
mock_do_download.assert_called_once()
|
||||
|
||||
check_attempts(3)
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ class Download(ModelHelper, Base):
|
|||
|
||||
class Downloader:
|
||||
name = None
|
||||
pretty_name = None
|
||||
|
||||
def __init__(self, priority: int = 50, name: str = None):
|
||||
"""
|
||||
|
|
@ -214,7 +215,8 @@ class DownloadManager:
|
|||
|
||||
def get_downloader(self, url: str) -> Tuple[Downloader, Dict]:
|
||||
for i in self.instances:
|
||||
valid, info = i.valid_url(url)
|
||||
result = i.valid_url(url)
|
||||
valid, info = result
|
||||
if valid:
|
||||
return i, info
|
||||
raise InvalidDownload(f'Invalid URL {url=}')
|
||||
|
|
@ -253,39 +255,17 @@ class DownloadManager:
|
|||
session.flush()
|
||||
return download
|
||||
|
||||
def create_download(self, url: str, session, skip_download: bool = False, reset_attempts: bool = False) -> Download:
|
||||
def create_download(self, url: str, session, downloader: str = None, skip_download: bool = False,
|
||||
reset_attempts: bool = False) -> Download:
|
||||
"""
|
||||
Schedule a URL for download. If the URL failed previously, it will be retried.
|
||||
Schedule a URL for download. If the URL failed previously, it may be retried.
|
||||
"""
|
||||
# Make sure the URL can be downloaded.
|
||||
downloader, info = self.get_downloader(url)
|
||||
if not isinstance(info, dict) and info is not None:
|
||||
logger.warning(f'info from {downloader}.is_valid is the wrong type! {type(info)=}')
|
||||
downloads = self.create_downloads([url], session, downloader, skip_download=skip_download,
|
||||
reset_attempts=reset_attempts)
|
||||
return downloads[0]
|
||||
|
||||
download = self.get_or_create_download(url, session)
|
||||
# Download may have failed, try again.
|
||||
download.renew(reset_attempts=reset_attempts)
|
||||
|
||||
download.downloader = downloader.name
|
||||
if isinstance(info, dict):
|
||||
download.info_json = info
|
||||
|
||||
session.commit()
|
||||
|
||||
if skip_download is True:
|
||||
return download
|
||||
|
||||
if PYTEST:
|
||||
# Download now (for testing).
|
||||
self._do_downloads(session)
|
||||
else:
|
||||
# Start downloading ASAP.
|
||||
self.start_downloads()
|
||||
|
||||
return download
|
||||
|
||||
def create_downloads(self, urls: List[str], session: Session = None, skip_download: bool = False,
|
||||
reset_attempts: bool = False) -> List[Download]:
|
||||
def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None,
|
||||
skip_download: bool = False, reset_attempts: bool = False) -> List[Download]:
|
||||
"""
|
||||
Schedule all URLs for download. If one cannot be downloaded, none will be added.
|
||||
"""
|
||||
|
|
@ -293,10 +273,16 @@ class DownloadManager:
|
|||
_, session = get_db_context()
|
||||
|
||||
downloads = []
|
||||
forced_downloader = self.get_downloader_by_name(downloader) if downloader else None
|
||||
|
||||
with session.transaction:
|
||||
for url in urls:
|
||||
downloader, info_json = self.get_downloader(url)
|
||||
info_json = None
|
||||
downloader = forced_downloader
|
||||
if not forced_downloader:
|
||||
# User has requested automatic downloader selection, try and find it.
|
||||
downloader, info_json = self.get_downloader(url)
|
||||
|
||||
download = self.get_or_create_download(url, session)
|
||||
# Download may have failed, try again.
|
||||
download.renew(reset_attempts=reset_attempts)
|
||||
|
|
@ -317,6 +303,9 @@ class DownloadManager:
|
|||
return downloads
|
||||
|
||||
def _do_downloads(self, session=None):
|
||||
"""
|
||||
This method calls the Downloader's do_download method.
|
||||
"""
|
||||
# This is a long-running function, lets get a session that can be used for a long time.
|
||||
if not session:
|
||||
_, session = get_db_context()
|
||||
|
|
@ -331,7 +320,8 @@ class DownloadManager:
|
|||
url = download.url
|
||||
|
||||
downloader = self.get_downloader_by_name(download.downloader)
|
||||
if not downloader or not download.info_json:
|
||||
info = len(download.info_json) if download.info_json else None
|
||||
if not downloader:
|
||||
downloader, info_json = self.get_downloader(download.url)
|
||||
download.downloader = downloader.name
|
||||
download.info_json = info_json
|
||||
|
|
@ -536,6 +526,18 @@ class DownloadManager:
|
|||
download.last_successful_download < one_month:
|
||||
session.delete(download)
|
||||
|
||||
def list_downloaders(self) -> List[Dict]:
|
||||
"""
|
||||
Return a list of the Downloaders available on this Download Manager.
|
||||
"""
|
||||
downloaders = []
|
||||
for i in self.instances:
|
||||
downloaders.append(dict(
|
||||
name=i.name,
|
||||
pretty_name=i.pretty_name,
|
||||
))
|
||||
return downloaders
|
||||
|
||||
|
||||
# The global DownloadManager. This should be used everywhere!
|
||||
download_manager = DownloadManager()
|
||||
|
|
|
|||
|
|
@ -181,7 +181,10 @@ def valid_regex(_: Request, data: dict):
|
|||
async def post_download(request: Request, data: dict):
|
||||
# URLs are provided in a textarea, lets split all lines.
|
||||
urls = [i.strip() for i in str(data['urls']).strip().splitlines()]
|
||||
download_manager.create_downloads(urls, reset_attempts=True)
|
||||
downloader = data.get('downloader')
|
||||
if not downloader or downloader in ('auto', 'None', 'null'):
|
||||
downloader = None
|
||||
download_manager.create_downloads(urls, downloader=downloader, reset_attempts=True)
|
||||
return response.empty()
|
||||
|
||||
|
||||
|
|
@ -210,6 +213,14 @@ async def delete_download(request: Request, download_id: int):
|
|||
return response.empty(HTTPStatus.NO_CONTENT if deleted else HTTPStatus.NOT_FOUND)
|
||||
|
||||
|
||||
@root_api.get('/downloaders')
|
||||
async def get_downloaders(request: Request):
|
||||
downloaders = download_manager.list_downloaders()
|
||||
downloaders.insert(0, dict(name='auto', pretty_name='Automatic'))
|
||||
ret = dict(downloaders=downloaders)
|
||||
return json_response(ret)
|
||||
|
||||
|
||||
class CustomJSONEncoder(json.JSONEncoder):
|
||||
|
||||
def default(self, obj):
|
||||
|
|
|
|||
|
|
@ -179,7 +179,8 @@ class EventsResponse:
|
|||
|
||||
|
||||
class DownloadRequest:
|
||||
urls = doc.String()
|
||||
urls = doc.String(required=True)
|
||||
downloader = doc.String()
|
||||
|
||||
|
||||
class Trinary(Field):
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@ class PermissiveDownloader(Downloader):
|
|||
"""
|
||||
name = 'permissive'
|
||||
|
||||
def __repr__(self):
|
||||
return '<TESTING Permissive Downloader>'
|
||||
|
||||
def valid_url(self, url: str):
|
||||
return True, None
|
||||
|
||||
|
|
@ -30,6 +33,9 @@ class HTTPDownloader(Downloader):
|
|||
"""
|
||||
name = 'http'
|
||||
|
||||
def __repr__(self):
|
||||
return '<TESTING HTTP Downloader>'
|
||||
|
||||
def valid_url(self, url: str):
|
||||
return url.startswith('https://') or url.startswith('http://'), None
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue