Downloader can be specified by the user.

This commit is contained in:
Roland 2021-12-30 18:20:37 -07:00
parent 4a169ba43c
commit 047b507119
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
12 changed files with 171 additions and 53 deletions

View file

@ -339,8 +339,8 @@ export async function fetchDomains() {
}
}
export async function postDownload(urls) {
let body = {'urls': urls};
export async function postDownload(urls, downloader) {
let body = {urls: urls, downloader: downloader};
let response = await apiPost(`${API_URI}/download`, body);
return response;
}
@ -350,3 +350,8 @@ export async function killDownload(download_id) {
let response = await apiPost(`${API_URI}/download/${download_id}/kill`);
return response;
}
export async function getDownloaders() {
let response = await apiGet(`${API_URI}/downloaders`);
return await response.json();
}

View file

@ -219,7 +219,8 @@ class StoppableRow extends React.Component {
handleStart = async (e) => {
e.preventDefault();
await postDownload(`${this.props.url}`);
let downloader = this.props.downloader || null;
await postDownload(`${this.props.url}`, downloader);
this.closeStart();
await this.props.fetchDownloads();
};

View file

@ -1,6 +1,6 @@
import React from "react";
import {Button, Form, Header, Segment, TextArea} from "semantic-ui-react";
import {postDownload} from "../api";
import {Button, Dropdown, Form, Header, Segment, TextArea} from "semantic-ui-react";
import {getDownloaders, postDownload} from "../api";
const validUrl = /^(http|https):\/\/[^ "]+$/;
@ -11,14 +11,29 @@ class Downloader extends React.Component {
urls: '',
valid: true,
pending: false,
downloaders: [],
downloader: null,
};
}
async componentDidMount() {
await this.fetchDownloaders();
}
fetchDownloaders = async () => {
let {downloaders} = await getDownloaders();
downloaders = downloaders.map((i) => {
return {key: i.name, text: i.pretty_name || i.name, value: i.name}
})
this.setState({downloaders});
}
submitDownload = async () => {
if (this.state.urls) {
let {urls, downloader} = this.state;
if (urls) {
this.setState({pending: true});
try {
let response = await postDownload(this.state.urls);
let response = await postDownload(urls, downloader);
if (response.status === 204) {
this.setState({urls: '', pending: false});
}
@ -52,14 +67,25 @@ class Downloader extends React.Component {
return (
<Form onSubmit={this.submitDownload}>
<Header as='h4'>Enter the URLs you wish to bring offline.</Header>
<TextArea
placeholder={textareaPlaceholder}
name='urls'
onChange={this.handleInputChange}
value={this.state.urls}
error={!this.state.valid}
/>
<Header as='h4'>Enter the URLs you wish to save offline.</Header>
<Form.Field>
<TextArea
placeholder={textareaPlaceholder}
name='urls'
onChange={this.handleInputChange}
value={this.state.urls}
error={!this.state.valid}
/>
</Form.Field>
<Form.Field>
<Dropdown
selection
name='downloader'
options={this.state.downloaders}
placeholder='Automatic'
onChange={this.handleInputChange}
/>
</Form.Field>
<Button primary style={{marginTop: '1em'}} disabled={disabled}>Download</Button>
</Form>
)

View file

@ -14,6 +14,7 @@ logger = logger.getChild(__name__)
class ArchiveDownloader(Downloader, ABC):
name = 'archive'
pretty_name = 'Archiver'
def __repr__(self):
return f'<ArchiveDownloader name={self.name}>'

View file

@ -244,5 +244,6 @@ def delete_channel(link):
def download_channel(link: str):
channel = get_channel(link=link, return_dict=False)
# TODO do not download the channel if it does not already have Download record.
with get_db_session(commit=True) as session:
download_manager.create_download(channel.url, session)

View file

@ -5,6 +5,7 @@ import re
from abc import ABC
from typing import Tuple, List, Optional
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound
from yt_dlp import YoutubeDL
from yt_dlp.extractor import YoutubeTabIE
@ -48,6 +49,7 @@ class ChannelDownloader(Downloader, ABC):
Handling downloading of all Videos in a Channel.
"""
name = 'video_channel'
pretty_name = 'Video Channel'
def __repr__(self):
return f'<ChannelDownloader name={self.name}>'
@ -95,6 +97,7 @@ class VideoDownloader(Downloader, ABC):
Download a single video. Store the video in it's channel's directory, otherwise store it in `videos/NO CHANNEL`.
"""
name = 'video'
pretty_name = 'Videos'
def __repr__(self):
return f'<VideoDownloader name={self.name}>'
@ -125,6 +128,15 @@ class VideoDownloader(Downloader, ABC):
url = download.url
info = download.info_json
if not info:
# Info was not fetched by the DownloadManager, lets get it.
valid, info = self.valid_url(url)
if not valid:
raise UnrecoverableDownloadError(f'{self} cannot download {url}')
session = Session.object_session(download)
download.info_json = info
session.commit()
channel_name = info.get('channel')
channel_id = info.get('channel_id')
channel = None

View file

@ -255,9 +255,6 @@ class Channel(ModelHelper, Base):
info_json = Column(JSON)
info_date = Column(Date)
# Will be filled later by directory path relative to the media directory.
_directory = None
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
def __repr__(self):

View file

@ -14,6 +14,7 @@ from wrolpi.dates import local_timezone
from wrolpi.db import get_db_session, get_db_context
from wrolpi.downloader import DownloadManager, Download, DownloadFrequency
from wrolpi.test.common import wrap_test_db, TestAPI
from wrolpi.test.test_downloader import HTTPDownloader
class FakeYDL:
@ -371,3 +372,57 @@ def test_download_info_save(test_session, video_download_manager):
mock_extract_info.return_value = {'some': 'info'}
video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session)
mock_extract_info.assert_called_once()
def test_selected_downloader(test_session, video_download_manager):
"""
A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
"""
# The test HTTP Downloader will claim to be able to handle any HTTP URL; will pretend to have downloaded anything.
http_downloader = HTTPDownloader()
mock_do_download = http_downloader.do_download = mock.MagicMock()
mock_http_valid_url = http_downloader.valid_url = mock.MagicMock()
mock_http_valid_url.return_value = (True, None)
video_download_manager.register_downloader(http_downloader)
def check_attempts(attempts):
assert test_session.query(Download).one().attempts == attempts
# DownloadManager will attempt to choose the Downloader.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_extract_info.side_effect = UnsupportedError('not this one')
video_download_manager.create_downloads(['https://example.com'], test_session)
mock_http_valid_url.assert_called_once()
mock_extract_info.assert_called_once()
mock_do_download.assert_called_once()
check_attempts(1)
mock_http_valid_url.reset_mock()
mock_do_download.reset_mock()
# DownloadManager will only use the HTTPDownloader.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
mock_extract_info.assert_not_called()
mock_http_valid_url.assert_not_called()
# DownloadManager called do_download.
mock_do_download.assert_called_once()
check_attempts(2)
mock_http_valid_url.reset_mock()
mock_do_download.reset_mock()
# DownloadManager will only use the HTTPDownloader, even if the download fails.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_do_download.side_effect = Exception('oh no!')
video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
mock_extract_info.assert_not_called()
mock_http_valid_url.assert_not_called()
# DownloadManager called do_download.
mock_do_download.assert_called_once()
check_attempts(3)

View file

@ -105,6 +105,7 @@ class Download(ModelHelper, Base):
class Downloader:
name = None
pretty_name = None
def __init__(self, priority: int = 50, name: str = None):
"""
@ -214,7 +215,8 @@ class DownloadManager:
def get_downloader(self, url: str) -> Tuple[Downloader, Dict]:
for i in self.instances:
valid, info = i.valid_url(url)
result = i.valid_url(url)
valid, info = result
if valid:
return i, info
raise InvalidDownload(f'Invalid URL {url=}')
@ -253,39 +255,17 @@ class DownloadManager:
session.flush()
return download
def create_download(self, url: str, session, skip_download: bool = False, reset_attempts: bool = False) -> Download:
def create_download(self, url: str, session, downloader: str = None, skip_download: bool = False,
reset_attempts: bool = False) -> Download:
"""
Schedule a URL for download. If the URL failed previously, it will be retried.
Schedule a URL for download. If the URL failed previously, it may be retried.
"""
# Make sure the URL can be downloaded.
downloader, info = self.get_downloader(url)
if not isinstance(info, dict) and info is not None:
logger.warning(f'info from {downloader}.is_valid is the wrong type! {type(info)=}')
downloads = self.create_downloads([url], session, downloader, skip_download=skip_download,
reset_attempts=reset_attempts)
return downloads[0]
download = self.get_or_create_download(url, session)
# Download may have failed, try again.
download.renew(reset_attempts=reset_attempts)
download.downloader = downloader.name
if isinstance(info, dict):
download.info_json = info
session.commit()
if skip_download is True:
return download
if PYTEST:
# Download now (for testing).
self._do_downloads(session)
else:
# Start downloading ASAP.
self.start_downloads()
return download
def create_downloads(self, urls: List[str], session: Session = None, skip_download: bool = False,
reset_attempts: bool = False) -> List[Download]:
def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None,
skip_download: bool = False, reset_attempts: bool = False) -> List[Download]:
"""
Schedule all URLs for download. If one cannot be downloaded, none will be added.
"""
@ -293,10 +273,16 @@ class DownloadManager:
_, session = get_db_context()
downloads = []
forced_downloader = self.get_downloader_by_name(downloader) if downloader else None
with session.transaction:
for url in urls:
downloader, info_json = self.get_downloader(url)
info_json = None
downloader = forced_downloader
if not forced_downloader:
# User has requested automatic downloader selection, try and find it.
downloader, info_json = self.get_downloader(url)
download = self.get_or_create_download(url, session)
# Download may have failed, try again.
download.renew(reset_attempts=reset_attempts)
@ -317,6 +303,9 @@ class DownloadManager:
return downloads
def _do_downloads(self, session=None):
"""
This method calls the Downloader's do_download method.
"""
# This is a long-running function, lets get a session that can be used for a long time.
if not session:
_, session = get_db_context()
@ -331,7 +320,8 @@ class DownloadManager:
url = download.url
downloader = self.get_downloader_by_name(download.downloader)
if not downloader or not download.info_json:
info = len(download.info_json) if download.info_json else None
if not downloader:
downloader, info_json = self.get_downloader(download.url)
download.downloader = downloader.name
download.info_json = info_json
@ -536,6 +526,18 @@ class DownloadManager:
download.last_successful_download < one_month:
session.delete(download)
def list_downloaders(self) -> List[Dict]:
"""
Return a list of the Downloaders available on this Download Manager.
"""
downloaders = []
for i in self.instances:
downloaders.append(dict(
name=i.name,
pretty_name=i.pretty_name,
))
return downloaders
# The global DownloadManager. This should be used everywhere!
download_manager = DownloadManager()

View file

@ -181,7 +181,10 @@ def valid_regex(_: Request, data: dict):
async def post_download(request: Request, data: dict):
# URLs are provided in a textarea, lets split all lines.
urls = [i.strip() for i in str(data['urls']).strip().splitlines()]
download_manager.create_downloads(urls, reset_attempts=True)
downloader = data.get('downloader')
if not downloader or downloader in ('auto', 'None', 'null'):
downloader = None
download_manager.create_downloads(urls, downloader=downloader, reset_attempts=True)
return response.empty()
@ -210,6 +213,14 @@ async def delete_download(request: Request, download_id: int):
return response.empty(HTTPStatus.NO_CONTENT if deleted else HTTPStatus.NOT_FOUND)
@root_api.get('/downloaders')
async def get_downloaders(request: Request):
downloaders = download_manager.list_downloaders()
downloaders.insert(0, dict(name='auto', pretty_name='Automatic'))
ret = dict(downloaders=downloaders)
return json_response(ret)
class CustomJSONEncoder(json.JSONEncoder):
def default(self, obj):

View file

@ -179,7 +179,8 @@ class EventsResponse:
class DownloadRequest:
urls = doc.String()
urls = doc.String(required=True)
downloader = doc.String()
class Trinary(Field):

View file

@ -20,6 +20,9 @@ class PermissiveDownloader(Downloader):
"""
name = 'permissive'
def __repr__(self):
return '<TESTING Permissive Downloader>'
def valid_url(self, url: str):
return True, None
@ -30,6 +33,9 @@ class HTTPDownloader(Downloader):
"""
name = 'http'
def __repr__(self):
return '<TESTING HTTP Downloader>'
def valid_url(self, url: str):
return url.startswith('https://') or url.startswith('http://'), None