diff --git a/app/src/api.js b/app/src/api.js
index f261a88d..a4f07dc3 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -339,8 +339,8 @@ export async function fetchDomains() {
}
}
-export async function postDownload(urls) {
- let body = {'urls': urls};
+export async function postDownload(urls, downloader) {
+ let body = {urls: urls, downloader: downloader};
let response = await apiPost(`${API_URI}/download`, body);
return response;
}
@@ -350,3 +350,8 @@ export async function killDownload(download_id) {
let response = await apiPost(`${API_URI}/download/${download_id}/kill`);
return response;
}
+
+export async function getDownloaders() {
+ let response = await apiGet(`${API_URI}/downloaders`);
+ return await response.json();
+}
diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js
index b7035cc7..d8a6f782 100644
--- a/app/src/components/Admin.js
+++ b/app/src/components/Admin.js
@@ -219,7 +219,8 @@ class StoppableRow extends React.Component {
handleStart = async (e) => {
e.preventDefault();
- await postDownload(`${this.props.url}`);
+ let downloader = this.props.downloader || null;
+ await postDownload(`${this.props.url}`, downloader);
this.closeStart();
await this.props.fetchDownloads();
};
diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js
index 1338ba61..451bd189 100644
--- a/app/src/components/Upload.js
+++ b/app/src/components/Upload.js
@@ -1,6 +1,6 @@
import React from "react";
-import {Button, Form, Header, Segment, TextArea} from "semantic-ui-react";
-import {postDownload} from "../api";
+import {Button, Dropdown, Form, Header, Segment, TextArea} from "semantic-ui-react";
+import {getDownloaders, postDownload} from "../api";
const validUrl = /^(http|https):\/\/[^ "]+$/;
@@ -11,14 +11,29 @@ class Downloader extends React.Component {
urls: '',
valid: true,
pending: false,
+ downloaders: [],
+ downloader: null,
};
}
+ async componentDidMount() {
+ await this.fetchDownloaders();
+ }
+
+ fetchDownloaders = async () => {
+ let {downloaders} = await getDownloaders();
+ downloaders = downloaders.map((i) => {
+ return {key: i.name, text: i.pretty_name || i.name, value: i.name}
+ })
+ this.setState({downloaders});
+ }
+
submitDownload = async () => {
- if (this.state.urls) {
+ let {urls, downloader} = this.state;
+ if (urls) {
this.setState({pending: true});
try {
- let response = await postDownload(this.state.urls);
+ let response = await postDownload(urls, downloader);
if (response.status === 204) {
this.setState({urls: '', pending: false});
}
@@ -52,14 +67,25 @@ class Downloader extends React.Component {
return (
+
+
+
+
+
)
diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py
index 4d3114b4..1f91a845 100644
--- a/modules/archive/__init__.py
+++ b/modules/archive/__init__.py
@@ -14,6 +14,7 @@ logger = logger.getChild(__name__)
class ArchiveDownloader(Downloader, ABC):
name = 'archive'
+ pretty_name = 'Archiver'
def __repr__(self):
return f''
diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py
index f9b6c234..76e7500c 100644
--- a/modules/videos/channel/lib.py
+++ b/modules/videos/channel/lib.py
@@ -244,5 +244,6 @@ def delete_channel(link):
def download_channel(link: str):
channel = get_channel(link=link, return_dict=False)
+ # TODO do not download the channel if it does not already have Download record.
with get_db_session(commit=True) as session:
download_manager.create_download(channel.url, session)
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index 3c0827fe..7d2002cc 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -5,6 +5,7 @@ import re
from abc import ABC
from typing import Tuple, List, Optional
+from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound
from yt_dlp import YoutubeDL
from yt_dlp.extractor import YoutubeTabIE
@@ -48,6 +49,7 @@ class ChannelDownloader(Downloader, ABC):
Handling downloading of all Videos in a Channel.
"""
name = 'video_channel'
+ pretty_name = 'Video Channel'
def __repr__(self):
return f''
@@ -95,6 +97,7 @@ class VideoDownloader(Downloader, ABC):
Download a single video. Store the video in it's channel's directory, otherwise store it in `videos/NO CHANNEL`.
"""
name = 'video'
+ pretty_name = 'Videos'
def __repr__(self):
return f''
@@ -125,6 +128,15 @@ class VideoDownloader(Downloader, ABC):
url = download.url
info = download.info_json
+ if not info:
+ # Info was not fetched by the DownloadManager, lets get it.
+ valid, info = self.valid_url(url)
+ if not valid:
+ raise UnrecoverableDownloadError(f'{self} cannot download {url}')
+ session = Session.object_session(download)
+ download.info_json = info
+ session.commit()
+
channel_name = info.get('channel')
channel_id = info.get('channel_id')
channel = None
diff --git a/modules/videos/models.py b/modules/videos/models.py
index e420fea6..1797ec9a 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -255,9 +255,6 @@ class Channel(ModelHelper, Base):
info_json = Column(JSON)
info_date = Column(Date)
- # Will be filled later by directory path relative to the media directory.
- _directory = None
-
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
def __repr__(self):
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index de05d3c9..68d6d1db 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -14,6 +14,7 @@ from wrolpi.dates import local_timezone
from wrolpi.db import get_db_session, get_db_context
from wrolpi.downloader import DownloadManager, Download, DownloadFrequency
from wrolpi.test.common import wrap_test_db, TestAPI
+from wrolpi.test.test_downloader import HTTPDownloader
class FakeYDL:
@@ -371,3 +372,57 @@ def test_download_info_save(test_session, video_download_manager):
mock_extract_info.return_value = {'some': 'info'}
video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session)
mock_extract_info.assert_called_once()
+
+
+def test_selected_downloader(test_session, video_download_manager):
+ """
+ A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
+ """
+ # The test HTTP Downloader will claim to be able to handle any HTTP URL; will pretend to have downloaded anything.
+ http_downloader = HTTPDownloader()
+ mock_do_download = http_downloader.do_download = mock.MagicMock()
+ mock_http_valid_url = http_downloader.valid_url = mock.MagicMock()
+ mock_http_valid_url.return_value = (True, None)
+ video_download_manager.register_downloader(http_downloader)
+
+ def check_attempts(attempts):
+ assert test_session.query(Download).one().attempts == attempts
+
+ # DownloadManager will attempt to choose the Downloader.
+ with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
+ mock_extract_info.side_effect = UnsupportedError('not this one')
+ video_download_manager.create_downloads(['https://example.com'], test_session)
+ mock_http_valid_url.assert_called_once()
+ mock_extract_info.assert_called_once()
+ mock_do_download.assert_called_once()
+
+ check_attempts(1)
+
+ mock_http_valid_url.reset_mock()
+ mock_do_download.reset_mock()
+
+ # DownloadManager will only use the HTTPDownloader.
+ with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
+ video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
+ # DownloadManager does not try to find the Download, it trusts the "downloader" param.
+ mock_extract_info.assert_not_called()
+ mock_http_valid_url.assert_not_called()
+ # DownloadManager called do_download.
+ mock_do_download.assert_called_once()
+
+ check_attempts(2)
+
+ mock_http_valid_url.reset_mock()
+ mock_do_download.reset_mock()
+
+ # DownloadManager will only use the HTTPDownloader, even if the download fails.
+ with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
+ mock_do_download.side_effect = Exception('oh no!')
+ video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
+ # DownloadManager does not try to find the Download, it trusts the "downloader" param.
+ mock_extract_info.assert_not_called()
+ mock_http_valid_url.assert_not_called()
+ # DownloadManager called do_download.
+ mock_do_download.assert_called_once()
+
+ check_attempts(3)
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index d64a929d..3d717ce0 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -105,6 +105,7 @@ class Download(ModelHelper, Base):
class Downloader:
name = None
+ pretty_name = None
def __init__(self, priority: int = 50, name: str = None):
"""
@@ -214,7 +215,8 @@ class DownloadManager:
def get_downloader(self, url: str) -> Tuple[Downloader, Dict]:
for i in self.instances:
- valid, info = i.valid_url(url)
+ result = i.valid_url(url)
+ valid, info = result
if valid:
return i, info
raise InvalidDownload(f'Invalid URL {url=}')
@@ -253,39 +255,17 @@ class DownloadManager:
session.flush()
return download
- def create_download(self, url: str, session, skip_download: bool = False, reset_attempts: bool = False) -> Download:
+ def create_download(self, url: str, session, downloader: str = None, skip_download: bool = False,
+ reset_attempts: bool = False) -> Download:
"""
- Schedule a URL for download. If the URL failed previously, it will be retried.
+ Schedule a URL for download. If the URL failed previously, it may be retried.
"""
- # Make sure the URL can be downloaded.
- downloader, info = self.get_downloader(url)
- if not isinstance(info, dict) and info is not None:
- logger.warning(f'info from {downloader}.is_valid is the wrong type! {type(info)=}')
+ downloads = self.create_downloads([url], session, downloader, skip_download=skip_download,
+ reset_attempts=reset_attempts)
+ return downloads[0]
- download = self.get_or_create_download(url, session)
- # Download may have failed, try again.
- download.renew(reset_attempts=reset_attempts)
-
- download.downloader = downloader.name
- if isinstance(info, dict):
- download.info_json = info
-
- session.commit()
-
- if skip_download is True:
- return download
-
- if PYTEST:
- # Download now (for testing).
- self._do_downloads(session)
- else:
- # Start downloading ASAP.
- self.start_downloads()
-
- return download
-
- def create_downloads(self, urls: List[str], session: Session = None, skip_download: bool = False,
- reset_attempts: bool = False) -> List[Download]:
+ def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None,
+ skip_download: bool = False, reset_attempts: bool = False) -> List[Download]:
"""
Schedule all URLs for download. If one cannot be downloaded, none will be added.
"""
@@ -293,10 +273,16 @@ class DownloadManager:
_, session = get_db_context()
downloads = []
+ forced_downloader = self.get_downloader_by_name(downloader) if downloader else None
with session.transaction:
for url in urls:
- downloader, info_json = self.get_downloader(url)
+ info_json = None
+ downloader = forced_downloader
+ if not forced_downloader:
+ # User has requested automatic downloader selection, try and find it.
+ downloader, info_json = self.get_downloader(url)
+
download = self.get_or_create_download(url, session)
# Download may have failed, try again.
download.renew(reset_attempts=reset_attempts)
@@ -317,6 +303,9 @@ class DownloadManager:
return downloads
def _do_downloads(self, session=None):
+ """
+ This method calls the Downloader's do_download method.
+ """
# This is a long-running function, lets get a session that can be used for a long time.
if not session:
_, session = get_db_context()
@@ -331,7 +320,8 @@ class DownloadManager:
url = download.url
downloader = self.get_downloader_by_name(download.downloader)
- if not downloader or not download.info_json:
+ info = len(download.info_json) if download.info_json else None
+ if not downloader:
downloader, info_json = self.get_downloader(download.url)
download.downloader = downloader.name
download.info_json = info_json
@@ -536,6 +526,18 @@ class DownloadManager:
download.last_successful_download < one_month:
session.delete(download)
+ def list_downloaders(self) -> List[Dict]:
+ """
+ Return a list of the Downloaders available on this Download Manager.
+ """
+ downloaders = []
+ for i in self.instances:
+ downloaders.append(dict(
+ name=i.name,
+ pretty_name=i.pretty_name,
+ ))
+ return downloaders
+
# The global DownloadManager. This should be used everywhere!
download_manager = DownloadManager()
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 83465668..4973d5e9 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -181,7 +181,10 @@ def valid_regex(_: Request, data: dict):
async def post_download(request: Request, data: dict):
# URLs are provided in a textarea, lets split all lines.
urls = [i.strip() for i in str(data['urls']).strip().splitlines()]
- download_manager.create_downloads(urls, reset_attempts=True)
+ downloader = data.get('downloader')
+ if not downloader or downloader in ('auto', 'None', 'null'):
+ downloader = None
+ download_manager.create_downloads(urls, downloader=downloader, reset_attempts=True)
return response.empty()
@@ -210,6 +213,14 @@ async def delete_download(request: Request, download_id: int):
return response.empty(HTTPStatus.NO_CONTENT if deleted else HTTPStatus.NOT_FOUND)
+@root_api.get('/downloaders')
+async def get_downloaders(request: Request):
+ downloaders = download_manager.list_downloaders()
+ downloaders.insert(0, dict(name='auto', pretty_name='Automatic'))
+ ret = dict(downloaders=downloaders)
+ return json_response(ret)
+
+
class CustomJSONEncoder(json.JSONEncoder):
def default(self, obj):
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index f0e37514..d95a0e93 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -179,7 +179,8 @@ class EventsResponse:
class DownloadRequest:
- urls = doc.String()
+ urls = doc.String(required=True)
+ downloader = doc.String()
class Trinary(Field):
diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py
index 88c00bf9..dc501812 100644
--- a/wrolpi/test/test_downloader.py
+++ b/wrolpi/test/test_downloader.py
@@ -20,6 +20,9 @@ class PermissiveDownloader(Downloader):
"""
name = 'permissive'
+ def __repr__(self):
+ return ''
+
def valid_url(self, url: str):
return True, None
@@ -30,6 +33,9 @@ class HTTPDownloader(Downloader):
"""
name = 'http'
+ def __repr__(self):
+ return ''
+
def valid_url(self, url: str):
return url.startswith('https://') or url.startswith('http://'), None