diff --git a/alembic/versions/1a70196c05bb_download_location.py b/alembic/versions/1a70196c05bb_download_location.py
new file mode 100644
index 00000000..3172a9fb
--- /dev/null
+++ b/alembic/versions/1a70196c05bb_download_location.py
@@ -0,0 +1,33 @@
+"""Download location
+
+Revision ID: 1a70196c05bb
+Revises: d8fcdb7773c0
+Create Date: 2022-05-02 12:01:51.210070
+
+"""
+import os
+
+from alembic import op
+from sqlalchemy.orm import Session
+
+# revision identifiers, used by Alembic.
+revision = '1a70196c05bb'
+down_revision = 'd8fcdb7773c0'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+ bind = op.get_bind()
+ session = Session(bind=bind)
+ session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS location TEXT')
+ session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS error TEXT')
+
+
+def downgrade():
+ bind = op.get_bind()
+ session = Session(bind=bind)
+ session.execute('ALTER TABLE download DROP COLUMN IF EXISTS location')
+ session.execute('ALTER TABLE download DROP COLUMN IF EXISTS error')
diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js
index 87543de9..f028672e 100644
--- a/app/src/components/Admin.js
+++ b/app/src/components/Admin.js
@@ -331,7 +331,7 @@ function ClearFailedDownloads({callback}) {
class DownloadRow extends React.Component {
render() {
- let {url, frequency, last_successful_download, status, next_download} = this.props;
+ let {url, frequency, last_successful_download, status, location, next_download} = this.props;
let positive = false;
if (status === 'pending') {
positive = true;
@@ -342,7 +342,7 @@ class DownloadRow extends React.Component {
{secondsToFrequency(frequency)}
{last_successful_download ? secondsToDate(last_successful_download) : null}
{secondsToDate(next_download)}
- {status}
+ {location && View}
);
}
@@ -354,6 +354,7 @@ class StoppableRow extends React.Component {
this.state = {
stopOpen: false,
startOpen: false,
+ errorModalOpen: false,
};
}
@@ -389,9 +390,10 @@ class StoppableRow extends React.Component {
};
render() {
- let {url, last_successful_download, status} = this.props;
- let {stopOpen, startOpen} = this.state;
+ let {url, last_successful_download, status, location, error} = this.props;
+ let {stopOpen, startOpen, errorModalOpen} = this.state;
+ let completedAtCell = last_successful_download ? secondsToDate(last_successful_download) : null;
let buttonCell = ;
let positive = false;
let negative = false;
@@ -418,10 +420,21 @@ class StoppableRow extends React.Component {
warning = status === 'deferred';
buttonCell = (
+
+
+ >Retry
);
+ } else if (status === 'complete' && location) {
+ buttonCell = (
+
+ View
+
+ );
+ }
+ if (error) {
+ completedAtCell = (
+ this.setState({errorModalOpen: false})}
+ onOpen={() => this.setState({errorModalOpen: true})}
+ open={errorModalOpen}
+ trigger={}
+ >
+ Error
+
+ {error}
+
+
+
+
+
+ )
}
return (
{textEllipsis(url, 50)}
- {last_successful_download ? secondsToDate(last_successful_download) : null}
{status}
+ {completedAtCell}
{buttonCell}
);
@@ -498,8 +536,8 @@ class Downloads extends React.Component {
URL
- Completed At
Status
+ Completed At
Control
@@ -512,7 +550,7 @@ class Downloads extends React.Component {
Download Frequency
Last Successful Download
Next Download
- Status
+ View
);
diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py
index 8d33572b..981561a6 100644
--- a/modules/archive/__init__.py
+++ b/modules/archive/__init__.py
@@ -2,7 +2,7 @@ from abc import ABC
from typing import Tuple
from wrolpi.common import logger
-from wrolpi.downloader import Downloader, Download
+from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from . import lib
from .api import bp # noqa
@@ -17,7 +17,7 @@ class ArchiveDownloader(Downloader, ABC):
pretty_name = 'Archive'
def __repr__(self):
- return f''
+ return f''
@classmethod
def valid_url(cls, url: str) -> Tuple[bool, None]:
@@ -26,12 +26,12 @@ class ArchiveDownloader(Downloader, ABC):
"""
return True, None
- def do_download(self, download: Download):
+ def do_download(self, download: Download) -> DownloadResult:
if download.attempts > 3:
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
lib.do_archive(download.url)
- return True
+ return DownloadResult(success=True)
# Archive downloader is the last downloader which should be used.
diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py
index 3b6f19bb..fca6017e 100644
--- a/modules/videos/channel/lib.py
+++ b/modules/videos/channel/lib.py
@@ -152,7 +152,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
# Verify that the URL/Name/directory aren't taken
check_for_channel_conflicts(
session,
- url=data.url,
+ url=data.url or None,
name=data.name,
directory=data.directory,
source_id=data.source_id,
@@ -162,7 +162,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
channel = Channel(
name=data.name,
- url=data.url,
+ url=data.url or None,
match_regex=data.match_regex,
directory=data.directory, # noqa
download_frequency=data.download_frequency,
diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py
index f2fe566b..1c77df11 100644
--- a/modules/videos/channel/test/test_api.py
+++ b/modules/videos/channel/test/test_api.py
@@ -24,7 +24,8 @@ def test_get_channels(test_directory, channel_factory, test_client):
assert len(response.json['channels']) == 3
-def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel):
+@mock.patch('modules.videos.lib.validate_video')
+def test_refresh_no_channel(_, test_session, test_client, test_directory, simple_channel):
# A regular user-defined channel
vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
vid1.touch()
@@ -41,22 +42,7 @@ def test_refresh_no_channel(test_session, test_client, test_directory, simple_ch
vid4.touch()
test_client.post('/api/videos/refresh')
-
- videos = test_session.query(Video).order_by(Video.title).all()
- assert len(videos) == 4
-
- # Add some fake videos, they should be removed.
- test_session.add(Video(title='vid5'))
- test_session.add(Video(title='vid6', channel_id=simple_channel.id))
- test_session.commit()
-
- videos = test_session.query(Video).order_by(Video.title).all()
- assert len(videos) == 6
-
- test_client.post('/api/videos/refresh')
-
- videos = test_session.query(Video).order_by(Video.title).all()
- assert len(videos) == 4
+ assert test_session.query(Video).count() == 4
def test_get_video(test_client, test_session, simple_channel, video_factory):
diff --git a/modules/videos/channel/test/test_lib.py b/modules/videos/channel/test/test_lib.py
index 1e8bc510..f1af902a 100644
--- a/modules/videos/channel/test/test_lib.py
+++ b/modules/videos/channel/test/test_lib.py
@@ -1,6 +1,9 @@
import pytest
+from modules.videos import schema
from modules.videos.channel import lib
+from modules.videos.lib import save_channels_config, import_channels_config
+from modules.videos.models import Channel
from wrolpi.errors import UnknownChannel
@@ -57,3 +60,32 @@ def test_get_channel(test_session, test_directory, channel_factory):
for p in params:
assert lib.get_channel(**p, return_dict=False) == channel_id_channel
assert lib.get_channel(**p)
+
+
+def test_channels_no_url(test_session, test_directory):
+ """Test that a Channel's URL is coerced to None if it is empty."""
+ channel1_directory = test_directory / 'channel1'
+ channel1_directory.mkdir()
+ channel2_directory = test_directory / 'channel2'
+ channel2_directory.mkdir()
+
+ channel1 = schema.ChannelPostRequest(
+ name='foo',
+ directory=str(channel1_directory),
+ )
+ channel1.url = ''
+ lib.create_channel(test_session, data=channel1)
+
+ channel2 = schema.ChannelPostRequest(
+ name='bar',
+ directory=str(channel2_directory),
+ url=''
+ )
+ lib.create_channel(test_session, data=channel2)
+
+ channels = list(test_session.query(Channel))
+ assert len(channels) == 2
+ assert all(i.url is None for i in channels), [(i.name, i.url) for i in channels]
+
+ save_channels_config()
+ import_channels_config()
diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py
index 915f7329..42a2348a 100644
--- a/modules/videos/conftest.py
+++ b/modules/videos/conftest.py
@@ -100,7 +100,8 @@ def video_factory(test_session, test_directory):
if channel_id:
path = test_directory / f'{title}.mp4'
else:
- path = test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4'
+ (test_directory / 'videos/NO CHANNEL').mkdir(parents=True, exist_ok=True)
+ path = test_directory / f'videos/NO CHANNEL/{title}.mp4'
if with_video_file:
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path)
@@ -166,3 +167,22 @@ def test_downloader_config(test_directory):
set_test_downloader_config(True)
yield config_path
set_test_downloader_config(False)
+
+
+@pytest.fixture
+def mock_video_extract_info():
+ with mock.patch('modules.videos.downloader.extract_info') as mock_extract_info:
+ yield mock_extract_info
+
+
+@pytest.fixture
+def mock_video_prepare_filename():
+ with mock.patch('modules.videos.downloader.prepare_filename') as mock_prepare_filename:
+ yield mock_prepare_filename
+
+
+@pytest.fixture
+def mock_video_process_runner():
+ with mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
+ mock_process_runner.return_value = (0, {})
+ yield mock_process_runner
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index e974e7ee..44e18560 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -2,9 +2,11 @@
import json
import pathlib
import re
+import traceback
from abc import ABC
from typing import Tuple, List, Optional
+import yt_dlp.utils
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from yt_dlp.extractor import YoutubeTabIE
@@ -14,13 +16,13 @@ from wrolpi.cmd import which
from wrolpi.common import logger, extract_domain, get_media_directory
from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs
-from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG
+from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG, DownloadResult
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import apply_info_json, get_no_channel_directory, \
get_videos_directory
-from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id, validate_video
+from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id
from .models import Video, Channel
from .schema import ChannelPostRequest
from .video_url_resolver import video_url_resolver
@@ -41,16 +43,24 @@ PREFERRED_VIDEO_EXTENSION = 'mp4'
PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p,18,bestvideo+bestaudio'
+def extract_info(url: str, ydl: YoutubeDL = YDL) -> dict:
+ """Get info about a video. Separated for testing."""
+ return ydl.extract_info(url, download=False, process=False)
+
+
+def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
+ """Get filename from YoutubeDL. Separated for testing."""
+ return ydl.prepare_filename(entry)
+
+
class ChannelDownloader(Downloader, ABC):
- """
- Handling downloading of all Videos in a Channel.
- """
+ """Handles downloading of all Videos in a Channel."""
name = 'video_channel'
pretty_name = 'Video Channel'
listable = False
def __repr__(self):
- return f''
+ return f''
@classmethod
def valid_url(cls, url) -> Tuple[bool, None]:
@@ -60,10 +70,8 @@ class ChannelDownloader(Downloader, ABC):
logger.debug(f'{cls.__name__} not suitable for {url}')
return False, None
- def do_download(self, download: Download):
- """
- Update a Channel's catalog, then schedule downloads of every missing video.
- """
+ def do_download(self, download: Download) -> DownloadResult:
+ """Update a Channel's catalog, then schedule downloads of every missing video."""
with get_db_session() as session:
channel = session.query(Channel).filter_by(url=download.url).one_or_none()
@@ -71,6 +79,7 @@ class ChannelDownloader(Downloader, ABC):
# Couldn't get channel by URL, this is probably a playlist. Find the channel for the playlist.
channel_source_id = get_channel_source_id(download.url)
channel = get_channel(source_id=channel_source_id, return_dict=False)
+ channel_id = channel.id
update_channel(channel)
@@ -91,7 +100,7 @@ class ChannelDownloader(Downloader, ABC):
else:
self.manager.start_downloads()
- return True
+ return DownloadResult(success=True, location=f'/videos/channel/{channel_id}/video')
YT_DLP_BIN = which(
@@ -110,7 +119,7 @@ class VideoDownloader(Downloader, ABC):
pretty_name = 'Videos'
def __repr__(self):
- return f''
+ return f''
@classmethod
def valid_url(cls, url) -> Tuple[bool, Optional[dict]]:
@@ -120,7 +129,7 @@ class VideoDownloader(Downloader, ABC):
for ie in YDL._ies.values():
if ie.suitable(url) and not ChannelDownloader.valid_url(url)[0]:
try:
- info = YDL.extract_info(url, download=False, process=False)
+ info = extract_info(url)
return True, info
except UnsupportedError:
logger.debug(f'Video downloader extract_info failed for {url}')
@@ -131,7 +140,7 @@ class VideoDownloader(Downloader, ABC):
logger.debug(f'{cls.__name__} not suitable for {url}')
return False, None
- def do_download(self, download: Download):
+ def do_download(self, download: Download) -> DownloadResult:
if download.attempts >= 10:
raise UnrecoverableDownloadError('Max download attempts reached')
@@ -160,6 +169,7 @@ class VideoDownloader(Downloader, ABC):
out_dir = channel.directory.path
out_dir.mkdir(exist_ok=True, parents=True)
+ logs = None # noqa
try:
video_path, entry = self.prepare_filename(url, out_dir)
# Do the real download.
@@ -177,15 +187,24 @@ class VideoDownloader(Downloader, ABC):
'-o', file_name_format,
url,
]
- return_code = self.process_runner(url, cmd, out_dir)
+ return_code, logs = self.process_runner(url, cmd, out_dir)
if return_code != 0:
raise ValueError(f'video downloader process exited with {return_code}')
if not video_path.is_file():
raise ValueError(f'Video file could not be found! {video_path}')
+
+ with get_db_session(commit=True) as session:
+ # If the video is from a channel, it will already be in the database.
+ source_id = entry['id']
+ existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none()
+ video = upsert_video(session, video_path, channel, id_=existing_video.id if existing_video else None)
+ video_id = video.id
except UnrecoverableDownloadError:
raise
+ except yt_dlp.utils.UnsupportedError as e:
+ raise UnrecoverableDownloadError('URL is not supported by yt-dlp') from e
except Exception as e:
logger.warning(f'VideoDownloader failed to download: {download.url}', exc_info=e)
if _skip_download(e):
@@ -204,18 +223,22 @@ class VideoDownloader(Downloader, ABC):
# Skipped downloads should not be tried again.
raise UnrecoverableDownloadError() from e
- return
+ # Download did not succeed, try again later.
+ if logs and (stderr := logs.get('stderr')):
+ error = f'{stderr}\n\n{traceback.format_exc()}'
+ else:
+ error = str(traceback.format_exc())
+ return DownloadResult(success=False, error=error)
- with get_db_session(commit=True) as session:
- # If the video is from a channel, it will already be in the database.
- source_id = entry['id']
- existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none()
- id_ = existing_video.id if existing_video else None
-
- upsert_video(session, video_path, channel, id_=id_)
-
- # Successful download.
- return True
+ if channel:
+ location = f'/videos/channel/{channel.id}/video/{video_id}'
+ else:
+ location = f'/videos/video/{video_id}'
+ result = DownloadResult(
+ success=True,
+ location=location,
+ )
+ return result
@staticmethod
def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
@@ -234,9 +257,13 @@ class VideoDownloader(Downloader, ABC):
ydl.params['logger'] = ydl_logger
ydl.add_default_info_extractors()
- # Perform the real download. This saves multiple files.
- entry = ydl.extract_info(url, download=False, process=True)
- final_filename = pathlib.Path(ydl.prepare_filename(entry))
+ # Get the path where the video will be saved.
+ entry = extract_info(url, ydl=ydl)
+ final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
+ if final_filename.suffix == '.NA':
+ # yt-dlp has odd behavior sometimes and ignores our preferred video extension.
+ final_filename = final_filename.with_suffix('.mp4')
+ logger.warning(f'Had to replace video extension: {final_filename}')
return final_filename, entry
@@ -284,10 +311,10 @@ def update_channel(channel: Channel):
It is expected that any missing videos will be downloaded later.
"""
logger.info(f'Downloading video list for {channel.name} at {channel.url} This may take several minutes.')
- info = YDL.extract_info(channel.url, download=False, process=False)
+ info = extract_info(channel.url)
if 'url' in info:
url = info['url']
- info = YDL.extract_info(url, download=False, process=False)
+ info = extract_info(url)
# Resolve all entries to dictionaries.
entries = info['entries'] = list(info['entries'])
@@ -299,7 +326,7 @@ def update_channel(channel: Channel):
for entry in entries:
if entry['title'] == 'Uploads':
logger.info('Youtube-DL gave back a list of URLs, found the "Uploads" URL and using it.')
- info = YDL.extract_info(entry['url'], download=False, process=False)
+ info = extract_info(entry['url'])
entries = info['entries'] = list(info['entries'])
break
@@ -359,7 +386,6 @@ def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]:
if channel_id:
where = 'AND channel_id = %s'
params = (channel_id,)
-
query = f'''
SELECT
video.id, video.source_id, video.channel_id
@@ -380,7 +406,7 @@ def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]:
return missing_videos
-def find_all_missing_videos(channel_id: str = None) -> Tuple[dict, dict]:
+def find_all_missing_videos(channel_id: int = None) -> Tuple[dict, dict]:
"""
Find all videos that don't have a video file, but are found in the DB (taken from the channel's info_json).
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 5c405d63..e49088e7 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -6,7 +6,6 @@ from typing import Tuple, Optional, List, Union
from uuid import uuid1
from sqlalchemy import or_
-from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
@@ -18,7 +17,8 @@ from wrolpi.media_path import MediaPath
from wrolpi.vars import PYTEST
from .captions import get_captions
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \
- is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError
+ is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
+ get_no_channel_directory
from .models import Channel, Video
logger = logger.getChild(__name__)
@@ -60,7 +60,8 @@ def refresh_channel_videos(channel: Channel):
logger.debug(f'{channel.name}: Added {video_path.name}')
with get_db_curs(commit=True) as curs:
- curs.execute('DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL RETURNING id', (channel.id,))
+ stmt = 'DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL AND video_path IS NOT NULL RETURNING id'
+ curs.execute(stmt, (channel.id,))
deleted_count = len(curs.fetchall())
if deleted_count:
@@ -544,6 +545,9 @@ def import_channels_config():
channel.name = data['name']
channel.directory = directory
+ # A URL should not be an empty string
+ data['url'] = data['url'] or None
+
# Copy existing channel data, update all values from the config. This is necessary to clear out
# values not in the config.
full_data = channel.dict()
@@ -571,7 +575,6 @@ def import_channels_config():
continue
channel_dir = channel.directory.path
else:
- from .downloader import get_no_channel_directory
channel_dir = get_no_channel_directory()
# Set favorite Videos of this Channel.
@@ -712,7 +715,11 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
raise ValueError(f'Video path is not absolute: {video_path}')
# This function can update or insert a Video.
- video = session.query(Video).filter_by(id=id_).one() if id_ else Video()
+ if id_:
+ video = session.query(Video).filter_by(id=id_).one()
+ else:
+ video = Video()
+ session.add(video)
# Set the file values, all other things can be found using these files.
poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path)
@@ -728,6 +735,9 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
video.channel_id = channel.id
video.idempotency = idempotency
+ session.flush()
+ session.refresh(video)
+
try:
# Fill in any missing data. Generate poster if enabled and necessary.
validate_video(video, channel.generate_posters if channel else False)
@@ -738,11 +748,6 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
# inserted.
logger.warning(f'Failed to validate {video}', exc_info=e)
- try:
- session.add(video)
- except InvalidRequestError:
- # Video is already in a session.
- pass
session.flush()
return video
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 0b8af617..31b93f47 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -338,7 +338,7 @@ class Channel(ModelHelper, Base):
name=self.name,
skip_download_videos=self.skip_download_videos or [],
source_id=self.source_id,
- url=self.url or '',
+ url=self.url or None,
)
return config
diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py
index 1905cf7e..059c5246 100644
--- a/modules/videos/test/test_common.py
+++ b/modules/videos/test/test_common.py
@@ -347,6 +347,7 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
assert download.frequency == channel1.download_frequency
assert download.downloader == 'video_channel'
assert download.next_download
+ assert not download.error
next_download = str(download.next_download)
import_channels_config()
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index 19a50957..de3eb3cf 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -1,6 +1,7 @@
import pathlib
import shutil
from copy import copy
+from itertools import zip_longest
from unittest import mock
import pytest
@@ -13,22 +14,11 @@ from modules.videos.models import Channel, Video
from wrolpi.db import get_db_context
from wrolpi.downloader import DownloadManager, Download
from wrolpi.errors import InvalidDownload
-from wrolpi.test.common import wrap_test_db, TestAPI
+from wrolpi.test.common import TestAPI
from wrolpi.test.test_downloader import HTTPDownloader
from wrolpi.vars import PROJECT_DIR
-class FakeYDL:
-
- @staticmethod
- def extract_info(*a, **kw):
- return {
- 'entries': [
- {'id': 1},
- ],
- }
-
-
def test_find_all_missing_videos(test_session, channel_factory, video_factory):
channel1, channel2 = channel_factory(url='some url'), channel_factory(url='some other url')
channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]}
@@ -155,68 +145,52 @@ class TestVideosDownloaders(TestAPI):
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance')[0], self.cd)
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/watch?v=31jPEBiAC3c')[0], self.vd)
- @staticmethod
- def _make_video_files(channel_dir):
- video_path = channel_dir / 'video.mp4'
- poster_path = channel_dir / 'video.jpg'
- description_path = channel_dir / 'video.description'
- info_json_path = channel_dir / 'video.info.json'
- video_path.touch()
- poster_path.touch()
- description_path.touch()
- info_json_path.touch()
- return video_path, poster_path, description_path, info_json_path
- @wrap_test_db
- def test_video_download_no_channel(self):
- _, session = get_db_context()
+def test_video_download_no_channel(test_session, video_download_manager, video_factory, test_directory,
+ mock_video_extract_info, mock_video_process_runner):
+ """A video can be downloaded even if it does not have a Channel."""
+ channel_dir = test_directory / 'NO CHANNEL'
+ channel_dir.mkdir(parents=True)
+ video_path = channel_dir / 'video.mp4'
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
- channel_dir = self.videos_dir / 'NO CHANNEL'
- channel_dir.mkdir(parents=True)
+ # Video has no channel
+ info_json = copy(example_video_json)
+ del info_json['channel']
+ del info_json['channel_id']
+ del info_json['channel_url']
- # Video paths
- video_path, _, _, _ = self._make_video_files(channel_dir)
+ url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
+ mock_video_extract_info.return_value = info_json
+ with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
+ mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
+ video_download_manager.create_download(url, test_session)
- # Video has no channel
- info_json = copy(example_video_json)
- del info_json['channel']
- del info_json['channel_id']
- del info_json['channel_url']
+ mock_video_process_runner.assert_called_once()
- url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
- with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
- mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
- mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
- mock_extract_info.return_value = info_json
- mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
- mock_process_runner.return_value = 0
-
- self.mgr.create_download(url, session)
-
- mock_process_runner.assert_called_once()
-
- video: Video = session.query(Video).one()
- self.assertEqual(str(video.video_path.path), f'{channel_dir}/video.mp4')
+ video: Video = test_session.query(Video).one()
+ assert str(video.video_path.path) == f'{channel_dir}/video.mp4'
-def test_channel_download(test_session, simple_channel, video_download_manager, video_file):
+def test_download_channel(test_session, simple_channel, video_download_manager, video_file,
+ mock_video_extract_info, mock_video_prepare_filename, mock_video_process_runner):
+ """Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos."""
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
- with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
- mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
- mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
- mock_extract_info.return_value = example_channel_json
- mock_prepare_filename.return_value = (video_file, {'id': 'foo'})
- mock_process_runner.return_value = 0
+ mock_video_prepare_filename.return_value = video_file
+ mock_video_extract_info.return_value = example_channel_json
+ with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
+ mock_get_channel.return_value = simple_channel
video_download_manager.create_download(url, test_session)
# Two videos are in the example channel.
- downloads = video_download_manager.get_new_downloads(test_session)
- expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o']
- for download, expected in zip(downloads, expected):
+ downloads = video_download_manager.get_once_downloads(test_session)
+ downloads = filter(lambda i: 'watch' in i.url, downloads)
+ expected = ['https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url']
+ for download, expected in zip_longest(downloads, expected):
assert download.url == expected
- # Download is skipped for test.
- assert download.status == 'new'
+ # Download is run only for test.
+ assert download.status == 'complete'
def test_get_or_create_channel(test_session):
@@ -259,23 +233,25 @@ def test_get_or_create_channel(test_session):
assert get_or_create_channel(source_id='quux') == channel
-def test_download_info_save(test_session, video_download_manager):
+def test_download_info_save(test_session, video_download_manager, mock_video_extract_info, mock_video_prepare_filename,
+ mock_video_process_runner):
"""
An info dict from Downloader.valid_url is saved to the Downloader and can be used by the Downloader later.
"""
- with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
- mock_extract_info.return_value = {'some': 'info'}
- video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session)
- mock_extract_info.assert_called_once()
+ mock_video_extract_info.return_value = {'some': 'info'}
+ mock_video_prepare_filename.return_value = 'some path'
+ video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session)
+ assert mock_video_extract_info.call_count == 2 # Only called twice. The "info" gathering call was skipped.
-def test_selected_downloader(test_session, video_download_manager):
+def test_selected_downloader(test_session, video_download_manager, successful_download):
"""
A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
"""
# The test HTTP Downloader will claim to be able to handle any HTTP URL; will pretend to have downloaded anything.
http_downloader = HTTPDownloader()
mock_do_download = http_downloader.do_download = mock.MagicMock()
+ mock_do_download.return_value = successful_download
mock_http_valid_url = http_downloader.valid_url = mock.MagicMock()
mock_http_valid_url.return_value = (True, None)
video_download_manager.register_downloader(http_downloader)
@@ -286,7 +262,7 @@ def test_selected_downloader(test_session, video_download_manager):
# DownloadManager will attempt to choose the Downloader.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_extract_info.side_effect = UnsupportedError('not this one')
- video_download_manager.create_downloads(['https://example.com'], test_session)
+ video_download_manager.create_download('https://example.com', test_session)
mock_http_valid_url.assert_called_once()
mock_extract_info.assert_called_once()
mock_do_download.assert_called_once()
@@ -298,7 +274,7 @@ def test_selected_downloader(test_session, video_download_manager):
# DownloadManager will only use the HTTPDownloader.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
- video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
+ video_download_manager.create_download('https://example.com', test_session, downloader='http')
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
mock_extract_info.assert_not_called()
mock_http_valid_url.assert_not_called()
@@ -313,7 +289,7 @@ def test_selected_downloader(test_session, video_download_manager):
# DownloadManager will only use the HTTPDownloader, even if the download fails.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_do_download.side_effect = Exception('oh no!')
- video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
+ video_download_manager.create_download('https://example.com', test_session, downloader='http')
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
mock_extract_info.assert_not_called()
mock_http_valid_url.assert_not_called()
@@ -351,22 +327,22 @@ def test_channel_download_no_download(test_session, video_download_manager, simp
download_channel(simple_channel.id)
-def test_invalid_download_url(test_session, test_download_manager, test_downloader_config):
+def test_invalid_download_url(test_session, test_download_manager, mock_video_extract_info):
"""An invalid url should not be attempted again."""
_, session = get_db_context()
with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \
- mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader, \
- mock.patch('modules.videos.downloader.YDL') as mock_ydl:
- mock_get_downloader.return_value = (VideoDownloader(), {'channel': 'foo', 'id': 'some id'})
- mock_valid_url.return_value = (True, None)
- mock_ydl.extract_info.side_effect = UnsupportedError('oops')
- test_download_manager.create_download('url', session)
+ mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader:
+ mock_get_downloader.return_value = (VideoDownloader(), None)
+ mock_valid_url.return_value = (True, {}) # url is valid, but is not a video URL.
+ mock_video_extract_info.side_effect = UnsupportedError('oops')
+ test_download_manager.create_download('invalid url', session)
- download = test_download_manager.get_downloads(test_session)[0]
+ download = session.query(Download).one()
assert download.status == 'failed'
-def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager):
+def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager,
+ mock_video_process_runner):
simple_channel.source_id = example_video_json['channel_id']
simple_channel.directory = test_directory / 'videos/channel name'
simple_channel.directory.mkdir(parents=True)
@@ -376,16 +352,14 @@ def test_video_download_1(test_session, test_directory, simple_channel, video_do
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
- mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
- mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
+ mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
mock_extract_info.return_value = example_video_json
mock_prepare_filename.return_value = (video_file, {'id': 'foo'})
- mock_process_runner.return_value = 0
video_download_manager.create_download(url, test_session)
- mock_process_runner.assert_called_once()
- video_url, _, out_dir = mock_process_runner.call_args[0]
+ mock_video_process_runner.assert_called_once()
+ video_url, _, out_dir = mock_video_process_runner.call_args[0]
download: Download = test_session.query(Download).one()
assert video_url == download.url
@@ -393,3 +367,21 @@ def test_video_download_1(test_session, test_directory, simple_channel, video_do
video: Video = test_session.query(Video).one()
assert video.video_path.path.is_absolute()
+
+
+def test_download_result(test_session, test_directory, video_download_manager, mock_video_process_runner,
+ mock_video_extract_info):
+ """VideoDownloader returns a DownloadResult when complete."""
+ channel_directory = test_directory / 'videos/channel name'
+ channel_directory.mkdir(parents=True)
+ video_file = channel_directory / 'the video.mp4'
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file)
+
+ mock_video_extract_info.return_value = example_video_json
+ with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
+ mock_prepare_filename.return_value = [video_file, {'id': 'foo'}]
+ video_download_manager.create_download('https://example.com', test_session)
+
+ download: Download = test_session.query(Download).one()
+ assert download.url == 'https://example.com'
+ assert download.location == '/videos/channel/1/video/1'
diff --git a/wrolpi/common.py b/wrolpi/common.py
index 8c56482a..b5d529bb 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -555,9 +555,7 @@ def get_relative_to_media_directory(path: Union[str, Path]) -> Path:
def minimize_dict(d: dict, keys: Iterable) -> Optional[dict]:
- """
- Return a new dictionary that contains only the keys provided.
- """
+ """Return a new dictionary that contains only the keys provided."""
if d is None:
return
return {k: d[k] for k in set(keys) & d.keys()}
diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py
index d7552df3..ab442b07 100644
--- a/wrolpi/conftest.py
+++ b/wrolpi/conftest.py
@@ -15,7 +15,7 @@ from sqlalchemy.orm import Session, sessionmaker
from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.dates import set_test_now
from wrolpi.db import postgres_engine, get_db_args
-from wrolpi.downloader import DownloadManager
+from wrolpi.downloader import DownloadManager, DownloadResult
from wrolpi.root_api import BLUEPRINTS, api_app
@@ -124,3 +124,13 @@ def fake_now():
finally:
# reset now() to its original functionality.
set_test_now(None) # noqa
+
+
+@pytest.fixture
+def successful_download():
+ return DownloadResult(success=True)
+
+
+@pytest.fixture
+def failed_download():
+ return DownloadResult(error='pytest.fixture failed_download error', success=False)
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index ae1b38af..37039462 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -2,7 +2,8 @@ import asyncio
import multiprocessing
import pathlib
import subprocess
-from collections import defaultdict
+import traceback
+from dataclasses import dataclass, field
from datetime import timedelta, datetime
from enum import Enum
from functools import partial
@@ -13,7 +14,7 @@ from sqlalchemy import Column, Integer, String, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Session
-from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag, ConfigFile
+from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile
from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz
from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
@@ -34,6 +35,15 @@ class DownloadFrequency(int, Enum):
days90 = 7776000
+@dataclass
+class DownloadResult:
+ downloads: List[str] = field(default_factory=list)
+ error: str = None
+ info_json: dict = field(default_factory=dict)
+ location: str = None
+ success: bool = False
+
+
class Download(ModelHelper, Base):
"""
Model that is used to schedule downloads.
@@ -49,6 +59,8 @@ class Download(ModelHelper, Base):
status = Column(String, default='new')
info_json = Column(JSONB)
downloader = Column(Text)
+ location = Column(Text)
+ error = Column(Text)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -67,6 +79,7 @@ class Download(ModelHelper, Base):
frequency=self.frequency,
id=self.id,
last_successful_download=self.last_successful_download,
+ location=self.location,
next_download=self.next_download,
status=self.status,
url=self.url,
@@ -74,39 +87,30 @@ class Download(ModelHelper, Base):
return d
def renew(self, reset_attempts: bool = False):
- """
- Mark this Download as "new" so it will be retried.
- """
+ """Mark this Download as "new" so it will be retried."""
self.status = 'new'
if reset_attempts:
self.attempts = 0
def defer(self):
- """
- Download should be tried again after a time.
- """
+ """Download should be tried again after a time."""
self.status = 'deferred'
def fail(self):
- """
- Download should not be attempted again. A recurring Download will raise an error.
- """
+ """Download should not be attempted again. A recurring Download will raise an error."""
if self.frequency:
raise ValueError('Recurring download should not be failed.')
self.status = 'failed'
def started(self):
- """
- Mark this Download as in progress.
- """
+ """Mark this Download as in progress."""
self.attempts += 1
self.status = 'pending'
def complete(self):
- """
- Mark this Download as successfully downloaded.
- """
+ """Mark this Download as successfully downloaded."""
self.status = 'complete'
+ self.error = None # clear any old errors
self.last_successful_download = now()
def get_downloader(self):
@@ -147,7 +151,7 @@ class Downloader:
def valid_url(cls, url) -> Tuple[bool, Optional[dict]]:
raise NotImplementedError()
- def do_download(self, download: Download):
+ def do_download(self, download: Download) -> DownloadResult:
raise NotImplementedError()
@property
@@ -174,7 +178,7 @@ class Downloader:
if self._kill.is_set():
self._kill.clear()
- def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> int:
+ def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> Tuple[int, dict]:
"""
Run a subprocess using the provided arguments. This process can be killed by the Download Manager.
"""
@@ -202,13 +206,10 @@ class Downloader:
# Output all logs from the process.
# TODO is there a way to stream this output while the process is running?
logger.debug(f'Download exited with {proc.returncode}')
- outs, errs = proc.communicate()
- for line in outs.decode().splitlines(): # noqa
- logger.debug(line)
- for line in errs.decode().splitlines(): # noqa
- logger.error(line)
+ stdout, stderr = proc.communicate()
+ logs = {'stdout': stdout, 'stderr': stderr}
- return proc.returncode
+ return proc.returncode, logs
class DownloadManager:
@@ -233,15 +234,14 @@ class DownloadManager:
def get_downloader(self, url: str) -> Tuple[Downloader, Dict]:
for i in self.instances:
- result = i.valid_url(url)
- valid, info = result
+ valid, info = i.valid_url(url)
if valid:
return i, info
raise InvalidDownload(f'Invalid URL {url=}')
def get_downloader_by_name(self, name: str) -> Downloader:
"""
- Attempts to find a registered Downloader by its name. Returns None if it cannot be found.
+ Tries to find a registered Downloader by its name. Returns None if it cannot be found.
"""
return self._instances.get(name)
@@ -263,9 +263,7 @@ class DownloadManager:
yield download
def get_or_create_download(self, url: str, session: Session) -> Download:
- """
- Get a Download by its URL, if it cannot be found create one.
- """
+ """Get a Download by its URL, if it cannot be found create one."""
if not url:
raise ValueError('Download must have a URL')
@@ -360,29 +358,34 @@ class DownloadManager:
download.started()
session.commit()
- success = True
- failure = False
+ try_again = True
try:
- success = downloader.do_download(download)
+ result = downloader.do_download(download)
except UnrecoverableDownloadError as e:
# Download failed and should not be retried.
logger.warning(f'UnrecoverableDownloadError for {url}', exc_info=e)
- failure = True
+ result = DownloadResult(success=False, error=str(traceback.format_exc()))
+ try_again = False
except Exception as e:
logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e)
+ result = DownloadResult(success=False, error=str(traceback.format_exc()))
# Long-running, get the download again.
with get_db_session(commit=True) as session_:
download = self.get_download(session_, id_=download_id)
- if failure:
+ # Use a new location if provided, keep the old location if no new location is provided, otherwise
+ # clear out an outdated location.
+ download.location = result.location or download.location or None
+ # Clear any old errors if the download succeeded.
+ download.error = result.error if download.error else None
+ download.next_download = self.get_next_download(download, session)
+
+ if try_again is False:
download.fail()
- elif success:
+ elif result.success:
download.complete()
- download.next_download = self.get_next_download(download, session)
else:
download.defer()
- download.next_download = self.get_next_download(download, session)
- session_.commit()
if download_count:
logger.info(f'Done doing {download_count} downloads.')
@@ -409,9 +412,7 @@ class DownloadManager:
@staticmethod
def reset_downloads():
- """
- Set any incomplete Downloads to `new` so they will be retried.
- """
+ """Set any incomplete Downloads to `new` so they will be retried."""
with get_db_curs(commit=True) as curs:
curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'")
@@ -430,20 +431,6 @@ class DownloadManager:
DOWNLOAD_SORT = ('pending', 'failed', 'new', 'deferred', 'complete')
- @classmethod
- @iterify(list)
- def _downloads_sorter(cls, downloads: List[Download]):
- """
- Downloads should be displayed to the user by status.
- """
- grouped_by_statuses = defaultdict(lambda: [])
- for download in downloads:
- grouped_by_statuses[download.status].append(download)
-
- # Yield the downloads in the order defined above.
- for sort in cls.DOWNLOAD_SORT:
- yield from grouped_by_statuses[sort] # converted to a list by iterify()
-
@optional_session
def get_recurring_downloads(self, session: Session = None, limit: int = None):
"""Get all Downloads that will be downloaded in the future."""
@@ -456,7 +443,6 @@ class DownloadManager:
if limit:
query = query.limit(limit)
downloads = query.all()
- downloads = self._downloads_sorter(downloads)
return downloads
@optional_session
@@ -471,14 +457,11 @@ class DownloadManager:
if limit:
query = query.limit(limit)
downloads = query.all()
- downloads = self._downloads_sorter(downloads)
return downloads
@optional_session
def renew_recurring_downloads(self, session: Session = None):
- """
- Mark any recurring downloads that are due for download as "new". Start a download.
- """
+ """Mark any recurring downloads that are due for download as "new". Start a download."""
now_ = now()
recurring = self.get_recurring_downloads(session)
@@ -523,9 +506,7 @@ class DownloadManager:
return False
def kill_download(self, download_id: int):
- """
- Fail a Download. If it is pending, kill the Downloader so the download stops.
- """
+ """Fail a Download. If it is pending, kill the Downloader so the download stops."""
with get_db_session(commit=True) as session:
download = self.get_download(session, id_=download_id)
downloader = download.get_downloader()
@@ -535,9 +516,7 @@ class DownloadManager:
download.fail()
def kill(self):
- """
- Kill all downloads.
- """
+ """Kill all downloads. Do not start new downloads."""
self.disabled.set()
try:
with get_db_session(commit=True) as session:
@@ -610,9 +589,11 @@ class DownloadManager:
stmt = f'''
SELECT
downloader,
+ error,
frequency,
id,
last_successful_download,
+ location,
next_download,
status,
url
diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py
index 048b6f61..c25ccc93 100644
--- a/wrolpi/test/test_downloader.py
+++ b/wrolpi/test/test_downloader.py
@@ -9,7 +9,7 @@ import pytest
from wrolpi import downloader
from wrolpi.dates import local_timezone, Seconds
from wrolpi.db import get_db_context
-from wrolpi.downloader import Downloader, Download, DownloadFrequency
+from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled
from wrolpi.test.common import wrap_test_db, TestAPI
@@ -72,7 +72,7 @@ class TestDownloader(TestAPI):
self.assertEqual(len(self.mgr.get_downloads(session)), 0)
http_downloader = HTTPDownloader()
- http_downloader.do_download.return_value = False
+ http_downloader.do_download.return_value = DownloadResult(success=True)
self.mgr.register_downloader(http_downloader)
self.mgr.create_download('https://example.com', session)
@@ -88,11 +88,11 @@ class TestDownloader(TestAPI):
_, session = get_db_context()
http_downloader = HTTPDownloader()
- http_downloader.do_download.return_value = True
+ http_downloader.do_download.return_value = DownloadResult(success=True)
self.mgr.register_downloader(http_downloader)
permissive_downloader = PermissiveDownloader(priority=100)
- permissive_downloader.do_download.return_value = False # returns a failure
+ permissive_downloader.do_download.return_value = DownloadResult(success=False)
self.mgr.register_downloader(permissive_downloader)
# https is handled by the HTTP Downloader.
@@ -117,7 +117,7 @@ class TestDownloader(TestAPI):
self.assertEqual(download.attempts, 2)
# finally success
- permissive_downloader.do_download.return_value = True
+ permissive_downloader.do_download.return_value = DownloadResult(success=True)
self.mgr.create_download('foo', session)
self.mgr._do_downloads(session)
download = self.mgr.get_download(session, 'foo')
@@ -126,32 +126,6 @@ class TestDownloader(TestAPI):
# No downloads left.
self.assertEqual(list(self.mgr.get_new_downloads(session)), [])
- @wrap_test_db
- def test_max_attempts(self):
- """
- A Download will only be attempted so many times, it will eventually be deleted.
- """
- _, session = get_db_context()
-
- http_downloader = HTTPDownloader()
- http_downloader.do_download.return_value = False
- self.mgr.register_downloader(http_downloader)
-
- self.mgr.create_download('https://example.com', session)
- download = session.query(Download).one()
- self.assertEqual(download.attempts, 1)
-
- self.mgr.create_download('https://example.com', session)
- download = session.query(Download).one()
- self.assertEqual(download.attempts, 2)
-
- # There are no further attempts.
- http_downloader.do_download.side_effect = UnrecoverableDownloadError()
- self.mgr.create_download('https://example.com', session)
- download = session.query(Download).one()
- self.assertEqual(download.attempts, 3)
- self.assertEqual(download.status, 'failed')
-
@wrap_test_db
@mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0)))
def test_delete_old_once_downloads(self):
@@ -186,12 +160,12 @@ class TestDownloader(TestAPI):
self.mgr.delete_old_once_downloads()
# Two old downloads are deleted.
- downloads = self.mgr._downloads_sorter(session.query(Download).all())
+ downloads = session.query(Download).order_by(Download.id).all()
expected = [
- dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'),
- dict(url='https://example.com/6', attempts=1, status='pending'),
dict(url='https://example.com/1', frequency=1, attempts=0, status='new'),
+ dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'),
dict(url='https://example.com/5', attempts=0, status='new'),
+ dict(url='https://example.com/6', attempts=1, status='pending'),
]
for download, expected in zip_longest(downloads, expected):
self.assertDictContains(download.dict(), expected)
@@ -298,7 +272,7 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
http_downloader = HTTPDownloader()
http_downloader.do_download = MagicMock()
- http_downloader.do_download.return_value = True
+ http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
# Download every hour.
@@ -337,7 +311,7 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
# Try the download, but it fails.
http_downloader.do_download.reset_mock()
- http_downloader.do_download.return_value = False
+ http_downloader.do_download.return_value = DownloadResult(success=False)
test_download_manager.do_downloads_sync()
http_downloader.do_download.assert_called_once()
download = session.query(Download).one()
@@ -345,13 +319,13 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
assert download.status == 'deferred'
assert download.last_successful_download == now
# Download should be retried after the DEFAULT_RETRY_FREQUENCY.
- expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1))
+ expected = local_timezone(datetime(2020, 1, 1, 3, 0, 0, 997200))
assert download.next_download == expected
# Try the download again, it finally succeeds.
http_downloader.do_download.reset_mock()
now = fake_now(datetime(2020, 1, 1, 4, 0, 1))
- http_downloader.do_download.return_value = True
+ http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.renew_recurring_downloads(session)
test_download_manager.do_downloads_sync()
http_downloader.do_download.assert_called_once()
@@ -360,3 +334,28 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
assert download.last_successful_download == now
# Floats cause slightly wrong date.
assert download.next_download == local_timezone(datetime(2020, 1, 1, 5, 0, 0, 997200))
+
+
+def test_max_attempts(test_session, test_download_manager):
+ """A Download will only be attempted so many times, it will eventually be deleted."""
+ _, session = get_db_context()
+
+ http_downloader = HTTPDownloader()
+ http_downloader.do_download = MagicMock()
+ http_downloader.do_download.return_value = DownloadResult(success=True)
+ test_download_manager.register_downloader(http_downloader)
+
+ test_download_manager.create_download('https://example.com', session)
+ download = session.query(Download).one()
+ assert download.attempts == 1
+
+ test_download_manager.create_download('https://example.com', session)
+ download = session.query(Download).one()
+ assert download.attempts == 2
+
+ # There are no further attempts.
+ http_downloader.do_download.side_effect = UnrecoverableDownloadError()
+ test_download_manager.create_download('https://example.com', session)
+ download = session.query(Download).one()
+ assert download.attempts == 3
+ assert download.status == 'failed'
diff --git a/wrolpi/version.py b/wrolpi/version.py
index d39599e6..860f433a 100644
--- a/wrolpi/version.py
+++ b/wrolpi/version.py
@@ -1,4 +1,4 @@
-__version__ = '0.4.7-beta'
+__version__ = '0.4.8-beta'
import subprocess