DownloadResult.try_again is not necessary.

Removing unused fixture in test.

Simplifying logic after download completes.  Removing unused download sorting method.

ChannelDownloader returns location.

Hiding View link when no location is provided.

Performing video upsert in try/except so any errors are attached to the Download.

Including stderr only.

Including logs when returning Downloader.process_runner.

Warning when extention is replaced.

Providing location link on Downloads page.  Showing error modal on errors that fail.

Handling empty URL strings on import.

Fixing channel download test.

Reworking how Channels schedule Video downloads.

Bumping version for DownloadResult.

Updating tests to use DownloadResult.

VideoDownloader returns a DownloadResult.
This commit is contained in:
Roland 2022-05-03 10:46:10 -06:00
parent 28e983a3b3
commit 785ad8274c
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
17 changed files with 393 additions and 272 deletions

View file

@ -0,0 +1,33 @@
"""Download location
Revision ID: 1a70196c05bb
Revises: d8fcdb7773c0
Create Date: 2022-05-02 12:01:51.210070
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '1a70196c05bb'
down_revision = 'd8fcdb7773c0'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS location TEXT')
session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS error TEXT')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE download DROP COLUMN IF EXISTS location')
session.execute('ALTER TABLE download DROP COLUMN IF EXISTS error')

View file

@ -331,7 +331,7 @@ function ClearFailedDownloads({callback}) {
class DownloadRow extends React.Component {
render() {
let {url, frequency, last_successful_download, status, next_download} = this.props;
let {url, frequency, last_successful_download, status, location, next_download} = this.props;
let positive = false;
if (status === 'pending') {
positive = true;
@ -342,7 +342,7 @@ class DownloadRow extends React.Component {
<Table.Cell>{secondsToFrequency(frequency)}</Table.Cell>
<Table.Cell>{last_successful_download ? secondsToDate(last_successful_download) : null}</Table.Cell>
<Table.Cell>{secondsToDate(next_download)}</Table.Cell>
<Table.Cell>{status}</Table.Cell>
<Table.Cell>{location && <a href={location}>View</a>}</Table.Cell>
</Table.Row>
);
}
@ -354,6 +354,7 @@ class StoppableRow extends React.Component {
this.state = {
stopOpen: false,
startOpen: false,
errorModalOpen: false,
};
}
@ -389,9 +390,10 @@ class StoppableRow extends React.Component {
};
render() {
let {url, last_successful_download, status} = this.props;
let {stopOpen, startOpen} = this.state;
let {url, last_successful_download, status, location, error} = this.props;
let {stopOpen, startOpen, errorModalOpen} = this.state;
let completedAtCell = last_successful_download ? secondsToDate(last_successful_download) : null;
let buttonCell = <Table.Cell/>;
let positive = false;
let negative = false;
@ -418,10 +420,21 @@ class StoppableRow extends React.Component {
warning = status === 'deferred';
buttonCell = (
<Table.Cell>
<Button
onClick={this.openStop}
color='red'
>Stop</Button>
<Confirm
open={stopOpen}
content='Are you sure you want to stop this download? It will not be retried.'
confirmButton='Stop'
onCancel={this.closeStop}
onConfirm={this.handleStop}
/>
<Button
onClick={this.openStart}
color='green'
>Start</Button>
>Retry</Button>
<Confirm
open={startOpen}
content='Are you sure you want to restart this download?'
@ -431,13 +444,38 @@ class StoppableRow extends React.Component {
/>
</Table.Cell>
);
} else if (status === 'complete' && location) {
buttonCell = (
<Table.Cell>
<a href={location}>View</a>
</Table.Cell>
);
}
if (error) {
completedAtCell = (
<Modal
closeIcon
onClose={() => this.setState({errorModalOpen: false})}
onOpen={() => this.setState({errorModalOpen: true})}
open={errorModalOpen}
trigger={<Button icon='exclamation circle' color='red'/>}
>
<Modal.Header>Error</Modal.Header>
<Modal.Content>
<pre>{error}</pre>
</Modal.Content>
<Modal.Actions>
<Button onClick={() => this.setState({errorModalOpen: false})}>Close</Button>
</Modal.Actions>
</Modal>
)
}
return (
<Table.Row positive={positive} negative={negative} warning={warning}>
<Table.Cell><a href={url} target='_blank'>{textEllipsis(url, 50)}</a></Table.Cell>
<Table.Cell>{last_successful_download ? secondsToDate(last_successful_download) : null}</Table.Cell>
<Table.Cell>{status}</Table.Cell>
<Table.Cell>{completedAtCell}</Table.Cell>
{buttonCell}
</Table.Row>
);
@ -498,8 +536,8 @@ class Downloads extends React.Component {
<Table.Header>
<Table.Row>
<Table.HeaderCell>URL</Table.HeaderCell>
<Table.HeaderCell>Completed At</Table.HeaderCell>
<Table.HeaderCell>Status</Table.HeaderCell>
<Table.HeaderCell>Completed At</Table.HeaderCell>
<Table.HeaderCell>Control</Table.HeaderCell>
</Table.Row>
</Table.Header>
@ -512,7 +550,7 @@ class Downloads extends React.Component {
<Table.HeaderCell>Download Frequency</Table.HeaderCell>
<Table.HeaderCell>Last Successful Download</Table.HeaderCell>
<Table.HeaderCell>Next Download</Table.HeaderCell>
<Table.HeaderCell>Status</Table.HeaderCell>
<Table.HeaderCell>View</Table.HeaderCell>
</Table.Row>
</Table.Header>
);

View file

@ -2,7 +2,7 @@ from abc import ABC
from typing import Tuple
from wrolpi.common import logger
from wrolpi.downloader import Downloader, Download
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from . import lib
from .api import bp # noqa
@ -17,7 +17,7 @@ class ArchiveDownloader(Downloader, ABC):
pretty_name = 'Archive'
def __repr__(self):
return f'<ArchiveDownloader name={self.name}>'
return f'<ArchiveDownloader>'
@classmethod
def valid_url(cls, url: str) -> Tuple[bool, None]:
@ -26,12 +26,12 @@ class ArchiveDownloader(Downloader, ABC):
"""
return True, None
def do_download(self, download: Download):
def do_download(self, download: Download) -> DownloadResult:
if download.attempts > 3:
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
lib.do_archive(download.url)
return True
return DownloadResult(success=True)
# Archive downloader is the last downloader which should be used.

View file

@ -152,7 +152,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
# Verify that the URL/Name/directory aren't taken
check_for_channel_conflicts(
session,
url=data.url,
url=data.url or None,
name=data.name,
directory=data.directory,
source_id=data.source_id,
@ -162,7 +162,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
channel = Channel(
name=data.name,
url=data.url,
url=data.url or None,
match_regex=data.match_regex,
directory=data.directory, # noqa
download_frequency=data.download_frequency,

View file

@ -24,7 +24,8 @@ def test_get_channels(test_directory, channel_factory, test_client):
assert len(response.json['channels']) == 3
def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel):
@mock.patch('modules.videos.lib.validate_video')
def test_refresh_no_channel(_, test_session, test_client, test_directory, simple_channel):
# A regular user-defined channel
vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
vid1.touch()
@ -41,22 +42,7 @@ def test_refresh_no_channel(test_session, test_client, test_directory, simple_ch
vid4.touch()
test_client.post('/api/videos/refresh')
videos = test_session.query(Video).order_by(Video.title).all()
assert len(videos) == 4
# Add some fake videos, they should be removed.
test_session.add(Video(title='vid5'))
test_session.add(Video(title='vid6', channel_id=simple_channel.id))
test_session.commit()
videos = test_session.query(Video).order_by(Video.title).all()
assert len(videos) == 6
test_client.post('/api/videos/refresh')
videos = test_session.query(Video).order_by(Video.title).all()
assert len(videos) == 4
assert test_session.query(Video).count() == 4
def test_get_video(test_client, test_session, simple_channel, video_factory):

View file

@ -1,6 +1,9 @@
import pytest
from modules.videos import schema
from modules.videos.channel import lib
from modules.videos.lib import save_channels_config, import_channels_config
from modules.videos.models import Channel
from wrolpi.errors import UnknownChannel
@ -57,3 +60,32 @@ def test_get_channel(test_session, test_directory, channel_factory):
for p in params:
assert lib.get_channel(**p, return_dict=False) == channel_id_channel
assert lib.get_channel(**p)
def test_channels_no_url(test_session, test_directory):
"""Test that a Channel's URL is coerced to None if it is empty."""
channel1_directory = test_directory / 'channel1'
channel1_directory.mkdir()
channel2_directory = test_directory / 'channel2'
channel2_directory.mkdir()
channel1 = schema.ChannelPostRequest(
name='foo',
directory=str(channel1_directory),
)
channel1.url = ''
lib.create_channel(test_session, data=channel1)
channel2 = schema.ChannelPostRequest(
name='bar',
directory=str(channel2_directory),
url=''
)
lib.create_channel(test_session, data=channel2)
channels = list(test_session.query(Channel))
assert len(channels) == 2
assert all(i.url is None for i in channels), [(i.name, i.url) for i in channels]
save_channels_config()
import_channels_config()

View file

@ -100,7 +100,8 @@ def video_factory(test_session, test_directory):
if channel_id:
path = test_directory / f'{title}.mp4'
else:
path = test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4'
(test_directory / 'videos/NO CHANNEL').mkdir(parents=True, exist_ok=True)
path = test_directory / f'videos/NO CHANNEL/{title}.mp4'
if with_video_file:
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path)
@ -166,3 +167,22 @@ def test_downloader_config(test_directory):
set_test_downloader_config(True)
yield config_path
set_test_downloader_config(False)
@pytest.fixture
def mock_video_extract_info():
with mock.patch('modules.videos.downloader.extract_info') as mock_extract_info:
yield mock_extract_info
@pytest.fixture
def mock_video_prepare_filename():
with mock.patch('modules.videos.downloader.prepare_filename') as mock_prepare_filename:
yield mock_prepare_filename
@pytest.fixture
def mock_video_process_runner():
with mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
mock_process_runner.return_value = (0, {})
yield mock_process_runner

View file

@ -2,9 +2,11 @@
import json
import pathlib
import re
import traceback
from abc import ABC
from typing import Tuple, List, Optional
import yt_dlp.utils
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from yt_dlp.extractor import YoutubeTabIE
@ -14,13 +16,13 @@ from wrolpi.cmd import which
from wrolpi.common import logger, extract_domain, get_media_directory
from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG
from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG, DownloadResult
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import apply_info_json, get_no_channel_directory, \
get_videos_directory
from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id, validate_video
from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id
from .models import Video, Channel
from .schema import ChannelPostRequest
from .video_url_resolver import video_url_resolver
@ -41,16 +43,24 @@ PREFERRED_VIDEO_EXTENSION = 'mp4'
PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p,18,bestvideo+bestaudio'
def extract_info(url: str, ydl: YoutubeDL = YDL) -> dict:
"""Get info about a video. Separated for testing."""
return ydl.extract_info(url, download=False, process=False)
def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
"""Get filename from YoutubeDL. Separated for testing."""
return ydl.prepare_filename(entry)
class ChannelDownloader(Downloader, ABC):
"""
Handling downloading of all Videos in a Channel.
"""
"""Handles downloading of all Videos in a Channel."""
name = 'video_channel'
pretty_name = 'Video Channel'
listable = False
def __repr__(self):
return f'<ChannelDownloader name={self.name}>'
return f'<ChannelDownloader>'
@classmethod
def valid_url(cls, url) -> Tuple[bool, None]:
@ -60,10 +70,8 @@ class ChannelDownloader(Downloader, ABC):
logger.debug(f'{cls.__name__} not suitable for {url}')
return False, None
def do_download(self, download: Download):
"""
Update a Channel's catalog, then schedule downloads of every missing video.
"""
def do_download(self, download: Download) -> DownloadResult:
"""Update a Channel's catalog, then schedule downloads of every missing video."""
with get_db_session() as session:
channel = session.query(Channel).filter_by(url=download.url).one_or_none()
@ -71,6 +79,7 @@ class ChannelDownloader(Downloader, ABC):
# Couldn't get channel by URL, this is probably a playlist. Find the channel for the playlist.
channel_source_id = get_channel_source_id(download.url)
channel = get_channel(source_id=channel_source_id, return_dict=False)
channel_id = channel.id
update_channel(channel)
@ -91,7 +100,7 @@ class ChannelDownloader(Downloader, ABC):
else:
self.manager.start_downloads()
return True
return DownloadResult(success=True, location=f'/videos/channel/{channel_id}/video')
YT_DLP_BIN = which(
@ -110,7 +119,7 @@ class VideoDownloader(Downloader, ABC):
pretty_name = 'Videos'
def __repr__(self):
return f'<VideoDownloader name={self.name}>'
return f'<VideoDownloader>'
@classmethod
def valid_url(cls, url) -> Tuple[bool, Optional[dict]]:
@ -120,7 +129,7 @@ class VideoDownloader(Downloader, ABC):
for ie in YDL._ies.values():
if ie.suitable(url) and not ChannelDownloader.valid_url(url)[0]:
try:
info = YDL.extract_info(url, download=False, process=False)
info = extract_info(url)
return True, info
except UnsupportedError:
logger.debug(f'Video downloader extract_info failed for {url}')
@ -131,7 +140,7 @@ class VideoDownloader(Downloader, ABC):
logger.debug(f'{cls.__name__} not suitable for {url}')
return False, None
def do_download(self, download: Download):
def do_download(self, download: Download) -> DownloadResult:
if download.attempts >= 10:
raise UnrecoverableDownloadError('Max download attempts reached')
@ -160,6 +169,7 @@ class VideoDownloader(Downloader, ABC):
out_dir = channel.directory.path
out_dir.mkdir(exist_ok=True, parents=True)
logs = None # noqa
try:
video_path, entry = self.prepare_filename(url, out_dir)
# Do the real download.
@ -177,15 +187,24 @@ class VideoDownloader(Downloader, ABC):
'-o', file_name_format,
url,
]
return_code = self.process_runner(url, cmd, out_dir)
return_code, logs = self.process_runner(url, cmd, out_dir)
if return_code != 0:
raise ValueError(f'video downloader process exited with {return_code}')
if not video_path.is_file():
raise ValueError(f'Video file could not be found! {video_path}')
with get_db_session(commit=True) as session:
# If the video is from a channel, it will already be in the database.
source_id = entry['id']
existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none()
video = upsert_video(session, video_path, channel, id_=existing_video.id if existing_video else None)
video_id = video.id
except UnrecoverableDownloadError:
raise
except yt_dlp.utils.UnsupportedError as e:
raise UnrecoverableDownloadError('URL is not supported by yt-dlp') from e
except Exception as e:
logger.warning(f'VideoDownloader failed to download: {download.url}', exc_info=e)
if _skip_download(e):
@ -204,18 +223,22 @@ class VideoDownloader(Downloader, ABC):
# Skipped downloads should not be tried again.
raise UnrecoverableDownloadError() from e
return
# Download did not succeed, try again later.
if logs and (stderr := logs.get('stderr')):
error = f'{stderr}\n\n{traceback.format_exc()}'
else:
error = str(traceback.format_exc())
return DownloadResult(success=False, error=error)
with get_db_session(commit=True) as session:
# If the video is from a channel, it will already be in the database.
source_id = entry['id']
existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none()
id_ = existing_video.id if existing_video else None
upsert_video(session, video_path, channel, id_=id_)
# Successful download.
return True
if channel:
location = f'/videos/channel/{channel.id}/video/{video_id}'
else:
location = f'/videos/video/{video_id}'
result = DownloadResult(
success=True,
location=location,
)
return result
@staticmethod
def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
@ -234,9 +257,13 @@ class VideoDownloader(Downloader, ABC):
ydl.params['logger'] = ydl_logger
ydl.add_default_info_extractors()
# Perform the real download. This saves multiple files.
entry = ydl.extract_info(url, download=False, process=True)
final_filename = pathlib.Path(ydl.prepare_filename(entry))
# Get the path where the video will be saved.
entry = extract_info(url, ydl=ydl)
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
if final_filename.suffix == '.NA':
# yt-dlp has odd behavior sometimes and ignores our preferred video extension.
final_filename = final_filename.with_suffix('.mp4')
logger.warning(f'Had to replace video extension: {final_filename}')
return final_filename, entry
@ -284,10 +311,10 @@ def update_channel(channel: Channel):
It is expected that any missing videos will be downloaded later.
"""
logger.info(f'Downloading video list for {channel.name} at {channel.url} This may take several minutes.')
info = YDL.extract_info(channel.url, download=False, process=False)
info = extract_info(channel.url)
if 'url' in info:
url = info['url']
info = YDL.extract_info(url, download=False, process=False)
info = extract_info(url)
# Resolve all entries to dictionaries.
entries = info['entries'] = list(info['entries'])
@ -299,7 +326,7 @@ def update_channel(channel: Channel):
for entry in entries:
if entry['title'] == 'Uploads':
logger.info('Youtube-DL gave back a list of URLs, found the "Uploads" URL and using it.')
info = YDL.extract_info(entry['url'], download=False, process=False)
info = extract_info(entry['url'])
entries = info['entries'] = list(info['entries'])
break
@ -359,7 +386,6 @@ def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]:
if channel_id:
where = 'AND channel_id = %s'
params = (channel_id,)
query = f'''
SELECT
video.id, video.source_id, video.channel_id
@ -380,7 +406,7 @@ def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]:
return missing_videos
def find_all_missing_videos(channel_id: str = None) -> Tuple[dict, dict]:
def find_all_missing_videos(channel_id: int = None) -> Tuple[dict, dict]:
"""
Find all videos that don't have a video file, but are found in the DB (taken from the channel's info_json).

View file

@ -6,7 +6,6 @@ from typing import Tuple, Optional, List, Union
from uuid import uuid1
from sqlalchemy import or_
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
@ -18,7 +17,8 @@ from wrolpi.media_path import MediaPath
from wrolpi.vars import PYTEST
from .captions import get_captions
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \
is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError
is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
get_no_channel_directory
from .models import Channel, Video
logger = logger.getChild(__name__)
@ -60,7 +60,8 @@ def refresh_channel_videos(channel: Channel):
logger.debug(f'{channel.name}: Added {video_path.name}')
with get_db_curs(commit=True) as curs:
curs.execute('DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL RETURNING id', (channel.id,))
stmt = 'DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL AND video_path IS NOT NULL RETURNING id'
curs.execute(stmt, (channel.id,))
deleted_count = len(curs.fetchall())
if deleted_count:
@ -544,6 +545,9 @@ def import_channels_config():
channel.name = data['name']
channel.directory = directory
# A URL should not be an empty string
data['url'] = data['url'] or None
# Copy existing channel data, update all values from the config. This is necessary to clear out
# values not in the config.
full_data = channel.dict()
@ -571,7 +575,6 @@ def import_channels_config():
continue
channel_dir = channel.directory.path
else:
from .downloader import get_no_channel_directory
channel_dir = get_no_channel_directory()
# Set favorite Videos of this Channel.
@ -712,7 +715,11 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
raise ValueError(f'Video path is not absolute: {video_path}')
# This function can update or insert a Video.
video = session.query(Video).filter_by(id=id_).one() if id_ else Video()
if id_:
video = session.query(Video).filter_by(id=id_).one()
else:
video = Video()
session.add(video)
# Set the file values, all other things can be found using these files.
poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path)
@ -728,6 +735,9 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
video.channel_id = channel.id
video.idempotency = idempotency
session.flush()
session.refresh(video)
try:
# Fill in any missing data. Generate poster if enabled and necessary.
validate_video(video, channel.generate_posters if channel else False)
@ -738,11 +748,6 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
# inserted.
logger.warning(f'Failed to validate {video}', exc_info=e)
try:
session.add(video)
except InvalidRequestError:
# Video is already in a session.
pass
session.flush()
return video

View file

@ -338,7 +338,7 @@ class Channel(ModelHelper, Base):
name=self.name,
skip_download_videos=self.skip_download_videos or [],
source_id=self.source_id,
url=self.url or '',
url=self.url or None,
)
return config

View file

@ -347,6 +347,7 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
assert download.frequency == channel1.download_frequency
assert download.downloader == 'video_channel'
assert download.next_download
assert not download.error
next_download = str(download.next_download)
import_channels_config()

View file

@ -1,6 +1,7 @@
import pathlib
import shutil
from copy import copy
from itertools import zip_longest
from unittest import mock
import pytest
@ -13,22 +14,11 @@ from modules.videos.models import Channel, Video
from wrolpi.db import get_db_context
from wrolpi.downloader import DownloadManager, Download
from wrolpi.errors import InvalidDownload
from wrolpi.test.common import wrap_test_db, TestAPI
from wrolpi.test.common import TestAPI
from wrolpi.test.test_downloader import HTTPDownloader
from wrolpi.vars import PROJECT_DIR
class FakeYDL:
@staticmethod
def extract_info(*a, **kw):
return {
'entries': [
{'id': 1},
],
}
def test_find_all_missing_videos(test_session, channel_factory, video_factory):
channel1, channel2 = channel_factory(url='some url'), channel_factory(url='some other url')
channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]}
@ -155,68 +145,52 @@ class TestVideosDownloaders(TestAPI):
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance')[0], self.cd)
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/watch?v=31jPEBiAC3c')[0], self.vd)
@staticmethod
def _make_video_files(channel_dir):
video_path = channel_dir / 'video.mp4'
poster_path = channel_dir / 'video.jpg'
description_path = channel_dir / 'video.description'
info_json_path = channel_dir / 'video.info.json'
video_path.touch()
poster_path.touch()
description_path.touch()
info_json_path.touch()
return video_path, poster_path, description_path, info_json_path
@wrap_test_db
def test_video_download_no_channel(self):
_, session = get_db_context()
def test_video_download_no_channel(test_session, video_download_manager, video_factory, test_directory,
mock_video_extract_info, mock_video_process_runner):
"""A video can be downloaded even if it does not have a Channel."""
channel_dir = test_directory / 'NO CHANNEL'
channel_dir.mkdir(parents=True)
video_path = channel_dir / 'video.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
channel_dir = self.videos_dir / 'NO CHANNEL'
channel_dir.mkdir(parents=True)
# Video has no channel
info_json = copy(example_video_json)
del info_json['channel']
del info_json['channel_id']
del info_json['channel_url']
# Video paths
video_path, _, _, _ = self._make_video_files(channel_dir)
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
mock_video_extract_info.return_value = info_json
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
video_download_manager.create_download(url, test_session)
# Video has no channel
info_json = copy(example_video_json)
del info_json['channel']
del info_json['channel_id']
del info_json['channel_url']
mock_video_process_runner.assert_called_once()
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
mock_extract_info.return_value = info_json
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
mock_process_runner.return_value = 0
self.mgr.create_download(url, session)
mock_process_runner.assert_called_once()
video: Video = session.query(Video).one()
self.assertEqual(str(video.video_path.path), f'{channel_dir}/video.mp4')
video: Video = test_session.query(Video).one()
assert str(video.video_path.path) == f'{channel_dir}/video.mp4'
def test_channel_download(test_session, simple_channel, video_download_manager, video_file):
def test_download_channel(test_session, simple_channel, video_download_manager, video_file,
mock_video_extract_info, mock_video_prepare_filename, mock_video_process_runner):
"""Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos."""
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
mock_extract_info.return_value = example_channel_json
mock_prepare_filename.return_value = (video_file, {'id': 'foo'})
mock_process_runner.return_value = 0
mock_video_prepare_filename.return_value = video_file
mock_video_extract_info.return_value = example_channel_json
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
mock_get_channel.return_value = simple_channel
video_download_manager.create_download(url, test_session)
# Two videos are in the example channel.
downloads = video_download_manager.get_new_downloads(test_session)
expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o']
for download, expected in zip(downloads, expected):
downloads = video_download_manager.get_once_downloads(test_session)
downloads = filter(lambda i: 'watch' in i.url, downloads)
expected = ['https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url']
for download, expected in zip_longest(downloads, expected):
assert download.url == expected
# Download is skipped for test.
assert download.status == 'new'
# Download is run only for test.
assert download.status == 'complete'
def test_get_or_create_channel(test_session):
@ -259,23 +233,25 @@ def test_get_or_create_channel(test_session):
assert get_or_create_channel(source_id='quux') == channel
def test_download_info_save(test_session, video_download_manager):
def test_download_info_save(test_session, video_download_manager, mock_video_extract_info, mock_video_prepare_filename,
mock_video_process_runner):
"""
An info dict from Downloader.valid_url is saved to the Downloader and can be used by the Downloader later.
"""
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_extract_info.return_value = {'some': 'info'}
video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session)
mock_extract_info.assert_called_once()
mock_video_extract_info.return_value = {'some': 'info'}
mock_video_prepare_filename.return_value = 'some path'
video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session)
assert mock_video_extract_info.call_count == 2 # Only called twice. The "info" gathering call was skipped.
def test_selected_downloader(test_session, video_download_manager):
def test_selected_downloader(test_session, video_download_manager, successful_download):
"""
A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
"""
# The test HTTP Downloader will claim to be able to handle any HTTP URL; will pretend to have downloaded anything.
http_downloader = HTTPDownloader()
mock_do_download = http_downloader.do_download = mock.MagicMock()
mock_do_download.return_value = successful_download
mock_http_valid_url = http_downloader.valid_url = mock.MagicMock()
mock_http_valid_url.return_value = (True, None)
video_download_manager.register_downloader(http_downloader)
@ -286,7 +262,7 @@ def test_selected_downloader(test_session, video_download_manager):
# DownloadManager will attempt to choose the Downloader.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_extract_info.side_effect = UnsupportedError('not this one')
video_download_manager.create_downloads(['https://example.com'], test_session)
video_download_manager.create_download('https://example.com', test_session)
mock_http_valid_url.assert_called_once()
mock_extract_info.assert_called_once()
mock_do_download.assert_called_once()
@ -298,7 +274,7 @@ def test_selected_downloader(test_session, video_download_manager):
# DownloadManager will only use the HTTPDownloader.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
video_download_manager.create_download('https://example.com', test_session, downloader='http')
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
mock_extract_info.assert_not_called()
mock_http_valid_url.assert_not_called()
@ -313,7 +289,7 @@ def test_selected_downloader(test_session, video_download_manager):
# DownloadManager will only use the HTTPDownloader, even if the download fails.
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
mock_do_download.side_effect = Exception('oh no!')
video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
video_download_manager.create_download('https://example.com', test_session, downloader='http')
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
mock_extract_info.assert_not_called()
mock_http_valid_url.assert_not_called()
@ -351,22 +327,22 @@ def test_channel_download_no_download(test_session, video_download_manager, simp
download_channel(simple_channel.id)
def test_invalid_download_url(test_session, test_download_manager, test_downloader_config):
def test_invalid_download_url(test_session, test_download_manager, mock_video_extract_info):
"""An invalid url should not be attempted again."""
_, session = get_db_context()
with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \
mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader, \
mock.patch('modules.videos.downloader.YDL') as mock_ydl:
mock_get_downloader.return_value = (VideoDownloader(), {'channel': 'foo', 'id': 'some id'})
mock_valid_url.return_value = (True, None)
mock_ydl.extract_info.side_effect = UnsupportedError('oops')
test_download_manager.create_download('url', session)
mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader:
mock_get_downloader.return_value = (VideoDownloader(), None)
mock_valid_url.return_value = (True, {}) # url is valid, but is not a video URL.
mock_video_extract_info.side_effect = UnsupportedError('oops')
test_download_manager.create_download('invalid url', session)
download = test_download_manager.get_downloads(test_session)[0]
download = session.query(Download).one()
assert download.status == 'failed'
def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager):
def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager,
mock_video_process_runner):
simple_channel.source_id = example_video_json['channel_id']
simple_channel.directory = test_directory / 'videos/channel name'
simple_channel.directory.mkdir(parents=True)
@ -376,16 +352,14 @@ def test_video_download_1(test_session, test_directory, simple_channel, video_do
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
mock_extract_info.return_value = example_video_json
mock_prepare_filename.return_value = (video_file, {'id': 'foo'})
mock_process_runner.return_value = 0
video_download_manager.create_download(url, test_session)
mock_process_runner.assert_called_once()
video_url, _, out_dir = mock_process_runner.call_args[0]
mock_video_process_runner.assert_called_once()
video_url, _, out_dir = mock_video_process_runner.call_args[0]
download: Download = test_session.query(Download).one()
assert video_url == download.url
@ -393,3 +367,21 @@ def test_video_download_1(test_session, test_directory, simple_channel, video_do
video: Video = test_session.query(Video).one()
assert video.video_path.path.is_absolute()
def test_download_result(test_session, test_directory, video_download_manager, mock_video_process_runner,
mock_video_extract_info):
"""VideoDownloader returns a DownloadResult when complete."""
channel_directory = test_directory / 'videos/channel name'
channel_directory.mkdir(parents=True)
video_file = channel_directory / 'the video.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file)
mock_video_extract_info.return_value = example_video_json
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
mock_prepare_filename.return_value = [video_file, {'id': 'foo'}]
video_download_manager.create_download('https://example.com', test_session)
download: Download = test_session.query(Download).one()
assert download.url == 'https://example.com'
assert download.location == '/videos/channel/1/video/1'

View file

@ -555,9 +555,7 @@ def get_relative_to_media_directory(path: Union[str, Path]) -> Path:
def minimize_dict(d: dict, keys: Iterable) -> Optional[dict]:
"""
Return a new dictionary that contains only the keys provided.
"""
"""Return a new dictionary that contains only the keys provided."""
if d is None:
return
return {k: d[k] for k in set(keys) & d.keys()}

View file

@ -15,7 +15,7 @@ from sqlalchemy.orm import Session, sessionmaker
from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.dates import set_test_now
from wrolpi.db import postgres_engine, get_db_args
from wrolpi.downloader import DownloadManager
from wrolpi.downloader import DownloadManager, DownloadResult
from wrolpi.root_api import BLUEPRINTS, api_app
@ -124,3 +124,13 @@ def fake_now():
finally:
# reset now() to its original functionality.
set_test_now(None) # noqa
@pytest.fixture
def successful_download():
return DownloadResult(success=True)
@pytest.fixture
def failed_download():
return DownloadResult(error='pytest.fixture failed_download error', success=False)

View file

@ -2,7 +2,8 @@ import asyncio
import multiprocessing
import pathlib
import subprocess
from collections import defaultdict
import traceback
from dataclasses import dataclass, field
from datetime import timedelta, datetime
from enum import Enum
from functools import partial
@ -13,7 +14,7 @@ from sqlalchemy import Column, Integer, String, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Session
from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag, ConfigFile
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile
from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz
from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
@ -34,6 +35,15 @@ class DownloadFrequency(int, Enum):
days90 = 7776000
@dataclass
class DownloadResult:
downloads: List[str] = field(default_factory=list)
error: str = None
info_json: dict = field(default_factory=dict)
location: str = None
success: bool = False
class Download(ModelHelper, Base):
"""
Model that is used to schedule downloads.
@ -49,6 +59,8 @@ class Download(ModelHelper, Base):
status = Column(String, default='new')
info_json = Column(JSONB)
downloader = Column(Text)
location = Column(Text)
error = Column(Text)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@ -67,6 +79,7 @@ class Download(ModelHelper, Base):
frequency=self.frequency,
id=self.id,
last_successful_download=self.last_successful_download,
location=self.location,
next_download=self.next_download,
status=self.status,
url=self.url,
@ -74,39 +87,30 @@ class Download(ModelHelper, Base):
return d
def renew(self, reset_attempts: bool = False):
"""
Mark this Download as "new" so it will be retried.
"""
"""Mark this Download as "new" so it will be retried."""
self.status = 'new'
if reset_attempts:
self.attempts = 0
def defer(self):
"""
Download should be tried again after a time.
"""
"""Download should be tried again after a time."""
self.status = 'deferred'
def fail(self):
"""
Download should not be attempted again. A recurring Download will raise an error.
"""
"""Download should not be attempted again. A recurring Download will raise an error."""
if self.frequency:
raise ValueError('Recurring download should not be failed.')
self.status = 'failed'
def started(self):
"""
Mark this Download as in progress.
"""
"""Mark this Download as in progress."""
self.attempts += 1
self.status = 'pending'
def complete(self):
"""
Mark this Download as successfully downloaded.
"""
"""Mark this Download as successfully downloaded."""
self.status = 'complete'
self.error = None # clear any old errors
self.last_successful_download = now()
def get_downloader(self):
@ -147,7 +151,7 @@ class Downloader:
def valid_url(cls, url) -> Tuple[bool, Optional[dict]]:
raise NotImplementedError()
def do_download(self, download: Download):
def do_download(self, download: Download) -> DownloadResult:
raise NotImplementedError()
@property
@ -174,7 +178,7 @@ class Downloader:
if self._kill.is_set():
self._kill.clear()
def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> int:
def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> Tuple[int, dict]:
"""
Run a subprocess using the provided arguments. This process can be killed by the Download Manager.
"""
@ -202,13 +206,10 @@ class Downloader:
# Output all logs from the process.
# TODO is there a way to stream this output while the process is running?
logger.debug(f'Download exited with {proc.returncode}')
outs, errs = proc.communicate()
for line in outs.decode().splitlines(): # noqa
logger.debug(line)
for line in errs.decode().splitlines(): # noqa
logger.error(line)
stdout, stderr = proc.communicate()
logs = {'stdout': stdout, 'stderr': stderr}
return proc.returncode
return proc.returncode, logs
class DownloadManager:
@ -233,15 +234,14 @@ class DownloadManager:
def get_downloader(self, url: str) -> Tuple[Downloader, Dict]:
for i in self.instances:
result = i.valid_url(url)
valid, info = result
valid, info = i.valid_url(url)
if valid:
return i, info
raise InvalidDownload(f'Invalid URL {url=}')
def get_downloader_by_name(self, name: str) -> Downloader:
"""
Attempts to find a registered Downloader by its name. Returns None if it cannot be found.
Tries to find a registered Downloader by its name. Returns None if it cannot be found.
"""
return self._instances.get(name)
@ -263,9 +263,7 @@ class DownloadManager:
yield download
def get_or_create_download(self, url: str, session: Session) -> Download:
"""
Get a Download by its URL, if it cannot be found create one.
"""
"""Get a Download by its URL, if it cannot be found create one."""
if not url:
raise ValueError('Download must have a URL')
@ -360,29 +358,34 @@ class DownloadManager:
download.started()
session.commit()
success = True
failure = False
try_again = True
try:
success = downloader.do_download(download)
result = downloader.do_download(download)
except UnrecoverableDownloadError as e:
# Download failed and should not be retried.
logger.warning(f'UnrecoverableDownloadError for {url}', exc_info=e)
failure = True
result = DownloadResult(success=False, error=str(traceback.format_exc()))
try_again = False
except Exception as e:
logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e)
result = DownloadResult(success=False, error=str(traceback.format_exc()))
# Long-running, get the download again.
with get_db_session(commit=True) as session_:
download = self.get_download(session_, id_=download_id)
if failure:
# Use a new location if provided, keep the old location if no new location is provided, otherwise
# clear out an outdated location.
download.location = result.location or download.location or None
# Clear any old errors if the download succeeded.
download.error = result.error if download.error else None
download.next_download = self.get_next_download(download, session)
if try_again is False:
download.fail()
elif success:
elif result.success:
download.complete()
download.next_download = self.get_next_download(download, session)
else:
download.defer()
download.next_download = self.get_next_download(download, session)
session_.commit()
if download_count:
logger.info(f'Done doing {download_count} downloads.')
@ -409,9 +412,7 @@ class DownloadManager:
@staticmethod
def reset_downloads():
"""
Set any incomplete Downloads to `new` so they will be retried.
"""
"""Set any incomplete Downloads to `new` so they will be retried."""
with get_db_curs(commit=True) as curs:
curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'")
@ -430,20 +431,6 @@ class DownloadManager:
DOWNLOAD_SORT = ('pending', 'failed', 'new', 'deferred', 'complete')
@classmethod
@iterify(list)
def _downloads_sorter(cls, downloads: List[Download]):
"""
Downloads should be displayed to the user by status.
"""
grouped_by_statuses = defaultdict(lambda: [])
for download in downloads:
grouped_by_statuses[download.status].append(download)
# Yield the downloads in the order defined above.
for sort in cls.DOWNLOAD_SORT:
yield from grouped_by_statuses[sort] # converted to a list by iterify()
@optional_session
def get_recurring_downloads(self, session: Session = None, limit: int = None):
"""Get all Downloads that will be downloaded in the future."""
@ -456,7 +443,6 @@ class DownloadManager:
if limit:
query = query.limit(limit)
downloads = query.all()
downloads = self._downloads_sorter(downloads)
return downloads
@optional_session
@ -471,14 +457,11 @@ class DownloadManager:
if limit:
query = query.limit(limit)
downloads = query.all()
downloads = self._downloads_sorter(downloads)
return downloads
@optional_session
def renew_recurring_downloads(self, session: Session = None):
"""
Mark any recurring downloads that are due for download as "new". Start a download.
"""
"""Mark any recurring downloads that are due for download as "new". Start a download."""
now_ = now()
recurring = self.get_recurring_downloads(session)
@ -523,9 +506,7 @@ class DownloadManager:
return False
def kill_download(self, download_id: int):
"""
Fail a Download. If it is pending, kill the Downloader so the download stops.
"""
"""Fail a Download. If it is pending, kill the Downloader so the download stops."""
with get_db_session(commit=True) as session:
download = self.get_download(session, id_=download_id)
downloader = download.get_downloader()
@ -535,9 +516,7 @@ class DownloadManager:
download.fail()
def kill(self):
"""
Kill all downloads.
"""
"""Kill all downloads. Do not start new downloads."""
self.disabled.set()
try:
with get_db_session(commit=True) as session:
@ -610,9 +589,11 @@ class DownloadManager:
stmt = f'''
SELECT
downloader,
error,
frequency,
id,
last_successful_download,
location,
next_download,
status,
url

View file

@ -9,7 +9,7 @@ import pytest
from wrolpi import downloader
from wrolpi.dates import local_timezone, Seconds
from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency
from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled
from wrolpi.test.common import wrap_test_db, TestAPI
@ -72,7 +72,7 @@ class TestDownloader(TestAPI):
self.assertEqual(len(self.mgr.get_downloads(session)), 0)
http_downloader = HTTPDownloader()
http_downloader.do_download.return_value = False
http_downloader.do_download.return_value = DownloadResult(success=True)
self.mgr.register_downloader(http_downloader)
self.mgr.create_download('https://example.com', session)
@ -88,11 +88,11 @@ class TestDownloader(TestAPI):
_, session = get_db_context()
http_downloader = HTTPDownloader()
http_downloader.do_download.return_value = True
http_downloader.do_download.return_value = DownloadResult(success=True)
self.mgr.register_downloader(http_downloader)
permissive_downloader = PermissiveDownloader(priority=100)
permissive_downloader.do_download.return_value = False # returns a failure
permissive_downloader.do_download.return_value = DownloadResult(success=False)
self.mgr.register_downloader(permissive_downloader)
# https is handled by the HTTP Downloader.
@ -117,7 +117,7 @@ class TestDownloader(TestAPI):
self.assertEqual(download.attempts, 2)
# finally success
permissive_downloader.do_download.return_value = True
permissive_downloader.do_download.return_value = DownloadResult(success=True)
self.mgr.create_download('foo', session)
self.mgr._do_downloads(session)
download = self.mgr.get_download(session, 'foo')
@ -126,32 +126,6 @@ class TestDownloader(TestAPI):
# No downloads left.
self.assertEqual(list(self.mgr.get_new_downloads(session)), [])
@wrap_test_db
def test_max_attempts(self):
"""
A Download will only be attempted so many times, it will eventually be deleted.
"""
_, session = get_db_context()
http_downloader = HTTPDownloader()
http_downloader.do_download.return_value = False
self.mgr.register_downloader(http_downloader)
self.mgr.create_download('https://example.com', session)
download = session.query(Download).one()
self.assertEqual(download.attempts, 1)
self.mgr.create_download('https://example.com', session)
download = session.query(Download).one()
self.assertEqual(download.attempts, 2)
# There are no further attempts.
http_downloader.do_download.side_effect = UnrecoverableDownloadError()
self.mgr.create_download('https://example.com', session)
download = session.query(Download).one()
self.assertEqual(download.attempts, 3)
self.assertEqual(download.status, 'failed')
@wrap_test_db
@mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0)))
def test_delete_old_once_downloads(self):
@ -186,12 +160,12 @@ class TestDownloader(TestAPI):
self.mgr.delete_old_once_downloads()
# Two old downloads are deleted.
downloads = self.mgr._downloads_sorter(session.query(Download).all())
downloads = session.query(Download).order_by(Download.id).all()
expected = [
dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'),
dict(url='https://example.com/6', attempts=1, status='pending'),
dict(url='https://example.com/1', frequency=1, attempts=0, status='new'),
dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'),
dict(url='https://example.com/5', attempts=0, status='new'),
dict(url='https://example.com/6', attempts=1, status='pending'),
]
for download, expected in zip_longest(downloads, expected):
self.assertDictContains(download.dict(), expected)
@ -298,7 +272,7 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
http_downloader = HTTPDownloader()
http_downloader.do_download = MagicMock()
http_downloader.do_download.return_value = True
http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
# Download every hour.
@ -337,7 +311,7 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
# Try the download, but it fails.
http_downloader.do_download.reset_mock()
http_downloader.do_download.return_value = False
http_downloader.do_download.return_value = DownloadResult(success=False)
test_download_manager.do_downloads_sync()
http_downloader.do_download.assert_called_once()
download = session.query(Download).one()
@ -345,13 +319,13 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
assert download.status == 'deferred'
assert download.last_successful_download == now
# Download should be retried after the DEFAULT_RETRY_FREQUENCY.
expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1))
expected = local_timezone(datetime(2020, 1, 1, 3, 0, 0, 997200))
assert download.next_download == expected
# Try the download again, it finally succeeds.
http_downloader.do_download.reset_mock()
now = fake_now(datetime(2020, 1, 1, 4, 0, 1))
http_downloader.do_download.return_value = True
http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.renew_recurring_downloads(session)
test_download_manager.do_downloads_sync()
http_downloader.do_download.assert_called_once()
@ -360,3 +334,28 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
assert download.last_successful_download == now
# Floats cause slightly wrong date.
assert download.next_download == local_timezone(datetime(2020, 1, 1, 5, 0, 0, 997200))
def test_max_attempts(test_session, test_download_manager):
"""A Download will only be attempted so many times, it will eventually be deleted."""
_, session = get_db_context()
http_downloader = HTTPDownloader()
http_downloader.do_download = MagicMock()
http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
test_download_manager.create_download('https://example.com', session)
download = session.query(Download).one()
assert download.attempts == 1
test_download_manager.create_download('https://example.com', session)
download = session.query(Download).one()
assert download.attempts == 2
# There are no further attempts.
http_downloader.do_download.side_effect = UnrecoverableDownloadError()
test_download_manager.create_download('https://example.com', session)
download = session.query(Download).one()
assert download.attempts == 3
assert download.status == 'failed'

View file

@ -1,4 +1,4 @@
__version__ = '0.4.7-beta'
__version__ = '0.4.8-beta'
import subprocess