diff --git a/alembic/versions/1a70196c05bb_download_location.py b/alembic/versions/1a70196c05bb_download_location.py new file mode 100644 index 00000000..3172a9fb --- /dev/null +++ b/alembic/versions/1a70196c05bb_download_location.py @@ -0,0 +1,33 @@ +"""Download location + +Revision ID: 1a70196c05bb +Revises: d8fcdb7773c0 +Create Date: 2022-05-02 12:01:51.210070 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '1a70196c05bb' +down_revision = 'd8fcdb7773c0' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS location TEXT') + session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS error TEXT') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE download DROP COLUMN IF EXISTS location') + session.execute('ALTER TABLE download DROP COLUMN IF EXISTS error') diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 87543de9..f028672e 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -331,7 +331,7 @@ function ClearFailedDownloads({callback}) { class DownloadRow extends React.Component { render() { - let {url, frequency, last_successful_download, status, next_download} = this.props; + let {url, frequency, last_successful_download, status, location, next_download} = this.props; let positive = false; if (status === 'pending') { positive = true; @@ -342,7 +342,7 @@ class DownloadRow extends React.Component { {secondsToFrequency(frequency)} {last_successful_download ? secondsToDate(last_successful_download) : null} {secondsToDate(next_download)} - {status} + {location && View} ); } @@ -354,6 +354,7 @@ class StoppableRow extends React.Component { this.state = { stopOpen: false, startOpen: false, + errorModalOpen: false, }; } @@ -389,9 +390,10 @@ class StoppableRow extends React.Component { }; render() { - let {url, last_successful_download, status} = this.props; - let {stopOpen, startOpen} = this.state; + let {url, last_successful_download, status, location, error} = this.props; + let {stopOpen, startOpen, errorModalOpen} = this.state; + let completedAtCell = last_successful_download ? secondsToDate(last_successful_download) : null; let buttonCell = ; let positive = false; let negative = false; @@ -418,10 +420,21 @@ class StoppableRow extends React.Component { warning = status === 'deferred'; buttonCell = ( + + + >Retry ); + } else if (status === 'complete' && location) { + buttonCell = ( + + View + + ); + } + if (error) { + completedAtCell = ( + this.setState({errorModalOpen: false})} + onOpen={() => this.setState({errorModalOpen: true})} + open={errorModalOpen} + trigger={ + + + ) } return ( {textEllipsis(url, 50)} - {last_successful_download ? secondsToDate(last_successful_download) : null} {status} + {completedAtCell} {buttonCell} ); @@ -498,8 +536,8 @@ class Downloads extends React.Component { URL - Completed At Status + Completed At Control @@ -512,7 +550,7 @@ class Downloads extends React.Component { Download Frequency Last Successful Download Next Download - Status + View ); diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 8d33572b..981561a6 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -2,7 +2,7 @@ from abc import ABC from typing import Tuple from wrolpi.common import logger -from wrolpi.downloader import Downloader, Download +from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError from . import lib from .api import bp # noqa @@ -17,7 +17,7 @@ class ArchiveDownloader(Downloader, ABC): pretty_name = 'Archive' def __repr__(self): - return f'' + return f'' @classmethod def valid_url(cls, url: str) -> Tuple[bool, None]: @@ -26,12 +26,12 @@ class ArchiveDownloader(Downloader, ABC): """ return True, None - def do_download(self, download: Download): + def do_download(self, download: Download) -> DownloadResult: if download.attempts > 3: raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}') lib.do_archive(download.url) - return True + return DownloadResult(success=True) # Archive downloader is the last downloader which should be used. diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index 3b6f19bb..fca6017e 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -152,7 +152,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic # Verify that the URL/Name/directory aren't taken check_for_channel_conflicts( session, - url=data.url, + url=data.url or None, name=data.name, directory=data.directory, source_id=data.source_id, @@ -162,7 +162,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic channel = Channel( name=data.name, - url=data.url, + url=data.url or None, match_regex=data.match_regex, directory=data.directory, # noqa download_frequency=data.download_frequency, diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index f2fe566b..1c77df11 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -24,7 +24,8 @@ def test_get_channels(test_directory, channel_factory, test_client): assert len(response.json['channels']) == 3 -def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel): +@mock.patch('modules.videos.lib.validate_video') +def test_refresh_no_channel(_, test_session, test_client, test_directory, simple_channel): # A regular user-defined channel vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4' vid1.touch() @@ -41,22 +42,7 @@ def test_refresh_no_channel(test_session, test_client, test_directory, simple_ch vid4.touch() test_client.post('/api/videos/refresh') - - videos = test_session.query(Video).order_by(Video.title).all() - assert len(videos) == 4 - - # Add some fake videos, they should be removed. - test_session.add(Video(title='vid5')) - test_session.add(Video(title='vid6', channel_id=simple_channel.id)) - test_session.commit() - - videos = test_session.query(Video).order_by(Video.title).all() - assert len(videos) == 6 - - test_client.post('/api/videos/refresh') - - videos = test_session.query(Video).order_by(Video.title).all() - assert len(videos) == 4 + assert test_session.query(Video).count() == 4 def test_get_video(test_client, test_session, simple_channel, video_factory): diff --git a/modules/videos/channel/test/test_lib.py b/modules/videos/channel/test/test_lib.py index 1e8bc510..f1af902a 100644 --- a/modules/videos/channel/test/test_lib.py +++ b/modules/videos/channel/test/test_lib.py @@ -1,6 +1,9 @@ import pytest +from modules.videos import schema from modules.videos.channel import lib +from modules.videos.lib import save_channels_config, import_channels_config +from modules.videos.models import Channel from wrolpi.errors import UnknownChannel @@ -57,3 +60,32 @@ def test_get_channel(test_session, test_directory, channel_factory): for p in params: assert lib.get_channel(**p, return_dict=False) == channel_id_channel assert lib.get_channel(**p) + + +def test_channels_no_url(test_session, test_directory): + """Test that a Channel's URL is coerced to None if it is empty.""" + channel1_directory = test_directory / 'channel1' + channel1_directory.mkdir() + channel2_directory = test_directory / 'channel2' + channel2_directory.mkdir() + + channel1 = schema.ChannelPostRequest( + name='foo', + directory=str(channel1_directory), + ) + channel1.url = '' + lib.create_channel(test_session, data=channel1) + + channel2 = schema.ChannelPostRequest( + name='bar', + directory=str(channel2_directory), + url='' + ) + lib.create_channel(test_session, data=channel2) + + channels = list(test_session.query(Channel)) + assert len(channels) == 2 + assert all(i.url is None for i in channels), [(i.name, i.url) for i in channels] + + save_channels_config() + import_channels_config() diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 915f7329..42a2348a 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -100,7 +100,8 @@ def video_factory(test_session, test_directory): if channel_id: path = test_directory / f'{title}.mp4' else: - path = test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4' + (test_directory / 'videos/NO CHANNEL').mkdir(parents=True, exist_ok=True) + path = test_directory / f'videos/NO CHANNEL/{title}.mp4' if with_video_file: shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path) @@ -166,3 +167,22 @@ def test_downloader_config(test_directory): set_test_downloader_config(True) yield config_path set_test_downloader_config(False) + + +@pytest.fixture +def mock_video_extract_info(): + with mock.patch('modules.videos.downloader.extract_info') as mock_extract_info: + yield mock_extract_info + + +@pytest.fixture +def mock_video_prepare_filename(): + with mock.patch('modules.videos.downloader.prepare_filename') as mock_prepare_filename: + yield mock_prepare_filename + + +@pytest.fixture +def mock_video_process_runner(): + with mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: + mock_process_runner.return_value = (0, {}) + yield mock_process_runner diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index e974e7ee..44e18560 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -2,9 +2,11 @@ import json import pathlib import re +import traceback from abc import ABC from typing import Tuple, List, Optional +import yt_dlp.utils from sqlalchemy.orm import Session from yt_dlp import YoutubeDL from yt_dlp.extractor import YoutubeTabIE @@ -14,13 +16,13 @@ from wrolpi.cmd import which from wrolpi.common import logger, extract_domain, get_media_directory from wrolpi.dates import now from wrolpi.db import get_db_session, get_db_curs -from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG +from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG, DownloadResult from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel from .common import apply_info_json, get_no_channel_directory, \ get_videos_directory -from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id, validate_video +from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id from .models import Video, Channel from .schema import ChannelPostRequest from .video_url_resolver import video_url_resolver @@ -41,16 +43,24 @@ PREFERRED_VIDEO_EXTENSION = 'mp4' PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p,18,bestvideo+bestaudio' +def extract_info(url: str, ydl: YoutubeDL = YDL) -> dict: + """Get info about a video. Separated for testing.""" + return ydl.extract_info(url, download=False, process=False) + + +def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str: + """Get filename from YoutubeDL. Separated for testing.""" + return ydl.prepare_filename(entry) + + class ChannelDownloader(Downloader, ABC): - """ - Handling downloading of all Videos in a Channel. - """ + """Handles downloading of all Videos in a Channel.""" name = 'video_channel' pretty_name = 'Video Channel' listable = False def __repr__(self): - return f'' + return f'' @classmethod def valid_url(cls, url) -> Tuple[bool, None]: @@ -60,10 +70,8 @@ class ChannelDownloader(Downloader, ABC): logger.debug(f'{cls.__name__} not suitable for {url}') return False, None - def do_download(self, download: Download): - """ - Update a Channel's catalog, then schedule downloads of every missing video. - """ + def do_download(self, download: Download) -> DownloadResult: + """Update a Channel's catalog, then schedule downloads of every missing video.""" with get_db_session() as session: channel = session.query(Channel).filter_by(url=download.url).one_or_none() @@ -71,6 +79,7 @@ class ChannelDownloader(Downloader, ABC): # Couldn't get channel by URL, this is probably a playlist. Find the channel for the playlist. channel_source_id = get_channel_source_id(download.url) channel = get_channel(source_id=channel_source_id, return_dict=False) + channel_id = channel.id update_channel(channel) @@ -91,7 +100,7 @@ class ChannelDownloader(Downloader, ABC): else: self.manager.start_downloads() - return True + return DownloadResult(success=True, location=f'/videos/channel/{channel_id}/video') YT_DLP_BIN = which( @@ -110,7 +119,7 @@ class VideoDownloader(Downloader, ABC): pretty_name = 'Videos' def __repr__(self): - return f'' + return f'' @classmethod def valid_url(cls, url) -> Tuple[bool, Optional[dict]]: @@ -120,7 +129,7 @@ class VideoDownloader(Downloader, ABC): for ie in YDL._ies.values(): if ie.suitable(url) and not ChannelDownloader.valid_url(url)[0]: try: - info = YDL.extract_info(url, download=False, process=False) + info = extract_info(url) return True, info except UnsupportedError: logger.debug(f'Video downloader extract_info failed for {url}') @@ -131,7 +140,7 @@ class VideoDownloader(Downloader, ABC): logger.debug(f'{cls.__name__} not suitable for {url}') return False, None - def do_download(self, download: Download): + def do_download(self, download: Download) -> DownloadResult: if download.attempts >= 10: raise UnrecoverableDownloadError('Max download attempts reached') @@ -160,6 +169,7 @@ class VideoDownloader(Downloader, ABC): out_dir = channel.directory.path out_dir.mkdir(exist_ok=True, parents=True) + logs = None # noqa try: video_path, entry = self.prepare_filename(url, out_dir) # Do the real download. @@ -177,15 +187,24 @@ class VideoDownloader(Downloader, ABC): '-o', file_name_format, url, ] - return_code = self.process_runner(url, cmd, out_dir) + return_code, logs = self.process_runner(url, cmd, out_dir) if return_code != 0: raise ValueError(f'video downloader process exited with {return_code}') if not video_path.is_file(): raise ValueError(f'Video file could not be found! {video_path}') + + with get_db_session(commit=True) as session: + # If the video is from a channel, it will already be in the database. + source_id = entry['id'] + existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none() + video = upsert_video(session, video_path, channel, id_=existing_video.id if existing_video else None) + video_id = video.id except UnrecoverableDownloadError: raise + except yt_dlp.utils.UnsupportedError as e: + raise UnrecoverableDownloadError('URL is not supported by yt-dlp') from e except Exception as e: logger.warning(f'VideoDownloader failed to download: {download.url}', exc_info=e) if _skip_download(e): @@ -204,18 +223,22 @@ class VideoDownloader(Downloader, ABC): # Skipped downloads should not be tried again. raise UnrecoverableDownloadError() from e - return + # Download did not succeed, try again later. + if logs and (stderr := logs.get('stderr')): + error = f'{stderr}\n\n{traceback.format_exc()}' + else: + error = str(traceback.format_exc()) + return DownloadResult(success=False, error=error) - with get_db_session(commit=True) as session: - # If the video is from a channel, it will already be in the database. - source_id = entry['id'] - existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none() - id_ = existing_video.id if existing_video else None - - upsert_video(session, video_path, channel, id_=id_) - - # Successful download. - return True + if channel: + location = f'/videos/channel/{channel.id}/video/{video_id}' + else: + location = f'/videos/video/{video_id}' + result = DownloadResult( + success=True, + location=location, + ) + return result @staticmethod def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]: @@ -234,9 +257,13 @@ class VideoDownloader(Downloader, ABC): ydl.params['logger'] = ydl_logger ydl.add_default_info_extractors() - # Perform the real download. This saves multiple files. - entry = ydl.extract_info(url, download=False, process=True) - final_filename = pathlib.Path(ydl.prepare_filename(entry)) + # Get the path where the video will be saved. + entry = extract_info(url, ydl=ydl) + final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute() + if final_filename.suffix == '.NA': + # yt-dlp has odd behavior sometimes and ignores our preferred video extension. + final_filename = final_filename.with_suffix('.mp4') + logger.warning(f'Had to replace video extension: {final_filename}') return final_filename, entry @@ -284,10 +311,10 @@ def update_channel(channel: Channel): It is expected that any missing videos will be downloaded later. """ logger.info(f'Downloading video list for {channel.name} at {channel.url} This may take several minutes.') - info = YDL.extract_info(channel.url, download=False, process=False) + info = extract_info(channel.url) if 'url' in info: url = info['url'] - info = YDL.extract_info(url, download=False, process=False) + info = extract_info(url) # Resolve all entries to dictionaries. entries = info['entries'] = list(info['entries']) @@ -299,7 +326,7 @@ def update_channel(channel: Channel): for entry in entries: if entry['title'] == 'Uploads': logger.info('Youtube-DL gave back a list of URLs, found the "Uploads" URL and using it.') - info = YDL.extract_info(entry['url'], download=False, process=False) + info = extract_info(entry['url']) entries = info['entries'] = list(info['entries']) break @@ -359,7 +386,6 @@ def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]: if channel_id: where = 'AND channel_id = %s' params = (channel_id,) - query = f''' SELECT video.id, video.source_id, video.channel_id @@ -380,7 +406,7 @@ def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]: return missing_videos -def find_all_missing_videos(channel_id: str = None) -> Tuple[dict, dict]: +def find_all_missing_videos(channel_id: int = None) -> Tuple[dict, dict]: """ Find all videos that don't have a video file, but are found in the DB (taken from the channel's info_json). diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 5c405d63..e49088e7 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -6,7 +6,6 @@ from typing import Tuple, Optional, List, Union from uuid import uuid1 from sqlalchemy import or_ -from sqlalchemy.exc import InvalidRequestError from sqlalchemy.orm import Session from yt_dlp import YoutubeDL @@ -18,7 +17,8 @@ from wrolpi.media_path import MediaPath from wrolpi.vars import PYTEST from .captions import get_captions from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \ - is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError + is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \ + get_no_channel_directory from .models import Channel, Video logger = logger.getChild(__name__) @@ -60,7 +60,8 @@ def refresh_channel_videos(channel: Channel): logger.debug(f'{channel.name}: Added {video_path.name}') with get_db_curs(commit=True) as curs: - curs.execute('DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL RETURNING id', (channel.id,)) + stmt = 'DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL AND video_path IS NOT NULL RETURNING id' + curs.execute(stmt, (channel.id,)) deleted_count = len(curs.fetchall()) if deleted_count: @@ -544,6 +545,9 @@ def import_channels_config(): channel.name = data['name'] channel.directory = directory + # A URL should not be an empty string + data['url'] = data['url'] or None + # Copy existing channel data, update all values from the config. This is necessary to clear out # values not in the config. full_data = channel.dict() @@ -571,7 +575,6 @@ def import_channels_config(): continue channel_dir = channel.directory.path else: - from .downloader import get_no_channel_directory channel_dir = get_no_channel_directory() # Set favorite Videos of this Channel. @@ -712,7 +715,11 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = raise ValueError(f'Video path is not absolute: {video_path}') # This function can update or insert a Video. - video = session.query(Video).filter_by(id=id_).one() if id_ else Video() + if id_: + video = session.query(Video).filter_by(id=id_).one() + else: + video = Video() + session.add(video) # Set the file values, all other things can be found using these files. poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path) @@ -728,6 +735,9 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = video.channel_id = channel.id video.idempotency = idempotency + session.flush() + session.refresh(video) + try: # Fill in any missing data. Generate poster if enabled and necessary. validate_video(video, channel.generate_posters if channel else False) @@ -738,11 +748,6 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = # inserted. logger.warning(f'Failed to validate {video}', exc_info=e) - try: - session.add(video) - except InvalidRequestError: - # Video is already in a session. - pass session.flush() return video diff --git a/modules/videos/models.py b/modules/videos/models.py index 0b8af617..31b93f47 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -338,7 +338,7 @@ class Channel(ModelHelper, Base): name=self.name, skip_download_videos=self.skip_download_videos or [], source_id=self.source_id, - url=self.url or '', + url=self.url or None, ) return config diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 1905cf7e..059c5246 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -347,6 +347,7 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c assert download.frequency == channel1.download_frequency assert download.downloader == 'video_channel' assert download.next_download + assert not download.error next_download = str(download.next_download) import_channels_config() diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 19a50957..de3eb3cf 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -1,6 +1,7 @@ import pathlib import shutil from copy import copy +from itertools import zip_longest from unittest import mock import pytest @@ -13,22 +14,11 @@ from modules.videos.models import Channel, Video from wrolpi.db import get_db_context from wrolpi.downloader import DownloadManager, Download from wrolpi.errors import InvalidDownload -from wrolpi.test.common import wrap_test_db, TestAPI +from wrolpi.test.common import TestAPI from wrolpi.test.test_downloader import HTTPDownloader from wrolpi.vars import PROJECT_DIR -class FakeYDL: - - @staticmethod - def extract_info(*a, **kw): - return { - 'entries': [ - {'id': 1}, - ], - } - - def test_find_all_missing_videos(test_session, channel_factory, video_factory): channel1, channel2 = channel_factory(url='some url'), channel_factory(url='some other url') channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]} @@ -155,68 +145,52 @@ class TestVideosDownloaders(TestAPI): self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance')[0], self.cd) self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/watch?v=31jPEBiAC3c')[0], self.vd) - @staticmethod - def _make_video_files(channel_dir): - video_path = channel_dir / 'video.mp4' - poster_path = channel_dir / 'video.jpg' - description_path = channel_dir / 'video.description' - info_json_path = channel_dir / 'video.info.json' - video_path.touch() - poster_path.touch() - description_path.touch() - info_json_path.touch() - return video_path, poster_path, description_path, info_json_path - @wrap_test_db - def test_video_download_no_channel(self): - _, session = get_db_context() +def test_video_download_no_channel(test_session, video_download_manager, video_factory, test_directory, + mock_video_extract_info, mock_video_process_runner): + """A video can be downloaded even if it does not have a Channel.""" + channel_dir = test_directory / 'NO CHANNEL' + channel_dir.mkdir(parents=True) + video_path = channel_dir / 'video.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path) - channel_dir = self.videos_dir / 'NO CHANNEL' - channel_dir.mkdir(parents=True) + # Video has no channel + info_json = copy(example_video_json) + del info_json['channel'] + del info_json['channel_id'] + del info_json['channel_url'] - # Video paths - video_path, _, _, _ = self._make_video_files(channel_dir) + url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' + mock_video_extract_info.return_value = info_json + with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename: + mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) + video_download_manager.create_download(url, test_session) - # Video has no channel - info_json = copy(example_video_json) - del info_json['channel'] - del info_json['channel_id'] - del info_json['channel_url'] + mock_video_process_runner.assert_called_once() - url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ - mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ - mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: - mock_extract_info.return_value = info_json - mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) - mock_process_runner.return_value = 0 - - self.mgr.create_download(url, session) - - mock_process_runner.assert_called_once() - - video: Video = session.query(Video).one() - self.assertEqual(str(video.video_path.path), f'{channel_dir}/video.mp4') + video: Video = test_session.query(Video).one() + assert str(video.video_path.path) == f'{channel_dir}/video.mp4' -def test_channel_download(test_session, simple_channel, video_download_manager, video_file): +def test_download_channel(test_session, simple_channel, video_download_manager, video_file, + mock_video_extract_info, mock_video_prepare_filename, mock_video_process_runner): + """Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos.""" url = 'https://www.youtube.com/c/LearningSelfReliance/videos' - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ - mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ - mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: - mock_extract_info.return_value = example_channel_json - mock_prepare_filename.return_value = (video_file, {'id': 'foo'}) - mock_process_runner.return_value = 0 + mock_video_prepare_filename.return_value = video_file + mock_video_extract_info.return_value = example_channel_json + with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel: + mock_get_channel.return_value = simple_channel video_download_manager.create_download(url, test_session) # Two videos are in the example channel. - downloads = video_download_manager.get_new_downloads(test_session) - expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o'] - for download, expected in zip(downloads, expected): + downloads = video_download_manager.get_once_downloads(test_session) + downloads = filter(lambda i: 'watch' in i.url, downloads) + expected = ['https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url'] + for download, expected in zip_longest(downloads, expected): assert download.url == expected - # Download is skipped for test. - assert download.status == 'new' + # Download is run only for test. + assert download.status == 'complete' def test_get_or_create_channel(test_session): @@ -259,23 +233,25 @@ def test_get_or_create_channel(test_session): assert get_or_create_channel(source_id='quux') == channel -def test_download_info_save(test_session, video_download_manager): +def test_download_info_save(test_session, video_download_manager, mock_video_extract_info, mock_video_prepare_filename, + mock_video_process_runner): """ An info dict from Downloader.valid_url is saved to the Downloader and can be used by the Downloader later. """ - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: - mock_extract_info.return_value = {'some': 'info'} - video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session) - mock_extract_info.assert_called_once() + mock_video_extract_info.return_value = {'some': 'info'} + mock_video_prepare_filename.return_value = 'some path' + video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session) + assert mock_video_extract_info.call_count == 2 # Only called twice. The "info" gathering call was skipped. -def test_selected_downloader(test_session, video_download_manager): +def test_selected_downloader(test_session, video_download_manager, successful_download): """ A user can specify which Downloader to use, no other should ever be used (even if the Download fails). """ # The test HTTP Downloader will claim to be able to handle any HTTP URL; will pretend to have downloaded anything. http_downloader = HTTPDownloader() mock_do_download = http_downloader.do_download = mock.MagicMock() + mock_do_download.return_value = successful_download mock_http_valid_url = http_downloader.valid_url = mock.MagicMock() mock_http_valid_url.return_value = (True, None) video_download_manager.register_downloader(http_downloader) @@ -286,7 +262,7 @@ def test_selected_downloader(test_session, video_download_manager): # DownloadManager will attempt to choose the Downloader. with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: mock_extract_info.side_effect = UnsupportedError('not this one') - video_download_manager.create_downloads(['https://example.com'], test_session) + video_download_manager.create_download('https://example.com', test_session) mock_http_valid_url.assert_called_once() mock_extract_info.assert_called_once() mock_do_download.assert_called_once() @@ -298,7 +274,7 @@ def test_selected_downloader(test_session, video_download_manager): # DownloadManager will only use the HTTPDownloader. with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: - video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http') + video_download_manager.create_download('https://example.com', test_session, downloader='http') # DownloadManager does not try to find the Download, it trusts the "downloader" param. mock_extract_info.assert_not_called() mock_http_valid_url.assert_not_called() @@ -313,7 +289,7 @@ def test_selected_downloader(test_session, video_download_manager): # DownloadManager will only use the HTTPDownloader, even if the download fails. with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: mock_do_download.side_effect = Exception('oh no!') - video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http') + video_download_manager.create_download('https://example.com', test_session, downloader='http') # DownloadManager does not try to find the Download, it trusts the "downloader" param. mock_extract_info.assert_not_called() mock_http_valid_url.assert_not_called() @@ -351,22 +327,22 @@ def test_channel_download_no_download(test_session, video_download_manager, simp download_channel(simple_channel.id) -def test_invalid_download_url(test_session, test_download_manager, test_downloader_config): +def test_invalid_download_url(test_session, test_download_manager, mock_video_extract_info): """An invalid url should not be attempted again.""" _, session = get_db_context() with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \ - mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader, \ - mock.patch('modules.videos.downloader.YDL') as mock_ydl: - mock_get_downloader.return_value = (VideoDownloader(), {'channel': 'foo', 'id': 'some id'}) - mock_valid_url.return_value = (True, None) - mock_ydl.extract_info.side_effect = UnsupportedError('oops') - test_download_manager.create_download('url', session) + mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader: + mock_get_downloader.return_value = (VideoDownloader(), None) + mock_valid_url.return_value = (True, {}) # url is valid, but is not a video URL. + mock_video_extract_info.side_effect = UnsupportedError('oops') + test_download_manager.create_download('invalid url', session) - download = test_download_manager.get_downloads(test_session)[0] + download = session.query(Download).one() assert download.status == 'failed' -def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager): +def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager, + mock_video_process_runner): simple_channel.source_id = example_video_json['channel_id'] simple_channel.directory = test_directory / 'videos/channel name' simple_channel.directory.mkdir(parents=True) @@ -376,16 +352,14 @@ def test_video_download_1(test_session, test_directory, simple_channel, video_do url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ - mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ - mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: + mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename: mock_extract_info.return_value = example_video_json mock_prepare_filename.return_value = (video_file, {'id': 'foo'}) - mock_process_runner.return_value = 0 video_download_manager.create_download(url, test_session) - mock_process_runner.assert_called_once() - video_url, _, out_dir = mock_process_runner.call_args[0] + mock_video_process_runner.assert_called_once() + video_url, _, out_dir = mock_video_process_runner.call_args[0] download: Download = test_session.query(Download).one() assert video_url == download.url @@ -393,3 +367,21 @@ def test_video_download_1(test_session, test_directory, simple_channel, video_do video: Video = test_session.query(Video).one() assert video.video_path.path.is_absolute() + + +def test_download_result(test_session, test_directory, video_download_manager, mock_video_process_runner, + mock_video_extract_info): + """VideoDownloader returns a DownloadResult when complete.""" + channel_directory = test_directory / 'videos/channel name' + channel_directory.mkdir(parents=True) + video_file = channel_directory / 'the video.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file) + + mock_video_extract_info.return_value = example_video_json + with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename: + mock_prepare_filename.return_value = [video_file, {'id': 'foo'}] + video_download_manager.create_download('https://example.com', test_session) + + download: Download = test_session.query(Download).one() + assert download.url == 'https://example.com' + assert download.location == '/videos/channel/1/video/1' diff --git a/wrolpi/common.py b/wrolpi/common.py index 8c56482a..b5d529bb 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -555,9 +555,7 @@ def get_relative_to_media_directory(path: Union[str, Path]) -> Path: def minimize_dict(d: dict, keys: Iterable) -> Optional[dict]: - """ - Return a new dictionary that contains only the keys provided. - """ + """Return a new dictionary that contains only the keys provided.""" if d is None: return return {k: d[k] for k in set(keys) & d.keys()} diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index d7552df3..ab442b07 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -15,7 +15,7 @@ from sqlalchemy.orm import Session, sessionmaker from wrolpi.common import set_test_media_directory, Base, set_test_config from wrolpi.dates import set_test_now from wrolpi.db import postgres_engine, get_db_args -from wrolpi.downloader import DownloadManager +from wrolpi.downloader import DownloadManager, DownloadResult from wrolpi.root_api import BLUEPRINTS, api_app @@ -124,3 +124,13 @@ def fake_now(): finally: # reset now() to its original functionality. set_test_now(None) # noqa + + +@pytest.fixture +def successful_download(): + return DownloadResult(success=True) + + +@pytest.fixture +def failed_download(): + return DownloadResult(error='pytest.fixture failed_download error', success=False) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index ae1b38af..37039462 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -2,7 +2,8 @@ import asyncio import multiprocessing import pathlib import subprocess -from collections import defaultdict +import traceback +from dataclasses import dataclass, field from datetime import timedelta, datetime from enum import Enum from functools import partial @@ -13,7 +14,7 @@ from sqlalchemy import Column, Integer, String, Text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Session -from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag, ConfigFile +from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError @@ -34,6 +35,15 @@ class DownloadFrequency(int, Enum): days90 = 7776000 +@dataclass +class DownloadResult: + downloads: List[str] = field(default_factory=list) + error: str = None + info_json: dict = field(default_factory=dict) + location: str = None + success: bool = False + + class Download(ModelHelper, Base): """ Model that is used to schedule downloads. @@ -49,6 +59,8 @@ class Download(ModelHelper, Base): status = Column(String, default='new') info_json = Column(JSONB) downloader = Column(Text) + location = Column(Text) + error = Column(Text) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -67,6 +79,7 @@ class Download(ModelHelper, Base): frequency=self.frequency, id=self.id, last_successful_download=self.last_successful_download, + location=self.location, next_download=self.next_download, status=self.status, url=self.url, @@ -74,39 +87,30 @@ class Download(ModelHelper, Base): return d def renew(self, reset_attempts: bool = False): - """ - Mark this Download as "new" so it will be retried. - """ + """Mark this Download as "new" so it will be retried.""" self.status = 'new' if reset_attempts: self.attempts = 0 def defer(self): - """ - Download should be tried again after a time. - """ + """Download should be tried again after a time.""" self.status = 'deferred' def fail(self): - """ - Download should not be attempted again. A recurring Download will raise an error. - """ + """Download should not be attempted again. A recurring Download will raise an error.""" if self.frequency: raise ValueError('Recurring download should not be failed.') self.status = 'failed' def started(self): - """ - Mark this Download as in progress. - """ + """Mark this Download as in progress.""" self.attempts += 1 self.status = 'pending' def complete(self): - """ - Mark this Download as successfully downloaded. - """ + """Mark this Download as successfully downloaded.""" self.status = 'complete' + self.error = None # clear any old errors self.last_successful_download = now() def get_downloader(self): @@ -147,7 +151,7 @@ class Downloader: def valid_url(cls, url) -> Tuple[bool, Optional[dict]]: raise NotImplementedError() - def do_download(self, download: Download): + def do_download(self, download: Download) -> DownloadResult: raise NotImplementedError() @property @@ -174,7 +178,7 @@ class Downloader: if self._kill.is_set(): self._kill.clear() - def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> int: + def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> Tuple[int, dict]: """ Run a subprocess using the provided arguments. This process can be killed by the Download Manager. """ @@ -202,13 +206,10 @@ class Downloader: # Output all logs from the process. # TODO is there a way to stream this output while the process is running? logger.debug(f'Download exited with {proc.returncode}') - outs, errs = proc.communicate() - for line in outs.decode().splitlines(): # noqa - logger.debug(line) - for line in errs.decode().splitlines(): # noqa - logger.error(line) + stdout, stderr = proc.communicate() + logs = {'stdout': stdout, 'stderr': stderr} - return proc.returncode + return proc.returncode, logs class DownloadManager: @@ -233,15 +234,14 @@ class DownloadManager: def get_downloader(self, url: str) -> Tuple[Downloader, Dict]: for i in self.instances: - result = i.valid_url(url) - valid, info = result + valid, info = i.valid_url(url) if valid: return i, info raise InvalidDownload(f'Invalid URL {url=}') def get_downloader_by_name(self, name: str) -> Downloader: """ - Attempts to find a registered Downloader by its name. Returns None if it cannot be found. + Tries to find a registered Downloader by its name. Returns None if it cannot be found. """ return self._instances.get(name) @@ -263,9 +263,7 @@ class DownloadManager: yield download def get_or_create_download(self, url: str, session: Session) -> Download: - """ - Get a Download by its URL, if it cannot be found create one. - """ + """Get a Download by its URL, if it cannot be found create one.""" if not url: raise ValueError('Download must have a URL') @@ -360,29 +358,34 @@ class DownloadManager: download.started() session.commit() - success = True - failure = False + try_again = True try: - success = downloader.do_download(download) + result = downloader.do_download(download) except UnrecoverableDownloadError as e: # Download failed and should not be retried. logger.warning(f'UnrecoverableDownloadError for {url}', exc_info=e) - failure = True + result = DownloadResult(success=False, error=str(traceback.format_exc())) + try_again = False except Exception as e: logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e) + result = DownloadResult(success=False, error=str(traceback.format_exc())) # Long-running, get the download again. with get_db_session(commit=True) as session_: download = self.get_download(session_, id_=download_id) - if failure: + # Use a new location if provided, keep the old location if no new location is provided, otherwise + # clear out an outdated location. + download.location = result.location or download.location or None + # Clear any old errors if the download succeeded. + download.error = result.error if download.error else None + download.next_download = self.get_next_download(download, session) + + if try_again is False: download.fail() - elif success: + elif result.success: download.complete() - download.next_download = self.get_next_download(download, session) else: download.defer() - download.next_download = self.get_next_download(download, session) - session_.commit() if download_count: logger.info(f'Done doing {download_count} downloads.') @@ -409,9 +412,7 @@ class DownloadManager: @staticmethod def reset_downloads(): - """ - Set any incomplete Downloads to `new` so they will be retried. - """ + """Set any incomplete Downloads to `new` so they will be retried.""" with get_db_curs(commit=True) as curs: curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'") @@ -430,20 +431,6 @@ class DownloadManager: DOWNLOAD_SORT = ('pending', 'failed', 'new', 'deferred', 'complete') - @classmethod - @iterify(list) - def _downloads_sorter(cls, downloads: List[Download]): - """ - Downloads should be displayed to the user by status. - """ - grouped_by_statuses = defaultdict(lambda: []) - for download in downloads: - grouped_by_statuses[download.status].append(download) - - # Yield the downloads in the order defined above. - for sort in cls.DOWNLOAD_SORT: - yield from grouped_by_statuses[sort] # converted to a list by iterify() - @optional_session def get_recurring_downloads(self, session: Session = None, limit: int = None): """Get all Downloads that will be downloaded in the future.""" @@ -456,7 +443,6 @@ class DownloadManager: if limit: query = query.limit(limit) downloads = query.all() - downloads = self._downloads_sorter(downloads) return downloads @optional_session @@ -471,14 +457,11 @@ class DownloadManager: if limit: query = query.limit(limit) downloads = query.all() - downloads = self._downloads_sorter(downloads) return downloads @optional_session def renew_recurring_downloads(self, session: Session = None): - """ - Mark any recurring downloads that are due for download as "new". Start a download. - """ + """Mark any recurring downloads that are due for download as "new". Start a download.""" now_ = now() recurring = self.get_recurring_downloads(session) @@ -523,9 +506,7 @@ class DownloadManager: return False def kill_download(self, download_id: int): - """ - Fail a Download. If it is pending, kill the Downloader so the download stops. - """ + """Fail a Download. If it is pending, kill the Downloader so the download stops.""" with get_db_session(commit=True) as session: download = self.get_download(session, id_=download_id) downloader = download.get_downloader() @@ -535,9 +516,7 @@ class DownloadManager: download.fail() def kill(self): - """ - Kill all downloads. - """ + """Kill all downloads. Do not start new downloads.""" self.disabled.set() try: with get_db_session(commit=True) as session: @@ -610,9 +589,11 @@ class DownloadManager: stmt = f''' SELECT downloader, + error, frequency, id, last_successful_download, + location, next_download, status, url diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 048b6f61..c25ccc93 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -9,7 +9,7 @@ import pytest from wrolpi import downloader from wrolpi.dates import local_timezone, Seconds from wrolpi.db import get_db_context -from wrolpi.downloader import Downloader, Download, DownloadFrequency +from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled from wrolpi.test.common import wrap_test_db, TestAPI @@ -72,7 +72,7 @@ class TestDownloader(TestAPI): self.assertEqual(len(self.mgr.get_downloads(session)), 0) http_downloader = HTTPDownloader() - http_downloader.do_download.return_value = False + http_downloader.do_download.return_value = DownloadResult(success=True) self.mgr.register_downloader(http_downloader) self.mgr.create_download('https://example.com', session) @@ -88,11 +88,11 @@ class TestDownloader(TestAPI): _, session = get_db_context() http_downloader = HTTPDownloader() - http_downloader.do_download.return_value = True + http_downloader.do_download.return_value = DownloadResult(success=True) self.mgr.register_downloader(http_downloader) permissive_downloader = PermissiveDownloader(priority=100) - permissive_downloader.do_download.return_value = False # returns a failure + permissive_downloader.do_download.return_value = DownloadResult(success=False) self.mgr.register_downloader(permissive_downloader) # https is handled by the HTTP Downloader. @@ -117,7 +117,7 @@ class TestDownloader(TestAPI): self.assertEqual(download.attempts, 2) # finally success - permissive_downloader.do_download.return_value = True + permissive_downloader.do_download.return_value = DownloadResult(success=True) self.mgr.create_download('foo', session) self.mgr._do_downloads(session) download = self.mgr.get_download(session, 'foo') @@ -126,32 +126,6 @@ class TestDownloader(TestAPI): # No downloads left. self.assertEqual(list(self.mgr.get_new_downloads(session)), []) - @wrap_test_db - def test_max_attempts(self): - """ - A Download will only be attempted so many times, it will eventually be deleted. - """ - _, session = get_db_context() - - http_downloader = HTTPDownloader() - http_downloader.do_download.return_value = False - self.mgr.register_downloader(http_downloader) - - self.mgr.create_download('https://example.com', session) - download = session.query(Download).one() - self.assertEqual(download.attempts, 1) - - self.mgr.create_download('https://example.com', session) - download = session.query(Download).one() - self.assertEqual(download.attempts, 2) - - # There are no further attempts. - http_downloader.do_download.side_effect = UnrecoverableDownloadError() - self.mgr.create_download('https://example.com', session) - download = session.query(Download).one() - self.assertEqual(download.attempts, 3) - self.assertEqual(download.status, 'failed') - @wrap_test_db @mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0))) def test_delete_old_once_downloads(self): @@ -186,12 +160,12 @@ class TestDownloader(TestAPI): self.mgr.delete_old_once_downloads() # Two old downloads are deleted. - downloads = self.mgr._downloads_sorter(session.query(Download).all()) + downloads = session.query(Download).order_by(Download.id).all() expected = [ - dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'), - dict(url='https://example.com/6', attempts=1, status='pending'), dict(url='https://example.com/1', frequency=1, attempts=0, status='new'), + dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'), dict(url='https://example.com/5', attempts=0, status='new'), + dict(url='https://example.com/6', attempts=1, status='pending'), ] for download, expected in zip_longest(downloads, expected): self.assertDictContains(download.dict(), expected) @@ -298,7 +272,7 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now): http_downloader = HTTPDownloader() http_downloader.do_download = MagicMock() - http_downloader.do_download.return_value = True + http_downloader.do_download.return_value = DownloadResult(success=True) test_download_manager.register_downloader(http_downloader) # Download every hour. @@ -337,7 +311,7 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now): # Try the download, but it fails. http_downloader.do_download.reset_mock() - http_downloader.do_download.return_value = False + http_downloader.do_download.return_value = DownloadResult(success=False) test_download_manager.do_downloads_sync() http_downloader.do_download.assert_called_once() download = session.query(Download).one() @@ -345,13 +319,13 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now): assert download.status == 'deferred' assert download.last_successful_download == now # Download should be retried after the DEFAULT_RETRY_FREQUENCY. - expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1)) + expected = local_timezone(datetime(2020, 1, 1, 3, 0, 0, 997200)) assert download.next_download == expected # Try the download again, it finally succeeds. http_downloader.do_download.reset_mock() now = fake_now(datetime(2020, 1, 1, 4, 0, 1)) - http_downloader.do_download.return_value = True + http_downloader.do_download.return_value = DownloadResult(success=True) test_download_manager.renew_recurring_downloads(session) test_download_manager.do_downloads_sync() http_downloader.do_download.assert_called_once() @@ -360,3 +334,28 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now): assert download.last_successful_download == now # Floats cause slightly wrong date. assert download.next_download == local_timezone(datetime(2020, 1, 1, 5, 0, 0, 997200)) + + +def test_max_attempts(test_session, test_download_manager): + """A Download will only be attempted so many times, it will eventually be deleted.""" + _, session = get_db_context() + + http_downloader = HTTPDownloader() + http_downloader.do_download = MagicMock() + http_downloader.do_download.return_value = DownloadResult(success=True) + test_download_manager.register_downloader(http_downloader) + + test_download_manager.create_download('https://example.com', session) + download = session.query(Download).one() + assert download.attempts == 1 + + test_download_manager.create_download('https://example.com', session) + download = session.query(Download).one() + assert download.attempts == 2 + + # There are no further attempts. + http_downloader.do_download.side_effect = UnrecoverableDownloadError() + test_download_manager.create_download('https://example.com', session) + download = session.query(Download).one() + assert download.attempts == 3 + assert download.status == 'failed' diff --git a/wrolpi/version.py b/wrolpi/version.py index d39599e6..860f433a 100644 --- a/wrolpi/version.py +++ b/wrolpi/version.py @@ -1,4 +1,4 @@ -__version__ = '0.4.7-beta' +__version__ = '0.4.8-beta' import subprocess