From 785ad8274cd34911af14e441385a260aa9323c8d Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 3 May 2022 10:46:10 -0600 Subject: [PATCH] DownloadResult.try_again is not necessary. Removing unused fixture in test. Simplifying logic after download completes. Removing unused download sorting method. ChannelDownloader returns location. Hiding View link when no location is provided. Performing video upsert in try/except so any errors are attached to the Download. Including stderr only. Including logs when returning Downloader.process_runner. Warning when extention is replaced. Providing location link on Downloads page. Showing error modal on errors that fail. Handling empty URL strings on import. Fixing channel download test. Reworking how Channels schedule Video downloads. Bumping version for DownloadResult. Updating tests to use DownloadResult. VideoDownloader returns a DownloadResult. --- .../1a70196c05bb_download_location.py | 33 ++++ app/src/components/Admin.js | 54 +++++- modules/archive/__init__.py | 8 +- modules/videos/channel/lib.py | 4 +- modules/videos/channel/test/test_api.py | 20 +-- modules/videos/channel/test/test_lib.py | 32 ++++ modules/videos/conftest.py | 22 ++- modules/videos/downloader.py | 94 ++++++---- modules/videos/lib.py | 25 +-- modules/videos/models.py | 2 +- modules/videos/test/test_common.py | 1 + modules/videos/test/test_downloader.py | 160 +++++++++--------- wrolpi/common.py | 4 +- wrolpi/conftest.py | 12 +- wrolpi/downloader.py | 117 ++++++------- wrolpi/test/test_downloader.py | 75 ++++---- wrolpi/version.py | 2 +- 17 files changed, 393 insertions(+), 272 deletions(-) create mode 100644 alembic/versions/1a70196c05bb_download_location.py diff --git a/alembic/versions/1a70196c05bb_download_location.py b/alembic/versions/1a70196c05bb_download_location.py new file mode 100644 index 00000000..3172a9fb --- /dev/null +++ b/alembic/versions/1a70196c05bb_download_location.py @@ -0,0 +1,33 @@ +"""Download location + +Revision ID: 1a70196c05bb +Revises: d8fcdb7773c0 +Create Date: 2022-05-02 12:01:51.210070 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '1a70196c05bb' +down_revision = 'd8fcdb7773c0' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS location TEXT') + session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS error TEXT') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE download DROP COLUMN IF EXISTS location') + session.execute('ALTER TABLE download DROP COLUMN IF EXISTS error') diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 87543de9..f028672e 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -331,7 +331,7 @@ function ClearFailedDownloads({callback}) { class DownloadRow extends React.Component { render() { - let {url, frequency, last_successful_download, status, next_download} = this.props; + let {url, frequency, last_successful_download, status, location, next_download} = this.props; let positive = false; if (status === 'pending') { positive = true; @@ -342,7 +342,7 @@ class DownloadRow extends React.Component { {secondsToFrequency(frequency)} {last_successful_download ? secondsToDate(last_successful_download) : null} {secondsToDate(next_download)} - {status} + {location && View} ); } @@ -354,6 +354,7 @@ class StoppableRow extends React.Component { this.state = { stopOpen: false, startOpen: false, + errorModalOpen: false, }; } @@ -389,9 +390,10 @@ class StoppableRow extends React.Component { }; render() { - let {url, last_successful_download, status} = this.props; - let {stopOpen, startOpen} = this.state; + let {url, last_successful_download, status, location, error} = this.props; + let {stopOpen, startOpen, errorModalOpen} = this.state; + let completedAtCell = last_successful_download ? secondsToDate(last_successful_download) : null; let buttonCell = ; let positive = false; let negative = false; @@ -418,10 +420,21 @@ class StoppableRow extends React.Component { warning = status === 'deferred'; buttonCell = ( + + + >Retry ); + } else if (status === 'complete' && location) { + buttonCell = ( + + View + + ); + } + if (error) { + completedAtCell = ( + this.setState({errorModalOpen: false})} + onOpen={() => this.setState({errorModalOpen: true})} + open={errorModalOpen} + trigger={ + + + ) } return ( {textEllipsis(url, 50)} - {last_successful_download ? secondsToDate(last_successful_download) : null} {status} + {completedAtCell} {buttonCell} ); @@ -498,8 +536,8 @@ class Downloads extends React.Component { URL - Completed At Status + Completed At Control @@ -512,7 +550,7 @@ class Downloads extends React.Component { Download Frequency Last Successful Download Next Download - Status + View ); diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 8d33572b..981561a6 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -2,7 +2,7 @@ from abc import ABC from typing import Tuple from wrolpi.common import logger -from wrolpi.downloader import Downloader, Download +from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError from . import lib from .api import bp # noqa @@ -17,7 +17,7 @@ class ArchiveDownloader(Downloader, ABC): pretty_name = 'Archive' def __repr__(self): - return f'' + return f'' @classmethod def valid_url(cls, url: str) -> Tuple[bool, None]: @@ -26,12 +26,12 @@ class ArchiveDownloader(Downloader, ABC): """ return True, None - def do_download(self, download: Download): + def do_download(self, download: Download) -> DownloadResult: if download.attempts > 3: raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}') lib.do_archive(download.url) - return True + return DownloadResult(success=True) # Archive downloader is the last downloader which should be used. diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index 3b6f19bb..fca6017e 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -152,7 +152,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic # Verify that the URL/Name/directory aren't taken check_for_channel_conflicts( session, - url=data.url, + url=data.url or None, name=data.name, directory=data.directory, source_id=data.source_id, @@ -162,7 +162,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic channel = Channel( name=data.name, - url=data.url, + url=data.url or None, match_regex=data.match_regex, directory=data.directory, # noqa download_frequency=data.download_frequency, diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index f2fe566b..1c77df11 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -24,7 +24,8 @@ def test_get_channels(test_directory, channel_factory, test_client): assert len(response.json['channels']) == 3 -def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel): +@mock.patch('modules.videos.lib.validate_video') +def test_refresh_no_channel(_, test_session, test_client, test_directory, simple_channel): # A regular user-defined channel vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4' vid1.touch() @@ -41,22 +42,7 @@ def test_refresh_no_channel(test_session, test_client, test_directory, simple_ch vid4.touch() test_client.post('/api/videos/refresh') - - videos = test_session.query(Video).order_by(Video.title).all() - assert len(videos) == 4 - - # Add some fake videos, they should be removed. - test_session.add(Video(title='vid5')) - test_session.add(Video(title='vid6', channel_id=simple_channel.id)) - test_session.commit() - - videos = test_session.query(Video).order_by(Video.title).all() - assert len(videos) == 6 - - test_client.post('/api/videos/refresh') - - videos = test_session.query(Video).order_by(Video.title).all() - assert len(videos) == 4 + assert test_session.query(Video).count() == 4 def test_get_video(test_client, test_session, simple_channel, video_factory): diff --git a/modules/videos/channel/test/test_lib.py b/modules/videos/channel/test/test_lib.py index 1e8bc510..f1af902a 100644 --- a/modules/videos/channel/test/test_lib.py +++ b/modules/videos/channel/test/test_lib.py @@ -1,6 +1,9 @@ import pytest +from modules.videos import schema from modules.videos.channel import lib +from modules.videos.lib import save_channels_config, import_channels_config +from modules.videos.models import Channel from wrolpi.errors import UnknownChannel @@ -57,3 +60,32 @@ def test_get_channel(test_session, test_directory, channel_factory): for p in params: assert lib.get_channel(**p, return_dict=False) == channel_id_channel assert lib.get_channel(**p) + + +def test_channels_no_url(test_session, test_directory): + """Test that a Channel's URL is coerced to None if it is empty.""" + channel1_directory = test_directory / 'channel1' + channel1_directory.mkdir() + channel2_directory = test_directory / 'channel2' + channel2_directory.mkdir() + + channel1 = schema.ChannelPostRequest( + name='foo', + directory=str(channel1_directory), + ) + channel1.url = '' + lib.create_channel(test_session, data=channel1) + + channel2 = schema.ChannelPostRequest( + name='bar', + directory=str(channel2_directory), + url='' + ) + lib.create_channel(test_session, data=channel2) + + channels = list(test_session.query(Channel)) + assert len(channels) == 2 + assert all(i.url is None for i in channels), [(i.name, i.url) for i in channels] + + save_channels_config() + import_channels_config() diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 915f7329..42a2348a 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -100,7 +100,8 @@ def video_factory(test_session, test_directory): if channel_id: path = test_directory / f'{title}.mp4' else: - path = test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4' + (test_directory / 'videos/NO CHANNEL').mkdir(parents=True, exist_ok=True) + path = test_directory / f'videos/NO CHANNEL/{title}.mp4' if with_video_file: shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path) @@ -166,3 +167,22 @@ def test_downloader_config(test_directory): set_test_downloader_config(True) yield config_path set_test_downloader_config(False) + + +@pytest.fixture +def mock_video_extract_info(): + with mock.patch('modules.videos.downloader.extract_info') as mock_extract_info: + yield mock_extract_info + + +@pytest.fixture +def mock_video_prepare_filename(): + with mock.patch('modules.videos.downloader.prepare_filename') as mock_prepare_filename: + yield mock_prepare_filename + + +@pytest.fixture +def mock_video_process_runner(): + with mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: + mock_process_runner.return_value = (0, {}) + yield mock_process_runner diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index e974e7ee..44e18560 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -2,9 +2,11 @@ import json import pathlib import re +import traceback from abc import ABC from typing import Tuple, List, Optional +import yt_dlp.utils from sqlalchemy.orm import Session from yt_dlp import YoutubeDL from yt_dlp.extractor import YoutubeTabIE @@ -14,13 +16,13 @@ from wrolpi.cmd import which from wrolpi.common import logger, extract_domain, get_media_directory from wrolpi.dates import now from wrolpi.db import get_db_session, get_db_curs -from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG +from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG, DownloadResult from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel from .common import apply_info_json, get_no_channel_directory, \ get_videos_directory -from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id, validate_video +from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id from .models import Video, Channel from .schema import ChannelPostRequest from .video_url_resolver import video_url_resolver @@ -41,16 +43,24 @@ PREFERRED_VIDEO_EXTENSION = 'mp4' PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p,18,bestvideo+bestaudio' +def extract_info(url: str, ydl: YoutubeDL = YDL) -> dict: + """Get info about a video. Separated for testing.""" + return ydl.extract_info(url, download=False, process=False) + + +def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str: + """Get filename from YoutubeDL. Separated for testing.""" + return ydl.prepare_filename(entry) + + class ChannelDownloader(Downloader, ABC): - """ - Handling downloading of all Videos in a Channel. - """ + """Handles downloading of all Videos in a Channel.""" name = 'video_channel' pretty_name = 'Video Channel' listable = False def __repr__(self): - return f'' + return f'' @classmethod def valid_url(cls, url) -> Tuple[bool, None]: @@ -60,10 +70,8 @@ class ChannelDownloader(Downloader, ABC): logger.debug(f'{cls.__name__} not suitable for {url}') return False, None - def do_download(self, download: Download): - """ - Update a Channel's catalog, then schedule downloads of every missing video. - """ + def do_download(self, download: Download) -> DownloadResult: + """Update a Channel's catalog, then schedule downloads of every missing video.""" with get_db_session() as session: channel = session.query(Channel).filter_by(url=download.url).one_or_none() @@ -71,6 +79,7 @@ class ChannelDownloader(Downloader, ABC): # Couldn't get channel by URL, this is probably a playlist. Find the channel for the playlist. channel_source_id = get_channel_source_id(download.url) channel = get_channel(source_id=channel_source_id, return_dict=False) + channel_id = channel.id update_channel(channel) @@ -91,7 +100,7 @@ class ChannelDownloader(Downloader, ABC): else: self.manager.start_downloads() - return True + return DownloadResult(success=True, location=f'/videos/channel/{channel_id}/video') YT_DLP_BIN = which( @@ -110,7 +119,7 @@ class VideoDownloader(Downloader, ABC): pretty_name = 'Videos' def __repr__(self): - return f'' + return f'' @classmethod def valid_url(cls, url) -> Tuple[bool, Optional[dict]]: @@ -120,7 +129,7 @@ class VideoDownloader(Downloader, ABC): for ie in YDL._ies.values(): if ie.suitable(url) and not ChannelDownloader.valid_url(url)[0]: try: - info = YDL.extract_info(url, download=False, process=False) + info = extract_info(url) return True, info except UnsupportedError: logger.debug(f'Video downloader extract_info failed for {url}') @@ -131,7 +140,7 @@ class VideoDownloader(Downloader, ABC): logger.debug(f'{cls.__name__} not suitable for {url}') return False, None - def do_download(self, download: Download): + def do_download(self, download: Download) -> DownloadResult: if download.attempts >= 10: raise UnrecoverableDownloadError('Max download attempts reached') @@ -160,6 +169,7 @@ class VideoDownloader(Downloader, ABC): out_dir = channel.directory.path out_dir.mkdir(exist_ok=True, parents=True) + logs = None # noqa try: video_path, entry = self.prepare_filename(url, out_dir) # Do the real download. @@ -177,15 +187,24 @@ class VideoDownloader(Downloader, ABC): '-o', file_name_format, url, ] - return_code = self.process_runner(url, cmd, out_dir) + return_code, logs = self.process_runner(url, cmd, out_dir) if return_code != 0: raise ValueError(f'video downloader process exited with {return_code}') if not video_path.is_file(): raise ValueError(f'Video file could not be found! {video_path}') + + with get_db_session(commit=True) as session: + # If the video is from a channel, it will already be in the database. + source_id = entry['id'] + existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none() + video = upsert_video(session, video_path, channel, id_=existing_video.id if existing_video else None) + video_id = video.id except UnrecoverableDownloadError: raise + except yt_dlp.utils.UnsupportedError as e: + raise UnrecoverableDownloadError('URL is not supported by yt-dlp') from e except Exception as e: logger.warning(f'VideoDownloader failed to download: {download.url}', exc_info=e) if _skip_download(e): @@ -204,18 +223,22 @@ class VideoDownloader(Downloader, ABC): # Skipped downloads should not be tried again. raise UnrecoverableDownloadError() from e - return + # Download did not succeed, try again later. + if logs and (stderr := logs.get('stderr')): + error = f'{stderr}\n\n{traceback.format_exc()}' + else: + error = str(traceback.format_exc()) + return DownloadResult(success=False, error=error) - with get_db_session(commit=True) as session: - # If the video is from a channel, it will already be in the database. - source_id = entry['id'] - existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none() - id_ = existing_video.id if existing_video else None - - upsert_video(session, video_path, channel, id_=id_) - - # Successful download. - return True + if channel: + location = f'/videos/channel/{channel.id}/video/{video_id}' + else: + location = f'/videos/video/{video_id}' + result = DownloadResult( + success=True, + location=location, + ) + return result @staticmethod def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]: @@ -234,9 +257,13 @@ class VideoDownloader(Downloader, ABC): ydl.params['logger'] = ydl_logger ydl.add_default_info_extractors() - # Perform the real download. This saves multiple files. - entry = ydl.extract_info(url, download=False, process=True) - final_filename = pathlib.Path(ydl.prepare_filename(entry)) + # Get the path where the video will be saved. + entry = extract_info(url, ydl=ydl) + final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute() + if final_filename.suffix == '.NA': + # yt-dlp has odd behavior sometimes and ignores our preferred video extension. + final_filename = final_filename.with_suffix('.mp4') + logger.warning(f'Had to replace video extension: {final_filename}') return final_filename, entry @@ -284,10 +311,10 @@ def update_channel(channel: Channel): It is expected that any missing videos will be downloaded later. """ logger.info(f'Downloading video list for {channel.name} at {channel.url} This may take several minutes.') - info = YDL.extract_info(channel.url, download=False, process=False) + info = extract_info(channel.url) if 'url' in info: url = info['url'] - info = YDL.extract_info(url, download=False, process=False) + info = extract_info(url) # Resolve all entries to dictionaries. entries = info['entries'] = list(info['entries']) @@ -299,7 +326,7 @@ def update_channel(channel: Channel): for entry in entries: if entry['title'] == 'Uploads': logger.info('Youtube-DL gave back a list of URLs, found the "Uploads" URL and using it.') - info = YDL.extract_info(entry['url'], download=False, process=False) + info = extract_info(entry['url']) entries = info['entries'] = list(info['entries']) break @@ -359,7 +386,6 @@ def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]: if channel_id: where = 'AND channel_id = %s' params = (channel_id,) - query = f''' SELECT video.id, video.source_id, video.channel_id @@ -380,7 +406,7 @@ def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]: return missing_videos -def find_all_missing_videos(channel_id: str = None) -> Tuple[dict, dict]: +def find_all_missing_videos(channel_id: int = None) -> Tuple[dict, dict]: """ Find all videos that don't have a video file, but are found in the DB (taken from the channel's info_json). diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 5c405d63..e49088e7 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -6,7 +6,6 @@ from typing import Tuple, Optional, List, Union from uuid import uuid1 from sqlalchemy import or_ -from sqlalchemy.exc import InvalidRequestError from sqlalchemy.orm import Session from yt_dlp import YoutubeDL @@ -18,7 +17,8 @@ from wrolpi.media_path import MediaPath from wrolpi.vars import PYTEST from .captions import get_captions from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \ - is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError + is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \ + get_no_channel_directory from .models import Channel, Video logger = logger.getChild(__name__) @@ -60,7 +60,8 @@ def refresh_channel_videos(channel: Channel): logger.debug(f'{channel.name}: Added {video_path.name}') with get_db_curs(commit=True) as curs: - curs.execute('DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL RETURNING id', (channel.id,)) + stmt = 'DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL AND video_path IS NOT NULL RETURNING id' + curs.execute(stmt, (channel.id,)) deleted_count = len(curs.fetchall()) if deleted_count: @@ -544,6 +545,9 @@ def import_channels_config(): channel.name = data['name'] channel.directory = directory + # A URL should not be an empty string + data['url'] = data['url'] or None + # Copy existing channel data, update all values from the config. This is necessary to clear out # values not in the config. full_data = channel.dict() @@ -571,7 +575,6 @@ def import_channels_config(): continue channel_dir = channel.directory.path else: - from .downloader import get_no_channel_directory channel_dir = get_no_channel_directory() # Set favorite Videos of this Channel. @@ -712,7 +715,11 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = raise ValueError(f'Video path is not absolute: {video_path}') # This function can update or insert a Video. - video = session.query(Video).filter_by(id=id_).one() if id_ else Video() + if id_: + video = session.query(Video).filter_by(id=id_).one() + else: + video = Video() + session.add(video) # Set the file values, all other things can be found using these files. poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path) @@ -728,6 +735,9 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = video.channel_id = channel.id video.idempotency = idempotency + session.flush() + session.refresh(video) + try: # Fill in any missing data. Generate poster if enabled and necessary. validate_video(video, channel.generate_posters if channel else False) @@ -738,11 +748,6 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = # inserted. logger.warning(f'Failed to validate {video}', exc_info=e) - try: - session.add(video) - except InvalidRequestError: - # Video is already in a session. - pass session.flush() return video diff --git a/modules/videos/models.py b/modules/videos/models.py index 0b8af617..31b93f47 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -338,7 +338,7 @@ class Channel(ModelHelper, Base): name=self.name, skip_download_videos=self.skip_download_videos or [], source_id=self.source_id, - url=self.url or '', + url=self.url or None, ) return config diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 1905cf7e..059c5246 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -347,6 +347,7 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c assert download.frequency == channel1.download_frequency assert download.downloader == 'video_channel' assert download.next_download + assert not download.error next_download = str(download.next_download) import_channels_config() diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 19a50957..de3eb3cf 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -1,6 +1,7 @@ import pathlib import shutil from copy import copy +from itertools import zip_longest from unittest import mock import pytest @@ -13,22 +14,11 @@ from modules.videos.models import Channel, Video from wrolpi.db import get_db_context from wrolpi.downloader import DownloadManager, Download from wrolpi.errors import InvalidDownload -from wrolpi.test.common import wrap_test_db, TestAPI +from wrolpi.test.common import TestAPI from wrolpi.test.test_downloader import HTTPDownloader from wrolpi.vars import PROJECT_DIR -class FakeYDL: - - @staticmethod - def extract_info(*a, **kw): - return { - 'entries': [ - {'id': 1}, - ], - } - - def test_find_all_missing_videos(test_session, channel_factory, video_factory): channel1, channel2 = channel_factory(url='some url'), channel_factory(url='some other url') channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]} @@ -155,68 +145,52 @@ class TestVideosDownloaders(TestAPI): self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance')[0], self.cd) self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/watch?v=31jPEBiAC3c')[0], self.vd) - @staticmethod - def _make_video_files(channel_dir): - video_path = channel_dir / 'video.mp4' - poster_path = channel_dir / 'video.jpg' - description_path = channel_dir / 'video.description' - info_json_path = channel_dir / 'video.info.json' - video_path.touch() - poster_path.touch() - description_path.touch() - info_json_path.touch() - return video_path, poster_path, description_path, info_json_path - @wrap_test_db - def test_video_download_no_channel(self): - _, session = get_db_context() +def test_video_download_no_channel(test_session, video_download_manager, video_factory, test_directory, + mock_video_extract_info, mock_video_process_runner): + """A video can be downloaded even if it does not have a Channel.""" + channel_dir = test_directory / 'NO CHANNEL' + channel_dir.mkdir(parents=True) + video_path = channel_dir / 'video.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path) - channel_dir = self.videos_dir / 'NO CHANNEL' - channel_dir.mkdir(parents=True) + # Video has no channel + info_json = copy(example_video_json) + del info_json['channel'] + del info_json['channel_id'] + del info_json['channel_url'] - # Video paths - video_path, _, _, _ = self._make_video_files(channel_dir) + url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' + mock_video_extract_info.return_value = info_json + with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename: + mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) + video_download_manager.create_download(url, test_session) - # Video has no channel - info_json = copy(example_video_json) - del info_json['channel'] - del info_json['channel_id'] - del info_json['channel_url'] + mock_video_process_runner.assert_called_once() - url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ - mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ - mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: - mock_extract_info.return_value = info_json - mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) - mock_process_runner.return_value = 0 - - self.mgr.create_download(url, session) - - mock_process_runner.assert_called_once() - - video: Video = session.query(Video).one() - self.assertEqual(str(video.video_path.path), f'{channel_dir}/video.mp4') + video: Video = test_session.query(Video).one() + assert str(video.video_path.path) == f'{channel_dir}/video.mp4' -def test_channel_download(test_session, simple_channel, video_download_manager, video_file): +def test_download_channel(test_session, simple_channel, video_download_manager, video_file, + mock_video_extract_info, mock_video_prepare_filename, mock_video_process_runner): + """Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos.""" url = 'https://www.youtube.com/c/LearningSelfReliance/videos' - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ - mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ - mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: - mock_extract_info.return_value = example_channel_json - mock_prepare_filename.return_value = (video_file, {'id': 'foo'}) - mock_process_runner.return_value = 0 + mock_video_prepare_filename.return_value = video_file + mock_video_extract_info.return_value = example_channel_json + with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel: + mock_get_channel.return_value = simple_channel video_download_manager.create_download(url, test_session) # Two videos are in the example channel. - downloads = video_download_manager.get_new_downloads(test_session) - expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o'] - for download, expected in zip(downloads, expected): + downloads = video_download_manager.get_once_downloads(test_session) + downloads = filter(lambda i: 'watch' in i.url, downloads) + expected = ['https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url'] + for download, expected in zip_longest(downloads, expected): assert download.url == expected - # Download is skipped for test. - assert download.status == 'new' + # Download is run only for test. + assert download.status == 'complete' def test_get_or_create_channel(test_session): @@ -259,23 +233,25 @@ def test_get_or_create_channel(test_session): assert get_or_create_channel(source_id='quux') == channel -def test_download_info_save(test_session, video_download_manager): +def test_download_info_save(test_session, video_download_manager, mock_video_extract_info, mock_video_prepare_filename, + mock_video_process_runner): """ An info dict from Downloader.valid_url is saved to the Downloader and can be used by the Downloader later. """ - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: - mock_extract_info.return_value = {'some': 'info'} - video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session) - mock_extract_info.assert_called_once() + mock_video_extract_info.return_value = {'some': 'info'} + mock_video_prepare_filename.return_value = 'some path' + video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session) + assert mock_video_extract_info.call_count == 2 # Only called twice. The "info" gathering call was skipped. -def test_selected_downloader(test_session, video_download_manager): +def test_selected_downloader(test_session, video_download_manager, successful_download): """ A user can specify which Downloader to use, no other should ever be used (even if the Download fails). """ # The test HTTP Downloader will claim to be able to handle any HTTP URL; will pretend to have downloaded anything. http_downloader = HTTPDownloader() mock_do_download = http_downloader.do_download = mock.MagicMock() + mock_do_download.return_value = successful_download mock_http_valid_url = http_downloader.valid_url = mock.MagicMock() mock_http_valid_url.return_value = (True, None) video_download_manager.register_downloader(http_downloader) @@ -286,7 +262,7 @@ def test_selected_downloader(test_session, video_download_manager): # DownloadManager will attempt to choose the Downloader. with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: mock_extract_info.side_effect = UnsupportedError('not this one') - video_download_manager.create_downloads(['https://example.com'], test_session) + video_download_manager.create_download('https://example.com', test_session) mock_http_valid_url.assert_called_once() mock_extract_info.assert_called_once() mock_do_download.assert_called_once() @@ -298,7 +274,7 @@ def test_selected_downloader(test_session, video_download_manager): # DownloadManager will only use the HTTPDownloader. with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: - video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http') + video_download_manager.create_download('https://example.com', test_session, downloader='http') # DownloadManager does not try to find the Download, it trusts the "downloader" param. mock_extract_info.assert_not_called() mock_http_valid_url.assert_not_called() @@ -313,7 +289,7 @@ def test_selected_downloader(test_session, video_download_manager): # DownloadManager will only use the HTTPDownloader, even if the download fails. with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: mock_do_download.side_effect = Exception('oh no!') - video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http') + video_download_manager.create_download('https://example.com', test_session, downloader='http') # DownloadManager does not try to find the Download, it trusts the "downloader" param. mock_extract_info.assert_not_called() mock_http_valid_url.assert_not_called() @@ -351,22 +327,22 @@ def test_channel_download_no_download(test_session, video_download_manager, simp download_channel(simple_channel.id) -def test_invalid_download_url(test_session, test_download_manager, test_downloader_config): +def test_invalid_download_url(test_session, test_download_manager, mock_video_extract_info): """An invalid url should not be attempted again.""" _, session = get_db_context() with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \ - mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader, \ - mock.patch('modules.videos.downloader.YDL') as mock_ydl: - mock_get_downloader.return_value = (VideoDownloader(), {'channel': 'foo', 'id': 'some id'}) - mock_valid_url.return_value = (True, None) - mock_ydl.extract_info.side_effect = UnsupportedError('oops') - test_download_manager.create_download('url', session) + mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader: + mock_get_downloader.return_value = (VideoDownloader(), None) + mock_valid_url.return_value = (True, {}) # url is valid, but is not a video URL. + mock_video_extract_info.side_effect = UnsupportedError('oops') + test_download_manager.create_download('invalid url', session) - download = test_download_manager.get_downloads(test_session)[0] + download = session.query(Download).one() assert download.status == 'failed' -def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager): +def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager, + mock_video_process_runner): simple_channel.source_id = example_video_json['channel_id'] simple_channel.directory = test_directory / 'videos/channel name' simple_channel.directory.mkdir(parents=True) @@ -376,16 +352,14 @@ def test_video_download_1(test_session, test_directory, simple_channel, video_do url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ - mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ - mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: + mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename: mock_extract_info.return_value = example_video_json mock_prepare_filename.return_value = (video_file, {'id': 'foo'}) - mock_process_runner.return_value = 0 video_download_manager.create_download(url, test_session) - mock_process_runner.assert_called_once() - video_url, _, out_dir = mock_process_runner.call_args[0] + mock_video_process_runner.assert_called_once() + video_url, _, out_dir = mock_video_process_runner.call_args[0] download: Download = test_session.query(Download).one() assert video_url == download.url @@ -393,3 +367,21 @@ def test_video_download_1(test_session, test_directory, simple_channel, video_do video: Video = test_session.query(Video).one() assert video.video_path.path.is_absolute() + + +def test_download_result(test_session, test_directory, video_download_manager, mock_video_process_runner, + mock_video_extract_info): + """VideoDownloader returns a DownloadResult when complete.""" + channel_directory = test_directory / 'videos/channel name' + channel_directory.mkdir(parents=True) + video_file = channel_directory / 'the video.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file) + + mock_video_extract_info.return_value = example_video_json + with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename: + mock_prepare_filename.return_value = [video_file, {'id': 'foo'}] + video_download_manager.create_download('https://example.com', test_session) + + download: Download = test_session.query(Download).one() + assert download.url == 'https://example.com' + assert download.location == '/videos/channel/1/video/1' diff --git a/wrolpi/common.py b/wrolpi/common.py index 8c56482a..b5d529bb 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -555,9 +555,7 @@ def get_relative_to_media_directory(path: Union[str, Path]) -> Path: def minimize_dict(d: dict, keys: Iterable) -> Optional[dict]: - """ - Return a new dictionary that contains only the keys provided. - """ + """Return a new dictionary that contains only the keys provided.""" if d is None: return return {k: d[k] for k in set(keys) & d.keys()} diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index d7552df3..ab442b07 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -15,7 +15,7 @@ from sqlalchemy.orm import Session, sessionmaker from wrolpi.common import set_test_media_directory, Base, set_test_config from wrolpi.dates import set_test_now from wrolpi.db import postgres_engine, get_db_args -from wrolpi.downloader import DownloadManager +from wrolpi.downloader import DownloadManager, DownloadResult from wrolpi.root_api import BLUEPRINTS, api_app @@ -124,3 +124,13 @@ def fake_now(): finally: # reset now() to its original functionality. set_test_now(None) # noqa + + +@pytest.fixture +def successful_download(): + return DownloadResult(success=True) + + +@pytest.fixture +def failed_download(): + return DownloadResult(error='pytest.fixture failed_download error', success=False) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index ae1b38af..37039462 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -2,7 +2,8 @@ import asyncio import multiprocessing import pathlib import subprocess -from collections import defaultdict +import traceback +from dataclasses import dataclass, field from datetime import timedelta, datetime from enum import Enum from functools import partial @@ -13,7 +14,7 @@ from sqlalchemy import Column, Integer, String, Text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Session -from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag, ConfigFile +from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError @@ -34,6 +35,15 @@ class DownloadFrequency(int, Enum): days90 = 7776000 +@dataclass +class DownloadResult: + downloads: List[str] = field(default_factory=list) + error: str = None + info_json: dict = field(default_factory=dict) + location: str = None + success: bool = False + + class Download(ModelHelper, Base): """ Model that is used to schedule downloads. @@ -49,6 +59,8 @@ class Download(ModelHelper, Base): status = Column(String, default='new') info_json = Column(JSONB) downloader = Column(Text) + location = Column(Text) + error = Column(Text) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -67,6 +79,7 @@ class Download(ModelHelper, Base): frequency=self.frequency, id=self.id, last_successful_download=self.last_successful_download, + location=self.location, next_download=self.next_download, status=self.status, url=self.url, @@ -74,39 +87,30 @@ class Download(ModelHelper, Base): return d def renew(self, reset_attempts: bool = False): - """ - Mark this Download as "new" so it will be retried. - """ + """Mark this Download as "new" so it will be retried.""" self.status = 'new' if reset_attempts: self.attempts = 0 def defer(self): - """ - Download should be tried again after a time. - """ + """Download should be tried again after a time.""" self.status = 'deferred' def fail(self): - """ - Download should not be attempted again. A recurring Download will raise an error. - """ + """Download should not be attempted again. A recurring Download will raise an error.""" if self.frequency: raise ValueError('Recurring download should not be failed.') self.status = 'failed' def started(self): - """ - Mark this Download as in progress. - """ + """Mark this Download as in progress.""" self.attempts += 1 self.status = 'pending' def complete(self): - """ - Mark this Download as successfully downloaded. - """ + """Mark this Download as successfully downloaded.""" self.status = 'complete' + self.error = None # clear any old errors self.last_successful_download = now() def get_downloader(self): @@ -147,7 +151,7 @@ class Downloader: def valid_url(cls, url) -> Tuple[bool, Optional[dict]]: raise NotImplementedError() - def do_download(self, download: Download): + def do_download(self, download: Download) -> DownloadResult: raise NotImplementedError() @property @@ -174,7 +178,7 @@ class Downloader: if self._kill.is_set(): self._kill.clear() - def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> int: + def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> Tuple[int, dict]: """ Run a subprocess using the provided arguments. This process can be killed by the Download Manager. """ @@ -202,13 +206,10 @@ class Downloader: # Output all logs from the process. # TODO is there a way to stream this output while the process is running? logger.debug(f'Download exited with {proc.returncode}') - outs, errs = proc.communicate() - for line in outs.decode().splitlines(): # noqa - logger.debug(line) - for line in errs.decode().splitlines(): # noqa - logger.error(line) + stdout, stderr = proc.communicate() + logs = {'stdout': stdout, 'stderr': stderr} - return proc.returncode + return proc.returncode, logs class DownloadManager: @@ -233,15 +234,14 @@ class DownloadManager: def get_downloader(self, url: str) -> Tuple[Downloader, Dict]: for i in self.instances: - result = i.valid_url(url) - valid, info = result + valid, info = i.valid_url(url) if valid: return i, info raise InvalidDownload(f'Invalid URL {url=}') def get_downloader_by_name(self, name: str) -> Downloader: """ - Attempts to find a registered Downloader by its name. Returns None if it cannot be found. + Tries to find a registered Downloader by its name. Returns None if it cannot be found. """ return self._instances.get(name) @@ -263,9 +263,7 @@ class DownloadManager: yield download def get_or_create_download(self, url: str, session: Session) -> Download: - """ - Get a Download by its URL, if it cannot be found create one. - """ + """Get a Download by its URL, if it cannot be found create one.""" if not url: raise ValueError('Download must have a URL') @@ -360,29 +358,34 @@ class DownloadManager: download.started() session.commit() - success = True - failure = False + try_again = True try: - success = downloader.do_download(download) + result = downloader.do_download(download) except UnrecoverableDownloadError as e: # Download failed and should not be retried. logger.warning(f'UnrecoverableDownloadError for {url}', exc_info=e) - failure = True + result = DownloadResult(success=False, error=str(traceback.format_exc())) + try_again = False except Exception as e: logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e) + result = DownloadResult(success=False, error=str(traceback.format_exc())) # Long-running, get the download again. with get_db_session(commit=True) as session_: download = self.get_download(session_, id_=download_id) - if failure: + # Use a new location if provided, keep the old location if no new location is provided, otherwise + # clear out an outdated location. + download.location = result.location or download.location or None + # Clear any old errors if the download succeeded. + download.error = result.error if download.error else None + download.next_download = self.get_next_download(download, session) + + if try_again is False: download.fail() - elif success: + elif result.success: download.complete() - download.next_download = self.get_next_download(download, session) else: download.defer() - download.next_download = self.get_next_download(download, session) - session_.commit() if download_count: logger.info(f'Done doing {download_count} downloads.') @@ -409,9 +412,7 @@ class DownloadManager: @staticmethod def reset_downloads(): - """ - Set any incomplete Downloads to `new` so they will be retried. - """ + """Set any incomplete Downloads to `new` so they will be retried.""" with get_db_curs(commit=True) as curs: curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'") @@ -430,20 +431,6 @@ class DownloadManager: DOWNLOAD_SORT = ('pending', 'failed', 'new', 'deferred', 'complete') - @classmethod - @iterify(list) - def _downloads_sorter(cls, downloads: List[Download]): - """ - Downloads should be displayed to the user by status. - """ - grouped_by_statuses = defaultdict(lambda: []) - for download in downloads: - grouped_by_statuses[download.status].append(download) - - # Yield the downloads in the order defined above. - for sort in cls.DOWNLOAD_SORT: - yield from grouped_by_statuses[sort] # converted to a list by iterify() - @optional_session def get_recurring_downloads(self, session: Session = None, limit: int = None): """Get all Downloads that will be downloaded in the future.""" @@ -456,7 +443,6 @@ class DownloadManager: if limit: query = query.limit(limit) downloads = query.all() - downloads = self._downloads_sorter(downloads) return downloads @optional_session @@ -471,14 +457,11 @@ class DownloadManager: if limit: query = query.limit(limit) downloads = query.all() - downloads = self._downloads_sorter(downloads) return downloads @optional_session def renew_recurring_downloads(self, session: Session = None): - """ - Mark any recurring downloads that are due for download as "new". Start a download. - """ + """Mark any recurring downloads that are due for download as "new". Start a download.""" now_ = now() recurring = self.get_recurring_downloads(session) @@ -523,9 +506,7 @@ class DownloadManager: return False def kill_download(self, download_id: int): - """ - Fail a Download. If it is pending, kill the Downloader so the download stops. - """ + """Fail a Download. If it is pending, kill the Downloader so the download stops.""" with get_db_session(commit=True) as session: download = self.get_download(session, id_=download_id) downloader = download.get_downloader() @@ -535,9 +516,7 @@ class DownloadManager: download.fail() def kill(self): - """ - Kill all downloads. - """ + """Kill all downloads. Do not start new downloads.""" self.disabled.set() try: with get_db_session(commit=True) as session: @@ -610,9 +589,11 @@ class DownloadManager: stmt = f''' SELECT downloader, + error, frequency, id, last_successful_download, + location, next_download, status, url diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 048b6f61..c25ccc93 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -9,7 +9,7 @@ import pytest from wrolpi import downloader from wrolpi.dates import local_timezone, Seconds from wrolpi.db import get_db_context -from wrolpi.downloader import Downloader, Download, DownloadFrequency +from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled from wrolpi.test.common import wrap_test_db, TestAPI @@ -72,7 +72,7 @@ class TestDownloader(TestAPI): self.assertEqual(len(self.mgr.get_downloads(session)), 0) http_downloader = HTTPDownloader() - http_downloader.do_download.return_value = False + http_downloader.do_download.return_value = DownloadResult(success=True) self.mgr.register_downloader(http_downloader) self.mgr.create_download('https://example.com', session) @@ -88,11 +88,11 @@ class TestDownloader(TestAPI): _, session = get_db_context() http_downloader = HTTPDownloader() - http_downloader.do_download.return_value = True + http_downloader.do_download.return_value = DownloadResult(success=True) self.mgr.register_downloader(http_downloader) permissive_downloader = PermissiveDownloader(priority=100) - permissive_downloader.do_download.return_value = False # returns a failure + permissive_downloader.do_download.return_value = DownloadResult(success=False) self.mgr.register_downloader(permissive_downloader) # https is handled by the HTTP Downloader. @@ -117,7 +117,7 @@ class TestDownloader(TestAPI): self.assertEqual(download.attempts, 2) # finally success - permissive_downloader.do_download.return_value = True + permissive_downloader.do_download.return_value = DownloadResult(success=True) self.mgr.create_download('foo', session) self.mgr._do_downloads(session) download = self.mgr.get_download(session, 'foo') @@ -126,32 +126,6 @@ class TestDownloader(TestAPI): # No downloads left. self.assertEqual(list(self.mgr.get_new_downloads(session)), []) - @wrap_test_db - def test_max_attempts(self): - """ - A Download will only be attempted so many times, it will eventually be deleted. - """ - _, session = get_db_context() - - http_downloader = HTTPDownloader() - http_downloader.do_download.return_value = False - self.mgr.register_downloader(http_downloader) - - self.mgr.create_download('https://example.com', session) - download = session.query(Download).one() - self.assertEqual(download.attempts, 1) - - self.mgr.create_download('https://example.com', session) - download = session.query(Download).one() - self.assertEqual(download.attempts, 2) - - # There are no further attempts. - http_downloader.do_download.side_effect = UnrecoverableDownloadError() - self.mgr.create_download('https://example.com', session) - download = session.query(Download).one() - self.assertEqual(download.attempts, 3) - self.assertEqual(download.status, 'failed') - @wrap_test_db @mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0))) def test_delete_old_once_downloads(self): @@ -186,12 +160,12 @@ class TestDownloader(TestAPI): self.mgr.delete_old_once_downloads() # Two old downloads are deleted. - downloads = self.mgr._downloads_sorter(session.query(Download).all()) + downloads = session.query(Download).order_by(Download.id).all() expected = [ - dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'), - dict(url='https://example.com/6', attempts=1, status='pending'), dict(url='https://example.com/1', frequency=1, attempts=0, status='new'), + dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'), dict(url='https://example.com/5', attempts=0, status='new'), + dict(url='https://example.com/6', attempts=1, status='pending'), ] for download, expected in zip_longest(downloads, expected): self.assertDictContains(download.dict(), expected) @@ -298,7 +272,7 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now): http_downloader = HTTPDownloader() http_downloader.do_download = MagicMock() - http_downloader.do_download.return_value = True + http_downloader.do_download.return_value = DownloadResult(success=True) test_download_manager.register_downloader(http_downloader) # Download every hour. @@ -337,7 +311,7 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now): # Try the download, but it fails. http_downloader.do_download.reset_mock() - http_downloader.do_download.return_value = False + http_downloader.do_download.return_value = DownloadResult(success=False) test_download_manager.do_downloads_sync() http_downloader.do_download.assert_called_once() download = session.query(Download).one() @@ -345,13 +319,13 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now): assert download.status == 'deferred' assert download.last_successful_download == now # Download should be retried after the DEFAULT_RETRY_FREQUENCY. - expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1)) + expected = local_timezone(datetime(2020, 1, 1, 3, 0, 0, 997200)) assert download.next_download == expected # Try the download again, it finally succeeds. http_downloader.do_download.reset_mock() now = fake_now(datetime(2020, 1, 1, 4, 0, 1)) - http_downloader.do_download.return_value = True + http_downloader.do_download.return_value = DownloadResult(success=True) test_download_manager.renew_recurring_downloads(session) test_download_manager.do_downloads_sync() http_downloader.do_download.assert_called_once() @@ -360,3 +334,28 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now): assert download.last_successful_download == now # Floats cause slightly wrong date. assert download.next_download == local_timezone(datetime(2020, 1, 1, 5, 0, 0, 997200)) + + +def test_max_attempts(test_session, test_download_manager): + """A Download will only be attempted so many times, it will eventually be deleted.""" + _, session = get_db_context() + + http_downloader = HTTPDownloader() + http_downloader.do_download = MagicMock() + http_downloader.do_download.return_value = DownloadResult(success=True) + test_download_manager.register_downloader(http_downloader) + + test_download_manager.create_download('https://example.com', session) + download = session.query(Download).one() + assert download.attempts == 1 + + test_download_manager.create_download('https://example.com', session) + download = session.query(Download).one() + assert download.attempts == 2 + + # There are no further attempts. + http_downloader.do_download.side_effect = UnrecoverableDownloadError() + test_download_manager.create_download('https://example.com', session) + download = session.query(Download).one() + assert download.attempts == 3 + assert download.status == 'failed' diff --git a/wrolpi/version.py b/wrolpi/version.py index d39599e6..860f433a 100644 --- a/wrolpi/version.py +++ b/wrolpi/version.py @@ -1,4 +1,4 @@ -__version__ = '0.4.7-beta' +__version__ = '0.4.8-beta' import subprocess