From 1e1c8f0b85df2e9165107446e6801f3f6f632d23 Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 8 Nov 2022 22:41:50 -0700 Subject: [PATCH] ChannelDownloader will add itself to Download.Settings, so VideoDownloader can download to the channel if yt-dlp cannot find the channel. --- modules/videos/common.py | 2 +- modules/videos/downloader.py | 66 +++++++++++++++++++++----- modules/videos/test/test_downloader.py | 18 ++++--- wrolpi/common.py | 5 +- wrolpi/downloader.py | 33 +++++++------ wrolpi/root_api.py | 5 +- wrolpi/test/test_downloader.py | 37 ++++++++++++++- wrolpi/test/test_root_api.py | 37 +-------------- 8 files changed, 126 insertions(+), 77 deletions(-) diff --git a/modules/videos/common.py b/modules/videos/common.py index 5c669248..bc006381 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -308,7 +308,7 @@ def apply_info_json(channel_id: int): Mark any videos not in the info_json as "censored". """ with get_db_session() as session: - channel = session.query(Channel).filter_by(id=channel_id).one() + channel: Channel = session.query(Channel).filter_by(id=channel_id).one() channel_name = channel.name info = channel.info_json diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 57379775..b7eb08ce 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -45,6 +45,8 @@ PREFERRED_VIDEO_FORMAT = ','.join([ 'res:720', # Use the best 720p available first '136+140', # 130=720p video-only, 140= high quality audio only '22', # 720p video with audio + 'mp4-480p', + 'mp4-360p', '18', # 360p video with audio 'bestvideo+bestaudio', # Download the highest resolution as a last resort (1080p is quite large). ]) @@ -79,7 +81,9 @@ class ChannelDownloader(Downloader, ABC): @staticmethod def is_a_playlist(info: dict): - # A playlist will have an id different from its channel. + if '_type' in info: + return info['_type'] == 'playlist' + # A playlist may have an id different from its channel. return info['id'] != info['channel_id'] async def do_download(self, download: Download) -> DownloadResult: @@ -92,12 +96,15 @@ class ChannelDownloader(Downloader, ABC): # May not have a session during testing. session.commit() - kind = info['uploader'] - channel_source_id = info['channel_id'] - channel = get_or_create_channel(channel_source_id, download.url, kind) + name = info.get('uploader') or info.get('webpage_url_basename') + if not name: + raise ValueError(f'Could not find name') + channel_source_id = info.get('channel_id') or info.get('id') + channel = get_or_create_channel(channel_source_id, download.url, name) channel.dict() # get all attributes while we have the session. location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None + settings = dict(channel_id=channel.id, channel_url=download.url) if channel else None is_a_playlist = self.is_a_playlist(info) try: @@ -105,13 +112,23 @@ class ChannelDownloader(Downloader, ABC): await self.prepare_channel_for_downloads(download, channel) downloads = self.get_missing_videos(download, channel) - return DownloadResult(success=True, location=location, downloads=downloads) + return DownloadResult( + success=True, + location=location, + downloads=downloads, + settings=settings, + ) except Exception: if PYTEST: raise kind = 'playlist' if is_a_playlist else 'channel' logger.warning(f'Failed to update catalog of {kind} {download.url}') - return DownloadResult(success=False, location=location, error=str(traceback.format_exc())) + return DownloadResult( + success=False, + location=location, + error=str(traceback.format_exc()), + settings=settings, + ) @classmethod async def prepare_channel_for_downloads(cls, download: Download, channel: Channel): @@ -209,12 +226,34 @@ class VideoDownloader(Downloader, ABC): download.info_json = info session.commit() + found_channel = None + channel_name = info.get('channel') - channel_id = info.get('channel_id') + source_channel_id = info.get('channel_id') + channel_url = info.get('channel_url') channel = None - if channel_name or channel_id: - channel_url = info.get('channel_url') - channel = get_or_create_channel(source_id=channel_id, url=channel_url, name=channel_name) + if channel_name or source_channel_id: + # Try to find the channel via info_json from yt-dlp. + channel = get_or_create_channel(source_id=source_channel_id, url=channel_url, name=channel_name) + if channel: + found_channel = 'yt_dlp' + + settings = download.settings or dict() + local_channel_id = settings.get('channel_id') + channel_url = settings.get('channel_url') + if not channel and (local_channel_id or channel_url): + # Could not find channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download. + logger.info(f'Using download.settings to find channel') + channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False) + if channel: + found_channel = 'download_settings' + + if found_channel == 'yt_dlp': + logger.debug('Found channel using yt_dlp') + elif found_channel == 'download_settings': + logger.info('Found channel using Download.settings') + else: + logger.info('Could not find channel') # Use the default directory if this video has no channel. out_dir = get_no_channel_directory() @@ -321,8 +360,7 @@ class VideoDownloader(Downloader, ABC): options = get_downloader_config().dict() options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}' options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION - - logger.debug(f'Downloading {url} to {out_dir}') + options['format'] = PREFERRED_VIDEO_FORMAT # Create a new YoutubeDL for the output directory. ydl = YoutubeDL(options) @@ -334,6 +372,8 @@ class VideoDownloader(Downloader, ABC): final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute() if final_filename.suffix.lower() != f'.{PREFERRED_VIDEO_EXTENSION}': raise DownloadError(f'Cannot download video {url} because yt-dlp filename is invalid.') + + logger.debug(f'Downloading {url} to {out_dir}') return final_filename, entry @@ -410,7 +450,7 @@ def update_channel_catalog(channel: Channel, info: dict): with get_db_session(commit=True) as session: # Get the channel in this new context. - channel = session.query(Channel).filter_by(id=channel.id).one() + channel: Channel = session.query(Channel).filter_by(id=channel.id).one() channel.info_json = info channel.info_date = now() diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 9dd98fe8..346ce6ca 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -153,8 +153,8 @@ class TestVideosDownloaders(TestAPI): @pytest.mark.asyncio -async def test_video_download_no_channel(test_session, video_download_manager, video_factory, test_directory, - mock_video_extract_info, mock_video_process_runner): +async def test_download_no_channel(test_session, video_download_manager, video_factory, test_directory, + mock_video_extract_info, mock_video_process_runner): """A video can be downloaded even if it does not have a Channel.""" channel_dir = test_directory / 'NO CHANNEL' channel_dir.mkdir(parents=True) @@ -216,12 +216,16 @@ async def test_download_channel(test_session, simple_channel, video_download_man await video_download_manager.wait_for_all_downloads() downloads = video_download_manager.get_once_downloads(test_session) assert [i.url for i in downloads] == ['https://youtube.com/watch?v=video_2_url'] + assert downloads[0].settings == { + 'channel_id': 1, + 'channel_url': 'https://www.youtube.com/c/LearningSelfReliance/videos', + } def test_get_or_create_channel(test_session): - """ - A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can - match it. + """A Channel may need to be created for an arbitrary download. + + Attempt to use an existing Channel if we can match it. """ c1 = Channel(name='foo', source_id='foo', url='foo') c2 = Channel(name='bar', source_id='bar') @@ -376,8 +380,8 @@ async def test_invalid_download_url(test_session, test_download_manager, mock_vi @pytest.mark.asyncio -async def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager, - mock_video_process_runner, image_file): +async def test_video_download(test_session, test_directory, simple_channel, video_download_manager, + mock_video_process_runner, image_file): """A video download is performed, files are associated.""" simple_channel.source_id = example_video_json['channel_id'] simple_channel.directory = test_directory / 'videos/channel name' diff --git a/wrolpi/common.py b/wrolpi/common.py index d9a713ff..10337360 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -187,7 +187,7 @@ class ConfigFile: raise ValueError(f'Refusing to save config file while testing: {config_file}') # Only one process can write to the file. - self.file_lock.acquire(block=True, timeout=5.0) + acquired = self.file_lock.acquire(block=True, timeout=5.0) try: # Config directory may not exist. @@ -206,7 +206,8 @@ class ConfigFile: with config_file.open('wt') as fh: yaml.dump(config, fh) finally: - self.file_lock.release() + if acquired: + self.file_lock.release() def get_file(self) -> Path: if not self.file_name: diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 2ecd8343..913b9487 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -50,6 +50,7 @@ class DownloadResult: info_json: dict = field(default_factory=dict) location: str = None success: bool = False + settings: dict = field(default_factory=dict) class Download(ModelHelper, Base): # noqa @@ -402,7 +403,10 @@ class DownloadManager: if result.downloads: worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}') urls = download.filter_excluded(result.downloads) - self.create_downloads(urls, session, downloader=download.sub_downloader) + downloads = self.create_downloads(urls, session, + downloader=download.sub_downloader, + settings=result.settings, + ) if try_again is False and not download.frequency: # Only once-downloads can fail. @@ -533,17 +537,9 @@ class DownloadManager: download.manager = self return download - @optional_session - def create_download(self, url: str, session: Session = None, downloader: str = None, reset_attempts: bool = False, - sub_downloader: str = None, excluded_urls: List[str] = None) -> Download: - """Schedule a URL for download. If the URL failed previously, it may be retried.""" - downloads = self.create_downloads([url], session=session, downloader=downloader, excluded_urls=excluded_urls, - reset_attempts=reset_attempts, sub_downloader=sub_downloader) - return downloads[0] - @optional_session def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None, - reset_attempts: bool = False, sub_downloader: str = None, excluded_urls: List[str] = None) \ + reset_attempts: bool = False, sub_downloader: str = None, settings: dict = None) \ -> List[Download]: """Schedule all URLs for download. If one cannot be downloaded, none will be added.""" if not all(urls): @@ -577,7 +573,7 @@ class DownloadManager: download.downloader = local_downloader.name download.info_json = info_json or download.info_json download.sub_downloader = sub_downloader - download.settings = {'excluded_urls': excluded_urls} + download.settings = settings downloads.append(download) try: @@ -592,17 +588,24 @@ class DownloadManager: return downloads + @optional_session + def create_download(self, url: str, session: Session = None, downloader: str = None, reset_attempts: bool = False, + sub_downloader: str = None, settings: Dict = None) -> Download: + """Schedule a URL for download. If the URL failed previously, it may be retried.""" + downloads = self.create_downloads([url], session=session, downloader=downloader, settings=settings, + reset_attempts=reset_attempts, sub_downloader=sub_downloader) + return downloads[0] + def recurring_download(self, url: str, frequency: int, downloader: str = None, sub_downloader: str = None, reset_attempts: bool = False, - excluded_urls: List[str] = None) -> Download: + settings: Dict = None) -> Download: """Schedule a recurring download.""" if not frequency: raise ValueError('Recurring download must have a frequency!') with get_db_session(commit=True) as session: - download, = self.create_downloads([url, ], session=session, downloader=downloader, - sub_downloader=sub_downloader, reset_attempts=reset_attempts, - excluded_urls=excluded_urls) + download, = self.create_downloads([url, ], session=session, downloader=downloader, settings=settings, + sub_downloader=sub_downloader, reset_attempts=reset_attempts) download.frequency = frequency return download diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 2f8257c1..254c618f 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -221,13 +221,14 @@ async def post_download(_: Request, body: DownloadRequest): downloader = body.downloader if not downloader or downloader in ('auto', 'None', 'null'): downloader = None + settings = dict(excluded_urls=excluded_urls) if body.frequency: download_manager.recurring_download(urls[0], body.frequency, downloader=downloader, sub_downloader=body.sub_downloader, reset_attempts=True, - excluded_urls=excluded_urls) + settings=settings) else: download_manager.create_downloads(urls, downloader=downloader, sub_downloader=body.sub_downloader, - reset_attempts=True, excluded_urls=excluded_urls) + reset_attempts=True, settings=settings) return response.empty() diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 8e4e5aac..9b200461 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -11,7 +11,8 @@ import pytest from wrolpi.dates import local_timezone, Seconds from wrolpi.db import get_db_context -from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult, import_downloads_config +from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult, import_downloads_config, \ + RSSDownloader from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled from wrolpi.test.common import assert_dict_contains @@ -481,3 +482,37 @@ async def test_downloads_config(test_client, test_session, test_download_manager # Import again, no duplicates. await import_downloads_config() assert_downloads(expected) + + +@pytest.mark.asyncio +async def test_download_excluded_urls(test_client, test_session, test_download_manager): + """Test that URLs that have been excluded are ignored by the download worker.""" + http_downloader = HTTPDownloader() + http_downloader.already_downloaded = lambda *i, **kw: [] + http_downloader.do_download = lambda *i, **kw: DownloadResult(success=True) + test_download_manager.register_downloader(http_downloader) + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + + with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed: + mock_parse_feed.return_value = dict( + bozo=0, + entries=[ + dict(link='https://example.com/a'), + dict(link='https://example.org/b'), # this should be ignored + dict(link='https://example.com/c'), + dict(link='https://example.gov/d'), + ] + ) + settings = dict(excluded_urls=['example.org', 'example.gov']) + feed_download: Download = test_download_manager.create_download('https://example.com/feed', + test_session, sub_downloader='http', + downloader=RSSDownloader.name, + settings=settings) + await test_download_manager.wait_for_all_downloads() + + downloads = list(test_download_manager.get_downloads(test_session)) + assert not feed_download.error, f'Feed download had error {feed_download.error}' + assert len(downloads) == 3, 'Domain was not ignored' + assert [i.url for i in downloads] == \ + ['https://example.com/feed', 'https://example.com/a', 'https://example.com/c'], f'Domain was not ignored' diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index a950b69a..087a571d 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -2,17 +2,15 @@ import json from http import HTTPStatus from itertools import zip_longest -import pytest from mock import mock from wrolpi.admin import HotspotStatus from wrolpi.common import get_config from wrolpi.dates import strptime from wrolpi.db import get_db_session -from wrolpi.downloader import Download, RSSDownloader, DownloadResult +from wrolpi.downloader import Download from wrolpi.root_api import api_app from wrolpi.test.common import TestAPI, wrap_test_db, skip_circleci -from wrolpi.test.test_downloader import HTTPDownloader class TestRootAPI(TestAPI): @@ -353,36 +351,3 @@ def test_get_downloaders(test_client): assert 'downloaders' in response.json, 'Downloaders not returned' assert isinstance(response.json['downloaders'], list) and len(response.json['downloaders']), \ 'No downloaders returned' - - -@pytest.mark.asyncio -async def test_download_excluded_urls(test_client, test_session, test_download_manager): - """Test that URLs that have been excluded are ignored by the download worker.""" - http_downloader = HTTPDownloader() - http_downloader.already_downloaded = lambda *i, **kw: [] - http_downloader.do_download = lambda *i, **kw: DownloadResult(success=True) - test_download_manager.register_downloader(http_downloader) - rss_downloader = RSSDownloader() - test_download_manager.register_downloader(rss_downloader) - - with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed: - mock_parse_feed.return_value = dict( - bozo=0, - entries=[ - dict(link='https://example.com/a'), - dict(link='https://example.org/b'), # this should be ignored - dict(link='https://example.com/c'), - dict(link='https://example.gov/d'), - ] - ) - feed_download: Download = test_download_manager.create_download('https://example.com/feed', - test_session, sub_downloader='http', - downloader=RSSDownloader.name, - excluded_urls=['example.org', 'example.gov']) - await test_download_manager.wait_for_all_downloads() - - downloads = list(test_download_manager.get_downloads(test_session)) - assert not feed_download.error, f'Feed download had error {feed_download.error}' - assert len(downloads) == 3, 'Domain was not ignored' - assert [i.url for i in downloads] == \ - ['https://example.com/feed', 'https://example.com/a', 'https://example.com/c'], f'Domain was not ignored'