diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index e8850db5..57379775 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -101,10 +101,10 @@ class ChannelDownloader(Downloader, ABC): is_a_playlist = self.is_a_playlist(info) try: - if is_a_playlist: - downloads = self.get_playlist_downloads(download) - else: - downloads = await self.get_channel_downloads(download, info, channel) + if not is_a_playlist: + await self.prepare_channel_for_downloads(download, channel) + + downloads = self.get_missing_videos(download, channel) return DownloadResult(success=True, location=location, downloads=downloads) except Exception: if PYTEST: @@ -113,14 +113,14 @@ class ChannelDownloader(Downloader, ABC): logger.warning(f'Failed to update catalog of {kind} {download.url}') return DownloadResult(success=False, location=location, error=str(traceback.format_exc())) - @staticmethod - async def get_channel_downloads(download: Download, info: dict, channel: Channel) -> List[str]: - """Get a list of all videos in a Channel, schedule downloads for any missing videos.""" + @classmethod + async def prepare_channel_for_downloads(cls, download: Download, channel: Channel): + """Update the Channel's video catalog. Refresh the Channel's files if necessary.""" channel_id = channel.id - update_channel_catalog(channel, info) + update_channel_catalog(channel, download.info_json) - with get_db_session(commit=True) as session: + with get_db_session() as session: channel = get_channel(session, channel_id=channel_id, return_dict=False) if not channel.videos or not channel.refreshed: logger.warning(f'Refreshing videos in {channel.directory} because {channel} has no video records!') @@ -128,20 +128,18 @@ class ChannelDownloader(Downloader, ABC): session.commit() # Commit the refresh. session.refresh(channel) - # Get all videos in the channel. - downloads = [i.get('webpage_url') or i.get('url') for i in download.info_json['entries']] - # YouTube Shorts are handled specially. - downloads = list(map(normalize_youtube_shorts_url, downloads)) - # Only download those that have not yet been downloaded. - already_downloaded = [i.url for i in video_downloader.already_downloaded(*downloads)] - downloads = [i for i in downloads if i not in already_downloaded] - return downloads - @staticmethod - def get_playlist_downloads(download: Download) -> List[str]: - """Get a list of all videos in a playlist, schedule downloads for any missing videos.""" - # Get all videos in the playlist - downloads = [i.get('webpage_url') or i.get('url') for i in download.info_json['entries']] + def get_missing_videos(download: Download, channel: Channel) -> List[str]: + """ + Return all URLs of Videos in the `info_json` which need to be downloaded. + """ + downloads = download.info_json['entries'] + if channel.match_regex: + # Only download Videos that have matching titles. + match_regex = re.compile(channel.match_regex) + downloads = [i for i in downloads if (title := i.get('title')) and match_regex.match(title)] + # Prefer `webpage_url` before `url` for all entries. + downloads = [i.get('webpage_url') or i.get('url') for i in downloads] # YouTube Shorts are handled specially. downloads = [normalize_youtube_shorts_url(i) for i in downloads] # Only download those that have not yet been downloaded. diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 1bb5af63..cbee07b5 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -184,7 +184,9 @@ async def test_video_download_no_channel(test_session, video_download_manager, v async def test_download_channel(test_session, simple_channel, video_download_manager, video_file, mock_video_extract_info, mock_video_prepare_filename, mock_video_process_runner): - """Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos.""" + """Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos. + + If a Channel has `match_regex` only those videos with matching titles will be downloaded.""" url = 'https://www.youtube.com/c/LearningSelfReliance/videos' mock_video_prepare_filename.return_value = video_file @@ -203,6 +205,17 @@ async def test_download_channel(test_session, simple_channel, video_download_man # Download is run only for test. assert download.status == 'complete' + # A channel with `match_regex` only returns matching video URLs. + simple_channel.match_regex = '.*(2).*' + test_session.query(Download).delete() + test_session.commit() + with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel: + mock_get_channel.return_value = simple_channel + video_download_manager.recurring_download(url, 100) + await video_download_manager.wait_for_all_downloads() + downloads = video_download_manager.get_once_downloads(test_session) + assert [i.url for i in downloads] == ['https://youtube.com/watch?v=video_2_url'] + def test_get_or_create_channel(test_session): """ @@ -502,7 +515,3 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac 'https://www.youtube.com/watch?v=video_2_url', # Shorts is converted to regular URL. 'https://youtube.com/watch?v=video_3_url', } - - -def test_channel_download_match_title(test_session, test_directory, simple_channel, video_download_manager): - pass diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index fe6d8765..bc08a36d 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -370,8 +370,7 @@ class DownloadManager: if result.downloads: worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}') - self.create_downloads(result.downloads, session, - downloader=download.sub_downloader) + self.create_downloads(result.downloads, session, downloader=download.sub_downloader) if try_again is False and not download.frequency: # Only once-downloads can fail. @@ -1001,14 +1000,14 @@ class RSSDownloader(Downloader, ABC): sub_downloader = self.manager.get_downloader_by_name(download.sub_downloader) for idx, entry in enumerate(feed['entries']): if url := entry.get('link'): - if not sub_downloader.already_downloaded(url): - urls.append(url) + urls.append(url.strip()) else: logger.warning(f'RSS entry {idx} did not have a link!') - session = Session.object_session(download) - self.manager.create_downloads(urls, session=session, downloader=download.sub_downloader) - return DownloadResult(success=True) + # Only download new URLs. + urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(*urls)]] + + return DownloadResult(success=True, downloads=urls) @staticmethod def acceptable_bozo_errors(feed): diff --git a/wrolpi/test/test_rss.py b/wrolpi/test/test_rss.py index 45d59f20..131a25dd 100644 --- a/wrolpi/test/test_rss.py +++ b/wrolpi/test/test_rss.py @@ -14,9 +14,9 @@ class RSSHTTPDownloader(HTTPDownloader): return DownloadResult(success=True) @optional_session - def already_downloaded(self, url: str, session=None): - download = session.query(Download).filter_by(url=url).one_or_none() - return download + def already_downloaded(self, *urls: List[str], session=None): + downloads = list(session.query(Download).filter(Download.url.in_(urls))) + return downloads @pytest.mark.asyncio @@ -30,10 +30,10 @@ async def test_rss_download(test_session, test_download_manager): def check_downloads(expected: List[dict]): downloads = test_session.query(Download).order_by(Download.id).all() for download, expected in zip_longest(downloads, expected): - assert download.id == expected['id'] \ - and download.status == expected['status'] \ - and download.url == expected['url'] \ - and download.attempts == expected['attempts'], f'{download} != {expected}' + assert download.id == expected['id'], 'Download id does not match' + assert download.status == expected['status'], 'Download status does not match' + assert download.url == expected['url'], 'Download URL does not match' + assert download.attempts == expected['attempts'], 'Download attempts do not match' with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed: mock_parse_feed.return_value = dict(