Restoring channel.match_regex functionality.

This commit is contained in:
Roland 2022-09-23 15:32:28 -06:00
parent 65322edf42
commit 5cc15c9328
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
4 changed files with 47 additions and 41 deletions

View file

@ -101,10 +101,10 @@ class ChannelDownloader(Downloader, ABC):
is_a_playlist = self.is_a_playlist(info)
try:
if is_a_playlist:
downloads = self.get_playlist_downloads(download)
else:
downloads = await self.get_channel_downloads(download, info, channel)
if not is_a_playlist:
await self.prepare_channel_for_downloads(download, channel)
downloads = self.get_missing_videos(download, channel)
return DownloadResult(success=True, location=location, downloads=downloads)
except Exception:
if PYTEST:
@ -113,14 +113,14 @@ class ChannelDownloader(Downloader, ABC):
logger.warning(f'Failed to update catalog of {kind} {download.url}')
return DownloadResult(success=False, location=location, error=str(traceback.format_exc()))
@staticmethod
async def get_channel_downloads(download: Download, info: dict, channel: Channel) -> List[str]:
"""Get a list of all videos in a Channel, schedule downloads for any missing videos."""
@classmethod
async def prepare_channel_for_downloads(cls, download: Download, channel: Channel):
"""Update the Channel's video catalog. Refresh the Channel's files if necessary."""
channel_id = channel.id
update_channel_catalog(channel, info)
update_channel_catalog(channel, download.info_json)
with get_db_session(commit=True) as session:
with get_db_session() as session:
channel = get_channel(session, channel_id=channel_id, return_dict=False)
if not channel.videos or not channel.refreshed:
logger.warning(f'Refreshing videos in {channel.directory} because {channel} has no video records!')
@ -128,20 +128,18 @@ class ChannelDownloader(Downloader, ABC):
session.commit() # Commit the refresh.
session.refresh(channel)
# Get all videos in the channel.
downloads = [i.get('webpage_url') or i.get('url') for i in download.info_json['entries']]
# YouTube Shorts are handled specially.
downloads = list(map(normalize_youtube_shorts_url, downloads))
# Only download those that have not yet been downloaded.
already_downloaded = [i.url for i in video_downloader.already_downloaded(*downloads)]
downloads = [i for i in downloads if i not in already_downloaded]
return downloads
@staticmethod
def get_playlist_downloads(download: Download) -> List[str]:
"""Get a list of all videos in a playlist, schedule downloads for any missing videos."""
# Get all videos in the playlist
downloads = [i.get('webpage_url') or i.get('url') for i in download.info_json['entries']]
def get_missing_videos(download: Download, channel: Channel) -> List[str]:
"""
Return all URLs of Videos in the `info_json` which need to be downloaded.
"""
downloads = download.info_json['entries']
if channel.match_regex:
# Only download Videos that have matching titles.
match_regex = re.compile(channel.match_regex)
downloads = [i for i in downloads if (title := i.get('title')) and match_regex.match(title)]
# Prefer `webpage_url` before `url` for all entries.
downloads = [i.get('webpage_url') or i.get('url') for i in downloads]
# YouTube Shorts are handled specially.
downloads = [normalize_youtube_shorts_url(i) for i in downloads]
# Only download those that have not yet been downloaded.

View file

@ -184,7 +184,9 @@ async def test_video_download_no_channel(test_session, video_download_manager, v
async def test_download_channel(test_session, simple_channel, video_download_manager, video_file,
mock_video_extract_info, mock_video_prepare_filename,
mock_video_process_runner):
"""Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos."""
"""Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos.
If a Channel has `match_regex` only those videos with matching titles will be downloaded."""
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
mock_video_prepare_filename.return_value = video_file
@ -203,6 +205,17 @@ async def test_download_channel(test_session, simple_channel, video_download_man
# Download is run only for test.
assert download.status == 'complete'
# A channel with `match_regex` only returns matching video URLs.
simple_channel.match_regex = '.*(2).*'
test_session.query(Download).delete()
test_session.commit()
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
mock_get_channel.return_value = simple_channel
video_download_manager.recurring_download(url, 100)
await video_download_manager.wait_for_all_downloads()
downloads = video_download_manager.get_once_downloads(test_session)
assert [i.url for i in downloads] == ['https://youtube.com/watch?v=video_2_url']
def test_get_or_create_channel(test_session):
"""
@ -502,7 +515,3 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac
'https://www.youtube.com/watch?v=video_2_url', # Shorts is converted to regular URL.
'https://youtube.com/watch?v=video_3_url',
}
def test_channel_download_match_title(test_session, test_directory, simple_channel, video_download_manager):
pass

View file

@ -370,8 +370,7 @@ class DownloadManager:
if result.downloads:
worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}')
self.create_downloads(result.downloads, session,
downloader=download.sub_downloader)
self.create_downloads(result.downloads, session, downloader=download.sub_downloader)
if try_again is False and not download.frequency:
# Only once-downloads can fail.
@ -1001,14 +1000,14 @@ class RSSDownloader(Downloader, ABC):
sub_downloader = self.manager.get_downloader_by_name(download.sub_downloader)
for idx, entry in enumerate(feed['entries']):
if url := entry.get('link'):
if not sub_downloader.already_downloaded(url):
urls.append(url)
urls.append(url.strip())
else:
logger.warning(f'RSS entry {idx} did not have a link!')
session = Session.object_session(download)
self.manager.create_downloads(urls, session=session, downloader=download.sub_downloader)
return DownloadResult(success=True)
# Only download new URLs.
urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(*urls)]]
return DownloadResult(success=True, downloads=urls)
@staticmethod
def acceptable_bozo_errors(feed):

View file

@ -14,9 +14,9 @@ class RSSHTTPDownloader(HTTPDownloader):
return DownloadResult(success=True)
@optional_session
def already_downloaded(self, url: str, session=None):
download = session.query(Download).filter_by(url=url).one_or_none()
return download
def already_downloaded(self, *urls: List[str], session=None):
downloads = list(session.query(Download).filter(Download.url.in_(urls)))
return downloads
@pytest.mark.asyncio
@ -30,10 +30,10 @@ async def test_rss_download(test_session, test_download_manager):
def check_downloads(expected: List[dict]):
downloads = test_session.query(Download).order_by(Download.id).all()
for download, expected in zip_longest(downloads, expected):
assert download.id == expected['id'] \
and download.status == expected['status'] \
and download.url == expected['url'] \
and download.attempts == expected['attempts'], f'{download} != {expected}'
assert download.id == expected['id'], 'Download id does not match'
assert download.status == expected['status'], 'Download status does not match'
assert download.url == expected['url'], 'Download URL does not match'
assert download.attempts == expected['attempts'], 'Download attempts do not match'
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
mock_parse_feed.return_value = dict(