mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Restoring channel.match_regex functionality.
This commit is contained in:
parent
65322edf42
commit
5cc15c9328
4 changed files with 47 additions and 41 deletions
|
|
@ -101,10 +101,10 @@ class ChannelDownloader(Downloader, ABC):
|
|||
|
||||
is_a_playlist = self.is_a_playlist(info)
|
||||
try:
|
||||
if is_a_playlist:
|
||||
downloads = self.get_playlist_downloads(download)
|
||||
else:
|
||||
downloads = await self.get_channel_downloads(download, info, channel)
|
||||
if not is_a_playlist:
|
||||
await self.prepare_channel_for_downloads(download, channel)
|
||||
|
||||
downloads = self.get_missing_videos(download, channel)
|
||||
return DownloadResult(success=True, location=location, downloads=downloads)
|
||||
except Exception:
|
||||
if PYTEST:
|
||||
|
|
@ -113,14 +113,14 @@ class ChannelDownloader(Downloader, ABC):
|
|||
logger.warning(f'Failed to update catalog of {kind} {download.url}')
|
||||
return DownloadResult(success=False, location=location, error=str(traceback.format_exc()))
|
||||
|
||||
@staticmethod
|
||||
async def get_channel_downloads(download: Download, info: dict, channel: Channel) -> List[str]:
|
||||
"""Get a list of all videos in a Channel, schedule downloads for any missing videos."""
|
||||
@classmethod
|
||||
async def prepare_channel_for_downloads(cls, download: Download, channel: Channel):
|
||||
"""Update the Channel's video catalog. Refresh the Channel's files if necessary."""
|
||||
channel_id = channel.id
|
||||
|
||||
update_channel_catalog(channel, info)
|
||||
update_channel_catalog(channel, download.info_json)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
with get_db_session() as session:
|
||||
channel = get_channel(session, channel_id=channel_id, return_dict=False)
|
||||
if not channel.videos or not channel.refreshed:
|
||||
logger.warning(f'Refreshing videos in {channel.directory} because {channel} has no video records!')
|
||||
|
|
@ -128,20 +128,18 @@ class ChannelDownloader(Downloader, ABC):
|
|||
session.commit() # Commit the refresh.
|
||||
session.refresh(channel)
|
||||
|
||||
# Get all videos in the channel.
|
||||
downloads = [i.get('webpage_url') or i.get('url') for i in download.info_json['entries']]
|
||||
# YouTube Shorts are handled specially.
|
||||
downloads = list(map(normalize_youtube_shorts_url, downloads))
|
||||
# Only download those that have not yet been downloaded.
|
||||
already_downloaded = [i.url for i in video_downloader.already_downloaded(*downloads)]
|
||||
downloads = [i for i in downloads if i not in already_downloaded]
|
||||
return downloads
|
||||
|
||||
@staticmethod
|
||||
def get_playlist_downloads(download: Download) -> List[str]:
|
||||
"""Get a list of all videos in a playlist, schedule downloads for any missing videos."""
|
||||
# Get all videos in the playlist
|
||||
downloads = [i.get('webpage_url') or i.get('url') for i in download.info_json['entries']]
|
||||
def get_missing_videos(download: Download, channel: Channel) -> List[str]:
|
||||
"""
|
||||
Return all URLs of Videos in the `info_json` which need to be downloaded.
|
||||
"""
|
||||
downloads = download.info_json['entries']
|
||||
if channel.match_regex:
|
||||
# Only download Videos that have matching titles.
|
||||
match_regex = re.compile(channel.match_regex)
|
||||
downloads = [i for i in downloads if (title := i.get('title')) and match_regex.match(title)]
|
||||
# Prefer `webpage_url` before `url` for all entries.
|
||||
downloads = [i.get('webpage_url') or i.get('url') for i in downloads]
|
||||
# YouTube Shorts are handled specially.
|
||||
downloads = [normalize_youtube_shorts_url(i) for i in downloads]
|
||||
# Only download those that have not yet been downloaded.
|
||||
|
|
|
|||
|
|
@ -184,7 +184,9 @@ async def test_video_download_no_channel(test_session, video_download_manager, v
|
|||
async def test_download_channel(test_session, simple_channel, video_download_manager, video_file,
|
||||
mock_video_extract_info, mock_video_prepare_filename,
|
||||
mock_video_process_runner):
|
||||
"""Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos."""
|
||||
"""Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos.
|
||||
|
||||
If a Channel has `match_regex` only those videos with matching titles will be downloaded."""
|
||||
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
|
||||
|
||||
mock_video_prepare_filename.return_value = video_file
|
||||
|
|
@ -203,6 +205,17 @@ async def test_download_channel(test_session, simple_channel, video_download_man
|
|||
# Download is run only for test.
|
||||
assert download.status == 'complete'
|
||||
|
||||
# A channel with `match_regex` only returns matching video URLs.
|
||||
simple_channel.match_regex = '.*(2).*'
|
||||
test_session.query(Download).delete()
|
||||
test_session.commit()
|
||||
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
|
||||
mock_get_channel.return_value = simple_channel
|
||||
video_download_manager.recurring_download(url, 100)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
downloads = video_download_manager.get_once_downloads(test_session)
|
||||
assert [i.url for i in downloads] == ['https://youtube.com/watch?v=video_2_url']
|
||||
|
||||
|
||||
def test_get_or_create_channel(test_session):
|
||||
"""
|
||||
|
|
@ -502,7 +515,3 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac
|
|||
'https://www.youtube.com/watch?v=video_2_url', # Shorts is converted to regular URL.
|
||||
'https://youtube.com/watch?v=video_3_url',
|
||||
}
|
||||
|
||||
|
||||
def test_channel_download_match_title(test_session, test_directory, simple_channel, video_download_manager):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -370,8 +370,7 @@ class DownloadManager:
|
|||
|
||||
if result.downloads:
|
||||
worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}')
|
||||
self.create_downloads(result.downloads, session,
|
||||
downloader=download.sub_downloader)
|
||||
self.create_downloads(result.downloads, session, downloader=download.sub_downloader)
|
||||
|
||||
if try_again is False and not download.frequency:
|
||||
# Only once-downloads can fail.
|
||||
|
|
@ -1001,14 +1000,14 @@ class RSSDownloader(Downloader, ABC):
|
|||
sub_downloader = self.manager.get_downloader_by_name(download.sub_downloader)
|
||||
for idx, entry in enumerate(feed['entries']):
|
||||
if url := entry.get('link'):
|
||||
if not sub_downloader.already_downloaded(url):
|
||||
urls.append(url)
|
||||
urls.append(url.strip())
|
||||
else:
|
||||
logger.warning(f'RSS entry {idx} did not have a link!')
|
||||
|
||||
session = Session.object_session(download)
|
||||
self.manager.create_downloads(urls, session=session, downloader=download.sub_downloader)
|
||||
return DownloadResult(success=True)
|
||||
# Only download new URLs.
|
||||
urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(*urls)]]
|
||||
|
||||
return DownloadResult(success=True, downloads=urls)
|
||||
|
||||
@staticmethod
|
||||
def acceptable_bozo_errors(feed):
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ class RSSHTTPDownloader(HTTPDownloader):
|
|||
return DownloadResult(success=True)
|
||||
|
||||
@optional_session
|
||||
def already_downloaded(self, url: str, session=None):
|
||||
download = session.query(Download).filter_by(url=url).one_or_none()
|
||||
return download
|
||||
def already_downloaded(self, *urls: List[str], session=None):
|
||||
downloads = list(session.query(Download).filter(Download.url.in_(urls)))
|
||||
return downloads
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -30,10 +30,10 @@ async def test_rss_download(test_session, test_download_manager):
|
|||
def check_downloads(expected: List[dict]):
|
||||
downloads = test_session.query(Download).order_by(Download.id).all()
|
||||
for download, expected in zip_longest(downloads, expected):
|
||||
assert download.id == expected['id'] \
|
||||
and download.status == expected['status'] \
|
||||
and download.url == expected['url'] \
|
||||
and download.attempts == expected['attempts'], f'{download} != {expected}'
|
||||
assert download.id == expected['id'], 'Download id does not match'
|
||||
assert download.status == expected['status'], 'Download status does not match'
|
||||
assert download.url == expected['url'], 'Download URL does not match'
|
||||
assert download.attempts == expected['attempts'], 'Download attempts do not match'
|
||||
|
||||
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
|
||||
mock_parse_feed.return_value = dict(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue