From a0df01f7b90462f34c121f4e65ce8c7eb48e0452 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 17 Feb 2023 16:00:44 -0700 Subject: [PATCH] Saving channel config when source_id is fetched during channel import. --- modules/videos/lib.py | 31 +++++++++--- modules/videos/test/test_downloader.py | 70 ++++++++++++-------------- requirements.txt | 2 +- 3 files changed, 58 insertions(+), 45 deletions(-) diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 92d0cba8..f09d06fb 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -377,20 +377,25 @@ def save_channels_config(session=None, preserve_favorites: bool = True): channels_config.update(config) +channel_import_logger = logger.getChild('channel_import') + + @before_startup @register_after_refresh def import_channels_config(): """Import channel settings to the DB. Existing channels will be updated.""" if PYTEST and not TEST_CHANNELS_CONFIG: - logger.warning(f'Not importing channels during this test. Use `test_channels_config` fixture if you would ' - f'like to call this.') + channel_import_logger.warning( + f'Not importing channels during this test. Use `test_channels_config` fixture if you would ' + f'like to call this.') return - logger.info('Importing videos config') + channel_import_logger.info('Importing videos config') try: config = get_channels_config() channels, favorites = config.channels, config.favorites + save_config = False with get_db_session(commit=True) as session: for data in channels: if isinstance(data, str): @@ -431,6 +436,20 @@ def import_channels_config(): if not channel.source_id and channel.url: # If we can download from a channel, we must have its source_id. channel.source_id = get_channel_source_id(channel.url) + save_config = True + if not channel.source_id: + channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}') + + channel_import_logger.debug(f'Updated {repr(channel.name)}' + f' url={channel.url}' + f' source_id={channel.source_id}' + f' directory={channel.directory}' + f' download_frequency={channel.download_frequency}' + ) + + if save_config: + # Information about the channel was fetched, store it. + config.save() with get_db_session(commit=True) as session: channels_by_link = {sanitize_link(i.name): i for i in session.query(Channel)} @@ -444,7 +463,7 @@ def import_channels_config(): # TODO remove these old favorites after beta. channel = channels_by_link.get(directory_) if not channel: - logger.warning(f'Cannot find channel {directory=} for favorites!') + channel_import_logger.warning(f'Cannot find channel {directory=} for favorites!') continue channel_dir = channel.directory else: @@ -460,9 +479,9 @@ def import_channels_config(): if video: video.favorite = data['favorite'] else: - logger.warning(f'Cannot find video to favorite: {video_path}') + channel_import_logger.warning(f'Cannot find video to favorite: {video_path}') except Exception as e: - logger.warning('Failed to load channels config!', exc_info=e) + channel_import_logger.warning('Failed to load channels config!', exc_info=e) if PYTEST: # Do not interrupt startup, only raise during testing. raise diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 223f1304..52c1ad76 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -1,5 +1,4 @@ import asyncio -import pathlib import shutil from copy import copy from unittest import mock @@ -9,13 +8,12 @@ from yt_dlp.utils import UnsupportedError from modules.videos.channel.lib import download_channel from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \ - ChannelDownloader, get_or_create_channel, channel_downloader + get_or_create_channel, channel_downloader, video_downloader from modules.videos.models import Channel, Video from wrolpi.db import get_db_context -from wrolpi.downloader import DownloadManager, Download, DownloadResult +from wrolpi.downloader import Download, DownloadResult from wrolpi.errors import InvalidDownload from wrolpi.files.models import File -from wrolpi.test.common import TestAPI from wrolpi.test.test_downloader import HTTPDownloader from wrolpi.vars import PROJECT_DIR @@ -108,48 +106,44 @@ example_channel_json = { } -class TestVideosDownloaders(TestAPI): +def test_video_valid_url(): + # A specific video can be downloaded. + assert video_downloader.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0] is True - def setUp(self) -> None: - super().setUp() - self.vd = VideoDownloader() - self.cd = ChannelDownloader() - self.mgr = DownloadManager() - self.mgr.register_downloader(self.vd) - self.mgr.register_downloader(self.cd) + # A channel cannot be downloaded. + assert video_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0] is False + assert video_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0] is False - self.videos_dir = pathlib.Path(self.tmp_dir.name) / 'videos' - self.videos_dir.mkdir() - def test_video_valid_url(self): - # A specific video can be downloaded. - self.assertTrue(self.vd.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0]) +def test_channel_valid_url(): + # An entire domain cannot be downloaded. + assert not channel_downloader.valid_url('https://example.com')[0] + assert not channel_downloader.valid_url('https://youtube.com')[0] - # A channel cannot be downloaded. - self.assertFalse(self.vd.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0]) - self.assertFalse(self.vd.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0]) + # Cannot download a single video. + assert not channel_downloader.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0] - def test_channel_valid_url(self): - # An entire domain cannot be downloaded. - self.assertFalse(self.cd.valid_url('https://example.com')[0]) - self.assertFalse(self.cd.valid_url('https://youtube.com')[0]) + # Can download entire channels. + assert channel_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0] + assert channel_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0] - # Cannot download a single video. - self.assertFalse(self.cd.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0]) + # Can download entire playlists. + assert channel_downloader.valid_url('https://www.youtube.com/playlist?list=PLCdlMQeP-TbG12nkBCt0E96yr3EfwcH4E')[0] - # Can download entire channels. - self.assertTrue(self.cd.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0]) - self.assertTrue(self.cd.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0]) - # Can download entire playlists. - self.assertTrue( - self.cd.valid_url('https://www.youtube.com/playlist?list=PLCdlMQeP-TbG12nkBCt0E96yr3EfwcH4E')[0]) - - def test_get_downloader(self): - # The correct Downloader is gotten. - self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance/videos')[0], self.cd) - self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance')[0], self.cd) - self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/watch?v=31jPEBiAC3c')[0], self.vd) +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'url,expected', [ + ('https://www.youtube.com/c/LearningSelfReliance/videos', channel_downloader), + ('https://www.youtube.com/c/LearningSelfReliance', channel_downloader), + ('https://www.youtube.com/watch?v=31jPEBiAC3c', video_downloader), + ] +) +async def test_get_downloader(test_session, test_download_manager, url, expected): + """The correct Downloader is gotten.""" + test_download_manager.register_downloader(channel_downloader) + test_download_manager.register_downloader(video_downloader) + assert test_download_manager.get_downloader(url)[0] == expected @pytest.mark.asyncio diff --git a/requirements.txt b/requirements.txt index 64dd0473..d310cfc0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,5 +24,5 @@ selenium==4.1.0 srt==3.5.1 websockets==10.1 webvtt-py==0.4.6 -yt-dlp==2023.1.6 +yt-dlp==2023.2.17