Saving channel config when source_id is fetched during channel import.

This commit is contained in:
Roland 2023-02-17 16:00:44 -07:00
parent b18cc7f18d
commit a0df01f7b9
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
3 changed files with 58 additions and 45 deletions

View file

@ -377,20 +377,25 @@ def save_channels_config(session=None, preserve_favorites: bool = True):
channels_config.update(config)
channel_import_logger = logger.getChild('channel_import')
@before_startup
@register_after_refresh
def import_channels_config():
"""Import channel settings to the DB. Existing channels will be updated."""
if PYTEST and not TEST_CHANNELS_CONFIG:
logger.warning(f'Not importing channels during this test. Use `test_channels_config` fixture if you would '
f'like to call this.')
channel_import_logger.warning(
f'Not importing channels during this test. Use `test_channels_config` fixture if you would '
f'like to call this.')
return
logger.info('Importing videos config')
channel_import_logger.info('Importing videos config')
try:
config = get_channels_config()
channels, favorites = config.channels, config.favorites
save_config = False
with get_db_session(commit=True) as session:
for data in channels:
if isinstance(data, str):
@ -431,6 +436,20 @@ def import_channels_config():
if not channel.source_id and channel.url:
# If we can download from a channel, we must have its source_id.
channel.source_id = get_channel_source_id(channel.url)
save_config = True
if not channel.source_id:
channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
channel_import_logger.debug(f'Updated {repr(channel.name)}'
f' url={channel.url}'
f' source_id={channel.source_id}'
f' directory={channel.directory}'
f' download_frequency={channel.download_frequency}'
)
if save_config:
# Information about the channel was fetched, store it.
config.save()
with get_db_session(commit=True) as session:
channels_by_link = {sanitize_link(i.name): i for i in session.query(Channel)}
@ -444,7 +463,7 @@ def import_channels_config():
# TODO remove these old favorites after beta.
channel = channels_by_link.get(directory_)
if not channel:
logger.warning(f'Cannot find channel {directory=} for favorites!')
channel_import_logger.warning(f'Cannot find channel {directory=} for favorites!')
continue
channel_dir = channel.directory
else:
@ -460,9 +479,9 @@ def import_channels_config():
if video:
video.favorite = data['favorite']
else:
logger.warning(f'Cannot find video to favorite: {video_path}')
channel_import_logger.warning(f'Cannot find video to favorite: {video_path}')
except Exception as e:
logger.warning('Failed to load channels config!', exc_info=e)
channel_import_logger.warning('Failed to load channels config!', exc_info=e)
if PYTEST:
# Do not interrupt startup, only raise during testing.
raise

View file

@ -1,5 +1,4 @@
import asyncio
import pathlib
import shutil
from copy import copy
from unittest import mock
@ -9,13 +8,12 @@ from yt_dlp.utils import UnsupportedError
from modules.videos.channel.lib import download_channel
from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \
ChannelDownloader, get_or_create_channel, channel_downloader
get_or_create_channel, channel_downloader, video_downloader
from modules.videos.models import Channel, Video
from wrolpi.db import get_db_context
from wrolpi.downloader import DownloadManager, Download, DownloadResult
from wrolpi.downloader import Download, DownloadResult
from wrolpi.errors import InvalidDownload
from wrolpi.files.models import File
from wrolpi.test.common import TestAPI
from wrolpi.test.test_downloader import HTTPDownloader
from wrolpi.vars import PROJECT_DIR
@ -108,48 +106,44 @@ example_channel_json = {
}
class TestVideosDownloaders(TestAPI):
def test_video_valid_url():
# A specific video can be downloaded.
assert video_downloader.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0] is True
def setUp(self) -> None:
super().setUp()
self.vd = VideoDownloader()
self.cd = ChannelDownloader()
self.mgr = DownloadManager()
self.mgr.register_downloader(self.vd)
self.mgr.register_downloader(self.cd)
# A channel cannot be downloaded.
assert video_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0] is False
assert video_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0] is False
self.videos_dir = pathlib.Path(self.tmp_dir.name) / 'videos'
self.videos_dir.mkdir()
def test_video_valid_url(self):
# A specific video can be downloaded.
self.assertTrue(self.vd.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0])
def test_channel_valid_url():
# An entire domain cannot be downloaded.
assert not channel_downloader.valid_url('https://example.com')[0]
assert not channel_downloader.valid_url('https://youtube.com')[0]
# A channel cannot be downloaded.
self.assertFalse(self.vd.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0])
self.assertFalse(self.vd.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0])
# Cannot download a single video.
assert not channel_downloader.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0]
def test_channel_valid_url(self):
# An entire domain cannot be downloaded.
self.assertFalse(self.cd.valid_url('https://example.com')[0])
self.assertFalse(self.cd.valid_url('https://youtube.com')[0])
# Can download entire channels.
assert channel_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0]
assert channel_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0]
# Cannot download a single video.
self.assertFalse(self.cd.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0])
# Can download entire playlists.
assert channel_downloader.valid_url('https://www.youtube.com/playlist?list=PLCdlMQeP-TbG12nkBCt0E96yr3EfwcH4E')[0]
# Can download entire channels.
self.assertTrue(self.cd.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0])
self.assertTrue(self.cd.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0])
# Can download entire playlists.
self.assertTrue(
self.cd.valid_url('https://www.youtube.com/playlist?list=PLCdlMQeP-TbG12nkBCt0E96yr3EfwcH4E')[0])
def test_get_downloader(self):
# The correct Downloader is gotten.
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance/videos')[0], self.cd)
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance')[0], self.cd)
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/watch?v=31jPEBiAC3c')[0], self.vd)
@pytest.mark.asyncio
@pytest.mark.parametrize(
'url,expected', [
('https://www.youtube.com/c/LearningSelfReliance/videos', channel_downloader),
('https://www.youtube.com/c/LearningSelfReliance', channel_downloader),
('https://www.youtube.com/watch?v=31jPEBiAC3c', video_downloader),
]
)
async def test_get_downloader(test_session, test_download_manager, url, expected):
"""The correct Downloader is gotten."""
test_download_manager.register_downloader(channel_downloader)
test_download_manager.register_downloader(video_downloader)
assert test_download_manager.get_downloader(url)[0] == expected
@pytest.mark.asyncio

View file

@ -24,5 +24,5 @@ selenium==4.1.0
srt==3.5.1
websockets==10.1
webvtt-py==0.4.6
yt-dlp==2023.1.6
yt-dlp==2023.2.17