mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Saving channel config when source_id is fetched during channel import.
This commit is contained in:
parent
b18cc7f18d
commit
a0df01f7b9
3 changed files with 58 additions and 45 deletions
|
|
@ -377,20 +377,25 @@ def save_channels_config(session=None, preserve_favorites: bool = True):
|
|||
channels_config.update(config)
|
||||
|
||||
|
||||
channel_import_logger = logger.getChild('channel_import')
|
||||
|
||||
|
||||
@before_startup
|
||||
@register_after_refresh
|
||||
def import_channels_config():
|
||||
"""Import channel settings to the DB. Existing channels will be updated."""
|
||||
if PYTEST and not TEST_CHANNELS_CONFIG:
|
||||
logger.warning(f'Not importing channels during this test. Use `test_channels_config` fixture if you would '
|
||||
f'like to call this.')
|
||||
channel_import_logger.warning(
|
||||
f'Not importing channels during this test. Use `test_channels_config` fixture if you would '
|
||||
f'like to call this.')
|
||||
return
|
||||
|
||||
logger.info('Importing videos config')
|
||||
channel_import_logger.info('Importing videos config')
|
||||
try:
|
||||
config = get_channels_config()
|
||||
channels, favorites = config.channels, config.favorites
|
||||
|
||||
save_config = False
|
||||
with get_db_session(commit=True) as session:
|
||||
for data in channels:
|
||||
if isinstance(data, str):
|
||||
|
|
@ -431,6 +436,20 @@ def import_channels_config():
|
|||
if not channel.source_id and channel.url:
|
||||
# If we can download from a channel, we must have its source_id.
|
||||
channel.source_id = get_channel_source_id(channel.url)
|
||||
save_config = True
|
||||
if not channel.source_id:
|
||||
channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
|
||||
|
||||
channel_import_logger.debug(f'Updated {repr(channel.name)}'
|
||||
f' url={channel.url}'
|
||||
f' source_id={channel.source_id}'
|
||||
f' directory={channel.directory}'
|
||||
f' download_frequency={channel.download_frequency}'
|
||||
)
|
||||
|
||||
if save_config:
|
||||
# Information about the channel was fetched, store it.
|
||||
config.save()
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
channels_by_link = {sanitize_link(i.name): i for i in session.query(Channel)}
|
||||
|
|
@ -444,7 +463,7 @@ def import_channels_config():
|
|||
# TODO remove these old favorites after beta.
|
||||
channel = channels_by_link.get(directory_)
|
||||
if not channel:
|
||||
logger.warning(f'Cannot find channel {directory=} for favorites!')
|
||||
channel_import_logger.warning(f'Cannot find channel {directory=} for favorites!')
|
||||
continue
|
||||
channel_dir = channel.directory
|
||||
else:
|
||||
|
|
@ -460,9 +479,9 @@ def import_channels_config():
|
|||
if video:
|
||||
video.favorite = data['favorite']
|
||||
else:
|
||||
logger.warning(f'Cannot find video to favorite: {video_path}')
|
||||
channel_import_logger.warning(f'Cannot find video to favorite: {video_path}')
|
||||
except Exception as e:
|
||||
logger.warning('Failed to load channels config!', exc_info=e)
|
||||
channel_import_logger.warning('Failed to load channels config!', exc_info=e)
|
||||
if PYTEST:
|
||||
# Do not interrupt startup, only raise during testing.
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import asyncio
|
||||
import pathlib
|
||||
import shutil
|
||||
from copy import copy
|
||||
from unittest import mock
|
||||
|
|
@ -9,13 +8,12 @@ from yt_dlp.utils import UnsupportedError
|
|||
|
||||
from modules.videos.channel.lib import download_channel
|
||||
from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \
|
||||
ChannelDownloader, get_or_create_channel, channel_downloader
|
||||
get_or_create_channel, channel_downloader, video_downloader
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import DownloadManager, Download, DownloadResult
|
||||
from wrolpi.downloader import Download, DownloadResult
|
||||
from wrolpi.errors import InvalidDownload
|
||||
from wrolpi.files.models import File
|
||||
from wrolpi.test.common import TestAPI
|
||||
from wrolpi.test.test_downloader import HTTPDownloader
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
|
@ -108,48 +106,44 @@ example_channel_json = {
|
|||
}
|
||||
|
||||
|
||||
class TestVideosDownloaders(TestAPI):
|
||||
def test_video_valid_url():
|
||||
# A specific video can be downloaded.
|
||||
assert video_downloader.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0] is True
|
||||
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.vd = VideoDownloader()
|
||||
self.cd = ChannelDownloader()
|
||||
self.mgr = DownloadManager()
|
||||
self.mgr.register_downloader(self.vd)
|
||||
self.mgr.register_downloader(self.cd)
|
||||
# A channel cannot be downloaded.
|
||||
assert video_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0] is False
|
||||
assert video_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0] is False
|
||||
|
||||
self.videos_dir = pathlib.Path(self.tmp_dir.name) / 'videos'
|
||||
self.videos_dir.mkdir()
|
||||
|
||||
def test_video_valid_url(self):
|
||||
# A specific video can be downloaded.
|
||||
self.assertTrue(self.vd.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0])
|
||||
def test_channel_valid_url():
|
||||
# An entire domain cannot be downloaded.
|
||||
assert not channel_downloader.valid_url('https://example.com')[0]
|
||||
assert not channel_downloader.valid_url('https://youtube.com')[0]
|
||||
|
||||
# A channel cannot be downloaded.
|
||||
self.assertFalse(self.vd.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0])
|
||||
self.assertFalse(self.vd.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0])
|
||||
# Cannot download a single video.
|
||||
assert not channel_downloader.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0]
|
||||
|
||||
def test_channel_valid_url(self):
|
||||
# An entire domain cannot be downloaded.
|
||||
self.assertFalse(self.cd.valid_url('https://example.com')[0])
|
||||
self.assertFalse(self.cd.valid_url('https://youtube.com')[0])
|
||||
# Can download entire channels.
|
||||
assert channel_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0]
|
||||
assert channel_downloader.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0]
|
||||
|
||||
# Cannot download a single video.
|
||||
self.assertFalse(self.cd.valid_url('https://www.youtube.com/watch?v=31jPEBiAC3c')[0])
|
||||
# Can download entire playlists.
|
||||
assert channel_downloader.valid_url('https://www.youtube.com/playlist?list=PLCdlMQeP-TbG12nkBCt0E96yr3EfwcH4E')[0]
|
||||
|
||||
# Can download entire channels.
|
||||
self.assertTrue(self.cd.valid_url('https://www.youtube.com/c/LearningSelfReliance/videos')[0])
|
||||
self.assertTrue(self.cd.valid_url('https://www.youtube.com/c/LearningSelfReliance')[0])
|
||||
|
||||
# Can download entire playlists.
|
||||
self.assertTrue(
|
||||
self.cd.valid_url('https://www.youtube.com/playlist?list=PLCdlMQeP-TbG12nkBCt0E96yr3EfwcH4E')[0])
|
||||
|
||||
def test_get_downloader(self):
|
||||
# The correct Downloader is gotten.
|
||||
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance/videos')[0], self.cd)
|
||||
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance')[0], self.cd)
|
||||
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/watch?v=31jPEBiAC3c')[0], self.vd)
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
'url,expected', [
|
||||
('https://www.youtube.com/c/LearningSelfReliance/videos', channel_downloader),
|
||||
('https://www.youtube.com/c/LearningSelfReliance', channel_downloader),
|
||||
('https://www.youtube.com/watch?v=31jPEBiAC3c', video_downloader),
|
||||
]
|
||||
)
|
||||
async def test_get_downloader(test_session, test_download_manager, url, expected):
|
||||
"""The correct Downloader is gotten."""
|
||||
test_download_manager.register_downloader(channel_downloader)
|
||||
test_download_manager.register_downloader(video_downloader)
|
||||
assert test_download_manager.get_downloader(url)[0] == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -24,5 +24,5 @@ selenium==4.1.0
|
|||
srt==3.5.1
|
||||
websockets==10.1
|
||||
webvtt-py==0.4.6
|
||||
yt-dlp==2023.1.6
|
||||
yt-dlp==2023.2.17
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue