From 287beabb9ce3263b48e135ad97bb5def8d327f5a Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 4 Jul 2024 19:16:33 -0600 Subject: [PATCH] Creating missing ChannelDownload when importing channels config. --- app/src/components/Channels.js | 6 +-- app/src/components/admin/Downloads.js | 10 +++++ modules/videos/channel/test/test_api.py | 3 +- modules/videos/channel/test/test_models.py | 18 +++++++- modules/videos/lib.py | 52 ++++++++++++++++------ modules/videos/models.py | 27 ++++++++--- modules/videos/test/test_common.py | 18 +++++--- modules/videos/test/test_lib.py | 21 ++++++++- wrolpi/downloader.py | 10 +++++ wrolpi/schema.py | 2 + 10 files changed, 135 insertions(+), 32 deletions(-) diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index c72bd0b4..19b76316 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -47,7 +47,7 @@ import {Media, ThemeContext} from "../contexts/contexts"; import {SortableTable} from "./SortableTable"; import {toast} from "react-semantic-toasts-2"; import {RecurringDownloadsTable} from "./admin/Downloads"; -import {TagsSelector, useTags} from "../Tags"; +import {TagsSelector} from "../Tags"; function ChannelStatistics({statistics}) { @@ -416,10 +416,10 @@ export function ChannelDownloadForm({channel_id, afterSave, closeModal, download const settings = download.settings ? download.settings : {}; const oldTagNames = download && download.settings && download.settings.tag_names ? download.settings.tag_names : []; const [state, setState] = React.useState({ - url: download ? download.url : '', frequency: download.frequency ? download.frequency : 604800, - title_match: settings.title_match ? settings.title_match : '', title_exclude: settings.title_exclude ? settings.title_exclude : '', + title_match: settings.title_match ? settings.title_match : '', + url: download ? download.url : '', }) const [disabled, setDisabled] = React.useState(useWROLMode()); const [loading, setLoading] = React.useState(false); diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js index 59230833..b95e436a 100644 --- a/app/src/components/admin/Downloads.js +++ b/app/src/components/admin/Downloads.js @@ -117,6 +117,7 @@ export function DownloadEditForm({afterSave, closeModal, download}) { url: download.url || '', frequency: download.frequency, excluded_urls: download.settings ? download.settings.excluded_urls : '', + destination: download.settings ? download.settings.destination : '', }) const [disabled, setDisabled] = React.useState(useWROLMode()); const [loading, setLoading] = React.useState(false); @@ -220,6 +221,15 @@ export function DownloadEditForm({afterSave, closeModal, download}) { onChange={handleInputChange} /> + + + diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index e8779439..7024a63a 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -414,12 +414,13 @@ async def test_create_channel_download(test_async_client, test_session, simple_c tag = tag_factory() # Create ChannelDownload (which includes a Download). - body = {'url': 'https://example.com/1', 'frequency': 42} + body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}} request, response = await test_async_client.post( f'/api/videos/channels/{simple_channel.id}/download', json=body) assert response.status_code == HTTPStatus.NO_CONTENT, response.content download: Download = test_session.query(Download).one() assert download.url == 'https://example.com/1' and not download.settings.get('tag_names') + assert download.settings.get('title_include') is None # Create another ChannelDownload. Previous ChannelDownload/Download are untouched. body = {'url': 'https://example.com/2', 'frequency': 55, 'settings': {'tag_names': [tag.name]}} diff --git a/modules/videos/channel/test/test_models.py b/modules/videos/channel/test/test_models.py index b1548e16..2744a959 100644 --- a/modules/videos/channel/test/test_models.py +++ b/modules/videos/channel/test/test_models.py @@ -3,9 +3,10 @@ from http import HTTPStatus import pytest +from modules.videos.downloader import VideoDownloader from modules.videos.models import Channel, ChannelDownload from modules.videos.models import Video -from wrolpi.downloader import Download, DownloadFrequency +from wrolpi.downloader import Download, DownloadFrequency, download_manager, RSSDownloader def test_delete_channel_no_url(test_session, test_client, channel_factory): @@ -92,6 +93,7 @@ async def test_channel_download_relationships(test_session, download_channel): assert channel.channel_downloads[0].download == download \ and channel.channel_downloads[0].download.url == 'https://example.com/channel1' assert download.frequency == DownloadFrequency.weekly + assert download.settings['destination'] == str(channel.directory) # Deleting Download deletes the ChannelDownload, but not the Channel. download.delete() @@ -107,3 +109,17 @@ async def test_channel_download_relationships(test_session, download_channel): download_channel.delete_with_videos() assert test_session.query(Download).count() == 0 assert test_session.query(ChannelDownload).count() == 0 + + +@pytest.mark.asyncio() +async def test_create_channel_download(test_session, simple_channel): + """A ChannelDownload is created if a Download is created for a Channel's directory.""" + test_session.flush() + + settings = dict(destination=str(simple_channel.directory)) + download_manager.create_download('https://example.com/2', RSSDownloader.name, test_session, + sub_downloader_name=VideoDownloader.name, settings=settings) + download = test_session.query(Download).one() + assert download.settings['destination'] == str(simple_channel.directory) + cd = test_session.query(ChannelDownload).one() + assert cd.channel == simple_channel diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 0ffb7d8b..6273343a 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -359,20 +359,6 @@ def get_channels_config_from_db(session: Session) -> dict: @optional_session() def save_channels_config(session: Session = None): """Get the Channel information from the DB, save it to the config.""" - # Create any missing ChannelDownload for any Channel.url that has a Download. - results = session.query(Channel, Download, ChannelDownload) \ - .outerjoin(ChannelDownload, ChannelDownload.download_url == Download.url) \ - .outerjoin(Channel, Channel.url == Download.url) \ - .all() - need_commit = False - for channel, download, cd in results: - if channel and download and not cd: - # Channel.url matches Download.url but there is no ChannelDownload. - channel.get_or_create_download(download.url, session) - need_commit = True - if need_commit: - session.commit() - config = get_channels_config_from_db(session) channels_config = get_channels_config() @@ -463,6 +449,9 @@ def import_channels_config(): logger.warning(f'Deleting {channel} because it is not in the config.') channel.delete_with_videos() + # Create any missing ChannelDownload. + link_channel_and_downloads() + if save_config: # Information about the channel was fetched, store it. save_channels_config() @@ -474,6 +463,41 @@ def import_channels_config(): raise +@optional_session +def link_channel_and_downloads(session: Session): + """Create any missing ChannelDownload for any Channel.url/Channel.directory that has a Download.""" + downloads = session.query(Download).all() + downloads_by_url = {i.url: i for i in downloads} + downloads_by_destination = {i.settings['destination']: i + for i in downloads if 'destination' in (i.settings or dict())} + channels = session.query(Channel).all() + + need_commit = False + for channel in channels: + directory = str(channel.directory) + download = downloads_by_destination.get(directory) or downloads_by_url.get(channel.url) + if download and not channel.get_download(download_id=download.id): + # Create ChannelDownload for the Download that is for this Channel. + cd = ChannelDownload(channel=channel, download=download) + session.add(cd) + channel_import_logger.info(f'Created missing ChannelDownload for {download}') + need_commit = True + + # Get any Downloads for a Channel's RSS feed. + rss_url = channel.get_rss_url() + download = downloads_by_url.get(rss_url) if rss_url else None + if rss_url and download: + cd = channel.get_download(rss_url) + if not cd: + # No ChannelDownload for this RSS Download. + cd = ChannelDownload(channel=channel, download=download) + session.add(cd) + channel_import_logger.info(f'Created missing ChannelDownload for {download}') + + if need_commit: + session.commit() + + YDL = YoutubeDL(dict(cachedir=YTDLP_CACHE_DIR)) ydl_logger = YDL.params['logger'] = logger.getChild('youtube-dl') YDL.add_default_info_extractors() diff --git a/modules/videos/models.py b/modules/videos/models.py index 04735bee..b915ba42 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -623,6 +623,10 @@ class Channel(ModelHelper, Base): raise FileNotFoundError(f'{path} does not exist!') return path + @staticmethod + def get_by_path(path: pathlib.Path, session: Session) -> Optional['Channel']: + return session.query(Channel).filter_by(directory=path).one_or_none() + def __json__(self) -> dict: d = dict( channel_downloads=self.channel_downloads, @@ -731,7 +735,8 @@ class Channel(ModelHelper, Base): @optional_session def get_or_create_download(self, url: str, session: Session = None, reset_attempts: bool = False) \ -> ChannelDownload: - """Get a ChannelDownload record, if it does not exist, create it.""" + """Get a ChannelDownload record, if it does not exist, create it. Create a Download if necessary + which goes into this Channel's directory.""" from modules.videos.downloader import ChannelDownloader, VideoDownloader if not url: @@ -743,10 +748,17 @@ class Channel(ModelHelper, Base): # VideoDownloader. download = session.query(Download).filter_by(url=url).one_or_none() if not download: - download_manager.create_download(url, ChannelDownloader.name, session=session, - sub_downloader_name=VideoDownloader.name, - reset_attempts=reset_attempts, - ) + download = download_manager.create_download(url, ChannelDownloader.name, session=session, + sub_downloader_name=VideoDownloader.name, + reset_attempts=reset_attempts, + ) + # Download into this Channel's directory by default. + destination = (download.settings or dict()).get('destination') + if not destination: + settings = (download.settings or dict()) + settings.update(dict(destination=str(self.directory))) + download.settings = settings + # Link between Channel and Download. cd = ChannelDownload.get_by_url(url, session) if not cd: cd = ChannelDownload(channel_id=self.id, download_url=url) @@ -754,3 +766,8 @@ class Channel(ModelHelper, Base): logger.debug(f'Created {cd} for {self}') return cd + + def get_rss_url(self) -> str | None: + """Return the RSS Feed URL for this Channel, if any is possible.""" + if self.url and self.source_id and 'youtube.com' in self.url: + return f'https://www.youtube.com/feeds/videos.xml?channel_id={self.source_id}' diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index daabf0f7..fc05e168 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -154,7 +154,7 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto import_channels_config() assert not channel1.channel_downloads assert len(test_session.query(Channel).all()) == 2 - assert test_session.query(Download).all() == [] + assert test_session.query(Download).count() == test_session.query(ChannelDownload).count() == 0 # Add a frequency to the Channel. channels_config = get_channels_config() @@ -166,27 +166,31 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto # Download record is created on import. import_channels_config() assert channel1.channel_downloads - assert len(test_session.query(Channel).all()) == 2 + assert len(test_session.query(Channel).all()) == 2, 'Both Channels should still exist' + assert test_session.query(ChannelDownload).count() == 1, 'One Channel has a Download' download: Download = test_session.query(Download).one() assert download.url == channel1.url assert download.frequency + # Download.next_download was not deleted. next_download = str(download.next_download) import_channels_config() download: Download = test_session.query(Download).one() assert next_download == str(download.next_download) - # Creating Download that matches Channel2's URL creates a ChannelDownload on save. - download = Download(url=channel2.url, frequency=DownloadFrequency.weekly) - test_session.add(download) + # Creating Download that matches Channel2's URL creates a ChannelDownload. Delete it and it should be re-created. + channel2.get_or_create_download(channel2.url, test_session) + save_channels_config() + test_session.query(ChannelDownload).filter_by(download_url=channel2.url).delete() test_session.commit() - save_channels_config() + # Missing ChannelDownload is re-created on import. + import_channels_config() channel2: Channel = test_session.query(Channel).filter_by(id=channel2.id).one() cds = test_session.query(ChannelDownload).all() assert len(cds) == 2, cds - assert channel2.channel_downloads and len(channel2.channel_downloads) == 1 assert channel1.channel_downloads and len(channel1.channel_downloads) == 1 + assert channel2.channel_downloads and len(channel2.channel_downloads) == 1 # Add a Download to Channel2 which does not match Channel.url. channel2.get_or_create_download('https://example.org') diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 6e4743ad..1a2b7cb8 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -5,7 +5,8 @@ import pytest from modules.videos import lib from modules.videos.lib import parse_video_file_name, validate_video, get_statistics -from modules.videos.models import Video +from modules.videos.models import Video, ChannelDownload +from wrolpi.downloader import Download, RSSDownloader from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup from wrolpi.vars import PROJECT_DIR @@ -148,3 +149,21 @@ async def test_orphaned_files(test_session, make_files_structure, test_directory vid2_poster_path, vid3_poster_path, ]) + + +def test_link_channel_and_downloads(test_session, channel_factory): + channel = channel_factory( + url='https://www.youtube.com/c/LearningSelfReliance/videos', + source_id='UCng5u6ASda3LNRXJN0JCQJA', + ) + download1 = Download(url=channel.get_rss_url(), downloader=RSSDownloader.name) + download2 = Download(url='https://example.com', downloader=RSSDownloader.name, + settings=dict(destination=str(channel.directory))) + test_session.add_all([download1, download2]) + test_session.commit() + assert test_session.query(Download).count() == 2 + assert test_session.query(ChannelDownload).count() == 0 + + lib.link_channel_and_downloads(session=test_session) + assert test_session.query(Download).count() == 2 + assert test_session.query(ChannelDownload).count() == 2 diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 41f7f163..f6f9029f 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -520,6 +520,16 @@ class DownloadManager: download.settings = settings if settings is not None else download.settings downloads.append(download) + # Create ChannelDownload for any download that is being downloaded into a Channel's Directory. + from modules.videos.models import Channel, ChannelDownload + if download.settings and (destination := download.settings.get('destination')): + channel = Channel.get_by_path(destination, session) + cd = ChannelDownload.get_by_url(url, session) + if channel and not cd: + cd = ChannelDownload(channel=channel, download=download) + session.add(cd) + + session.flush(downloads) try: # Start downloading ASAP. api_app.add_task(self.dispatch_downloads()) diff --git a/wrolpi/schema.py b/wrolpi/schema.py index ce85b8d0..9b8e3db0 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -72,6 +72,8 @@ class DownloadSettings: self.destination = str(get_media_directory() / self.destination) if self.destination else None if self.suffix and not self.suffix.startswith('.'): raise ValidationError('suffix must start with .') + self.title_exclude = self.title_exclude or None + self.title_include = self.title_include or None @dataclass