diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js
index c72bd0b4..19b76316 100644
--- a/app/src/components/Channels.js
+++ b/app/src/components/Channels.js
@@ -47,7 +47,7 @@ import {Media, ThemeContext} from "../contexts/contexts";
import {SortableTable} from "./SortableTable";
import {toast} from "react-semantic-toasts-2";
import {RecurringDownloadsTable} from "./admin/Downloads";
-import {TagsSelector, useTags} from "../Tags";
+import {TagsSelector} from "../Tags";
function ChannelStatistics({statistics}) {
@@ -416,10 +416,10 @@ export function ChannelDownloadForm({channel_id, afterSave, closeModal, download
const settings = download.settings ? download.settings : {};
const oldTagNames = download && download.settings && download.settings.tag_names ? download.settings.tag_names : [];
const [state, setState] = React.useState({
- url: download ? download.url : '',
frequency: download.frequency ? download.frequency : 604800,
- title_match: settings.title_match ? settings.title_match : '',
title_exclude: settings.title_exclude ? settings.title_exclude : '',
+ title_match: settings.title_match ? settings.title_match : '',
+ url: download ? download.url : '',
})
const [disabled, setDisabled] = React.useState(useWROLMode());
const [loading, setLoading] = React.useState(false);
diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js
index 59230833..b95e436a 100644
--- a/app/src/components/admin/Downloads.js
+++ b/app/src/components/admin/Downloads.js
@@ -117,6 +117,7 @@ export function DownloadEditForm({afterSave, closeModal, download}) {
url: download.url || '',
frequency: download.frequency,
excluded_urls: download.settings ? download.settings.excluded_urls : '',
+ destination: download.settings ? download.settings.destination : '',
})
const [disabled, setDisabled] = React.useState(useWROLMode());
const [loading, setLoading] = React.useState(false);
@@ -220,6 +221,15 @@ export function DownloadEditForm({afterSave, closeModal, download}) {
onChange={handleInputChange}
/>
+
+
+
diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py
index e8779439..7024a63a 100644
--- a/modules/videos/channel/test/test_api.py
+++ b/modules/videos/channel/test/test_api.py
@@ -414,12 +414,13 @@ async def test_create_channel_download(test_async_client, test_session, simple_c
tag = tag_factory()
# Create ChannelDownload (which includes a Download).
- body = {'url': 'https://example.com/1', 'frequency': 42}
+ body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
request, response = await test_async_client.post(
f'/api/videos/channels/{simple_channel.id}/download', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download: Download = test_session.query(Download).one()
assert download.url == 'https://example.com/1' and not download.settings.get('tag_names')
+ assert download.settings.get('title_include') is None
# Create another ChannelDownload. Previous ChannelDownload/Download are untouched.
body = {'url': 'https://example.com/2', 'frequency': 55, 'settings': {'tag_names': [tag.name]}}
diff --git a/modules/videos/channel/test/test_models.py b/modules/videos/channel/test/test_models.py
index b1548e16..2744a959 100644
--- a/modules/videos/channel/test/test_models.py
+++ b/modules/videos/channel/test/test_models.py
@@ -3,9 +3,10 @@ from http import HTTPStatus
import pytest
+from modules.videos.downloader import VideoDownloader
from modules.videos.models import Channel, ChannelDownload
from modules.videos.models import Video
-from wrolpi.downloader import Download, DownloadFrequency
+from wrolpi.downloader import Download, DownloadFrequency, download_manager, RSSDownloader
def test_delete_channel_no_url(test_session, test_client, channel_factory):
@@ -92,6 +93,7 @@ async def test_channel_download_relationships(test_session, download_channel):
assert channel.channel_downloads[0].download == download \
and channel.channel_downloads[0].download.url == 'https://example.com/channel1'
assert download.frequency == DownloadFrequency.weekly
+ assert download.settings['destination'] == str(channel.directory)
# Deleting Download deletes the ChannelDownload, but not the Channel.
download.delete()
@@ -107,3 +109,17 @@ async def test_channel_download_relationships(test_session, download_channel):
download_channel.delete_with_videos()
assert test_session.query(Download).count() == 0
assert test_session.query(ChannelDownload).count() == 0
+
+
+@pytest.mark.asyncio()
+async def test_create_channel_download(test_session, simple_channel):
+ """A ChannelDownload is created if a Download is created for a Channel's directory."""
+ test_session.flush()
+
+ settings = dict(destination=str(simple_channel.directory))
+ download_manager.create_download('https://example.com/2', RSSDownloader.name, test_session,
+ sub_downloader_name=VideoDownloader.name, settings=settings)
+ download = test_session.query(Download).one()
+ assert download.settings['destination'] == str(simple_channel.directory)
+ cd = test_session.query(ChannelDownload).one()
+ assert cd.channel == simple_channel
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 0ffb7d8b..6273343a 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -359,20 +359,6 @@ def get_channels_config_from_db(session: Session) -> dict:
@optional_session()
def save_channels_config(session: Session = None):
"""Get the Channel information from the DB, save it to the config."""
- # Create any missing ChannelDownload for any Channel.url that has a Download.
- results = session.query(Channel, Download, ChannelDownload) \
- .outerjoin(ChannelDownload, ChannelDownload.download_url == Download.url) \
- .outerjoin(Channel, Channel.url == Download.url) \
- .all()
- need_commit = False
- for channel, download, cd in results:
- if channel and download and not cd:
- # Channel.url matches Download.url but there is no ChannelDownload.
- channel.get_or_create_download(download.url, session)
- need_commit = True
- if need_commit:
- session.commit()
-
config = get_channels_config_from_db(session)
channels_config = get_channels_config()
@@ -463,6 +449,9 @@ def import_channels_config():
logger.warning(f'Deleting {channel} because it is not in the config.')
channel.delete_with_videos()
+ # Create any missing ChannelDownload.
+ link_channel_and_downloads()
+
if save_config:
# Information about the channel was fetched, store it.
save_channels_config()
@@ -474,6 +463,41 @@ def import_channels_config():
raise
+@optional_session
+def link_channel_and_downloads(session: Session):
+ """Create any missing ChannelDownload for any Channel.url/Channel.directory that has a Download."""
+ downloads = session.query(Download).all()
+ downloads_by_url = {i.url: i for i in downloads}
+ downloads_by_destination = {i.settings['destination']: i
+ for i in downloads if 'destination' in (i.settings or dict())}
+ channels = session.query(Channel).all()
+
+ need_commit = False
+ for channel in channels:
+ directory = str(channel.directory)
+ download = downloads_by_destination.get(directory) or downloads_by_url.get(channel.url)
+ if download and not channel.get_download(download_id=download.id):
+ # Create ChannelDownload for the Download that is for this Channel.
+ cd = ChannelDownload(channel=channel, download=download)
+ session.add(cd)
+ channel_import_logger.info(f'Created missing ChannelDownload for {download}')
+ need_commit = True
+
+ # Get any Downloads for a Channel's RSS feed.
+ rss_url = channel.get_rss_url()
+ download = downloads_by_url.get(rss_url) if rss_url else None
+ if rss_url and download:
+ cd = channel.get_download(rss_url)
+ if not cd:
+ # No ChannelDownload for this RSS Download.
+ cd = ChannelDownload(channel=channel, download=download)
+ session.add(cd)
+ channel_import_logger.info(f'Created missing ChannelDownload for {download}')
+
+ if need_commit:
+ session.commit()
+
+
YDL = YoutubeDL(dict(cachedir=YTDLP_CACHE_DIR))
ydl_logger = YDL.params['logger'] = logger.getChild('youtube-dl')
YDL.add_default_info_extractors()
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 04735bee..b915ba42 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -623,6 +623,10 @@ class Channel(ModelHelper, Base):
raise FileNotFoundError(f'{path} does not exist!')
return path
+ @staticmethod
+ def get_by_path(path: pathlib.Path, session: Session) -> Optional['Channel']:
+ return session.query(Channel).filter_by(directory=path).one_or_none()
+
def __json__(self) -> dict:
d = dict(
channel_downloads=self.channel_downloads,
@@ -731,7 +735,8 @@ class Channel(ModelHelper, Base):
@optional_session
def get_or_create_download(self, url: str, session: Session = None, reset_attempts: bool = False) \
-> ChannelDownload:
- """Get a ChannelDownload record, if it does not exist, create it."""
+ """Get a ChannelDownload record, if it does not exist, create it. Create a Download if necessary
+ which goes into this Channel's directory."""
from modules.videos.downloader import ChannelDownloader, VideoDownloader
if not url:
@@ -743,10 +748,17 @@ class Channel(ModelHelper, Base):
# VideoDownloader.
download = session.query(Download).filter_by(url=url).one_or_none()
if not download:
- download_manager.create_download(url, ChannelDownloader.name, session=session,
- sub_downloader_name=VideoDownloader.name,
- reset_attempts=reset_attempts,
- )
+ download = download_manager.create_download(url, ChannelDownloader.name, session=session,
+ sub_downloader_name=VideoDownloader.name,
+ reset_attempts=reset_attempts,
+ )
+ # Download into this Channel's directory by default.
+ destination = (download.settings or dict()).get('destination')
+ if not destination:
+ settings = (download.settings or dict())
+ settings.update(dict(destination=str(self.directory)))
+ download.settings = settings
+ # Link between Channel and Download.
cd = ChannelDownload.get_by_url(url, session)
if not cd:
cd = ChannelDownload(channel_id=self.id, download_url=url)
@@ -754,3 +766,8 @@ class Channel(ModelHelper, Base):
logger.debug(f'Created {cd} for {self}')
return cd
+
+ def get_rss_url(self) -> str | None:
+ """Return the RSS Feed URL for this Channel, if any is possible."""
+ if self.url and self.source_id and 'youtube.com' in self.url:
+ return f'https://www.youtube.com/feeds/videos.xml?channel_id={self.source_id}'
diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py
index daabf0f7..fc05e168 100644
--- a/modules/videos/test/test_common.py
+++ b/modules/videos/test/test_common.py
@@ -154,7 +154,7 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto
import_channels_config()
assert not channel1.channel_downloads
assert len(test_session.query(Channel).all()) == 2
- assert test_session.query(Download).all() == []
+ assert test_session.query(Download).count() == test_session.query(ChannelDownload).count() == 0
# Add a frequency to the Channel.
channels_config = get_channels_config()
@@ -166,27 +166,31 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto
# Download record is created on import.
import_channels_config()
assert channel1.channel_downloads
- assert len(test_session.query(Channel).all()) == 2
+ assert len(test_session.query(Channel).all()) == 2, 'Both Channels should still exist'
+ assert test_session.query(ChannelDownload).count() == 1, 'One Channel has a Download'
download: Download = test_session.query(Download).one()
assert download.url == channel1.url
assert download.frequency
+ # Download.next_download was not deleted.
next_download = str(download.next_download)
import_channels_config()
download: Download = test_session.query(Download).one()
assert next_download == str(download.next_download)
- # Creating Download that matches Channel2's URL creates a ChannelDownload on save.
- download = Download(url=channel2.url, frequency=DownloadFrequency.weekly)
- test_session.add(download)
+ # Creating Download that matches Channel2's URL creates a ChannelDownload. Delete it and it should be re-created.
+ channel2.get_or_create_download(channel2.url, test_session)
+ save_channels_config()
+ test_session.query(ChannelDownload).filter_by(download_url=channel2.url).delete()
test_session.commit()
- save_channels_config()
+ # Missing ChannelDownload is re-created on import.
+ import_channels_config()
channel2: Channel = test_session.query(Channel).filter_by(id=channel2.id).one()
cds = test_session.query(ChannelDownload).all()
assert len(cds) == 2, cds
- assert channel2.channel_downloads and len(channel2.channel_downloads) == 1
assert channel1.channel_downloads and len(channel1.channel_downloads) == 1
+ assert channel2.channel_downloads and len(channel2.channel_downloads) == 1
# Add a Download to Channel2 which does not match Channel.url.
channel2.get_or_create_download('https://example.org')
diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py
index 6e4743ad..1a2b7cb8 100644
--- a/modules/videos/test/test_lib.py
+++ b/modules/videos/test/test_lib.py
@@ -5,7 +5,8 @@ import pytest
from modules.videos import lib
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
-from modules.videos.models import Video
+from modules.videos.models import Video, ChannelDownload
+from wrolpi.downloader import Download, RSSDownloader
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.vars import PROJECT_DIR
@@ -148,3 +149,21 @@ async def test_orphaned_files(test_session, make_files_structure, test_directory
vid2_poster_path,
vid3_poster_path,
])
+
+
+def test_link_channel_and_downloads(test_session, channel_factory):
+ channel = channel_factory(
+ url='https://www.youtube.com/c/LearningSelfReliance/videos',
+ source_id='UCng5u6ASda3LNRXJN0JCQJA',
+ )
+ download1 = Download(url=channel.get_rss_url(), downloader=RSSDownloader.name)
+ download2 = Download(url='https://example.com', downloader=RSSDownloader.name,
+ settings=dict(destination=str(channel.directory)))
+ test_session.add_all([download1, download2])
+ test_session.commit()
+ assert test_session.query(Download).count() == 2
+ assert test_session.query(ChannelDownload).count() == 0
+
+ lib.link_channel_and_downloads(session=test_session)
+ assert test_session.query(Download).count() == 2
+ assert test_session.query(ChannelDownload).count() == 2
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index 41f7f163..f6f9029f 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -520,6 +520,16 @@ class DownloadManager:
download.settings = settings if settings is not None else download.settings
downloads.append(download)
+ # Create ChannelDownload for any download that is being downloaded into a Channel's Directory.
+ from modules.videos.models import Channel, ChannelDownload
+ if download.settings and (destination := download.settings.get('destination')):
+ channel = Channel.get_by_path(destination, session)
+ cd = ChannelDownload.get_by_url(url, session)
+ if channel and not cd:
+ cd = ChannelDownload(channel=channel, download=download)
+ session.add(cd)
+
+ session.flush(downloads)
try:
# Start downloading ASAP.
api_app.add_task(self.dispatch_downloads())
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index ce85b8d0..9b8e3db0 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -72,6 +72,8 @@ class DownloadSettings:
self.destination = str(get_media_directory() / self.destination) if self.destination else None
if self.suffix and not self.suffix.startswith('.'):
raise ValidationError('suffix must start with .')
+ self.title_exclude = self.title_exclude or None
+ self.title_include = self.title_include or None
@dataclass