rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 19:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Creating missing ChannelDownload when importing channels config.

Browse source
This commit is contained in:
Roland 2024-07-04 19:16:33 -06:00
parent 7a784d7c45
commit 287beabb9c
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
10 changed files with 135 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

6
app/src/components/Channels.js
View file

@ -47,7 +47,7 @@ import {Media, ThemeContext} from "../contexts/contexts";
import {SortableTable} from "./SortableTable";
import {toast} from "react-semantic-toasts-2";
import {RecurringDownloadsTable} from "./admin/Downloads";
import {TagsSelector, useTags} from "../Tags";
import {TagsSelector} from "../Tags";
function ChannelStatistics({statistics}) {
@ -416,10 +416,10 @@ export function ChannelDownloadForm({channel_id, afterSave, closeModal, download
const settings = download.settings ? download.settings : {};
const oldTagNames = download && download.settings && download.settings.tag_names ? download.settings.tag_names : [];
const [state, setState] = React.useState({
url: download ? download.url : '',
frequency: download.frequency ? download.frequency : 604800,
title_match: settings.title_match ? settings.title_match : '',
title_exclude: settings.title_exclude ? settings.title_exclude : '',
title_match: settings.title_match ? settings.title_match : '',
url: download ? download.url : '',
})
const [disabled, setDisabled] = React.useState(useWROLMode());
const [loading, setLoading] = React.useState(false);

10
app/src/components/admin/Downloads.js
View file

@ -117,6 +117,7 @@ export function DownloadEditForm({afterSave, closeModal, download}) {
url: download.url || '',
frequency: download.frequency,
excluded_urls: download.settings ? download.settings.excluded_urls : '',
destination: download.settings ? download.settings.destination : '',
})
const [disabled, setDisabled] = React.useState(useWROLMode());
const [loading, setLoading] = React.useState(false);
@ -220,6 +221,15 @@ export function DownloadEditForm({afterSave, closeModal, download}) {
onChange={handleInputChange}
/>
</Grid.Column>
<Grid.Column>
<FormInput
label='Destination'
name='destination'
type='text'
value={state.destination}
onChange={handleInputChange}
/>
</Grid.Column>
</Grid.Row>
<Grid.Row columns={1}>
<Grid.Column textAlign='right'>

3
modules/videos/channel/test/test_api.py
View file

@ -414,12 +414,13 @@ async def test_create_channel_download(test_async_client, test_session, simple_c
tag = tag_factory()
# Create ChannelDownload (which includes a Download).
body = {'url': 'https://example.com/1', 'frequency': 42}
body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
request, response = await test_async_client.post(
f'/api/videos/channels/{simple_channel.id}/download', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download: Download = test_session.query(Download).one()
assert download.url == 'https://example.com/1' and not download.settings.get('tag_names')
assert download.settings.get('title_include') is None
# Create another ChannelDownload. Previous ChannelDownload/Download are untouched.
body = {'url': 'https://example.com/2', 'frequency': 55, 'settings': {'tag_names': [tag.name]}}

18
modules/videos/channel/test/test_models.py
View file

@ -3,9 +3,10 @@ from http import HTTPStatus
import pytest
from modules.videos.downloader import VideoDownloader
from modules.videos.models import Channel, ChannelDownload
from modules.videos.models import Video
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.downloader import Download, DownloadFrequency, download_manager, RSSDownloader
def test_delete_channel_no_url(test_session, test_client, channel_factory):
@ -92,6 +93,7 @@ async def test_channel_download_relationships(test_session, download_channel):
assert channel.channel_downloads[0].download == download \
and channel.channel_downloads[0].download.url == 'https://example.com/channel1'
assert download.frequency == DownloadFrequency.weekly
assert download.settings['destination'] == str(channel.directory)
# Deleting Download deletes the ChannelDownload, but not the Channel.
download.delete()
@ -107,3 +109,17 @@ async def test_channel_download_relationships(test_session, download_channel):
download_channel.delete_with_videos()
assert test_session.query(Download).count() == 0
assert test_session.query(ChannelDownload).count() == 0
@pytest.mark.asyncio()
async def test_create_channel_download(test_session, simple_channel):
"""A ChannelDownload is created if a Download is created for a Channel's directory."""
test_session.flush()
settings = dict(destination=str(simple_channel.directory))
download_manager.create_download('https://example.com/2', RSSDownloader.name, test_session,
sub_downloader_name=VideoDownloader.name, settings=settings)
download = test_session.query(Download).one()
assert download.settings['destination'] == str(simple_channel.directory)
cd = test_session.query(ChannelDownload).one()
assert cd.channel == simple_channel

52
modules/videos/lib.py
View file

@ -359,20 +359,6 @@ def get_channels_config_from_db(session: Session) -> dict:
@optional_session()
def save_channels_config(session: Session = None):
"""Get the Channel information from the DB, save it to the config."""
# Create any missing ChannelDownload for any Channel.url that has a Download.
results = session.query(Channel, Download, ChannelDownload) \
.outerjoin(ChannelDownload, ChannelDownload.download_url == Download.url) \
.outerjoin(Channel, Channel.url == Download.url) \
.all()
need_commit = False
for channel, download, cd in results:
if channel and download and not cd:
# Channel.url matches Download.url but there is no ChannelDownload.
channel.get_or_create_download(download.url, session)
need_commit = True
if need_commit:
session.commit()
config = get_channels_config_from_db(session)
channels_config = get_channels_config()
@ -463,6 +449,9 @@ def import_channels_config():
logger.warning(f'Deleting {channel} because it is not in the config.')
channel.delete_with_videos()
# Create any missing ChannelDownload.
link_channel_and_downloads()
if save_config:
# Information about the channel was fetched, store it.
save_channels_config()
@ -474,6 +463,41 @@ def import_channels_config():
raise
@optional_session
def link_channel_and_downloads(session: Session):
"""Create any missing ChannelDownload for any Channel.url/Channel.directory that has a Download."""
downloads = session.query(Download).all()
downloads_by_url = {i.url: i for i in downloads}
downloads_by_destination = {i.settings['destination']: i
for i in downloads if 'destination' in (i.settings or dict())}
channels = session.query(Channel).all()
need_commit = False
for channel in channels:
directory = str(channel.directory)
download = downloads_by_destination.get(directory) or downloads_by_url.get(channel.url)
if download and not channel.get_download(download_id=download.id):
# Create ChannelDownload for the Download that is for this Channel.
cd = ChannelDownload(channel=channel, download=download)
session.add(cd)
channel_import_logger.info(f'Created missing ChannelDownload for {download}')
need_commit = True
# Get any Downloads for a Channel's RSS feed.
rss_url = channel.get_rss_url()
download = downloads_by_url.get(rss_url) if rss_url else None
if rss_url and download:
cd = channel.get_download(rss_url)
if not cd:
# No ChannelDownload for this RSS Download.
cd = ChannelDownload(channel=channel, download=download)
session.add(cd)
channel_import_logger.info(f'Created missing ChannelDownload for {download}')
if need_commit:
session.commit()
YDL = YoutubeDL(dict(cachedir=YTDLP_CACHE_DIR))
ydl_logger = YDL.params['logger'] = logger.getChild('youtube-dl')
YDL.add_default_info_extractors()

27
modules/videos/models.py
View file

@ -623,6 +623,10 @@ class Channel(ModelHelper, Base):
raise FileNotFoundError(f'{path} does not exist!')
return path
@staticmethod
def get_by_path(path: pathlib.Path, session: Session) -> Optional['Channel']:
return session.query(Channel).filter_by(directory=path).one_or_none()
def __json__(self) -> dict:
d = dict(
channel_downloads=self.channel_downloads,
@ -731,7 +735,8 @@ class Channel(ModelHelper, Base):
@optional_session
def get_or_create_download(self, url: str, session: Session = None, reset_attempts: bool = False) \
-> ChannelDownload:
"""Get a ChannelDownload record, if it does not exist, create it."""
"""Get a ChannelDownload record, if it does not exist, create it. Create a Download if necessary
which goes into this Channel's directory."""
from modules.videos.downloader import ChannelDownloader, VideoDownloader
if not url:
@ -743,10 +748,17 @@ class Channel(ModelHelper, Base):
# VideoDownloader.
download = session.query(Download).filter_by(url=url).one_or_none()
if not download:
download_manager.create_download(url, ChannelDownloader.name, session=session,
sub_downloader_name=VideoDownloader.name,
reset_attempts=reset_attempts,
)
download = download_manager.create_download(url, ChannelDownloader.name, session=session,
sub_downloader_name=VideoDownloader.name,
reset_attempts=reset_attempts,
)
# Download into this Channel's directory by default.
destination = (download.settings or dict()).get('destination')
if not destination:
settings = (download.settings or dict())
settings.update(dict(destination=str(self.directory)))
download.settings = settings
# Link between Channel and Download.
cd = ChannelDownload.get_by_url(url, session)
if not cd:
cd = ChannelDownload(channel_id=self.id, download_url=url)
@ -754,3 +766,8 @@ class Channel(ModelHelper, Base):
logger.debug(f'Created {cd} for {self}')
return cd
def get_rss_url(self) -> str | None:
"""Return the RSS Feed URL for this Channel, if any is possible."""
if self.url and self.source_id and 'youtube.com' in self.url:
return f'https://www.youtube.com/feeds/videos.xml?channel_id={self.source_id}'

18
modules/videos/test/test_common.py
View file

@ -154,7 +154,7 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto
import_channels_config()
assert not channel1.channel_downloads
assert len(test_session.query(Channel).all()) == 2
assert test_session.query(Download).all() == []
assert test_session.query(Download).count() == test_session.query(ChannelDownload).count() == 0
# Add a frequency to the Channel.
channels_config = get_channels_config()
@ -166,27 +166,31 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto
# Download record is created on import.
import_channels_config()
assert channel1.channel_downloads
assert len(test_session.query(Channel).all()) == 2
assert len(test_session.query(Channel).all()) == 2, 'Both Channels should still exist'
assert test_session.query(ChannelDownload).count() == 1, 'One Channel has a Download'
download: Download = test_session.query(Download).one()
assert download.url == channel1.url
assert download.frequency
# Download.next_download was not deleted.
next_download = str(download.next_download)
import_channels_config()
download: Download = test_session.query(Download).one()
assert next_download == str(download.next_download)
# Creating Download that matches Channel2's URL creates a ChannelDownload on save.
download = Download(url=channel2.url, frequency=DownloadFrequency.weekly)
test_session.add(download)
# Creating Download that matches Channel2's URL creates a ChannelDownload. Delete it and it should be re-created.
channel2.get_or_create_download(channel2.url, test_session)
save_channels_config()
test_session.query(ChannelDownload).filter_by(download_url=channel2.url).delete()
test_session.commit()
save_channels_config()
# Missing ChannelDownload is re-created on import.
import_channels_config()
channel2: Channel = test_session.query(Channel).filter_by(id=channel2.id).one()
cds = test_session.query(ChannelDownload).all()
assert len(cds) == 2, cds
assert channel2.channel_downloads and len(channel2.channel_downloads) == 1
assert channel1.channel_downloads and len(channel1.channel_downloads) == 1
assert channel2.channel_downloads and len(channel2.channel_downloads) == 1
# Add a Download to Channel2 which does not match Channel.url.
channel2.get_or_create_download('https://example.org')

21
modules/videos/test/test_lib.py
View file

@ -5,7 +5,8 @@ import pytest
from modules.videos import lib
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
from modules.videos.models import Video
from modules.videos.models import Video, ChannelDownload
from wrolpi.downloader import Download, RSSDownloader
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.vars import PROJECT_DIR
@ -148,3 +149,21 @@ async def test_orphaned_files(test_session, make_files_structure, test_directory
vid2_poster_path,
vid3_poster_path,
])
def test_link_channel_and_downloads(test_session, channel_factory):
channel = channel_factory(
url='https://www.youtube.com/c/LearningSelfReliance/videos',
source_id='UCng5u6ASda3LNRXJN0JCQJA',
)
download1 = Download(url=channel.get_rss_url(), downloader=RSSDownloader.name)
download2 = Download(url='https://example.com', downloader=RSSDownloader.name,
settings=dict(destination=str(channel.directory)))
test_session.add_all([download1, download2])
test_session.commit()
assert test_session.query(Download).count() == 2
assert test_session.query(ChannelDownload).count() == 0
lib.link_channel_and_downloads(session=test_session)
assert test_session.query(Download).count() == 2
assert test_session.query(ChannelDownload).count() == 2

10
wrolpi/downloader.py
View file

@ -520,6 +520,16 @@ class DownloadManager:
download.settings = settings if settings is not None else download.settings
downloads.append(download)
# Create ChannelDownload for any download that is being downloaded into a Channel's Directory.
from modules.videos.models import Channel, ChannelDownload
if download.settings and (destination := download.settings.get('destination')):
channel = Channel.get_by_path(destination, session)
cd = ChannelDownload.get_by_url(url, session)
if channel and not cd:
cd = ChannelDownload(channel=channel, download=download)
session.add(cd)
session.flush(downloads)
try:
# Start downloading ASAP.
api_app.add_task(self.dispatch_downloads())

2
wrolpi/schema.py
View file

@ -72,6 +72,8 @@ class DownloadSettings:
self.destination = str(get_media_directory() / self.destination) if self.destination else None
if self.suffix and not self.suffix.startswith('.'):
raise ValidationError('suffix must start with .')
self.title_exclude = self.title_exclude or None
self.title_include = self.title_include or None
@dataclass