Downloads without a frequency cannot be a Channel Download.

This commit is contained in:
Roland 2024-07-22 17:15:37 -06:00
parent 53d2402d32
commit 7933d9625c
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
3 changed files with 51 additions and 34 deletions

View file

@ -384,7 +384,7 @@ def import_channels_config():
logger.warning('Skipping import_channels_config for this test')
return
channel_import_logger.info('Importing videos config')
channel_import_logger.info('Importing channels config')
try:
config = get_channels_config()
channels = config.channels
@ -473,16 +473,23 @@ def import_channels_config():
def link_channel_and_downloads(session: Session):
"""Create any missing Downloads for any Channel.url/Channel.directory that has a Download. Associate any Download
related to a Channel."""
downloads = list(session.query(Download).all())
# Only Downloads with a frequency can be a Channel Download.
downloads = list(session.query(Download).filter(Download.frequency.isnot(None)).all())
# Download.url is unique and cannot be null.
downloads_by_url = {i.url: i for i in downloads}
downloads_by_destination = {i.settings['destination']: i
for i in downloads if 'destination' in (i.settings or dict())}
# Many Downloads may share the same destination.
downloads_with_destination = [i for i in downloads if (i.settings or dict()).get('destination')]
channels = session.query(Channel).all()
need_commit = False
for channel in channels:
directory = str(channel.directory)
download = downloads_by_destination.get(directory) or downloads_by_url.get(channel.url)
for download in downloads_with_destination:
if download.settings['destination'] == directory and not download.channel_id:
download.channel_id = channel.id
need_commit = True
download = downloads_by_url.get(channel.url)
if download and not download.channel_id:
download.channel_id = channel.id
need_commit = True
@ -491,6 +498,7 @@ def link_channel_and_downloads(session: Session):
rss_url = channel.get_rss_url()
if rss_url and (download := downloads_by_url.get(rss_url)):
download.channel_id = channel.id
need_commit = True
# Associate any Download which shares a Channel's URL.
for download in downloads:
@ -499,13 +507,6 @@ def link_channel_and_downloads(session: Session):
download.channel_id = channel.id
need_commit = True
# Associate any Download which shares a Channel's directory.
for destination, download in downloads_by_destination.items():
if destination and (channel := Channel.get_by_directory(destination)):
if not download.channel_id:
download.channel_id = channel.id
need_commit = True
if need_commit:
session.commit()

View file

@ -11,7 +11,7 @@ from modules.videos.errors import UnknownVideo, UnknownChannel
from wrolpi.captions import read_captions
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.downloader import Download, download_manager, DownloadFrequency
from wrolpi.downloader import Download, download_manager
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
from wrolpi.media_path import MediaPathType
@ -563,15 +563,20 @@ class Channel(ModelHelper, Base):
if isinstance(download, str):
url = download
elif isinstance(download, dict):
if not download.get('frequency'):
logger.error('Refusing to create channel download without frequency')
continue
url = download['url']
elif isinstance(download, Download):
if not download.frequency:
logger.error('Refusing to create channel download without frequency')
continue
url = download.url
if not url:
raise RuntimeError(f'Unknown download type: {download}')
download_ = self.get_or_create_download(url, session=session, reset_attempts=True)
# If `download` is dict, use that frequency. Try to keep the currently set frequency, finally 30 days.
frequency = download['frequency'] \
if isinstance(download, dict) else download_.frequency or DownloadFrequency.days30
# If `download` is dict, use that frequency. Try to keep the currently set frequency.
frequency = download['frequency'] if isinstance(download, dict) else download_.frequency
download_.frequency = frequency
session.flush()

View file

@ -4,9 +4,9 @@ import shutil
import pytest
from modules.videos import lib
from modules.videos.downloader import ChannelDownloader
from modules.videos.downloader import ChannelDownloader, VideoDownloader
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
from modules.videos.models import Video
from modules.videos.models import Video, Channel
from wrolpi.downloader import Download, RSSDownloader
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
@ -158,34 +158,39 @@ def test_link_channel_and_downloads(test_session, channel_factory, test_download
source_id='UCng5u6ASda3LNRXJN0JCQJA',
)
download1 = Download(url=channel.get_rss_url(), downloader=RSSDownloader.name)
download2 = Download(url='https://example.com', downloader=RSSDownloader.name,
download2 = Download(url='https://example.com/videos', downloader=RSSDownloader.name, frequency=99,
settings=dict(destination=str(channel.directory)))
test_session.add_all([download1, download2])
download3 = Download(url='https://example.com/video/1', downloader=RSSDownloader.name,
settings=dict(destination=str(channel.directory)))
test_session.add_all([download1, download2, download3])
test_session.commit()
assert test_session.query(Download).count() == 2
assert test_session.query(Download).count() == 3
assert not any(i.channel_id for i in test_download_manager.get_downloads(test_session))
# `link_channel_and_downloads` creates missing ChannelDownloads.
# `link_channel_and_downloads` creates missing Downloads.
lib.link_channel_and_downloads(session=test_session)
assert test_session.query(Download).count() == 2
assert all(i.channel_id for i in test_download_manager.get_downloads(test_session))
assert test_session.query(Download).count() == 3
assert all(i.channel_id for i in test_download_manager.get_recurring_downloads(test_session))
assert not any(i.channel_id for i in test_download_manager.get_once_downloads(test_session))
@pytest.mark.asyncio
def test_channel_channel_downloads_migration(test_async_client, test_session, channel_factory, test_download_manager):
def test_link_channel_and_downloads_migration(test_async_client, test_session, channel_factory, test_download_manager):
"""Test the 8d0d81bc9c34_channel_channel_downloads.py migration."""
# A simple Channel which has a download for its URL.
channel1 = channel_factory(url='https://example.com/channel1')
# A Channel which has two Downloads. One for it's URL, another which is an RSS feed in its directory.
channel2 = channel_factory(url='https://example.com/channel2')
download1 = Download(url=channel1.url, downloader=ChannelDownloader.name)
download2a = Download(url='https://example.com/channel2/rss', downloader=RSSDownloader.name,
download1 = Download(url='https://example.com/channel1', downloader=ChannelDownloader.name, frequency=1)
download2a = Download(url='https://example.com/channel2/rss', downloader=RSSDownloader.name, frequency=1,
settings=dict(destination=str(channel2.directory)))
download2b = Download(url='https://example.com/channel2', downloader=ChannelDownloader.name,
download2b = Download(url='https://example.com/channel2', downloader=ChannelDownloader.name, frequency=1,
settings=dict(destination=str(channel2.directory)))
# Download does not have a frequency, so it cannot be a channel download.
download2c = Download(url='https://example.com/channel2/video/1', downloader=VideoDownloader.name)
# A Channel which has a URL, but no Downloads.
channel3 = channel_factory()
test_session.add_all([download1, download2a, download2b])
test_session.add_all([download1, download2a, download2b, download2c])
test_session.commit()
assert not channel1.downloads
@ -198,8 +203,14 @@ def test_channel_channel_downloads_migration(test_async_client, test_session, ch
assert channel2.downloads
assert not channel3.downloads
assert test_session.query(Download).count() == 3
cd1, cd2a, cd2b = test_session.query(Download).order_by(Download.url).all()
assert cd1.url == channel1.url
assert cd2a.url == 'https://example.com/channel2'
assert cd2b.url == 'https://example.com/channel2/rss'
assert test_session.query(Channel).count() == 3
assert test_session.query(Download).count() == 4
d1, d2a, d2b, d2c = test_session.query(Download).order_by(Download.url).all()
assert d1.url == channel1.url == 'https://example.com/channel1' and d1.frequency == 1
assert d1.channel_id
assert d2a.url == 'https://example.com/channel2' and d2a.frequency == 1
assert d2a.channel_id
assert d2b.url == 'https://example.com/channel2/rss' and d2b.frequency == 1
assert d2b.channel_id
assert d2c.url == 'https://example.com/channel2/video/1' and d2c.frequency is None
assert not d2c.channel_id