mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
Downloads without a frequency cannot be a Channel Download.
This commit is contained in:
parent
53d2402d32
commit
7933d9625c
3 changed files with 51 additions and 34 deletions
|
|
@ -384,7 +384,7 @@ def import_channels_config():
|
|||
logger.warning('Skipping import_channels_config for this test')
|
||||
return
|
||||
|
||||
channel_import_logger.info('Importing videos config')
|
||||
channel_import_logger.info('Importing channels config')
|
||||
try:
|
||||
config = get_channels_config()
|
||||
channels = config.channels
|
||||
|
|
@ -473,16 +473,23 @@ def import_channels_config():
|
|||
def link_channel_and_downloads(session: Session):
|
||||
"""Create any missing Downloads for any Channel.url/Channel.directory that has a Download. Associate any Download
|
||||
related to a Channel."""
|
||||
downloads = list(session.query(Download).all())
|
||||
# Only Downloads with a frequency can be a Channel Download.
|
||||
downloads = list(session.query(Download).filter(Download.frequency.isnot(None)).all())
|
||||
# Download.url is unique and cannot be null.
|
||||
downloads_by_url = {i.url: i for i in downloads}
|
||||
downloads_by_destination = {i.settings['destination']: i
|
||||
for i in downloads if 'destination' in (i.settings or dict())}
|
||||
# Many Downloads may share the same destination.
|
||||
downloads_with_destination = [i for i in downloads if (i.settings or dict()).get('destination')]
|
||||
channels = session.query(Channel).all()
|
||||
|
||||
need_commit = False
|
||||
for channel in channels:
|
||||
directory = str(channel.directory)
|
||||
download = downloads_by_destination.get(directory) or downloads_by_url.get(channel.url)
|
||||
for download in downloads_with_destination:
|
||||
if download.settings['destination'] == directory and not download.channel_id:
|
||||
download.channel_id = channel.id
|
||||
need_commit = True
|
||||
|
||||
download = downloads_by_url.get(channel.url)
|
||||
if download and not download.channel_id:
|
||||
download.channel_id = channel.id
|
||||
need_commit = True
|
||||
|
|
@ -491,6 +498,7 @@ def link_channel_and_downloads(session: Session):
|
|||
rss_url = channel.get_rss_url()
|
||||
if rss_url and (download := downloads_by_url.get(rss_url)):
|
||||
download.channel_id = channel.id
|
||||
need_commit = True
|
||||
|
||||
# Associate any Download which shares a Channel's URL.
|
||||
for download in downloads:
|
||||
|
|
@ -499,13 +507,6 @@ def link_channel_and_downloads(session: Session):
|
|||
download.channel_id = channel.id
|
||||
need_commit = True
|
||||
|
||||
# Associate any Download which shares a Channel's directory.
|
||||
for destination, download in downloads_by_destination.items():
|
||||
if destination and (channel := Channel.get_by_directory(destination)):
|
||||
if not download.channel_id:
|
||||
download.channel_id = channel.id
|
||||
need_commit = True
|
||||
|
||||
if need_commit:
|
||||
session.commit()
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from modules.videos.errors import UnknownVideo, UnknownChannel
|
|||
from wrolpi.captions import read_captions
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, background_task, replace_file
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.downloader import Download, download_manager, DownloadFrequency
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
|
@ -563,15 +563,20 @@ class Channel(ModelHelper, Base):
|
|||
if isinstance(download, str):
|
||||
url = download
|
||||
elif isinstance(download, dict):
|
||||
if not download.get('frequency'):
|
||||
logger.error('Refusing to create channel download without frequency')
|
||||
continue
|
||||
url = download['url']
|
||||
elif isinstance(download, Download):
|
||||
if not download.frequency:
|
||||
logger.error('Refusing to create channel download without frequency')
|
||||
continue
|
||||
url = download.url
|
||||
if not url:
|
||||
raise RuntimeError(f'Unknown download type: {download}')
|
||||
download_ = self.get_or_create_download(url, session=session, reset_attempts=True)
|
||||
# If `download` is dict, use that frequency. Try to keep the currently set frequency, finally 30 days.
|
||||
frequency = download['frequency'] \
|
||||
if isinstance(download, dict) else download_.frequency or DownloadFrequency.days30
|
||||
# If `download` is dict, use that frequency. Try to keep the currently set frequency.
|
||||
frequency = download['frequency'] if isinstance(download, dict) else download_.frequency
|
||||
download_.frequency = frequency
|
||||
|
||||
session.flush()
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ import shutil
|
|||
import pytest
|
||||
|
||||
from modules.videos import lib
|
||||
from modules.videos.downloader import ChannelDownloader
|
||||
from modules.videos.downloader import ChannelDownloader, VideoDownloader
|
||||
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
|
||||
from modules.videos.models import Video
|
||||
from modules.videos.models import Video, Channel
|
||||
from wrolpi.downloader import Download, RSSDownloader
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
|
@ -158,34 +158,39 @@ def test_link_channel_and_downloads(test_session, channel_factory, test_download
|
|||
source_id='UCng5u6ASda3LNRXJN0JCQJA',
|
||||
)
|
||||
download1 = Download(url=channel.get_rss_url(), downloader=RSSDownloader.name)
|
||||
download2 = Download(url='https://example.com', downloader=RSSDownloader.name,
|
||||
download2 = Download(url='https://example.com/videos', downloader=RSSDownloader.name, frequency=99,
|
||||
settings=dict(destination=str(channel.directory)))
|
||||
test_session.add_all([download1, download2])
|
||||
download3 = Download(url='https://example.com/video/1', downloader=RSSDownloader.name,
|
||||
settings=dict(destination=str(channel.directory)))
|
||||
test_session.add_all([download1, download2, download3])
|
||||
test_session.commit()
|
||||
assert test_session.query(Download).count() == 2
|
||||
assert test_session.query(Download).count() == 3
|
||||
assert not any(i.channel_id for i in test_download_manager.get_downloads(test_session))
|
||||
|
||||
# `link_channel_and_downloads` creates missing ChannelDownloads.
|
||||
# `link_channel_and_downloads` creates missing Downloads.
|
||||
lib.link_channel_and_downloads(session=test_session)
|
||||
assert test_session.query(Download).count() == 2
|
||||
assert all(i.channel_id for i in test_download_manager.get_downloads(test_session))
|
||||
assert test_session.query(Download).count() == 3
|
||||
assert all(i.channel_id for i in test_download_manager.get_recurring_downloads(test_session))
|
||||
assert not any(i.channel_id for i in test_download_manager.get_once_downloads(test_session))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
def test_channel_channel_downloads_migration(test_async_client, test_session, channel_factory, test_download_manager):
|
||||
def test_link_channel_and_downloads_migration(test_async_client, test_session, channel_factory, test_download_manager):
|
||||
"""Test the 8d0d81bc9c34_channel_channel_downloads.py migration."""
|
||||
# A simple Channel which has a download for its URL.
|
||||
channel1 = channel_factory(url='https://example.com/channel1')
|
||||
# A Channel which has two Downloads. One for it's URL, another which is an RSS feed in its directory.
|
||||
channel2 = channel_factory(url='https://example.com/channel2')
|
||||
download1 = Download(url=channel1.url, downloader=ChannelDownloader.name)
|
||||
download2a = Download(url='https://example.com/channel2/rss', downloader=RSSDownloader.name,
|
||||
download1 = Download(url='https://example.com/channel1', downloader=ChannelDownloader.name, frequency=1)
|
||||
download2a = Download(url='https://example.com/channel2/rss', downloader=RSSDownloader.name, frequency=1,
|
||||
settings=dict(destination=str(channel2.directory)))
|
||||
download2b = Download(url='https://example.com/channel2', downloader=ChannelDownloader.name,
|
||||
download2b = Download(url='https://example.com/channel2', downloader=ChannelDownloader.name, frequency=1,
|
||||
settings=dict(destination=str(channel2.directory)))
|
||||
# Download does not have a frequency, so it cannot be a channel download.
|
||||
download2c = Download(url='https://example.com/channel2/video/1', downloader=VideoDownloader.name)
|
||||
# A Channel which has a URL, but no Downloads.
|
||||
channel3 = channel_factory()
|
||||
test_session.add_all([download1, download2a, download2b])
|
||||
test_session.add_all([download1, download2a, download2b, download2c])
|
||||
test_session.commit()
|
||||
|
||||
assert not channel1.downloads
|
||||
|
|
@ -198,8 +203,14 @@ def test_channel_channel_downloads_migration(test_async_client, test_session, ch
|
|||
assert channel2.downloads
|
||||
assert not channel3.downloads
|
||||
|
||||
assert test_session.query(Download).count() == 3
|
||||
cd1, cd2a, cd2b = test_session.query(Download).order_by(Download.url).all()
|
||||
assert cd1.url == channel1.url
|
||||
assert cd2a.url == 'https://example.com/channel2'
|
||||
assert cd2b.url == 'https://example.com/channel2/rss'
|
||||
assert test_session.query(Channel).count() == 3
|
||||
assert test_session.query(Download).count() == 4
|
||||
d1, d2a, d2b, d2c = test_session.query(Download).order_by(Download.url).all()
|
||||
assert d1.url == channel1.url == 'https://example.com/channel1' and d1.frequency == 1
|
||||
assert d1.channel_id
|
||||
assert d2a.url == 'https://example.com/channel2' and d2a.frequency == 1
|
||||
assert d2a.channel_id
|
||||
assert d2b.url == 'https://example.com/channel2/rss' and d2b.frequency == 1
|
||||
assert d2b.channel_id
|
||||
assert d2c.url == 'https://example.com/channel2/video/1' and d2c.frequency is None
|
||||
assert not d2c.channel_id
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue