Removing ChannelDownload in favor of Download.channel_id.

This commit is contained in:
Roland 2024-07-20 15:35:16 -06:00
parent 958dbb899d
commit 63738bd70f
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
17 changed files with 290 additions and 364 deletions

View file

@ -1,4 +1,4 @@
"""Channel.channel_downloads
"""Channel.downloads / Download.channel
Revision ID: 8d0d81bc9c34
Revises: 00f11f309c53
@ -11,6 +11,8 @@ from alembic import op
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from modules.videos.lib import link_channel_and_downloads
# revision identifiers, used by Alembic.
revision = '8d0d81bc9c34'
down_revision = '00f11f309c53'
@ -27,29 +29,21 @@ def upgrade():
session = Session(bind=bind)
session.execute('ALTER TABLE download ADD CONSTRAINT download_url_unique UNIQUE (url)')
session.execute('''
CREATE TABLE channel_download (
channel_id INTEGER REFERENCES channel (id),
download_url TEXT REFERENCES download (url),
primary key (channel_id, download_url)
)''')
session.execute('ALTER TABLE download ADD COLUMN channel_id INTEGER REFERENCES channel(id)')
# Migration is separated for testing. See `test_channel_channel_downloads_migration`
from wrolpi.migration import migrate_channel_downloads
migrate_channel_downloads(session)
link_channel_and_downloads(session)
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS match_regex')
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS download_frequency')
if not DOCKERIZED:
session.execute('ALTER TABLE public.channel_download OWNER TO wrolpi')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('DROP TABLE IF EXISTS channel_download')
session.execute('ALTER TABLE download DROP COLUMN IF EXISTS channel_id')
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS channel_id')
session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS match_regex TEXT')
session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS download_frequency INTEGER')
session.execute('ALTER TABLE download DROP CONSTRAINT IF EXISTS download_url_unique')

View file

@ -240,7 +240,7 @@ function ChannelPage({create, header}) {
}
}
let channelDownloads = channel && channel.channel_downloads ? channel.channel_downloads : null;
let downloads = channel && channel.downloads ? channel.downloads : null;
const afterNewDownloadSave = async () => {
await fetchChannel();
@ -399,7 +399,7 @@ function ChannelPage({create, header}) {
</ModalContent>
</Modal>
<RecurringDownloadsTable downloads={channelDownloads} fetchDownloads={fetchChannel}/>
<RecurringDownloadsTable downloads={downloads} fetchDownloads={fetchChannel}/>
</Segment>}
<div style={{marginTop: '2em'}}>

View file

@ -252,17 +252,6 @@ class RecurringDownloadRow extends React.Component {
}
}
handleDelete = async () => {
const {id} = this.props;
try {
await deleteDownload(id);
} finally {
if (this.props.fetchDownloads) {
this.props.fetchDownloads();
}
}
}
handleRestart = async () => {
const {id} = this.props;
try {
@ -295,7 +284,6 @@ class RecurringDownloadRow extends React.Component {
next_download,
error,
downloader,
channel_id
} = this.props;
const {errorModalOpen} = this.state;
@ -319,17 +307,6 @@ class RecurringDownloadRow extends React.Component {
</ModalActions>
</Modal>;
const deleteButton = <>
<APIButton
color='red'
icon='trash'
confirmContent='Are you sure you want to delete this download?'
confirmButton='Delete'
onClick={this.handleDelete}
obeyWROLMode={true}
/>
</>;
const editButton = <>
<Button icon='edit' onClick={() => this.setState({editModalOpen: true})}/>
</>

View file

@ -8,12 +8,13 @@ from sqlalchemy.orm.exc import NoResultFound
from wrolpi.common import run_after, logger, \
get_media_directory
from wrolpi.db import get_db_curs, optional_session, get_db_session
from wrolpi.errors import UnknownDirectory, APIError, ValidationError, InvalidDownload
from wrolpi.downloader import save_downloads_config, download_manager
from wrolpi.errors import UnknownDirectory, APIError, ValidationError
from .. import schema
from ..common import check_for_channel_conflicts
from ..errors import UnknownChannel
from ..lib import save_channels_config
from ..models import Channel, Video, ChannelDownload
from ..models import Channel, Video
logger = logger.getChild(__name__)
@ -33,8 +34,7 @@ async def get_minimal_channels() -> List[dict]:
(
select min(d.frequency)
from download d
left join channel_download cd on cd.download_url = d.url
where cd.channel_id = c.id
where d.channel_id = c.id
) AS minimum_frequency
FROM
channel AS c
@ -217,32 +217,33 @@ async def search_channels_by_name(name: str, limit: int = 5, session: Session =
return channels
async def create_channel_download(channel_id: int, url: str, frequency: int, settings: dict) -> ChannelDownload:
"""Create ChannelDownload."""
async def create_channel_download(channel_id: int, url: str, frequency: int, settings: dict):
"""Create Download for Channel."""
with get_db_session(commit=True) as session:
channel = get_channel(session, channel_id=channel_id, return_dict=False)
cd = channel.get_or_create_download(url, session, reset_attempts=True)
cd.download.frequency = cd.download.frequency or frequency
cd.download.settings = settings
channel = Channel.find_by_id(channel_id, session=session)
download = channel.get_or_create_download(url, session, reset_attempts=True)
download.frequency = download.frequency or frequency
download.settings = settings
save_channels_config()
await save_downloads_config()
return cd
return download
async def update_channel_download(channel_id: int, download_id: int, url: str, frequency: int, settings: dict) \
-> ChannelDownload:
"""Fetch ChannelDownload, update its properties."""
async def update_channel_download(channel_id: int, download_id: int, url: str, frequency: int, settings: dict):
"""Fetch Channel's Download, update its properties."""
with get_db_session(commit=True) as session:
channel = get_channel(session, channel_id=channel_id, return_dict=False)
cd = channel.get_download(download_id=download_id)
if not cd:
raise InvalidDownload('No Download with that ID on this Channel')
# Ensure that the Channel exists.
Channel.find_by_id(channel_id, session=session)
download = download_manager.get_download(session, id_=download_id)
download.url = url
download.frequency = download.frequency or frequency
download.settings = settings
cd.download.url = url
cd.download.frequency = cd.download.frequency or frequency
cd.download.settings = settings
download_manager.remove_from_skip_list(url)
save_channels_config()
await save_downloads_config()
return cd
return download

View file

@ -9,7 +9,7 @@ import mock
import pytest
from modules.videos.downloader import ChannelDownloader
from modules.videos.models import Channel, ChannelDownload
from modules.videos.models import Channel
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import download_manager, Download, DownloadResult
@ -58,29 +58,31 @@ def test_channel_no_download_frequency(test_client, test_session, test_directory
# Get the Channel
request, response = test_client.get('/api/videos/channels/1')
assert response.status_code == HTTPStatus.OK
channel = response.json['channel']
assert not channel['channel_downloads']
assert not channel['downloads']
# Update the Channel with a frequency.
new_download = {'urls': [simple_channel.url], 'frequency': 10, 'downloader': ChannelDownloader.name}
request, response = test_client.post('/api/download', json=new_download)
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
# Download is scheduled. ChannelDownload was created.
# Download is scheduled. Download is related to Channel.
assert len(download_manager.get_downloads(test_session)) == 1
request, response = test_client.get('/api/videos/channels/1')
channel = response.json['channel']
assert channel['channel_downloads']
channel_downloads = channel['channel_downloads']
assert len(channel_downloads) == 1 and channel_downloads[0]['url'] == simple_channel.url
assert test_session.query(ChannelDownload).one()
download_id = channel['channel_downloads'][0]['id']
assert channel['downloads']
downloads = channel['downloads']
assert len(downloads) == 1 and downloads[0]['url'] == simple_channel.url
assert test_session.query(Download).one()
download_id = channel['downloads'][0]['id']
# Deleting the Download deletes the ChannelDownload.
# Deleting Download does not delete Channel.
request, response = test_client.delete(f'/api/download/{download_id}')
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
assert not download_manager.get_downloads(test_session)
assert not test_session.query(ChannelDownload).all()
assert not test_session.query(Download).all()
assert test_session.query(Channel).count() == 1
def test_video_file_name(test_session, simple_video, test_client):
@ -188,7 +190,7 @@ async def test_channel_download_requires_refresh(
Videos already downloaded are not downloaded again."""
vid = video_factory(channel_id=download_channel.id, with_video_file=True, with_poster_ext='jpg')
vid.file_group.url = 'https://example.com/1'
d = download_channel.channel_downloads[0].download
d = download_channel.downloads[0]
d.next_download = None
test_session.commit()
assert not d.next_download
@ -413,7 +415,7 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto
async def test_create_channel_download(test_async_client, test_session, simple_channel, tag_factory):
tag = tag_factory()
# Create ChannelDownload (which includes a Download).
# Create Download.
body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
request, response = await test_async_client.post(
f'/api/videos/channels/{simple_channel.id}/download', json=body)
@ -422,7 +424,7 @@ async def test_create_channel_download(test_async_client, test_session, simple_c
assert download.url == 'https://example.com/1' and not download.settings.get('tag_names')
assert download.settings.get('title_include') is None
# Create another ChannelDownload. Previous ChannelDownload/Download are untouched.
# Create another Download. Previous Downloads is untouched.
body = {'url': 'https://example.com/2', 'frequency': 55, 'settings': {'tag_names': [tag.name]}}
request, response = await test_async_client.post(
f'/api/videos/channels/{simple_channel.id}/download', json=body)
@ -431,9 +433,9 @@ async def test_create_channel_download(test_async_client, test_session, simple_c
assert download1.url == 'https://example.com/1' and not download1.settings.get('tag_names')
assert download2.url == 'https://example.com/2' and download2.settings.get('tag_names') == [tag.name, ]
test_session.flush(simple_channel)
assert len(simple_channel.channel_downloads) == 2
assert len(simple_channel.downloads) == 2
# Change frequency of first ChannelDownload.
# Change frequency of first Download.
body = {'url': 'https://example.com/1', 'frequency': 123, 'settings': {'title_include': 'foo,bar'}}
request, response = await test_async_client.put(
f'/api/videos/channels/{simple_channel.id}/download/{download1.id}', json=body)
@ -443,10 +445,10 @@ async def test_create_channel_download(test_async_client, test_session, simple_c
assert download1.settings == {'title_include': 'foo,bar'}
assert download2.url == 'https://example.com/2' and download2.settings.get('tag_names') == [tag.name, ]
test_session.flush(simple_channel)
assert len(simple_channel.channel_downloads) == 2
assert len(simple_channel.downloads) == 2
# Deleting Download delets ChannelDownload.
# Deleting Download does not delete Channel.
request, response = await test_async_client.delete(f'/api/download/{download1.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(ChannelDownload).count() == 1
assert test_session.query(Download).count() == 1
assert test_session.query(Channel).count() == 1

View file

@ -4,7 +4,7 @@ from http import HTTPStatus
import pytest
from modules.videos.downloader import VideoDownloader
from modules.videos.models import Channel, ChannelDownload
from modules.videos.models import Channel
from modules.videos.models import Video
from wrolpi.downloader import Download, DownloadFrequency, download_manager, RSSDownloader
@ -73,47 +73,40 @@ async def test_nested_channel_directories(test_session, test_async_client, test_
@pytest.mark.asyncio()
async def test_channel_download_relationships(test_session, download_channel):
"""Test relationships of ChannelDownload."""
"""Test relationships of Channel and Download."""
test_session.flush()
download = test_session.query(Download).one()
cd = test_session.query(ChannelDownload).one()
channel = test_session.query(Channel).one()
assert len(channel.channel_downloads) == 1
assert channel.channel_downloads[0] == cd
assert channel.channel_downloads[0].download == download
assert len(channel.downloads) == 1
assert channel.downloads
assert download in channel.downloads
test_session.commit()
download = test_session.query(Download).one()
cd = test_session.query(ChannelDownload).one()
channel = test_session.query(Channel).one()
assert len(channel.channel_downloads) == 1
assert channel.channel_downloads[0] == cd
assert channel.channel_downloads[0].download == download \
and channel.channel_downloads[0].download.url == 'https://example.com/channel1'
download: Download = test_session.query(Download).one()
channel: Channel = test_session.query(Channel).one()
assert len(channel.downloads) == 1
assert channel.downloads[0] == download and channel.downloads[0].url == 'https://example.com/channel1'
assert download.frequency == DownloadFrequency.weekly
assert download.settings['destination'] == str(channel.directory)
# Deleting Download deletes the ChannelDownload, but not the Channel.
# Deleting Download deletes the Download, but not the Channel.
download.delete()
assert not test_session.query(Download).all()
assert not test_session.query(ChannelDownload).all()
assert test_session.query(Channel).one(), 'Channel should not have been deleted.'
# Create a Download again.
download_channel.get_or_create_download('https://example.com/1', test_session, reset_attempts=True)
assert test_session.query(Download).count() == 1
assert test_session.query(ChannelDownload).count() == 1
# Delete the Channel, and any relationships.
download_channel.delete_with_videos()
assert test_session.query(Download).count() == 0
assert test_session.query(ChannelDownload).count() == 0
@pytest.mark.asyncio()
async def test_create_channel_download(test_session, simple_channel):
"""A ChannelDownload is created if a Download is created for a Channel's directory."""
"""A Download will be related to a Channel if a Download is created in the Channel's directory"""
test_session.flush()
settings = dict(destination=str(simple_channel.directory))
@ -121,5 +114,5 @@ async def test_create_channel_download(test_session, simple_channel):
sub_downloader_name=VideoDownloader.name, settings=settings)
download = test_session.query(Download).one()
assert download.settings['destination'] == str(simple_channel.directory)
cd = test_session.query(ChannelDownload).one()
assert cd.channel == simple_channel
download = test_session.query(Download).one()
assert download.channel == simple_channel

View file

@ -34,7 +34,7 @@ def simple_channel(test_session, test_directory) -> Channel:
@pytest.fixture
def channel_factory(test_session, test_directory):
"""Create a random Channel with a directory, ChannelDownload, and Download."""
"""Create a random Channel with a directory, and Download."""
def factory(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None,
directory: pathlib.Path = None):
@ -52,10 +52,10 @@ def channel_factory(test_session, test_directory):
test_session.flush([channel])
assert channel.id and channel.url
if download_frequency:
cd = channel.get_or_create_download(channel.url)
cd.download.frequency = download_frequency
assert cd and cd.download_url == channel.url
assert channel.channel_downloads
download = channel.get_or_create_download(channel.url)
download.frequency = download_frequency
assert download.url == channel.url
assert channel.downloads
test_session.commit()
return channel
@ -71,9 +71,9 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch
test_session.add(channel)
test_session.flush([channel, ])
assert channel and channel.id and channel.url
cd = channel.get_or_create_download(channel.url, test_session)
assert cd.download_url == channel.url
cd.download.frequency = DownloadFrequency.weekly
download = channel.get_or_create_download(channel.url, test_session)
assert download.url == channel.url
download.frequency = DownloadFrequency.weekly
test_session.commit()
return channel

View file

@ -22,7 +22,7 @@ from .common import is_valid_poster, convert_image, \
generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
get_video_duration
from .errors import UnknownChannel
from .models import Channel, Video, ChannelDownload
from .models import Channel, Video
logger = logger.getChild(__name__)
@ -455,7 +455,7 @@ def import_channels_config():
logger.warning(f'Deleting {channel} because it is not in the config.')
channel.delete_with_videos()
# Create any missing ChannelDownload.
# Create any missing Downloads. Associated Downloads with any necessary Channels.
link_channel_and_downloads()
if save_config:
@ -471,8 +471,9 @@ def import_channels_config():
@optional_session
def link_channel_and_downloads(session: Session):
"""Create any missing ChannelDownload for any Channel.url/Channel.directory that has a Download."""
downloads = session.query(Download).all()
"""Create any missing Downloads for any Channel.url/Channel.directory that has a Download. Associate any Download
related to a Channel."""
downloads = list(session.query(Download).all())
downloads_by_url = {i.url: i for i in downloads}
downloads_by_destination = {i.settings['destination']: i
for i in downloads if 'destination' in (i.settings or dict())}
@ -482,23 +483,29 @@ def link_channel_and_downloads(session: Session):
for channel in channels:
directory = str(channel.directory)
download = downloads_by_destination.get(directory) or downloads_by_url.get(channel.url)
if download and not channel.get_download(download_id=download.id):
# Create ChannelDownload for the Download that is for this Channel.
cd = ChannelDownload(channel=channel, download=download)
session.add(cd)
channel_import_logger.info(f'Created missing ChannelDownload for {download}')
if download and not download.channel_id:
download.channel_id = channel.id
need_commit = True
# Get any Downloads for a Channel's RSS feed.
rss_url = channel.get_rss_url()
download = downloads_by_url.get(rss_url) if rss_url else None
if rss_url and download:
cd = channel.get_download(rss_url)
if not cd:
# No ChannelDownload for this RSS Download.
cd = ChannelDownload(channel=channel, download=download)
session.add(cd)
channel_import_logger.info(f'Created missing ChannelDownload for {download}')
if download:
download.channel_id = channel.id
# Associate any Download which shares a Channel's URL.
for download in downloads:
channel = Channel.get_by_url(download.url)
if channel and not download.channel_id:
download.channel_id = channel.id
need_commit = True
# Associate any Download which shares a Channel's directory.
for destination, download in downloads_by_destination.items():
if destination and (channel := Channel.get_by_directory(destination)):
if not download.channel_id:
download.channel_id = channel.id
need_commit = True
if need_commit:
session.commit()

View file

@ -20,7 +20,7 @@ from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
__all__ = ['Video', 'Channel', 'ChannelDownload']
__all__ = ['Video', 'Channel']
class Video(ModelHelper, Base):
@ -496,43 +496,11 @@ class Video(ModelHelper, Base):
self.file_group.append_files(info_json_path)
class ChannelDownload(Base):
"""One-to-many join table between Channel and Download. Expects to link using Download.url."""
__tablename__ = 'channel_download'
channel_id = Column(Integer, ForeignKey('channel.id'), primary_key=True)
channel = relationship('Channel', primaryjoin='ChannelDownload.channel_id==Channel.id',
back_populates='channel_downloads')
download_url = Column(String, ForeignKey('download.url'), primary_key=True)
download = relationship('Download', primaryjoin='ChannelDownload.download_url==Download.url')
def __repr__(self):
return f'<ChannelDownload url={self.download_url} channel_id={self.channel_id}>'
def __json__(self) -> dict:
d = self.download.__json__()
d['channel_id'] = self.channel_id
return d
@staticmethod
def get_by_id(id: int, session: Session) -> Optional['ChannelDownload']:
return session.query(ChannelDownload).filter(ChannelDownload.channel_id == id).one_or_none()
@staticmethod
def get_by_url(url: str, session: Session) -> Optional['ChannelDownload']:
return session.query(ChannelDownload).filter(ChannelDownload.download_url == url).one_or_none()
def delete(self):
session = Session.object_session(self)
if self.download:
self.download.delete(skip=False)
session.delete(self)
class Channel(ModelHelper, Base):
__tablename__ = 'channel'
id = Column(Integer, primary_key=True)
name = Column(String)
url = Column(String, unique=True) # will only be downloaded if ChannelDownload exists.
url = Column(String, unique=True) # will only be downloaded if related Download exists.
directory: pathlib.Path = Column(MediaPathType)
generate_posters = Column(Boolean, default=False) # generating posters may delete files, and can be slow.
calculate_duration = Column(Boolean, default=True) # use ffmpeg to extract duration (slower than info json).
@ -543,8 +511,7 @@ class Channel(ModelHelper, Base):
info_date = Column(Date)
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
channel_downloads: InstrumentedList = \
relationship('ChannelDownload', primaryjoin='ChannelDownload.channel_id==Channel.id')
downloads: InstrumentedList = relationship('Download', primaryjoin='Download.channel_id==Channel.id')
def __repr__(self):
return f'<Channel id={self.id} name={repr(self.name)} url={self.url} directory={self.directory}>'
@ -558,13 +525,13 @@ class Channel(ModelHelper, Base):
for video in videos:
video.channel = None
for cd in self.channel_downloads:
cd: ChannelDownload
cd.delete()
for download in self.downloads:
download: Download
download.delete()
# Must commit before final deletion.
# TODO why?
session.commit()
# session.commit()
session.delete(self)
def update(self, data: dict):
@ -577,7 +544,7 @@ class Channel(ModelHelper, Base):
if 'url' in data:
data['url'] = data['url'] or None
channel_downloads = data.pop('channel_downloads', [])
downloads = data.pop('downloads', [])
for key, value in data.items():
setattr(self, key, value)
@ -590,15 +557,22 @@ class Channel(ModelHelper, Base):
session: Session = Session.object_session(self)
for cd in channel_downloads:
# `cd` may be string from config file.
url = cd['url'] if isinstance(cd, dict) else cd
cd_ = ChannelDownload.get_by_url(url, session=session)
if not cd_:
cd_ = self.get_or_create_download(url, session=session)
# If `cd` is dict, use that frequency. Try to keep the currently set frequency, finally 30 days.
frequency = cd['frequency'] if isinstance(cd, dict) else cd_.download.frequency or DownloadFrequency.days30
cd_.download.frequency = frequency
for download in downloads:
# `download` may be string from config file.
url = None
if isinstance(download, str):
url = download
elif isinstance(download, dict):
url = download['url']
elif isinstance(download, Download):
url = download.url
if not url:
raise RuntimeError(f'Unknown download type: {download}')
download_ = self.get_or_create_download(url, session=session, reset_attempts=True)
# If `download` is dict, use that frequency. Try to keep the currently set frequency, finally 30 days.
frequency = download['frequency'] \
if isinstance(download, dict) else download_.frequency or DownloadFrequency.days30
download_.frequency = frequency
session.flush()
@ -613,7 +587,7 @@ class Channel(ModelHelper, Base):
name=self.name,
source_id=self.source_id,
url=self.url or None,
channel_downloads=[i.download_url for i in self.channel_downloads]
downloads=[i.url for i in self.downloads]
)
return config
@ -629,7 +603,7 @@ class Channel(ModelHelper, Base):
def __json__(self) -> dict:
d = dict(
channel_downloads=self.channel_downloads,
downloads=self.downloads,
directory=self.directory,
id=self.id,
name=self.name,
@ -643,7 +617,7 @@ class Channel(ModelHelper, Base):
if with_statistics:
d['statistics'] = self.get_statistics()
if with_downloads:
d['channel_downloads'] = self.channel_downloads
d['downloads'] = self.downloads
return d
def get_statistics(self):
@ -721,53 +695,41 @@ class Channel(ModelHelper, Base):
elif len(matching_entries) > 1:
raise RuntimeError(f'More than one info_json entry matches {video_source_id}')
def get_download(self, url: str = None, download_id: int = None) -> Optional[ChannelDownload]:
"""Attempt to get the ChannelDownload that matches the provided parameters."""
if not url and not download_id:
raise RuntimeError('Must provide either url or download_id')
for cd in self.channel_downloads:
if download_id and cd.download.id == download_id:
return cd
if url and cd.download.url == url:
return cd
@optional_session
def get_or_create_download(self, url: str, session: Session = None, reset_attempts: bool = False) \
-> ChannelDownload:
"""Get a ChannelDownload record, if it does not exist, create it. Create a Download if necessary
def get_or_create_download(self, url: str, session: Session = None, reset_attempts: bool = False) -> Download:
"""Get a Download record, if it does not exist, create it. Create a Download if necessary
which goes into this Channel's directory."""
if not isinstance(url, str) or not url:
raise RuntimeError(f'Cannot get Download without url')
download = download_manager.get_or_create_download(url, session=session, reset_attempts=reset_attempts)
download.channel_id = self.id
download.settings = download.settings or dict(destination=str(self.directory))
from modules.videos.downloader import ChannelDownloader, VideoDownloader
download.downloader = ChannelDownloader.name
download.sub_downloader = VideoDownloader.name
if not url:
raise RuntimeError(f'Cannot get ChannelDownload without url')
cd = ChannelDownload.get_by_url(url, session)
if not cd:
# A Channel is downloaded using the ChannelDownloader first. Then, any missing videos are passed to
# VideoDownloader.
download = session.query(Download).filter_by(url=url).one_or_none()
if not download:
download = download_manager.create_download(url, ChannelDownloader.name, session=session,
sub_downloader_name=VideoDownloader.name,
reset_attempts=reset_attempts,
)
# Download into this Channel's directory by default.
destination = (download.settings or dict()).get('destination')
if not destination:
settings = (download.settings or dict())
settings.update(dict(destination=str(self.directory)))
download.settings = settings
# Link between Channel and Download.
cd = ChannelDownload.get_by_url(url, session)
if not cd:
cd = ChannelDownload(channel_id=self.id, download_url=url)
session.add(cd)
logger.debug(f'Created {cd} for {self}')
return cd
return download
def get_rss_url(self) -> str | None:
"""Return the RSS Feed URL for this Channel, if any is possible."""
if self.url and self.source_id and 'youtube.com' in self.url:
return f'https://www.youtube.com/feeds/videos.xml?channel_id={self.source_id}'
@staticmethod
@optional_session
def get_by_url(url: str, session: Session = None) -> Optional['Channel']:
if not url:
raise RuntimeError('Must provide URL to get Channel')
channel = session.query(Channel).filter_by(url=url).one_or_none()
return channel
@staticmethod
@optional_session
def get_by_directory(directory: pathlib.Path | str, session: Session = None) -> Optional['Channel']:
if not directory:
raise RuntimeError('Must provide directory to get Channel')
directory = pathlib.Path(directory) if isinstance(directory, str) else directory
directory = str(directory.absolute()) if directory.is_absolute() else str(get_media_directory() / directory)
channel = session.query(Channel).filter_by(directory=directory).one_or_none()
return channel

View file

@ -1,12 +1,13 @@
import pathlib
import subprocess
import tempfile
from copy import copy
from unittest import mock
import pytest
from PIL import Image
from modules.videos.models import Channel, Video, ChannelDownload
from modules.videos.models import Channel, Video
from wrolpi.common import get_absolute_media_path, get_wrolpi_config
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.files import lib as files_lib
@ -149,25 +150,26 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto
conf.channels = old_config['channels']
conf.save()
# Config has no channels with a download_frequency.
# Two Channels, but neither have Downloads.
save_channels_config()
import_channels_config()
assert not channel1.channel_downloads
assert not channel1.downloads
assert not channel2.downloads
assert len(test_session.query(Channel).all()) == 2
assert test_session.query(Download).count() == test_session.query(ChannelDownload).count() == 0
assert test_session.query(Download).count() == 0
# Add a frequency to the Channel.
# Add a frequency to `channel1`.
channels_config = get_channels_config()
update_channel_config(channels_config, 'foo',
{'channel_downloads': [
{'downloads': [
{'url': 'https://example.com/channel1', 'frequency': DownloadFrequency.weekly}
]})
# Download record is created on import.
import_channels_config()
assert channel1.channel_downloads
assert channel1.downloads
assert len(test_session.query(Channel).all()) == 2, 'Both Channels should still exist'
assert test_session.query(ChannelDownload).count() == 1, 'One Channel has a Download'
assert test_session.query(Download).count() == 1, 'One Channel has a Download'
download: Download = test_session.query(Download).one()
assert download.url == channel1.url
assert download.frequency
@ -178,22 +180,23 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto
download: Download = test_session.query(Download).one()
assert next_download == str(download.next_download)
# Creating Download that matches Channel2's URL creates a ChannelDownload. Delete it and it should be re-created.
# Creating Download that matches Channel2's URL means they are related. Delete it and it should be re-created.
channel2.get_or_create_download(channel2.url, test_session)
save_channels_config()
test_session.query(ChannelDownload).filter_by(download_url=channel2.url).delete()
Download.find_by_url(channel2.url).delete(skip=False)
test_session.commit()
# Missing ChannelDownload is re-created on import.
# Missing Download is re-created on import.
import_channels_config()
channel2: Channel = test_session.query(Channel).filter_by(id=channel2.id).one()
cds = test_session.query(ChannelDownload).all()
assert len(cds) == 2, cds
assert channel1.channel_downloads and len(channel1.channel_downloads) == 1
assert channel2.channel_downloads and len(channel2.channel_downloads) == 1
channel2 = Channel.get_by_id(channel2.id)
downloads = test_session.query(Download).all()
assert len(downloads) == 1, downloads
assert len(channel1.downloads) == 1
assert len(channel2.downloads) == 0
# Add a Download to Channel2 which does not match Channel.url.
channel2.get_or_create_download('https://example.org')
channel2.get_or_create_download('https://example.org', session=test_session)
channel2.get_or_create_download('https://example.com/channel2', session=test_session)
test_session.commit()
save_channels_config()
# Check config is written to match new URLs.
@ -202,21 +205,23 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto
for channel_config in config.channels:
if channel_config['source_id'] == channel2.source_id:
assert channel_config['source_id'] == 'bar'
assert len(channel_config['channel_downloads']) == 2
assert set(channel_config['channel_downloads']) == {'https://example.com/channel2', 'https://example.org'}
assert len(channel_config['downloads']) == 2
assert set(channel_config['downloads']) == {'https://example.com/channel2', 'https://example.org'}
else:
assert channel_config['source_id'] == 'foo'
# Reset Downloads. Downloads and ChannelDownloads should be recreated.
# Get Channels with their Downloads.
channels_backup = copy(get_channels_config().channels)
# Reset Downloads. Downloads should be recreated from the channels config file.
[i.delete(skip=False) for i in test_session.query(Download).all()]
[i.delete() for i in test_session.query(ChannelDownload).all()]
# Write config with Channel Downloads now that the delete() has removed them from the config.
get_channels_config().channels = channels_backup
import_channels_config()
channel1, channel2 = test_session.query(Channel).order_by(Channel.url).all()
assert channel1.url == 'https://example.com/channel1'
assert channel2.url == 'https://example.com/channel2'
assert {i.download.url for i in channel1.channel_downloads} == {'https://example.com/channel1'}
assert {i.download.url for i in channel2.channel_downloads} == {'https://example.com/channel2',
'https://example.org'}
assert {i.url for i in channel1.downloads} == {'https://example.com/channel1'}
assert {i.url for i in channel2.downloads} == {'https://example.com/channel2', 'https://example.org'}
def test_import_channel_delete_missing_channels(test_session, channel_factory, test_channels_config):

View file

@ -7,7 +7,7 @@ import pytest
from modules.videos.downloader import VideoDownloader, \
get_or_create_channel, channel_downloader, video_downloader
from modules.videos.models import Channel, Video, ChannelDownload
from modules.videos.models import Channel, Video
from wrolpi.downloader import Download, DownloadResult
from wrolpi.errors import InvalidDownload
from wrolpi.vars import PROJECT_DIR
@ -142,7 +142,6 @@ async def test_download_channel(test_session, simple_channel, video_download_man
await video_download_manager.wait_for_all_downloads()
def reset_downloads():
[i.delete() for i in test_session.query(ChannelDownload)]
[i.delete(skip=False) for i in test_session.query(Download)]
# Let background tasks run.
@ -161,9 +160,9 @@ async def test_download_channel(test_session, simple_channel, video_download_man
test_session.commit()
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
mock_get_channel.return_value = simple_channel
cd = simple_channel.get_or_create_download(url, test_session)
cd.download.frequency = 100
cd.download.settings = {'title_include': '2'}
download = simple_channel.get_or_create_download(url, test_session)
download.frequency = 100
download.settings = {'title_include': '2'}
await video_download_manager.wait_for_all_downloads()
downloads = video_download_manager.get_once_downloads(test_session)
assert {i.url for i in downloads} == {'https://youtube.com/watch?v=video_2_url'}
@ -178,9 +177,9 @@ async def test_download_channel(test_session, simple_channel, video_download_man
test_session.commit()
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
mock_get_channel.return_value = simple_channel
cd = simple_channel.get_or_create_download(url, test_session)
cd.download.frequency = 100
cd.download.settings = {'title_exclude': '2'}
download = simple_channel.get_or_create_download(url, test_session)
download.frequency = 100
download.settings = {'title_exclude': '2'}
await video_download_manager.wait_for_all_downloads()
downloads = video_download_manager.get_once_downloads(test_session)
assert {i.url for i in downloads} == {'https://youtube.com/watch?v=video_1_url'}

View file

@ -4,8 +4,9 @@ import shutil
import pytest
from modules.videos import lib
from modules.videos.downloader import ChannelDownloader
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
from modules.videos.models import Video, ChannelDownload
from modules.videos.models import Video
from wrolpi.downloader import Download, RSSDownloader
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
@ -151,7 +152,7 @@ async def test_orphaned_files(test_session, make_files_structure, test_directory
])
def test_link_channel_and_downloads(test_session, channel_factory):
def test_link_channel_and_downloads(test_session, channel_factory, test_download_manager):
channel = channel_factory(
url='https://www.youtube.com/c/LearningSelfReliance/videos',
source_id='UCng5u6ASda3LNRXJN0JCQJA',
@ -162,9 +163,43 @@ def test_link_channel_and_downloads(test_session, channel_factory):
test_session.add_all([download1, download2])
test_session.commit()
assert test_session.query(Download).count() == 2
assert test_session.query(ChannelDownload).count() == 0
assert not any(i.channel_id for i in test_download_manager.get_downloads(test_session))
# `link_channel_and_downloads` creates missing ChannelDownloads.
lib.link_channel_and_downloads(session=test_session)
assert test_session.query(Download).count() == 2
assert test_session.query(ChannelDownload).count() == 2
assert all(i.channel_id for i in test_download_manager.get_downloads(test_session))
@pytest.mark.asyncio
def test_channel_channel_downloads_migration(test_async_client, test_session, channel_factory, test_download_manager):
"""Test the 8d0d81bc9c34_channel_channel_downloads.py migration."""
# A simple Channel which has a download for its URL.
channel1 = channel_factory(url='https://example.com/channel1')
# A Channel which has two Downloads. One for it's URL, another which is an RSS feed in its directory.
channel2 = channel_factory(url='https://example.com/channel2')
download1 = Download(url=channel1.url, downloader=ChannelDownloader.name)
download2a = Download(url='https://example.com/channel2/rss', downloader=RSSDownloader.name,
settings=dict(destination=str(channel2.directory)))
download2b = Download(url='https://example.com/channel2', downloader=ChannelDownloader.name,
settings=dict(destination=str(channel2.directory)))
# A Channel which has a URL, but no Downloads.
channel3 = channel_factory()
test_session.add_all([download1, download2a, download2b])
test_session.commit()
assert not channel1.downloads
assert channel2.directory and not channel2.downloads
assert not channel3.downloads
lib.link_channel_and_downloads(test_session)
assert channel1.downloads
assert channel2.downloads
assert not channel3.downloads
assert test_session.query(Download).count() == 3
cd1, cd2a, cd2b = test_session.query(Download).order_by(Download.url).all()
assert cd1.url == channel1.url
assert cd2a.url == 'https://example.com/channel2'
assert cd2b.url == 'https://example.com/channel2/rss'

View file

@ -1,40 +0,0 @@
import pytest
from modules.videos.downloader import ChannelDownloader
from modules.videos.models import ChannelDownload
from wrolpi.downloader import Download, RSSDownloader
from wrolpi.migration import migrate_channel_downloads
@pytest.mark.asyncio
def test_channel_channel_downloads_migration(test_async_client, test_session, channel_factory, test_download_manager):
"""Test the 8d0d81bc9c34_channel_channel_downloads.py migration."""
# A simple Channel which has a download for its URL.
channel1 = channel_factory(url='https://example.com/channel1')
# A Channel which has two Downloads. One for it's URL, another which is an RSS feed in its directory.
channel2 = channel_factory(url='https://example.com/channel2')
download1 = Download(url=channel1.url, downloader=ChannelDownloader.name)
download2a = Download(url='https://example.com/channel2/rss', downloader=RSSDownloader.name,
settings=dict(destination=str(channel2.directory)))
download2b = Download(url='https://example.com/channel2', downloader=ChannelDownloader.name,
settings=dict(destination=str(channel2.directory)))
# A Channel which has a URL, but no Downloads.
channel3 = channel_factory()
test_session.add_all([download1, download2a, download2b])
test_session.commit()
assert not channel1.channel_downloads
assert channel2.directory and not channel2.channel_downloads
assert not channel3.channel_downloads
migrate_channel_downloads(test_session)
assert channel1.channel_downloads
assert channel2.channel_downloads
assert not channel3.channel_downloads
assert test_session.query(ChannelDownload).count() == 3
cd1, cd2a, cd2b = test_session.query(ChannelDownload).order_by(ChannelDownload.download_url).all()
assert cd1.download_url == channel1.url
assert cd2a.download_url == 'https://example.com/channel2'
assert cd2b.download_url == 'https://example.com/channel2/rss'

View file

@ -18,9 +18,9 @@ from urllib.parse import urlparse
import feedparser
import pytz
from feedparser import FeedParserDict
from sqlalchemy import Column, Integer, String, Text
from sqlalchemy import Column, Integer, String, Text, ForeignKey
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Session
from sqlalchemy.orm import Session, relationship
from sqlalchemy.sql import Delete
from wrolpi import flags
@ -30,7 +30,7 @@ from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, C
limit_concurrent, wrol_mode_enabled
from wrolpi.dates import TZDateTime, now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload
from wrolpi.vars import PYTEST, SIMULTANEOUS_DOWNLOAD_DOMAINS
logger = logger.getChild(__name__)
@ -111,7 +111,10 @@ class Download(ModelHelper, Base): # noqa
next_download = Column(TZDateTime)
settings = Column(JSONB) # information about how the download should happen (destination, etc.)
status = Column(String, default=DownloadStatus.new) # `DownloadStatus` enum.
_manager: 'DownloadManager' = None
# A Download may be associated with a Channel (downloads all Channel videos, or a playlist, etc.).
channel_id = Column(Integer, ForeignKey('channel.id'))
channel = relationship('Channel', primaryjoin='Download.channel_id==Channel.id', back_populates='downloads')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@ -126,6 +129,7 @@ class Download(ModelHelper, Base): # noqa
def __json__(self) -> dict:
d = dict(
channel_id=self.channel_id,
downloader=self.downloader,
frequency=self.frequency,
id=self.id,
@ -215,17 +219,46 @@ class Download(ModelHelper, Base): # noqa
session = Session.object_session(self)
# Delete ChannelDownload, if any exists.
from modules.videos.models import ChannelDownload
cds = session.query(ChannelDownload).filter(ChannelDownload.download == self).all()
for cd in cds:
session.delete(cd)
session.delete(self)
session.commit()
if self.channel_id:
# Save Channels config if this download was associated with a Channel.
from modules.videos.lib import save_channels_config
save_channels_config()
# Save download config again because this download is now removed from the download lists.
api_app.add_task(save_downloads_config())
@staticmethod
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
download = session.query(Download).filter(Download.id == id_).one_or_none()
return download
@classmethod
@optional_session
def find_by_id(cls, id_: int, session: Session = None) -> 'Download':
if download := cls.get_by_id(id_, session):
return download
raise UnknownDownload(f'Cannot find Download with id: {id_}')
@staticmethod
@optional_session
def get_by_url(url: str, session: Session = None) -> Optional['Download']:
download = session.query(Download).filter(Download.url == url).one_or_none()
return download
@classmethod
@optional_session
def find_by_url(cls, url: str, session: Session = None) -> 'Download':
if download := cls.get_by_url(url, session):
return download
raise UnknownDownload(f'Cannot find Download with URL: {url}')
@staticmethod
@optional_session
def get_by_channel_id(channel_id: int, session: Session = None) -> Optional['Download']:
download = session.query(Download).filter(Download.channel_id == channel_id).one_or_none()
return download
class Downloader:
name: str = None
@ -474,20 +507,6 @@ class DownloadManager:
download = Download(url=url, status='new')
session.add(download)
from modules.videos.models import Channel, ChannelDownload
channel: Channel = session.query(Channel).filter(Channel.url == url).one_or_none()
if channel:
# This Download needs to be associated with the Channel.
cd = ChannelDownload.get_by_url(url, session)
if not cd:
cd = ChannelDownload(channel=channel, download=download)
session.add(cd)
else:
cd.download = download
if cd not in channel.channel_downloads:
channel.channel_downloads.append(cd)
session.flush([cd, channel])
return download
@optional_session
@ -518,16 +537,15 @@ class DownloadManager:
download.downloader = downloader_name
download.sub_downloader = sub_downloader_name
download.settings = settings if settings is not None else download.settings
downloads.append(download)
# Create ChannelDownload for any download that is being downloaded into a Channel's Directory.
from modules.videos.models import Channel, ChannelDownload
if download.settings and (destination := download.settings.get('destination')):
channel = Channel.get_by_path(destination, session)
cd = ChannelDownload.get_by_url(url, session)
if channel and not cd:
cd = ChannelDownload(channel=channel, download=download)
session.add(cd)
from modules.videos.models import Channel
if channel := Channel.get_by_url(url=download.url, session=session):
download.channel_id = channel.id
if destination := (download.settings or dict()).get('destination'):
if channel := Channel.get_by_directory(destination):
download.channel_id = channel.id
downloads.append(download)
session.flush(downloads)
try:
@ -852,9 +870,8 @@ class DownloadManager:
status,
sub_downloader,
url,
cd.channel_id
channel_id
FROM download
LEFT OUTER JOIN channel_download cd on download.url = cd.download_url
WHERE frequency IS NOT NULL
ORDER BY
{self._status_order},
@ -1055,7 +1072,7 @@ async def signal_download_download(download_id: int, download_url: str):
with get_db_session(commit=True) as session:
# Mark the download as started in new session so the change is committed.
download = session.query(Download).filter_by(id=download_id).one()
download = Download.find_by_id(download_id, session=session)
download.started()
download_domain = download.domain

View file

@ -51,6 +51,12 @@ class InvalidDownload(APIError):
status_code = HTTPStatus.BAD_REQUEST
class UnknownDownload(APIError):
code = 'UNKNOWN_DOWNLOAD'
message = 'The Download cannot be found.'
status_code = HTTPStatus.NOT_FOUND
class UnrecoverableDownloadError(APIError):
code = 'UNRECOVERABLE_DOWNLOAD_ERROR'
message = 'Download experienced an error which cannot be recovered. Download will be deleted.'

View file

@ -1,32 +0,0 @@
from modules.videos.models import Channel, ChannelDownload
from wrolpi.downloader import Download
def migrate_channel_downloads(session):
channels_by_url = {i.url: i for i in session.query(Channel).all()}
downloads = session.query(Download).all()
need_commit = False
for download in downloads:
channel = channels_by_url.get(download.url)
if channel:
cd = session.query(ChannelDownload).filter_by(download_url=download.url).one_or_none()
if not cd:
cd = ChannelDownload(channel_id=channel.id, download_url=download.url)
session.add(cd)
need_commit = True
for download in downloads:
destination = download.settings.get('destination') if download.settings else None
if not destination:
continue
channel = session.query(Channel).filter_by(directory=destination).one_or_none()
if channel:
cd = session.query(ChannelDownload).filter_by(download_url=download.url).one_or_none()
if not cd:
cd = ChannelDownload(channel_id=channel.id, download_url=download.url)
session.add(cd)
need_commit = True
if need_commit:
session.commit()

View file

@ -535,8 +535,8 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
await assert_results(
dict(search_str='foo'),
[
{'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'channel_downloads': []},
{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'channel_downloads': []},
{'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []},
{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'downloads': []},
],
[{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}],
)
@ -545,14 +545,14 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
await assert_results(
dict(search_str='foo l'),
[
{'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'channel_downloads': []}],
{'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}],
[],
)
await assert_results(
dict(search_str='bar'),
[
{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'channel_downloads': []}
{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'downloads': []}
],
[{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}],
)