mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
Removing ChannelDownload in favor of Download.channel_id.
This commit is contained in:
parent
958dbb899d
commit
63738bd70f
17 changed files with 290 additions and 364 deletions
|
|
@ -1,4 +1,4 @@
|
|||
"""Channel.channel_downloads
|
||||
"""Channel.downloads / Download.channel
|
||||
|
||||
Revision ID: 8d0d81bc9c34
|
||||
Revises: 00f11f309c53
|
||||
|
|
@ -11,6 +11,8 @@ from alembic import op
|
|||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modules.videos.lib import link_channel_and_downloads
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '8d0d81bc9c34'
|
||||
down_revision = '00f11f309c53'
|
||||
|
|
@ -27,29 +29,21 @@ def upgrade():
|
|||
session = Session(bind=bind)
|
||||
|
||||
session.execute('ALTER TABLE download ADD CONSTRAINT download_url_unique UNIQUE (url)')
|
||||
session.execute('''
|
||||
CREATE TABLE channel_download (
|
||||
channel_id INTEGER REFERENCES channel (id),
|
||||
download_url TEXT REFERENCES download (url),
|
||||
primary key (channel_id, download_url)
|
||||
)''')
|
||||
session.execute('ALTER TABLE download ADD COLUMN channel_id INTEGER REFERENCES channel(id)')
|
||||
|
||||
# Migration is separated for testing. See `test_channel_channel_downloads_migration`
|
||||
from wrolpi.migration import migrate_channel_downloads
|
||||
migrate_channel_downloads(session)
|
||||
link_channel_and_downloads(session)
|
||||
|
||||
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS match_regex')
|
||||
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS download_frequency')
|
||||
|
||||
if not DOCKERIZED:
|
||||
session.execute('ALTER TABLE public.channel_download OWNER TO wrolpi')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
|
||||
session.execute('DROP TABLE IF EXISTS channel_download')
|
||||
session.execute('ALTER TABLE download DROP COLUMN IF EXISTS channel_id')
|
||||
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS channel_id')
|
||||
session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS match_regex TEXT')
|
||||
session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS download_frequency INTEGER')
|
||||
session.execute('ALTER TABLE download DROP CONSTRAINT IF EXISTS download_url_unique')
|
||||
|
|
|
|||
|
|
@ -240,7 +240,7 @@ function ChannelPage({create, header}) {
|
|||
}
|
||||
}
|
||||
|
||||
let channelDownloads = channel && channel.channel_downloads ? channel.channel_downloads : null;
|
||||
let downloads = channel && channel.downloads ? channel.downloads : null;
|
||||
|
||||
const afterNewDownloadSave = async () => {
|
||||
await fetchChannel();
|
||||
|
|
@ -399,7 +399,7 @@ function ChannelPage({create, header}) {
|
|||
</ModalContent>
|
||||
</Modal>
|
||||
|
||||
<RecurringDownloadsTable downloads={channelDownloads} fetchDownloads={fetchChannel}/>
|
||||
<RecurringDownloadsTable downloads={downloads} fetchDownloads={fetchChannel}/>
|
||||
</Segment>}
|
||||
|
||||
<div style={{marginTop: '2em'}}>
|
||||
|
|
|
|||
|
|
@ -252,17 +252,6 @@ class RecurringDownloadRow extends React.Component {
|
|||
}
|
||||
}
|
||||
|
||||
handleDelete = async () => {
|
||||
const {id} = this.props;
|
||||
try {
|
||||
await deleteDownload(id);
|
||||
} finally {
|
||||
if (this.props.fetchDownloads) {
|
||||
this.props.fetchDownloads();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
handleRestart = async () => {
|
||||
const {id} = this.props;
|
||||
try {
|
||||
|
|
@ -295,7 +284,6 @@ class RecurringDownloadRow extends React.Component {
|
|||
next_download,
|
||||
error,
|
||||
downloader,
|
||||
channel_id
|
||||
} = this.props;
|
||||
const {errorModalOpen} = this.state;
|
||||
|
||||
|
|
@ -319,17 +307,6 @@ class RecurringDownloadRow extends React.Component {
|
|||
</ModalActions>
|
||||
</Modal>;
|
||||
|
||||
const deleteButton = <>
|
||||
<APIButton
|
||||
color='red'
|
||||
icon='trash'
|
||||
confirmContent='Are you sure you want to delete this download?'
|
||||
confirmButton='Delete'
|
||||
onClick={this.handleDelete}
|
||||
obeyWROLMode={true}
|
||||
/>
|
||||
</>;
|
||||
|
||||
const editButton = <>
|
||||
<Button icon='edit' onClick={() => this.setState({editModalOpen: true})}/>
|
||||
</>
|
||||
|
|
|
|||
|
|
@ -8,12 +8,13 @@ from sqlalchemy.orm.exc import NoResultFound
|
|||
from wrolpi.common import run_after, logger, \
|
||||
get_media_directory
|
||||
from wrolpi.db import get_db_curs, optional_session, get_db_session
|
||||
from wrolpi.errors import UnknownDirectory, APIError, ValidationError, InvalidDownload
|
||||
from wrolpi.downloader import save_downloads_config, download_manager
|
||||
from wrolpi.errors import UnknownDirectory, APIError, ValidationError
|
||||
from .. import schema
|
||||
from ..common import check_for_channel_conflicts
|
||||
from ..errors import UnknownChannel
|
||||
from ..lib import save_channels_config
|
||||
from ..models import Channel, Video, ChannelDownload
|
||||
from ..models import Channel, Video
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -33,8 +34,7 @@ async def get_minimal_channels() -> List[dict]:
|
|||
(
|
||||
select min(d.frequency)
|
||||
from download d
|
||||
left join channel_download cd on cd.download_url = d.url
|
||||
where cd.channel_id = c.id
|
||||
where d.channel_id = c.id
|
||||
) AS minimum_frequency
|
||||
FROM
|
||||
channel AS c
|
||||
|
|
@ -217,32 +217,33 @@ async def search_channels_by_name(name: str, limit: int = 5, session: Session =
|
|||
return channels
|
||||
|
||||
|
||||
async def create_channel_download(channel_id: int, url: str, frequency: int, settings: dict) -> ChannelDownload:
|
||||
"""Create ChannelDownload."""
|
||||
async def create_channel_download(channel_id: int, url: str, frequency: int, settings: dict):
|
||||
"""Create Download for Channel."""
|
||||
with get_db_session(commit=True) as session:
|
||||
channel = get_channel(session, channel_id=channel_id, return_dict=False)
|
||||
cd = channel.get_or_create_download(url, session, reset_attempts=True)
|
||||
cd.download.frequency = cd.download.frequency or frequency
|
||||
cd.download.settings = settings
|
||||
channel = Channel.find_by_id(channel_id, session=session)
|
||||
download = channel.get_or_create_download(url, session, reset_attempts=True)
|
||||
download.frequency = download.frequency or frequency
|
||||
download.settings = settings
|
||||
|
||||
save_channels_config()
|
||||
await save_downloads_config()
|
||||
|
||||
return cd
|
||||
return download
|
||||
|
||||
|
||||
async def update_channel_download(channel_id: int, download_id: int, url: str, frequency: int, settings: dict) \
|
||||
-> ChannelDownload:
|
||||
"""Fetch ChannelDownload, update its properties."""
|
||||
async def update_channel_download(channel_id: int, download_id: int, url: str, frequency: int, settings: dict):
|
||||
"""Fetch Channel's Download, update its properties."""
|
||||
with get_db_session(commit=True) as session:
|
||||
channel = get_channel(session, channel_id=channel_id, return_dict=False)
|
||||
cd = channel.get_download(download_id=download_id)
|
||||
if not cd:
|
||||
raise InvalidDownload('No Download with that ID on this Channel')
|
||||
# Ensure that the Channel exists.
|
||||
Channel.find_by_id(channel_id, session=session)
|
||||
download = download_manager.get_download(session, id_=download_id)
|
||||
download.url = url
|
||||
download.frequency = download.frequency or frequency
|
||||
download.settings = settings
|
||||
|
||||
cd.download.url = url
|
||||
cd.download.frequency = cd.download.frequency or frequency
|
||||
cd.download.settings = settings
|
||||
download_manager.remove_from_skip_list(url)
|
||||
|
||||
save_channels_config()
|
||||
await save_downloads_config()
|
||||
|
||||
return cd
|
||||
return download
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import mock
|
|||
import pytest
|
||||
|
||||
from modules.videos.downloader import ChannelDownloader
|
||||
from modules.videos.models import Channel, ChannelDownload
|
||||
from modules.videos.models import Channel
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.downloader import download_manager, Download, DownloadResult
|
||||
|
|
@ -58,29 +58,31 @@ def test_channel_no_download_frequency(test_client, test_session, test_directory
|
|||
|
||||
# Get the Channel
|
||||
request, response = test_client.get('/api/videos/channels/1')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
channel = response.json['channel']
|
||||
assert not channel['channel_downloads']
|
||||
assert not channel['downloads']
|
||||
|
||||
# Update the Channel with a frequency.
|
||||
new_download = {'urls': [simple_channel.url], 'frequency': 10, 'downloader': ChannelDownloader.name}
|
||||
request, response = test_client.post('/api/download', json=new_download)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
||||
|
||||
# Download is scheduled. ChannelDownload was created.
|
||||
# Download is scheduled. Download is related to Channel.
|
||||
assert len(download_manager.get_downloads(test_session)) == 1
|
||||
request, response = test_client.get('/api/videos/channels/1')
|
||||
channel = response.json['channel']
|
||||
assert channel['channel_downloads']
|
||||
channel_downloads = channel['channel_downloads']
|
||||
assert len(channel_downloads) == 1 and channel_downloads[0]['url'] == simple_channel.url
|
||||
assert test_session.query(ChannelDownload).one()
|
||||
download_id = channel['channel_downloads'][0]['id']
|
||||
assert channel['downloads']
|
||||
downloads = channel['downloads']
|
||||
assert len(downloads) == 1 and downloads[0]['url'] == simple_channel.url
|
||||
assert test_session.query(Download).one()
|
||||
download_id = channel['downloads'][0]['id']
|
||||
|
||||
# Deleting the Download deletes the ChannelDownload.
|
||||
# Deleting Download does not delete Channel.
|
||||
request, response = test_client.delete(f'/api/download/{download_id}')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
||||
assert not download_manager.get_downloads(test_session)
|
||||
assert not test_session.query(ChannelDownload).all()
|
||||
assert not test_session.query(Download).all()
|
||||
assert test_session.query(Channel).count() == 1
|
||||
|
||||
|
||||
def test_video_file_name(test_session, simple_video, test_client):
|
||||
|
|
@ -188,7 +190,7 @@ async def test_channel_download_requires_refresh(
|
|||
Videos already downloaded are not downloaded again."""
|
||||
vid = video_factory(channel_id=download_channel.id, with_video_file=True, with_poster_ext='jpg')
|
||||
vid.file_group.url = 'https://example.com/1'
|
||||
d = download_channel.channel_downloads[0].download
|
||||
d = download_channel.downloads[0]
|
||||
d.next_download = None
|
||||
test_session.commit()
|
||||
assert not d.next_download
|
||||
|
|
@ -413,7 +415,7 @@ def test_get_channel_videos_pagination(test_session, simple_channel, video_facto
|
|||
async def test_create_channel_download(test_async_client, test_session, simple_channel, tag_factory):
|
||||
tag = tag_factory()
|
||||
|
||||
# Create ChannelDownload (which includes a Download).
|
||||
# Create Download.
|
||||
body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
|
||||
request, response = await test_async_client.post(
|
||||
f'/api/videos/channels/{simple_channel.id}/download', json=body)
|
||||
|
|
@ -422,7 +424,7 @@ async def test_create_channel_download(test_async_client, test_session, simple_c
|
|||
assert download.url == 'https://example.com/1' and not download.settings.get('tag_names')
|
||||
assert download.settings.get('title_include') is None
|
||||
|
||||
# Create another ChannelDownload. Previous ChannelDownload/Download are untouched.
|
||||
# Create another Download. Previous Downloads is untouched.
|
||||
body = {'url': 'https://example.com/2', 'frequency': 55, 'settings': {'tag_names': [tag.name]}}
|
||||
request, response = await test_async_client.post(
|
||||
f'/api/videos/channels/{simple_channel.id}/download', json=body)
|
||||
|
|
@ -431,9 +433,9 @@ async def test_create_channel_download(test_async_client, test_session, simple_c
|
|||
assert download1.url == 'https://example.com/1' and not download1.settings.get('tag_names')
|
||||
assert download2.url == 'https://example.com/2' and download2.settings.get('tag_names') == [tag.name, ]
|
||||
test_session.flush(simple_channel)
|
||||
assert len(simple_channel.channel_downloads) == 2
|
||||
assert len(simple_channel.downloads) == 2
|
||||
|
||||
# Change frequency of first ChannelDownload.
|
||||
# Change frequency of first Download.
|
||||
body = {'url': 'https://example.com/1', 'frequency': 123, 'settings': {'title_include': 'foo,bar'}}
|
||||
request, response = await test_async_client.put(
|
||||
f'/api/videos/channels/{simple_channel.id}/download/{download1.id}', json=body)
|
||||
|
|
@ -443,10 +445,10 @@ async def test_create_channel_download(test_async_client, test_session, simple_c
|
|||
assert download1.settings == {'title_include': 'foo,bar'}
|
||||
assert download2.url == 'https://example.com/2' and download2.settings.get('tag_names') == [tag.name, ]
|
||||
test_session.flush(simple_channel)
|
||||
assert len(simple_channel.channel_downloads) == 2
|
||||
assert len(simple_channel.downloads) == 2
|
||||
|
||||
# Deleting Download delets ChannelDownload.
|
||||
# Deleting Download does not delete Channel.
|
||||
request, response = await test_async_client.delete(f'/api/download/{download1.id}')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert test_session.query(ChannelDownload).count() == 1
|
||||
assert test_session.query(Download).count() == 1
|
||||
assert test_session.query(Channel).count() == 1
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from http import HTTPStatus
|
|||
import pytest
|
||||
|
||||
from modules.videos.downloader import VideoDownloader
|
||||
from modules.videos.models import Channel, ChannelDownload
|
||||
from modules.videos.models import Channel
|
||||
from modules.videos.models import Video
|
||||
from wrolpi.downloader import Download, DownloadFrequency, download_manager, RSSDownloader
|
||||
|
||||
|
|
@ -73,47 +73,40 @@ async def test_nested_channel_directories(test_session, test_async_client, test_
|
|||
|
||||
@pytest.mark.asyncio()
|
||||
async def test_channel_download_relationships(test_session, download_channel):
|
||||
"""Test relationships of ChannelDownload."""
|
||||
"""Test relationships of Channel and Download."""
|
||||
test_session.flush()
|
||||
|
||||
download = test_session.query(Download).one()
|
||||
cd = test_session.query(ChannelDownload).one()
|
||||
channel = test_session.query(Channel).one()
|
||||
assert len(channel.channel_downloads) == 1
|
||||
assert channel.channel_downloads[0] == cd
|
||||
assert channel.channel_downloads[0].download == download
|
||||
assert len(channel.downloads) == 1
|
||||
assert channel.downloads
|
||||
assert download in channel.downloads
|
||||
|
||||
test_session.commit()
|
||||
|
||||
download = test_session.query(Download).one()
|
||||
cd = test_session.query(ChannelDownload).one()
|
||||
channel = test_session.query(Channel).one()
|
||||
assert len(channel.channel_downloads) == 1
|
||||
assert channel.channel_downloads[0] == cd
|
||||
assert channel.channel_downloads[0].download == download \
|
||||
and channel.channel_downloads[0].download.url == 'https://example.com/channel1'
|
||||
download: Download = test_session.query(Download).one()
|
||||
channel: Channel = test_session.query(Channel).one()
|
||||
assert len(channel.downloads) == 1
|
||||
assert channel.downloads[0] == download and channel.downloads[0].url == 'https://example.com/channel1'
|
||||
assert download.frequency == DownloadFrequency.weekly
|
||||
assert download.settings['destination'] == str(channel.directory)
|
||||
|
||||
# Deleting Download deletes the ChannelDownload, but not the Channel.
|
||||
# Deleting Download deletes the Download, but not the Channel.
|
||||
download.delete()
|
||||
assert not test_session.query(Download).all()
|
||||
assert not test_session.query(ChannelDownload).all()
|
||||
assert test_session.query(Channel).one(), 'Channel should not have been deleted.'
|
||||
|
||||
# Create a Download again.
|
||||
download_channel.get_or_create_download('https://example.com/1', test_session, reset_attempts=True)
|
||||
assert test_session.query(Download).count() == 1
|
||||
assert test_session.query(ChannelDownload).count() == 1
|
||||
# Delete the Channel, and any relationships.
|
||||
download_channel.delete_with_videos()
|
||||
assert test_session.query(Download).count() == 0
|
||||
assert test_session.query(ChannelDownload).count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
async def test_create_channel_download(test_session, simple_channel):
|
||||
"""A ChannelDownload is created if a Download is created for a Channel's directory."""
|
||||
"""A Download will be related to a Channel if a Download is created in the Channel's directory"""
|
||||
test_session.flush()
|
||||
|
||||
settings = dict(destination=str(simple_channel.directory))
|
||||
|
|
@ -121,5 +114,5 @@ async def test_create_channel_download(test_session, simple_channel):
|
|||
sub_downloader_name=VideoDownloader.name, settings=settings)
|
||||
download = test_session.query(Download).one()
|
||||
assert download.settings['destination'] == str(simple_channel.directory)
|
||||
cd = test_session.query(ChannelDownload).one()
|
||||
assert cd.channel == simple_channel
|
||||
download = test_session.query(Download).one()
|
||||
assert download.channel == simple_channel
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ def simple_channel(test_session, test_directory) -> Channel:
|
|||
|
||||
@pytest.fixture
|
||||
def channel_factory(test_session, test_directory):
|
||||
"""Create a random Channel with a directory, ChannelDownload, and Download."""
|
||||
"""Create a random Channel with a directory, and Download."""
|
||||
|
||||
def factory(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None,
|
||||
directory: pathlib.Path = None):
|
||||
|
|
@ -52,10 +52,10 @@ def channel_factory(test_session, test_directory):
|
|||
test_session.flush([channel])
|
||||
assert channel.id and channel.url
|
||||
if download_frequency:
|
||||
cd = channel.get_or_create_download(channel.url)
|
||||
cd.download.frequency = download_frequency
|
||||
assert cd and cd.download_url == channel.url
|
||||
assert channel.channel_downloads
|
||||
download = channel.get_or_create_download(channel.url)
|
||||
download.frequency = download_frequency
|
||||
assert download.url == channel.url
|
||||
assert channel.downloads
|
||||
test_session.commit()
|
||||
return channel
|
||||
|
||||
|
|
@ -71,9 +71,9 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch
|
|||
test_session.add(channel)
|
||||
test_session.flush([channel, ])
|
||||
assert channel and channel.id and channel.url
|
||||
cd = channel.get_or_create_download(channel.url, test_session)
|
||||
assert cd.download_url == channel.url
|
||||
cd.download.frequency = DownloadFrequency.weekly
|
||||
download = channel.get_or_create_download(channel.url, test_session)
|
||||
assert download.url == channel.url
|
||||
download.frequency = DownloadFrequency.weekly
|
||||
test_session.commit()
|
||||
return channel
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from .common import is_valid_poster, convert_image, \
|
|||
generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
|
||||
get_video_duration
|
||||
from .errors import UnknownChannel
|
||||
from .models import Channel, Video, ChannelDownload
|
||||
from .models import Channel, Video
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -455,7 +455,7 @@ def import_channels_config():
|
|||
logger.warning(f'Deleting {channel} because it is not in the config.')
|
||||
channel.delete_with_videos()
|
||||
|
||||
# Create any missing ChannelDownload.
|
||||
# Create any missing Downloads. Associated Downloads with any necessary Channels.
|
||||
link_channel_and_downloads()
|
||||
|
||||
if save_config:
|
||||
|
|
@ -471,8 +471,9 @@ def import_channels_config():
|
|||
|
||||
@optional_session
|
||||
def link_channel_and_downloads(session: Session):
|
||||
"""Create any missing ChannelDownload for any Channel.url/Channel.directory that has a Download."""
|
||||
downloads = session.query(Download).all()
|
||||
"""Create any missing Downloads for any Channel.url/Channel.directory that has a Download. Associate any Download
|
||||
related to a Channel."""
|
||||
downloads = list(session.query(Download).all())
|
||||
downloads_by_url = {i.url: i for i in downloads}
|
||||
downloads_by_destination = {i.settings['destination']: i
|
||||
for i in downloads if 'destination' in (i.settings or dict())}
|
||||
|
|
@ -482,23 +483,29 @@ def link_channel_and_downloads(session: Session):
|
|||
for channel in channels:
|
||||
directory = str(channel.directory)
|
||||
download = downloads_by_destination.get(directory) or downloads_by_url.get(channel.url)
|
||||
if download and not channel.get_download(download_id=download.id):
|
||||
# Create ChannelDownload for the Download that is for this Channel.
|
||||
cd = ChannelDownload(channel=channel, download=download)
|
||||
session.add(cd)
|
||||
channel_import_logger.info(f'Created missing ChannelDownload for {download}')
|
||||
if download and not download.channel_id:
|
||||
download.channel_id = channel.id
|
||||
need_commit = True
|
||||
|
||||
# Get any Downloads for a Channel's RSS feed.
|
||||
rss_url = channel.get_rss_url()
|
||||
download = downloads_by_url.get(rss_url) if rss_url else None
|
||||
if rss_url and download:
|
||||
cd = channel.get_download(rss_url)
|
||||
if not cd:
|
||||
# No ChannelDownload for this RSS Download.
|
||||
cd = ChannelDownload(channel=channel, download=download)
|
||||
session.add(cd)
|
||||
channel_import_logger.info(f'Created missing ChannelDownload for {download}')
|
||||
if download:
|
||||
download.channel_id = channel.id
|
||||
|
||||
# Associate any Download which shares a Channel's URL.
|
||||
for download in downloads:
|
||||
channel = Channel.get_by_url(download.url)
|
||||
if channel and not download.channel_id:
|
||||
download.channel_id = channel.id
|
||||
need_commit = True
|
||||
|
||||
# Associate any Download which shares a Channel's directory.
|
||||
for destination, download in downloads_by_destination.items():
|
||||
if destination and (channel := Channel.get_by_directory(destination)):
|
||||
if not download.channel_id:
|
||||
download.channel_id = channel.id
|
||||
need_commit = True
|
||||
|
||||
if need_commit:
|
||||
session.commit()
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from wrolpi.vars import PYTEST
|
|||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = ['Video', 'Channel', 'ChannelDownload']
|
||||
__all__ = ['Video', 'Channel']
|
||||
|
||||
|
||||
class Video(ModelHelper, Base):
|
||||
|
|
@ -496,43 +496,11 @@ class Video(ModelHelper, Base):
|
|||
self.file_group.append_files(info_json_path)
|
||||
|
||||
|
||||
class ChannelDownload(Base):
|
||||
"""One-to-many join table between Channel and Download. Expects to link using Download.url."""
|
||||
__tablename__ = 'channel_download'
|
||||
channel_id = Column(Integer, ForeignKey('channel.id'), primary_key=True)
|
||||
channel = relationship('Channel', primaryjoin='ChannelDownload.channel_id==Channel.id',
|
||||
back_populates='channel_downloads')
|
||||
download_url = Column(String, ForeignKey('download.url'), primary_key=True)
|
||||
download = relationship('Download', primaryjoin='ChannelDownload.download_url==Download.url')
|
||||
|
||||
def __repr__(self):
|
||||
return f'<ChannelDownload url={self.download_url} channel_id={self.channel_id}>'
|
||||
|
||||
def __json__(self) -> dict:
|
||||
d = self.download.__json__()
|
||||
d['channel_id'] = self.channel_id
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def get_by_id(id: int, session: Session) -> Optional['ChannelDownload']:
|
||||
return session.query(ChannelDownload).filter(ChannelDownload.channel_id == id).one_or_none()
|
||||
|
||||
@staticmethod
|
||||
def get_by_url(url: str, session: Session) -> Optional['ChannelDownload']:
|
||||
return session.query(ChannelDownload).filter(ChannelDownload.download_url == url).one_or_none()
|
||||
|
||||
def delete(self):
|
||||
session = Session.object_session(self)
|
||||
if self.download:
|
||||
self.download.delete(skip=False)
|
||||
session.delete(self)
|
||||
|
||||
|
||||
class Channel(ModelHelper, Base):
|
||||
__tablename__ = 'channel'
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String)
|
||||
url = Column(String, unique=True) # will only be downloaded if ChannelDownload exists.
|
||||
url = Column(String, unique=True) # will only be downloaded if related Download exists.
|
||||
directory: pathlib.Path = Column(MediaPathType)
|
||||
generate_posters = Column(Boolean, default=False) # generating posters may delete files, and can be slow.
|
||||
calculate_duration = Column(Boolean, default=True) # use ffmpeg to extract duration (slower than info json).
|
||||
|
|
@ -543,8 +511,7 @@ class Channel(ModelHelper, Base):
|
|||
info_date = Column(Date)
|
||||
|
||||
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
|
||||
channel_downloads: InstrumentedList = \
|
||||
relationship('ChannelDownload', primaryjoin='ChannelDownload.channel_id==Channel.id')
|
||||
downloads: InstrumentedList = relationship('Download', primaryjoin='Download.channel_id==Channel.id')
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Channel id={self.id} name={repr(self.name)} url={self.url} directory={self.directory}>'
|
||||
|
|
@ -558,13 +525,13 @@ class Channel(ModelHelper, Base):
|
|||
for video in videos:
|
||||
video.channel = None
|
||||
|
||||
for cd in self.channel_downloads:
|
||||
cd: ChannelDownload
|
||||
cd.delete()
|
||||
for download in self.downloads:
|
||||
download: Download
|
||||
download.delete()
|
||||
|
||||
# Must commit before final deletion.
|
||||
# TODO why?
|
||||
session.commit()
|
||||
# session.commit()
|
||||
session.delete(self)
|
||||
|
||||
def update(self, data: dict):
|
||||
|
|
@ -577,7 +544,7 @@ class Channel(ModelHelper, Base):
|
|||
if 'url' in data:
|
||||
data['url'] = data['url'] or None
|
||||
|
||||
channel_downloads = data.pop('channel_downloads', [])
|
||||
downloads = data.pop('downloads', [])
|
||||
|
||||
for key, value in data.items():
|
||||
setattr(self, key, value)
|
||||
|
|
@ -590,15 +557,22 @@ class Channel(ModelHelper, Base):
|
|||
|
||||
session: Session = Session.object_session(self)
|
||||
|
||||
for cd in channel_downloads:
|
||||
# `cd` may be string from config file.
|
||||
url = cd['url'] if isinstance(cd, dict) else cd
|
||||
cd_ = ChannelDownload.get_by_url(url, session=session)
|
||||
if not cd_:
|
||||
cd_ = self.get_or_create_download(url, session=session)
|
||||
# If `cd` is dict, use that frequency. Try to keep the currently set frequency, finally 30 days.
|
||||
frequency = cd['frequency'] if isinstance(cd, dict) else cd_.download.frequency or DownloadFrequency.days30
|
||||
cd_.download.frequency = frequency
|
||||
for download in downloads:
|
||||
# `download` may be string from config file.
|
||||
url = None
|
||||
if isinstance(download, str):
|
||||
url = download
|
||||
elif isinstance(download, dict):
|
||||
url = download['url']
|
||||
elif isinstance(download, Download):
|
||||
url = download.url
|
||||
if not url:
|
||||
raise RuntimeError(f'Unknown download type: {download}')
|
||||
download_ = self.get_or_create_download(url, session=session, reset_attempts=True)
|
||||
# If `download` is dict, use that frequency. Try to keep the currently set frequency, finally 30 days.
|
||||
frequency = download['frequency'] \
|
||||
if isinstance(download, dict) else download_.frequency or DownloadFrequency.days30
|
||||
download_.frequency = frequency
|
||||
|
||||
session.flush()
|
||||
|
||||
|
|
@ -613,7 +587,7 @@ class Channel(ModelHelper, Base):
|
|||
name=self.name,
|
||||
source_id=self.source_id,
|
||||
url=self.url or None,
|
||||
channel_downloads=[i.download_url for i in self.channel_downloads]
|
||||
downloads=[i.url for i in self.downloads]
|
||||
)
|
||||
return config
|
||||
|
||||
|
|
@ -629,7 +603,7 @@ class Channel(ModelHelper, Base):
|
|||
|
||||
def __json__(self) -> dict:
|
||||
d = dict(
|
||||
channel_downloads=self.channel_downloads,
|
||||
downloads=self.downloads,
|
||||
directory=self.directory,
|
||||
id=self.id,
|
||||
name=self.name,
|
||||
|
|
@ -643,7 +617,7 @@ class Channel(ModelHelper, Base):
|
|||
if with_statistics:
|
||||
d['statistics'] = self.get_statistics()
|
||||
if with_downloads:
|
||||
d['channel_downloads'] = self.channel_downloads
|
||||
d['downloads'] = self.downloads
|
||||
return d
|
||||
|
||||
def get_statistics(self):
|
||||
|
|
@ -721,53 +695,41 @@ class Channel(ModelHelper, Base):
|
|||
elif len(matching_entries) > 1:
|
||||
raise RuntimeError(f'More than one info_json entry matches {video_source_id}')
|
||||
|
||||
def get_download(self, url: str = None, download_id: int = None) -> Optional[ChannelDownload]:
|
||||
"""Attempt to get the ChannelDownload that matches the provided parameters."""
|
||||
if not url and not download_id:
|
||||
raise RuntimeError('Must provide either url or download_id')
|
||||
|
||||
for cd in self.channel_downloads:
|
||||
if download_id and cd.download.id == download_id:
|
||||
return cd
|
||||
if url and cd.download.url == url:
|
||||
return cd
|
||||
|
||||
@optional_session
|
||||
def get_or_create_download(self, url: str, session: Session = None, reset_attempts: bool = False) \
|
||||
-> ChannelDownload:
|
||||
"""Get a ChannelDownload record, if it does not exist, create it. Create a Download if necessary
|
||||
def get_or_create_download(self, url: str, session: Session = None, reset_attempts: bool = False) -> Download:
|
||||
"""Get a Download record, if it does not exist, create it. Create a Download if necessary
|
||||
which goes into this Channel's directory."""
|
||||
if not isinstance(url, str) or not url:
|
||||
raise RuntimeError(f'Cannot get Download without url')
|
||||
|
||||
download = download_manager.get_or_create_download(url, session=session, reset_attempts=reset_attempts)
|
||||
download.channel_id = self.id
|
||||
download.settings = download.settings or dict(destination=str(self.directory))
|
||||
from modules.videos.downloader import ChannelDownloader, VideoDownloader
|
||||
download.downloader = ChannelDownloader.name
|
||||
download.sub_downloader = VideoDownloader.name
|
||||
|
||||
if not url:
|
||||
raise RuntimeError(f'Cannot get ChannelDownload without url')
|
||||
|
||||
cd = ChannelDownload.get_by_url(url, session)
|
||||
if not cd:
|
||||
# A Channel is downloaded using the ChannelDownloader first. Then, any missing videos are passed to
|
||||
# VideoDownloader.
|
||||
download = session.query(Download).filter_by(url=url).one_or_none()
|
||||
if not download:
|
||||
download = download_manager.create_download(url, ChannelDownloader.name, session=session,
|
||||
sub_downloader_name=VideoDownloader.name,
|
||||
reset_attempts=reset_attempts,
|
||||
)
|
||||
# Download into this Channel's directory by default.
|
||||
destination = (download.settings or dict()).get('destination')
|
||||
if not destination:
|
||||
settings = (download.settings or dict())
|
||||
settings.update(dict(destination=str(self.directory)))
|
||||
download.settings = settings
|
||||
# Link between Channel and Download.
|
||||
cd = ChannelDownload.get_by_url(url, session)
|
||||
if not cd:
|
||||
cd = ChannelDownload(channel_id=self.id, download_url=url)
|
||||
session.add(cd)
|
||||
logger.debug(f'Created {cd} for {self}')
|
||||
|
||||
return cd
|
||||
return download
|
||||
|
||||
def get_rss_url(self) -> str | None:
|
||||
"""Return the RSS Feed URL for this Channel, if any is possible."""
|
||||
if self.url and self.source_id and 'youtube.com' in self.url:
|
||||
return f'https://www.youtube.com/feeds/videos.xml?channel_id={self.source_id}'
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def get_by_url(url: str, session: Session = None) -> Optional['Channel']:
|
||||
if not url:
|
||||
raise RuntimeError('Must provide URL to get Channel')
|
||||
channel = session.query(Channel).filter_by(url=url).one_or_none()
|
||||
return channel
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def get_by_directory(directory: pathlib.Path | str, session: Session = None) -> Optional['Channel']:
|
||||
if not directory:
|
||||
raise RuntimeError('Must provide directory to get Channel')
|
||||
directory = pathlib.Path(directory) if isinstance(directory, str) else directory
|
||||
directory = str(directory.absolute()) if directory.is_absolute() else str(get_media_directory() / directory)
|
||||
channel = session.query(Channel).filter_by(directory=directory).one_or_none()
|
||||
return channel
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
from copy import copy
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from modules.videos.models import Channel, Video, ChannelDownload
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.common import get_absolute_media_path, get_wrolpi_config
|
||||
from wrolpi.downloader import Download, DownloadFrequency
|
||||
from wrolpi.files import lib as files_lib
|
||||
|
|
@ -149,25 +150,26 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto
|
|||
conf.channels = old_config['channels']
|
||||
conf.save()
|
||||
|
||||
# Config has no channels with a download_frequency.
|
||||
# Two Channels, but neither have Downloads.
|
||||
save_channels_config()
|
||||
import_channels_config()
|
||||
assert not channel1.channel_downloads
|
||||
assert not channel1.downloads
|
||||
assert not channel2.downloads
|
||||
assert len(test_session.query(Channel).all()) == 2
|
||||
assert test_session.query(Download).count() == test_session.query(ChannelDownload).count() == 0
|
||||
assert test_session.query(Download).count() == 0
|
||||
|
||||
# Add a frequency to the Channel.
|
||||
# Add a frequency to `channel1`.
|
||||
channels_config = get_channels_config()
|
||||
update_channel_config(channels_config, 'foo',
|
||||
{'channel_downloads': [
|
||||
{'downloads': [
|
||||
{'url': 'https://example.com/channel1', 'frequency': DownloadFrequency.weekly}
|
||||
]})
|
||||
|
||||
# Download record is created on import.
|
||||
import_channels_config()
|
||||
assert channel1.channel_downloads
|
||||
assert channel1.downloads
|
||||
assert len(test_session.query(Channel).all()) == 2, 'Both Channels should still exist'
|
||||
assert test_session.query(ChannelDownload).count() == 1, 'One Channel has a Download'
|
||||
assert test_session.query(Download).count() == 1, 'One Channel has a Download'
|
||||
download: Download = test_session.query(Download).one()
|
||||
assert download.url == channel1.url
|
||||
assert download.frequency
|
||||
|
|
@ -178,22 +180,23 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto
|
|||
download: Download = test_session.query(Download).one()
|
||||
assert next_download == str(download.next_download)
|
||||
|
||||
# Creating Download that matches Channel2's URL creates a ChannelDownload. Delete it and it should be re-created.
|
||||
# Creating Download that matches Channel2's URL means they are related. Delete it and it should be re-created.
|
||||
channel2.get_or_create_download(channel2.url, test_session)
|
||||
save_channels_config()
|
||||
test_session.query(ChannelDownload).filter_by(download_url=channel2.url).delete()
|
||||
Download.find_by_url(channel2.url).delete(skip=False)
|
||||
test_session.commit()
|
||||
|
||||
# Missing ChannelDownload is re-created on import.
|
||||
# Missing Download is re-created on import.
|
||||
import_channels_config()
|
||||
channel2: Channel = test_session.query(Channel).filter_by(id=channel2.id).one()
|
||||
cds = test_session.query(ChannelDownload).all()
|
||||
assert len(cds) == 2, cds
|
||||
assert channel1.channel_downloads and len(channel1.channel_downloads) == 1
|
||||
assert channel2.channel_downloads and len(channel2.channel_downloads) == 1
|
||||
channel2 = Channel.get_by_id(channel2.id)
|
||||
downloads = test_session.query(Download).all()
|
||||
assert len(downloads) == 1, downloads
|
||||
assert len(channel1.downloads) == 1
|
||||
assert len(channel2.downloads) == 0
|
||||
|
||||
# Add a Download to Channel2 which does not match Channel.url.
|
||||
channel2.get_or_create_download('https://example.org')
|
||||
channel2.get_or_create_download('https://example.org', session=test_session)
|
||||
channel2.get_or_create_download('https://example.com/channel2', session=test_session)
|
||||
test_session.commit()
|
||||
save_channels_config()
|
||||
# Check config is written to match new URLs.
|
||||
|
|
@ -202,21 +205,23 @@ def test_import_channel_downloads(test_async_client, test_session, channel_facto
|
|||
for channel_config in config.channels:
|
||||
if channel_config['source_id'] == channel2.source_id:
|
||||
assert channel_config['source_id'] == 'bar'
|
||||
assert len(channel_config['channel_downloads']) == 2
|
||||
assert set(channel_config['channel_downloads']) == {'https://example.com/channel2', 'https://example.org'}
|
||||
assert len(channel_config['downloads']) == 2
|
||||
assert set(channel_config['downloads']) == {'https://example.com/channel2', 'https://example.org'}
|
||||
else:
|
||||
assert channel_config['source_id'] == 'foo'
|
||||
|
||||
# Reset Downloads. Downloads and ChannelDownloads should be recreated.
|
||||
# Get Channels with their Downloads.
|
||||
channels_backup = copy(get_channels_config().channels)
|
||||
# Reset Downloads. Downloads should be recreated from the channels config file.
|
||||
[i.delete(skip=False) for i in test_session.query(Download).all()]
|
||||
[i.delete() for i in test_session.query(ChannelDownload).all()]
|
||||
# Write config with Channel Downloads now that the delete() has removed them from the config.
|
||||
get_channels_config().channels = channels_backup
|
||||
import_channels_config()
|
||||
channel1, channel2 = test_session.query(Channel).order_by(Channel.url).all()
|
||||
assert channel1.url == 'https://example.com/channel1'
|
||||
assert channel2.url == 'https://example.com/channel2'
|
||||
assert {i.download.url for i in channel1.channel_downloads} == {'https://example.com/channel1'}
|
||||
assert {i.download.url for i in channel2.channel_downloads} == {'https://example.com/channel2',
|
||||
'https://example.org'}
|
||||
assert {i.url for i in channel1.downloads} == {'https://example.com/channel1'}
|
||||
assert {i.url for i in channel2.downloads} == {'https://example.com/channel2', 'https://example.org'}
|
||||
|
||||
|
||||
def test_import_channel_delete_missing_channels(test_session, channel_factory, test_channels_config):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import pytest
|
|||
|
||||
from modules.videos.downloader import VideoDownloader, \
|
||||
get_or_create_channel, channel_downloader, video_downloader
|
||||
from modules.videos.models import Channel, Video, ChannelDownload
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.downloader import Download, DownloadResult
|
||||
from wrolpi.errors import InvalidDownload
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
|
@ -142,7 +142,6 @@ async def test_download_channel(test_session, simple_channel, video_download_man
|
|||
await video_download_manager.wait_for_all_downloads()
|
||||
|
||||
def reset_downloads():
|
||||
[i.delete() for i in test_session.query(ChannelDownload)]
|
||||
[i.delete(skip=False) for i in test_session.query(Download)]
|
||||
|
||||
# Let background tasks run.
|
||||
|
|
@ -161,9 +160,9 @@ async def test_download_channel(test_session, simple_channel, video_download_man
|
|||
test_session.commit()
|
||||
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
|
||||
mock_get_channel.return_value = simple_channel
|
||||
cd = simple_channel.get_or_create_download(url, test_session)
|
||||
cd.download.frequency = 100
|
||||
cd.download.settings = {'title_include': '2'}
|
||||
download = simple_channel.get_or_create_download(url, test_session)
|
||||
download.frequency = 100
|
||||
download.settings = {'title_include': '2'}
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
downloads = video_download_manager.get_once_downloads(test_session)
|
||||
assert {i.url for i in downloads} == {'https://youtube.com/watch?v=video_2_url'}
|
||||
|
|
@ -178,9 +177,9 @@ async def test_download_channel(test_session, simple_channel, video_download_man
|
|||
test_session.commit()
|
||||
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
|
||||
mock_get_channel.return_value = simple_channel
|
||||
cd = simple_channel.get_or_create_download(url, test_session)
|
||||
cd.download.frequency = 100
|
||||
cd.download.settings = {'title_exclude': '2'}
|
||||
download = simple_channel.get_or_create_download(url, test_session)
|
||||
download.frequency = 100
|
||||
download.settings = {'title_exclude': '2'}
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
downloads = video_download_manager.get_once_downloads(test_session)
|
||||
assert {i.url for i in downloads} == {'https://youtube.com/watch?v=video_1_url'}
|
||||
|
|
|
|||
|
|
@ -4,8 +4,9 @@ import shutil
|
|||
import pytest
|
||||
|
||||
from modules.videos import lib
|
||||
from modules.videos.downloader import ChannelDownloader
|
||||
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
|
||||
from modules.videos.models import Video, ChannelDownload
|
||||
from modules.videos.models import Video
|
||||
from wrolpi.downloader import Download, RSSDownloader
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
|
@ -151,7 +152,7 @@ async def test_orphaned_files(test_session, make_files_structure, test_directory
|
|||
])
|
||||
|
||||
|
||||
def test_link_channel_and_downloads(test_session, channel_factory):
|
||||
def test_link_channel_and_downloads(test_session, channel_factory, test_download_manager):
|
||||
channel = channel_factory(
|
||||
url='https://www.youtube.com/c/LearningSelfReliance/videos',
|
||||
source_id='UCng5u6ASda3LNRXJN0JCQJA',
|
||||
|
|
@ -162,9 +163,43 @@ def test_link_channel_and_downloads(test_session, channel_factory):
|
|||
test_session.add_all([download1, download2])
|
||||
test_session.commit()
|
||||
assert test_session.query(Download).count() == 2
|
||||
assert test_session.query(ChannelDownload).count() == 0
|
||||
assert not any(i.channel_id for i in test_download_manager.get_downloads(test_session))
|
||||
|
||||
# `link_channel_and_downloads` creates missing ChannelDownloads.
|
||||
lib.link_channel_and_downloads(session=test_session)
|
||||
assert test_session.query(Download).count() == 2
|
||||
assert test_session.query(ChannelDownload).count() == 2
|
||||
assert all(i.channel_id for i in test_download_manager.get_downloads(test_session))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
def test_channel_channel_downloads_migration(test_async_client, test_session, channel_factory, test_download_manager):
|
||||
"""Test the 8d0d81bc9c34_channel_channel_downloads.py migration."""
|
||||
# A simple Channel which has a download for its URL.
|
||||
channel1 = channel_factory(url='https://example.com/channel1')
|
||||
# A Channel which has two Downloads. One for it's URL, another which is an RSS feed in its directory.
|
||||
channel2 = channel_factory(url='https://example.com/channel2')
|
||||
download1 = Download(url=channel1.url, downloader=ChannelDownloader.name)
|
||||
download2a = Download(url='https://example.com/channel2/rss', downloader=RSSDownloader.name,
|
||||
settings=dict(destination=str(channel2.directory)))
|
||||
download2b = Download(url='https://example.com/channel2', downloader=ChannelDownloader.name,
|
||||
settings=dict(destination=str(channel2.directory)))
|
||||
# A Channel which has a URL, but no Downloads.
|
||||
channel3 = channel_factory()
|
||||
test_session.add_all([download1, download2a, download2b])
|
||||
test_session.commit()
|
||||
|
||||
assert not channel1.downloads
|
||||
assert channel2.directory and not channel2.downloads
|
||||
assert not channel3.downloads
|
||||
|
||||
lib.link_channel_and_downloads(test_session)
|
||||
|
||||
assert channel1.downloads
|
||||
assert channel2.downloads
|
||||
assert not channel3.downloads
|
||||
|
||||
assert test_session.query(Download).count() == 3
|
||||
cd1, cd2a, cd2b = test_session.query(Download).order_by(Download.url).all()
|
||||
assert cd1.url == channel1.url
|
||||
assert cd2a.url == 'https://example.com/channel2'
|
||||
assert cd2b.url == 'https://example.com/channel2/rss'
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
import pytest
|
||||
|
||||
from modules.videos.downloader import ChannelDownloader
|
||||
from modules.videos.models import ChannelDownload
|
||||
from wrolpi.downloader import Download, RSSDownloader
|
||||
from wrolpi.migration import migrate_channel_downloads
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
def test_channel_channel_downloads_migration(test_async_client, test_session, channel_factory, test_download_manager):
|
||||
"""Test the 8d0d81bc9c34_channel_channel_downloads.py migration."""
|
||||
# A simple Channel which has a download for its URL.
|
||||
channel1 = channel_factory(url='https://example.com/channel1')
|
||||
# A Channel which has two Downloads. One for it's URL, another which is an RSS feed in its directory.
|
||||
channel2 = channel_factory(url='https://example.com/channel2')
|
||||
download1 = Download(url=channel1.url, downloader=ChannelDownloader.name)
|
||||
download2a = Download(url='https://example.com/channel2/rss', downloader=RSSDownloader.name,
|
||||
settings=dict(destination=str(channel2.directory)))
|
||||
download2b = Download(url='https://example.com/channel2', downloader=ChannelDownloader.name,
|
||||
settings=dict(destination=str(channel2.directory)))
|
||||
# A Channel which has a URL, but no Downloads.
|
||||
channel3 = channel_factory()
|
||||
test_session.add_all([download1, download2a, download2b])
|
||||
test_session.commit()
|
||||
|
||||
assert not channel1.channel_downloads
|
||||
assert channel2.directory and not channel2.channel_downloads
|
||||
assert not channel3.channel_downloads
|
||||
|
||||
migrate_channel_downloads(test_session)
|
||||
|
||||
assert channel1.channel_downloads
|
||||
assert channel2.channel_downloads
|
||||
assert not channel3.channel_downloads
|
||||
|
||||
assert test_session.query(ChannelDownload).count() == 3
|
||||
cd1, cd2a, cd2b = test_session.query(ChannelDownload).order_by(ChannelDownload.download_url).all()
|
||||
assert cd1.download_url == channel1.url
|
||||
assert cd2a.download_url == 'https://example.com/channel2'
|
||||
assert cd2b.download_url == 'https://example.com/channel2/rss'
|
||||
|
|
@ -18,9 +18,9 @@ from urllib.parse import urlparse
|
|||
import feedparser
|
||||
import pytz
|
||||
from feedparser import FeedParserDict
|
||||
from sqlalchemy import Column, Integer, String, Text
|
||||
from sqlalchemy import Column, Integer, String, Text, ForeignKey
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm import Session, relationship
|
||||
from sqlalchemy.sql import Delete
|
||||
|
||||
from wrolpi import flags
|
||||
|
|
@ -30,7 +30,7 @@ from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, C
|
|||
limit_concurrent, wrol_mode_enabled
|
||||
from wrolpi.dates import TZDateTime, now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs, optional_session
|
||||
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
|
||||
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload
|
||||
from wrolpi.vars import PYTEST, SIMULTANEOUS_DOWNLOAD_DOMAINS
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -111,7 +111,10 @@ class Download(ModelHelper, Base): # noqa
|
|||
next_download = Column(TZDateTime)
|
||||
settings = Column(JSONB) # information about how the download should happen (destination, etc.)
|
||||
status = Column(String, default=DownloadStatus.new) # `DownloadStatus` enum.
|
||||
_manager: 'DownloadManager' = None
|
||||
|
||||
# A Download may be associated with a Channel (downloads all Channel videos, or a playlist, etc.).
|
||||
channel_id = Column(Integer, ForeignKey('channel.id'))
|
||||
channel = relationship('Channel', primaryjoin='Download.channel_id==Channel.id', back_populates='downloads')
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
|
@ -126,6 +129,7 @@ class Download(ModelHelper, Base): # noqa
|
|||
|
||||
def __json__(self) -> dict:
|
||||
d = dict(
|
||||
channel_id=self.channel_id,
|
||||
downloader=self.downloader,
|
||||
frequency=self.frequency,
|
||||
id=self.id,
|
||||
|
|
@ -215,17 +219,46 @@ class Download(ModelHelper, Base): # noqa
|
|||
|
||||
session = Session.object_session(self)
|
||||
|
||||
# Delete ChannelDownload, if any exists.
|
||||
from modules.videos.models import ChannelDownload
|
||||
cds = session.query(ChannelDownload).filter(ChannelDownload.download == self).all()
|
||||
for cd in cds:
|
||||
session.delete(cd)
|
||||
|
||||
session.delete(self)
|
||||
session.commit()
|
||||
if self.channel_id:
|
||||
# Save Channels config if this download was associated with a Channel.
|
||||
from modules.videos.lib import save_channels_config
|
||||
save_channels_config()
|
||||
# Save download config again because this download is now removed from the download lists.
|
||||
api_app.add_task(save_downloads_config())
|
||||
|
||||
@staticmethod
|
||||
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
|
||||
download = session.query(Download).filter(Download.id == id_).one_or_none()
|
||||
return download
|
||||
|
||||
@classmethod
|
||||
@optional_session
|
||||
def find_by_id(cls, id_: int, session: Session = None) -> 'Download':
|
||||
if download := cls.get_by_id(id_, session):
|
||||
return download
|
||||
raise UnknownDownload(f'Cannot find Download with id: {id_}')
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def get_by_url(url: str, session: Session = None) -> Optional['Download']:
|
||||
download = session.query(Download).filter(Download.url == url).one_or_none()
|
||||
return download
|
||||
|
||||
@classmethod
|
||||
@optional_session
|
||||
def find_by_url(cls, url: str, session: Session = None) -> 'Download':
|
||||
if download := cls.get_by_url(url, session):
|
||||
return download
|
||||
raise UnknownDownload(f'Cannot find Download with URL: {url}')
|
||||
|
||||
@staticmethod
|
||||
@optional_session
|
||||
def get_by_channel_id(channel_id: int, session: Session = None) -> Optional['Download']:
|
||||
download = session.query(Download).filter(Download.channel_id == channel_id).one_or_none()
|
||||
return download
|
||||
|
||||
|
||||
class Downloader:
|
||||
name: str = None
|
||||
|
|
@ -474,20 +507,6 @@ class DownloadManager:
|
|||
download = Download(url=url, status='new')
|
||||
session.add(download)
|
||||
|
||||
from modules.videos.models import Channel, ChannelDownload
|
||||
channel: Channel = session.query(Channel).filter(Channel.url == url).one_or_none()
|
||||
if channel:
|
||||
# This Download needs to be associated with the Channel.
|
||||
cd = ChannelDownload.get_by_url(url, session)
|
||||
if not cd:
|
||||
cd = ChannelDownload(channel=channel, download=download)
|
||||
session.add(cd)
|
||||
else:
|
||||
cd.download = download
|
||||
if cd not in channel.channel_downloads:
|
||||
channel.channel_downloads.append(cd)
|
||||
|
||||
session.flush([cd, channel])
|
||||
return download
|
||||
|
||||
@optional_session
|
||||
|
|
@ -518,16 +537,15 @@ class DownloadManager:
|
|||
download.downloader = downloader_name
|
||||
download.sub_downloader = sub_downloader_name
|
||||
download.settings = settings if settings is not None else download.settings
|
||||
downloads.append(download)
|
||||
|
||||
# Create ChannelDownload for any download that is being downloaded into a Channel's Directory.
|
||||
from modules.videos.models import Channel, ChannelDownload
|
||||
if download.settings and (destination := download.settings.get('destination')):
|
||||
channel = Channel.get_by_path(destination, session)
|
||||
cd = ChannelDownload.get_by_url(url, session)
|
||||
if channel and not cd:
|
||||
cd = ChannelDownload(channel=channel, download=download)
|
||||
session.add(cd)
|
||||
from modules.videos.models import Channel
|
||||
if channel := Channel.get_by_url(url=download.url, session=session):
|
||||
download.channel_id = channel.id
|
||||
if destination := (download.settings or dict()).get('destination'):
|
||||
if channel := Channel.get_by_directory(destination):
|
||||
download.channel_id = channel.id
|
||||
|
||||
downloads.append(download)
|
||||
|
||||
session.flush(downloads)
|
||||
try:
|
||||
|
|
@ -852,9 +870,8 @@ class DownloadManager:
|
|||
status,
|
||||
sub_downloader,
|
||||
url,
|
||||
cd.channel_id
|
||||
channel_id
|
||||
FROM download
|
||||
LEFT OUTER JOIN channel_download cd on download.url = cd.download_url
|
||||
WHERE frequency IS NOT NULL
|
||||
ORDER BY
|
||||
{self._status_order},
|
||||
|
|
@ -1055,7 +1072,7 @@ async def signal_download_download(download_id: int, download_url: str):
|
|||
|
||||
with get_db_session(commit=True) as session:
|
||||
# Mark the download as started in new session so the change is committed.
|
||||
download = session.query(Download).filter_by(id=download_id).one()
|
||||
download = Download.find_by_id(download_id, session=session)
|
||||
download.started()
|
||||
download_domain = download.domain
|
||||
|
||||
|
|
|
|||
|
|
@ -51,6 +51,12 @@ class InvalidDownload(APIError):
|
|||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class UnknownDownload(APIError):
|
||||
code = 'UNKNOWN_DOWNLOAD'
|
||||
message = 'The Download cannot be found.'
|
||||
status_code = HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class UnrecoverableDownloadError(APIError):
|
||||
code = 'UNRECOVERABLE_DOWNLOAD_ERROR'
|
||||
message = 'Download experienced an error which cannot be recovered. Download will be deleted.'
|
||||
|
|
|
|||
|
|
@ -1,32 +0,0 @@
|
|||
from modules.videos.models import Channel, ChannelDownload
|
||||
from wrolpi.downloader import Download
|
||||
|
||||
|
||||
def migrate_channel_downloads(session):
|
||||
channels_by_url = {i.url: i for i in session.query(Channel).all()}
|
||||
downloads = session.query(Download).all()
|
||||
need_commit = False
|
||||
for download in downloads:
|
||||
channel = channels_by_url.get(download.url)
|
||||
if channel:
|
||||
cd = session.query(ChannelDownload).filter_by(download_url=download.url).one_or_none()
|
||||
if not cd:
|
||||
cd = ChannelDownload(channel_id=channel.id, download_url=download.url)
|
||||
session.add(cd)
|
||||
need_commit = True
|
||||
|
||||
for download in downloads:
|
||||
destination = download.settings.get('destination') if download.settings else None
|
||||
if not destination:
|
||||
continue
|
||||
|
||||
channel = session.query(Channel).filter_by(directory=destination).one_or_none()
|
||||
if channel:
|
||||
cd = session.query(ChannelDownload).filter_by(download_url=download.url).one_or_none()
|
||||
if not cd:
|
||||
cd = ChannelDownload(channel_id=channel.id, download_url=download.url)
|
||||
session.add(cd)
|
||||
need_commit = True
|
||||
|
||||
if need_commit:
|
||||
session.commit()
|
||||
|
|
@ -535,8 +535,8 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
|
|||
await assert_results(
|
||||
dict(search_str='foo'),
|
||||
[
|
||||
{'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'channel_downloads': []},
|
||||
{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'channel_downloads': []},
|
||||
{'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []},
|
||||
{'directory': 'Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'downloads': []},
|
||||
],
|
||||
[{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 1}],
|
||||
)
|
||||
|
|
@ -545,14 +545,14 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
|
|||
await assert_results(
|
||||
dict(search_str='foo l'),
|
||||
[
|
||||
{'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'channel_downloads': []}],
|
||||
{'directory': 'Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}],
|
||||
[],
|
||||
)
|
||||
|
||||
await assert_results(
|
||||
dict(search_str='bar'),
|
||||
[
|
||||
{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'channel_downloads': []}
|
||||
{'directory': 'Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'downloads': []}
|
||||
],
|
||||
[{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 2}],
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue