wrolpi/modules/videos/channel/lib.py
Roland f14a230bc6
RSSDownloader can schedule downloads for other Downloaders using sub_downloader.
RSS download can be submitted in UI.

Checking if entry has link.

A video playlist can be downloaded.

Passing sub_downloader when submitting RSS feed.

Adding Archive downloader button.

Playlists need more info.

Removing expunge, it causes errors elsewhere.  Fixing tests.

Showing errors on recurring downloads.  Ignoring CharacterEncodingOverride bozo error for rss feed.

Changing download button order.

Fixing bug where channel attributes were unavailable after session.  Simplifying video format selection.

Calling validate_video in Video.validate() to avoid duplicate code.

Adding more frequent downloads for RSS Feeds.

DownloadManager skips downloads in it's list.

Renaming get_next_download to more clear calculate_next_download.

Stopping download manager when server is shutting down.

Not shutting down downloads during tests, not all tests use downloads.

Making methods static.
2022-06-08 11:56:42 -06:00

202 lines
6.4 KiB
Python

from pathlib import Path
from typing import List, Dict, Union
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound
from wrolpi.common import run_after, logger, \
get_media_directory
from wrolpi.db import get_db_curs, optional_session
from wrolpi.downloader import download_manager
from wrolpi.errors import UnknownChannel, UnknownDirectory, APIError, ValidationError, InvalidDownload
from .. import schema
from ..common import check_for_channel_conflicts
from ..lib import save_channels_config
from ..models import Channel
logger = logger.getChild(__name__)
async def get_minimal_channels() -> List[dict]:
"""
Get the minimum amount of information necessary about all channels.
"""
with get_db_curs() as curs:
# Get all channels, even if they don't have videos.
stmt = '''
SELECT
c.id, name, directory, url, download_frequency, info_date,
(select next_download from download d where d.url=c.url) AS next_download
FROM
channel AS c
ORDER BY LOWER(name)
'''
curs.execute(stmt)
channels = list(map(dict, curs.fetchall()))
video_counts = await get_channels_video_count()
for channel in channels:
channel_id = channel['id']
channel['video_count'] = video_counts[channel_id]
return channels
async def get_channels_video_count() -> Dict[int, int]:
"""
Add video counts to all channels
"""
with get_db_curs() as curs:
stmt = 'SELECT id FROM channel'
curs.execute(stmt)
# Get all channel IDs, start them with a count of 0.
video_counts = {int(i['id']): 0 for i in curs.fetchall()}
stmt = '''
SELECT
c.id, COUNT(v.id) AS video_count
FROM
channel AS c
LEFT JOIN video AS v ON v.channel_id = c.id
WHERE
v.video_path IS NOT NULL
GROUP BY 1
'''
curs.execute(stmt)
# Replace all the counts of those channels with videos.
video_counts.update({int(i['id']): int(i['video_count']) for i in curs.fetchall()})
return video_counts
COD = Union[dict, Channel]
@optional_session
def get_channel(session: Session, *, channel_id: int = None, source_id: str = None, url: str = None,
directory: str = None, name: str = None, return_dict: bool = True) -> COD:
"""
Attempt to find a Channel using the provided params. The params are in order of reliability.
Raises UnknownChannel if no channel is found.
"""
channel: COD = None # noqa
# Try to find the channel by the most reliable methods first.
if channel_id:
channel = session.query(Channel).filter_by(id=channel_id).one_or_none()
if not channel and source_id:
channel = session.query(Channel).filter_by(source_id=source_id).one_or_none()
if not channel and url:
channel = session.query(Channel).filter_by(url=url).one_or_none()
if not channel and directory:
directory = Path(directory)
if not directory.is_absolute():
directory = get_media_directory() / directory
channel = session.query(Channel).filter_by(directory=directory).one_or_none()
if not channel and name:
channel = session.query(Channel).filter_by(name=name).one_or_none()
if not channel:
raise UnknownChannel(f'No channel matches {channel_id=} {source_id=} {url=} {directory=}')
logger.debug(f'Found {channel=} using {channel_id=} {source_id=} {url=} {directory=}')
session.refresh(channel)
if return_dict:
statistics = channel.get_statistics()
channel = channel.dict()
channel['statistics'] = statistics
return channel
@run_after(save_channels_config)
@optional_session
def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
"""Update a Channel's DB record"""
try:
channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
except NoResultFound:
raise UnknownChannel()
# Only update directory if it was empty
if data.directory and not channel.directory:
data.directory = get_media_directory() / data.directory
if not data.directory.is_dir():
if data.mkdir:
data.directory.mkdir()
else:
raise UnknownDirectory()
# Verify that the URL/Name/directory aren't taken
check_for_channel_conflicts(
session,
id_=channel.id,
url=data.url,
name=data.name,
directory=data.directory,
)
# Apply the changes now that we've OK'd them
channel.update(data.__dict__)
session.commit()
return channel
@run_after(save_channels_config)
@optional_session
def create_channel(session: Session, data: schema.ChannelPostRequest, return_dict: bool = True) -> Union[Channel, dict]:
"""
Create a new Channel. Check for conflicts with existing Channels.
"""
try:
# Verify that the URL/Name/directory aren't taken
check_for_channel_conflicts(
session,
url=data.url or None,
name=data.name,
directory=data.directory,
source_id=data.source_id,
)
except APIError as e:
raise ValidationError() from e
channel = Channel(
name=data.name,
url=data.url or None,
match_regex=data.match_regex,
directory=data.directory, # noqa
download_frequency=data.download_frequency,
source_id=data.source_id,
)
session.add(channel)
session.commit()
session.flush()
session.refresh(channel)
return channel.dict() if return_dict else channel
@run_after(save_channels_config)
@optional_session
def delete_channel(session: Session, *, channel_id: int):
try:
channel = session.query(Channel).filter_by(id=channel_id).one()
except NoResultFound:
raise UnknownChannel()
channel.delete_with_videos()
session.commit()
def download_channel(id_: int):
"""Create a Download record for a Channel's entire catalog. Start downloading."""
channel = get_channel(channel_id=id_, return_dict=False)
session = Session.object_session(channel)
download = channel.get_download()
if not download:
raise InvalidDownload(f'Channel {channel.name} does not have a download!')
download.renew(reset_attempts=True)
session.commit()
download_manager.start_downloads()