From aa203cc102b3074bc130e836002b5418f5f39ef7 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 8 Apr 2022 09:57:04 -0600 Subject: [PATCH 1/2] Moving non-video functions to wrolpi.common. Handling Channel/Downlaod relationship in Channel.update. Removing default download freqeuncy so channels will not be downloaded without a url and frequency. Adding channel unique url constraint. Handling when a Channel's directory may still be relative in DB. Adding TODOs to remove old favorites. Removing outdated favorites from config after favorites have been created. Testing that old-style favorited videos in the config are imported. Removing channels_by_link. Using list to store channels in config. Moving channel import function into lib. Supporting old channel import with links. Bumping version for channel.link removal. Removing all reference to Channel.link in app and api. Adding relative property to MediaPathType. Testing that a channel can be downloaded after it has a frequency added. All tests pass. Video and Channel tests pass without channel.link. Removing channel.link usage in channel api. Changing video Statistics page to About. --- .../d8fcdb7773c0_drop_channel_link.py | 37 + app/src/App.js | 2 +- app/src/api.js | 26 +- app/src/components/Channels.js | 22 +- app/src/components/Common.js | 4 +- app/src/components/VideoPlayer.js | 8 +- app/src/components/Videos.js | 24 +- modules/archive/lib.py | 1 - modules/files/lib.py | 1 + modules/files/test/test_api.py | 6 +- modules/videos/api.py | 38 +- modules/videos/channel/api.py | 38 +- modules/videos/channel/lib.py | 190 +++-- modules/videos/channel/test/test_api.py | 655 ++++++++---------- modules/videos/channel/test/test_lib.py | 59 ++ modules/videos/common.py | 204 +----- modules/videos/conftest.py | 41 +- modules/videos/downloader.py | 65 +- modules/videos/lib.py | 155 ++++- modules/videos/models.py | 61 +- modules/videos/schema.py | 23 +- modules/videos/test/common.py | 75 -- modules/videos/test/test_api.py | 72 ++ modules/videos/test/test_common.py | 214 +++--- modules/videos/test/test_downloader.py | 230 +++--- modules/videos/video/api.py | 2 +- modules/videos/video/lib.py | 8 +- modules/videos/video/test/test_api.py | 234 +++---- wrolpi/__init__.py | 3 + wrolpi/cmd.py | 2 +- wrolpi/common.py | 15 +- wrolpi/conftest.py | 1 - wrolpi/errors.py | 2 +- wrolpi/media_path.py | 11 +- wrolpi/schema.py | 2 +- wrolpi/version.py | 6 +- 36 files changed, 1229 insertions(+), 1308 deletions(-) create mode 100644 alembic/versions/d8fcdb7773c0_drop_channel_link.py create mode 100644 modules/videos/channel/test/test_lib.py delete mode 100644 modules/videos/test/common.py create mode 100644 modules/videos/test/test_api.py diff --git a/alembic/versions/d8fcdb7773c0_drop_channel_link.py b/alembic/versions/d8fcdb7773c0_drop_channel_link.py new file mode 100644 index 00000000..7ec3a943 --- /dev/null +++ b/alembic/versions/d8fcdb7773c0_drop_channel_link.py @@ -0,0 +1,37 @@ +"""Drop channel.link. + +Revision ID: d8fcdb7773c0 +Revises: 2fb39c1861c7 +Create Date: 2022-04-08 12:52:02.877449 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'd8fcdb7773c0' +down_revision = '2fb39c1861c7' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE channel DROP CONSTRAINT IF EXISTS channel_link_key') + session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS link') + session.execute('DROP INDEX IF EXISTS channel_url_key') + session.execute("UPDATE channel SET url=null WHERE url=''") + session.execute('CREATE UNIQUE INDEX channel_url_key ON channel (url) WHERE url IS NOT NULL') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE channel ADD CONSTRAINT channel_link_key UNIQUE (link)') + session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS link TEXT') + # Notice we are not removing the channel_url_key constraint. diff --git a/app/src/App.js b/app/src/App.js index 2ed6b604..a9725839 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -82,7 +82,7 @@ export default function App() { <> - + diff --git a/app/src/api.js b/app/src/api.js index 444f4110..d906eeb7 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -95,9 +95,9 @@ let apiPatch = async (url, body) => { return await apiCall(url, 'PATCH', body) }; -export async function updateChannel(link, channel) { +export async function updateChannel(id, channel) { channel = emptyToNull(channel); - return await apiPut(`${VIDEOS_API}/channels/${link}`, channel); + return await apiPut(`${VIDEOS_API}/channels/${id}`, channel); } export async function createChannel(channel) { @@ -105,8 +105,8 @@ export async function createChannel(channel) { return await apiPost(`${VIDEOS_API}/channels`, channel); } -export async function deleteChannel(channel_link) { - return apiDelete(`${VIDEOS_API}/channels/${channel_link}`); +export async function deleteChannel(id) { + return apiDelete(`${VIDEOS_API}/channels/${id}`); } export async function getChannels() { @@ -114,12 +114,12 @@ export async function getChannels() { return (await response.json())['channels']; } -export async function getChannel(link) { - let response = await apiGet(`${VIDEOS_API}/channels/${link}`); +export async function getChannel(id) { + let response = await apiGet(`${VIDEOS_API}/channels/${id}`); return (await response.json())['channel']; } -export async function searchVideos(offset, limit, channel_link, searchStr, order_by, filters) { +export async function searchVideos(offset, limit, channel_id, searchStr, order_by, filters) { // Build a search query to retrieve a list of videos from the API offset = offset || 0; limit = limit || DEFAULT_LIMIT; @@ -128,8 +128,8 @@ export async function searchVideos(offset, limit, channel_link, searchStr, order if (searchStr) { body.search_str = searchStr; } - if (channel_link) { - body.channel_link = channel_link; + if (channel_id) { + body.channel_id = channel_id; } body.order_by = order_by ? order_by : 'rank' @@ -211,8 +211,8 @@ export async function download() { return await response.json(); } -export async function downloadChannel(link) { - let url = `${VIDEOS_API}/download/${link}`; +export async function downloadChannel(id) { + let url = `${VIDEOS_API}/download/${id}`; let response = await apiPost(url); let json = await response.json(); if (response.status === 400 && json['code'] === 30) { @@ -225,8 +225,8 @@ export async function downloadChannel(link) { } } -export async function refreshChannel(link) { - let url = `${VIDEOS_API}/refresh/${link}`; +export async function refreshChannel(id) { + let url = `${VIDEOS_API}/refresh/${id}`; await fetch(url, {method: 'POST'}); } diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index 4b690f80..fe1a4a87 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -154,8 +154,8 @@ class ChannelPage extends APIForm { async componentDidMount() { if (!this.props.create) { - let channel_link = this.props.match.params.channel_link; - let channel = await getChannel(channel_link); + let channel_id = this.props.match.params.channel_id; + let channel = await getChannel(channel_id); let statistics = channel.statistics; if (statistics) { this.setState({statistics: statistics || null}); @@ -173,7 +173,7 @@ class ChannelPage extends APIForm { handleConfirm = async () => { this.setState({deleteOpen: false}); - let response = await deleteChannel(this.props.match.params.channel_link); + let response = await deleteChannel(this.props.match.params.channel_id); if (response.status === 204) { this.props.history.push({ pathname: '/videos/channel' @@ -194,7 +194,7 @@ class ChannelPage extends APIForm { if (this.props.create) { response = await createChannel(inputs); } else { - response = await updateChannel(this.props.match.params.channel_link, inputs); + response = await updateChannel(this.props.match.params.channel_id, inputs); } } finally { @@ -215,7 +215,7 @@ class ChannelPage extends APIForm { this.setSuccess( 'Channel created', - Your channel was created. View it here + Your channel was created. View it here ); } else { @@ -262,12 +262,12 @@ class ChannelPage extends APIForm { downloadChannel = async (e) => { e.preventDefault(); - return await downloadChannel(this.props.match.params.channel_link); + return await downloadChannel(this.props.match.params.channel_id); } refreshChannel = async (e) => { e.preventDefault(); - return await refreshChannel(this.props.match.params.channel_link); + return await refreshChannel(this.props.match.params.channel_id); } handleDeleteButton = (e) => { @@ -470,8 +470,8 @@ export function NewChannel(props) { class ChannelRow extends React.Component { constructor(props) { super(props); - this.editTo = `/videos/channel/${props.channel.link}/edit`; - this.videosTo = `/videos/channel/${props.channel.link}/video`; + this.editTo = `/videos/channel/${props.channel.id}/edit`; + this.videosTo = `/videos/channel/${props.channel.id}/video`; } render() { @@ -646,7 +646,7 @@ export class Channels extends React.Component { {tableHeader} - {this.state.results.map((channel) => )}
@@ -655,7 +655,7 @@ export class Channels extends React.Component { {this.state.results.map((channel) => - )} + )}
diff --git a/app/src/components/Common.js b/app/src/components/Common.js index d3e8f0ef..8d46c1e4 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -135,8 +135,8 @@ export function VideoCard({video}) { let channel = video.channel ? video.channel : null; let channel_url = null; if (channel) { - channel_url = `/videos/channel/${channel.link}/video`; - video_url = `/videos/channel/${channel.link}/video/${video.id}`; + channel_url = `/videos/channel/${channel.id}/video`; + video_url = `/videos/channel/${channel.id}/video/${video.id}`; } let poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null; diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index e27be091..526ebd5d 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -76,8 +76,8 @@ function VideoPage(props) { }; - let statisticsPane = { - menuItem: 'Statistics', render: () => + let aboutPane = { + menuItem: 'About', render: () =>

Size

{video.size ? humanFileSize(video.size) : '???'}

@@ -104,7 +104,7 @@ function VideoPage(props) {
}; - let tabPanes = [descriptionPane, statisticsPane, captionsPane]; + let tabPanes = [descriptionPane, aboutPane, captionsPane]; return ( <> @@ -135,7 +135,7 @@ function VideoPage(props) {

{video.title || video.video_path}

{video.upload_date &&

{uploadDate(video.upload_date)}

}

- {channel && + {channel && {channel.name} }

diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index f068f252..7cf3b1a6 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -103,8 +103,8 @@ export class VideoWrapper extends React.Component { async fetchVideo() { // Get and display the Video specified in the Router match let [video, prev, next] = await getVideo(this.props.match.params.video_id); - let channel_link = this.props.match.params.channel_link; - let channel = channel_link ? await getChannel(channel_link) : null; + let channel_id = this.props.match.params.channel_id; + let channel = channel_id ? await getChannel(channel_id) : null; let no_channel = false; if (!channel) { no_channel = true; @@ -222,7 +222,7 @@ class Videos extends React.Component { } async componentDidMount() { - if (this.props.match.params.channel_link) { + if (this.props.match.params.channel_id) { await this.fetchChannel(); } else { await this.fetchVideos(); @@ -233,7 +233,7 @@ class Videos extends React.Component { async componentDidUpdate(prevProps, prevState, snapshot) { let params = this.props.match.params; - let channelChanged = params.channel_link !== prevProps.match.params.channel_link; + let channelChanged = params.channel_id !== prevProps.match.params.channel_id; let pageChanged = ( prevState.activePage !== this.state.activePage || prevState.searchOrder !== this.state.searchOrder || @@ -269,10 +269,10 @@ class Videos extends React.Component { async fetchChannel() { // Get and display the channel specified in the Router match - let channel_link = this.props.match.params.channel_link; + let channel_id = this.props.match.params.channel_id; let channel = null; - if (channel_link) { - channel = await getChannel(channel_link); + if (channel_id) { + channel = await getChannel(channel_id); } this.setState({ channel, @@ -300,13 +300,13 @@ class Videos extends React.Component { async fetchVideos() { this.setState({videos: null}); let offset = this.state.limit * this.state.activePage - this.state.limit; - let channel_link = this.state.channel ? this.state.channel.link : null; + let channel_id = this.state.channel ? this.state.channel.id : null; let {queryStr, searchOrder, limit} = this.state; // Pass any enabled filters to the search. let filtersArr = this.enabledFilters(); - let [videos, total] = await searchVideos(offset, limit, channel_link, queryStr, searchOrder, filtersArr); + let [videos, total] = await searchVideos(offset, limit, channel_id, queryStr, searchOrder, filtersArr); let totalPages = Math.round(total / this.state.limit) || 1; this.setState({videos, totalPages}); @@ -428,7 +428,7 @@ class Videos extends React.Component { <>     - + @@ -527,7 +527,7 @@ export function VideosRoute(props) { - } /> - + ) } diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 5a6aac98..16379707 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -609,7 +609,6 @@ def search(search_str: str, domain: str, limit: int, offset: int) -> Tuple[List[ OFFSET %(offset)s LIMIT %(limit)s ''' - print(stmt, offset, limit) curs.execute(stmt, params) results = [dict(i) for i in curs.fetchall()] total = results[0]['total'] if results else 0 diff --git a/modules/files/lib.py b/modules/files/lib.py index 4a215094..8e4d6c24 100644 --- a/modules/files/lib.py +++ b/modules/files/lib.py @@ -17,6 +17,7 @@ from wrolpi.vars import PYTEST try: import magic + have_magic = True except ImportError: logger.error('Could not import magic. Is libmagic installed?') diff --git a/modules/files/test/test_api.py b/modules/files/test/test_api.py index 2b9875c4..eb79d452 100644 --- a/modules/files/test/test_api.py +++ b/modules/files/test/test_api.py @@ -6,7 +6,7 @@ from unittest import mock import pytest from wrolpi.errors import API_ERRORS, WROLModeEnabled -from wrolpi.test.common import assert_dict_contains, skip_circleci +from wrolpi.test.common import assert_dict_contains from wrolpi.vars import PROJECT_DIR @@ -128,8 +128,6 @@ def do_search(test_client, search_str, total, expected): assert file['path'] == file['key'] -# TODO this fails on CircleCI. The order is wrong. Why? -@skip_circleci def test_files_search(test_session, test_client, make_files_structure): # You can search an empty directory. do_search(test_client, 'nothing', 0, []) @@ -154,8 +152,8 @@ def test_files_search(test_session, test_client, make_files_structure): do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)]) do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)]) do_search(test_client, 'baz', 2, [ - dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736), dict(path='baz.mp4', mimetype='video/mp4', size=1055736), + dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736), ]) do_search(test_client, 'two', 1, [dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736)]) do_search(test_client, 'nothing', 0, []) diff --git a/modules/videos/api.py b/modules/videos/api.py index a07a6366..6e41c581 100644 --- a/modules/videos/api.py +++ b/modules/videos/api.py @@ -1,6 +1,7 @@ import asyncio from functools import wraps from http import HTTPStatus +from typing import List from sanic import Blueprint, response from sanic.request import Request @@ -32,12 +33,12 @@ refresh_queue, refresh_event = create_websocket_feed('refresh', '/feeds/refresh' @content_bp.post('/refresh') -@content_bp.post('/refresh/') +@content_bp.post('/refresh/') @openapi.description('Search for videos that have previously been downloaded and stored.') @openapi.response(HTTPStatus.OK, schema.StreamResponse) @openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) @wrol_mode_check -async def refresh(_, link: str = None): +async def refresh(_, channel_id: int = None): refresh_logger = logger.getChild('refresh') stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/refresh') @@ -51,8 +52,8 @@ async def refresh(_, link: str = None): try: refresh_logger.info('refresh started') - channel_links = [link] if link else None - await refresh_videos(channel_links) + channel_ids = [channel_id] if channel_id else None + await refresh_videos(channel_ids) refresh_logger.info('refresh complete') except Exception: @@ -66,34 +67,19 @@ async def refresh(_, link: str = None): return response.json({'code': 'stream-started', 'stream_url': stream_url}) -download_queue, download_event = create_websocket_feed('download', '/feeds/download', content_bp) - - -@content_bp.post('/download') -@content_bp.post('/download/') -@openapi.description('Update channel catalogs, download any missing videos') +@content_bp.post('/download/') +@openapi.description('Update a channel catalog, download any missing videos') @openapi.response(HTTPStatus.OK, schema.StreamResponse) @openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) @wrol_mode_check -def download(_, link: str = None): - download_logger = logger.getChild('download') - - stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/download') - # Only one download can run at a time - if download_event.is_set(): - return response.json({'error': 'download already running', 'stream_url': stream_url}, HTTPStatus.CONFLICT) - if refresh_event.is_set(): - return response.json({'error': 'Refresh is running. Cannot download.'}) - - channel_lib.download_channel(link) - - download_logger.debug('do_download scheduled') - return response.json({'code': 'stream-started', 'stream_url': stream_url}) +def download(_, channel_id: int = None): + channel_lib.download_channel(channel_id) + return response.empty() @wraps(lib.refresh_videos) -async def refresh_videos(channel_links: list = None): - return lib.refresh_videos(channel_links=channel_links) +async def refresh_videos(channel_ids: List[int] = None): + return lib.refresh_videos(channel_ids=channel_ids) @content_bp.post('/favorite') diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py index 6e942537..1ec6adba 100644 --- a/modules/videos/channel/api.py +++ b/modules/videos/channel/api.py @@ -7,11 +7,13 @@ from sanic_ext import validate from sanic_ext.extensions.openapi import openapi from wrolpi.common import logger, wrol_mode_check, make_media_directory, \ - get_media_directory + get_media_directory, run_after from wrolpi.root_api import json_response from wrolpi.schema import JSONErrorResponse +from wrolpi.vars import PYTEST from . import lib from .. import schema +from ..lib import save_channels_config channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels') @@ -26,11 +28,11 @@ async def get_channels(_: Request): return json_response({'channels': channels}) -@channel_bp.get('/') +@channel_bp.get('/') @openapi.description('Get a Channel') @openapi.response(HTTPStatus.OK, schema.ChannelResponse) -def channel_get(_: Request, link: str): - channel = lib.get_channel(link=link) +def channel_get(_: Request, channel_id: int = None): + channel = lib.get_channel(channel_id=channel_id) channel.pop('info_json') return json_response({'channel': channel}) @@ -49,20 +51,19 @@ def channel_post(_: Request, body: schema.ChannelPostRequest): if not body.directory.is_dir() and body.mkdir: make_media_directory(body.directory) - bd_edit = body.__dict__ - bd_edit['name'] = bd_edit['name'].strip() - channel = lib.create_channel(bd_edit) # TODO don't use the dataclass as a dict. + channel = lib.create_channel(data=body, return_dict=False) # Refresh the videos asynchronously - from ..api import refresh_videos - coro = refresh_videos([channel['link']]) - asyncio.ensure_future(coro) + if not PYTEST: + from ..api import refresh_videos + coro = refresh_videos([channel.id]) + asyncio.ensure_future(coro) return response.json({'success': 'Channel created successfully'}, HTTPStatus.CREATED, - {'Location': f'/api/videos/channels/{channel["link"]}'}) + {'Location': f'/api/videos/channels/{channel.id}'}) -@channel_bp.put('/') +@channel_bp.put('/') @openapi.definition( description='Update a Channel', body=schema.ChannelPutRequest, @@ -70,17 +71,18 @@ def channel_post(_: Request, body: schema.ChannelPostRequest): @openapi.response(HTTPStatus.NO_CONTENT) @openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) @validate(schema.ChannelPutRequest) +@run_after(save_channels_config) @wrol_mode_check -def channel_update(_: Request, link: str, body: schema.ChannelPutRequest): - channel = lib.update_channel(body.__dict__, link) +def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest): + channel = lib.update_channel(data=body, channel_id=channel_id) return response.raw('', HTTPStatus.NO_CONTENT, - headers={'Location': f'/api/videos/channels/{channel.link}'}) + headers={'Location': f'/api/videos/channels/{channel.id}'}) -@channel_bp.delete('/') +@channel_bp.delete('/') @openapi.description('Delete a Channel') @openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) @wrol_mode_check -def channel_delete(_: Request, link: str): - lib.delete_channel(link) +def channel_delete(_: Request, channel_id: int): + lib.delete_channel(channel_id=channel_id) return response.raw('', HTTPStatus.NO_CONTENT) diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index 85f165d4..4cc0739f 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -1,15 +1,18 @@ from collections import defaultdict from datetime import timedelta +from pathlib import Path from typing import List, Dict, Union, Generator from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound -from wrolpi.common import sanitize_link, run_after, get_relative_to_media_directory, make_media_directory, logger +from wrolpi.common import run_after, logger, \ + get_media_directory from wrolpi.dates import today -from wrolpi.db import get_db_session, get_db_curs +from wrolpi.db import get_db_curs, optional_session from wrolpi.downloader import download_manager from wrolpi.errors import UnknownChannel, UnknownDirectory, APIError, ValidationError, InvalidDownload +from .. import schema from ..common import check_for_channel_conflicts from ..lib import save_channels_config from ..models import Channel @@ -25,7 +28,7 @@ async def get_minimal_channels() -> List[dict]: # Get all channels, even if they don't have videos. stmt = ''' SELECT - c.id, name, link, directory, url, download_frequency, info_date, + c.id, name, directory, url, download_frequency, info_date, (select next_download from download d where d.url=c.url) AS next_download FROM channel AS c @@ -72,31 +75,39 @@ async def get_channels_video_count() -> Dict[int, int]: COD = Union[dict, Channel] -def get_channel(source_id: str = None, link: str = None, url: str = None, return_dict: bool = True) -> COD: +@optional_session +def get_channel(session: Session, *, channel_id: int = None, source_id: str = None, url: str = None, + directory: str = None, name: str = None, return_dict: bool = True) -> COD: """ Attempt to find a Channel using the provided params. The params are in order of reliability. Raises UnknownChannel if no channel is found. """ - with get_db_session() as session: - channel: COD = None # noqa - # Try by source_id first. - if source_id: - channel = session.query(Channel).filter_by(source_id=source_id).one_or_none() - if not channel and link: - channel = session.query(Channel).filter_by(link=link).one_or_none() - if not channel and url: - channel = session.query(Channel).filter_by(url=url).one_or_none() + channel: COD = None # noqa + # Try to find the channel by the most reliable methods first. + if channel_id: + channel = session.query(Channel).filter_by(id=channel_id).one_or_none() + if not channel and source_id: + channel = session.query(Channel).filter_by(source_id=source_id).one_or_none() + if not channel and url: + channel = session.query(Channel).filter_by(url=url).one_or_none() + if not channel and directory: + directory = Path(directory) + if not directory.is_absolute(): + directory = get_media_directory() / directory + channel = session.query(Channel).filter_by(directory=directory).one_or_none() + if not channel and name: + channel = session.query(Channel).filter_by(name=name).one_or_none() - if not channel: - raise UnknownChannel(f'No channel matches {link=} {source_id=} {url=}') + if not channel: + raise UnknownChannel(f'No channel matches {channel_id=} {source_id=} {url=} {directory=}') - logger.debug(f'Found {channel=} using {source_id=} {link=} {url=}') - if return_dict: - statistics = channel.get_statistics() - channel = channel.dict() - channel['statistics'] = statistics - return channel + logger.debug(f'Found {channel=} using {channel_id=} {source_id=} {url=} {directory=}') + if return_dict: + statistics = channel.get_statistics() + channel = channel.dict() + channel['statistics'] = statistics + return channel def _spread_by_frequency(channels: List[Channel]) -> Generator[Dict, None, None]: @@ -117,103 +128,90 @@ def _spread_by_frequency(channels: List[Channel]) -> Generator[Dict, None, None] @run_after(save_channels_config) -def update_channel(data: dict, link: str) -> Channel: +@optional_session +def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel: """Update a Channel's DB record""" - with get_db_session(commit=True) as session: - try: - channel: Channel = session.query(Channel).filter_by(link=link).one() - except NoResultFound: - raise UnknownChannel() + try: + channel: Channel = session.query(Channel).filter_by(id=channel_id).one() + except NoResultFound: + raise UnknownChannel() - # Only update directory if it was empty - if data.get('directory') and not channel.directory: - try: - data['directory'] = get_relative_to_media_directory(data['directory']) - except UnknownDirectory: - if data['mkdir']: - make_media_directory(data['directory']) - data['directory'] = get_relative_to_media_directory(data['directory']) - else: - raise + # Only update directory if it was empty + if data.directory and not channel.directory: + data.directory = get_media_directory() / data.directory + if not data.directory.is_dir(): + if data.mkdir: + data.directory.mkdir() + else: + raise UnknownDirectory() - if 'directory' in data: - data['directory'] = str(data['directory']) + # Verify that the URL/Name/directory aren't taken + check_for_channel_conflicts( + session, + id_=channel.id, + url=data.url, + name=data.name, + directory=data.directory, + ) - if data.get('match_regex') in ('None', 'null'): - data['match_regex'] = None + # Apply the changes now that we've OK'd them + channel.update(data.__dict__) - data['link'] = data.get('link') or link - - # Verify that the URL/Name/Link aren't taken - check_for_channel_conflicts( - session, - id_=channel.id, - url=data.get('url'), - name=data.get('name'), - link=data['link'], - directory=data.get('directory'), - ) - - # Apply the changes now that we've OK'd them - channel.update(data) + session.commit() return channel @run_after(save_channels_config) -def create_channel(data: dict, return_dict: bool = True) -> Union[Channel, dict]: +@optional_session +def create_channel(session: Session, data: schema.ChannelPostRequest, return_dict: bool = True) -> Union[Channel, dict]: """ Create a new Channel. Check for conflicts with existing Channels. """ - with get_db_session(commit=True) as session: - link = sanitize_link(data['name']) - try: - # Verify that the URL/Name/Link aren't taken - check_for_channel_conflicts( - session, - url=data.get('url'), - name=data['name'], - link=link, - directory=str(data['directory']), - source_id=data.get('source_id'), - ) - except APIError as e: - raise ValidationError() from e - - channel = Channel( - name=data['name'], - url=data.get('url'), - match_regex=data.get('match_regex'), - link=link, - directory=str(data['directory']), - download_frequency=data.get('download_frequency'), - source_id=data.get('source_id'), + try: + # Verify that the URL/Name/directory aren't taken + check_for_channel_conflicts( + session, + url=data.url, + name=data.name, + directory=data.directory, + source_id=data.source_id, ) - session.add(channel) - session.commit() - session.flush() - session.refresh(channel) + except APIError as e: + raise ValidationError() from e - if return_dict: - return channel.dict() - else: - return channel + channel = Channel( + name=data.name, + url=data.url, + match_regex=data.match_regex, + directory=data.directory, # noqa + download_frequency=data.download_frequency, + source_id=data.source_id, + ) + session.add(channel) + session.commit() + session.flush() + session.refresh(channel) + + return channel.dict() if return_dict else channel @run_after(save_channels_config) -def delete_channel(link): - with get_db_session(commit=True) as session: - try: - channel = session.query(Channel).filter_by(link=link).one() - except NoResultFound: - raise UnknownChannel() +@optional_session +def delete_channel(session: Session, *, channel_id: int): + try: + channel = session.query(Channel).filter_by(id=channel_id).one() + except NoResultFound: + raise UnknownChannel() - channel.delete_with_videos() + channel.delete_with_videos() + + session.commit() -def download_channel(link: str): +def download_channel(id_: int): """Create a Download record for a Channel's entire catalog. Start downloading.""" - channel = get_channel(link=link, return_dict=False) + channel = get_channel(channel_id=id_, return_dict=False) session = Session.object_session(channel) download = channel.get_download() if not download: diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index 31c45fd1..f2fe566b 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -1,7 +1,5 @@ import json -import os import pathlib -import shutil import tempfile from datetime import timedelta from http import HTTPStatus @@ -11,301 +9,116 @@ import mock from modules.videos.channel.lib import delete_channel from modules.videos.common import get_no_channel_directory from modules.videos.models import Channel, Video -from modules.videos.test.common import create_channel_structure from wrolpi.dates import now from wrolpi.db import get_db_session from wrolpi.downloader import download_manager, Download -from wrolpi.errors import UnknownFile -from wrolpi.root_api import api_app -from wrolpi.test.common import wrap_test_db, TestAPI -from wrolpi.vars import PROJECT_DIR +from wrolpi.test.common import assert_dict_contains -class TestVideoAPI(TestAPI): +def test_get_channels(test_directory, channel_factory, test_client): + channel_factory() + channel_factory() + channel_factory() + request, response = test_client.get('/api/videos/channels') + assert response.status_code == HTTPStatus.OK + assert len(response.json['channels']) == 3 - @wrap_test_db - @create_channel_structure( - { - 'Foo': ['vid1.mp4'], - 'Bar': ['vid2.mp4'], - 'Baz': [], - } + +def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel): + # A regular user-defined channel + vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4' + vid1.touch() + vid2 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title2.mp4' + vid2.touch() + + # The special NO CHANNEL directory. + no_channel_path = get_no_channel_directory() + vid3 = no_channel_path / 'some video.mp4' + vid3.touch() + no_channel_subdir = no_channel_path / 'subdir' + no_channel_subdir.mkdir() + vid4 = no_channel_subdir / 'other video.mp4' + vid4.touch() + + test_client.post('/api/videos/refresh') + + videos = test_session.query(Video).order_by(Video.title).all() + assert len(videos) == 4 + + # Add some fake videos, they should be removed. + test_session.add(Video(title='vid5')) + test_session.add(Video(title='vid6', channel_id=simple_channel.id)) + test_session.commit() + + videos = test_session.query(Video).order_by(Video.title).all() + assert len(videos) == 6 + + test_client.post('/api/videos/refresh') + + videos = test_session.query(Video).order_by(Video.title).all() + assert len(videos) == 4 + + +def test_get_video(test_client, test_session, simple_channel, video_factory): + """ + Test that you get can information about a video. Test that video file can be gotten. + """ + now_ = now() + video1 = video_factory(channel_id=simple_channel.id) + video1.upload_date, video1.title = now_, 'vid1' + video2 = video_factory(channel_id=simple_channel.id) + video2.upload_date, video2.title = now_ + timedelta(seconds=1), 'vid2' + test_session.commit() + + # Test that a 404 is returned when no video exists + _, response = test_client.get('/api/videos/video/10') + assert response.status_code == HTTPStatus.NOT_FOUND, response.json + assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''} + + # Get the video info we inserted + _, response = test_client.get('/api/videos/video/1') + assert response.status_code == HTTPStatus.OK, response.json + assert_dict_contains(response.json['video'], {'title': 'vid1'}) + + # The next video is included. + assert response.json['prev'] is None + assert_dict_contains(response.json['next'], {'title': 'vid2'}) + + +def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel): + """A channel does not require a download frequency.""" + # No downloads are scheduled. + assert len(download_manager.get_downloads(test_session)) == 0 + + # Get the Channel + request, response = test_client.get('/api/videos/channels/1') + channel = response.json['channel'] + assert channel['download_frequency'] is None + + # Update the Channel with a frequency. + new_channel = dict( + directory=channel['directory'], + name=channel['name'], + url=channel['url'], + download_frequency=10, ) - def test_get_channels(self, tempdir): - request, response = api_app.test_client.get('/api/videos/channels') - assert response.status_code == HTTPStatus.OK - # Channels are sorted by name - self.assertDictContains(response.json['channels'][0], {'id': 2, 'name': 'Bar', 'video_count': 1}) - self.assertDictContains(response.json['channels'][1], {'id': 3, 'name': 'Baz', 'video_count': 0}) - self.assertDictContains(response.json['channels'][2], {'id': 1, 'name': 'Foo', 'video_count': 1}) + request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}', + content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.NO_CONTENT, response.json - @wrap_test_db - def test_channel(self): - channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name - channel_directory_name = channel_directory.split('/')[-1] - pathlib.Path(channel_directory).mkdir() + # Download is scheduled. + assert len(download_manager.get_downloads(test_session)) == 1 - new_channel = dict( - directory=channel_directory, - match_regex='asdf', - name=' Example Channel 1 ', - url='https://example.com/channel1', - ) - - # Channel doesn't exist - request, response = api_app.test_client.get('/api/videos/channels/examplechannel1') - assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}' - - # Create it - request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel)) - assert response.status_code == HTTPStatus.CREATED, response.json - location = response.headers['Location'] - request, response = api_app.test_client.get(location) - created = response.json['channel'] - self.assertIsNotNone(created) - self.assertIsNotNone(created['id']) - - # Channel name leading/trailing whitespace should be stripped - assert created['name'] == 'Example Channel 1' - - # Channel directory should be relative to the media directory - assert not pathlib.Path(created['directory']).is_absolute(), \ - f'Channel directory is absolute: {created["directory"]}' - - # Get the link that was computed - new_channel['link'] = response.json['channel']['link'] - assert new_channel['link'] - - # Can't create it again - request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel)) - assert response.status_code == HTTPStatus.BAD_REQUEST - - # Update it - new_channel['name'] = 'Example Channel 2' - new_channel['directory'] = str(new_channel['directory']) - request, response = api_app.test_client.put(location, content=json.dumps(new_channel)) - assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code - request, response = api_app.test_client.get(location) - assert response.status_code == HTTPStatus.OK - self.assertDictContains(response.json['channel'], { - 'id': 1, - 'name': 'Example Channel 2', - 'directory': channel_directory_name, - 'match_regex': 'asdf', - 'url': 'https://example.com/channel1', - }) - - # Can't update channel that doesn't exist - request, response = api_app.test_client.put('/api/videos/channels/doesnt_exist', - content=json.dumps(new_channel)) - assert response.status_code == HTTPStatus.NOT_FOUND - - # Delete the new channel - request, response = api_app.test_client.delete(location) - assert response.status_code == HTTPStatus.NO_CONTENT - - # Cant delete it again - request, response = api_app.test_client.delete(location) - assert response.status_code == HTTPStatus.NOT_FOUND - - @wrap_test_db - def test_refresh_no_channel(self): - with get_db_session() as session: - videos_path = pathlib.Path(self.tmp_dir.name) / 'videos' - videos_path.mkdir() - - # A regular user-defined channel - channel1_path = videos_path / 'channel1' - channel1_path.mkdir() - vid1 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title1.mp4' - vid1.touch() - vid2 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title2.mp4' - vid2.touch() - - channel1 = Channel(name='channel1', link='channel1', directory=channel1_path) - session.add(channel1) - session.commit() - - # The special NO CHANNEL directory. - no_channel_path = get_no_channel_directory() - vid3 = no_channel_path / 'some video.mp4' - vid3.touch() - no_channel_subdir = no_channel_path / 'subdir' - no_channel_subdir.mkdir() - vid4 = no_channel_subdir / 'other video.mp4' - vid4.touch() - - api_app.test_client.post('/api/videos/refresh') - - videos = session.query(Video).order_by(Video.title).all() - self.assertEqual(len(videos), 4) - - # Add some fake videos, they should be removed. - session.add(Video(title='vid5')) - session.add(Video(title='vid6', channel_id=channel1.id)) - session.commit() - - videos = session.query(Video).order_by(Video.title).all() - self.assertEqual(len(videos), 6) - - api_app.test_client.post('/api/videos/refresh') - - videos = session.query(Video).order_by(Video.title).all() - self.assertEqual(len(videos), 4) - - @wrap_test_db - def test_get_channel_videos(self): - with get_db_session(commit=True) as session: - channel1 = Channel(name='Foo', link='foo') - channel2 = Channel(name='Bar', link='bar') - session.add(channel1) - session.add(channel2) - session.flush() - session.refresh(channel1) - session.refresh(channel2) - - # Channels don't have videos yet - d = dict(channel_link=channel1.link) - request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d)) - assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == 0 - - with get_db_session(commit=True) as session: - vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo') - vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo') - session.add(vid1) - session.add(vid2) - - # Videos are gotten by their respective channels - request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d)) - assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == 1 - assert response.json['totals']['videos'] == 1 - self.assertDictContains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id)) - - d = dict(channel_link=channel2.link) - request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d)) - assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == 1 - self.assertDictContains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id)) - - @wrap_test_db - def test_get_video(self): - """ - Test that you get can information about a video. Test that video file can be gotten. - """ - - def raise_unknown_file(_): - raise UnknownFile() - - with get_db_session(commit=True) as session, \ - mock.patch('modules.videos.common.get_absolute_video_info_json', raise_unknown_file): - channel = Channel(name='Foo', link='foo') - session.add(channel) - session.flush() - session.refresh(channel) - now_ = now() - session.add(Video(title='vid1', channel_id=channel.id, upload_date=now_)) - session.add(Video(title='vid2', channel_id=channel.id, upload_date=now_ + timedelta(seconds=1))) - - # Test that a 404 is returned when no video exists - _, response = api_app.test_client.get('/api/videos/video/10') - assert response.status_code == HTTPStatus.NOT_FOUND, response.json - assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''} - - # Get the video info we inserted - _, response = api_app.test_client.get('/api/videos/video/1') - assert response.status_code == HTTPStatus.OK, response.json - self.assertDictContains(response.json['video'], {'title': 'vid1'}) - - # The next video is included. - self.assertIsNone(response.json['prev']) - self.assertDictContains(response.json['next'], {'title': 'vid2'}) - - @wrap_test_db - def test_get_channel_videos_pagination(self): - with get_db_session(commit=True) as session: - channel1 = Channel(name='Foo', link='foo') - session.add(channel1) - session.flush() - session.refresh(channel1) - - for i in range(50): - session.add(Video(title=f'Foo.Video{i}', channel_id=channel1.id, video_path='foo')) - - channel2 = Channel(name='Bar', link='bar') - session.add(channel2) - session.flush() - session.refresh(channel2) - session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo')) - - # Get first, second, third, and empty pages of videos. - tests = [ - # (offset, video_count) - (0, 20), - (20, 20), - (40, 10), - (50, 0), - ] - last_ids = [] - for offset, video_count in tests: - d = dict(channel_link=channel1.link, order_by='id', offset=offset) - _, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d)) - assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == video_count - current_ids = [i['id'] for i in response.json['videos']] - assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}' - last_ids = current_ids - - @wrap_test_db - def test_channel_no_download_frequency(self): - """A channel does not require a download frequency.""" - channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name - os.mkdir(channel_directory) - - new_channel = dict( - directory=channel_directory, - name='Example Channel 1', - url='https://example.com/channel1', - download_frequency=None, - ) - - # Create the Channel - request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel)) - assert response.status_code == HTTPStatus.CREATED - location = response.headers['Location'] - - # No downloads are scheduled. - with get_db_session() as session: - downloads = list(download_manager.get_new_downloads(session)) - self.assertEqual(len(downloads), 0) - - # Get the Channel - request, response = api_app.test_client.get(location) - channel = response.json['channel'] - self.assertEqual(channel['download_frequency'], None) - - # Update the Channel with a frequency. - new_channel = dict( - directory=channel['directory'], - name=channel['name'], - url=channel['url'], - download_frequency=10, - ) - request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}', - content=json.dumps(new_channel)) - self.assertEqual(response.status_code, 204, response.json) - - # Remove the frequency. - new_channel = dict( - directory=channel['directory'], - name=channel['name'], - url=channel['url'], - download_frequency=None, - ) - request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}', - content=json.dumps(new_channel)) - self.assertEqual(response.status_code, 204, response.json) + # Remove the frequency. + new_channel = dict( + directory=channel['directory'], + name=channel['name'], + url=channel['url'], + download_frequency=None, + ) + request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}', + content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.NO_CONTENT, response.json def test_channel_frequency_update(download_channel, test_client, test_session): @@ -321,7 +134,7 @@ def test_channel_frequency_update(download_channel, test_client, test_session): url=download_channel.url, download_frequency=100, ) - request, response = test_client.put(f'/api/videos/channels/{download_channel.link}', json=data) + request, response = test_client.put(f'/api/videos/channels/{download_channel.id}', json=data) assert response.status_code == HTTPStatus.NO_CONTENT, response.json download = download_channel.get_download() @@ -332,20 +145,59 @@ def test_channel_frequency_update(download_channel, test_client, test_session): assert len(list(downloads)) == 1 -def test_delete_channel_delete_download(download_channel, test_session): - """ - Deleting a Channel deletes it's Download. - """ - downloads = test_session.query(Download).all() - assert all(i.url == download_channel.url for i in downloads) +def test_channel_download_crud(test_session, simple_channel): + """Modifying a Channel modifies it's Download.""" + assert simple_channel.url + assert simple_channel.download_frequency is None - delete_channel(download_channel.link) + simple_channel.update({'download_frequency': 1000}) + test_session.commit() + + download: Download = test_session.query(Download).one() + assert simple_channel.get_download() == download + assert simple_channel.url == download.url + assert simple_channel.download_frequency == download.frequency + + # Increase the frequency to 1 second. + simple_channel.update({'download_frequency': 1}) + test_session.commit() + assert simple_channel.url == download.url + assert simple_channel.download_frequency == download.frequency == 1 + assert test_session.query(Download).count() == 1 + + # Changing the Channel's URL changes the download's URL. + simple_channel.update({'url': 'https://example.com/new url'}) + test_session.commit() + assert simple_channel.url == 'https://example.com/new url' + assert simple_channel.download_frequency == download.frequency == 1 + assert test_session.query(Download).count() == 1 + + # Removing the Channel's frequency removes the download. + simple_channel.update({'download_frequency': None}) + test_session.commit() + assert test_session.query(Download).count() == 0 + + # Removing the URL is supported. + simple_channel.update({'url': None}) + test_session.commit() + + # Adding a URL is supported. + simple_channel.update({'url': 'https://example.com'}) + test_session.commit() + + +def test_delete_channel_delete_download(download_channel, test_session): + """Deleting a Channel deletes it's Download.""" + downloads = test_session.query(Download).all() + assert downloads and all(i.url == download_channel.url for i in downloads) + + delete_channel(channel_id=download_channel.id) downloads = test_session.query(Download).all() assert not list(downloads) -def test_video_search(test_session): +def test_video_search(test_client, test_session): """ Test that videos can be searched and that their order is by their textsearch rank. """ @@ -363,7 +215,7 @@ def test_video_search(test_session): def do_search(search_str, limit=20): d = json.dumps({'search_str': search_str, 'limit': limit}) - _, resp = api_app.test_client.post('/api/videos/search', content=d) + _, resp = test_client.post('/api/videos/search', content=d) return resp def search_is_as_expected(resp, expected): @@ -414,7 +266,7 @@ def test_video_file_name(test_session, simple_video, test_client): assert resp.json['video'].get('stem') == 'simple_video' -def test_channel_conflicts(test_session, test_directory): +def test_channel_conflicts(test_client, test_session, test_directory): channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name pathlib.Path(channel_directory).mkdir() new_channel = dict( @@ -425,7 +277,7 @@ def test_channel_conflicts(test_session, test_directory): ) def _post_channel(channel): - return api_app.test_client.post('/api/videos/channels', content=json.dumps(channel)) + return test_client.post('/api/videos/channels', content=json.dumps(channel)) # Create it request, response = _post_channel(new_channel) @@ -443,18 +295,6 @@ def test_channel_conflicts(test_session, test_directory): assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10, 'cause': {'error': 'The channel name is already taken.', 'code': 5}} - # Name matches when converted to link - channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name - pathlib.Path(channel_directory2).mkdir() - new_channel = dict( - directory=channel_directory2, - name='Example channel 1', - ) - request, response = _post_channel(new_channel) - assert response.status_code == HTTPStatus.BAD_REQUEST - assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10, - 'cause': {'code': 11, 'error': 'Channel link already used by another channel'}} - # Directory was already used new_channel = dict( directory=channel_directory, @@ -479,7 +319,7 @@ def test_channel_conflicts(test_session, test_directory): 'cause': {'code': 6, 'error': 'The URL is already used by another channel.'}} -def test_channel_empty_url_doesnt_conflict(test_session, test_directory): +def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_directory): """Two channels with empty URLs shouldn't conflict""" channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name pathlib.Path(channel_directory).mkdir() @@ -488,7 +328,7 @@ def test_channel_empty_url_doesnt_conflict(test_session, test_directory): 'name': 'Fooz', 'directory': channel_directory, } - request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel)) + request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel)) assert response.status_code == HTTPStatus.CREATED, response.json location = response.headers['Location'] @@ -498,7 +338,7 @@ def test_channel_empty_url_doesnt_conflict(test_session, test_directory): 'name': 'Barz', 'directory': channel_directory2, } - request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel)) + request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel)) assert response.status_code == HTTPStatus.CREATED, response.json assert location != response.headers['Location'] @@ -524,11 +364,11 @@ def test_download_channel_no_refresh(test_session, download_channel, video_downl def test_channel_post_directory(test_session, test_client, test_directory): """A Channel can be created with or without an existing directory.""" - # Channel can be created with a missing directory. + # Channel can be created with a directory which is not on disk. data = dict(name='foo', directory='foo') request, response = test_client.post('/api/videos/channels', content=json.dumps(data)) assert response.status_code == HTTPStatus.CREATED - directory = test_session.query(Channel).filter_by(link='foo').one().directory.path + directory = test_session.query(Channel).filter_by(id=1).one().directory.path assert (test_directory / 'foo') == directory assert not directory.is_dir() assert directory.is_absolute() @@ -537,56 +377,147 @@ def test_channel_post_directory(test_session, test_client, test_directory): data = dict(name='bar', directory='bar', mkdir=True) request, response = test_client.post('/api/videos/channels', content=json.dumps(data)) assert response.status_code == HTTPStatus.CREATED - directory = test_session.query(Channel).filter_by(link='bar').one().directory.path + directory = test_session.query(Channel).filter_by(id=2).one().directory.path assert (test_directory / 'bar') == directory assert directory.is_dir() assert directory.is_absolute() -def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory): - subdir = test_directory / 'subdir' - subdir.mkdir() +def test_channel_by_id(test_session, test_client, simple_channel, simple_video): + request, response = test_client.get(f'/api/videos/channels/{simple_channel.id}') + assert response.status_code == HTTPStatus.OK - # video1 is in a subdirectory. - video1 = video_factory(simple_channel.id, True, True, 'jpg') - test_session.commit() - shutil.move(video1.video_path.path, subdir / video1.video_path.path.name) - shutil.move(video1.poster_path.path, subdir / video1.poster_path.path.name) - video1.video_path = subdir / video1.video_path.path.name - video1.poster_path = None - # video2 is in the test directory. - video2 = video_factory(simple_channel.id, True, True, 'jpg') - video2.poster_path = video1.poster_path = None + +def test_channel_crud(test_session, test_client, test_directory): + channel_directory = test_directory / 'channel directory' + channel_directory.mkdir() + + new_channel = dict( + directory=str(channel_directory), + match_regex='asdf', + name=' Example Channel 1 ', + url='https://example.com/channel1', + ) + + # Channel doesn't exist + request, response = test_client.get('/api/videos/channels/examplechannel1') + assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}' + + # Create it + request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.CREATED, response.json + location = response.headers['Location'] + + request, response = test_client.get(location) + assert response.status_code == HTTPStatus.OK, response.json + created = response.json['channel'] + assert created + assert created['id'] + # No frequency was provided. No download exists. + assert not created['download_frequency'] + assert test_session.query(Download).count() == 0 + + # Channel name leading/trailing whitespace should be stripped + assert created['name'] == 'Example Channel 1' + + # Channel directory should be relative to the media directory + assert not pathlib.Path(created['directory']).is_absolute(), \ + f'Channel directory is absolute: {created["directory"]}' + + # Can't create it again + request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.BAD_REQUEST + + # Update it + new_channel['name'] = 'Example Channel 2' + new_channel['directory'] = str(new_channel['directory']) # noqa + request, response = test_client.put(location, content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code + request, response = test_client.get(location) + assert response.status_code == HTTPStatus.OK + channel = response.json['channel'] + assert channel['id'] == 1 + assert channel['name'] == 'Example Channel 2' + assert channel['directory'] == channel_directory.name + assert channel['match_regex'] == 'asdf' + assert channel['url'] == 'https://example.com/channel1' + + # Can't update channel that doesn't exist + request, response = test_client.put('/api/videos/channels/doesnt_exist', + content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.NOT_FOUND + + # Delete the new channel + request, response = test_client.delete(location) + assert response.status_code == HTTPStatus.NO_CONTENT + + # Cant delete it again + request, response = test_client.delete(location) + assert response.status_code == HTTPStatus.NOT_FOUND + + +def test_search_channel_videos(test_client, test_session): + with get_db_session(commit=True) as session: + channel1 = Channel(name='Foo') + channel2 = Channel(name='Bar') + session.add(channel1) + session.add(channel2) + session.flush() + session.refresh(channel1) + session.refresh(channel2) + + # Channels don't have videos yet + d = dict(channel_id=channel1.id) + request, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) + assert response.status_code == HTTPStatus.OK + assert len(response.json['videos']) == 0 + + with get_db_session(commit=True) as session: + vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo') + vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo') + session.add(vid1) + session.add(vid2) + + # Videos are gotten by their respective channels + request, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) + assert response.status_code == HTTPStatus.OK + assert len(response.json['videos']) == 1 + assert response.json['totals']['videos'] == 1 + assert_dict_contains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id)) + + d = dict(channel_id=channel2.id) + request, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) + assert response.status_code == HTTPStatus.OK + assert len(response.json['videos']) == 1 + assert_dict_contains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id)) + + +def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory): + for i in range(50): + video_factory(channel_id=simple_channel.id) + + channel2 = Channel(name='Bar') + test_session.add(channel2) + test_session.flush() + test_session.refresh(channel2) + video_factory(channel_id=channel2.id) + test_session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo')) test_session.commit() - assert not video1.size, 'video1 should not have size during creation' - - # Create a video not in the DB. - vid3 = test_directory / 'vid3.mp4' - shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', vid3) - - # Orphan poster should be ignored. - orphan_poster = pathlib.Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg') - orphan_poster.touch() - - # Create a bogus video in the channel. - bogus = Video(video_path='foo', channel_id=simple_channel.id) - test_session.add(bogus) - - test_client.post('/api/videos/refresh') - - # Posters were found during refresh. - assert video1.poster_path - assert 'subdir' in str(video1.poster_path.path) - assert video2.poster_path - # Missing video3 was found - video3: Video = test_session.query(Video).filter_by(id=4).one() - assert video3.video_path.path == vid3 - assert video3.video_path != video1.video_path - assert video3.video_path != video2.video_path - # Bogus video was removed. - assert not any('foo' in str(i.video_path.path) for i in test_session.query(Video).all()) - # Orphan file was not deleted. - assert orphan_poster.is_file(), 'Orphan poster was removed!' - - assert video1.size, 'video1 size was not found' + # Get first, second, third, and empty pages of videos. + tests = [ + # (offset, video_count) + (0, 20), + (20, 20), + (40, 10), + (50, 0), + ] + last_ids = [] + for offset, video_count in tests: + d = dict(channel_id=simple_channel.id, order_by='id', offset=offset) + _, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) + assert response.status_code == HTTPStatus.OK + assert len(response.json['videos']) == video_count + current_ids = [i['id'] for i in response.json['videos']] + assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}' + last_ids = current_ids diff --git a/modules/videos/channel/test/test_lib.py b/modules/videos/channel/test/test_lib.py new file mode 100644 index 00000000..1e8bc510 --- /dev/null +++ b/modules/videos/channel/test/test_lib.py @@ -0,0 +1,59 @@ +import pytest + +from modules.videos.channel import lib +from wrolpi.errors import UnknownChannel + + +@pytest.mark.parametrize('params', [ + dict(), + dict(channel_id=None), + dict(source_id=None), + dict(url=None), + dict(directory=None), +]) +def test_get_bad_channel(test_session, test_directory, params): + """A Channel cannot be found without data.""" + with pytest.raises(UnknownChannel): + lib.get_channel(**params) + + +def test_get_channel(test_session, test_directory, channel_factory): + """`get_channel` can find a Channel in multiple ways.""" + channel_id_channel = channel_factory() + + source_id_channel = channel_factory() + source_id_channel.source_id = 'the source id' + + url_channel = channel_factory() + url_channel.url = 'https://example.com/channel' + + directory_channel = channel_factory() + directory_channel.directory = test_directory / 'some directory' + + test_session.commit() + + channel = lib.get_channel(channel_id=channel_id_channel.id, return_dict=False) + assert channel == channel_id_channel + + channel = lib.get_channel(source_id='the source id', return_dict=False) + assert channel == source_id_channel + + channel = lib.get_channel(url='https://example.com/channel', return_dict=False) + assert channel == url_channel + + # Directory can be relative, or absolute. + channel = lib.get_channel(directory='some directory', return_dict=False) + assert channel == directory_channel + channel = lib.get_channel(directory=f'{test_directory}/some directory', return_dict=False) + assert channel == directory_channel + + # Channel is found using the priority of params, incorrect data is ignored. + params = [ + dict(channel_id=channel_id_channel.id, directory='bad directory'), + dict(channel_id=channel_id_channel.id, directory='some directory'), + dict(channel_id=channel_id_channel.id, source_id='bad source id'), + dict(channel_id=channel_id_channel.id, source_id='the source id'), + ] + for p in params: + assert lib.get_channel(**p, return_dict=False) == channel_id_channel + assert lib.get_channel(**p) diff --git a/modules/videos/common.py b/modules/videos/common.py index 64842eba..4008b77d 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -10,16 +10,13 @@ from typing import Union, Tuple, List, Set, Iterable, Optional import PIL from PIL import Image from sqlalchemy.orm import Session -from yt_dlp import YoutubeDL -from wrolpi import before_startup from wrolpi.cmd import which from wrolpi.common import logger, iterify, get_media_directory, \ - minimize_dict, api_param_limiter + minimize_dict, any_extensions from wrolpi.db import get_db_session, get_db_curs -from wrolpi.downloader import download_manager -from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \ - ChannelLinkConflict, ChannelDirectoryConflict, ChannelSourceIdConflict +from wrolpi.errors import UnknownFile, ChannelNameConflict, ChannelURLConflict, \ + ChannelDirectoryConflict, ChannelSourceIdConflict from wrolpi.media_path import MediaPath from wrolpi.vars import DEFAULT_FILE_PERMISSIONS from .models import Channel, Video @@ -27,27 +24,21 @@ from .models import Channel, Video logger = logger.getChild(__name__) REQUIRED_OPTIONS = ['name', 'directory'] -MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count', 'link'} +MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count'} MINIMUM_INFO_JSON_KEYS = {'description', 'view_count', 'webpage_url'} MINIMUM_VIDEO_KEYS = {'id', 'title', 'upload_date', 'duration', 'channel', 'channel_id', 'favorite', 'size', 'poster_path', 'caption_path', 'video_path', 'info_json', 'channel', 'viewed', 'source_id', 'view_count'} -VIDEO_QUERY_LIMIT = 20 -VIDEO_QUERY_MAX_LIMIT = 100 # These are the supported video formats. These are in order of their preference. VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv') -video_limiter = api_param_limiter(100) - class ConfigError(Exception): pass def get_videos_directory() -> pathlib.Path: - """ - Get the "videos" directory in the media directory. Make it if it does not exist. - """ + """Get the "videos" directory in the media directory. Make it if it does not exist.""" directory = get_media_directory() / 'videos' if not directory.is_dir(): directory.mkdir(parents=True) @@ -55,89 +46,13 @@ def get_videos_directory() -> pathlib.Path: def get_no_channel_directory() -> pathlib.Path: - """ - Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist. - """ + """Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist.""" directory = get_videos_directory() / 'NO CHANNEL' if not directory.is_dir(): directory.mkdir(parents=True) return directory -@before_startup -def import_videos_config(): - """Import channel settings to the DB. Existing channels will be updated.""" - logger.info('Importing videos config') - try: - from .lib import get_channels_config - from .downloader import ChannelDownloader - config = get_channels_config() - config, favorites = config.channels, config.favorites - - with get_db_session(commit=True) as session: - for link in config: - for option in (i for i in REQUIRED_OPTIONS if i not in config[link]): - raise ConfigError(f'Channel "{link}" is required to have "{option}"') - - name, directory = config[link]['name'], config[link]['directory'] - - channel = session.query(Channel).filter(Channel.link == link).one_or_none() - if not channel: - # Channel not yet in the DB, add it. - channel = Channel(link=link) - - # Only name and directory are required - channel.name = name - channel.directory = str(directory) - - channel.calculate_duration = config[link].get('calculate_duration') or channel.calculate_duration - channel.download_frequency = config[link].get('download_frequency') or channel.download_frequency - channel.generate_posters = config[link].get('generate_posters') or channel.generate_posters - channel.match_regex = config[link].get('match_regex') or channel.match_regex - channel.skip_download_videos = list(set(config[link].get('skip_download_videos', {}))) - channel.source_id = config[link].get('source_id') or channel.source_id - channel.url = config[link].get('url') or channel.url - - if not channel.source_id and channel.url: - # If we can download from a channel, we must have its source_id. - channel.source_id = get_channel_source_id(channel.url) - - if channel.download_frequency and channel.url: - download = download_manager.get_or_create_download(channel.url, session=session) - download.frequency = channel.download_frequency - download.downloader = ChannelDownloader.name - download.next_download = download.next_download or \ - download_manager.get_next_download(download, session=session) - - session.add(channel) - - with get_db_session(commit=True) as session: - for link, favorites in favorites.items(): - if link != 'NO CHANNEL': - channel: Channel = session.query(Channel).filter_by(link=link).one_or_none() - if not channel: - logger.warning(f'Cannot find channel {link=} for favorites!') - continue - channel_dir = channel.directory.path - else: - from .downloader import get_no_channel_directory - channel_dir = get_no_channel_directory() - - # Set favorite Videos of this Channel. - for video_path, data in favorites.items(): - # Favorite in the config is the name of the video_path. Add the channel directory onto this - # video_path, so we can match the complete path for the Video. - video_path = channel_dir / video_path - video = session.query(Video).filter_by(video_path=video_path).one_or_none() - # If no Video is found, it may be that we need to refresh. - if video: - video.favorite = data['favorite'] - else: - logger.warning(f'Cannot find video to favorite: {video_path}') - except Exception as e: - logger.warning('Failed to load channels config!', exc_info=e) - - VALID_VIDEO_KINDS = {'video', 'caption', 'poster', 'description', 'info_json'} @@ -150,66 +65,11 @@ def get_absolute_video_path(video: Video, kind: str = 'video') -> MediaPath: raise UnknownFile() -get_absolute_video_caption = partial(get_absolute_video_path, kind='caption') -get_absolute_video_poster = partial(get_absolute_video_path, kind='poster') -get_absolute_video_description = partial(get_absolute_video_path, kind='description') -get_absolute_video_info_json = partial(get_absolute_video_path, kind='info_json') - - -def get_absolute_video_files(video: Video) -> List[MediaPath]: - """ - Get all video files that exist. - """ - getters = [ - get_absolute_video_description, - get_absolute_video_caption, - get_absolute_video_poster, - get_absolute_video_info_json, - get_absolute_video_path, - ] - - def _get(): - for getter in getters: - try: - yield getter(video) - except UnknownFile: - pass - - return list(_get()) - - -def get_video_info_json(video) -> Optional[dict]: - """Get the info_json object from a video's meta-file. Return an empty dict if not possible.""" - if not video.channel or not video.channel.directory: - return - - try: - path = get_absolute_video_info_json(video) - if path.exists(): - with open(str(path), 'rb') as fh: - contents = json.load(fh) - return contents - except UnknownFile: - return - except UnknownDirectory: - return - - -def any_extensions(filename: str, extensions: Iterable = ()): - """ - Return True only if the file name ends with any of the possible extensions. - Matches lower or upper case of the extension. - """ - return any(filename.lower().endswith(ext) for ext in extensions) - - match_video_extensions = partial(any_extensions, extensions=VIDEO_EXTENSIONS) def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, pathlib.Path]: - """ - Generate a list of video paths in the provided directory. - """ + """Generate a list of video paths in the provided directory.""" directory = pathlib.Path(directory) for child in directory.iterdir(): @@ -223,9 +83,11 @@ def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, path @iterify(set) def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]: """ - Remove any duplicate paths from a given list. Duplicate is defined as any file that shares the EXACT same - name as another file, but with a different extension. If a duplicate is found, only yield one of the paths. The - path that will be yielded will be whichever is first in the VIDEO_EXTESIONS tuple. + Remove any duplicate paths from a given list. + + Duplicate is defined as any file that shares the EXACT same name as another file, but with a different extension. + If a duplicate is found, only yield one of the paths. The path that will be yielded will be whichever is first in + the VIDEO_EXTESIONS tuple. i.e. >>> remove_duplicate_video_paths([Path('one.mp4'), Path('two.mp4'), Path('one.ogg')]) @@ -262,15 +124,15 @@ def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]: yield sorted(paths)[0] -def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, link=None, directory=None, +def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, directory=None, source_id=None): """ Search for any channels that conflict with the provided args, raise a relevant exception if any conflicts are found. """ - if not any([id_, url, name, link, directory]): + if not any([id_, url, name, directory]): raise Exception('Cannot search for channel with no arguments') - logger.debug(f'Checking for channel conflicts: id={id_} url={url} name={name} link={link} directory={directory}') + logger.debug(f'Checking for channel conflicts: id={id_} url={url} name={name} directory={directory}') # A channel can't conflict with itself if id_: @@ -286,10 +148,6 @@ def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, conflicts = base_where.filter(Channel.name == name) if list(conflicts): raise ChannelNameConflict() - if link: - conflicts = base_where.filter(Channel.link == link) - if list(conflicts): - raise ChannelLinkConflict() if directory: conflicts = base_where.filter(Channel.directory == directory) if list(conflicts): @@ -329,29 +187,13 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]: return paths -def replace_extension(path: pathlib.Path, new_ext) -> pathlib.Path: - """Swap the extension of a file's path. - - Example: - >>> foo = pathlib.Path('foo.bar') - >>> replace_extension(foo, 'baz') - 'foo.baz' - """ - parent = path.parent - existing_ext = path.suffix - path = str(path) - name, _, _ = path.rpartition(existing_ext) - path = pathlib.Path(str(parent / name) + new_ext) - return path - - def generate_video_poster(video_path: Path) -> Path: """ Create a poster (aka thumbnail) next to the provided video_path. """ - poster_path = replace_extension(video_path, '.jpg') - cmd = ['/usr/bin/ffmpeg', '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000', - str(poster_path)] + poster_path = video_path.with_suffix('.jpg') + cmd = ('/usr/bin/ffmpeg', '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000', + str(poster_path)) try: subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) logger.info(f'Generated poster at {poster_path}') @@ -474,13 +316,3 @@ def minimize_video(video: dict) -> dict: video['info_json'] = minimize_video_info_json(video['info_json']) return video - - -YDL = YoutubeDL() -YDL.params['logger'] = logger.getChild('youtube-dl') -YDL.add_default_info_extractors() - - -def get_channel_source_id(url: str) -> str: - channel_info = YDL.extract_info(url, download=False, process=False) - return channel_info.get('uploader_id') or channel_info['id'] diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 336deb7f..915f7329 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -10,22 +10,18 @@ from PIL import Image from modules.videos.downloader import VideoDownloader, ChannelDownloader from modules.videos.lib import set_test_channels_config, set_test_downloader_config from modules.videos.models import Channel, Video -from wrolpi.common import sanitize_link from wrolpi.downloader import DownloadFrequency, DownloadManager, Download from wrolpi.vars import PROJECT_DIR @pytest.fixture def simple_channel(test_session, test_directory) -> Channel: - """ - Get a Channel with the minimum properties. This Channel has no download! - """ + """Get a Channel with the minimum properties. This Channel has no download!""" channel = Channel( directory=test_directory, name='Simple Channel', url='https://example.com/channel1', download_frequency=None, # noqa - link='simplechannel', ) test_session.add(channel) test_session.commit() @@ -34,20 +30,18 @@ def simple_channel(test_session, test_directory) -> Channel: @pytest.fixture def channel_factory(test_session, test_directory): - """ - Create a random Channel with a directory, but no frequency. - """ + """Create a random Channel with a directory, but no frequency.""" - def _(): - name = str(uuid4()) + def _(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None): + name = name or str(uuid4()) directory = test_directory / name directory.mkdir() channel = Channel( directory=directory, # noqa name=name, - url=f'https://example.com/{name}', - download_frequency=None, # noqa - link=sanitize_link(name), + url=url or f'https://example.com/{name}', + download_frequency=download_frequency, + source_id=source_id, ) test_session.add(channel) test_session.commit() @@ -67,7 +61,6 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch name='Download Channel', url='https://example.com/channel1', download_frequency=DownloadFrequency.weekly, - link='downloadchannel', ) download = Download(url=channel.url, downloader='video_channel', frequency=channel.download_frequency) test_session.add_all([channel, download]) @@ -77,9 +70,7 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch @pytest.fixture def simple_video(test_session, test_directory, simple_channel) -> Video: - """ - A Video with a video file whose channel is the Simple Channel. - """ + """A Video with an empty video file whose channel is the Simple Channel.""" video_path = test_directory / 'simple_video.mp4' video_path.touch() video = Video(video_path=video_path, channel_id=simple_channel.id) @@ -103,9 +94,9 @@ def video_file(test_directory) -> pathlib.Path: def video_factory(test_session, test_directory): """Creates Videos for testing.""" - def _(channel_id: int = None, with_video_file: bool = False, with_info_json: dict = None, - with_poster_ext: str = None): - title = str(uuid4()) + def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file: bool = False, + with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False): + title = title or str(uuid4()) if channel_id: path = test_directory / f'{title}.mp4' else: @@ -129,12 +120,18 @@ def video_factory(test_session, test_directory): poster_path = path.with_suffix(f'.{with_poster_ext}') Image.new('RGB', (25, 25)).save(poster_path) + caption_path = None + if with_caption_file: + caption_path = path.with_suffix('.en.vtt') + caption_path.touch() + video = Video(video_path=str(path), title=title, channel_id=channel_id, source_id=title, - info_json_path=info_json_path, poster_path=poster_path) + info_json_path=info_json_path, poster_path=poster_path, caption_path=caption_path, + upload_date=upload_date) test_session.add(video) return video - return _ + return factory @pytest.fixture diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 4aad0c56..e9050b46 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -6,23 +6,23 @@ from abc import ABC from typing import Tuple, List, Optional from sqlalchemy.orm import Session -from sqlalchemy.orm.exc import NoResultFound from yt_dlp import YoutubeDL from yt_dlp.extractor import YoutubeTabIE from yt_dlp.utils import UnsupportedError, DownloadError from wrolpi.cmd import which -from wrolpi.common import logger, sanitize_link, extract_domain +from wrolpi.common import logger, extract_domain, get_media_directory from wrolpi.dates import now from wrolpi.db import get_db_session, get_db_curs from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel -from .common import apply_info_json, get_channel_source_id, get_no_channel_directory, \ +from .common import apply_info_json, get_no_channel_directory, \ get_videos_directory -from .lib import upsert_video, refresh_channel_videos, get_downloader_config +from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id from .models import Video, Channel +from .schema import ChannelPostRequest from .video_url_resolver import video_url_resolver logger = logger.getChild(__name__) @@ -70,12 +70,12 @@ class ChannelDownloader(Downloader, ABC): if not channel: # Couldn't get channel by URL, this is probably a playlist. Find the channel for the playlist. channel_source_id = get_channel_source_id(download.url) - channel = get_channel(channel_source_id, return_dict=False) + channel = get_channel(source_id=channel_source_id, return_dict=False) update_channel(channel) domain = extract_domain(download.url) - missing_videos = find_all_missing_videos(channel.link) + missing_videos = find_all_missing_videos(channel.id) with get_db_session(commit=True) as session: for video_id, source_id, missing_video in missing_videos: url = video_url_resolver(domain, missing_video) @@ -152,7 +152,7 @@ class VideoDownloader(Downloader, ABC): channel = None if channel_name or channel_id: channel_url = info.get('channel_url') - channel = get_or_create_channel(channel_id, None, channel_url, channel_name) + channel = get_or_create_channel(source_id=channel_id, url=channel_url, name=channel_name) # Use the default directory if this video has no channel. out_dir = get_no_channel_directory() @@ -245,34 +245,32 @@ channel_downloader = ChannelDownloader() video_downloader = VideoDownloader(40) -def get_or_create_channel(source_id: str = None, link: str = None, url: str = None, name: str = None) -> Channel: +def get_or_create_channel(source_id: str = None, url: str = None, name: str = None) -> Channel: """ Attempt to find a Channel using the provided params. The params are in order of reliability. Creates a new Channel if one cannot be found. """ - if not link and name: - link = sanitize_link(name) - try: - channel = get_channel(link=link, source_id=source_id, url=url, return_dict=False) + channel = get_channel(source_id=source_id, url=url, name=name, return_dict=False) return channel except UnknownChannel: pass + if not name: + raise ValueError(f'Cannot create channel without a name') + # Channel does not exist. Create one in the video directory. channel_directory = get_videos_directory() / name if not channel_directory.is_dir(): channel_directory.mkdir(parents=True) - data = dict( + data = ChannelPostRequest( + source_id=source_id, name=name, url=url, - link=link, - domain=extract_domain(url), - directory=str(channel_directory), - source_id=source_id, + directory=str(channel_directory.relative_to(get_media_directory())), ) - channel = create_channel(data, return_dict=False) + channel = create_channel(data=data, return_dict=False) # Create the directory now that the channel is approved. channel_directory.mkdir(exist_ok=True) @@ -348,21 +346,17 @@ def update_channel(channel: Channel): apply_info_json(channel_id) -def _find_all_missing_videos(link: str = None) -> List[Tuple]: +def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]: """Get all Video entries which don't have the required media files (i.e. hasn't been downloaded). - Restrict to a single channel if "link" is provided. + Restrict to a single channel if `channel_id` is provided. """ with get_db_curs() as curs: # Get all channels by default. where = '' params = () - if link: - # Restrict by channel when link is provided. - query = 'SELECT id FROM channel WHERE link = %s' - curs.execute(query, (link,)) - channel_id = curs.fetchall()[0][0] + if channel_id: where = 'AND channel_id = %s' params = (channel_id,) @@ -386,18 +380,15 @@ def _find_all_missing_videos(link: str = None) -> List[Tuple]: return missing_videos -def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]: +def find_all_missing_videos(channel_id: str = None) -> Tuple[dict, dict]: """ Find all videos that don't have a video file, but are found in the DB (taken from the channel's info_json). Yields a Channel Dict object, our Video id, and the "entry" of the video from the channel's info_json['entries']. """ with get_db_session() as session: - if link: - try: - channel = session.query(Channel).filter_by(link=link).one() - except NoResultFound: - raise UnknownChannel(f'No channel with link: {link}') + if channel_id: + channel = get_channel(channel_id=channel_id, return_dict=False) if not channel.url: raise ChannelURLEmpty('No URL for this channel') channels = [channel, ] @@ -420,12 +411,12 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]: # Convert the channel video entries into a form that allows them to be quickly retrieved without searching through # the entire entries list. channels_entries = {} - for id_, channel in channels.items(): - channels_entries[id_] = {i['id']: i for i in channel.info_json['entries']} + for channel_id_, channel in channels.items(): + channels_entries[channel_id_] = {i['id']: i for i in channel.info_json['entries']} - missing_videos = _find_all_missing_videos(link) + missing_videos = _find_all_missing_videos(channel_id) - for id_, source_id, channel_id in missing_videos: + for video_id, source_id, channel_id in missing_videos: channel = channels[channel_id] if channel.skip_download_videos and source_id in channel.skip_download_videos: @@ -435,13 +426,13 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]: try: missing_video = channels_entries[channel_id][source_id] except KeyError: - logger.warning(f'Video {id_} / {source_id} is not in {channel.name} info_json') + logger.warning(f'Video {channel.name} / {source_id} is not in {channel.name} info_json') continue match_regex: re.compile = match_regexen.get(channel_id) if not match_regex or (match_regex and missing_video['title'] and match_regex.match(missing_video['title'])): # No title match regex, or the title matches the regex. - yield id_, source_id, missing_video + yield video_id, source_id, missing_video UNRECOVERABLE_ERRORS = { diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 5ee00572..54e81003 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -2,20 +2,23 @@ import html import pathlib import re from collections import defaultdict -from typing import Tuple, Optional, List +from typing import Tuple, Optional, List, Union from uuid import uuid1 +from sqlalchemy import or_ from sqlalchemy.exc import InvalidRequestError from sqlalchemy.orm import Session +from yt_dlp import YoutubeDL -from wrolpi.common import logger, chunks, ConfigFile +from wrolpi import before_startup +from wrolpi.common import chunks, ConfigFile, get_media_directory, sanitize_link from wrolpi.dates import from_timestamp, Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.media_path import MediaPath from wrolpi.vars import PYTEST from .captions import get_captions -from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, import_videos_config, \ - get_video_duration, is_valid_poster, convert_image, generate_video_poster +from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \ + is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError from .models import Channel, Video logger = logger.getChild(__name__) @@ -66,8 +69,9 @@ def refresh_channel_videos(channel: Channel): logger.info(f'{channel.name}: {len(new_videos)} new videos, {len(existing_paths)} already existed. ') - channel.refreshed = True - Session.object_session(channel).commit() + with get_db_session(commit=True) as session: + channel = session.query(Channel).filter_by(id=channel.id).one() + channel.refreshed = True apply_info_json(channel.id) @@ -155,7 +159,7 @@ def validate_videos(): video.validated = True except Exception as e: # This video failed to validate, continue validation for the rest of the videos. - logger.warning(f'Failed to validate {video=}', exc_info=e) + logger.warning(f'Failed to validate {video}', exc_info=e) def validate_video(video: Video, channel_generate_poster: bool): @@ -246,7 +250,7 @@ def convert_or_generate_poster(video: Video) -> Optional[pathlib.Path]: logger.error(f'Failed to generate poster for {video}', exc_info=e) -def refresh_videos(channel_links: list = None): +def refresh_videos(channel_ids: List[int] = None): """ Find any videos in the channel directories and add them to the DB. Delete DB records of any videos not in the file system. @@ -257,15 +261,15 @@ def refresh_videos(channel_links: list = None): """ logger.info('Refreshing video files') with get_db_session() as session: - if channel_links: - channels = session.query(Channel).filter(Channel.link.in_(channel_links)) + if channel_ids: + channels = session.query(Channel).filter(Channel.id.in_(channel_ids)) else: channels = session.query(Channel).all() channels = list(channels) - if not channels and channel_links: - raise Exception(f'No channels match links(s): {channel_links}') + if not channels and channel_ids: + raise Exception(f'No channels match id(s): {channel_ids}') elif not channels: raise Exception(f'No channels in DB. Have you created any?') @@ -276,13 +280,13 @@ def refresh_videos(channel_links: list = None): logger.fatal(f'Failed to refresh videos for channel {channel.name}!', exc_info=e) pass - if not channel_links: + if not channel_ids: # Refresh NO CHANNEL videos when not refreshing a specific channel. refresh_no_channel_videos() # Fill in any missing data for all videos. if not PYTEST: - import_videos_config() + import_channels_config() validate_videos() @@ -460,8 +464,8 @@ def set_test_downloader_config(enabled: bool): def get_channels_config_from_db(session: Session) -> dict: """Create a dictionary that contains all the Channels from the DB.""" - channels = session.query(Channel).order_by(Channel.link).all() - channels = {i.link: i.config_view() for i in channels} + channels = session.query(Channel).order_by(Channel.directory).all() + channels = sorted((i.config_view() for i in channels), key=lambda i: i['directory']) # Get all Videos that are favorites. Store them in their own config section, so they can be preserved if a channel # is deleted or the DB is wiped. @@ -469,7 +473,7 @@ def get_channels_config_from_db(session: Session) -> dict: favorites = defaultdict(lambda: {}) for video in favorite_videos: if video.channel: - favorites[video.channel.link][video.video_path.path.name] = dict(favorite=video.favorite) + favorites[str(video.channel.directory.relative)][video.video_path.path.name] = dict(favorite=video.favorite) else: favorites['NO CHANNEL'][video.video_path.path.name] = dict(favorite=video.favorite) favorites = dict(favorites) @@ -477,18 +481,128 @@ def get_channels_config_from_db(session: Session) -> dict: return dict(channels=channels, favorites=favorites) +def _detect_old_favorites(config: Union[ChannelsConfig, dict]) -> bool: + # TODO remove these old favorites after beta. + media_directory = get_media_directory() + favorites = config.favorites if hasattr(config, 'favorites') else config['favorites'] + for channel, favorites in favorites.items(): + if channel == 'NO CHANNEL': + continue + if not (media_directory / channel).is_dir(): + return True + return False + + @optional_session() def save_channels_config(session=None, preserve_favorites: bool = True): - """ - Pull the Channel information from the DB, save it to the config. - """ + """Get the Channel information from the DB, save it to the config.""" config = get_channels_config_from_db(session) channels_config = get_channels_config() + # TODO remove these old favorites after beta. + old_favorites = _detect_old_favorites(channels_config) + if old_favorites and config['favorites']: + # There are favorites in the database, remove the old favorites. + channels_config.favorites = {} if preserve_favorites: config['favorites'].update(channels_config.favorites or {}) channels_config.update(config) +@before_startup +def import_channels_config(): + """Import channel settings to the DB. Existing channels will be updated.""" + logger.info('Importing videos config') + try: + from .downloader import ChannelDownloader + config = get_channels_config() + channels, favorites = config.channels, config.favorites + + with get_db_session(commit=True) as session: + for data in channels: + if isinstance(data, str): + # Outdated style config. + # TODO remove this after beta. + data = channels[data] + # A Channel's directory is saved (in config) relative to the media directory. + directory = get_media_directory() / data['directory'] + for option in (i for i in REQUIRED_OPTIONS if i not in data): + raise ConfigError(f'Channel "{directory}" is required to have "{option}"') + + channel = session.query(Channel).filter( + or_( + Channel.directory == str(directory), + Channel.directory == str(data['directory']), + Channel.directory == str(directory.relative_to(get_media_directory())), + ), + ).one_or_none() + if not channel: + # Channel not yet in the DB, add it. + channel = Channel(directory=directory) + session.add(channel) + + # Only name and directory are required + channel.name = data['name'] + channel.directory = directory + + # Copy existing channel data, update all values from the config. This is necessary to clear out + # values not in the config. + full_data = channel.dict() + full_data.update(data) + print(f'{full_data=}') + full_data['skip_download_videos'] = list(set(data.get('skip_download_videos', {}))) + channel.update(full_data) + + if not channel.source_id and channel.url: + # If we can download from a channel, we must have its source_id. + channel.source_id = get_channel_source_id(channel.url) + + with get_db_session(commit=True) as session: + channels_by_link = {sanitize_link(i.name): i for i in session.query(Channel)} + for directory_, favorites in favorites.items(): + if directory_ != 'NO CHANNEL': + # A Channel's directory is saved (in the config) relative to the media directory. + directory = get_media_directory() / directory_ + channel: Channel = session.query(Channel).filter_by(directory=directory).one_or_none() + if not channel: + # Directory may be the outdated "link". + # TODO remove these old favorites after beta. + channel = channels_by_link.get(directory_) + if not channel: + logger.warning(f'Cannot find channel {directory=} for favorites!') + continue + channel_dir = channel.directory.path + else: + from .downloader import get_no_channel_directory + channel_dir = get_no_channel_directory() + + # Set favorite Videos of this Channel. + for video_path, data in favorites.items(): + # Favorite in the config is the name of the video_path. Add the channel directory onto this + # video_path, so we can match the complete path for the Video. + video_path = channel_dir / video_path + video = session.query(Video).filter_by(video_path=video_path).one_or_none() + # If no Video is found, it may be that we need to refresh. + if video: + video.favorite = data['favorite'] + else: + logger.warning(f'Cannot find video to favorite: {video_path}') + except Exception as e: + logger.warning('Failed to load channels config!', exc_info=e) + if PYTEST: + # Do not interrupt startup, only raise during testing. + raise + + +YDL = YoutubeDL() +YDL.params['logger'] = logger.getChild('youtube-dl') +YDL.add_default_info_extractors() + + +def get_channel_source_id(url: str) -> str: + channel_info = YDL.extract_info(url, download=False, process=False) + return channel_info.get('uploader_id') or channel_info['id'] + + async def get_statistics(): with get_db_curs() as curs: curs.execute(''' @@ -608,7 +722,6 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = video.info_json_path = info_json_path video.poster_path = poster_path video.video_path = video_path - if channel and not str(video_path).startswith(str(channel.directory.path)): raise ValueError(f'Video path is not within its channel {video_path=} not in {channel.directory=}') diff --git a/modules/videos/models.py b/modules/videos/models.py index 91156226..0b8af617 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -1,5 +1,4 @@ import json -import pathlib from datetime import datetime from pathlib import Path from typing import Optional, Generator @@ -12,7 +11,7 @@ from wrolpi.common import Base, tsvector, ModelHelper, logger, get_media_directo from wrolpi.dates import now, TZDateTime from wrolpi.db import get_db_curs from wrolpi.downloader import Download, download_manager -from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory, InvalidDownload +from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory from wrolpi.media_path import MediaPathType, MediaPath logger = logger.getChild(__name__) @@ -116,9 +115,7 @@ class Video(ModelHelper, Base): save_channels_config(preserve_favorites=False) def add_to_skip_list(self): - """ - Add this video to it's Channel's skip list. - """ + """Add this video to it's Channel's skip list.""" if self.channel and self.source_id: self.channel.add_video_to_skip_list(self.source_id) @@ -130,18 +127,16 @@ class Video(ModelHelper, Base): self.viewed = now() def get_info_json(self) -> Optional[JSON]: - """ - If this Video has an info_json file, return it's contents. Otherwise, return None. - """ + """If this Video has an info_json file, return it's contents. Otherwise, return None.""" + if not self.info_json_path: + return + try: - if isinstance(self.info_json_path, pathlib.Path): - with open(self.info_json_path, 'rb') as fh: - contents = json.load(fh) - return contents - elif self.info_json_path: - with open(self.info_json_path.path, 'rb') as fh: - contents = json.load(fh) - return contents + info_json_path = self.info_json_path.path if \ + isinstance(self.info_json_path, MediaPath) else self.info_json_path + with open(info_json_path, 'rb') as fh: + contents = json.load(fh) + return contents except UnknownFile: pass except UnknownDirectory: @@ -265,9 +260,8 @@ class Channel(ModelHelper, Base): __tablename__ = 'channel' id = Column(Integer, primary_key=True) name = Column(String) - link = Column(String, nullable=False) idempotency = Column(String) - url = Column(String) + url = Column(String, unique=True) match_regex = Column(String) directory: MediaPath = Column(MediaPathType) skip_download_videos = Column(ARRAY(String)) @@ -307,18 +301,28 @@ class Channel(ModelHelper, Base): def update(self, data: dict): """ - Update the values of this Channel. Will also update the Channel's Download, if it has one. + Update the attributes of this Channel. Will also update the Channel's Download, if it has one. """ + # Get the download before we change the URL. + download = self.get_download() + for key, value in data.items(): setattr(self, key, value) - if not self.url: - return - download = self.get_download() - if download: - session = Session.object_session(self) + # All channels with a URL and download_frequency should have a download. + session = Session.object_session(self) + if download and not self.download_frequency: + download_manager.delete_download(download.id, session) + elif download and self.download_frequency: download.frequency = self.download_frequency + download.url = self.url download.next_download = download_manager.get_next_download(download, session) + elif not download and self.download_frequency and self.url: + download = Download(frequency=self.download_frequency, url=self.url, downloader='video_channel') + session.add(download) + session.flush() + download.next_download = download_manager.get_next_download(download, session) + session.flush() def config_view(self) -> dict: """ @@ -349,7 +353,7 @@ class Channel(ModelHelper, Base): Get the Download row for this Channel. If there isn't a Download, return None. """ if not self.url: - raise InvalidDownload(f'Channel {self.name} does not have a URL to download!') + return None session = Session.object_session(self) download = session.query(Download).filter_by(url=self.url).one_or_none() @@ -361,10 +365,15 @@ class Channel(ModelHelper, Base): name=self.name, directory=self.directory, url=self.url, - link=self.link, ) return d + def dict(self, with_statistics: bool = False): + d = super(Channel, self).dict() + if with_statistics: + d['statistics'] = self.get_statistics() + return d + def get_statistics(self): """ Get statistics about this channel. diff --git a/modules/videos/schema.py b/modules/videos/schema.py index 38440038..360dd842 100644 --- a/modules/videos/schema.py +++ b/modules/videos/schema.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field from datetime import date, datetime from typing import List, Optional -from modules.videos.lib import DEFAULT_DOWNLOAD_FREQUENCY from modules.videos.video.lib import DEFAULT_VIDEO_ORDER, VIDEO_QUERY_LIMIT @@ -11,25 +10,37 @@ class ChannelPostRequest: name: str directory: str calculate_duration: Optional[bool] = None - download_frequency: Optional[int] = DEFAULT_DOWNLOAD_FREQUENCY + download_frequency: Optional[int] = None generate_posters: Optional[bool] = None match_regex: Optional[str] = None mkdir: Optional[bool] = None url: Optional[str] = None + source_id: Optional[str] = None + + def __post_init__(self): + self.name = self.name.strip() or None + self.directory = self.directory.strip() or None + self.url = self.url.strip() if self.url else None + self.source_id = self.source_id.strip() if self.source_id else None @dataclass class ChannelPutRequest: calculate_duration: Optional[bool] = None directory: Optional[str] = None - download_frequency: Optional[int] = DEFAULT_DOWNLOAD_FREQUENCY + download_frequency: Optional[int] = None generate_posters: Optional[bool] = None - link: Optional[str] = None match_regex: Optional[str] = None mkdir: Optional[bool] = None name: Optional[str] = None url: Optional[str] = None + def __post_init__(self): + self.name = self.name.strip() or None + self.directory = self.directory.strip() or None + self.url = self.url.strip() if self.url else None + self.match_regex = None if self.match_regex in ('None', '') else self.match_regex + @dataclass class RefreshRequest: @@ -42,7 +53,6 @@ class ChannelModel: url: str name: str match_regex: str - link: str directory: str @@ -60,7 +70,6 @@ class ChannelPostResponse: class ChannelsChannelModel: id: int name: str - link: str @dataclass @@ -132,7 +141,7 @@ class VideoSearchRequest: offset: Optional[int] = None limit: Optional[int] = VIDEO_QUERY_LIMIT order_by: Optional[str] = DEFAULT_VIDEO_ORDER - channel_link: Optional[str] = None + channel_id: Optional[int] = None @dataclass diff --git a/modules/videos/test/common.py b/modules/videos/test/common.py deleted file mode 100644 index 81f6af90..00000000 --- a/modules/videos/test/common.py +++ /dev/null @@ -1,75 +0,0 @@ -import pathlib -import tempfile -from functools import wraps - -from modules.videos.lib import refresh_channel_videos -from modules.videos.models import Channel -from wrolpi.common import insert_parameter, set_test_media_directory -from wrolpi.db import get_db_session -from wrolpi.test.common import wrap_test_db, build_test_directories - - -def create_channel_structure(structure): - """ - Create a directory containing the specified structure of channels and videos. Create DB entries for these - channels and videos. - - Example: - >>> s = {'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4', 'vid2.mp4', 'vid2.en.vtt']} - >>> create_channel_structure(s) - - Creates directories like so: - channel1/vid1.mp4 - channel2/vid1.mp4 - channel2/vid2.mp4 - channel2/vid2.en.vtt - - Channels like so: - Channel(name='channel1', directory='channel1') - Channel(name='channel2', directory='channel2') - - And, Videos like so: - Video(channel_id=1, video_path='vid1.mp4') - Video(channel_id=2, video_path='vid1.mp4') - Video(channel_id=2, video_path='vid2.mp4', caption_path='vid2.en.vtt') - """ - - def wrapper(func): - @wraps(func) - @wrap_test_db - def wrapped(*args, **kwargs): - if args and (test := args[0]) and hasattr(test, 'tmp_dir'): - tmp_dir = pathlib.Path(test.tmp_dir.name) - else: - tmp_dir = pathlib.Path(tempfile.TemporaryDirectory().name) - set_test_media_directory(tmp_dir) - - # Convert the channel/video structure to a file structure for the test. - file_structure = [] - for channel, paths in structure.items(): - for path in paths: - file_structure.append(f'{channel}/{path}') - file_structure.append(f'{channel}/') - - with build_test_directories(file_structure, tmp_dir) as tempdir: - args, kwargs = insert_parameter(func, 'tempdir', tempdir, args, kwargs) - - with get_db_session(commit=True) as session: - for channel in structure: - channel_ = Channel(directory=str(tempdir / channel), name=channel, link=channel) - session.add(channel_) - session.flush() - session.refresh(channel_) - - with get_db_session(commit=True) as session: - for channel_ in session.query(Channel).all(): - refresh_channel_videos(channel_) - - try: - return func(*args, **kwargs) - finally: - set_test_media_directory(None) - - return wrapped - - return wrapper diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py new file mode 100644 index 00000000..3fb24242 --- /dev/null +++ b/modules/videos/test/test_api.py @@ -0,0 +1,72 @@ +import shutil +from http import HTTPStatus +from pathlib import Path + +from modules.videos.models import Video +from wrolpi.downloader import DownloadFrequency +from wrolpi.vars import PROJECT_DIR + + +def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory): + subdir = test_directory / 'subdir' + subdir.mkdir() + + # video1 is in a subdirectory. + video1 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True, + with_poster_ext='jpg') + test_session.commit() + shutil.move(video1.video_path.path, subdir / video1.video_path.path.name) + shutil.move(video1.poster_path.path, subdir / video1.poster_path.path.name) + video1.video_path = subdir / video1.video_path.path.name + video1.poster_path = None + # video2 is in the test directory. + video2 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True, + with_poster_ext='jpg') + video2.poster_path = video1.poster_path = None + test_session.commit() + + assert not video1.size, 'video1 should not have size during creation' + + # Create a video not in the DB. + vid3 = test_directory / 'vid3.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', vid3) + + # Orphan poster should be ignored. + orphan_poster = Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg') + orphan_poster.touch() + + # Create a bogus video in the channel. + bogus = Video(video_path='foo', channel_id=simple_channel.id) + test_session.add(bogus) + + test_client.post('/api/videos/refresh') + + # Posters were found during refresh. + assert video1.poster_path + assert 'subdir' in str(video1.poster_path.path) + assert video2.poster_path + # Missing video3 was found + video3: Video = test_session.query(Video).filter_by(id=4).one() + assert video3.video_path.path == vid3 + assert video3.video_path != video1.video_path + assert video3.video_path != video2.video_path + # Bogus video was removed. + assert not any('foo' in str(i.video_path.path) for i in test_session.query(Video).all()) + # Orphan file was not deleted. + assert orphan_poster.is_file(), 'Orphan poster was removed!' + + assert video1.size, 'video1 size was not found' + + +def test_api_download_channel(test_session, test_client, simple_channel): + """A Channel download (a catalog update) can be triggered via the API.""" + # simple_channel does not have a download record. + request, response = test_client.post(f'/api/videos/download/{simple_channel.id}') + assert response.status_code == HTTPStatus.BAD_REQUEST, response.json + assert 'not have a download' in response.json['message'] + + # Add a download frequency to the channel, this should also create a download. + simple_channel.update(dict(download_frequency=DownloadFrequency.daily)) + test_session.commit() + request, response = test_client.post(f'/api/videos/download/{simple_channel.id}') + assert response.status_code == HTTPStatus.NO_CONTENT diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 1930e02f..1905cf7e 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -4,23 +4,19 @@ import tempfile from datetime import datetime from typing import List from unittest import mock -from unittest.mock import MagicMock import pytest from PIL import Image from modules.videos.models import Channel, Video -from modules.videos.test.common import create_channel_structure -from wrolpi.common import get_absolute_media_path -from wrolpi.dates import local_timezone -from wrolpi.db import get_db_session +from wrolpi.common import get_absolute_media_path, sanitize_link +from wrolpi.dates import local_timezone, now from wrolpi.downloader import Download, DownloadFrequency -from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI +from wrolpi.test.common import build_test_directories, TestAPI from wrolpi.vars import PROJECT_DIR from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \ - apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \ - import_videos_config -from ..lib import save_channels_config, get_channels_config + apply_info_json, get_video_duration, generate_video_poster, is_valid_poster +from ..lib import save_channels_config, get_channels_config, import_channels_config class TestCommon(TestAPI): @@ -134,113 +130,50 @@ def test_is_valid_poster(ext, expected, test_directory): assert is_valid_poster(image_path) == expected, f'is_valid_poster({image_path}) should be {expected}' -@pytest.mark.parametrize( - '_structure,paths', - ( - ( - {'channel1': ['vid1.mp4']}, - [ - 'channel1/vid1.mp4', - ], - ), - ( - {'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4']}, - [ - 'channel1/vid1.mp4', - 'channel2/vid1.mp4', - ], - ), - ( - {'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4', 'vid2.mp4', 'vid2.en.vtt']}, - [ - 'channel1/vid1.mp4', - 'channel2/vid1.mp4', - 'channel2/vid2.mp4', - 'channel2/vid2.en.vtt', - ], - ), - ) -) -def test_create_db_structure(_structure, paths): - @create_channel_structure(_structure) - def test_func(_, tempdir): - assert isinstance(tempdir, pathlib.Path) - for path in paths: - path = (tempdir / path) - assert path.exists() - assert path.is_file() - - with get_db_session() as session: - for channel_name in _structure: - channel = session.query(Channel).filter_by(name=channel_name).one() - assert (tempdir / channel_name).is_dir() - assert channel - assert channel.directory == tempdir / channel_name - assert len(channel.videos) == len([i for i in _structure[channel_name] if i.endswith('mp4')]) - - with tempfile.TemporaryDirectory() as tmp_dir, \ - mock.patch('wrolpi.media_path.get_media_directory') as mock_get_directory: - mock_get_directory.return_value = pathlib.Path(tmp_dir) - test_ = MagicMock() - test_.tmp_dir.name = tmp_dir - test_func(test_) - - -@wrap_test_db -@create_channel_structure( - { - 'channel1': ['vid1.mp4', 'vid1.jpg'], - 'channel2': ['vid2.mp4'], - 'channel3': ['vid3.mp4', 'vid4.mp4'], - } -) -def test_update_view_count(tempdir: pathlib.Path): +def test_update_view_count(test_session, channel_factory, video_factory): def check_view_counts(view_counts): - with get_db_session() as session_: - for source_id, view_count in view_counts.items(): - vid = session_.query(Video).filter_by(source_id=source_id).one() - assert vid.view_count == view_count + for source_id, view_count in view_counts.items(): + video = test_session.query(Video).filter_by(source_id=source_id).one() + assert video.view_count == view_count - with get_db_session(commit=True) as session: - channel1, channel2, channel3 = session.query(Channel).order_by(Channel.id).all() - channel1.info_json = {'entries': [{'id': 'vid1.mp4', 'view_count': 10}]} - channel2.info_json = {'entries': [{'id': 'vid2.mp4', 'view_count': 11}, {'id': 'bad_id', 'view_count': 12}]} - channel3.info_json = {'entries': [{'id': 'vid3.mp4', 'view_count': 13}, {'id': 'vid4.mp4', 'view_count': 14}]} - - # Use the video file name as a unique source_id - for v in session.query(Video).all(): - v.source_id = str(v.video_path.path).split('/')[-1] + channel1, channel2, channel3 = channel_factory(), channel_factory(), channel_factory() + video_factory(channel_id=channel1.id, with_poster_ext='jpg', title='vid1') + video_factory(channel_id=channel2.id, title='vid2') + video_factory(channel_id=channel3.id, with_poster_ext='jpg', title='vid3') + video_factory(channel_id=channel3.id, title='vid4') + channel1.info_json = {'entries': [{'id': 'vid1', 'view_count': 10}]} + channel2.info_json = {'entries': [{'id': 'vid2', 'view_count': 11}, {'id': 'bad_id', 'view_count': 12}]} + channel3.info_json = {'entries': [{'id': 'vid3', 'view_count': 13}, {'id': 'vid4', 'view_count': 14}]} + test_session.commit() # Check all videos are empty. - check_view_counts({'vid1.mp4': None, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None}) + check_view_counts({'vid1': None, 'vid2': None, 'vid3': None, 'vid4': None}) # Channel 1 is updated, the other channels are left alone. apply_info_json(channel1.id) - check_view_counts({'vid1.mp4': 10, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None}) + check_view_counts({'vid1': 10, 'vid2': None, 'vid3': None, 'vid4': None}) # Channel 2 is updated, the other channels are left alone. The 'bad_id' video is ignored. apply_info_json(channel2.id) - check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': None, 'vid4.mp4': None}) + check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': None, 'vid4': None}) # All videos are updated. apply_info_json(channel3.id) - check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14}) + check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14}) # An outdated view count will be overwritten. - with get_db_session(commit=True) as session: - vid = session.query(Video).filter_by(id=1).one() - vid.view_count = 8 - check_view_counts({'vid1.mp4': 8, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14}) + vid = test_session.query(Video).filter_by(id=1).one() + vid.view_count = 8 + check_view_counts({'vid1': 8, 'vid2': 11, 'vid3': 13, 'vid4': 14}) apply_info_json(channel1.id) - check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14}) + check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14}) def test_generate_video_poster(video_file): """ A poster can be generated from a video file. """ - poster_path = replace_extension(video_file, '.jpg') - assert not poster_path.is_file(), f'{poster_path} already exists!' + poster_path = video_file.with_suffix('.jpg') generate_video_poster(video_file) assert poster_path.is_file(), f'{poster_path} was not created!' assert poster_path.stat().st_size > 0 @@ -326,7 +259,7 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan vid2_video_path = vid2.video_path.path favorites = { - simple_channel.link: {vid3.video_path.path.name: {'favorite': favorite}}, + str(simple_channel.directory.relative): {vid3.video_path.path.name: {'favorite': favorite}}, 'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}, } @@ -336,8 +269,8 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan assert config.favorites == favorites def import_and_verify(favorited_ids: List[int]): - with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'): - import_videos_config() + with mock.patch('modules.videos.lib.get_channel_source_id', lambda i: 'foo'): + import_channels_config() for video in test_session.query(Video).all(): if video.id in favorited_ids: assert video.favorite @@ -371,10 +304,8 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan def test_import_channel_downloads(test_session, channel_factory, test_channels_config): """Importing the Channels' config should create any missing download records""" - channel1 = channel_factory() - channel2 = channel_factory() - channel1.source_id = 'foo' - channel2.source_id = 'bar' + channel1 = channel_factory(source_id='foo') + channel2 = channel_factory(source_id='bar') # channel2 has no url, but has a frequency. Import should not create a download record. channel2.url = None channel2.download_frequency = DownloadFrequency.biweekly @@ -383,20 +314,25 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c assert len(test_session.query(Channel).all()) == 2 assert test_session.query(Download).all() == [] + def update_channel_config(conf, source_id, d): + for c in conf.channels: + if c['source_id'] == source_id: + c.update(d) + conf.save() + # Config has no channels with a download_frequency. save_channels_config() - import_videos_config() + import_channels_config() assert channel1.download_frequency is None assert len(test_session.query(Channel).all()) == 2 assert test_session.query(Download).all() == [] # Add a frequency to the Channel. channels_config = get_channels_config() - channels_config.channels[channel1.link]['download_frequency'] = DownloadFrequency.biweekly - channels_config.save() + update_channel_config(channels_config, 'foo', {'download_frequency': DownloadFrequency.biweekly}) # Download record is created on import. - import_videos_config() + import_channels_config() assert channel1.download_frequency is not None assert len(test_session.query(Channel).all()) == 2 download: Download = test_session.query(Download).one() @@ -404,9 +340,8 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c assert download.frequency == channel1.download_frequency # Download frequency is adjusted when config file changes. - channels_config.channels[channel1.link]['download_frequency'] = DownloadFrequency.weekly - channels_config.save() - import_videos_config() + update_channel_config(channels_config, 'foo', {'download_frequency': DownloadFrequency.weekly}) + import_channels_config() assert len(test_session.query(Channel).all()) == 2 assert download.url == channel1.url assert download.frequency == channel1.download_frequency @@ -414,6 +349,67 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c assert download.next_download next_download = str(download.next_download) - import_videos_config() + import_channels_config() download: Download = test_session.query(Download).one() assert next_download == str(download.next_download) + + +def test_import_channels_config_outdated(test_session, test_directory, channel_factory, test_channels_config, + video_factory): + """The Channels' config used to use a "link" to differentiate channels, test that old configs can be imported.""" + channel1 = channel_factory(name='Channel1', download_frequency=DownloadFrequency.weekly) + channel2 = channel_factory(name='Channel2', download_frequency=DownloadFrequency.weekly) + test_session.commit() + + vid1 = video_factory(title='vid1', channel_id=channel1.id) + vid2 = video_factory(title='vid2', channel_id=channel2.id) + vid3 = video_factory(title='vid3') + vid4 = video_factory(title='vid4') + test_session.commit() + + channel1_name, channel_1_directory = channel1.name, channel1.directory.path + channel2_name, channel_2_directory = channel2.name, channel2.directory.path + + assert channel1.directory.path.exists() and channel1.directory.path.is_absolute() + assert channel2.directory.path.exists() and channel2.directory.path.is_absolute() + + # Channel's used to have a relative directory in the DB. + test_session.execute('UPDATE channel SET directory=:directory WHERE id=2', + dict(directory=str(channel_2_directory.relative_to(test_directory))), + ) + + save_channels_config() + + # The config has a list of Channels, use the old method of a "dict" with the sanitized name as the key. + channels_config = get_channels_config() + channels_config.channels = {sanitize_link(i['name']): i for i in channels_config.channels} + # Change frequency to verify that the channels are updated. + for link, channel in channels_config.channels.items(): + channel['download_frequency'] = DownloadFrequency.biweekly + # Change the favorites to the old "link" method as well. + channels_config.favorites = { + sanitize_link(channel1.name): {str(vid1.video_path.path): {'favorite': now()}}, + sanitize_link(channel2.name): {str(vid2.video_path.path): {'favorite': now()}}, + 'NO CHANNEL': {str(vid3.video_path.path): {'favorite': now()}}, + } + channels_config.save() + + assert not vid1.favorite and not vid2.favorite and not vid3.favorite and not vid3.favorite + + assert test_session.query(Channel).count() == 2 + + with mock.patch('modules.videos.lib.get_channel_source_id') as mock_get_channel_source_id: + mock_get_channel_source_id.return_value = 'some source id' + import_channels_config() + + # No new channels were created. Existing channels were updated. + assert test_session.query(Channel).count() == 2 + channel1, channel2 = test_session.query(Channel).order_by(Channel.id) + assert channel1.name == channel1_name and str(channel1.directory.path) == str(channel_1_directory) + # Channel2's directory is now absolute. + assert channel2.name == channel2_name and str(channel2.directory.path) == str(channel_2_directory) + assert channel1.download_frequency == DownloadFrequency.biweekly + assert channel2.download_frequency == DownloadFrequency.biweekly + # Videos were favorited by their channel "link". + assert vid1.favorite and vid2.favorite and vid3.favorite + assert not vid4.favorite diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index d15b3033..19a50957 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -1,4 +1,5 @@ import pathlib +import shutil from copy import copy from unittest import mock @@ -9,12 +10,12 @@ from modules.videos.channel.lib import download_channel from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \ ChannelDownloader, get_or_create_channel from modules.videos.models import Channel, Video -from modules.videos.test.common import create_channel_structure -from wrolpi.db import get_db_session, get_db_context +from wrolpi.db import get_db_context from wrolpi.downloader import DownloadManager, Download from wrolpi.errors import InvalidDownload from wrolpi.test.common import wrap_test_db, TestAPI from wrolpi.test.test_downloader import HTTPDownloader +from wrolpi.vars import PROJECT_DIR class FakeYDL: @@ -28,37 +29,32 @@ class FakeYDL: } -class TestDownloader(TestAPI): - @wrap_test_db - @create_channel_structure( - { - 'channel1': ['vid1.mp4'], - 'channel2': ['vid2.mp4'] - } - ) - def test_find_all_missing_videos(self, tempdir): - with get_db_session(commit=True) as session: - channel1, channel2 = session.query(Channel).order_by(Channel.id).all() - channel1.url = channel2.url = 'some url' - channel1.info_json = {'entries': [{'id': 'foo'}]} +def test_find_all_missing_videos(test_session, channel_factory, video_factory): + channel1, channel2 = channel_factory(url='some url'), channel_factory(url='some other url') + channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]} + test_session.commit() - # No missing videos - self.assertEqual([], list(find_all_missing_videos())) + # Two videos are already downloaded. + video_factory() + video_factory() - # Create a video that has no video file. - with get_db_session(commit=True) as session: - video = Video(title='needs to be downloaded', channel_id=channel1.id, source_id='foo') - session.add(video) + # No missing videos + assert [] == list(find_all_missing_videos()) - missing = list(find_all_missing_videos()) - self.assertEqual(len(missing), 1) + # Create a video that has no video file. + video = Video(title='needs to be downloaded', channel_id=channel1.id, source_id='foo') + test_session.add(video) + test_session.commit() - # The video returned is the one we faked. - id_, source_id, entry = missing[0] - # Two videos were created for this test already. - self.assertEqual(id_, 3) - # The fake entry we added is regurgitated back. - self.assertEqual(entry, channel1.info_json['entries'][0]) + missing = list(find_all_missing_videos()) + assert len(missing) == 1 + + # The video returned is the one we faked. + id_, source_id, entry = missing[0] + # Two videos were created for this test already. + assert id_ == 3 + # The fake entry we added is regurgitated back. + assert entry == channel1.info_json['entries'][0] example_video_json = { @@ -171,43 +167,6 @@ class TestVideosDownloaders(TestAPI): info_json_path.touch() return video_path, poster_path, description_path, info_json_path - @wrap_test_db - def test_video_download(self): - channel_dir = self.videos_dir / 'channel name' - channel_dir.mkdir(parents=True) - - # Video paths - video_path, poster_path, description_path, info_json_path = self._make_video_files(channel_dir) - - with get_db_session() as session: - url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ - mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ - mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: - mock_extract_info.return_value = example_video_json - mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) - mock_process_runner.return_value = 0 - - self.mgr.create_download(url, session) - - mock_process_runner.assert_called_once() - video_url, _, out_dir = mock_process_runner.call_args[0] - - download: Download = session.query(Download).one() - self.assertEqual(video_url, download.url) - channel: Channel = session.query(Channel).one() - self.assertEqual(channel.directory, channel_dir) - - video: Video = session.query(Video).one() - self.assertEqual(video.video_path, video_path) - self.assertEqual(video.description_path, description_path) - self.assertEqual(video.poster_path, poster_path) - self.assertEqual(video.info_json_path, info_json_path) - self.assertTrue(video.video_path.path.is_absolute()) - self.assertTrue(video.poster_path.path.is_absolute()) - self.assertTrue(video.description_path.path.is_absolute()) - self.assertTrue(video.info_json_path.path.is_absolute()) - @wrap_test_db def test_video_download_no_channel(self): _, session = get_db_context() @@ -239,73 +198,65 @@ class TestVideosDownloaders(TestAPI): video: Video = session.query(Video).one() self.assertEqual(str(video.video_path.path), f'{channel_dir}/video.mp4') - @wrap_test_db - def test_channel_download(self): - _, session = get_db_context() - url = 'https://www.youtube.com/c/LearningSelfReliance/videos' - session.add(Channel(url=url, link='foo')) +def test_channel_download(test_session, simple_channel, video_download_manager, video_file): + url = 'https://www.youtube.com/c/LearningSelfReliance/videos' - channel_dir = self.videos_dir / 'NO CHANNEL' - channel_dir.mkdir(parents=True) - video_path, _, _, _ = self._make_video_files(channel_dir) + with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ + mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ + mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: + mock_extract_info.return_value = example_channel_json + mock_prepare_filename.return_value = (video_file, {'id': 'foo'}) + mock_process_runner.return_value = 0 + video_download_manager.create_download(url, test_session) - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ - mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ - mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: - mock_extract_info.return_value = example_channel_json - mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) - mock_process_runner.return_value = 0 - self.mgr.create_download(url, session) + # Two videos are in the example channel. + downloads = video_download_manager.get_new_downloads(test_session) + expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o'] + for download, expected in zip(downloads, expected): + assert download.url == expected + # Download is skipped for test. + assert download.status == 'new' - # Two videos are in the example channel. - downloads = self.mgr.get_new_downloads(session) - expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o'] - for download, expected in zip(downloads, expected): - self.assertEqual(download.url, expected) - # Download is skipped for test. - self.assertEqual(download.status, 'new') - @wrap_test_db - def test_get_or_create_channel(self): - """ - A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can - match it. - """ - with get_db_session(commit=True) as session: - c1 = Channel(name='foo', link='foo', source_id='foo', url='foo') - c2 = Channel(name='bar', link='bar', source_id='bar') - c3 = Channel(name='baz', link='baz', source_id='baz', url='baz') - c4 = Channel(name='qux', link='qux') - session.add_all([c1, c2, c3, c4]) +def test_get_or_create_channel(test_session): + """ + A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can + match it. + """ + c1 = Channel(name='foo', source_id='foo', url='foo') + c2 = Channel(name='bar', source_id='bar') + c3 = Channel(name='baz', source_id='baz', url='baz') + c4 = Channel(name='qux') + test_session.add_all([c1, c2, c3, c4]) + test_session.commit() - # All existing channels should be used. - tests = [ - (dict(source_id='foo'), c1), - (dict(link='foo'), c1), - (dict(url='foo'), c1), - (dict(url='foo', source_id='bar'), c2), # source_id is preferred. - (dict(name='foo', source_id='bar'), c2), - (dict(source_id='bar'), c2), - (dict(source_id='baz'), c3), - (dict(name='qux'), c4), - (dict(link='qux'), c4), - ] - for kwargs, expected in tests: + # All existing channels should be used. + tests = [ + (dict(source_id='foo'), c1), + (dict(url='foo'), c1), + (dict(url='foo', source_id='bar'), c2), # source_id has priority. + (dict(name='foo', source_id='bar'), c2), + (dict(source_id='bar'), c2), + (dict(source_id='baz'), c3), + (dict(name='qux'), c4), + ] + for kwargs, expected in tests: + try: channel = get_or_create_channel(**kwargs) - self.assertEqual(expected.id, channel.id, f'Expected {expected} got {channel}') + except Exception as e: + raise Exception(f'get_or_create_channel failed with {kwargs=}') from e + assert expected.id == channel.id, f'Expected {expected} for {kwargs} but got {channel}' - # A new channel is created. It will not be automatically downloaded. - channel = get_or_create_channel(source_id='quux', name='quux', url='quux') - self.assertEqual(channel.id, 5) - self.assertEqual(channel.source_id, 'quux') - self.assertEqual(channel.name, 'quux') - self.assertEqual(channel.link, 'quux') - self.assertEqual(channel.url, 'quux') - self.assertIsNone(channel.download_frequency) + # A new channel is created. It will not be automatically downloaded. + channel = get_or_create_channel(source_id='quux', name='quux', url='quux') + assert channel.id == 5 + assert channel.source_id == 'quux' + assert channel.name == 'quux' + assert channel.url == 'quux' - # New channel can be retrieved. - self.assertEqual(get_or_create_channel(source_id='quux'), channel) + # New channel can be retrieved. + assert get_or_create_channel(source_id='quux') == channel def test_download_info_save(test_session, video_download_manager): @@ -397,7 +348,7 @@ def test_channel_download_no_download(test_session, video_download_manager, simp assert video_download_manager.get_downloads(test_session) == [] with pytest.raises(InvalidDownload): - download_channel(simple_channel.link) + download_channel(simple_channel.id) def test_invalid_download_url(test_session, test_download_manager, test_downloader_config): @@ -413,3 +364,32 @@ def test_invalid_download_url(test_session, test_download_manager, test_download download = test_download_manager.get_downloads(test_session)[0] assert download.status == 'failed' + + +def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager): + simple_channel.source_id = example_video_json['channel_id'] + simple_channel.directory = test_directory / 'videos/channel name' + simple_channel.directory.mkdir(parents=True) + + video_file = simple_channel.directory / 'a video.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file) + + url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' + with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ + mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ + mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: + mock_extract_info.return_value = example_video_json + mock_prepare_filename.return_value = (video_file, {'id': 'foo'}) + mock_process_runner.return_value = 0 + + video_download_manager.create_download(url, test_session) + + mock_process_runner.assert_called_once() + video_url, _, out_dir = mock_process_runner.call_args[0] + + download: Download = test_session.query(Download).one() + assert video_url == download.url + assert test_session.query(Channel).one() + + video: Video = test_session.query(Video).one() + assert video.video_path.path.is_absolute() diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py index 7dfcf39b..8d9ff7fe 100644 --- a/modules/videos/video/api.py +++ b/modules/videos/video/api.py @@ -44,7 +44,7 @@ async def search(_: Request, body: schema.VideoSearchRequest): body.search_str, body.offset, body.limit, - body.channel_link, + body.channel_id, body.order_by, body.filters, ) diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index 806e3df0..edb3111a 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -80,16 +80,16 @@ def video_search( search_str: str = None, offset: int = None, limit: int = VIDEO_QUERY_LIMIT, - channel_link: str = None, + channel_id: int = None, order_by: str = None, filters: List[str] = None, ) -> Tuple[List[dict], int]: with get_db_curs() as curs: params = dict(search_str=search_str, offset=offset) channel_where = '' - if channel_link: - channel_where = 'AND channel_id = (select id from channel where link=%(channel_link)s)' - params['channel_link'] = channel_link + if channel_id: + channel_where = 'AND channel_id = %(channel_id)s' + params['channel_id'] = channel_id # Filter for/against favorites, if it was provided favorites_where = '' diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py index a30a1a92..41c71a26 100644 --- a/modules/videos/video/test/test_api.py +++ b/modules/videos/video/test/test_api.py @@ -1,163 +1,141 @@ -import pathlib from datetime import timedelta from http import HTTPStatus from json import dumps from unittest import mock -from uuid import uuid4 -from modules.videos.models import Channel, Video -from modules.videos.test.common import create_channel_structure +from modules.videos.models import Video from modules.videos.video.lib import get_video_for_app from wrolpi.dates import now -from wrolpi.db import get_db_session from wrolpi.errors import API_ERRORS, WROLModeEnabled from wrolpi.root_api import api_app -from wrolpi.test.common import wrap_test_db, TestAPI -class TestVideoFunctions(TestAPI): +def test_get_video_prev_next(test_session, channel_factory, video_factory): + """ + Test that the previous and next videos will be retrieved when fetching a video. + """ + channel1, channel2, channel3, channel4 = channel_factory(), channel_factory(), channel_factory(), channel_factory() - @wrap_test_db - def test_get_video_prev_next(self): - """ - Test that the previous and next videos will be retrieved when fetching a video. - """ + now_ = now() + second = timedelta(seconds=1) - with get_db_session(commit=True) as session: - for _ in range(4): - session.add(Channel(link=str(uuid4()))) - channel1, channel2, channel3, channel4 = session.query(Channel).all() + # The upload_date decides the order of the prev/next videos. + video_factory(title='vid1', channel_id=channel1.id, upload_date=now_) + video_factory(title='vid2', channel_id=channel1.id, upload_date=now_ + second) + video_factory(title='vid3', channel_id=channel2.id, upload_date=now_ + (second * 4)) + video_factory(title='vid4', channel_id=channel1.id, upload_date=now_ + (second * 3)) + video_factory(title='vid5', channel_id=channel2.id, upload_date=now_ + (second * 2)) + video_factory(title='vid6', channel_id=channel2.id, upload_date=now_ + (second * 5)) + video_factory(title='vid7', channel_id=channel1.id) + video_factory(title='vid8', channel_id=channel2.id, upload_date=now_ + (second * 7)) + video_factory(title='vid9', channel_id=channel3.id, upload_date=now_ + (second * 8)) + # Videos without an upload date can't be placed. + video_factory(title='vid10', channel_id=channel4.id) + video_factory(title='vid11', channel_id=channel4.id) - now_ = now() - second = timedelta(seconds=1) + test_session.commit() - # The upload_date decides the order of the prev/next videos. - session.add(Video(title=f'vid1', channel_id=channel1.id, upload_date=now_)) - session.add(Video(title=f'vid2', channel_id=channel1.id, upload_date=now_ + second)) - session.add(Video(title=f'vid3', channel_id=channel2.id, upload_date=now_ + (second * 4))) - session.add(Video(title=f'vid4', channel_id=channel1.id, upload_date=now_ + (second * 3))) - session.add(Video(title=f'vid5', channel_id=channel2.id, upload_date=now_ + (second * 2))) - session.add(Video(title=f'vid6', channel_id=channel2.id, upload_date=now_ + (second * 5))) - session.add(Video(title=f'vid7', channel_id=channel1.id)) - session.add(Video(title=f'vid8', channel_id=channel2.id, upload_date=now_ + (second * 7))) - session.add(Video(title=f'vid9', channel_id=channel3.id, upload_date=now_ + (second * 8))) - session.add(Video(title=f'vid10', channel_id=channel4.id)) - session.add(Video(title=f'vid11', channel_id=channel4.id)) + tests = [ + # Channel 1's videos were inserted in upload_date order. + (1, (None, 'vid2')), + (2, ('vid1', 'vid4')), + (4, ('vid2', None)), # 7 has no upload_date, so it doesn't come after 4. + (7, (None, None)), # 7 has no upload_date, so we don't know the order of it. + # Channel 3 has only one video. + (9, (None, None)), + # Channel 2 was inserted out of order. + (5, (None, 'vid3')), + (3, ('vid5', 'vid6')), + (8, ('vid6', None)), + # Channel 4's videos have no upload date, so we don't know what is previous/next. + (10, (None, None)), + ] - tests = [ - # Channel 1's videos were inserted in upload_date order. - (1, (None, 'vid2')), - (2, ('vid1', 'vid4')), - (4, ('vid2', None)), # 7 has no upload_date, so it doesn't come after 4. - (7, (None, None)), # 7 has no upload_date, so we don't know the order of it. - # Channel 3 has only one video. - (9, (None, None)), - # Channel 2 was inserted out of order. - (5, (None, 'vid3')), - (3, ('vid5', 'vid6')), - (8, ('vid6', None)), - # Channel 4's videos have no upload date, so we don't know what is previous/next. - (10, (None, None)), - ] - - for id_, (prev_title, next_title) in tests: - video = session.query(Video).filter_by(id=id_).one() - prev_video, next_video = video.get_surrounding_videos() + for id_, (prev_title, next_title) in tests: + video = test_session.query(Video).filter_by(id=id_).one() + prev_video, next_video = video.get_surrounding_videos() + try: if prev_title is None: - self.assertIsNone(prev_video) + assert prev_video is None else: - self.assertDictContains(prev_video, {'title': prev_title}) + assert prev_video and prev_video.title == prev_title if next_title is None: - self.assertIsNone(next_video) + assert next_video is None else: - self.assertDictContains(next_video, {'title': next_title}) + assert next_video and next_video.title == next_title + except AssertionError as e: + raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e - @wrap_test_db - @create_channel_structure({ - 'channel1': ['vid1.mp4'] - }) - def test_wrol_mode(self, tempdir): - """ - Many methods are blocked when WROL Mode is enabled. - """ - channel = dumps(dict(name='foo', directory='foo')) - favorite = dumps(dict(video_id=1, favorite=True)) - with mock.patch('wrolpi.common.wrol_mode_enabled', lambda: True): - # Can't create, update, or delete a channel. - _, resp = api_app.test_client.post('/api/videos/channels', content=channel) - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) - _, resp = api_app.test_client.put('/api/videos/channels/channel1', content=channel) - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) - _, resp = api_app.test_client.delete('/api/videos/channels/channel1') - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) +def test_get_video_for_app(test_session, simple_channel, simple_video): + vid, prev, next_ = get_video_for_app(simple_video.id) + assert vid['id'] == simple_video.id - # Can't delete a video - _, resp = api_app.test_client.delete('/api/videos/video/1') - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) - # Can't refresh or download - _, resp = api_app.test_client.post('/api/videos/refresh') - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) - _, resp = api_app.test_client.post('/api/videos/download') - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) +def test_video_delete(test_session, test_directory, channel_factory, video_factory): + """Video.delete() removes the video's files, but leave the DB record.""" + channel1, channel2 = channel_factory(), channel_factory() + vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True) + vid2 = video_factory(channel_id=channel2.id, with_video_file=True, with_info_json=True) + test_session.commit() - # THE REST OF THESE METHODS ARE ALLOWED - _, resp = api_app.test_client.post('/api/videos/favorite', content=favorite) - self.assertEqual(resp.status_code, HTTPStatus.OK) + vid1_video_path, vid1_caption_path = vid1.video_path.path, vid1.caption_path.path + vid2_video_path, vid2_info_json_path = vid2.video_path.path, vid2.info_json_path.path - @wrap_test_db - @create_channel_structure({ - 'channel1': [ - 'vid1.mp4', - 'vid1.en.vtt', - ], - }) - def test_get_video_for_app(self, tempdir): - with get_db_session(commit=True) as session: - vid1 = session.query(Video).one() + # No videos have been deleted yet. + assert vid1_video_path.is_file() and vid1_caption_path.is_file() + assert vid2_video_path.is_file() and vid2_info_json_path.is_file() + assert channel1.skip_download_videos is None + assert channel2.skip_download_videos is None - vid, prev, next_ = get_video_for_app(vid1.id) - self.assertEqual(vid['id'], vid1.id) + vid1.delete() - @wrap_test_db - @create_channel_structure({ - 'channel1': [ - 'vid1.mp4', - 'vid1.en.vtt', - ], - 'channel2': [ - 'vid2.mp4', - 'vid2.info.json', - ], - }) - def test_delete_video(self, tempdir: pathlib.Path): - with get_db_session(commit=True) as session: - channel1 = session.query(Channel).filter_by(name='channel1').one() - channel2 = session.query(Channel).filter_by(name='channel2').one() - vid1, vid2 = session.query(Video).order_by(Video.video_path).all() - vid1.source_id = 'foo' - vid2.source_id = 'bar' + # Video was added to skip list. + assert len(channel1.skip_download_videos) == 1 + # Deleting a video leaves it's entry in the DB, but its files are deleted. + assert test_session.query(Video).count() == 2 + assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False + assert vid2_video_path.is_file() and vid2_info_json_path.is_file() - # No videos have been deleted yet. - self.assertIsNone(channel1.skip_download_videos) - self.assertIsNone(channel2.skip_download_videos) - self.assertTrue((tempdir / 'channel1/vid1.mp4').is_file()) + vid2.delete() - vid1.delete() + assert test_session.query(Video).count() == 2 + assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False + assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False - channel1 = session.query(Channel).filter_by(name='channel1').one() - # Video was added to skip list. - self.assertEqual(len(channel1.skip_download_videos), 1) - # Deleting a video leaves it's entry in the DB, but its files are deleted. - self.assertEqual(session.query(Video).count(), 2) - self.assertFalse((tempdir / 'channel1/vid1.mp4').is_file()) - self.assertTrue((tempdir / 'channel2/vid2.mp4').is_file()) - vid2.delete() +def test_wrol_mode(test_directory, simple_channel, simple_video): + """Many methods are blocked when WROL Mode is enabled.""" + channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory))) + favorite = dumps(dict(video_id=simple_video.id, favorite=True)) - self.assertEqual(session.query(Video).count(), 2) - self.assertFalse((tempdir / 'channel1/vid1.mp4').is_file()) - self.assertFalse((tempdir / 'channel2/vid2.mp4').is_file()) + with mock.patch('wrolpi.common.wrol_mode_enabled', lambda: True): + # Can't create, update, or delete a channel. + _, resp = api_app.test_client.post('/api/videos/channels', content=channel) + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + _, resp = api_app.test_client.put(f'/api/videos/channels/{simple_channel.id}', content=channel) + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + _, resp = api_app.test_client.delete(f'/api/videos/channels/{simple_channel.id}') + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + + # Can't delete a video + _, resp = api_app.test_client.delete(f'/api/videos/video/{simple_video.id}') + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + + # Can't refresh or download + _, resp = api_app.test_client.post('/api/videos/refresh') + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + _, resp = api_app.test_client.post(f'/api/videos/download/{simple_channel.id}') + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + + # THE REST OF THESE METHODS ARE ALLOWED + _, resp = api_app.test_client.post('/api/videos/favorite', content=favorite) + assert resp.status_code == HTTPStatus.OK diff --git a/wrolpi/__init__.py b/wrolpi/__init__.py index 7fe8a8d8..d479333b 100644 --- a/wrolpi/__init__.py +++ b/wrolpi/__init__.py @@ -1,5 +1,6 @@ import inspect import multiprocessing +from functools import wraps BEFORE_STARTUP_FUNCTIONS = [] @@ -29,6 +30,7 @@ def limit_concurrent(limit: int): def wrapper(func: callable): if inspect.iscoroutinefunction(func): + @wraps(func) async def wrapped(*a, **kw): acquired = sema.acquire(block=False) if not acquired: @@ -37,6 +39,7 @@ def limit_concurrent(limit: int): return wrapped else: + @wraps(func) def wrapped(*a, **kw): acquired = sema.acquire(block=False) if not acquired: diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py index bbc002cf..329c618a 100644 --- a/wrolpi/cmd.py +++ b/wrolpi/cmd.py @@ -20,7 +20,7 @@ def which(*possible_paths: str, warn: bool = False) -> Path: Returns None if no executable can be found. """ - possible_paths = map(Path, possible_paths) + possible_paths = list(map(Path, possible_paths)) for path in possible_paths: if is_executable(path): # `path` was an absolute file which is executable. diff --git a/wrolpi/common.py b/wrolpi/common.py index dd8b6404..a23a7c35 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -44,7 +44,7 @@ Base = declarative_base() class ModelHelper: - def dict(self) -> dict: + def dict(self, *_, **__) -> dict: d = {i.name: getattr(self, i.name) for i in self.__table__.columns} # noqa return d @@ -732,11 +732,6 @@ def native_only(func: callable): return wrapped -def read_yaml(path: Path): - with path.open('rt') as fh: - return yaml.load(fh, Loader=yaml.Loader) - - def recursive_map(obj: Any, func: callable): """ Apply `func` to all values of any dictionaries; or items in any list/set/tuple. @@ -755,3 +750,11 @@ def recursive_map(obj: Any, func: callable): type_ = type(obj) return type_(map(lambda i: recursive_map(i, func), obj)) return func(obj) + + +def any_extensions(filename: str, extensions: Iterable = ()): + """Return True only if the file name ends with any of the possible extensions. + + Matches lower or upper case of the extension. + """ + return any(filename.lower().endswith(ext) for ext in extensions) diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 69d79308..49bd2d29 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -1,7 +1,6 @@ """ Fixtures for Pytest tests. """ -import os import pathlib import tempfile from typing import Tuple diff --git a/wrolpi/errors.py b/wrolpi/errors.py index 22a03e9d..f052a99f 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -158,7 +158,7 @@ API_ERRORS = { }, UnknownChannel: { 'code': next(error_codes), - 'message': 'The channel could not be found by its link.', + 'message': 'The channel could not be found.', 'status': HTTPStatus.NOT_FOUND, }, UnknownDirectory: { diff --git a/wrolpi/media_path.py b/wrolpi/media_path.py index ff097225..099aa72d 100644 --- a/wrolpi/media_path.py +++ b/wrolpi/media_path.py @@ -18,10 +18,10 @@ class MediaPath: self._path = path.absolute() def __repr__(self): - return f'' + return f'' def __str__(self): - return str(self.path.relative_to(get_media_directory())) + return str(self.relative) @classmethod def _validate_path(cls, path: PATH_TYPE): @@ -40,7 +40,7 @@ class MediaPath: def __json__(self): # Always return a relative path to frontend. - return str(self.path.relative_to(get_media_directory())) + return str(self.relative) @property def path(self) -> pathlib.Path: @@ -60,6 +60,11 @@ class MediaPath: def startswith(self, other): return str(self._path).startswith(other) + @property + def relative(self): + """Get the path relative to the media directory.""" + return self._path.relative_to(get_media_directory()) + class MediaPathType(types.TypeDecorator): # noqa impl = types.String diff --git a/wrolpi/schema.py b/wrolpi/schema.py index b94cef0f..a1969cdd 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional +from typing import Optional @dataclass diff --git a/wrolpi/version.py b/wrolpi/version.py index 41b97933..8cad1f4d 100644 --- a/wrolpi/version.py +++ b/wrolpi/version.py @@ -1,4 +1,4 @@ -__version__ = '0.4.3-beta' +__version__ = '0.4.4-beta' import subprocess @@ -25,7 +25,5 @@ def git_branch(): def get_version_string(): - """ - Return a string containing the WROLPi version, git branch and git revision hash. - """ + """Return a string containing the WROLPi version, git branch and git revision hash.""" return f'{__version__} (git: {git_branch()} {git_revision()})' From d75ddf097db73f3ab1f05414e811fe4ade4f052c Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 29 Apr 2022 19:09:49 -0600 Subject: [PATCH 2/2] Removing unused function. --- modules/videos/channel/lib.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index 4cc0739f..3b6f19bb 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -1,14 +1,11 @@ -from collections import defaultdict -from datetime import timedelta from pathlib import Path -from typing import List, Dict, Union, Generator +from typing import List, Dict, Union from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from wrolpi.common import run_after, logger, \ get_media_directory -from wrolpi.dates import today from wrolpi.db import get_db_curs, optional_session from wrolpi.downloader import download_manager from wrolpi.errors import UnknownChannel, UnknownDirectory, APIError, ValidationError, InvalidDownload @@ -110,23 +107,6 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No return channel -def _spread_by_frequency(channels: List[Channel]) -> Generator[Dict, None, None]: - channels_by_frequency = defaultdict(lambda: []) - for channel in channels: - channels_by_frequency[channel.download_frequency].append(channel) - - for frequency, channels in channels_by_frequency.items(): - # The seconds between each download. - chunk = frequency // len(channels) - for channel in channels: - # My position in the channel group. - index = channels.index(channel) - # My next download will be distributed by my frequency and my position. - position = chunk * index - next_download = today() + timedelta(seconds=frequency + position) - yield dict(url=channel.url, frequency=frequency, next_download=next_download) - - @run_after(save_channels_config) @optional_session def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel: