diff --git a/alembic/versions/d8fcdb7773c0_drop_channel_link.py b/alembic/versions/d8fcdb7773c0_drop_channel_link.py new file mode 100644 index 00000000..7ec3a943 --- /dev/null +++ b/alembic/versions/d8fcdb7773c0_drop_channel_link.py @@ -0,0 +1,37 @@ +"""Drop channel.link. + +Revision ID: d8fcdb7773c0 +Revises: 2fb39c1861c7 +Create Date: 2022-04-08 12:52:02.877449 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = 'd8fcdb7773c0' +down_revision = '2fb39c1861c7' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE channel DROP CONSTRAINT IF EXISTS channel_link_key') + session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS link') + session.execute('DROP INDEX IF EXISTS channel_url_key') + session.execute("UPDATE channel SET url=null WHERE url=''") + session.execute('CREATE UNIQUE INDEX channel_url_key ON channel (url) WHERE url IS NOT NULL') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE channel ADD CONSTRAINT channel_link_key UNIQUE (link)') + session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS link TEXT') + # Notice we are not removing the channel_url_key constraint. diff --git a/app/src/App.js b/app/src/App.js index 2ed6b604..a9725839 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -82,7 +82,7 @@ export default function App() { <> - + diff --git a/app/src/api.js b/app/src/api.js index 444f4110..d906eeb7 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -95,9 +95,9 @@ let apiPatch = async (url, body) => { return await apiCall(url, 'PATCH', body) }; -export async function updateChannel(link, channel) { +export async function updateChannel(id, channel) { channel = emptyToNull(channel); - return await apiPut(`${VIDEOS_API}/channels/${link}`, channel); + return await apiPut(`${VIDEOS_API}/channels/${id}`, channel); } export async function createChannel(channel) { @@ -105,8 +105,8 @@ export async function createChannel(channel) { return await apiPost(`${VIDEOS_API}/channels`, channel); } -export async function deleteChannel(channel_link) { - return apiDelete(`${VIDEOS_API}/channels/${channel_link}`); +export async function deleteChannel(id) { + return apiDelete(`${VIDEOS_API}/channels/${id}`); } export async function getChannels() { @@ -114,12 +114,12 @@ export async function getChannels() { return (await response.json())['channels']; } -export async function getChannel(link) { - let response = await apiGet(`${VIDEOS_API}/channels/${link}`); +export async function getChannel(id) { + let response = await apiGet(`${VIDEOS_API}/channels/${id}`); return (await response.json())['channel']; } -export async function searchVideos(offset, limit, channel_link, searchStr, order_by, filters) { +export async function searchVideos(offset, limit, channel_id, searchStr, order_by, filters) { // Build a search query to retrieve a list of videos from the API offset = offset || 0; limit = limit || DEFAULT_LIMIT; @@ -128,8 +128,8 @@ export async function searchVideos(offset, limit, channel_link, searchStr, order if (searchStr) { body.search_str = searchStr; } - if (channel_link) { - body.channel_link = channel_link; + if (channel_id) { + body.channel_id = channel_id; } body.order_by = order_by ? order_by : 'rank' @@ -211,8 +211,8 @@ export async function download() { return await response.json(); } -export async function downloadChannel(link) { - let url = `${VIDEOS_API}/download/${link}`; +export async function downloadChannel(id) { + let url = `${VIDEOS_API}/download/${id}`; let response = await apiPost(url); let json = await response.json(); if (response.status === 400 && json['code'] === 30) { @@ -225,8 +225,8 @@ export async function downloadChannel(link) { } } -export async function refreshChannel(link) { - let url = `${VIDEOS_API}/refresh/${link}`; +export async function refreshChannel(id) { + let url = `${VIDEOS_API}/refresh/${id}`; await fetch(url, {method: 'POST'}); } diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index 4b690f80..fe1a4a87 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -154,8 +154,8 @@ class ChannelPage extends APIForm { async componentDidMount() { if (!this.props.create) { - let channel_link = this.props.match.params.channel_link; - let channel = await getChannel(channel_link); + let channel_id = this.props.match.params.channel_id; + let channel = await getChannel(channel_id); let statistics = channel.statistics; if (statistics) { this.setState({statistics: statistics || null}); @@ -173,7 +173,7 @@ class ChannelPage extends APIForm { handleConfirm = async () => { this.setState({deleteOpen: false}); - let response = await deleteChannel(this.props.match.params.channel_link); + let response = await deleteChannel(this.props.match.params.channel_id); if (response.status === 204) { this.props.history.push({ pathname: '/videos/channel' @@ -194,7 +194,7 @@ class ChannelPage extends APIForm { if (this.props.create) { response = await createChannel(inputs); } else { - response = await updateChannel(this.props.match.params.channel_link, inputs); + response = await updateChannel(this.props.match.params.channel_id, inputs); } } finally { @@ -215,7 +215,7 @@ class ChannelPage extends APIForm { this.setSuccess( 'Channel created', - Your channel was created. View it here + Your channel was created. View it here ); } else { @@ -262,12 +262,12 @@ class ChannelPage extends APIForm { downloadChannel = async (e) => { e.preventDefault(); - return await downloadChannel(this.props.match.params.channel_link); + return await downloadChannel(this.props.match.params.channel_id); } refreshChannel = async (e) => { e.preventDefault(); - return await refreshChannel(this.props.match.params.channel_link); + return await refreshChannel(this.props.match.params.channel_id); } handleDeleteButton = (e) => { @@ -470,8 +470,8 @@ export function NewChannel(props) { class ChannelRow extends React.Component { constructor(props) { super(props); - this.editTo = `/videos/channel/${props.channel.link}/edit`; - this.videosTo = `/videos/channel/${props.channel.link}/video`; + this.editTo = `/videos/channel/${props.channel.id}/edit`; + this.videosTo = `/videos/channel/${props.channel.id}/video`; } render() { @@ -646,7 +646,7 @@ export class Channels extends React.Component { {tableHeader} - {this.state.results.map((channel) => )}
@@ -655,7 +655,7 @@ export class Channels extends React.Component { {this.state.results.map((channel) => - )} + )}
diff --git a/app/src/components/Common.js b/app/src/components/Common.js index d3e8f0ef..8d46c1e4 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -135,8 +135,8 @@ export function VideoCard({video}) { let channel = video.channel ? video.channel : null; let channel_url = null; if (channel) { - channel_url = `/videos/channel/${channel.link}/video`; - video_url = `/videos/channel/${channel.link}/video/${video.id}`; + channel_url = `/videos/channel/${channel.id}/video`; + video_url = `/videos/channel/${channel.id}/video/${video.id}`; } let poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null; diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index e27be091..526ebd5d 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -76,8 +76,8 @@ function VideoPage(props) { }; - let statisticsPane = { - menuItem: 'Statistics', render: () => + let aboutPane = { + menuItem: 'About', render: () =>

Size

{video.size ? humanFileSize(video.size) : '???'}

@@ -104,7 +104,7 @@ function VideoPage(props) {
}; - let tabPanes = [descriptionPane, statisticsPane, captionsPane]; + let tabPanes = [descriptionPane, aboutPane, captionsPane]; return ( <> @@ -135,7 +135,7 @@ function VideoPage(props) {

{video.title || video.video_path}

{video.upload_date &&

{uploadDate(video.upload_date)}

}

- {channel && + {channel && {channel.name} }

diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index f068f252..7cf3b1a6 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -103,8 +103,8 @@ export class VideoWrapper extends React.Component { async fetchVideo() { // Get and display the Video specified in the Router match let [video, prev, next] = await getVideo(this.props.match.params.video_id); - let channel_link = this.props.match.params.channel_link; - let channel = channel_link ? await getChannel(channel_link) : null; + let channel_id = this.props.match.params.channel_id; + let channel = channel_id ? await getChannel(channel_id) : null; let no_channel = false; if (!channel) { no_channel = true; @@ -222,7 +222,7 @@ class Videos extends React.Component { } async componentDidMount() { - if (this.props.match.params.channel_link) { + if (this.props.match.params.channel_id) { await this.fetchChannel(); } else { await this.fetchVideos(); @@ -233,7 +233,7 @@ class Videos extends React.Component { async componentDidUpdate(prevProps, prevState, snapshot) { let params = this.props.match.params; - let channelChanged = params.channel_link !== prevProps.match.params.channel_link; + let channelChanged = params.channel_id !== prevProps.match.params.channel_id; let pageChanged = ( prevState.activePage !== this.state.activePage || prevState.searchOrder !== this.state.searchOrder || @@ -269,10 +269,10 @@ class Videos extends React.Component { async fetchChannel() { // Get and display the channel specified in the Router match - let channel_link = this.props.match.params.channel_link; + let channel_id = this.props.match.params.channel_id; let channel = null; - if (channel_link) { - channel = await getChannel(channel_link); + if (channel_id) { + channel = await getChannel(channel_id); } this.setState({ channel, @@ -300,13 +300,13 @@ class Videos extends React.Component { async fetchVideos() { this.setState({videos: null}); let offset = this.state.limit * this.state.activePage - this.state.limit; - let channel_link = this.state.channel ? this.state.channel.link : null; + let channel_id = this.state.channel ? this.state.channel.id : null; let {queryStr, searchOrder, limit} = this.state; // Pass any enabled filters to the search. let filtersArr = this.enabledFilters(); - let [videos, total] = await searchVideos(offset, limit, channel_link, queryStr, searchOrder, filtersArr); + let [videos, total] = await searchVideos(offset, limit, channel_id, queryStr, searchOrder, filtersArr); let totalPages = Math.round(total / this.state.limit) || 1; this.setState({videos, totalPages}); @@ -428,7 +428,7 @@ class Videos extends React.Component { <>     - + @@ -527,7 +527,7 @@ export function VideosRoute(props) { - } /> - + ) } diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 5a6aac98..16379707 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -609,7 +609,6 @@ def search(search_str: str, domain: str, limit: int, offset: int) -> Tuple[List[ OFFSET %(offset)s LIMIT %(limit)s ''' - print(stmt, offset, limit) curs.execute(stmt, params) results = [dict(i) for i in curs.fetchall()] total = results[0]['total'] if results else 0 diff --git a/modules/files/lib.py b/modules/files/lib.py index 4a215094..8e4d6c24 100644 --- a/modules/files/lib.py +++ b/modules/files/lib.py @@ -17,6 +17,7 @@ from wrolpi.vars import PYTEST try: import magic + have_magic = True except ImportError: logger.error('Could not import magic. Is libmagic installed?') diff --git a/modules/files/test/test_api.py b/modules/files/test/test_api.py index 2b9875c4..eb79d452 100644 --- a/modules/files/test/test_api.py +++ b/modules/files/test/test_api.py @@ -6,7 +6,7 @@ from unittest import mock import pytest from wrolpi.errors import API_ERRORS, WROLModeEnabled -from wrolpi.test.common import assert_dict_contains, skip_circleci +from wrolpi.test.common import assert_dict_contains from wrolpi.vars import PROJECT_DIR @@ -128,8 +128,6 @@ def do_search(test_client, search_str, total, expected): assert file['path'] == file['key'] -# TODO this fails on CircleCI. The order is wrong. Why? -@skip_circleci def test_files_search(test_session, test_client, make_files_structure): # You can search an empty directory. do_search(test_client, 'nothing', 0, []) @@ -154,8 +152,8 @@ def test_files_search(test_session, test_client, make_files_structure): do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)]) do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)]) do_search(test_client, 'baz', 2, [ - dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736), dict(path='baz.mp4', mimetype='video/mp4', size=1055736), + dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736), ]) do_search(test_client, 'two', 1, [dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736)]) do_search(test_client, 'nothing', 0, []) diff --git a/modules/videos/api.py b/modules/videos/api.py index a07a6366..6e41c581 100644 --- a/modules/videos/api.py +++ b/modules/videos/api.py @@ -1,6 +1,7 @@ import asyncio from functools import wraps from http import HTTPStatus +from typing import List from sanic import Blueprint, response from sanic.request import Request @@ -32,12 +33,12 @@ refresh_queue, refresh_event = create_websocket_feed('refresh', '/feeds/refresh' @content_bp.post('/refresh') -@content_bp.post('/refresh/') +@content_bp.post('/refresh/') @openapi.description('Search for videos that have previously been downloaded and stored.') @openapi.response(HTTPStatus.OK, schema.StreamResponse) @openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) @wrol_mode_check -async def refresh(_, link: str = None): +async def refresh(_, channel_id: int = None): refresh_logger = logger.getChild('refresh') stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/refresh') @@ -51,8 +52,8 @@ async def refresh(_, link: str = None): try: refresh_logger.info('refresh started') - channel_links = [link] if link else None - await refresh_videos(channel_links) + channel_ids = [channel_id] if channel_id else None + await refresh_videos(channel_ids) refresh_logger.info('refresh complete') except Exception: @@ -66,34 +67,19 @@ async def refresh(_, link: str = None): return response.json({'code': 'stream-started', 'stream_url': stream_url}) -download_queue, download_event = create_websocket_feed('download', '/feeds/download', content_bp) - - -@content_bp.post('/download') -@content_bp.post('/download/') -@openapi.description('Update channel catalogs, download any missing videos') +@content_bp.post('/download/') +@openapi.description('Update a channel catalog, download any missing videos') @openapi.response(HTTPStatus.OK, schema.StreamResponse) @openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) @wrol_mode_check -def download(_, link: str = None): - download_logger = logger.getChild('download') - - stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/download') - # Only one download can run at a time - if download_event.is_set(): - return response.json({'error': 'download already running', 'stream_url': stream_url}, HTTPStatus.CONFLICT) - if refresh_event.is_set(): - return response.json({'error': 'Refresh is running. Cannot download.'}) - - channel_lib.download_channel(link) - - download_logger.debug('do_download scheduled') - return response.json({'code': 'stream-started', 'stream_url': stream_url}) +def download(_, channel_id: int = None): + channel_lib.download_channel(channel_id) + return response.empty() @wraps(lib.refresh_videos) -async def refresh_videos(channel_links: list = None): - return lib.refresh_videos(channel_links=channel_links) +async def refresh_videos(channel_ids: List[int] = None): + return lib.refresh_videos(channel_ids=channel_ids) @content_bp.post('/favorite') diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py index 6e942537..1ec6adba 100644 --- a/modules/videos/channel/api.py +++ b/modules/videos/channel/api.py @@ -7,11 +7,13 @@ from sanic_ext import validate from sanic_ext.extensions.openapi import openapi from wrolpi.common import logger, wrol_mode_check, make_media_directory, \ - get_media_directory + get_media_directory, run_after from wrolpi.root_api import json_response from wrolpi.schema import JSONErrorResponse +from wrolpi.vars import PYTEST from . import lib from .. import schema +from ..lib import save_channels_config channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels') @@ -26,11 +28,11 @@ async def get_channels(_: Request): return json_response({'channels': channels}) -@channel_bp.get('/') +@channel_bp.get('/') @openapi.description('Get a Channel') @openapi.response(HTTPStatus.OK, schema.ChannelResponse) -def channel_get(_: Request, link: str): - channel = lib.get_channel(link=link) +def channel_get(_: Request, channel_id: int = None): + channel = lib.get_channel(channel_id=channel_id) channel.pop('info_json') return json_response({'channel': channel}) @@ -49,20 +51,19 @@ def channel_post(_: Request, body: schema.ChannelPostRequest): if not body.directory.is_dir() and body.mkdir: make_media_directory(body.directory) - bd_edit = body.__dict__ - bd_edit['name'] = bd_edit['name'].strip() - channel = lib.create_channel(bd_edit) # TODO don't use the dataclass as a dict. + channel = lib.create_channel(data=body, return_dict=False) # Refresh the videos asynchronously - from ..api import refresh_videos - coro = refresh_videos([channel['link']]) - asyncio.ensure_future(coro) + if not PYTEST: + from ..api import refresh_videos + coro = refresh_videos([channel.id]) + asyncio.ensure_future(coro) return response.json({'success': 'Channel created successfully'}, HTTPStatus.CREATED, - {'Location': f'/api/videos/channels/{channel["link"]}'}) + {'Location': f'/api/videos/channels/{channel.id}'}) -@channel_bp.put('/') +@channel_bp.put('/') @openapi.definition( description='Update a Channel', body=schema.ChannelPutRequest, @@ -70,17 +71,18 @@ def channel_post(_: Request, body: schema.ChannelPostRequest): @openapi.response(HTTPStatus.NO_CONTENT) @openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) @validate(schema.ChannelPutRequest) +@run_after(save_channels_config) @wrol_mode_check -def channel_update(_: Request, link: str, body: schema.ChannelPutRequest): - channel = lib.update_channel(body.__dict__, link) +def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest): + channel = lib.update_channel(data=body, channel_id=channel_id) return response.raw('', HTTPStatus.NO_CONTENT, - headers={'Location': f'/api/videos/channels/{channel.link}'}) + headers={'Location': f'/api/videos/channels/{channel.id}'}) -@channel_bp.delete('/') +@channel_bp.delete('/') @openapi.description('Delete a Channel') @openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) @wrol_mode_check -def channel_delete(_: Request, link: str): - lib.delete_channel(link) +def channel_delete(_: Request, channel_id: int): + lib.delete_channel(channel_id=channel_id) return response.raw('', HTTPStatus.NO_CONTENT) diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index 85f165d4..3b6f19bb 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -1,15 +1,15 @@ -from collections import defaultdict -from datetime import timedelta -from typing import List, Dict, Union, Generator +from pathlib import Path +from typing import List, Dict, Union from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound -from wrolpi.common import sanitize_link, run_after, get_relative_to_media_directory, make_media_directory, logger -from wrolpi.dates import today -from wrolpi.db import get_db_session, get_db_curs +from wrolpi.common import run_after, logger, \ + get_media_directory +from wrolpi.db import get_db_curs, optional_session from wrolpi.downloader import download_manager from wrolpi.errors import UnknownChannel, UnknownDirectory, APIError, ValidationError, InvalidDownload +from .. import schema from ..common import check_for_channel_conflicts from ..lib import save_channels_config from ..models import Channel @@ -25,7 +25,7 @@ async def get_minimal_channels() -> List[dict]: # Get all channels, even if they don't have videos. stmt = ''' SELECT - c.id, name, link, directory, url, download_frequency, info_date, + c.id, name, directory, url, download_frequency, info_date, (select next_download from download d where d.url=c.url) AS next_download FROM channel AS c @@ -72,148 +72,126 @@ async def get_channels_video_count() -> Dict[int, int]: COD = Union[dict, Channel] -def get_channel(source_id: str = None, link: str = None, url: str = None, return_dict: bool = True) -> COD: +@optional_session +def get_channel(session: Session, *, channel_id: int = None, source_id: str = None, url: str = None, + directory: str = None, name: str = None, return_dict: bool = True) -> COD: """ Attempt to find a Channel using the provided params. The params are in order of reliability. Raises UnknownChannel if no channel is found. """ - with get_db_session() as session: - channel: COD = None # noqa - # Try by source_id first. - if source_id: - channel = session.query(Channel).filter_by(source_id=source_id).one_or_none() - if not channel and link: - channel = session.query(Channel).filter_by(link=link).one_or_none() - if not channel and url: - channel = session.query(Channel).filter_by(url=url).one_or_none() + channel: COD = None # noqa + # Try to find the channel by the most reliable methods first. + if channel_id: + channel = session.query(Channel).filter_by(id=channel_id).one_or_none() + if not channel and source_id: + channel = session.query(Channel).filter_by(source_id=source_id).one_or_none() + if not channel and url: + channel = session.query(Channel).filter_by(url=url).one_or_none() + if not channel and directory: + directory = Path(directory) + if not directory.is_absolute(): + directory = get_media_directory() / directory + channel = session.query(Channel).filter_by(directory=directory).one_or_none() + if not channel and name: + channel = session.query(Channel).filter_by(name=name).one_or_none() - if not channel: - raise UnknownChannel(f'No channel matches {link=} {source_id=} {url=}') + if not channel: + raise UnknownChannel(f'No channel matches {channel_id=} {source_id=} {url=} {directory=}') - logger.debug(f'Found {channel=} using {source_id=} {link=} {url=}') - if return_dict: - statistics = channel.get_statistics() - channel = channel.dict() - channel['statistics'] = statistics - return channel - - -def _spread_by_frequency(channels: List[Channel]) -> Generator[Dict, None, None]: - channels_by_frequency = defaultdict(lambda: []) - for channel in channels: - channels_by_frequency[channel.download_frequency].append(channel) - - for frequency, channels in channels_by_frequency.items(): - # The seconds between each download. - chunk = frequency // len(channels) - for channel in channels: - # My position in the channel group. - index = channels.index(channel) - # My next download will be distributed by my frequency and my position. - position = chunk * index - next_download = today() + timedelta(seconds=frequency + position) - yield dict(url=channel.url, frequency=frequency, next_download=next_download) + logger.debug(f'Found {channel=} using {channel_id=} {source_id=} {url=} {directory=}') + if return_dict: + statistics = channel.get_statistics() + channel = channel.dict() + channel['statistics'] = statistics + return channel @run_after(save_channels_config) -def update_channel(data: dict, link: str) -> Channel: +@optional_session +def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel: """Update a Channel's DB record""" - with get_db_session(commit=True) as session: - try: - channel: Channel = session.query(Channel).filter_by(link=link).one() - except NoResultFound: - raise UnknownChannel() + try: + channel: Channel = session.query(Channel).filter_by(id=channel_id).one() + except NoResultFound: + raise UnknownChannel() - # Only update directory if it was empty - if data.get('directory') and not channel.directory: - try: - data['directory'] = get_relative_to_media_directory(data['directory']) - except UnknownDirectory: - if data['mkdir']: - make_media_directory(data['directory']) - data['directory'] = get_relative_to_media_directory(data['directory']) - else: - raise + # Only update directory if it was empty + if data.directory and not channel.directory: + data.directory = get_media_directory() / data.directory + if not data.directory.is_dir(): + if data.mkdir: + data.directory.mkdir() + else: + raise UnknownDirectory() - if 'directory' in data: - data['directory'] = str(data['directory']) + # Verify that the URL/Name/directory aren't taken + check_for_channel_conflicts( + session, + id_=channel.id, + url=data.url, + name=data.name, + directory=data.directory, + ) - if data.get('match_regex') in ('None', 'null'): - data['match_regex'] = None + # Apply the changes now that we've OK'd them + channel.update(data.__dict__) - data['link'] = data.get('link') or link - - # Verify that the URL/Name/Link aren't taken - check_for_channel_conflicts( - session, - id_=channel.id, - url=data.get('url'), - name=data.get('name'), - link=data['link'], - directory=data.get('directory'), - ) - - # Apply the changes now that we've OK'd them - channel.update(data) + session.commit() return channel @run_after(save_channels_config) -def create_channel(data: dict, return_dict: bool = True) -> Union[Channel, dict]: +@optional_session +def create_channel(session: Session, data: schema.ChannelPostRequest, return_dict: bool = True) -> Union[Channel, dict]: """ Create a new Channel. Check for conflicts with existing Channels. """ - with get_db_session(commit=True) as session: - link = sanitize_link(data['name']) - try: - # Verify that the URL/Name/Link aren't taken - check_for_channel_conflicts( - session, - url=data.get('url'), - name=data['name'], - link=link, - directory=str(data['directory']), - source_id=data.get('source_id'), - ) - except APIError as e: - raise ValidationError() from e - - channel = Channel( - name=data['name'], - url=data.get('url'), - match_regex=data.get('match_regex'), - link=link, - directory=str(data['directory']), - download_frequency=data.get('download_frequency'), - source_id=data.get('source_id'), + try: + # Verify that the URL/Name/directory aren't taken + check_for_channel_conflicts( + session, + url=data.url, + name=data.name, + directory=data.directory, + source_id=data.source_id, ) - session.add(channel) - session.commit() - session.flush() - session.refresh(channel) + except APIError as e: + raise ValidationError() from e - if return_dict: - return channel.dict() - else: - return channel + channel = Channel( + name=data.name, + url=data.url, + match_regex=data.match_regex, + directory=data.directory, # noqa + download_frequency=data.download_frequency, + source_id=data.source_id, + ) + session.add(channel) + session.commit() + session.flush() + session.refresh(channel) + + return channel.dict() if return_dict else channel @run_after(save_channels_config) -def delete_channel(link): - with get_db_session(commit=True) as session: - try: - channel = session.query(Channel).filter_by(link=link).one() - except NoResultFound: - raise UnknownChannel() +@optional_session +def delete_channel(session: Session, *, channel_id: int): + try: + channel = session.query(Channel).filter_by(id=channel_id).one() + except NoResultFound: + raise UnknownChannel() - channel.delete_with_videos() + channel.delete_with_videos() + + session.commit() -def download_channel(link: str): +def download_channel(id_: int): """Create a Download record for a Channel's entire catalog. Start downloading.""" - channel = get_channel(link=link, return_dict=False) + channel = get_channel(channel_id=id_, return_dict=False) session = Session.object_session(channel) download = channel.get_download() if not download: diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index 31c45fd1..f2fe566b 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -1,7 +1,5 @@ import json -import os import pathlib -import shutil import tempfile from datetime import timedelta from http import HTTPStatus @@ -11,301 +9,116 @@ import mock from modules.videos.channel.lib import delete_channel from modules.videos.common import get_no_channel_directory from modules.videos.models import Channel, Video -from modules.videos.test.common import create_channel_structure from wrolpi.dates import now from wrolpi.db import get_db_session from wrolpi.downloader import download_manager, Download -from wrolpi.errors import UnknownFile -from wrolpi.root_api import api_app -from wrolpi.test.common import wrap_test_db, TestAPI -from wrolpi.vars import PROJECT_DIR +from wrolpi.test.common import assert_dict_contains -class TestVideoAPI(TestAPI): +def test_get_channels(test_directory, channel_factory, test_client): + channel_factory() + channel_factory() + channel_factory() + request, response = test_client.get('/api/videos/channels') + assert response.status_code == HTTPStatus.OK + assert len(response.json['channels']) == 3 - @wrap_test_db - @create_channel_structure( - { - 'Foo': ['vid1.mp4'], - 'Bar': ['vid2.mp4'], - 'Baz': [], - } + +def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel): + # A regular user-defined channel + vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4' + vid1.touch() + vid2 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title2.mp4' + vid2.touch() + + # The special NO CHANNEL directory. + no_channel_path = get_no_channel_directory() + vid3 = no_channel_path / 'some video.mp4' + vid3.touch() + no_channel_subdir = no_channel_path / 'subdir' + no_channel_subdir.mkdir() + vid4 = no_channel_subdir / 'other video.mp4' + vid4.touch() + + test_client.post('/api/videos/refresh') + + videos = test_session.query(Video).order_by(Video.title).all() + assert len(videos) == 4 + + # Add some fake videos, they should be removed. + test_session.add(Video(title='vid5')) + test_session.add(Video(title='vid6', channel_id=simple_channel.id)) + test_session.commit() + + videos = test_session.query(Video).order_by(Video.title).all() + assert len(videos) == 6 + + test_client.post('/api/videos/refresh') + + videos = test_session.query(Video).order_by(Video.title).all() + assert len(videos) == 4 + + +def test_get_video(test_client, test_session, simple_channel, video_factory): + """ + Test that you get can information about a video. Test that video file can be gotten. + """ + now_ = now() + video1 = video_factory(channel_id=simple_channel.id) + video1.upload_date, video1.title = now_, 'vid1' + video2 = video_factory(channel_id=simple_channel.id) + video2.upload_date, video2.title = now_ + timedelta(seconds=1), 'vid2' + test_session.commit() + + # Test that a 404 is returned when no video exists + _, response = test_client.get('/api/videos/video/10') + assert response.status_code == HTTPStatus.NOT_FOUND, response.json + assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''} + + # Get the video info we inserted + _, response = test_client.get('/api/videos/video/1') + assert response.status_code == HTTPStatus.OK, response.json + assert_dict_contains(response.json['video'], {'title': 'vid1'}) + + # The next video is included. + assert response.json['prev'] is None + assert_dict_contains(response.json['next'], {'title': 'vid2'}) + + +def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel): + """A channel does not require a download frequency.""" + # No downloads are scheduled. + assert len(download_manager.get_downloads(test_session)) == 0 + + # Get the Channel + request, response = test_client.get('/api/videos/channels/1') + channel = response.json['channel'] + assert channel['download_frequency'] is None + + # Update the Channel with a frequency. + new_channel = dict( + directory=channel['directory'], + name=channel['name'], + url=channel['url'], + download_frequency=10, ) - def test_get_channels(self, tempdir): - request, response = api_app.test_client.get('/api/videos/channels') - assert response.status_code == HTTPStatus.OK - # Channels are sorted by name - self.assertDictContains(response.json['channels'][0], {'id': 2, 'name': 'Bar', 'video_count': 1}) - self.assertDictContains(response.json['channels'][1], {'id': 3, 'name': 'Baz', 'video_count': 0}) - self.assertDictContains(response.json['channels'][2], {'id': 1, 'name': 'Foo', 'video_count': 1}) + request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}', + content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.NO_CONTENT, response.json - @wrap_test_db - def test_channel(self): - channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name - channel_directory_name = channel_directory.split('/')[-1] - pathlib.Path(channel_directory).mkdir() + # Download is scheduled. + assert len(download_manager.get_downloads(test_session)) == 1 - new_channel = dict( - directory=channel_directory, - match_regex='asdf', - name=' Example Channel 1 ', - url='https://example.com/channel1', - ) - - # Channel doesn't exist - request, response = api_app.test_client.get('/api/videos/channels/examplechannel1') - assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}' - - # Create it - request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel)) - assert response.status_code == HTTPStatus.CREATED, response.json - location = response.headers['Location'] - request, response = api_app.test_client.get(location) - created = response.json['channel'] - self.assertIsNotNone(created) - self.assertIsNotNone(created['id']) - - # Channel name leading/trailing whitespace should be stripped - assert created['name'] == 'Example Channel 1' - - # Channel directory should be relative to the media directory - assert not pathlib.Path(created['directory']).is_absolute(), \ - f'Channel directory is absolute: {created["directory"]}' - - # Get the link that was computed - new_channel['link'] = response.json['channel']['link'] - assert new_channel['link'] - - # Can't create it again - request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel)) - assert response.status_code == HTTPStatus.BAD_REQUEST - - # Update it - new_channel['name'] = 'Example Channel 2' - new_channel['directory'] = str(new_channel['directory']) - request, response = api_app.test_client.put(location, content=json.dumps(new_channel)) - assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code - request, response = api_app.test_client.get(location) - assert response.status_code == HTTPStatus.OK - self.assertDictContains(response.json['channel'], { - 'id': 1, - 'name': 'Example Channel 2', - 'directory': channel_directory_name, - 'match_regex': 'asdf', - 'url': 'https://example.com/channel1', - }) - - # Can't update channel that doesn't exist - request, response = api_app.test_client.put('/api/videos/channels/doesnt_exist', - content=json.dumps(new_channel)) - assert response.status_code == HTTPStatus.NOT_FOUND - - # Delete the new channel - request, response = api_app.test_client.delete(location) - assert response.status_code == HTTPStatus.NO_CONTENT - - # Cant delete it again - request, response = api_app.test_client.delete(location) - assert response.status_code == HTTPStatus.NOT_FOUND - - @wrap_test_db - def test_refresh_no_channel(self): - with get_db_session() as session: - videos_path = pathlib.Path(self.tmp_dir.name) / 'videos' - videos_path.mkdir() - - # A regular user-defined channel - channel1_path = videos_path / 'channel1' - channel1_path.mkdir() - vid1 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title1.mp4' - vid1.touch() - vid2 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title2.mp4' - vid2.touch() - - channel1 = Channel(name='channel1', link='channel1', directory=channel1_path) - session.add(channel1) - session.commit() - - # The special NO CHANNEL directory. - no_channel_path = get_no_channel_directory() - vid3 = no_channel_path / 'some video.mp4' - vid3.touch() - no_channel_subdir = no_channel_path / 'subdir' - no_channel_subdir.mkdir() - vid4 = no_channel_subdir / 'other video.mp4' - vid4.touch() - - api_app.test_client.post('/api/videos/refresh') - - videos = session.query(Video).order_by(Video.title).all() - self.assertEqual(len(videos), 4) - - # Add some fake videos, they should be removed. - session.add(Video(title='vid5')) - session.add(Video(title='vid6', channel_id=channel1.id)) - session.commit() - - videos = session.query(Video).order_by(Video.title).all() - self.assertEqual(len(videos), 6) - - api_app.test_client.post('/api/videos/refresh') - - videos = session.query(Video).order_by(Video.title).all() - self.assertEqual(len(videos), 4) - - @wrap_test_db - def test_get_channel_videos(self): - with get_db_session(commit=True) as session: - channel1 = Channel(name='Foo', link='foo') - channel2 = Channel(name='Bar', link='bar') - session.add(channel1) - session.add(channel2) - session.flush() - session.refresh(channel1) - session.refresh(channel2) - - # Channels don't have videos yet - d = dict(channel_link=channel1.link) - request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d)) - assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == 0 - - with get_db_session(commit=True) as session: - vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo') - vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo') - session.add(vid1) - session.add(vid2) - - # Videos are gotten by their respective channels - request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d)) - assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == 1 - assert response.json['totals']['videos'] == 1 - self.assertDictContains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id)) - - d = dict(channel_link=channel2.link) - request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d)) - assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == 1 - self.assertDictContains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id)) - - @wrap_test_db - def test_get_video(self): - """ - Test that you get can information about a video. Test that video file can be gotten. - """ - - def raise_unknown_file(_): - raise UnknownFile() - - with get_db_session(commit=True) as session, \ - mock.patch('modules.videos.common.get_absolute_video_info_json', raise_unknown_file): - channel = Channel(name='Foo', link='foo') - session.add(channel) - session.flush() - session.refresh(channel) - now_ = now() - session.add(Video(title='vid1', channel_id=channel.id, upload_date=now_)) - session.add(Video(title='vid2', channel_id=channel.id, upload_date=now_ + timedelta(seconds=1))) - - # Test that a 404 is returned when no video exists - _, response = api_app.test_client.get('/api/videos/video/10') - assert response.status_code == HTTPStatus.NOT_FOUND, response.json - assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''} - - # Get the video info we inserted - _, response = api_app.test_client.get('/api/videos/video/1') - assert response.status_code == HTTPStatus.OK, response.json - self.assertDictContains(response.json['video'], {'title': 'vid1'}) - - # The next video is included. - self.assertIsNone(response.json['prev']) - self.assertDictContains(response.json['next'], {'title': 'vid2'}) - - @wrap_test_db - def test_get_channel_videos_pagination(self): - with get_db_session(commit=True) as session: - channel1 = Channel(name='Foo', link='foo') - session.add(channel1) - session.flush() - session.refresh(channel1) - - for i in range(50): - session.add(Video(title=f'Foo.Video{i}', channel_id=channel1.id, video_path='foo')) - - channel2 = Channel(name='Bar', link='bar') - session.add(channel2) - session.flush() - session.refresh(channel2) - session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo')) - - # Get first, second, third, and empty pages of videos. - tests = [ - # (offset, video_count) - (0, 20), - (20, 20), - (40, 10), - (50, 0), - ] - last_ids = [] - for offset, video_count in tests: - d = dict(channel_link=channel1.link, order_by='id', offset=offset) - _, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d)) - assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == video_count - current_ids = [i['id'] for i in response.json['videos']] - assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}' - last_ids = current_ids - - @wrap_test_db - def test_channel_no_download_frequency(self): - """A channel does not require a download frequency.""" - channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name - os.mkdir(channel_directory) - - new_channel = dict( - directory=channel_directory, - name='Example Channel 1', - url='https://example.com/channel1', - download_frequency=None, - ) - - # Create the Channel - request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel)) - assert response.status_code == HTTPStatus.CREATED - location = response.headers['Location'] - - # No downloads are scheduled. - with get_db_session() as session: - downloads = list(download_manager.get_new_downloads(session)) - self.assertEqual(len(downloads), 0) - - # Get the Channel - request, response = api_app.test_client.get(location) - channel = response.json['channel'] - self.assertEqual(channel['download_frequency'], None) - - # Update the Channel with a frequency. - new_channel = dict( - directory=channel['directory'], - name=channel['name'], - url=channel['url'], - download_frequency=10, - ) - request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}', - content=json.dumps(new_channel)) - self.assertEqual(response.status_code, 204, response.json) - - # Remove the frequency. - new_channel = dict( - directory=channel['directory'], - name=channel['name'], - url=channel['url'], - download_frequency=None, - ) - request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}', - content=json.dumps(new_channel)) - self.assertEqual(response.status_code, 204, response.json) + # Remove the frequency. + new_channel = dict( + directory=channel['directory'], + name=channel['name'], + url=channel['url'], + download_frequency=None, + ) + request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}', + content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.NO_CONTENT, response.json def test_channel_frequency_update(download_channel, test_client, test_session): @@ -321,7 +134,7 @@ def test_channel_frequency_update(download_channel, test_client, test_session): url=download_channel.url, download_frequency=100, ) - request, response = test_client.put(f'/api/videos/channels/{download_channel.link}', json=data) + request, response = test_client.put(f'/api/videos/channels/{download_channel.id}', json=data) assert response.status_code == HTTPStatus.NO_CONTENT, response.json download = download_channel.get_download() @@ -332,20 +145,59 @@ def test_channel_frequency_update(download_channel, test_client, test_session): assert len(list(downloads)) == 1 -def test_delete_channel_delete_download(download_channel, test_session): - """ - Deleting a Channel deletes it's Download. - """ - downloads = test_session.query(Download).all() - assert all(i.url == download_channel.url for i in downloads) +def test_channel_download_crud(test_session, simple_channel): + """Modifying a Channel modifies it's Download.""" + assert simple_channel.url + assert simple_channel.download_frequency is None - delete_channel(download_channel.link) + simple_channel.update({'download_frequency': 1000}) + test_session.commit() + + download: Download = test_session.query(Download).one() + assert simple_channel.get_download() == download + assert simple_channel.url == download.url + assert simple_channel.download_frequency == download.frequency + + # Increase the frequency to 1 second. + simple_channel.update({'download_frequency': 1}) + test_session.commit() + assert simple_channel.url == download.url + assert simple_channel.download_frequency == download.frequency == 1 + assert test_session.query(Download).count() == 1 + + # Changing the Channel's URL changes the download's URL. + simple_channel.update({'url': 'https://example.com/new url'}) + test_session.commit() + assert simple_channel.url == 'https://example.com/new url' + assert simple_channel.download_frequency == download.frequency == 1 + assert test_session.query(Download).count() == 1 + + # Removing the Channel's frequency removes the download. + simple_channel.update({'download_frequency': None}) + test_session.commit() + assert test_session.query(Download).count() == 0 + + # Removing the URL is supported. + simple_channel.update({'url': None}) + test_session.commit() + + # Adding a URL is supported. + simple_channel.update({'url': 'https://example.com'}) + test_session.commit() + + +def test_delete_channel_delete_download(download_channel, test_session): + """Deleting a Channel deletes it's Download.""" + downloads = test_session.query(Download).all() + assert downloads and all(i.url == download_channel.url for i in downloads) + + delete_channel(channel_id=download_channel.id) downloads = test_session.query(Download).all() assert not list(downloads) -def test_video_search(test_session): +def test_video_search(test_client, test_session): """ Test that videos can be searched and that their order is by their textsearch rank. """ @@ -363,7 +215,7 @@ def test_video_search(test_session): def do_search(search_str, limit=20): d = json.dumps({'search_str': search_str, 'limit': limit}) - _, resp = api_app.test_client.post('/api/videos/search', content=d) + _, resp = test_client.post('/api/videos/search', content=d) return resp def search_is_as_expected(resp, expected): @@ -414,7 +266,7 @@ def test_video_file_name(test_session, simple_video, test_client): assert resp.json['video'].get('stem') == 'simple_video' -def test_channel_conflicts(test_session, test_directory): +def test_channel_conflicts(test_client, test_session, test_directory): channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name pathlib.Path(channel_directory).mkdir() new_channel = dict( @@ -425,7 +277,7 @@ def test_channel_conflicts(test_session, test_directory): ) def _post_channel(channel): - return api_app.test_client.post('/api/videos/channels', content=json.dumps(channel)) + return test_client.post('/api/videos/channels', content=json.dumps(channel)) # Create it request, response = _post_channel(new_channel) @@ -443,18 +295,6 @@ def test_channel_conflicts(test_session, test_directory): assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10, 'cause': {'error': 'The channel name is already taken.', 'code': 5}} - # Name matches when converted to link - channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name - pathlib.Path(channel_directory2).mkdir() - new_channel = dict( - directory=channel_directory2, - name='Example channel 1', - ) - request, response = _post_channel(new_channel) - assert response.status_code == HTTPStatus.BAD_REQUEST - assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10, - 'cause': {'code': 11, 'error': 'Channel link already used by another channel'}} - # Directory was already used new_channel = dict( directory=channel_directory, @@ -479,7 +319,7 @@ def test_channel_conflicts(test_session, test_directory): 'cause': {'code': 6, 'error': 'The URL is already used by another channel.'}} -def test_channel_empty_url_doesnt_conflict(test_session, test_directory): +def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_directory): """Two channels with empty URLs shouldn't conflict""" channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name pathlib.Path(channel_directory).mkdir() @@ -488,7 +328,7 @@ def test_channel_empty_url_doesnt_conflict(test_session, test_directory): 'name': 'Fooz', 'directory': channel_directory, } - request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel)) + request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel)) assert response.status_code == HTTPStatus.CREATED, response.json location = response.headers['Location'] @@ -498,7 +338,7 @@ def test_channel_empty_url_doesnt_conflict(test_session, test_directory): 'name': 'Barz', 'directory': channel_directory2, } - request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel)) + request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel)) assert response.status_code == HTTPStatus.CREATED, response.json assert location != response.headers['Location'] @@ -524,11 +364,11 @@ def test_download_channel_no_refresh(test_session, download_channel, video_downl def test_channel_post_directory(test_session, test_client, test_directory): """A Channel can be created with or without an existing directory.""" - # Channel can be created with a missing directory. + # Channel can be created with a directory which is not on disk. data = dict(name='foo', directory='foo') request, response = test_client.post('/api/videos/channels', content=json.dumps(data)) assert response.status_code == HTTPStatus.CREATED - directory = test_session.query(Channel).filter_by(link='foo').one().directory.path + directory = test_session.query(Channel).filter_by(id=1).one().directory.path assert (test_directory / 'foo') == directory assert not directory.is_dir() assert directory.is_absolute() @@ -537,56 +377,147 @@ def test_channel_post_directory(test_session, test_client, test_directory): data = dict(name='bar', directory='bar', mkdir=True) request, response = test_client.post('/api/videos/channels', content=json.dumps(data)) assert response.status_code == HTTPStatus.CREATED - directory = test_session.query(Channel).filter_by(link='bar').one().directory.path + directory = test_session.query(Channel).filter_by(id=2).one().directory.path assert (test_directory / 'bar') == directory assert directory.is_dir() assert directory.is_absolute() -def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory): - subdir = test_directory / 'subdir' - subdir.mkdir() +def test_channel_by_id(test_session, test_client, simple_channel, simple_video): + request, response = test_client.get(f'/api/videos/channels/{simple_channel.id}') + assert response.status_code == HTTPStatus.OK - # video1 is in a subdirectory. - video1 = video_factory(simple_channel.id, True, True, 'jpg') - test_session.commit() - shutil.move(video1.video_path.path, subdir / video1.video_path.path.name) - shutil.move(video1.poster_path.path, subdir / video1.poster_path.path.name) - video1.video_path = subdir / video1.video_path.path.name - video1.poster_path = None - # video2 is in the test directory. - video2 = video_factory(simple_channel.id, True, True, 'jpg') - video2.poster_path = video1.poster_path = None + +def test_channel_crud(test_session, test_client, test_directory): + channel_directory = test_directory / 'channel directory' + channel_directory.mkdir() + + new_channel = dict( + directory=str(channel_directory), + match_regex='asdf', + name=' Example Channel 1 ', + url='https://example.com/channel1', + ) + + # Channel doesn't exist + request, response = test_client.get('/api/videos/channels/examplechannel1') + assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}' + + # Create it + request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.CREATED, response.json + location = response.headers['Location'] + + request, response = test_client.get(location) + assert response.status_code == HTTPStatus.OK, response.json + created = response.json['channel'] + assert created + assert created['id'] + # No frequency was provided. No download exists. + assert not created['download_frequency'] + assert test_session.query(Download).count() == 0 + + # Channel name leading/trailing whitespace should be stripped + assert created['name'] == 'Example Channel 1' + + # Channel directory should be relative to the media directory + assert not pathlib.Path(created['directory']).is_absolute(), \ + f'Channel directory is absolute: {created["directory"]}' + + # Can't create it again + request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.BAD_REQUEST + + # Update it + new_channel['name'] = 'Example Channel 2' + new_channel['directory'] = str(new_channel['directory']) # noqa + request, response = test_client.put(location, content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code + request, response = test_client.get(location) + assert response.status_code == HTTPStatus.OK + channel = response.json['channel'] + assert channel['id'] == 1 + assert channel['name'] == 'Example Channel 2' + assert channel['directory'] == channel_directory.name + assert channel['match_regex'] == 'asdf' + assert channel['url'] == 'https://example.com/channel1' + + # Can't update channel that doesn't exist + request, response = test_client.put('/api/videos/channels/doesnt_exist', + content=json.dumps(new_channel)) + assert response.status_code == HTTPStatus.NOT_FOUND + + # Delete the new channel + request, response = test_client.delete(location) + assert response.status_code == HTTPStatus.NO_CONTENT + + # Cant delete it again + request, response = test_client.delete(location) + assert response.status_code == HTTPStatus.NOT_FOUND + + +def test_search_channel_videos(test_client, test_session): + with get_db_session(commit=True) as session: + channel1 = Channel(name='Foo') + channel2 = Channel(name='Bar') + session.add(channel1) + session.add(channel2) + session.flush() + session.refresh(channel1) + session.refresh(channel2) + + # Channels don't have videos yet + d = dict(channel_id=channel1.id) + request, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) + assert response.status_code == HTTPStatus.OK + assert len(response.json['videos']) == 0 + + with get_db_session(commit=True) as session: + vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo') + vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo') + session.add(vid1) + session.add(vid2) + + # Videos are gotten by their respective channels + request, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) + assert response.status_code == HTTPStatus.OK + assert len(response.json['videos']) == 1 + assert response.json['totals']['videos'] == 1 + assert_dict_contains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id)) + + d = dict(channel_id=channel2.id) + request, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) + assert response.status_code == HTTPStatus.OK + assert len(response.json['videos']) == 1 + assert_dict_contains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id)) + + +def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory): + for i in range(50): + video_factory(channel_id=simple_channel.id) + + channel2 = Channel(name='Bar') + test_session.add(channel2) + test_session.flush() + test_session.refresh(channel2) + video_factory(channel_id=channel2.id) + test_session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo')) test_session.commit() - assert not video1.size, 'video1 should not have size during creation' - - # Create a video not in the DB. - vid3 = test_directory / 'vid3.mp4' - shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', vid3) - - # Orphan poster should be ignored. - orphan_poster = pathlib.Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg') - orphan_poster.touch() - - # Create a bogus video in the channel. - bogus = Video(video_path='foo', channel_id=simple_channel.id) - test_session.add(bogus) - - test_client.post('/api/videos/refresh') - - # Posters were found during refresh. - assert video1.poster_path - assert 'subdir' in str(video1.poster_path.path) - assert video2.poster_path - # Missing video3 was found - video3: Video = test_session.query(Video).filter_by(id=4).one() - assert video3.video_path.path == vid3 - assert video3.video_path != video1.video_path - assert video3.video_path != video2.video_path - # Bogus video was removed. - assert not any('foo' in str(i.video_path.path) for i in test_session.query(Video).all()) - # Orphan file was not deleted. - assert orphan_poster.is_file(), 'Orphan poster was removed!' - - assert video1.size, 'video1 size was not found' + # Get first, second, third, and empty pages of videos. + tests = [ + # (offset, video_count) + (0, 20), + (20, 20), + (40, 10), + (50, 0), + ] + last_ids = [] + for offset, video_count in tests: + d = dict(channel_id=simple_channel.id, order_by='id', offset=offset) + _, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) + assert response.status_code == HTTPStatus.OK + assert len(response.json['videos']) == video_count + current_ids = [i['id'] for i in response.json['videos']] + assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}' + last_ids = current_ids diff --git a/modules/videos/channel/test/test_lib.py b/modules/videos/channel/test/test_lib.py new file mode 100644 index 00000000..1e8bc510 --- /dev/null +++ b/modules/videos/channel/test/test_lib.py @@ -0,0 +1,59 @@ +import pytest + +from modules.videos.channel import lib +from wrolpi.errors import UnknownChannel + + +@pytest.mark.parametrize('params', [ + dict(), + dict(channel_id=None), + dict(source_id=None), + dict(url=None), + dict(directory=None), +]) +def test_get_bad_channel(test_session, test_directory, params): + """A Channel cannot be found without data.""" + with pytest.raises(UnknownChannel): + lib.get_channel(**params) + + +def test_get_channel(test_session, test_directory, channel_factory): + """`get_channel` can find a Channel in multiple ways.""" + channel_id_channel = channel_factory() + + source_id_channel = channel_factory() + source_id_channel.source_id = 'the source id' + + url_channel = channel_factory() + url_channel.url = 'https://example.com/channel' + + directory_channel = channel_factory() + directory_channel.directory = test_directory / 'some directory' + + test_session.commit() + + channel = lib.get_channel(channel_id=channel_id_channel.id, return_dict=False) + assert channel == channel_id_channel + + channel = lib.get_channel(source_id='the source id', return_dict=False) + assert channel == source_id_channel + + channel = lib.get_channel(url='https://example.com/channel', return_dict=False) + assert channel == url_channel + + # Directory can be relative, or absolute. + channel = lib.get_channel(directory='some directory', return_dict=False) + assert channel == directory_channel + channel = lib.get_channel(directory=f'{test_directory}/some directory', return_dict=False) + assert channel == directory_channel + + # Channel is found using the priority of params, incorrect data is ignored. + params = [ + dict(channel_id=channel_id_channel.id, directory='bad directory'), + dict(channel_id=channel_id_channel.id, directory='some directory'), + dict(channel_id=channel_id_channel.id, source_id='bad source id'), + dict(channel_id=channel_id_channel.id, source_id='the source id'), + ] + for p in params: + assert lib.get_channel(**p, return_dict=False) == channel_id_channel + assert lib.get_channel(**p) diff --git a/modules/videos/common.py b/modules/videos/common.py index 64842eba..4008b77d 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -10,16 +10,13 @@ from typing import Union, Tuple, List, Set, Iterable, Optional import PIL from PIL import Image from sqlalchemy.orm import Session -from yt_dlp import YoutubeDL -from wrolpi import before_startup from wrolpi.cmd import which from wrolpi.common import logger, iterify, get_media_directory, \ - minimize_dict, api_param_limiter + minimize_dict, any_extensions from wrolpi.db import get_db_session, get_db_curs -from wrolpi.downloader import download_manager -from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \ - ChannelLinkConflict, ChannelDirectoryConflict, ChannelSourceIdConflict +from wrolpi.errors import UnknownFile, ChannelNameConflict, ChannelURLConflict, \ + ChannelDirectoryConflict, ChannelSourceIdConflict from wrolpi.media_path import MediaPath from wrolpi.vars import DEFAULT_FILE_PERMISSIONS from .models import Channel, Video @@ -27,27 +24,21 @@ from .models import Channel, Video logger = logger.getChild(__name__) REQUIRED_OPTIONS = ['name', 'directory'] -MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count', 'link'} +MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count'} MINIMUM_INFO_JSON_KEYS = {'description', 'view_count', 'webpage_url'} MINIMUM_VIDEO_KEYS = {'id', 'title', 'upload_date', 'duration', 'channel', 'channel_id', 'favorite', 'size', 'poster_path', 'caption_path', 'video_path', 'info_json', 'channel', 'viewed', 'source_id', 'view_count'} -VIDEO_QUERY_LIMIT = 20 -VIDEO_QUERY_MAX_LIMIT = 100 # These are the supported video formats. These are in order of their preference. VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv') -video_limiter = api_param_limiter(100) - class ConfigError(Exception): pass def get_videos_directory() -> pathlib.Path: - """ - Get the "videos" directory in the media directory. Make it if it does not exist. - """ + """Get the "videos" directory in the media directory. Make it if it does not exist.""" directory = get_media_directory() / 'videos' if not directory.is_dir(): directory.mkdir(parents=True) @@ -55,89 +46,13 @@ def get_videos_directory() -> pathlib.Path: def get_no_channel_directory() -> pathlib.Path: - """ - Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist. - """ + """Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist.""" directory = get_videos_directory() / 'NO CHANNEL' if not directory.is_dir(): directory.mkdir(parents=True) return directory -@before_startup -def import_videos_config(): - """Import channel settings to the DB. Existing channels will be updated.""" - logger.info('Importing videos config') - try: - from .lib import get_channels_config - from .downloader import ChannelDownloader - config = get_channels_config() - config, favorites = config.channels, config.favorites - - with get_db_session(commit=True) as session: - for link in config: - for option in (i for i in REQUIRED_OPTIONS if i not in config[link]): - raise ConfigError(f'Channel "{link}" is required to have "{option}"') - - name, directory = config[link]['name'], config[link]['directory'] - - channel = session.query(Channel).filter(Channel.link == link).one_or_none() - if not channel: - # Channel not yet in the DB, add it. - channel = Channel(link=link) - - # Only name and directory are required - channel.name = name - channel.directory = str(directory) - - channel.calculate_duration = config[link].get('calculate_duration') or channel.calculate_duration - channel.download_frequency = config[link].get('download_frequency') or channel.download_frequency - channel.generate_posters = config[link].get('generate_posters') or channel.generate_posters - channel.match_regex = config[link].get('match_regex') or channel.match_regex - channel.skip_download_videos = list(set(config[link].get('skip_download_videos', {}))) - channel.source_id = config[link].get('source_id') or channel.source_id - channel.url = config[link].get('url') or channel.url - - if not channel.source_id and channel.url: - # If we can download from a channel, we must have its source_id. - channel.source_id = get_channel_source_id(channel.url) - - if channel.download_frequency and channel.url: - download = download_manager.get_or_create_download(channel.url, session=session) - download.frequency = channel.download_frequency - download.downloader = ChannelDownloader.name - download.next_download = download.next_download or \ - download_manager.get_next_download(download, session=session) - - session.add(channel) - - with get_db_session(commit=True) as session: - for link, favorites in favorites.items(): - if link != 'NO CHANNEL': - channel: Channel = session.query(Channel).filter_by(link=link).one_or_none() - if not channel: - logger.warning(f'Cannot find channel {link=} for favorites!') - continue - channel_dir = channel.directory.path - else: - from .downloader import get_no_channel_directory - channel_dir = get_no_channel_directory() - - # Set favorite Videos of this Channel. - for video_path, data in favorites.items(): - # Favorite in the config is the name of the video_path. Add the channel directory onto this - # video_path, so we can match the complete path for the Video. - video_path = channel_dir / video_path - video = session.query(Video).filter_by(video_path=video_path).one_or_none() - # If no Video is found, it may be that we need to refresh. - if video: - video.favorite = data['favorite'] - else: - logger.warning(f'Cannot find video to favorite: {video_path}') - except Exception as e: - logger.warning('Failed to load channels config!', exc_info=e) - - VALID_VIDEO_KINDS = {'video', 'caption', 'poster', 'description', 'info_json'} @@ -150,66 +65,11 @@ def get_absolute_video_path(video: Video, kind: str = 'video') -> MediaPath: raise UnknownFile() -get_absolute_video_caption = partial(get_absolute_video_path, kind='caption') -get_absolute_video_poster = partial(get_absolute_video_path, kind='poster') -get_absolute_video_description = partial(get_absolute_video_path, kind='description') -get_absolute_video_info_json = partial(get_absolute_video_path, kind='info_json') - - -def get_absolute_video_files(video: Video) -> List[MediaPath]: - """ - Get all video files that exist. - """ - getters = [ - get_absolute_video_description, - get_absolute_video_caption, - get_absolute_video_poster, - get_absolute_video_info_json, - get_absolute_video_path, - ] - - def _get(): - for getter in getters: - try: - yield getter(video) - except UnknownFile: - pass - - return list(_get()) - - -def get_video_info_json(video) -> Optional[dict]: - """Get the info_json object from a video's meta-file. Return an empty dict if not possible.""" - if not video.channel or not video.channel.directory: - return - - try: - path = get_absolute_video_info_json(video) - if path.exists(): - with open(str(path), 'rb') as fh: - contents = json.load(fh) - return contents - except UnknownFile: - return - except UnknownDirectory: - return - - -def any_extensions(filename: str, extensions: Iterable = ()): - """ - Return True only if the file name ends with any of the possible extensions. - Matches lower or upper case of the extension. - """ - return any(filename.lower().endswith(ext) for ext in extensions) - - match_video_extensions = partial(any_extensions, extensions=VIDEO_EXTENSIONS) def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, pathlib.Path]: - """ - Generate a list of video paths in the provided directory. - """ + """Generate a list of video paths in the provided directory.""" directory = pathlib.Path(directory) for child in directory.iterdir(): @@ -223,9 +83,11 @@ def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, path @iterify(set) def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]: """ - Remove any duplicate paths from a given list. Duplicate is defined as any file that shares the EXACT same - name as another file, but with a different extension. If a duplicate is found, only yield one of the paths. The - path that will be yielded will be whichever is first in the VIDEO_EXTESIONS tuple. + Remove any duplicate paths from a given list. + + Duplicate is defined as any file that shares the EXACT same name as another file, but with a different extension. + If a duplicate is found, only yield one of the paths. The path that will be yielded will be whichever is first in + the VIDEO_EXTESIONS tuple. i.e. >>> remove_duplicate_video_paths([Path('one.mp4'), Path('two.mp4'), Path('one.ogg')]) @@ -262,15 +124,15 @@ def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]: yield sorted(paths)[0] -def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, link=None, directory=None, +def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, directory=None, source_id=None): """ Search for any channels that conflict with the provided args, raise a relevant exception if any conflicts are found. """ - if not any([id_, url, name, link, directory]): + if not any([id_, url, name, directory]): raise Exception('Cannot search for channel with no arguments') - logger.debug(f'Checking for channel conflicts: id={id_} url={url} name={name} link={link} directory={directory}') + logger.debug(f'Checking for channel conflicts: id={id_} url={url} name={name} directory={directory}') # A channel can't conflict with itself if id_: @@ -286,10 +148,6 @@ def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, conflicts = base_where.filter(Channel.name == name) if list(conflicts): raise ChannelNameConflict() - if link: - conflicts = base_where.filter(Channel.link == link) - if list(conflicts): - raise ChannelLinkConflict() if directory: conflicts = base_where.filter(Channel.directory == directory) if list(conflicts): @@ -329,29 +187,13 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]: return paths -def replace_extension(path: pathlib.Path, new_ext) -> pathlib.Path: - """Swap the extension of a file's path. - - Example: - >>> foo = pathlib.Path('foo.bar') - >>> replace_extension(foo, 'baz') - 'foo.baz' - """ - parent = path.parent - existing_ext = path.suffix - path = str(path) - name, _, _ = path.rpartition(existing_ext) - path = pathlib.Path(str(parent / name) + new_ext) - return path - - def generate_video_poster(video_path: Path) -> Path: """ Create a poster (aka thumbnail) next to the provided video_path. """ - poster_path = replace_extension(video_path, '.jpg') - cmd = ['/usr/bin/ffmpeg', '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000', - str(poster_path)] + poster_path = video_path.with_suffix('.jpg') + cmd = ('/usr/bin/ffmpeg', '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000', + str(poster_path)) try: subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) logger.info(f'Generated poster at {poster_path}') @@ -474,13 +316,3 @@ def minimize_video(video: dict) -> dict: video['info_json'] = minimize_video_info_json(video['info_json']) return video - - -YDL = YoutubeDL() -YDL.params['logger'] = logger.getChild('youtube-dl') -YDL.add_default_info_extractors() - - -def get_channel_source_id(url: str) -> str: - channel_info = YDL.extract_info(url, download=False, process=False) - return channel_info.get('uploader_id') or channel_info['id'] diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 336deb7f..915f7329 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -10,22 +10,18 @@ from PIL import Image from modules.videos.downloader import VideoDownloader, ChannelDownloader from modules.videos.lib import set_test_channels_config, set_test_downloader_config from modules.videos.models import Channel, Video -from wrolpi.common import sanitize_link from wrolpi.downloader import DownloadFrequency, DownloadManager, Download from wrolpi.vars import PROJECT_DIR @pytest.fixture def simple_channel(test_session, test_directory) -> Channel: - """ - Get a Channel with the minimum properties. This Channel has no download! - """ + """Get a Channel with the minimum properties. This Channel has no download!""" channel = Channel( directory=test_directory, name='Simple Channel', url='https://example.com/channel1', download_frequency=None, # noqa - link='simplechannel', ) test_session.add(channel) test_session.commit() @@ -34,20 +30,18 @@ def simple_channel(test_session, test_directory) -> Channel: @pytest.fixture def channel_factory(test_session, test_directory): - """ - Create a random Channel with a directory, but no frequency. - """ + """Create a random Channel with a directory, but no frequency.""" - def _(): - name = str(uuid4()) + def _(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None): + name = name or str(uuid4()) directory = test_directory / name directory.mkdir() channel = Channel( directory=directory, # noqa name=name, - url=f'https://example.com/{name}', - download_frequency=None, # noqa - link=sanitize_link(name), + url=url or f'https://example.com/{name}', + download_frequency=download_frequency, + source_id=source_id, ) test_session.add(channel) test_session.commit() @@ -67,7 +61,6 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch name='Download Channel', url='https://example.com/channel1', download_frequency=DownloadFrequency.weekly, - link='downloadchannel', ) download = Download(url=channel.url, downloader='video_channel', frequency=channel.download_frequency) test_session.add_all([channel, download]) @@ -77,9 +70,7 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch @pytest.fixture def simple_video(test_session, test_directory, simple_channel) -> Video: - """ - A Video with a video file whose channel is the Simple Channel. - """ + """A Video with an empty video file whose channel is the Simple Channel.""" video_path = test_directory / 'simple_video.mp4' video_path.touch() video = Video(video_path=video_path, channel_id=simple_channel.id) @@ -103,9 +94,9 @@ def video_file(test_directory) -> pathlib.Path: def video_factory(test_session, test_directory): """Creates Videos for testing.""" - def _(channel_id: int = None, with_video_file: bool = False, with_info_json: dict = None, - with_poster_ext: str = None): - title = str(uuid4()) + def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file: bool = False, + with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False): + title = title or str(uuid4()) if channel_id: path = test_directory / f'{title}.mp4' else: @@ -129,12 +120,18 @@ def video_factory(test_session, test_directory): poster_path = path.with_suffix(f'.{with_poster_ext}') Image.new('RGB', (25, 25)).save(poster_path) + caption_path = None + if with_caption_file: + caption_path = path.with_suffix('.en.vtt') + caption_path.touch() + video = Video(video_path=str(path), title=title, channel_id=channel_id, source_id=title, - info_json_path=info_json_path, poster_path=poster_path) + info_json_path=info_json_path, poster_path=poster_path, caption_path=caption_path, + upload_date=upload_date) test_session.add(video) return video - return _ + return factory @pytest.fixture diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 4aad0c56..e9050b46 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -6,23 +6,23 @@ from abc import ABC from typing import Tuple, List, Optional from sqlalchemy.orm import Session -from sqlalchemy.orm.exc import NoResultFound from yt_dlp import YoutubeDL from yt_dlp.extractor import YoutubeTabIE from yt_dlp.utils import UnsupportedError, DownloadError from wrolpi.cmd import which -from wrolpi.common import logger, sanitize_link, extract_domain +from wrolpi.common import logger, extract_domain, get_media_directory from wrolpi.dates import now from wrolpi.db import get_db_session, get_db_curs from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel -from .common import apply_info_json, get_channel_source_id, get_no_channel_directory, \ +from .common import apply_info_json, get_no_channel_directory, \ get_videos_directory -from .lib import upsert_video, refresh_channel_videos, get_downloader_config +from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id from .models import Video, Channel +from .schema import ChannelPostRequest from .video_url_resolver import video_url_resolver logger = logger.getChild(__name__) @@ -70,12 +70,12 @@ class ChannelDownloader(Downloader, ABC): if not channel: # Couldn't get channel by URL, this is probably a playlist. Find the channel for the playlist. channel_source_id = get_channel_source_id(download.url) - channel = get_channel(channel_source_id, return_dict=False) + channel = get_channel(source_id=channel_source_id, return_dict=False) update_channel(channel) domain = extract_domain(download.url) - missing_videos = find_all_missing_videos(channel.link) + missing_videos = find_all_missing_videos(channel.id) with get_db_session(commit=True) as session: for video_id, source_id, missing_video in missing_videos: url = video_url_resolver(domain, missing_video) @@ -152,7 +152,7 @@ class VideoDownloader(Downloader, ABC): channel = None if channel_name or channel_id: channel_url = info.get('channel_url') - channel = get_or_create_channel(channel_id, None, channel_url, channel_name) + channel = get_or_create_channel(source_id=channel_id, url=channel_url, name=channel_name) # Use the default directory if this video has no channel. out_dir = get_no_channel_directory() @@ -245,34 +245,32 @@ channel_downloader = ChannelDownloader() video_downloader = VideoDownloader(40) -def get_or_create_channel(source_id: str = None, link: str = None, url: str = None, name: str = None) -> Channel: +def get_or_create_channel(source_id: str = None, url: str = None, name: str = None) -> Channel: """ Attempt to find a Channel using the provided params. The params are in order of reliability. Creates a new Channel if one cannot be found. """ - if not link and name: - link = sanitize_link(name) - try: - channel = get_channel(link=link, source_id=source_id, url=url, return_dict=False) + channel = get_channel(source_id=source_id, url=url, name=name, return_dict=False) return channel except UnknownChannel: pass + if not name: + raise ValueError(f'Cannot create channel without a name') + # Channel does not exist. Create one in the video directory. channel_directory = get_videos_directory() / name if not channel_directory.is_dir(): channel_directory.mkdir(parents=True) - data = dict( + data = ChannelPostRequest( + source_id=source_id, name=name, url=url, - link=link, - domain=extract_domain(url), - directory=str(channel_directory), - source_id=source_id, + directory=str(channel_directory.relative_to(get_media_directory())), ) - channel = create_channel(data, return_dict=False) + channel = create_channel(data=data, return_dict=False) # Create the directory now that the channel is approved. channel_directory.mkdir(exist_ok=True) @@ -348,21 +346,17 @@ def update_channel(channel: Channel): apply_info_json(channel_id) -def _find_all_missing_videos(link: str = None) -> List[Tuple]: +def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]: """Get all Video entries which don't have the required media files (i.e. hasn't been downloaded). - Restrict to a single channel if "link" is provided. + Restrict to a single channel if `channel_id` is provided. """ with get_db_curs() as curs: # Get all channels by default. where = '' params = () - if link: - # Restrict by channel when link is provided. - query = 'SELECT id FROM channel WHERE link = %s' - curs.execute(query, (link,)) - channel_id = curs.fetchall()[0][0] + if channel_id: where = 'AND channel_id = %s' params = (channel_id,) @@ -386,18 +380,15 @@ def _find_all_missing_videos(link: str = None) -> List[Tuple]: return missing_videos -def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]: +def find_all_missing_videos(channel_id: str = None) -> Tuple[dict, dict]: """ Find all videos that don't have a video file, but are found in the DB (taken from the channel's info_json). Yields a Channel Dict object, our Video id, and the "entry" of the video from the channel's info_json['entries']. """ with get_db_session() as session: - if link: - try: - channel = session.query(Channel).filter_by(link=link).one() - except NoResultFound: - raise UnknownChannel(f'No channel with link: {link}') + if channel_id: + channel = get_channel(channel_id=channel_id, return_dict=False) if not channel.url: raise ChannelURLEmpty('No URL for this channel') channels = [channel, ] @@ -420,12 +411,12 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]: # Convert the channel video entries into a form that allows them to be quickly retrieved without searching through # the entire entries list. channels_entries = {} - for id_, channel in channels.items(): - channels_entries[id_] = {i['id']: i for i in channel.info_json['entries']} + for channel_id_, channel in channels.items(): + channels_entries[channel_id_] = {i['id']: i for i in channel.info_json['entries']} - missing_videos = _find_all_missing_videos(link) + missing_videos = _find_all_missing_videos(channel_id) - for id_, source_id, channel_id in missing_videos: + for video_id, source_id, channel_id in missing_videos: channel = channels[channel_id] if channel.skip_download_videos and source_id in channel.skip_download_videos: @@ -435,13 +426,13 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]: try: missing_video = channels_entries[channel_id][source_id] except KeyError: - logger.warning(f'Video {id_} / {source_id} is not in {channel.name} info_json') + logger.warning(f'Video {channel.name} / {source_id} is not in {channel.name} info_json') continue match_regex: re.compile = match_regexen.get(channel_id) if not match_regex or (match_regex and missing_video['title'] and match_regex.match(missing_video['title'])): # No title match regex, or the title matches the regex. - yield id_, source_id, missing_video + yield video_id, source_id, missing_video UNRECOVERABLE_ERRORS = { diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 5ee00572..54e81003 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -2,20 +2,23 @@ import html import pathlib import re from collections import defaultdict -from typing import Tuple, Optional, List +from typing import Tuple, Optional, List, Union from uuid import uuid1 +from sqlalchemy import or_ from sqlalchemy.exc import InvalidRequestError from sqlalchemy.orm import Session +from yt_dlp import YoutubeDL -from wrolpi.common import logger, chunks, ConfigFile +from wrolpi import before_startup +from wrolpi.common import chunks, ConfigFile, get_media_directory, sanitize_link from wrolpi.dates import from_timestamp, Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.media_path import MediaPath from wrolpi.vars import PYTEST from .captions import get_captions -from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, import_videos_config, \ - get_video_duration, is_valid_poster, convert_image, generate_video_poster +from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \ + is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError from .models import Channel, Video logger = logger.getChild(__name__) @@ -66,8 +69,9 @@ def refresh_channel_videos(channel: Channel): logger.info(f'{channel.name}: {len(new_videos)} new videos, {len(existing_paths)} already existed. ') - channel.refreshed = True - Session.object_session(channel).commit() + with get_db_session(commit=True) as session: + channel = session.query(Channel).filter_by(id=channel.id).one() + channel.refreshed = True apply_info_json(channel.id) @@ -155,7 +159,7 @@ def validate_videos(): video.validated = True except Exception as e: # This video failed to validate, continue validation for the rest of the videos. - logger.warning(f'Failed to validate {video=}', exc_info=e) + logger.warning(f'Failed to validate {video}', exc_info=e) def validate_video(video: Video, channel_generate_poster: bool): @@ -246,7 +250,7 @@ def convert_or_generate_poster(video: Video) -> Optional[pathlib.Path]: logger.error(f'Failed to generate poster for {video}', exc_info=e) -def refresh_videos(channel_links: list = None): +def refresh_videos(channel_ids: List[int] = None): """ Find any videos in the channel directories and add them to the DB. Delete DB records of any videos not in the file system. @@ -257,15 +261,15 @@ def refresh_videos(channel_links: list = None): """ logger.info('Refreshing video files') with get_db_session() as session: - if channel_links: - channels = session.query(Channel).filter(Channel.link.in_(channel_links)) + if channel_ids: + channels = session.query(Channel).filter(Channel.id.in_(channel_ids)) else: channels = session.query(Channel).all() channels = list(channels) - if not channels and channel_links: - raise Exception(f'No channels match links(s): {channel_links}') + if not channels and channel_ids: + raise Exception(f'No channels match id(s): {channel_ids}') elif not channels: raise Exception(f'No channels in DB. Have you created any?') @@ -276,13 +280,13 @@ def refresh_videos(channel_links: list = None): logger.fatal(f'Failed to refresh videos for channel {channel.name}!', exc_info=e) pass - if not channel_links: + if not channel_ids: # Refresh NO CHANNEL videos when not refreshing a specific channel. refresh_no_channel_videos() # Fill in any missing data for all videos. if not PYTEST: - import_videos_config() + import_channels_config() validate_videos() @@ -460,8 +464,8 @@ def set_test_downloader_config(enabled: bool): def get_channels_config_from_db(session: Session) -> dict: """Create a dictionary that contains all the Channels from the DB.""" - channels = session.query(Channel).order_by(Channel.link).all() - channels = {i.link: i.config_view() for i in channels} + channels = session.query(Channel).order_by(Channel.directory).all() + channels = sorted((i.config_view() for i in channels), key=lambda i: i['directory']) # Get all Videos that are favorites. Store them in their own config section, so they can be preserved if a channel # is deleted or the DB is wiped. @@ -469,7 +473,7 @@ def get_channels_config_from_db(session: Session) -> dict: favorites = defaultdict(lambda: {}) for video in favorite_videos: if video.channel: - favorites[video.channel.link][video.video_path.path.name] = dict(favorite=video.favorite) + favorites[str(video.channel.directory.relative)][video.video_path.path.name] = dict(favorite=video.favorite) else: favorites['NO CHANNEL'][video.video_path.path.name] = dict(favorite=video.favorite) favorites = dict(favorites) @@ -477,18 +481,128 @@ def get_channels_config_from_db(session: Session) -> dict: return dict(channels=channels, favorites=favorites) +def _detect_old_favorites(config: Union[ChannelsConfig, dict]) -> bool: + # TODO remove these old favorites after beta. + media_directory = get_media_directory() + favorites = config.favorites if hasattr(config, 'favorites') else config['favorites'] + for channel, favorites in favorites.items(): + if channel == 'NO CHANNEL': + continue + if not (media_directory / channel).is_dir(): + return True + return False + + @optional_session() def save_channels_config(session=None, preserve_favorites: bool = True): - """ - Pull the Channel information from the DB, save it to the config. - """ + """Get the Channel information from the DB, save it to the config.""" config = get_channels_config_from_db(session) channels_config = get_channels_config() + # TODO remove these old favorites after beta. + old_favorites = _detect_old_favorites(channels_config) + if old_favorites and config['favorites']: + # There are favorites in the database, remove the old favorites. + channels_config.favorites = {} if preserve_favorites: config['favorites'].update(channels_config.favorites or {}) channels_config.update(config) +@before_startup +def import_channels_config(): + """Import channel settings to the DB. Existing channels will be updated.""" + logger.info('Importing videos config') + try: + from .downloader import ChannelDownloader + config = get_channels_config() + channels, favorites = config.channels, config.favorites + + with get_db_session(commit=True) as session: + for data in channels: + if isinstance(data, str): + # Outdated style config. + # TODO remove this after beta. + data = channels[data] + # A Channel's directory is saved (in config) relative to the media directory. + directory = get_media_directory() / data['directory'] + for option in (i for i in REQUIRED_OPTIONS if i not in data): + raise ConfigError(f'Channel "{directory}" is required to have "{option}"') + + channel = session.query(Channel).filter( + or_( + Channel.directory == str(directory), + Channel.directory == str(data['directory']), + Channel.directory == str(directory.relative_to(get_media_directory())), + ), + ).one_or_none() + if not channel: + # Channel not yet in the DB, add it. + channel = Channel(directory=directory) + session.add(channel) + + # Only name and directory are required + channel.name = data['name'] + channel.directory = directory + + # Copy existing channel data, update all values from the config. This is necessary to clear out + # values not in the config. + full_data = channel.dict() + full_data.update(data) + print(f'{full_data=}') + full_data['skip_download_videos'] = list(set(data.get('skip_download_videos', {}))) + channel.update(full_data) + + if not channel.source_id and channel.url: + # If we can download from a channel, we must have its source_id. + channel.source_id = get_channel_source_id(channel.url) + + with get_db_session(commit=True) as session: + channels_by_link = {sanitize_link(i.name): i for i in session.query(Channel)} + for directory_, favorites in favorites.items(): + if directory_ != 'NO CHANNEL': + # A Channel's directory is saved (in the config) relative to the media directory. + directory = get_media_directory() / directory_ + channel: Channel = session.query(Channel).filter_by(directory=directory).one_or_none() + if not channel: + # Directory may be the outdated "link". + # TODO remove these old favorites after beta. + channel = channels_by_link.get(directory_) + if not channel: + logger.warning(f'Cannot find channel {directory=} for favorites!') + continue + channel_dir = channel.directory.path + else: + from .downloader import get_no_channel_directory + channel_dir = get_no_channel_directory() + + # Set favorite Videos of this Channel. + for video_path, data in favorites.items(): + # Favorite in the config is the name of the video_path. Add the channel directory onto this + # video_path, so we can match the complete path for the Video. + video_path = channel_dir / video_path + video = session.query(Video).filter_by(video_path=video_path).one_or_none() + # If no Video is found, it may be that we need to refresh. + if video: + video.favorite = data['favorite'] + else: + logger.warning(f'Cannot find video to favorite: {video_path}') + except Exception as e: + logger.warning('Failed to load channels config!', exc_info=e) + if PYTEST: + # Do not interrupt startup, only raise during testing. + raise + + +YDL = YoutubeDL() +YDL.params['logger'] = logger.getChild('youtube-dl') +YDL.add_default_info_extractors() + + +def get_channel_source_id(url: str) -> str: + channel_info = YDL.extract_info(url, download=False, process=False) + return channel_info.get('uploader_id') or channel_info['id'] + + async def get_statistics(): with get_db_curs() as curs: curs.execute(''' @@ -608,7 +722,6 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = video.info_json_path = info_json_path video.poster_path = poster_path video.video_path = video_path - if channel and not str(video_path).startswith(str(channel.directory.path)): raise ValueError(f'Video path is not within its channel {video_path=} not in {channel.directory=}') diff --git a/modules/videos/models.py b/modules/videos/models.py index 91156226..0b8af617 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -1,5 +1,4 @@ import json -import pathlib from datetime import datetime from pathlib import Path from typing import Optional, Generator @@ -12,7 +11,7 @@ from wrolpi.common import Base, tsvector, ModelHelper, logger, get_media_directo from wrolpi.dates import now, TZDateTime from wrolpi.db import get_db_curs from wrolpi.downloader import Download, download_manager -from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory, InvalidDownload +from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory from wrolpi.media_path import MediaPathType, MediaPath logger = logger.getChild(__name__) @@ -116,9 +115,7 @@ class Video(ModelHelper, Base): save_channels_config(preserve_favorites=False) def add_to_skip_list(self): - """ - Add this video to it's Channel's skip list. - """ + """Add this video to it's Channel's skip list.""" if self.channel and self.source_id: self.channel.add_video_to_skip_list(self.source_id) @@ -130,18 +127,16 @@ class Video(ModelHelper, Base): self.viewed = now() def get_info_json(self) -> Optional[JSON]: - """ - If this Video has an info_json file, return it's contents. Otherwise, return None. - """ + """If this Video has an info_json file, return it's contents. Otherwise, return None.""" + if not self.info_json_path: + return + try: - if isinstance(self.info_json_path, pathlib.Path): - with open(self.info_json_path, 'rb') as fh: - contents = json.load(fh) - return contents - elif self.info_json_path: - with open(self.info_json_path.path, 'rb') as fh: - contents = json.load(fh) - return contents + info_json_path = self.info_json_path.path if \ + isinstance(self.info_json_path, MediaPath) else self.info_json_path + with open(info_json_path, 'rb') as fh: + contents = json.load(fh) + return contents except UnknownFile: pass except UnknownDirectory: @@ -265,9 +260,8 @@ class Channel(ModelHelper, Base): __tablename__ = 'channel' id = Column(Integer, primary_key=True) name = Column(String) - link = Column(String, nullable=False) idempotency = Column(String) - url = Column(String) + url = Column(String, unique=True) match_regex = Column(String) directory: MediaPath = Column(MediaPathType) skip_download_videos = Column(ARRAY(String)) @@ -307,18 +301,28 @@ class Channel(ModelHelper, Base): def update(self, data: dict): """ - Update the values of this Channel. Will also update the Channel's Download, if it has one. + Update the attributes of this Channel. Will also update the Channel's Download, if it has one. """ + # Get the download before we change the URL. + download = self.get_download() + for key, value in data.items(): setattr(self, key, value) - if not self.url: - return - download = self.get_download() - if download: - session = Session.object_session(self) + # All channels with a URL and download_frequency should have a download. + session = Session.object_session(self) + if download and not self.download_frequency: + download_manager.delete_download(download.id, session) + elif download and self.download_frequency: download.frequency = self.download_frequency + download.url = self.url download.next_download = download_manager.get_next_download(download, session) + elif not download and self.download_frequency and self.url: + download = Download(frequency=self.download_frequency, url=self.url, downloader='video_channel') + session.add(download) + session.flush() + download.next_download = download_manager.get_next_download(download, session) + session.flush() def config_view(self) -> dict: """ @@ -349,7 +353,7 @@ class Channel(ModelHelper, Base): Get the Download row for this Channel. If there isn't a Download, return None. """ if not self.url: - raise InvalidDownload(f'Channel {self.name} does not have a URL to download!') + return None session = Session.object_session(self) download = session.query(Download).filter_by(url=self.url).one_or_none() @@ -361,10 +365,15 @@ class Channel(ModelHelper, Base): name=self.name, directory=self.directory, url=self.url, - link=self.link, ) return d + def dict(self, with_statistics: bool = False): + d = super(Channel, self).dict() + if with_statistics: + d['statistics'] = self.get_statistics() + return d + def get_statistics(self): """ Get statistics about this channel. diff --git a/modules/videos/schema.py b/modules/videos/schema.py index 38440038..360dd842 100644 --- a/modules/videos/schema.py +++ b/modules/videos/schema.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field from datetime import date, datetime from typing import List, Optional -from modules.videos.lib import DEFAULT_DOWNLOAD_FREQUENCY from modules.videos.video.lib import DEFAULT_VIDEO_ORDER, VIDEO_QUERY_LIMIT @@ -11,25 +10,37 @@ class ChannelPostRequest: name: str directory: str calculate_duration: Optional[bool] = None - download_frequency: Optional[int] = DEFAULT_DOWNLOAD_FREQUENCY + download_frequency: Optional[int] = None generate_posters: Optional[bool] = None match_regex: Optional[str] = None mkdir: Optional[bool] = None url: Optional[str] = None + source_id: Optional[str] = None + + def __post_init__(self): + self.name = self.name.strip() or None + self.directory = self.directory.strip() or None + self.url = self.url.strip() if self.url else None + self.source_id = self.source_id.strip() if self.source_id else None @dataclass class ChannelPutRequest: calculate_duration: Optional[bool] = None directory: Optional[str] = None - download_frequency: Optional[int] = DEFAULT_DOWNLOAD_FREQUENCY + download_frequency: Optional[int] = None generate_posters: Optional[bool] = None - link: Optional[str] = None match_regex: Optional[str] = None mkdir: Optional[bool] = None name: Optional[str] = None url: Optional[str] = None + def __post_init__(self): + self.name = self.name.strip() or None + self.directory = self.directory.strip() or None + self.url = self.url.strip() if self.url else None + self.match_regex = None if self.match_regex in ('None', '') else self.match_regex + @dataclass class RefreshRequest: @@ -42,7 +53,6 @@ class ChannelModel: url: str name: str match_regex: str - link: str directory: str @@ -60,7 +70,6 @@ class ChannelPostResponse: class ChannelsChannelModel: id: int name: str - link: str @dataclass @@ -132,7 +141,7 @@ class VideoSearchRequest: offset: Optional[int] = None limit: Optional[int] = VIDEO_QUERY_LIMIT order_by: Optional[str] = DEFAULT_VIDEO_ORDER - channel_link: Optional[str] = None + channel_id: Optional[int] = None @dataclass diff --git a/modules/videos/test/common.py b/modules/videos/test/common.py deleted file mode 100644 index 81f6af90..00000000 --- a/modules/videos/test/common.py +++ /dev/null @@ -1,75 +0,0 @@ -import pathlib -import tempfile -from functools import wraps - -from modules.videos.lib import refresh_channel_videos -from modules.videos.models import Channel -from wrolpi.common import insert_parameter, set_test_media_directory -from wrolpi.db import get_db_session -from wrolpi.test.common import wrap_test_db, build_test_directories - - -def create_channel_structure(structure): - """ - Create a directory containing the specified structure of channels and videos. Create DB entries for these - channels and videos. - - Example: - >>> s = {'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4', 'vid2.mp4', 'vid2.en.vtt']} - >>> create_channel_structure(s) - - Creates directories like so: - channel1/vid1.mp4 - channel2/vid1.mp4 - channel2/vid2.mp4 - channel2/vid2.en.vtt - - Channels like so: - Channel(name='channel1', directory='channel1') - Channel(name='channel2', directory='channel2') - - And, Videos like so: - Video(channel_id=1, video_path='vid1.mp4') - Video(channel_id=2, video_path='vid1.mp4') - Video(channel_id=2, video_path='vid2.mp4', caption_path='vid2.en.vtt') - """ - - def wrapper(func): - @wraps(func) - @wrap_test_db - def wrapped(*args, **kwargs): - if args and (test := args[0]) and hasattr(test, 'tmp_dir'): - tmp_dir = pathlib.Path(test.tmp_dir.name) - else: - tmp_dir = pathlib.Path(tempfile.TemporaryDirectory().name) - set_test_media_directory(tmp_dir) - - # Convert the channel/video structure to a file structure for the test. - file_structure = [] - for channel, paths in structure.items(): - for path in paths: - file_structure.append(f'{channel}/{path}') - file_structure.append(f'{channel}/') - - with build_test_directories(file_structure, tmp_dir) as tempdir: - args, kwargs = insert_parameter(func, 'tempdir', tempdir, args, kwargs) - - with get_db_session(commit=True) as session: - for channel in structure: - channel_ = Channel(directory=str(tempdir / channel), name=channel, link=channel) - session.add(channel_) - session.flush() - session.refresh(channel_) - - with get_db_session(commit=True) as session: - for channel_ in session.query(Channel).all(): - refresh_channel_videos(channel_) - - try: - return func(*args, **kwargs) - finally: - set_test_media_directory(None) - - return wrapped - - return wrapper diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py new file mode 100644 index 00000000..3fb24242 --- /dev/null +++ b/modules/videos/test/test_api.py @@ -0,0 +1,72 @@ +import shutil +from http import HTTPStatus +from pathlib import Path + +from modules.videos.models import Video +from wrolpi.downloader import DownloadFrequency +from wrolpi.vars import PROJECT_DIR + + +def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory): + subdir = test_directory / 'subdir' + subdir.mkdir() + + # video1 is in a subdirectory. + video1 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True, + with_poster_ext='jpg') + test_session.commit() + shutil.move(video1.video_path.path, subdir / video1.video_path.path.name) + shutil.move(video1.poster_path.path, subdir / video1.poster_path.path.name) + video1.video_path = subdir / video1.video_path.path.name + video1.poster_path = None + # video2 is in the test directory. + video2 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True, + with_poster_ext='jpg') + video2.poster_path = video1.poster_path = None + test_session.commit() + + assert not video1.size, 'video1 should not have size during creation' + + # Create a video not in the DB. + vid3 = test_directory / 'vid3.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', vid3) + + # Orphan poster should be ignored. + orphan_poster = Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg') + orphan_poster.touch() + + # Create a bogus video in the channel. + bogus = Video(video_path='foo', channel_id=simple_channel.id) + test_session.add(bogus) + + test_client.post('/api/videos/refresh') + + # Posters were found during refresh. + assert video1.poster_path + assert 'subdir' in str(video1.poster_path.path) + assert video2.poster_path + # Missing video3 was found + video3: Video = test_session.query(Video).filter_by(id=4).one() + assert video3.video_path.path == vid3 + assert video3.video_path != video1.video_path + assert video3.video_path != video2.video_path + # Bogus video was removed. + assert not any('foo' in str(i.video_path.path) for i in test_session.query(Video).all()) + # Orphan file was not deleted. + assert orphan_poster.is_file(), 'Orphan poster was removed!' + + assert video1.size, 'video1 size was not found' + + +def test_api_download_channel(test_session, test_client, simple_channel): + """A Channel download (a catalog update) can be triggered via the API.""" + # simple_channel does not have a download record. + request, response = test_client.post(f'/api/videos/download/{simple_channel.id}') + assert response.status_code == HTTPStatus.BAD_REQUEST, response.json + assert 'not have a download' in response.json['message'] + + # Add a download frequency to the channel, this should also create a download. + simple_channel.update(dict(download_frequency=DownloadFrequency.daily)) + test_session.commit() + request, response = test_client.post(f'/api/videos/download/{simple_channel.id}') + assert response.status_code == HTTPStatus.NO_CONTENT diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 1930e02f..1905cf7e 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -4,23 +4,19 @@ import tempfile from datetime import datetime from typing import List from unittest import mock -from unittest.mock import MagicMock import pytest from PIL import Image from modules.videos.models import Channel, Video -from modules.videos.test.common import create_channel_structure -from wrolpi.common import get_absolute_media_path -from wrolpi.dates import local_timezone -from wrolpi.db import get_db_session +from wrolpi.common import get_absolute_media_path, sanitize_link +from wrolpi.dates import local_timezone, now from wrolpi.downloader import Download, DownloadFrequency -from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI +from wrolpi.test.common import build_test_directories, TestAPI from wrolpi.vars import PROJECT_DIR from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \ - apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \ - import_videos_config -from ..lib import save_channels_config, get_channels_config + apply_info_json, get_video_duration, generate_video_poster, is_valid_poster +from ..lib import save_channels_config, get_channels_config, import_channels_config class TestCommon(TestAPI): @@ -134,113 +130,50 @@ def test_is_valid_poster(ext, expected, test_directory): assert is_valid_poster(image_path) == expected, f'is_valid_poster({image_path}) should be {expected}' -@pytest.mark.parametrize( - '_structure,paths', - ( - ( - {'channel1': ['vid1.mp4']}, - [ - 'channel1/vid1.mp4', - ], - ), - ( - {'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4']}, - [ - 'channel1/vid1.mp4', - 'channel2/vid1.mp4', - ], - ), - ( - {'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4', 'vid2.mp4', 'vid2.en.vtt']}, - [ - 'channel1/vid1.mp4', - 'channel2/vid1.mp4', - 'channel2/vid2.mp4', - 'channel2/vid2.en.vtt', - ], - ), - ) -) -def test_create_db_structure(_structure, paths): - @create_channel_structure(_structure) - def test_func(_, tempdir): - assert isinstance(tempdir, pathlib.Path) - for path in paths: - path = (tempdir / path) - assert path.exists() - assert path.is_file() - - with get_db_session() as session: - for channel_name in _structure: - channel = session.query(Channel).filter_by(name=channel_name).one() - assert (tempdir / channel_name).is_dir() - assert channel - assert channel.directory == tempdir / channel_name - assert len(channel.videos) == len([i for i in _structure[channel_name] if i.endswith('mp4')]) - - with tempfile.TemporaryDirectory() as tmp_dir, \ - mock.patch('wrolpi.media_path.get_media_directory') as mock_get_directory: - mock_get_directory.return_value = pathlib.Path(tmp_dir) - test_ = MagicMock() - test_.tmp_dir.name = tmp_dir - test_func(test_) - - -@wrap_test_db -@create_channel_structure( - { - 'channel1': ['vid1.mp4', 'vid1.jpg'], - 'channel2': ['vid2.mp4'], - 'channel3': ['vid3.mp4', 'vid4.mp4'], - } -) -def test_update_view_count(tempdir: pathlib.Path): +def test_update_view_count(test_session, channel_factory, video_factory): def check_view_counts(view_counts): - with get_db_session() as session_: - for source_id, view_count in view_counts.items(): - vid = session_.query(Video).filter_by(source_id=source_id).one() - assert vid.view_count == view_count + for source_id, view_count in view_counts.items(): + video = test_session.query(Video).filter_by(source_id=source_id).one() + assert video.view_count == view_count - with get_db_session(commit=True) as session: - channel1, channel2, channel3 = session.query(Channel).order_by(Channel.id).all() - channel1.info_json = {'entries': [{'id': 'vid1.mp4', 'view_count': 10}]} - channel2.info_json = {'entries': [{'id': 'vid2.mp4', 'view_count': 11}, {'id': 'bad_id', 'view_count': 12}]} - channel3.info_json = {'entries': [{'id': 'vid3.mp4', 'view_count': 13}, {'id': 'vid4.mp4', 'view_count': 14}]} - - # Use the video file name as a unique source_id - for v in session.query(Video).all(): - v.source_id = str(v.video_path.path).split('/')[-1] + channel1, channel2, channel3 = channel_factory(), channel_factory(), channel_factory() + video_factory(channel_id=channel1.id, with_poster_ext='jpg', title='vid1') + video_factory(channel_id=channel2.id, title='vid2') + video_factory(channel_id=channel3.id, with_poster_ext='jpg', title='vid3') + video_factory(channel_id=channel3.id, title='vid4') + channel1.info_json = {'entries': [{'id': 'vid1', 'view_count': 10}]} + channel2.info_json = {'entries': [{'id': 'vid2', 'view_count': 11}, {'id': 'bad_id', 'view_count': 12}]} + channel3.info_json = {'entries': [{'id': 'vid3', 'view_count': 13}, {'id': 'vid4', 'view_count': 14}]} + test_session.commit() # Check all videos are empty. - check_view_counts({'vid1.mp4': None, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None}) + check_view_counts({'vid1': None, 'vid2': None, 'vid3': None, 'vid4': None}) # Channel 1 is updated, the other channels are left alone. apply_info_json(channel1.id) - check_view_counts({'vid1.mp4': 10, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None}) + check_view_counts({'vid1': 10, 'vid2': None, 'vid3': None, 'vid4': None}) # Channel 2 is updated, the other channels are left alone. The 'bad_id' video is ignored. apply_info_json(channel2.id) - check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': None, 'vid4.mp4': None}) + check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': None, 'vid4': None}) # All videos are updated. apply_info_json(channel3.id) - check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14}) + check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14}) # An outdated view count will be overwritten. - with get_db_session(commit=True) as session: - vid = session.query(Video).filter_by(id=1).one() - vid.view_count = 8 - check_view_counts({'vid1.mp4': 8, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14}) + vid = test_session.query(Video).filter_by(id=1).one() + vid.view_count = 8 + check_view_counts({'vid1': 8, 'vid2': 11, 'vid3': 13, 'vid4': 14}) apply_info_json(channel1.id) - check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14}) + check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14}) def test_generate_video_poster(video_file): """ A poster can be generated from a video file. """ - poster_path = replace_extension(video_file, '.jpg') - assert not poster_path.is_file(), f'{poster_path} already exists!' + poster_path = video_file.with_suffix('.jpg') generate_video_poster(video_file) assert poster_path.is_file(), f'{poster_path} was not created!' assert poster_path.stat().st_size > 0 @@ -326,7 +259,7 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan vid2_video_path = vid2.video_path.path favorites = { - simple_channel.link: {vid3.video_path.path.name: {'favorite': favorite}}, + str(simple_channel.directory.relative): {vid3.video_path.path.name: {'favorite': favorite}}, 'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}, } @@ -336,8 +269,8 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan assert config.favorites == favorites def import_and_verify(favorited_ids: List[int]): - with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'): - import_videos_config() + with mock.patch('modules.videos.lib.get_channel_source_id', lambda i: 'foo'): + import_channels_config() for video in test_session.query(Video).all(): if video.id in favorited_ids: assert video.favorite @@ -371,10 +304,8 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan def test_import_channel_downloads(test_session, channel_factory, test_channels_config): """Importing the Channels' config should create any missing download records""" - channel1 = channel_factory() - channel2 = channel_factory() - channel1.source_id = 'foo' - channel2.source_id = 'bar' + channel1 = channel_factory(source_id='foo') + channel2 = channel_factory(source_id='bar') # channel2 has no url, but has a frequency. Import should not create a download record. channel2.url = None channel2.download_frequency = DownloadFrequency.biweekly @@ -383,20 +314,25 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c assert len(test_session.query(Channel).all()) == 2 assert test_session.query(Download).all() == [] + def update_channel_config(conf, source_id, d): + for c in conf.channels: + if c['source_id'] == source_id: + c.update(d) + conf.save() + # Config has no channels with a download_frequency. save_channels_config() - import_videos_config() + import_channels_config() assert channel1.download_frequency is None assert len(test_session.query(Channel).all()) == 2 assert test_session.query(Download).all() == [] # Add a frequency to the Channel. channels_config = get_channels_config() - channels_config.channels[channel1.link]['download_frequency'] = DownloadFrequency.biweekly - channels_config.save() + update_channel_config(channels_config, 'foo', {'download_frequency': DownloadFrequency.biweekly}) # Download record is created on import. - import_videos_config() + import_channels_config() assert channel1.download_frequency is not None assert len(test_session.query(Channel).all()) == 2 download: Download = test_session.query(Download).one() @@ -404,9 +340,8 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c assert download.frequency == channel1.download_frequency # Download frequency is adjusted when config file changes. - channels_config.channels[channel1.link]['download_frequency'] = DownloadFrequency.weekly - channels_config.save() - import_videos_config() + update_channel_config(channels_config, 'foo', {'download_frequency': DownloadFrequency.weekly}) + import_channels_config() assert len(test_session.query(Channel).all()) == 2 assert download.url == channel1.url assert download.frequency == channel1.download_frequency @@ -414,6 +349,67 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c assert download.next_download next_download = str(download.next_download) - import_videos_config() + import_channels_config() download: Download = test_session.query(Download).one() assert next_download == str(download.next_download) + + +def test_import_channels_config_outdated(test_session, test_directory, channel_factory, test_channels_config, + video_factory): + """The Channels' config used to use a "link" to differentiate channels, test that old configs can be imported.""" + channel1 = channel_factory(name='Channel1', download_frequency=DownloadFrequency.weekly) + channel2 = channel_factory(name='Channel2', download_frequency=DownloadFrequency.weekly) + test_session.commit() + + vid1 = video_factory(title='vid1', channel_id=channel1.id) + vid2 = video_factory(title='vid2', channel_id=channel2.id) + vid3 = video_factory(title='vid3') + vid4 = video_factory(title='vid4') + test_session.commit() + + channel1_name, channel_1_directory = channel1.name, channel1.directory.path + channel2_name, channel_2_directory = channel2.name, channel2.directory.path + + assert channel1.directory.path.exists() and channel1.directory.path.is_absolute() + assert channel2.directory.path.exists() and channel2.directory.path.is_absolute() + + # Channel's used to have a relative directory in the DB. + test_session.execute('UPDATE channel SET directory=:directory WHERE id=2', + dict(directory=str(channel_2_directory.relative_to(test_directory))), + ) + + save_channels_config() + + # The config has a list of Channels, use the old method of a "dict" with the sanitized name as the key. + channels_config = get_channels_config() + channels_config.channels = {sanitize_link(i['name']): i for i in channels_config.channels} + # Change frequency to verify that the channels are updated. + for link, channel in channels_config.channels.items(): + channel['download_frequency'] = DownloadFrequency.biweekly + # Change the favorites to the old "link" method as well. + channels_config.favorites = { + sanitize_link(channel1.name): {str(vid1.video_path.path): {'favorite': now()}}, + sanitize_link(channel2.name): {str(vid2.video_path.path): {'favorite': now()}}, + 'NO CHANNEL': {str(vid3.video_path.path): {'favorite': now()}}, + } + channels_config.save() + + assert not vid1.favorite and not vid2.favorite and not vid3.favorite and not vid3.favorite + + assert test_session.query(Channel).count() == 2 + + with mock.patch('modules.videos.lib.get_channel_source_id') as mock_get_channel_source_id: + mock_get_channel_source_id.return_value = 'some source id' + import_channels_config() + + # No new channels were created. Existing channels were updated. + assert test_session.query(Channel).count() == 2 + channel1, channel2 = test_session.query(Channel).order_by(Channel.id) + assert channel1.name == channel1_name and str(channel1.directory.path) == str(channel_1_directory) + # Channel2's directory is now absolute. + assert channel2.name == channel2_name and str(channel2.directory.path) == str(channel_2_directory) + assert channel1.download_frequency == DownloadFrequency.biweekly + assert channel2.download_frequency == DownloadFrequency.biweekly + # Videos were favorited by their channel "link". + assert vid1.favorite and vid2.favorite and vid3.favorite + assert not vid4.favorite diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index d15b3033..19a50957 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -1,4 +1,5 @@ import pathlib +import shutil from copy import copy from unittest import mock @@ -9,12 +10,12 @@ from modules.videos.channel.lib import download_channel from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \ ChannelDownloader, get_or_create_channel from modules.videos.models import Channel, Video -from modules.videos.test.common import create_channel_structure -from wrolpi.db import get_db_session, get_db_context +from wrolpi.db import get_db_context from wrolpi.downloader import DownloadManager, Download from wrolpi.errors import InvalidDownload from wrolpi.test.common import wrap_test_db, TestAPI from wrolpi.test.test_downloader import HTTPDownloader +from wrolpi.vars import PROJECT_DIR class FakeYDL: @@ -28,37 +29,32 @@ class FakeYDL: } -class TestDownloader(TestAPI): - @wrap_test_db - @create_channel_structure( - { - 'channel1': ['vid1.mp4'], - 'channel2': ['vid2.mp4'] - } - ) - def test_find_all_missing_videos(self, tempdir): - with get_db_session(commit=True) as session: - channel1, channel2 = session.query(Channel).order_by(Channel.id).all() - channel1.url = channel2.url = 'some url' - channel1.info_json = {'entries': [{'id': 'foo'}]} +def test_find_all_missing_videos(test_session, channel_factory, video_factory): + channel1, channel2 = channel_factory(url='some url'), channel_factory(url='some other url') + channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]} + test_session.commit() - # No missing videos - self.assertEqual([], list(find_all_missing_videos())) + # Two videos are already downloaded. + video_factory() + video_factory() - # Create a video that has no video file. - with get_db_session(commit=True) as session: - video = Video(title='needs to be downloaded', channel_id=channel1.id, source_id='foo') - session.add(video) + # No missing videos + assert [] == list(find_all_missing_videos()) - missing = list(find_all_missing_videos()) - self.assertEqual(len(missing), 1) + # Create a video that has no video file. + video = Video(title='needs to be downloaded', channel_id=channel1.id, source_id='foo') + test_session.add(video) + test_session.commit() - # The video returned is the one we faked. - id_, source_id, entry = missing[0] - # Two videos were created for this test already. - self.assertEqual(id_, 3) - # The fake entry we added is regurgitated back. - self.assertEqual(entry, channel1.info_json['entries'][0]) + missing = list(find_all_missing_videos()) + assert len(missing) == 1 + + # The video returned is the one we faked. + id_, source_id, entry = missing[0] + # Two videos were created for this test already. + assert id_ == 3 + # The fake entry we added is regurgitated back. + assert entry == channel1.info_json['entries'][0] example_video_json = { @@ -171,43 +167,6 @@ class TestVideosDownloaders(TestAPI): info_json_path.touch() return video_path, poster_path, description_path, info_json_path - @wrap_test_db - def test_video_download(self): - channel_dir = self.videos_dir / 'channel name' - channel_dir.mkdir(parents=True) - - # Video paths - video_path, poster_path, description_path, info_json_path = self._make_video_files(channel_dir) - - with get_db_session() as session: - url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ - mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ - mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: - mock_extract_info.return_value = example_video_json - mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) - mock_process_runner.return_value = 0 - - self.mgr.create_download(url, session) - - mock_process_runner.assert_called_once() - video_url, _, out_dir = mock_process_runner.call_args[0] - - download: Download = session.query(Download).one() - self.assertEqual(video_url, download.url) - channel: Channel = session.query(Channel).one() - self.assertEqual(channel.directory, channel_dir) - - video: Video = session.query(Video).one() - self.assertEqual(video.video_path, video_path) - self.assertEqual(video.description_path, description_path) - self.assertEqual(video.poster_path, poster_path) - self.assertEqual(video.info_json_path, info_json_path) - self.assertTrue(video.video_path.path.is_absolute()) - self.assertTrue(video.poster_path.path.is_absolute()) - self.assertTrue(video.description_path.path.is_absolute()) - self.assertTrue(video.info_json_path.path.is_absolute()) - @wrap_test_db def test_video_download_no_channel(self): _, session = get_db_context() @@ -239,73 +198,65 @@ class TestVideosDownloaders(TestAPI): video: Video = session.query(Video).one() self.assertEqual(str(video.video_path.path), f'{channel_dir}/video.mp4') - @wrap_test_db - def test_channel_download(self): - _, session = get_db_context() - url = 'https://www.youtube.com/c/LearningSelfReliance/videos' - session.add(Channel(url=url, link='foo')) +def test_channel_download(test_session, simple_channel, video_download_manager, video_file): + url = 'https://www.youtube.com/c/LearningSelfReliance/videos' - channel_dir = self.videos_dir / 'NO CHANNEL' - channel_dir.mkdir(parents=True) - video_path, _, _, _ = self._make_video_files(channel_dir) + with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ + mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ + mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: + mock_extract_info.return_value = example_channel_json + mock_prepare_filename.return_value = (video_file, {'id': 'foo'}) + mock_process_runner.return_value = 0 + video_download_manager.create_download(url, test_session) - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ - mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ - mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: - mock_extract_info.return_value = example_channel_json - mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) - mock_process_runner.return_value = 0 - self.mgr.create_download(url, session) + # Two videos are in the example channel. + downloads = video_download_manager.get_new_downloads(test_session) + expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o'] + for download, expected in zip(downloads, expected): + assert download.url == expected + # Download is skipped for test. + assert download.status == 'new' - # Two videos are in the example channel. - downloads = self.mgr.get_new_downloads(session) - expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o'] - for download, expected in zip(downloads, expected): - self.assertEqual(download.url, expected) - # Download is skipped for test. - self.assertEqual(download.status, 'new') - @wrap_test_db - def test_get_or_create_channel(self): - """ - A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can - match it. - """ - with get_db_session(commit=True) as session: - c1 = Channel(name='foo', link='foo', source_id='foo', url='foo') - c2 = Channel(name='bar', link='bar', source_id='bar') - c3 = Channel(name='baz', link='baz', source_id='baz', url='baz') - c4 = Channel(name='qux', link='qux') - session.add_all([c1, c2, c3, c4]) +def test_get_or_create_channel(test_session): + """ + A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can + match it. + """ + c1 = Channel(name='foo', source_id='foo', url='foo') + c2 = Channel(name='bar', source_id='bar') + c3 = Channel(name='baz', source_id='baz', url='baz') + c4 = Channel(name='qux') + test_session.add_all([c1, c2, c3, c4]) + test_session.commit() - # All existing channels should be used. - tests = [ - (dict(source_id='foo'), c1), - (dict(link='foo'), c1), - (dict(url='foo'), c1), - (dict(url='foo', source_id='bar'), c2), # source_id is preferred. - (dict(name='foo', source_id='bar'), c2), - (dict(source_id='bar'), c2), - (dict(source_id='baz'), c3), - (dict(name='qux'), c4), - (dict(link='qux'), c4), - ] - for kwargs, expected in tests: + # All existing channels should be used. + tests = [ + (dict(source_id='foo'), c1), + (dict(url='foo'), c1), + (dict(url='foo', source_id='bar'), c2), # source_id has priority. + (dict(name='foo', source_id='bar'), c2), + (dict(source_id='bar'), c2), + (dict(source_id='baz'), c3), + (dict(name='qux'), c4), + ] + for kwargs, expected in tests: + try: channel = get_or_create_channel(**kwargs) - self.assertEqual(expected.id, channel.id, f'Expected {expected} got {channel}') + except Exception as e: + raise Exception(f'get_or_create_channel failed with {kwargs=}') from e + assert expected.id == channel.id, f'Expected {expected} for {kwargs} but got {channel}' - # A new channel is created. It will not be automatically downloaded. - channel = get_or_create_channel(source_id='quux', name='quux', url='quux') - self.assertEqual(channel.id, 5) - self.assertEqual(channel.source_id, 'quux') - self.assertEqual(channel.name, 'quux') - self.assertEqual(channel.link, 'quux') - self.assertEqual(channel.url, 'quux') - self.assertIsNone(channel.download_frequency) + # A new channel is created. It will not be automatically downloaded. + channel = get_or_create_channel(source_id='quux', name='quux', url='quux') + assert channel.id == 5 + assert channel.source_id == 'quux' + assert channel.name == 'quux' + assert channel.url == 'quux' - # New channel can be retrieved. - self.assertEqual(get_or_create_channel(source_id='quux'), channel) + # New channel can be retrieved. + assert get_or_create_channel(source_id='quux') == channel def test_download_info_save(test_session, video_download_manager): @@ -397,7 +348,7 @@ def test_channel_download_no_download(test_session, video_download_manager, simp assert video_download_manager.get_downloads(test_session) == [] with pytest.raises(InvalidDownload): - download_channel(simple_channel.link) + download_channel(simple_channel.id) def test_invalid_download_url(test_session, test_download_manager, test_downloader_config): @@ -413,3 +364,32 @@ def test_invalid_download_url(test_session, test_download_manager, test_download download = test_download_manager.get_downloads(test_session)[0] assert download.status == 'failed' + + +def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager): + simple_channel.source_id = example_video_json['channel_id'] + simple_channel.directory = test_directory / 'videos/channel name' + simple_channel.directory.mkdir(parents=True) + + video_file = simple_channel.directory / 'a video.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file) + + url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' + with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ + mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \ + mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner: + mock_extract_info.return_value = example_video_json + mock_prepare_filename.return_value = (video_file, {'id': 'foo'}) + mock_process_runner.return_value = 0 + + video_download_manager.create_download(url, test_session) + + mock_process_runner.assert_called_once() + video_url, _, out_dir = mock_process_runner.call_args[0] + + download: Download = test_session.query(Download).one() + assert video_url == download.url + assert test_session.query(Channel).one() + + video: Video = test_session.query(Video).one() + assert video.video_path.path.is_absolute() diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py index 7dfcf39b..8d9ff7fe 100644 --- a/modules/videos/video/api.py +++ b/modules/videos/video/api.py @@ -44,7 +44,7 @@ async def search(_: Request, body: schema.VideoSearchRequest): body.search_str, body.offset, body.limit, - body.channel_link, + body.channel_id, body.order_by, body.filters, ) diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index 806e3df0..edb3111a 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -80,16 +80,16 @@ def video_search( search_str: str = None, offset: int = None, limit: int = VIDEO_QUERY_LIMIT, - channel_link: str = None, + channel_id: int = None, order_by: str = None, filters: List[str] = None, ) -> Tuple[List[dict], int]: with get_db_curs() as curs: params = dict(search_str=search_str, offset=offset) channel_where = '' - if channel_link: - channel_where = 'AND channel_id = (select id from channel where link=%(channel_link)s)' - params['channel_link'] = channel_link + if channel_id: + channel_where = 'AND channel_id = %(channel_id)s' + params['channel_id'] = channel_id # Filter for/against favorites, if it was provided favorites_where = '' diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py index a30a1a92..41c71a26 100644 --- a/modules/videos/video/test/test_api.py +++ b/modules/videos/video/test/test_api.py @@ -1,163 +1,141 @@ -import pathlib from datetime import timedelta from http import HTTPStatus from json import dumps from unittest import mock -from uuid import uuid4 -from modules.videos.models import Channel, Video -from modules.videos.test.common import create_channel_structure +from modules.videos.models import Video from modules.videos.video.lib import get_video_for_app from wrolpi.dates import now -from wrolpi.db import get_db_session from wrolpi.errors import API_ERRORS, WROLModeEnabled from wrolpi.root_api import api_app -from wrolpi.test.common import wrap_test_db, TestAPI -class TestVideoFunctions(TestAPI): +def test_get_video_prev_next(test_session, channel_factory, video_factory): + """ + Test that the previous and next videos will be retrieved when fetching a video. + """ + channel1, channel2, channel3, channel4 = channel_factory(), channel_factory(), channel_factory(), channel_factory() - @wrap_test_db - def test_get_video_prev_next(self): - """ - Test that the previous and next videos will be retrieved when fetching a video. - """ + now_ = now() + second = timedelta(seconds=1) - with get_db_session(commit=True) as session: - for _ in range(4): - session.add(Channel(link=str(uuid4()))) - channel1, channel2, channel3, channel4 = session.query(Channel).all() + # The upload_date decides the order of the prev/next videos. + video_factory(title='vid1', channel_id=channel1.id, upload_date=now_) + video_factory(title='vid2', channel_id=channel1.id, upload_date=now_ + second) + video_factory(title='vid3', channel_id=channel2.id, upload_date=now_ + (second * 4)) + video_factory(title='vid4', channel_id=channel1.id, upload_date=now_ + (second * 3)) + video_factory(title='vid5', channel_id=channel2.id, upload_date=now_ + (second * 2)) + video_factory(title='vid6', channel_id=channel2.id, upload_date=now_ + (second * 5)) + video_factory(title='vid7', channel_id=channel1.id) + video_factory(title='vid8', channel_id=channel2.id, upload_date=now_ + (second * 7)) + video_factory(title='vid9', channel_id=channel3.id, upload_date=now_ + (second * 8)) + # Videos without an upload date can't be placed. + video_factory(title='vid10', channel_id=channel4.id) + video_factory(title='vid11', channel_id=channel4.id) - now_ = now() - second = timedelta(seconds=1) + test_session.commit() - # The upload_date decides the order of the prev/next videos. - session.add(Video(title=f'vid1', channel_id=channel1.id, upload_date=now_)) - session.add(Video(title=f'vid2', channel_id=channel1.id, upload_date=now_ + second)) - session.add(Video(title=f'vid3', channel_id=channel2.id, upload_date=now_ + (second * 4))) - session.add(Video(title=f'vid4', channel_id=channel1.id, upload_date=now_ + (second * 3))) - session.add(Video(title=f'vid5', channel_id=channel2.id, upload_date=now_ + (second * 2))) - session.add(Video(title=f'vid6', channel_id=channel2.id, upload_date=now_ + (second * 5))) - session.add(Video(title=f'vid7', channel_id=channel1.id)) - session.add(Video(title=f'vid8', channel_id=channel2.id, upload_date=now_ + (second * 7))) - session.add(Video(title=f'vid9', channel_id=channel3.id, upload_date=now_ + (second * 8))) - session.add(Video(title=f'vid10', channel_id=channel4.id)) - session.add(Video(title=f'vid11', channel_id=channel4.id)) + tests = [ + # Channel 1's videos were inserted in upload_date order. + (1, (None, 'vid2')), + (2, ('vid1', 'vid4')), + (4, ('vid2', None)), # 7 has no upload_date, so it doesn't come after 4. + (7, (None, None)), # 7 has no upload_date, so we don't know the order of it. + # Channel 3 has only one video. + (9, (None, None)), + # Channel 2 was inserted out of order. + (5, (None, 'vid3')), + (3, ('vid5', 'vid6')), + (8, ('vid6', None)), + # Channel 4's videos have no upload date, so we don't know what is previous/next. + (10, (None, None)), + ] - tests = [ - # Channel 1's videos were inserted in upload_date order. - (1, (None, 'vid2')), - (2, ('vid1', 'vid4')), - (4, ('vid2', None)), # 7 has no upload_date, so it doesn't come after 4. - (7, (None, None)), # 7 has no upload_date, so we don't know the order of it. - # Channel 3 has only one video. - (9, (None, None)), - # Channel 2 was inserted out of order. - (5, (None, 'vid3')), - (3, ('vid5', 'vid6')), - (8, ('vid6', None)), - # Channel 4's videos have no upload date, so we don't know what is previous/next. - (10, (None, None)), - ] - - for id_, (prev_title, next_title) in tests: - video = session.query(Video).filter_by(id=id_).one() - prev_video, next_video = video.get_surrounding_videos() + for id_, (prev_title, next_title) in tests: + video = test_session.query(Video).filter_by(id=id_).one() + prev_video, next_video = video.get_surrounding_videos() + try: if prev_title is None: - self.assertIsNone(prev_video) + assert prev_video is None else: - self.assertDictContains(prev_video, {'title': prev_title}) + assert prev_video and prev_video.title == prev_title if next_title is None: - self.assertIsNone(next_video) + assert next_video is None else: - self.assertDictContains(next_video, {'title': next_title}) + assert next_video and next_video.title == next_title + except AssertionError as e: + raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e - @wrap_test_db - @create_channel_structure({ - 'channel1': ['vid1.mp4'] - }) - def test_wrol_mode(self, tempdir): - """ - Many methods are blocked when WROL Mode is enabled. - """ - channel = dumps(dict(name='foo', directory='foo')) - favorite = dumps(dict(video_id=1, favorite=True)) - with mock.patch('wrolpi.common.wrol_mode_enabled', lambda: True): - # Can't create, update, or delete a channel. - _, resp = api_app.test_client.post('/api/videos/channels', content=channel) - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) - _, resp = api_app.test_client.put('/api/videos/channels/channel1', content=channel) - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) - _, resp = api_app.test_client.delete('/api/videos/channels/channel1') - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) +def test_get_video_for_app(test_session, simple_channel, simple_video): + vid, prev, next_ = get_video_for_app(simple_video.id) + assert vid['id'] == simple_video.id - # Can't delete a video - _, resp = api_app.test_client.delete('/api/videos/video/1') - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) - # Can't refresh or download - _, resp = api_app.test_client.post('/api/videos/refresh') - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) - _, resp = api_app.test_client.post('/api/videos/download') - self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code']) +def test_video_delete(test_session, test_directory, channel_factory, video_factory): + """Video.delete() removes the video's files, but leave the DB record.""" + channel1, channel2 = channel_factory(), channel_factory() + vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True) + vid2 = video_factory(channel_id=channel2.id, with_video_file=True, with_info_json=True) + test_session.commit() - # THE REST OF THESE METHODS ARE ALLOWED - _, resp = api_app.test_client.post('/api/videos/favorite', content=favorite) - self.assertEqual(resp.status_code, HTTPStatus.OK) + vid1_video_path, vid1_caption_path = vid1.video_path.path, vid1.caption_path.path + vid2_video_path, vid2_info_json_path = vid2.video_path.path, vid2.info_json_path.path - @wrap_test_db - @create_channel_structure({ - 'channel1': [ - 'vid1.mp4', - 'vid1.en.vtt', - ], - }) - def test_get_video_for_app(self, tempdir): - with get_db_session(commit=True) as session: - vid1 = session.query(Video).one() + # No videos have been deleted yet. + assert vid1_video_path.is_file() and vid1_caption_path.is_file() + assert vid2_video_path.is_file() and vid2_info_json_path.is_file() + assert channel1.skip_download_videos is None + assert channel2.skip_download_videos is None - vid, prev, next_ = get_video_for_app(vid1.id) - self.assertEqual(vid['id'], vid1.id) + vid1.delete() - @wrap_test_db - @create_channel_structure({ - 'channel1': [ - 'vid1.mp4', - 'vid1.en.vtt', - ], - 'channel2': [ - 'vid2.mp4', - 'vid2.info.json', - ], - }) - def test_delete_video(self, tempdir: pathlib.Path): - with get_db_session(commit=True) as session: - channel1 = session.query(Channel).filter_by(name='channel1').one() - channel2 = session.query(Channel).filter_by(name='channel2').one() - vid1, vid2 = session.query(Video).order_by(Video.video_path).all() - vid1.source_id = 'foo' - vid2.source_id = 'bar' + # Video was added to skip list. + assert len(channel1.skip_download_videos) == 1 + # Deleting a video leaves it's entry in the DB, but its files are deleted. + assert test_session.query(Video).count() == 2 + assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False + assert vid2_video_path.is_file() and vid2_info_json_path.is_file() - # No videos have been deleted yet. - self.assertIsNone(channel1.skip_download_videos) - self.assertIsNone(channel2.skip_download_videos) - self.assertTrue((tempdir / 'channel1/vid1.mp4').is_file()) + vid2.delete() - vid1.delete() + assert test_session.query(Video).count() == 2 + assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False + assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False - channel1 = session.query(Channel).filter_by(name='channel1').one() - # Video was added to skip list. - self.assertEqual(len(channel1.skip_download_videos), 1) - # Deleting a video leaves it's entry in the DB, but its files are deleted. - self.assertEqual(session.query(Video).count(), 2) - self.assertFalse((tempdir / 'channel1/vid1.mp4').is_file()) - self.assertTrue((tempdir / 'channel2/vid2.mp4').is_file()) - vid2.delete() +def test_wrol_mode(test_directory, simple_channel, simple_video): + """Many methods are blocked when WROL Mode is enabled.""" + channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory))) + favorite = dumps(dict(video_id=simple_video.id, favorite=True)) - self.assertEqual(session.query(Video).count(), 2) - self.assertFalse((tempdir / 'channel1/vid1.mp4').is_file()) - self.assertFalse((tempdir / 'channel2/vid2.mp4').is_file()) + with mock.patch('wrolpi.common.wrol_mode_enabled', lambda: True): + # Can't create, update, or delete a channel. + _, resp = api_app.test_client.post('/api/videos/channels', content=channel) + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + _, resp = api_app.test_client.put(f'/api/videos/channels/{simple_channel.id}', content=channel) + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + _, resp = api_app.test_client.delete(f'/api/videos/channels/{simple_channel.id}') + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + + # Can't delete a video + _, resp = api_app.test_client.delete(f'/api/videos/video/{simple_video.id}') + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + + # Can't refresh or download + _, resp = api_app.test_client.post('/api/videos/refresh') + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + _, resp = api_app.test_client.post(f'/api/videos/download/{simple_channel.id}') + assert resp.status_code == HTTPStatus.FORBIDDEN + assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + + # THE REST OF THESE METHODS ARE ALLOWED + _, resp = api_app.test_client.post('/api/videos/favorite', content=favorite) + assert resp.status_code == HTTPStatus.OK diff --git a/wrolpi/__init__.py b/wrolpi/__init__.py index 7fe8a8d8..d479333b 100644 --- a/wrolpi/__init__.py +++ b/wrolpi/__init__.py @@ -1,5 +1,6 @@ import inspect import multiprocessing +from functools import wraps BEFORE_STARTUP_FUNCTIONS = [] @@ -29,6 +30,7 @@ def limit_concurrent(limit: int): def wrapper(func: callable): if inspect.iscoroutinefunction(func): + @wraps(func) async def wrapped(*a, **kw): acquired = sema.acquire(block=False) if not acquired: @@ -37,6 +39,7 @@ def limit_concurrent(limit: int): return wrapped else: + @wraps(func) def wrapped(*a, **kw): acquired = sema.acquire(block=False) if not acquired: diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py index bbc002cf..329c618a 100644 --- a/wrolpi/cmd.py +++ b/wrolpi/cmd.py @@ -20,7 +20,7 @@ def which(*possible_paths: str, warn: bool = False) -> Path: Returns None if no executable can be found. """ - possible_paths = map(Path, possible_paths) + possible_paths = list(map(Path, possible_paths)) for path in possible_paths: if is_executable(path): # `path` was an absolute file which is executable. diff --git a/wrolpi/common.py b/wrolpi/common.py index dd8b6404..a23a7c35 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -44,7 +44,7 @@ Base = declarative_base() class ModelHelper: - def dict(self) -> dict: + def dict(self, *_, **__) -> dict: d = {i.name: getattr(self, i.name) for i in self.__table__.columns} # noqa return d @@ -732,11 +732,6 @@ def native_only(func: callable): return wrapped -def read_yaml(path: Path): - with path.open('rt') as fh: - return yaml.load(fh, Loader=yaml.Loader) - - def recursive_map(obj: Any, func: callable): """ Apply `func` to all values of any dictionaries; or items in any list/set/tuple. @@ -755,3 +750,11 @@ def recursive_map(obj: Any, func: callable): type_ = type(obj) return type_(map(lambda i: recursive_map(i, func), obj)) return func(obj) + + +def any_extensions(filename: str, extensions: Iterable = ()): + """Return True only if the file name ends with any of the possible extensions. + + Matches lower or upper case of the extension. + """ + return any(filename.lower().endswith(ext) for ext in extensions) diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 69d79308..49bd2d29 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -1,7 +1,6 @@ """ Fixtures for Pytest tests. """ -import os import pathlib import tempfile from typing import Tuple diff --git a/wrolpi/errors.py b/wrolpi/errors.py index 22a03e9d..f052a99f 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -158,7 +158,7 @@ API_ERRORS = { }, UnknownChannel: { 'code': next(error_codes), - 'message': 'The channel could not be found by its link.', + 'message': 'The channel could not be found.', 'status': HTTPStatus.NOT_FOUND, }, UnknownDirectory: { diff --git a/wrolpi/media_path.py b/wrolpi/media_path.py index ff097225..099aa72d 100644 --- a/wrolpi/media_path.py +++ b/wrolpi/media_path.py @@ -18,10 +18,10 @@ class MediaPath: self._path = path.absolute() def __repr__(self): - return f'' + return f'' def __str__(self): - return str(self.path.relative_to(get_media_directory())) + return str(self.relative) @classmethod def _validate_path(cls, path: PATH_TYPE): @@ -40,7 +40,7 @@ class MediaPath: def __json__(self): # Always return a relative path to frontend. - return str(self.path.relative_to(get_media_directory())) + return str(self.relative) @property def path(self) -> pathlib.Path: @@ -60,6 +60,11 @@ class MediaPath: def startswith(self, other): return str(self._path).startswith(other) + @property + def relative(self): + """Get the path relative to the media directory.""" + return self._path.relative_to(get_media_directory()) + class MediaPathType(types.TypeDecorator): # noqa impl = types.String diff --git a/wrolpi/schema.py b/wrolpi/schema.py index b94cef0f..a1969cdd 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional +from typing import Optional @dataclass diff --git a/wrolpi/version.py b/wrolpi/version.py index 41b97933..8cad1f4d 100644 --- a/wrolpi/version.py +++ b/wrolpi/version.py @@ -1,4 +1,4 @@ -__version__ = '0.4.3-beta' +__version__ = '0.4.4-beta' import subprocess @@ -25,7 +25,5 @@ def git_branch(): def get_version_string(): - """ - Return a string containing the WROLPi version, git branch and git revision hash. - """ + """Return a string containing the WROLPi version, git branch and git revision hash.""" return f'{__version__} (git: {git_branch()} {git_revision()})'