Merge pull request #118 from lrnselfreliance/feature/remove-channel-link

Feature/remove channel link
This commit is contained in:
lrnselfreliance 2022-04-29 19:13:33 -06:00 • committed by GitHub
commit c8810fbeba
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 1230 additions and 1329 deletions

View file

@ -0,0 +1,37 @@
"""Drop channel.link.
Revision ID: d8fcdb7773c0
Revises: 2fb39c1861c7
Create Date: 2022-04-08 12:52:02.877449
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = 'd8fcdb7773c0'
down_revision = '2fb39c1861c7'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE channel DROP CONSTRAINT IF EXISTS channel_link_key')
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS link')
session.execute('DROP INDEX IF EXISTS channel_url_key')
session.execute("UPDATE channel SET url=null WHERE url=''")
session.execute('CREATE UNIQUE INDEX channel_url_key ON channel (url) WHERE url IS NOT NULL')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE channel ADD CONSTRAINT channel_link_key UNIQUE (link)')
session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS link TEXT')
# Notice we are not removing the channel_url_key constraint.

View file

@ -82,7 +82,7 @@ export default function App() {
<>
<Switch>
<Route path='/videos/video/:video_id' exact component={VideoWrapper}/>
<Route path='/videos/channel/:channel_link/video/:video_id' exact component={VideoWrapper}/>
<Route path='/videos/channel/:channel_id/video/:video_id' exact component={VideoWrapper}/>
<Route path="/" exact={true} component={Welcome}/>
<Route path="/videos" component={VideosRoute}/>
<Route path="/admin" component={Admin}/>

View file

@ -95,9 +95,9 @@ let apiPatch = async (url, body) => {
return await apiCall(url, 'PATCH', body)
};
export async function updateChannel(link, channel) {
export async function updateChannel(id, channel) {
channel = emptyToNull(channel);
return await apiPut(`${VIDEOS_API}/channels/${link}`, channel);
return await apiPut(`${VIDEOS_API}/channels/${id}`, channel);
}
export async function createChannel(channel) {
@ -105,8 +105,8 @@ export async function createChannel(channel) {
return await apiPost(`${VIDEOS_API}/channels`, channel);
}
export async function deleteChannel(channel_link) {
return apiDelete(`${VIDEOS_API}/channels/${channel_link}`);
export async function deleteChannel(id) {
return apiDelete(`${VIDEOS_API}/channels/${id}`);
}
export async function getChannels() {
@ -114,12 +114,12 @@ export async function getChannels() {
return (await response.json())['channels'];
}
export async function getChannel(link) {
let response = await apiGet(`${VIDEOS_API}/channels/${link}`);
export async function getChannel(id) {
let response = await apiGet(`${VIDEOS_API}/channels/${id}`);
return (await response.json())['channel'];
}
export async function searchVideos(offset, limit, channel_link, searchStr, order_by, filters) {
export async function searchVideos(offset, limit, channel_id, searchStr, order_by, filters) {
// Build a search query to retrieve a list of videos from the API
offset = offset || 0;
limit = limit || DEFAULT_LIMIT;
@ -128,8 +128,8 @@ export async function searchVideos(offset, limit, channel_link, searchStr, order
if (searchStr) {
body.search_str = searchStr;
}
if (channel_link) {
body.channel_link = channel_link;
if (channel_id) {
body.channel_id = channel_id;
}
body.order_by = order_by ? order_by : 'rank'
@ -211,8 +211,8 @@ export async function download() {
return await response.json();
}
export async function downloadChannel(link) {
let url = `${VIDEOS_API}/download/${link}`;
export async function downloadChannel(id) {
let url = `${VIDEOS_API}/download/${id}`;
let response = await apiPost(url);
let json = await response.json();
if (response.status === 400 && json['code'] === 30) {
@ -225,8 +225,8 @@ export async function downloadChannel(link) {
}
}
export async function refreshChannel(link) {
let url = `${VIDEOS_API}/refresh/${link}`;
export async function refreshChannel(id) {
let url = `${VIDEOS_API}/refresh/${id}`;
await fetch(url, {method: 'POST'});
}

View file

@ -154,8 +154,8 @@ class ChannelPage extends APIForm {
async componentDidMount() {
if (!this.props.create) {
let channel_link = this.props.match.params.channel_link;
let channel = await getChannel(channel_link);
let channel_id = this.props.match.params.channel_id;
let channel = await getChannel(channel_id);
let statistics = channel.statistics;
if (statistics) {
this.setState({statistics: statistics || null});
@ -173,7 +173,7 @@ class ChannelPage extends APIForm {
handleConfirm = async () => {
this.setState({deleteOpen: false});
let response = await deleteChannel(this.props.match.params.channel_link);
let response = await deleteChannel(this.props.match.params.channel_id);
if (response.status === 204) {
this.props.history.push({
pathname: '/videos/channel'
@ -194,7 +194,7 @@ class ChannelPage extends APIForm {
if (this.props.create) {
response = await createChannel(inputs);
} else {
response = await updateChannel(this.props.match.params.channel_link, inputs);
response = await updateChannel(this.props.match.params.channel_id, inputs);
}
} finally {
@ -215,7 +215,7 @@ class ChannelPage extends APIForm {
this.setSuccess(
'Channel created',
<span>
Your channel was created. View it <Link to={`/videos/channel/${channel.link}/edit`}>here</Link>
Your channel was created. View it <Link to={`/videos/channel/${channel.id}/edit`}>here</Link>
</span>
);
} else {
@ -262,12 +262,12 @@ class ChannelPage extends APIForm {
downloadChannel = async (e) => {
e.preventDefault();
return await downloadChannel(this.props.match.params.channel_link);
return await downloadChannel(this.props.match.params.channel_id);
}
refreshChannel = async (e) => {
e.preventDefault();
return await refreshChannel(this.props.match.params.channel_link);
return await refreshChannel(this.props.match.params.channel_id);
}
handleDeleteButton = (e) => {
@ -470,8 +470,8 @@ export function NewChannel(props) {
class ChannelRow extends React.Component {
constructor(props) {
super(props);
this.editTo = `/videos/channel/${props.channel.link}/edit`;
this.videosTo = `/videos/channel/${props.channel.link}/video`;
this.editTo = `/videos/channel/${props.channel.id}/edit`;
this.videosTo = `/videos/channel/${props.channel.id}/video`;
}
render() {
@ -646,7 +646,7 @@ export class Channels extends React.Component {
<Table striped basic size='large'>
{tableHeader}
<Table.Body>
{this.state.results.map((channel) => <ChannelRow key={channel.link}
{this.state.results.map((channel) => <ChannelRow key={channel.id}
channel={channel}/>)}
</Table.Body>
</Table>
@ -655,7 +655,7 @@ export class Channels extends React.Component {
<Table striped basic unstackable size='small'>
<Table.Body>
{this.state.results.map((channel) =>
<MobileChannelRow key={channel.link} channel={channel}/>)}
<MobileChannelRow key={channel.id} channel={channel}/>)}
</Table.Body>
</Table>
</Responsive>

View file

@ -135,8 +135,8 @@ export function VideoCard({video}) {
let channel = video.channel ? video.channel : null;
let channel_url = null;
if (channel) {
channel_url = `/videos/channel/${channel.link}/video`;
video_url = `/videos/channel/${channel.link}/video/${video.id}`;
channel_url = `/videos/channel/${channel.id}/video`;
video_url = `/videos/channel/${channel.id}/video/${video.id}`;
}
let poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null;

View file

@ -76,8 +76,8 @@ function VideoPage(props) {
</Tab.Pane>
};
let statisticsPane = {
menuItem: 'Statistics', render: () => <Tab.Pane>
let aboutPane = {
menuItem: 'About', render: () => <Tab.Pane>
<h3>Size</h3>
<p>{video.size ? humanFileSize(video.size) : '???'}</p>
@ -104,7 +104,7 @@ function VideoPage(props) {
</Tab.Pane>
};
let tabPanes = [descriptionPane, statisticsPane, captionsPane];
let tabPanes = [descriptionPane, aboutPane, captionsPane];
return (
<>
@ -135,7 +135,7 @@ function VideoPage(props) {
<h2>{video.title || video.video_path}</h2>
{video.upload_date && <h3>{uploadDate(video.upload_date)}</h3>}
<h3>
{channel && <Link to={`/videos/channel/${channel.link}/video`}>
{channel && <Link to={`/videos/channel/${channel.id}/video`}>
{channel.name}
</Link>}
</h3>

View file

@ -103,8 +103,8 @@ export class VideoWrapper extends React.Component {
async fetchVideo() {
// Get and display the Video specified in the Router match
let [video, prev, next] = await getVideo(this.props.match.params.video_id);
let channel_link = this.props.match.params.channel_link;
let channel = channel_link ? await getChannel(channel_link) : null;
let channel_id = this.props.match.params.channel_id;
let channel = channel_id ? await getChannel(channel_id) : null;
let no_channel = false;
if (!channel) {
no_channel = true;
@ -222,7 +222,7 @@ class Videos extends React.Component {
}
async componentDidMount() {
if (this.props.match.params.channel_link) {
if (this.props.match.params.channel_id) {
await this.fetchChannel();
} else {
await this.fetchVideos();
@ -233,7 +233,7 @@ class Videos extends React.Component {
async componentDidUpdate(prevProps, prevState, snapshot) {
let params = this.props.match.params;
let channelChanged = params.channel_link !== prevProps.match.params.channel_link;
let channelChanged = params.channel_id !== prevProps.match.params.channel_id;
let pageChanged = (
prevState.activePage !== this.state.activePage ||
prevState.searchOrder !== this.state.searchOrder ||
@ -269,10 +269,10 @@ class Videos extends React.Component {
async fetchChannel() {
// Get and display the channel specified in the Router match
let channel_link = this.props.match.params.channel_link;
let channel_id = this.props.match.params.channel_id;
let channel = null;
if (channel_link) {
channel = await getChannel(channel_link);
if (channel_id) {
channel = await getChannel(channel_id);
}
this.setState({
channel,
@ -300,13 +300,13 @@ class Videos extends React.Component {
async fetchVideos() {
this.setState({videos: null});
let offset = this.state.limit * this.state.activePage - this.state.limit;
let channel_link = this.state.channel ? this.state.channel.link : null;
let channel_id = this.state.channel ? this.state.channel.id : null;
let {queryStr, searchOrder, limit} = this.state;
// Pass any enabled filters to the search.
let filtersArr = this.enabledFilters();
let [videos, total] = await searchVideos(offset, limit, channel_link, queryStr, searchOrder, filtersArr);
let [videos, total] = await searchVideos(offset, limit, channel_id, queryStr, searchOrder, filtersArr);
let totalPages = Math.round(total / this.state.limit) || 1;
this.setState({videos, totalPages});
@ -428,7 +428,7 @@ class Videos extends React.Component {
<>
&nbsp;
&nbsp;
<Link to={`/videos/channel/${channel.link}/edit`}>
<Link to={`/videos/channel/${channel.id}/edit`}>
<Icon name="edit"/>
</Link>
</>
@ -527,7 +527,7 @@ export function VideosRoute(props) {
<Route path='/videos/channel' exact component={Channels}/>
<Route path='/videos/manage' exact component={ManageTab}/>
<Route path='/videos/channel/new' exact component={NewChannel}/>
<Route path='/videos/channel/:channel_link/edit' exact
<Route path='/videos/channel/:channel_id/edit' exact
component={(i) =>
<EditChannel
match={i.match}
@ -535,7 +535,7 @@ export function VideosRoute(props) {
/>
}
/>
<Route path='/videos/channel/:channel_link/video' exact component={Videos}/>
<Route path='/videos/channel/:channel_id/video' exact component={Videos}/>
</PageContainer>
)
}

View file

@ -609,7 +609,6 @@ def search(search_str: str, domain: str, limit: int, offset: int) -> Tuple[List[
OFFSET %(offset)s
LIMIT %(limit)s
'''
print(stmt, offset, limit)
curs.execute(stmt, params)
results = [dict(i) for i in curs.fetchall()]
total = results[0]['total'] if results else 0

View file

@ -17,6 +17,7 @@ from wrolpi.vars import PYTEST
try:
import magic
have_magic = True
except ImportError:
logger.error('Could not import magic. Is libmagic installed?')

View file

@ -6,7 +6,7 @@ from unittest import mock
import pytest
from wrolpi.errors import API_ERRORS, WROLModeEnabled
from wrolpi.test.common import assert_dict_contains, skip_circleci
from wrolpi.test.common import assert_dict_contains
from wrolpi.vars import PROJECT_DIR
@ -128,8 +128,6 @@ def do_search(test_client, search_str, total, expected):
assert file['path'] == file['key']
# TODO this fails on CircleCI. The order is wrong. Why?
@skip_circleci
def test_files_search(test_session, test_client, make_files_structure):
# You can search an empty directory.
do_search(test_client, 'nothing', 0, [])
@ -154,8 +152,8 @@ def test_files_search(test_session, test_client, make_files_structure):
do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)])
do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)])
do_search(test_client, 'baz', 2, [
dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736),
dict(path='baz.mp4', mimetype='video/mp4', size=1055736),
dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736),
])
do_search(test_client, 'two', 1, [dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736)])
do_search(test_client, 'nothing', 0, [])

View file

@ -1,6 +1,7 @@
import asyncio
from functools import wraps
from http import HTTPStatus
from typing import List
from sanic import Blueprint, response
from sanic.request import Request
@ -32,12 +33,12 @@ refresh_queue, refresh_event = create_websocket_feed('refresh', '/feeds/refresh'
@content_bp.post('/refresh')
@content_bp.post('/refresh/<link:str>')
@content_bp.post('/refresh/<channel_id:str>')
@openapi.description('Search for videos that have previously been downloaded and stored.')
@openapi.response(HTTPStatus.OK, schema.StreamResponse)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
@wrol_mode_check
async def refresh(_, link: str = None):
async def refresh(_, channel_id: int = None):
refresh_logger = logger.getChild('refresh')
stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/refresh')
@ -51,8 +52,8 @@ async def refresh(_, link: str = None):
try:
refresh_logger.info('refresh started')
channel_links = [link] if link else None
await refresh_videos(channel_links)
channel_ids = [channel_id] if channel_id else None
await refresh_videos(channel_ids)
refresh_logger.info('refresh complete')
except Exception:
@ -66,34 +67,19 @@ async def refresh(_, link: str = None):
return response.json({'code': 'stream-started', 'stream_url': stream_url})
download_queue, download_event = create_websocket_feed('download', '/feeds/download', content_bp)
@content_bp.post('/download')
@content_bp.post('/download/<link:str>')
@openapi.description('Update channel catalogs, download any missing videos')
@content_bp.post('/download/<channel_id:int>')
@openapi.description('Update a channel catalog, download any missing videos')
@openapi.response(HTTPStatus.OK, schema.StreamResponse)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
@wrol_mode_check
def download(_, link: str = None):
download_logger = logger.getChild('download')
stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/download')
# Only one download can run at a time
if download_event.is_set():
return response.json({'error': 'download already running', 'stream_url': stream_url}, HTTPStatus.CONFLICT)
if refresh_event.is_set():
return response.json({'error': 'Refresh is running. Cannot download.'})
channel_lib.download_channel(link)
download_logger.debug('do_download scheduled')
return response.json({'code': 'stream-started', 'stream_url': stream_url})
def download(_, channel_id: int = None):
channel_lib.download_channel(channel_id)
return response.empty()
@wraps(lib.refresh_videos)
async def refresh_videos(channel_links: list = None):
return lib.refresh_videos(channel_links=channel_links)
async def refresh_videos(channel_ids: List[int] = None):
return lib.refresh_videos(channel_ids=channel_ids)
@content_bp.post('/favorite')

View file

@ -7,11 +7,13 @@ from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.common import logger, wrol_mode_check, make_media_directory, \
get_media_directory
get_media_directory, run_after
from wrolpi.root_api import json_response
from wrolpi.schema import JSONErrorResponse
from wrolpi.vars import PYTEST
from . import lib
from .. import schema
from ..lib import save_channels_config
channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels')
@ -26,11 +28,11 @@ async def get_channels(_: Request):
return json_response({'channels': channels})
@channel_bp.get('/<link:str>')
@channel_bp.get('/<channel_id:int>')
@openapi.description('Get a Channel')
@openapi.response(HTTPStatus.OK, schema.ChannelResponse)
def channel_get(_: Request, link: str):
channel = lib.get_channel(link=link)
def channel_get(_: Request, channel_id: int = None):
channel = lib.get_channel(channel_id=channel_id)
channel.pop('info_json')
return json_response({'channel': channel})
@ -49,20 +51,19 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
if not body.directory.is_dir() and body.mkdir:
make_media_directory(body.directory)
bd_edit = body.__dict__
bd_edit['name'] = bd_edit['name'].strip()
channel = lib.create_channel(bd_edit) # TODO don't use the dataclass as a dict.
channel = lib.create_channel(data=body, return_dict=False)
# Refresh the videos asynchronously
from ..api import refresh_videos
coro = refresh_videos([channel['link']])
asyncio.ensure_future(coro)
if not PYTEST:
from ..api import refresh_videos
coro = refresh_videos([channel.id])
asyncio.ensure_future(coro)
return response.json({'success': 'Channel created successfully'}, HTTPStatus.CREATED,
{'Location': f'/api/videos/channels/{channel["link"]}'})
{'Location': f'/api/videos/channels/{channel.id}'})
@channel_bp.put('/<link:str>')
@channel_bp.put('/<channel_id:int>')
@openapi.definition(
description='Update a Channel',
body=schema.ChannelPutRequest,
@ -70,17 +71,18 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
@openapi.response(HTTPStatus.NO_CONTENT)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
@validate(schema.ChannelPutRequest)
@run_after(save_channels_config)
@wrol_mode_check
def channel_update(_: Request, link: str, body: schema.ChannelPutRequest):
channel = lib.update_channel(body.__dict__, link)
def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest):
channel = lib.update_channel(data=body, channel_id=channel_id)
return response.raw('', HTTPStatus.NO_CONTENT,
headers={'Location': f'/api/videos/channels/{channel.link}'})
headers={'Location': f'/api/videos/channels/{channel.id}'})
@channel_bp.delete('/<link:str>')
@channel_bp.delete('/<channel_id:int>')
@openapi.description('Delete a Channel')
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@wrol_mode_check
def channel_delete(_: Request, link: str):
lib.delete_channel(link)
def channel_delete(_: Request, channel_id: int):
lib.delete_channel(channel_id=channel_id)
return response.raw('', HTTPStatus.NO_CONTENT)

View file

@ -1,15 +1,15 @@
from collections import defaultdict
from datetime import timedelta
from typing import List, Dict, Union, Generator
from pathlib import Path
from typing import List, Dict, Union
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound
from wrolpi.common import sanitize_link, run_after, get_relative_to_media_directory, make_media_directory, logger
from wrolpi.dates import today
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.common import run_after, logger, \
get_media_directory
from wrolpi.db import get_db_curs, optional_session
from wrolpi.downloader import download_manager
from wrolpi.errors import UnknownChannel, UnknownDirectory, APIError, ValidationError, InvalidDownload
from .. import schema
from ..common import check_for_channel_conflicts
from ..lib import save_channels_config
from ..models import Channel
@ -25,7 +25,7 @@ async def get_minimal_channels() -> List[dict]:
# Get all channels, even if they don't have videos.
stmt = '''
SELECT
c.id, name, link, directory, url, download_frequency, info_date,
c.id, name, directory, url, download_frequency, info_date,
(select next_download from download d where d.url=c.url) AS next_download
FROM
channel AS c
@ -72,148 +72,126 @@ async def get_channels_video_count() -> Dict[int, int]:
COD = Union[dict, Channel]
def get_channel(source_id: str = None, link: str = None, url: str = None, return_dict: bool = True) -> COD:
@optional_session
def get_channel(session: Session, *, channel_id: int = None, source_id: str = None, url: str = None,
directory: str = None, name: str = None, return_dict: bool = True) -> COD:
"""
Attempt to find a Channel using the provided params. The params are in order of reliability.
Raises UnknownChannel if no channel is found.
"""
with get_db_session() as session:
channel: COD = None # noqa
# Try by source_id first.
if source_id:
channel = session.query(Channel).filter_by(source_id=source_id).one_or_none()
if not channel and link:
channel = session.query(Channel).filter_by(link=link).one_or_none()
if not channel and url:
channel = session.query(Channel).filter_by(url=url).one_or_none()
channel: COD = None # noqa
# Try to find the channel by the most reliable methods first.
if channel_id:
channel = session.query(Channel).filter_by(id=channel_id).one_or_none()
if not channel and source_id:
channel = session.query(Channel).filter_by(source_id=source_id).one_or_none()
if not channel and url:
channel = session.query(Channel).filter_by(url=url).one_or_none()
if not channel and directory:
directory = Path(directory)
if not directory.is_absolute():
directory = get_media_directory() / directory
channel = session.query(Channel).filter_by(directory=directory).one_or_none()
if not channel and name:
channel = session.query(Channel).filter_by(name=name).one_or_none()
if not channel:
raise UnknownChannel(f'No channel matches {link=} {source_id=} {url=}')
if not channel:
raise UnknownChannel(f'No channel matches {channel_id=} {source_id=} {url=} {directory=}')
logger.debug(f'Found {channel=} using {source_id=} {link=} {url=}')
if return_dict:
statistics = channel.get_statistics()
channel = channel.dict()
channel['statistics'] = statistics
return channel
def _spread_by_frequency(channels: List[Channel]) -> Generator[Dict, None, None]:
channels_by_frequency = defaultdict(lambda: [])
for channel in channels:
channels_by_frequency[channel.download_frequency].append(channel)
for frequency, channels in channels_by_frequency.items():
# The seconds between each download.
chunk = frequency // len(channels)
for channel in channels:
# My position in the channel group.
index = channels.index(channel)
# My next download will be distributed by my frequency and my position.
position = chunk * index
next_download = today() + timedelta(seconds=frequency + position)
yield dict(url=channel.url, frequency=frequency, next_download=next_download)
logger.debug(f'Found {channel=} using {channel_id=} {source_id=} {url=} {directory=}')
if return_dict:
statistics = channel.get_statistics()
channel = channel.dict()
channel['statistics'] = statistics
return channel
@run_after(save_channels_config)
def update_channel(data: dict, link: str) -> Channel:
@optional_session
def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
"""Update a Channel's DB record"""
with get_db_session(commit=True) as session:
try:
channel: Channel = session.query(Channel).filter_by(link=link).one()
except NoResultFound:
raise UnknownChannel()
try:
channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
except NoResultFound:
raise UnknownChannel()
# Only update directory if it was empty
if data.get('directory') and not channel.directory:
try:
data['directory'] = get_relative_to_media_directory(data['directory'])
except UnknownDirectory:
if data['mkdir']:
make_media_directory(data['directory'])
data['directory'] = get_relative_to_media_directory(data['directory'])
else:
raise
# Only update directory if it was empty
if data.directory and not channel.directory:
data.directory = get_media_directory() / data.directory
if not data.directory.is_dir():
if data.mkdir:
data.directory.mkdir()
else:
raise UnknownDirectory()
if 'directory' in data:
data['directory'] = str(data['directory'])
# Verify that the URL/Name/directory aren't taken
check_for_channel_conflicts(
session,
id_=channel.id,
url=data.url,
name=data.name,
directory=data.directory,
)
if data.get('match_regex') in ('None', 'null'):
data['match_regex'] = None
# Apply the changes now that we've OK'd them
channel.update(data.__dict__)
data['link'] = data.get('link') or link
# Verify that the URL/Name/Link aren't taken
check_for_channel_conflicts(
session,
id_=channel.id,
url=data.get('url'),
name=data.get('name'),
link=data['link'],
directory=data.get('directory'),
)
# Apply the changes now that we've OK'd them
channel.update(data)
session.commit()
return channel
@run_after(save_channels_config)
def create_channel(data: dict, return_dict: bool = True) -> Union[Channel, dict]:
@optional_session
def create_channel(session: Session, data: schema.ChannelPostRequest, return_dict: bool = True) -> Union[Channel, dict]:
"""
Create a new Channel. Check for conflicts with existing Channels.
"""
with get_db_session(commit=True) as session:
link = sanitize_link(data['name'])
try:
# Verify that the URL/Name/Link aren't taken
check_for_channel_conflicts(
session,
url=data.get('url'),
name=data['name'],
link=link,
directory=str(data['directory']),
source_id=data.get('source_id'),
)
except APIError as e:
raise ValidationError() from e
channel = Channel(
name=data['name'],
url=data.get('url'),
match_regex=data.get('match_regex'),
link=link,
directory=str(data['directory']),
download_frequency=data.get('download_frequency'),
source_id=data.get('source_id'),
try:
# Verify that the URL/Name/directory aren't taken
check_for_channel_conflicts(
session,
url=data.url,
name=data.name,
directory=data.directory,
source_id=data.source_id,
)
session.add(channel)
session.commit()
session.flush()
session.refresh(channel)
except APIError as e:
raise ValidationError() from e
if return_dict:
return channel.dict()
else:
return channel
channel = Channel(
name=data.name,
url=data.url,
match_regex=data.match_regex,
directory=data.directory, # noqa
download_frequency=data.download_frequency,
source_id=data.source_id,
)
session.add(channel)
session.commit()
session.flush()
session.refresh(channel)
return channel.dict() if return_dict else channel
@run_after(save_channels_config)
def delete_channel(link):
with get_db_session(commit=True) as session:
try:
channel = session.query(Channel).filter_by(link=link).one()
except NoResultFound:
raise UnknownChannel()
@optional_session
def delete_channel(session: Session, *, channel_id: int):
try:
channel = session.query(Channel).filter_by(id=channel_id).one()
except NoResultFound:
raise UnknownChannel()
channel.delete_with_videos()
channel.delete_with_videos()
session.commit()
def download_channel(link: str):
def download_channel(id_: int):
"""Create a Download record for a Channel's entire catalog. Start downloading."""
channel = get_channel(link=link, return_dict=False)
channel = get_channel(channel_id=id_, return_dict=False)
session = Session.object_session(channel)
download = channel.get_download()
if not download:

View file

@ -1,7 +1,5 @@
import json
import os
import pathlib
import shutil
import tempfile
from datetime import timedelta
from http import HTTPStatus
@ -11,301 +9,116 @@ import mock
from modules.videos.channel.lib import delete_channel
from modules.videos.common import get_no_channel_directory
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import download_manager, Download
from wrolpi.errors import UnknownFile
from wrolpi.root_api import api_app
from wrolpi.test.common import wrap_test_db, TestAPI
from wrolpi.vars import PROJECT_DIR
from wrolpi.test.common import assert_dict_contains
class TestVideoAPI(TestAPI):
def test_get_channels(test_directory, channel_factory, test_client):
channel_factory()
channel_factory()
channel_factory()
request, response = test_client.get('/api/videos/channels')
assert response.status_code == HTTPStatus.OK
assert len(response.json['channels']) == 3
@wrap_test_db
@create_channel_structure(
{
'Foo': ['vid1.mp4'],
'Bar': ['vid2.mp4'],
'Baz': [],
}
def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel):
# A regular user-defined channel
vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
vid1.touch()
vid2 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title2.mp4'
vid2.touch()
# The special NO CHANNEL directory.
no_channel_path = get_no_channel_directory()
vid3 = no_channel_path / 'some video.mp4'
vid3.touch()
no_channel_subdir = no_channel_path / 'subdir'
no_channel_subdir.mkdir()
vid4 = no_channel_subdir / 'other video.mp4'
vid4.touch()
test_client.post('/api/videos/refresh')
videos = test_session.query(Video).order_by(Video.title).all()
assert len(videos) == 4
# Add some fake videos, they should be removed.
test_session.add(Video(title='vid5'))
test_session.add(Video(title='vid6', channel_id=simple_channel.id))
test_session.commit()
videos = test_session.query(Video).order_by(Video.title).all()
assert len(videos) == 6
test_client.post('/api/videos/refresh')
videos = test_session.query(Video).order_by(Video.title).all()
assert len(videos) == 4
def test_get_video(test_client, test_session, simple_channel, video_factory):
"""
Test that you get can information about a video. Test that video file can be gotten.
"""
now_ = now()
video1 = video_factory(channel_id=simple_channel.id)
video1.upload_date, video1.title = now_, 'vid1'
video2 = video_factory(channel_id=simple_channel.id)
video2.upload_date, video2.title = now_ + timedelta(seconds=1), 'vid2'
test_session.commit()
# Test that a 404 is returned when no video exists
_, response = test_client.get('/api/videos/video/10')
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''}
# Get the video info we inserted
_, response = test_client.get('/api/videos/video/1')
assert response.status_code == HTTPStatus.OK, response.json
assert_dict_contains(response.json['video'], {'title': 'vid1'})
# The next video is included.
assert response.json['prev'] is None
assert_dict_contains(response.json['next'], {'title': 'vid2'})
def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel):
"""A channel does not require a download frequency."""
# No downloads are scheduled.
assert len(download_manager.get_downloads(test_session)) == 0
# Get the Channel
request, response = test_client.get('/api/videos/channels/1')
channel = response.json['channel']
assert channel['download_frequency'] is None
# Update the Channel with a frequency.
new_channel = dict(
directory=channel['directory'],
name=channel['name'],
url=channel['url'],
download_frequency=10,
)
def test_get_channels(self, tempdir):
request, response = api_app.test_client.get('/api/videos/channels')
assert response.status_code == HTTPStatus.OK
# Channels are sorted by name
self.assertDictContains(response.json['channels'][0], {'id': 2, 'name': 'Bar', 'video_count': 1})
self.assertDictContains(response.json['channels'][1], {'id': 3, 'name': 'Baz', 'video_count': 0})
self.assertDictContains(response.json['channels'][2], {'id': 1, 'name': 'Foo', 'video_count': 1})
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
@wrap_test_db
def test_channel(self):
channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name
channel_directory_name = channel_directory.split('/')[-1]
pathlib.Path(channel_directory).mkdir()
# Download is scheduled.
assert len(download_manager.get_downloads(test_session)) == 1
new_channel = dict(
directory=channel_directory,
match_regex='asdf',
name=' Example Channel 1 ',
url='https://example.com/channel1',
)
# Channel doesn't exist
request, response = api_app.test_client.get('/api/videos/channels/examplechannel1')
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
# Create it
request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.CREATED, response.json
location = response.headers['Location']
request, response = api_app.test_client.get(location)
created = response.json['channel']
self.assertIsNotNone(created)
self.assertIsNotNone(created['id'])
# Channel name leading/trailing whitespace should be stripped
assert created['name'] == 'Example Channel 1'
# Channel directory should be relative to the media directory
assert not pathlib.Path(created['directory']).is_absolute(), \
f'Channel directory is absolute: {created["directory"]}'
# Get the link that was computed
new_channel['link'] = response.json['channel']['link']
assert new_channel['link']
# Can't create it again
request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.BAD_REQUEST
# Update it
new_channel['name'] = 'Example Channel 2'
new_channel['directory'] = str(new_channel['directory'])
request, response = api_app.test_client.put(location, content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = api_app.test_client.get(location)
assert response.status_code == HTTPStatus.OK
self.assertDictContains(response.json['channel'], {
'id': 1,
'name': 'Example Channel 2',
'directory': channel_directory_name,
'match_regex': 'asdf',
'url': 'https://example.com/channel1',
})
# Can't update channel that doesn't exist
request, response = api_app.test_client.put('/api/videos/channels/doesnt_exist',
content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NOT_FOUND
# Delete the new channel
request, response = api_app.test_client.delete(location)
assert response.status_code == HTTPStatus.NO_CONTENT
# Cant delete it again
request, response = api_app.test_client.delete(location)
assert response.status_code == HTTPStatus.NOT_FOUND
@wrap_test_db
def test_refresh_no_channel(self):
with get_db_session() as session:
videos_path = pathlib.Path(self.tmp_dir.name) / 'videos'
videos_path.mkdir()
# A regular user-defined channel
channel1_path = videos_path / 'channel1'
channel1_path.mkdir()
vid1 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
vid1.touch()
vid2 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title2.mp4'
vid2.touch()
channel1 = Channel(name='channel1', link='channel1', directory=channel1_path)
session.add(channel1)
session.commit()
# The special NO CHANNEL directory.
no_channel_path = get_no_channel_directory()
vid3 = no_channel_path / 'some video.mp4'
vid3.touch()
no_channel_subdir = no_channel_path / 'subdir'
no_channel_subdir.mkdir()
vid4 = no_channel_subdir / 'other video.mp4'
vid4.touch()
api_app.test_client.post('/api/videos/refresh')
videos = session.query(Video).order_by(Video.title).all()
self.assertEqual(len(videos), 4)
# Add some fake videos, they should be removed.
session.add(Video(title='vid5'))
session.add(Video(title='vid6', channel_id=channel1.id))
session.commit()
videos = session.query(Video).order_by(Video.title).all()
self.assertEqual(len(videos), 6)
api_app.test_client.post('/api/videos/refresh')
videos = session.query(Video).order_by(Video.title).all()
self.assertEqual(len(videos), 4)
@wrap_test_db
def test_get_channel_videos(self):
with get_db_session(commit=True) as session:
channel1 = Channel(name='Foo', link='foo')
channel2 = Channel(name='Bar', link='bar')
session.add(channel1)
session.add(channel2)
session.flush()
session.refresh(channel1)
session.refresh(channel2)
# Channels don't have videos yet
d = dict(channel_link=channel1.link)
request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 0
with get_db_session(commit=True) as session:
vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo')
vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo')
session.add(vid1)
session.add(vid2)
# Videos are gotten by their respective channels
request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 1
assert response.json['totals']['videos'] == 1
self.assertDictContains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id))
d = dict(channel_link=channel2.link)
request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 1
self.assertDictContains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id))
@wrap_test_db
def test_get_video(self):
"""
Test that you get can information about a video. Test that video file can be gotten.
"""
def raise_unknown_file(_):
raise UnknownFile()
with get_db_session(commit=True) as session, \
mock.patch('modules.videos.common.get_absolute_video_info_json', raise_unknown_file):
channel = Channel(name='Foo', link='foo')
session.add(channel)
session.flush()
session.refresh(channel)
now_ = now()
session.add(Video(title='vid1', channel_id=channel.id, upload_date=now_))
session.add(Video(title='vid2', channel_id=channel.id, upload_date=now_ + timedelta(seconds=1)))
# Test that a 404 is returned when no video exists
_, response = api_app.test_client.get('/api/videos/video/10')
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''}
# Get the video info we inserted
_, response = api_app.test_client.get('/api/videos/video/1')
assert response.status_code == HTTPStatus.OK, response.json
self.assertDictContains(response.json['video'], {'title': 'vid1'})
# The next video is included.
self.assertIsNone(response.json['prev'])
self.assertDictContains(response.json['next'], {'title': 'vid2'})
@wrap_test_db
def test_get_channel_videos_pagination(self):
with get_db_session(commit=True) as session:
channel1 = Channel(name='Foo', link='foo')
session.add(channel1)
session.flush()
session.refresh(channel1)
for i in range(50):
session.add(Video(title=f'Foo.Video{i}', channel_id=channel1.id, video_path='foo'))
channel2 = Channel(name='Bar', link='bar')
session.add(channel2)
session.flush()
session.refresh(channel2)
session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo'))
# Get first, second, third, and empty pages of videos.
tests = [
# (offset, video_count)
(0, 20),
(20, 20),
(40, 10),
(50, 0),
]
last_ids = []
for offset, video_count in tests:
d = dict(channel_link=channel1.link, order_by='id', offset=offset)
_, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == video_count
current_ids = [i['id'] for i in response.json['videos']]
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
last_ids = current_ids
@wrap_test_db
def test_channel_no_download_frequency(self):
"""A channel does not require a download frequency."""
channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name
os.mkdir(channel_directory)
new_channel = dict(
directory=channel_directory,
name='Example Channel 1',
url='https://example.com/channel1',
download_frequency=None,
)
# Create the Channel
request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.CREATED
location = response.headers['Location']
# No downloads are scheduled.
with get_db_session() as session:
downloads = list(download_manager.get_new_downloads(session))
self.assertEqual(len(downloads), 0)
# Get the Channel
request, response = api_app.test_client.get(location)
channel = response.json['channel']
self.assertEqual(channel['download_frequency'], None)
# Update the Channel with a frequency.
new_channel = dict(
directory=channel['directory'],
name=channel['name'],
url=channel['url'],
download_frequency=10,
)
request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}',
content=json.dumps(new_channel))
self.assertEqual(response.status_code, 204, response.json)
# Remove the frequency.
new_channel = dict(
directory=channel['directory'],
name=channel['name'],
url=channel['url'],
download_frequency=None,
)
request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}',
content=json.dumps(new_channel))
self.assertEqual(response.status_code, 204, response.json)
# Remove the frequency.
new_channel = dict(
directory=channel['directory'],
name=channel['name'],
url=channel['url'],
download_frequency=None,
)
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
def test_channel_frequency_update(download_channel, test_client, test_session):
@ -321,7 +134,7 @@ def test_channel_frequency_update(download_channel, test_client, test_session):
url=download_channel.url,
download_frequency=100,
)
request, response = test_client.put(f'/api/videos/channels/{download_channel.link}', json=data)
request, response = test_client.put(f'/api/videos/channels/{download_channel.id}', json=data)
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
download = download_channel.get_download()
@ -332,20 +145,59 @@ def test_channel_frequency_update(download_channel, test_client, test_session):
assert len(list(downloads)) == 1
def test_delete_channel_delete_download(download_channel, test_session):
"""
Deleting a Channel deletes it's Download.
"""
downloads = test_session.query(Download).all()
assert all(i.url == download_channel.url for i in downloads)
def test_channel_download_crud(test_session, simple_channel):
"""Modifying a Channel modifies it's Download."""
assert simple_channel.url
assert simple_channel.download_frequency is None
delete_channel(download_channel.link)
simple_channel.update({'download_frequency': 1000})
test_session.commit()
download: Download = test_session.query(Download).one()
assert simple_channel.get_download() == download
assert simple_channel.url == download.url
assert simple_channel.download_frequency == download.frequency
# Increase the frequency to 1 second.
simple_channel.update({'download_frequency': 1})
test_session.commit()
assert simple_channel.url == download.url
assert simple_channel.download_frequency == download.frequency == 1
assert test_session.query(Download).count() == 1
# Changing the Channel's URL changes the download's URL.
simple_channel.update({'url': 'https://example.com/new url'})
test_session.commit()
assert simple_channel.url == 'https://example.com/new url'
assert simple_channel.download_frequency == download.frequency == 1
assert test_session.query(Download).count() == 1
# Removing the Channel's frequency removes the download.
simple_channel.update({'download_frequency': None})
test_session.commit()
assert test_session.query(Download).count() == 0
# Removing the URL is supported.
simple_channel.update({'url': None})
test_session.commit()
# Adding a URL is supported.
simple_channel.update({'url': 'https://example.com'})
test_session.commit()
def test_delete_channel_delete_download(download_channel, test_session):
"""Deleting a Channel deletes it's Download."""
downloads = test_session.query(Download).all()
assert downloads and all(i.url == download_channel.url for i in downloads)
delete_channel(channel_id=download_channel.id)
downloads = test_session.query(Download).all()
assert not list(downloads)
def test_video_search(test_session):
def test_video_search(test_client, test_session):
"""
Test that videos can be searched and that their order is by their textsearch rank.
"""
@ -363,7 +215,7 @@ def test_video_search(test_session):
def do_search(search_str, limit=20):
d = json.dumps({'search_str': search_str, 'limit': limit})
_, resp = api_app.test_client.post('/api/videos/search', content=d)
_, resp = test_client.post('/api/videos/search', content=d)
return resp
def search_is_as_expected(resp, expected):
@ -414,7 +266,7 @@ def test_video_file_name(test_session, simple_video, test_client):
assert resp.json['video'].get('stem') == 'simple_video'
def test_channel_conflicts(test_session, test_directory):
def test_channel_conflicts(test_client, test_session, test_directory):
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory).mkdir()
new_channel = dict(
@ -425,7 +277,7 @@ def test_channel_conflicts(test_session, test_directory):
)
def _post_channel(channel):
return api_app.test_client.post('/api/videos/channels', content=json.dumps(channel))
return test_client.post('/api/videos/channels', content=json.dumps(channel))
# Create it
request, response = _post_channel(new_channel)
@ -443,18 +295,6 @@ def test_channel_conflicts(test_session, test_directory):
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
'cause': {'error': 'The channel name is already taken.', 'code': 5}}
# Name matches when converted to link
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory2).mkdir()
new_channel = dict(
directory=channel_directory2,
name='Example channel 1',
)
request, response = _post_channel(new_channel)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
'cause': {'code': 11, 'error': 'Channel link already used by another channel'}}
# Directory was already used
new_channel = dict(
directory=channel_directory,
@ -479,7 +319,7 @@ def test_channel_conflicts(test_session, test_directory):
'cause': {'code': 6, 'error': 'The URL is already used by another channel.'}}
def test_channel_empty_url_doesnt_conflict(test_session, test_directory):
def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_directory):
"""Two channels with empty URLs shouldn't conflict"""
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory).mkdir()
@ -488,7 +328,7 @@ def test_channel_empty_url_doesnt_conflict(test_session, test_directory):
'name': 'Fooz',
'directory': channel_directory,
}
request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.CREATED, response.json
location = response.headers['Location']
@ -498,7 +338,7 @@ def test_channel_empty_url_doesnt_conflict(test_session, test_directory):
'name': 'Barz',
'directory': channel_directory2,
}
request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.CREATED, response.json
assert location != response.headers['Location']
@ -524,11 +364,11 @@ def test_download_channel_no_refresh(test_session, download_channel, video_downl
def test_channel_post_directory(test_session, test_client, test_directory):
"""A Channel can be created with or without an existing directory."""
# Channel can be created with a missing directory.
# Channel can be created with a directory which is not on disk.
data = dict(name='foo', directory='foo')
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
assert response.status_code == HTTPStatus.CREATED
directory = test_session.query(Channel).filter_by(link='foo').one().directory.path
directory = test_session.query(Channel).filter_by(id=1).one().directory.path
assert (test_directory / 'foo') == directory
assert not directory.is_dir()
assert directory.is_absolute()
@ -537,56 +377,147 @@ def test_channel_post_directory(test_session, test_client, test_directory):
data = dict(name='bar', directory='bar', mkdir=True)
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
assert response.status_code == HTTPStatus.CREATED
directory = test_session.query(Channel).filter_by(link='bar').one().directory.path
directory = test_session.query(Channel).filter_by(id=2).one().directory.path
assert (test_directory / 'bar') == directory
assert directory.is_dir()
assert directory.is_absolute()
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory):
subdir = test_directory / 'subdir'
subdir.mkdir()
def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
request, response = test_client.get(f'/api/videos/channels/{simple_channel.id}')
assert response.status_code == HTTPStatus.OK
# video1 is in a subdirectory.
video1 = video_factory(simple_channel.id, True, True, 'jpg')
test_session.commit()
shutil.move(video1.video_path.path, subdir / video1.video_path.path.name)
shutil.move(video1.poster_path.path, subdir / video1.poster_path.path.name)
video1.video_path = subdir / video1.video_path.path.name
video1.poster_path = None
# video2 is in the test directory.
video2 = video_factory(simple_channel.id, True, True, 'jpg')
video2.poster_path = video1.poster_path = None
def test_channel_crud(test_session, test_client, test_directory):
channel_directory = test_directory / 'channel directory'
channel_directory.mkdir()
new_channel = dict(
directory=str(channel_directory),
match_regex='asdf',
name=' Example Channel 1 ',
url='https://example.com/channel1',
)
# Channel doesn't exist
request, response = test_client.get('/api/videos/channels/examplechannel1')
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
# Create it
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.CREATED, response.json
location = response.headers['Location']
request, response = test_client.get(location)
assert response.status_code == HTTPStatus.OK, response.json
created = response.json['channel']
assert created
assert created['id']
# No frequency was provided. No download exists.
assert not created['download_frequency']
assert test_session.query(Download).count() == 0
# Channel name leading/trailing whitespace should be stripped
assert created['name'] == 'Example Channel 1'
# Channel directory should be relative to the media directory
assert not pathlib.Path(created['directory']).is_absolute(), \
f'Channel directory is absolute: {created["directory"]}'
# Can't create it again
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.BAD_REQUEST
# Update it
new_channel['name'] = 'Example Channel 2'
new_channel['directory'] = str(new_channel['directory']) # noqa
request, response = test_client.put(location, content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = test_client.get(location)
assert response.status_code == HTTPStatus.OK
channel = response.json['channel']
assert channel['id'] == 1
assert channel['name'] == 'Example Channel 2'
assert channel['directory'] == channel_directory.name
assert channel['match_regex'] == 'asdf'
assert channel['url'] == 'https://example.com/channel1'
# Can't update channel that doesn't exist
request, response = test_client.put('/api/videos/channels/doesnt_exist',
content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NOT_FOUND
# Delete the new channel
request, response = test_client.delete(location)
assert response.status_code == HTTPStatus.NO_CONTENT
# Cant delete it again
request, response = test_client.delete(location)
assert response.status_code == HTTPStatus.NOT_FOUND
def test_search_channel_videos(test_client, test_session):
with get_db_session(commit=True) as session:
channel1 = Channel(name='Foo')
channel2 = Channel(name='Bar')
session.add(channel1)
session.add(channel2)
session.flush()
session.refresh(channel1)
session.refresh(channel2)
# Channels don't have videos yet
d = dict(channel_id=channel1.id)
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 0
with get_db_session(commit=True) as session:
vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo')
vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo')
session.add(vid1)
session.add(vid2)
# Videos are gotten by their respective channels
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 1
assert response.json['totals']['videos'] == 1
assert_dict_contains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id))
d = dict(channel_id=channel2.id)
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == 1
assert_dict_contains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id))
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory):
for i in range(50):
video_factory(channel_id=simple_channel.id)
channel2 = Channel(name='Bar')
test_session.add(channel2)
test_session.flush()
test_session.refresh(channel2)
video_factory(channel_id=channel2.id)
test_session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo'))
test_session.commit()
assert not video1.size, 'video1 should not have size during creation'
# Create a video not in the DB.
vid3 = test_directory / 'vid3.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', vid3)
# Orphan poster should be ignored.
orphan_poster = pathlib.Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg')
orphan_poster.touch()
# Create a bogus video in the channel.
bogus = Video(video_path='foo', channel_id=simple_channel.id)
test_session.add(bogus)
test_client.post('/api/videos/refresh')
# Posters were found during refresh.
assert video1.poster_path
assert 'subdir' in str(video1.poster_path.path)
assert video2.poster_path
# Missing video3 was found
video3: Video = test_session.query(Video).filter_by(id=4).one()
assert video3.video_path.path == vid3
assert video3.video_path != video1.video_path
assert video3.video_path != video2.video_path
# Bogus video was removed.
assert not any('foo' in str(i.video_path.path) for i in test_session.query(Video).all())
# Orphan file was not deleted.
assert orphan_poster.is_file(), 'Orphan poster was removed!'
assert video1.size, 'video1 size was not found'
# Get first, second, third, and empty pages of videos.
tests = [
# (offset, video_count)
(0, 20),
(20, 20),
(40, 10),
(50, 0),
]
last_ids = []
for offset, video_count in tests:
d = dict(channel_id=simple_channel.id, order_by='id', offset=offset)
_, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
assert response.status_code == HTTPStatus.OK
assert len(response.json['videos']) == video_count
current_ids = [i['id'] for i in response.json['videos']]
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
last_ids = current_ids

View file

@ -0,0 +1,59 @@
import pytest
from modules.videos.channel import lib
from wrolpi.errors import UnknownChannel
@pytest.mark.parametrize('params', [
dict(),
dict(channel_id=None),
dict(source_id=None),
dict(url=None),
dict(directory=None),
])
def test_get_bad_channel(test_session, test_directory, params):
"""A Channel cannot be found without data."""
with pytest.raises(UnknownChannel):
lib.get_channel(**params)
def test_get_channel(test_session, test_directory, channel_factory):
"""`get_channel` can find a Channel in multiple ways."""
channel_id_channel = channel_factory()
source_id_channel = channel_factory()
source_id_channel.source_id = 'the source id'
url_channel = channel_factory()
url_channel.url = 'https://example.com/channel'
directory_channel = channel_factory()
directory_channel.directory = test_directory / 'some directory'
test_session.commit()
channel = lib.get_channel(channel_id=channel_id_channel.id, return_dict=False)
assert channel == channel_id_channel
channel = lib.get_channel(source_id='the source id', return_dict=False)
assert channel == source_id_channel
channel = lib.get_channel(url='https://example.com/channel', return_dict=False)
assert channel == url_channel
# Directory can be relative, or absolute.
channel = lib.get_channel(directory='some directory', return_dict=False)
assert channel == directory_channel
channel = lib.get_channel(directory=f'{test_directory}/some directory', return_dict=False)
assert channel == directory_channel
# Channel is found using the priority of params, incorrect data is ignored.
params = [
dict(channel_id=channel_id_channel.id, directory='bad directory'),
dict(channel_id=channel_id_channel.id, directory='some directory'),
dict(channel_id=channel_id_channel.id, source_id='bad source id'),
dict(channel_id=channel_id_channel.id, source_id='the source id'),
]
for p in params:
assert lib.get_channel(**p, return_dict=False) == channel_id_channel
assert lib.get_channel(**p)

View file

@ -10,16 +10,13 @@ from typing import Union, Tuple, List, Set, Iterable, Optional
import PIL
from PIL import Image
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from wrolpi import before_startup
from wrolpi.cmd import which
from wrolpi.common import logger, iterify, get_media_directory, \
minimize_dict, api_param_limiter
minimize_dict, any_extensions
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.downloader import download_manager
from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \
ChannelLinkConflict, ChannelDirectoryConflict, ChannelSourceIdConflict
from wrolpi.errors import UnknownFile, ChannelNameConflict, ChannelURLConflict, \
ChannelDirectoryConflict, ChannelSourceIdConflict
from wrolpi.media_path import MediaPath
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
from .models import Channel, Video
@ -27,27 +24,21 @@ from .models import Channel, Video
logger = logger.getChild(__name__)
REQUIRED_OPTIONS = ['name', 'directory']
MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count', 'link'}
MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count'}
MINIMUM_INFO_JSON_KEYS = {'description', 'view_count', 'webpage_url'}
MINIMUM_VIDEO_KEYS = {'id', 'title', 'upload_date', 'duration', 'channel', 'channel_id', 'favorite', 'size',
'poster_path', 'caption_path', 'video_path', 'info_json', 'channel', 'viewed', 'source_id',
'view_count'}
VIDEO_QUERY_LIMIT = 20
VIDEO_QUERY_MAX_LIMIT = 100
# These are the supported video formats. These are in order of their preference.
VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv')
video_limiter = api_param_limiter(100)
class ConfigError(Exception):
pass
def get_videos_directory() -> pathlib.Path:
"""
Get the "videos" directory in the media directory. Make it if it does not exist.
"""
"""Get the "videos" directory in the media directory. Make it if it does not exist."""
directory = get_media_directory() / 'videos'
if not directory.is_dir():
directory.mkdir(parents=True)
@ -55,89 +46,13 @@ def get_videos_directory() -> pathlib.Path:
def get_no_channel_directory() -> pathlib.Path:
"""
Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist.
"""
"""Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist."""
directory = get_videos_directory() / 'NO CHANNEL'
if not directory.is_dir():
directory.mkdir(parents=True)
return directory
@before_startup
def import_videos_config():
"""Import channel settings to the DB. Existing channels will be updated."""
logger.info('Importing videos config')
try:
from .lib import get_channels_config
from .downloader import ChannelDownloader
config = get_channels_config()
config, favorites = config.channels, config.favorites
with get_db_session(commit=True) as session:
for link in config:
for option in (i for i in REQUIRED_OPTIONS if i not in config[link]):
raise ConfigError(f'Channel "{link}" is required to have "{option}"')
name, directory = config[link]['name'], config[link]['directory']
channel = session.query(Channel).filter(Channel.link == link).one_or_none()
if not channel:
# Channel not yet in the DB, add it.
channel = Channel(link=link)
# Only name and directory are required
channel.name = name
channel.directory = str(directory)
channel.calculate_duration = config[link].get('calculate_duration') or channel.calculate_duration
channel.download_frequency = config[link].get('download_frequency') or channel.download_frequency
channel.generate_posters = config[link].get('generate_posters') or channel.generate_posters
channel.match_regex = config[link].get('match_regex') or channel.match_regex
channel.skip_download_videos = list(set(config[link].get('skip_download_videos', {})))
channel.source_id = config[link].get('source_id') or channel.source_id
channel.url = config[link].get('url') or channel.url
if not channel.source_id and channel.url:
# If we can download from a channel, we must have its source_id.
channel.source_id = get_channel_source_id(channel.url)
if channel.download_frequency and channel.url:
download = download_manager.get_or_create_download(channel.url, session=session)
download.frequency = channel.download_frequency
download.downloader = ChannelDownloader.name
download.next_download = download.next_download or \
download_manager.get_next_download(download, session=session)
session.add(channel)
with get_db_session(commit=True) as session:
for link, favorites in favorites.items():
if link != 'NO CHANNEL':
channel: Channel = session.query(Channel).filter_by(link=link).one_or_none()
if not channel:
logger.warning(f'Cannot find channel {link=} for favorites!')
continue
channel_dir = channel.directory.path
else:
from .downloader import get_no_channel_directory
channel_dir = get_no_channel_directory()
# Set favorite Videos of this Channel.
for video_path, data in favorites.items():
# Favorite in the config is the name of the video_path. Add the channel directory onto this
# video_path, so we can match the complete path for the Video.
video_path = channel_dir / video_path
video = session.query(Video).filter_by(video_path=video_path).one_or_none()
# If no Video is found, it may be that we need to refresh.
if video:
video.favorite = data['favorite']
else:
logger.warning(f'Cannot find video to favorite: {video_path}')
except Exception as e:
logger.warning('Failed to load channels config!', exc_info=e)
VALID_VIDEO_KINDS = {'video', 'caption', 'poster', 'description', 'info_json'}
@ -150,66 +65,11 @@ def get_absolute_video_path(video: Video, kind: str = 'video') -> MediaPath:
raise UnknownFile()
get_absolute_video_caption = partial(get_absolute_video_path, kind='caption')
get_absolute_video_poster = partial(get_absolute_video_path, kind='poster')
get_absolute_video_description = partial(get_absolute_video_path, kind='description')
get_absolute_video_info_json = partial(get_absolute_video_path, kind='info_json')
def get_absolute_video_files(video: Video) -> List[MediaPath]:
"""
Get all video files that exist.
"""
getters = [
get_absolute_video_description,
get_absolute_video_caption,
get_absolute_video_poster,
get_absolute_video_info_json,
get_absolute_video_path,
]
def _get():
for getter in getters:
try:
yield getter(video)
except UnknownFile:
pass
return list(_get())
def get_video_info_json(video) -> Optional[dict]:
"""Get the info_json object from a video's meta-file. Return an empty dict if not possible."""
if not video.channel or not video.channel.directory:
return
try:
path = get_absolute_video_info_json(video)
if path.exists():
with open(str(path), 'rb') as fh:
contents = json.load(fh)
return contents
except UnknownFile:
return
except UnknownDirectory:
return
def any_extensions(filename: str, extensions: Iterable = ()):
"""
Return True only if the file name ends with any of the possible extensions.
Matches lower or upper case of the extension.
"""
return any(filename.lower().endswith(ext) for ext in extensions)
match_video_extensions = partial(any_extensions, extensions=VIDEO_EXTENSIONS)
def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, pathlib.Path]:
"""
Generate a list of video paths in the provided directory.
"""
"""Generate a list of video paths in the provided directory."""
directory = pathlib.Path(directory)
for child in directory.iterdir():
@ -223,9 +83,11 @@ def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, path
@iterify(set)
def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]:
"""
Remove any duplicate paths from a given list. Duplicate is defined as any file that shares the EXACT same
name as another file, but with a different extension. If a duplicate is found, only yield one of the paths. The
path that will be yielded will be whichever is first in the VIDEO_EXTESIONS tuple.
Remove any duplicate paths from a given list.
Duplicate is defined as any file that shares the EXACT same name as another file, but with a different extension.
If a duplicate is found, only yield one of the paths. The path that will be yielded will be whichever is first in
the VIDEO_EXTESIONS tuple.
i.e.
>>> remove_duplicate_video_paths([Path('one.mp4'), Path('two.mp4'), Path('one.ogg')])
@ -262,15 +124,15 @@ def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]:
yield sorted(paths)[0]
def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, link=None, directory=None,
def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, directory=None,
source_id=None):
"""
Search for any channels that conflict with the provided args, raise a relevant exception if any conflicts are found.
"""
if not any([id_, url, name, link, directory]):
if not any([id_, url, name, directory]):
raise Exception('Cannot search for channel with no arguments')
logger.debug(f'Checking for channel conflicts: id={id_} url={url} name={name} link={link} directory={directory}')
logger.debug(f'Checking for channel conflicts: id={id_} url={url} name={name} directory={directory}')
# A channel can't conflict with itself
if id_:
@ -286,10 +148,6 @@ def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None,
conflicts = base_where.filter(Channel.name == name)
if list(conflicts):
raise ChannelNameConflict()
if link:
conflicts = base_where.filter(Channel.link == link)
if list(conflicts):
raise ChannelLinkConflict()
if directory:
conflicts = base_where.filter(Channel.directory == directory)
if list(conflicts):
@ -329,29 +187,13 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]:
return paths
def replace_extension(path: pathlib.Path, new_ext) -> pathlib.Path:
"""Swap the extension of a file's path.
Example:
>>> foo = pathlib.Path('foo.bar')
>>> replace_extension(foo, 'baz')
'foo.baz'
"""
parent = path.parent
existing_ext = path.suffix
path = str(path)
name, _, _ = path.rpartition(existing_ext)
path = pathlib.Path(str(parent / name) + new_ext)
return path
def generate_video_poster(video_path: Path) -> Path:
"""
Create a poster (aka thumbnail) next to the provided video_path.
"""
poster_path = replace_extension(video_path, '.jpg')
cmd = ['/usr/bin/ffmpeg', '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000',
str(poster_path)]
poster_path = video_path.with_suffix('.jpg')
cmd = ('/usr/bin/ffmpeg', '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000',
str(poster_path))
try:
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logger.info(f'Generated poster at {poster_path}')
@ -474,13 +316,3 @@ def minimize_video(video: dict) -> dict:
video['info_json'] = minimize_video_info_json(video['info_json'])
return video
YDL = YoutubeDL()
YDL.params['logger'] = logger.getChild('youtube-dl')
YDL.add_default_info_extractors()
def get_channel_source_id(url: str) -> str:
channel_info = YDL.extract_info(url, download=False, process=False)
return channel_info.get('uploader_id') or channel_info['id']

View file

@ -10,22 +10,18 @@ from PIL import Image
from modules.videos.downloader import VideoDownloader, ChannelDownloader
from modules.videos.lib import set_test_channels_config, set_test_downloader_config
from modules.videos.models import Channel, Video
from wrolpi.common import sanitize_link
from wrolpi.downloader import DownloadFrequency, DownloadManager, Download
from wrolpi.vars import PROJECT_DIR
@pytest.fixture
def simple_channel(test_session, test_directory) -> Channel:
"""
Get a Channel with the minimum properties. This Channel has no download!
"""
"""Get a Channel with the minimum properties. This Channel has no download!"""
channel = Channel(
directory=test_directory,
name='Simple Channel',
url='https://example.com/channel1',
download_frequency=None, # noqa
link='simplechannel',
)
test_session.add(channel)
test_session.commit()
@ -34,20 +30,18 @@ def simple_channel(test_session, test_directory) -> Channel:
@pytest.fixture
def channel_factory(test_session, test_directory):
"""
Create a random Channel with a directory, but no frequency.
"""
"""Create a random Channel with a directory, but no frequency."""
def _():
name = str(uuid4())
def _(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None):
name = name or str(uuid4())
directory = test_directory / name
directory.mkdir()
channel = Channel(
directory=directory, # noqa
name=name,
url=f'https://example.com/{name}',
download_frequency=None, # noqa
link=sanitize_link(name),
url=url or f'https://example.com/{name}',
download_frequency=download_frequency,
source_id=source_id,
)
test_session.add(channel)
test_session.commit()
@ -67,7 +61,6 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch
name='Download Channel',
url='https://example.com/channel1',
download_frequency=DownloadFrequency.weekly,
link='downloadchannel',
)
download = Download(url=channel.url, downloader='video_channel', frequency=channel.download_frequency)
test_session.add_all([channel, download])
@ -77,9 +70,7 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch
@pytest.fixture
def simple_video(test_session, test_directory, simple_channel) -> Video:
"""
A Video with a video file whose channel is the Simple Channel.
"""
"""A Video with an empty video file whose channel is the Simple Channel."""
video_path = test_directory / 'simple_video.mp4'
video_path.touch()
video = Video(video_path=video_path, channel_id=simple_channel.id)
@ -103,9 +94,9 @@ def video_file(test_directory) -> pathlib.Path:
def video_factory(test_session, test_directory):
"""Creates Videos for testing."""
def _(channel_id: int = None, with_video_file: bool = False, with_info_json: dict = None,
with_poster_ext: str = None):
title = str(uuid4())
def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file: bool = False,
with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False):
title = title or str(uuid4())
if channel_id:
path = test_directory / f'{title}.mp4'
else:
@ -129,12 +120,18 @@ def video_factory(test_session, test_directory):
poster_path = path.with_suffix(f'.{with_poster_ext}')
Image.new('RGB', (25, 25)).save(poster_path)
caption_path = None
if with_caption_file:
caption_path = path.with_suffix('.en.vtt')
caption_path.touch()
video = Video(video_path=str(path), title=title, channel_id=channel_id, source_id=title,
info_json_path=info_json_path, poster_path=poster_path)
info_json_path=info_json_path, poster_path=poster_path, caption_path=caption_path,
upload_date=upload_date)
test_session.add(video)
return video
return _
return factory
@pytest.fixture

View file

@ -6,23 +6,23 @@ from abc import ABC
from typing import Tuple, List, Optional
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound
from yt_dlp import YoutubeDL
from yt_dlp.extractor import YoutubeTabIE
from yt_dlp.utils import UnsupportedError, DownloadError
from wrolpi.cmd import which
from wrolpi.common import logger, sanitize_link, extract_domain
from wrolpi.common import logger, extract_domain, get_media_directory
from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import apply_info_json, get_channel_source_id, get_no_channel_directory, \
from .common import apply_info_json, get_no_channel_directory, \
get_videos_directory
from .lib import upsert_video, refresh_channel_videos, get_downloader_config
from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id
from .models import Video, Channel
from .schema import ChannelPostRequest
from .video_url_resolver import video_url_resolver
logger = logger.getChild(__name__)
@ -70,12 +70,12 @@ class ChannelDownloader(Downloader, ABC):
if not channel:
# Couldn't get channel by URL, this is probably a playlist. Find the channel for the playlist.
channel_source_id = get_channel_source_id(download.url)
channel = get_channel(channel_source_id, return_dict=False)
channel = get_channel(source_id=channel_source_id, return_dict=False)
update_channel(channel)
domain = extract_domain(download.url)
missing_videos = find_all_missing_videos(channel.link)
missing_videos = find_all_missing_videos(channel.id)
with get_db_session(commit=True) as session:
for video_id, source_id, missing_video in missing_videos:
url = video_url_resolver(domain, missing_video)
@ -152,7 +152,7 @@ class VideoDownloader(Downloader, ABC):
channel = None
if channel_name or channel_id:
channel_url = info.get('channel_url')
channel = get_or_create_channel(channel_id, None, channel_url, channel_name)
channel = get_or_create_channel(source_id=channel_id, url=channel_url, name=channel_name)
# Use the default directory if this video has no channel.
out_dir = get_no_channel_directory()
@ -245,34 +245,32 @@ channel_downloader = ChannelDownloader()
video_downloader = VideoDownloader(40)
def get_or_create_channel(source_id: str = None, link: str = None, url: str = None, name: str = None) -> Channel:
def get_or_create_channel(source_id: str = None, url: str = None, name: str = None) -> Channel:
"""
Attempt to find a Channel using the provided params. The params are in order of reliability.
Creates a new Channel if one cannot be found.
"""
if not link and name:
link = sanitize_link(name)
try:
channel = get_channel(link=link, source_id=source_id, url=url, return_dict=False)
channel = get_channel(source_id=source_id, url=url, name=name, return_dict=False)
return channel
except UnknownChannel:
pass
if not name:
raise ValueError(f'Cannot create channel without a name')
# Channel does not exist. Create one in the video directory.
channel_directory = get_videos_directory() / name
if not channel_directory.is_dir():
channel_directory.mkdir(parents=True)
data = dict(
data = ChannelPostRequest(
source_id=source_id,
name=name,
url=url,
link=link,
domain=extract_domain(url),
directory=str(channel_directory),
source_id=source_id,
directory=str(channel_directory.relative_to(get_media_directory())),
)
channel = create_channel(data, return_dict=False)
channel = create_channel(data=data, return_dict=False)
# Create the directory now that the channel is approved.
channel_directory.mkdir(exist_ok=True)
@ -348,21 +346,17 @@ def update_channel(channel: Channel):
apply_info_json(channel_id)
def _find_all_missing_videos(link: str = None) -> List[Tuple]:
def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]:
"""Get all Video entries which don't have the required media files (i.e. hasn't been downloaded).
Restrict to a single channel if "link" is provided.
Restrict to a single channel if `channel_id` is provided.
"""
with get_db_curs() as curs:
# Get all channels by default.
where = ''
params = ()
if link:
# Restrict by channel when link is provided.
query = 'SELECT id FROM channel WHERE link = %s'
curs.execute(query, (link,))
channel_id = curs.fetchall()[0][0]
if channel_id:
where = 'AND channel_id = %s'
params = (channel_id,)
@ -386,18 +380,15 @@ def _find_all_missing_videos(link: str = None) -> List[Tuple]:
return missing_videos
def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
def find_all_missing_videos(channel_id: str = None) -> Tuple[dict, dict]:
"""
Find all videos that don't have a video file, but are found in the DB (taken from the channel's info_json).
Yields a Channel Dict object, our Video id, and the "entry" of the video from the channel's info_json['entries'].
"""
with get_db_session() as session:
if link:
try:
channel = session.query(Channel).filter_by(link=link).one()
except NoResultFound:
raise UnknownChannel(f'No channel with link: {link}')
if channel_id:
channel = get_channel(channel_id=channel_id, return_dict=False)
if not channel.url:
raise ChannelURLEmpty('No URL for this channel')
channels = [channel, ]
@ -420,12 +411,12 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
# Convert the channel video entries into a form that allows them to be quickly retrieved without searching through
# the entire entries list.
channels_entries = {}
for id_, channel in channels.items():
channels_entries[id_] = {i['id']: i for i in channel.info_json['entries']}
for channel_id_, channel in channels.items():
channels_entries[channel_id_] = {i['id']: i for i in channel.info_json['entries']}
missing_videos = _find_all_missing_videos(link)
missing_videos = _find_all_missing_videos(channel_id)
for id_, source_id, channel_id in missing_videos:
for video_id, source_id, channel_id in missing_videos:
channel = channels[channel_id]
if channel.skip_download_videos and source_id in channel.skip_download_videos:
@ -435,13 +426,13 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
try:
missing_video = channels_entries[channel_id][source_id]
except KeyError:
logger.warning(f'Video {id_} / {source_id} is not in {channel.name} info_json')
logger.warning(f'Video {channel.name} / {source_id} is not in {channel.name} info_json')
continue
match_regex: re.compile = match_regexen.get(channel_id)
if not match_regex or (match_regex and missing_video['title'] and match_regex.match(missing_video['title'])):
# No title match regex, or the title matches the regex.
yield id_, source_id, missing_video
yield video_id, source_id, missing_video
UNRECOVERABLE_ERRORS = {

View file

@ -2,20 +2,23 @@ import html
import pathlib
import re
from collections import defaultdict
from typing import Tuple, Optional, List
from typing import Tuple, Optional, List, Union
from uuid import uuid1
from sqlalchemy import or_
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from wrolpi.common import logger, chunks, ConfigFile
from wrolpi import before_startup
from wrolpi.common import chunks, ConfigFile, get_media_directory, sanitize_link
from wrolpi.dates import from_timestamp, Seconds
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.media_path import MediaPath
from wrolpi.vars import PYTEST
from .captions import get_captions
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, import_videos_config, \
get_video_duration, is_valid_poster, convert_image, generate_video_poster
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \
is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError
from .models import Channel, Video
logger = logger.getChild(__name__)
@ -66,8 +69,9 @@ def refresh_channel_videos(channel: Channel):
logger.info(f'{channel.name}: {len(new_videos)} new videos, {len(existing_paths)} already existed. ')
channel.refreshed = True
Session.object_session(channel).commit()
with get_db_session(commit=True) as session:
channel = session.query(Channel).filter_by(id=channel.id).one()
channel.refreshed = True
apply_info_json(channel.id)
@ -155,7 +159,7 @@ def validate_videos():
video.validated = True
except Exception as e:
# This video failed to validate, continue validation for the rest of the videos.
logger.warning(f'Failed to validate {video=}', exc_info=e)
logger.warning(f'Failed to validate {video}', exc_info=e)
def validate_video(video: Video, channel_generate_poster: bool):
@ -246,7 +250,7 @@ def convert_or_generate_poster(video: Video) -> Optional[pathlib.Path]:
logger.error(f'Failed to generate poster for {video}', exc_info=e)
def refresh_videos(channel_links: list = None):
def refresh_videos(channel_ids: List[int] = None):
"""
Find any videos in the channel directories and add them to the DB. Delete DB records of any videos not in the
file system.
@ -257,15 +261,15 @@ def refresh_videos(channel_links: list = None):
"""
logger.info('Refreshing video files')
with get_db_session() as session:
if channel_links:
channels = session.query(Channel).filter(Channel.link.in_(channel_links))
if channel_ids:
channels = session.query(Channel).filter(Channel.id.in_(channel_ids))
else:
channels = session.query(Channel).all()
channels = list(channels)
if not channels and channel_links:
raise Exception(f'No channels match links(s): {channel_links}')
if not channels and channel_ids:
raise Exception(f'No channels match id(s): {channel_ids}')
elif not channels:
raise Exception(f'No channels in DB. Have you created any?')
@ -276,13 +280,13 @@ def refresh_videos(channel_links: list = None):
logger.fatal(f'Failed to refresh videos for channel {channel.name}!', exc_info=e)
pass
if not channel_links:
if not channel_ids:
# Refresh NO CHANNEL videos when not refreshing a specific channel.
refresh_no_channel_videos()
# Fill in any missing data for all videos.
if not PYTEST:
import_videos_config()
import_channels_config()
validate_videos()
@ -460,8 +464,8 @@ def set_test_downloader_config(enabled: bool):
def get_channels_config_from_db(session: Session) -> dict:
"""Create a dictionary that contains all the Channels from the DB."""
channels = session.query(Channel).order_by(Channel.link).all()
channels = {i.link: i.config_view() for i in channels}
channels = session.query(Channel).order_by(Channel.directory).all()
channels = sorted((i.config_view() for i in channels), key=lambda i: i['directory'])
# Get all Videos that are favorites. Store them in their own config section, so they can be preserved if a channel
# is deleted or the DB is wiped.
@ -469,7 +473,7 @@ def get_channels_config_from_db(session: Session) -> dict:
favorites = defaultdict(lambda: {})
for video in favorite_videos:
if video.channel:
favorites[video.channel.link][video.video_path.path.name] = dict(favorite=video.favorite)
favorites[str(video.channel.directory.relative)][video.video_path.path.name] = dict(favorite=video.favorite)
else:
favorites['NO CHANNEL'][video.video_path.path.name] = dict(favorite=video.favorite)
favorites = dict(favorites)
@ -477,18 +481,128 @@ def get_channels_config_from_db(session: Session) -> dict:
return dict(channels=channels, favorites=favorites)
def _detect_old_favorites(config: Union[ChannelsConfig, dict]) -> bool:
# TODO remove these old favorites after beta.
media_directory = get_media_directory()
favorites = config.favorites if hasattr(config, 'favorites') else config['favorites']
for channel, favorites in favorites.items():
if channel == 'NO CHANNEL':
continue
if not (media_directory / channel).is_dir():
return True
return False
@optional_session()
def save_channels_config(session=None, preserve_favorites: bool = True):
"""
Pull the Channel information from the DB, save it to the config.
"""
"""Get the Channel information from the DB, save it to the config."""
config = get_channels_config_from_db(session)
channels_config = get_channels_config()
# TODO remove these old favorites after beta.
old_favorites = _detect_old_favorites(channels_config)
if old_favorites and config['favorites']:
# There are favorites in the database, remove the old favorites.
channels_config.favorites = {}
if preserve_favorites:
config['favorites'].update(channels_config.favorites or {})
channels_config.update(config)
@before_startup
def import_channels_config():
"""Import channel settings to the DB. Existing channels will be updated."""
logger.info('Importing videos config')
try:
from .downloader import ChannelDownloader
config = get_channels_config()
channels, favorites = config.channels, config.favorites
with get_db_session(commit=True) as session:
for data in channels:
if isinstance(data, str):
# Outdated style config.
# TODO remove this after beta.
data = channels[data]
# A Channel's directory is saved (in config) relative to the media directory.
directory = get_media_directory() / data['directory']
for option in (i for i in REQUIRED_OPTIONS if i not in data):
raise ConfigError(f'Channel "{directory}" is required to have "{option}"')
channel = session.query(Channel).filter(
or_(
Channel.directory == str(directory),
Channel.directory == str(data['directory']),
Channel.directory == str(directory.relative_to(get_media_directory())),
),
).one_or_none()
if not channel:
# Channel not yet in the DB, add it.
channel = Channel(directory=directory)
session.add(channel)
# Only name and directory are required
channel.name = data['name']
channel.directory = directory
# Copy existing channel data, update all values from the config. This is necessary to clear out
# values not in the config.
full_data = channel.dict()
full_data.update(data)
print(f'{full_data=}')
full_data['skip_download_videos'] = list(set(data.get('skip_download_videos', {})))
channel.update(full_data)
if not channel.source_id and channel.url:
# If we can download from a channel, we must have its source_id.
channel.source_id = get_channel_source_id(channel.url)
with get_db_session(commit=True) as session:
channels_by_link = {sanitize_link(i.name): i for i in session.query(Channel)}
for directory_, favorites in favorites.items():
if directory_ != 'NO CHANNEL':
# A Channel's directory is saved (in the config) relative to the media directory.
directory = get_media_directory() / directory_
channel: Channel = session.query(Channel).filter_by(directory=directory).one_or_none()
if not channel:
# Directory may be the outdated "link".
# TODO remove these old favorites after beta.
channel = channels_by_link.get(directory_)
if not channel:
logger.warning(f'Cannot find channel {directory=} for favorites!')
continue
channel_dir = channel.directory.path
else:
from .downloader import get_no_channel_directory
channel_dir = get_no_channel_directory()
# Set favorite Videos of this Channel.
for video_path, data in favorites.items():
# Favorite in the config is the name of the video_path. Add the channel directory onto this
# video_path, so we can match the complete path for the Video.
video_path = channel_dir / video_path
video = session.query(Video).filter_by(video_path=video_path).one_or_none()
# If no Video is found, it may be that we need to refresh.
if video:
video.favorite = data['favorite']
else:
logger.warning(f'Cannot find video to favorite: {video_path}')
except Exception as e:
logger.warning('Failed to load channels config!', exc_info=e)
if PYTEST:
# Do not interrupt startup, only raise during testing.
raise
YDL = YoutubeDL()
YDL.params['logger'] = logger.getChild('youtube-dl')
YDL.add_default_info_extractors()
def get_channel_source_id(url: str) -> str:
channel_info = YDL.extract_info(url, download=False, process=False)
return channel_info.get('uploader_id') or channel_info['id']
async def get_statistics():
with get_db_curs() as curs:
curs.execute('''
@ -608,7 +722,6 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
video.info_json_path = info_json_path
video.poster_path = poster_path
video.video_path = video_path
if channel and not str(video_path).startswith(str(channel.directory.path)):
raise ValueError(f'Video path is not within its channel {video_path=} not in {channel.directory=}')

View file

@ -1,5 +1,4 @@
import json
import pathlib
from datetime import datetime
from pathlib import Path
from typing import Optional, Generator
@ -12,7 +11,7 @@ from wrolpi.common import Base, tsvector, ModelHelper, logger, get_media_directo
from wrolpi.dates import now, TZDateTime
from wrolpi.db import get_db_curs
from wrolpi.downloader import Download, download_manager
from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory, InvalidDownload
from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory
from wrolpi.media_path import MediaPathType, MediaPath
logger = logger.getChild(__name__)
@ -116,9 +115,7 @@ class Video(ModelHelper, Base):
save_channels_config(preserve_favorites=False)
def add_to_skip_list(self):
"""
Add this video to it's Channel's skip list.
"""
"""Add this video to it's Channel's skip list."""
if self.channel and self.source_id:
self.channel.add_video_to_skip_list(self.source_id)
@ -130,18 +127,16 @@ class Video(ModelHelper, Base):
self.viewed = now()
def get_info_json(self) -> Optional[JSON]:
"""
If this Video has an info_json file, return it's contents. Otherwise, return None.
"""
"""If this Video has an info_json file, return it's contents. Otherwise, return None."""
if not self.info_json_path:
return
try:
if isinstance(self.info_json_path, pathlib.Path):
with open(self.info_json_path, 'rb') as fh:
contents = json.load(fh)
return contents
elif self.info_json_path:
with open(self.info_json_path.path, 'rb') as fh:
contents = json.load(fh)
return contents
info_json_path = self.info_json_path.path if \
isinstance(self.info_json_path, MediaPath) else self.info_json_path
with open(info_json_path, 'rb') as fh:
contents = json.load(fh)
return contents
except UnknownFile:
pass
except UnknownDirectory:
@ -265,9 +260,8 @@ class Channel(ModelHelper, Base):
__tablename__ = 'channel'
id = Column(Integer, primary_key=True)
name = Column(String)
link = Column(String, nullable=False)
idempotency = Column(String)
url = Column(String)
url = Column(String, unique=True)
match_regex = Column(String)
directory: MediaPath = Column(MediaPathType)
skip_download_videos = Column(ARRAY(String))
@ -307,18 +301,28 @@ class Channel(ModelHelper, Base):
def update(self, data: dict):
"""
Update the values of this Channel. Will also update the Channel's Download, if it has one.
Update the attributes of this Channel. Will also update the Channel's Download, if it has one.
"""
# Get the download before we change the URL.
download = self.get_download()
for key, value in data.items():
setattr(self, key, value)
if not self.url:
return
download = self.get_download()
if download:
session = Session.object_session(self)
# All channels with a URL and download_frequency should have a download.
session = Session.object_session(self)
if download and not self.download_frequency:
download_manager.delete_download(download.id, session)
elif download and self.download_frequency:
download.frequency = self.download_frequency
download.url = self.url
download.next_download = download_manager.get_next_download(download, session)
elif not download and self.download_frequency and self.url:
download = Download(frequency=self.download_frequency, url=self.url, downloader='video_channel')
session.add(download)
session.flush()
download.next_download = download_manager.get_next_download(download, session)
session.flush()
def config_view(self) -> dict:
"""
@ -349,7 +353,7 @@ class Channel(ModelHelper, Base):
Get the Download row for this Channel. If there isn't a Download, return None.
"""
if not self.url:
raise InvalidDownload(f'Channel {self.name} does not have a URL to download!')
return None
session = Session.object_session(self)
download = session.query(Download).filter_by(url=self.url).one_or_none()
@ -361,10 +365,15 @@ class Channel(ModelHelper, Base):
name=self.name,
directory=self.directory,
url=self.url,
link=self.link,
)
return d
def dict(self, with_statistics: bool = False):
d = super(Channel, self).dict()
if with_statistics:
d['statistics'] = self.get_statistics()
return d
def get_statistics(self):
"""
Get statistics about this channel.

View file

@ -2,7 +2,6 @@ from dataclasses import dataclass, field
from datetime import date, datetime
from typing import List, Optional
from modules.videos.lib import DEFAULT_DOWNLOAD_FREQUENCY
from modules.videos.video.lib import DEFAULT_VIDEO_ORDER, VIDEO_QUERY_LIMIT
@ -11,25 +10,37 @@ class ChannelPostRequest:
name: str
directory: str
calculate_duration: Optional[bool] = None
download_frequency: Optional[int] = DEFAULT_DOWNLOAD_FREQUENCY
download_frequency: Optional[int] = None
generate_posters: Optional[bool] = None
match_regex: Optional[str] = None
mkdir: Optional[bool] = None
url: Optional[str] = None
source_id: Optional[str] = None
def __post_init__(self):
self.name = self.name.strip() or None
self.directory = self.directory.strip() or None
self.url = self.url.strip() if self.url else None
self.source_id = self.source_id.strip() if self.source_id else None
@dataclass
class ChannelPutRequest:
calculate_duration: Optional[bool] = None
directory: Optional[str] = None
download_frequency: Optional[int] = DEFAULT_DOWNLOAD_FREQUENCY
download_frequency: Optional[int] = None
generate_posters: Optional[bool] = None
link: Optional[str] = None
match_regex: Optional[str] = None
mkdir: Optional[bool] = None
name: Optional[str] = None
url: Optional[str] = None
def __post_init__(self):
self.name = self.name.strip() or None
self.directory = self.directory.strip() or None
self.url = self.url.strip() if self.url else None
self.match_regex = None if self.match_regex in ('None', '') else self.match_regex
@dataclass
class RefreshRequest:
@ -42,7 +53,6 @@ class ChannelModel:
url: str
name: str
match_regex: str
link: str
directory: str
@ -60,7 +70,6 @@ class ChannelPostResponse:
class ChannelsChannelModel:
id: int
name: str
link: str
@dataclass
@ -132,7 +141,7 @@ class VideoSearchRequest:
offset: Optional[int] = None
limit: Optional[int] = VIDEO_QUERY_LIMIT
order_by: Optional[str] = DEFAULT_VIDEO_ORDER
channel_link: Optional[str] = None
channel_id: Optional[int] = None
@dataclass

View file

@ -1,75 +0,0 @@
import pathlib
import tempfile
from functools import wraps
from modules.videos.lib import refresh_channel_videos
from modules.videos.models import Channel
from wrolpi.common import insert_parameter, set_test_media_directory
from wrolpi.db import get_db_session
from wrolpi.test.common import wrap_test_db, build_test_directories
def create_channel_structure(structure):
"""
Create a directory containing the specified structure of channels and videos. Create DB entries for these
channels and videos.
Example:
>>> s = {'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4', 'vid2.mp4', 'vid2.en.vtt']}
>>> create_channel_structure(s)
Creates directories like so:
channel1/vid1.mp4
channel2/vid1.mp4
channel2/vid2.mp4
channel2/vid2.en.vtt
Channels like so:
Channel(name='channel1', directory='channel1')
Channel(name='channel2', directory='channel2')
And, Videos like so:
Video(channel_id=1, video_path='vid1.mp4')
Video(channel_id=2, video_path='vid1.mp4')
Video(channel_id=2, video_path='vid2.mp4', caption_path='vid2.en.vtt')
"""
def wrapper(func):
@wraps(func)
@wrap_test_db
def wrapped(*args, **kwargs):
if args and (test := args[0]) and hasattr(test, 'tmp_dir'):
tmp_dir = pathlib.Path(test.tmp_dir.name)
else:
tmp_dir = pathlib.Path(tempfile.TemporaryDirectory().name)
set_test_media_directory(tmp_dir)
# Convert the channel/video structure to a file structure for the test.
file_structure = []
for channel, paths in structure.items():
for path in paths:
file_structure.append(f'{channel}/{path}')
file_structure.append(f'{channel}/')
with build_test_directories(file_structure, tmp_dir) as tempdir:
args, kwargs = insert_parameter(func, 'tempdir', tempdir, args, kwargs)
with get_db_session(commit=True) as session:
for channel in structure:
channel_ = Channel(directory=str(tempdir / channel), name=channel, link=channel)
session.add(channel_)
session.flush()
session.refresh(channel_)
with get_db_session(commit=True) as session:
for channel_ in session.query(Channel).all():
refresh_channel_videos(channel_)
try:
return func(*args, **kwargs)
finally:
set_test_media_directory(None)
return wrapped
return wrapper

View file

@ -0,0 +1,72 @@
import shutil
from http import HTTPStatus
from pathlib import Path
from modules.videos.models import Video
from wrolpi.downloader import DownloadFrequency
from wrolpi.vars import PROJECT_DIR
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory):
subdir = test_directory / 'subdir'
subdir.mkdir()
# video1 is in a subdirectory.
video1 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True,
with_poster_ext='jpg')
test_session.commit()
shutil.move(video1.video_path.path, subdir / video1.video_path.path.name)
shutil.move(video1.poster_path.path, subdir / video1.poster_path.path.name)
video1.video_path = subdir / video1.video_path.path.name
video1.poster_path = None
# video2 is in the test directory.
video2 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True,
with_poster_ext='jpg')
video2.poster_path = video1.poster_path = None
test_session.commit()
assert not video1.size, 'video1 should not have size during creation'
# Create a video not in the DB.
vid3 = test_directory / 'vid3.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', vid3)
# Orphan poster should be ignored.
orphan_poster = Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg')
orphan_poster.touch()
# Create a bogus video in the channel.
bogus = Video(video_path='foo', channel_id=simple_channel.id)
test_session.add(bogus)
test_client.post('/api/videos/refresh')
# Posters were found during refresh.
assert video1.poster_path
assert 'subdir' in str(video1.poster_path.path)
assert video2.poster_path
# Missing video3 was found
video3: Video = test_session.query(Video).filter_by(id=4).one()
assert video3.video_path.path == vid3
assert video3.video_path != video1.video_path
assert video3.video_path != video2.video_path
# Bogus video was removed.
assert not any('foo' in str(i.video_path.path) for i in test_session.query(Video).all())
# Orphan file was not deleted.
assert orphan_poster.is_file(), 'Orphan poster was removed!'
assert video1.size, 'video1 size was not found'
def test_api_download_channel(test_session, test_client, simple_channel):
"""A Channel download (a catalog update) can be triggered via the API."""
# simple_channel does not have a download record.
request, response = test_client.post(f'/api/videos/download/{simple_channel.id}')
assert response.status_code == HTTPStatus.BAD_REQUEST, response.json
assert 'not have a download' in response.json['message']
# Add a download frequency to the channel, this should also create a download.
simple_channel.update(dict(download_frequency=DownloadFrequency.daily))
test_session.commit()
request, response = test_client.post(f'/api/videos/download/{simple_channel.id}')
assert response.status_code == HTTPStatus.NO_CONTENT

View file

@ -4,23 +4,19 @@ import tempfile
from datetime import datetime
from typing import List
from unittest import mock
from unittest.mock import MagicMock
import pytest
from PIL import Image
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
from wrolpi.common import get_absolute_media_path
from wrolpi.dates import local_timezone
from wrolpi.db import get_db_session
from wrolpi.common import get_absolute_media_path, sanitize_link
from wrolpi.dates import local_timezone, now
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI
from wrolpi.test.common import build_test_directories, TestAPI
from wrolpi.vars import PROJECT_DIR
from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \
apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \
import_videos_config
from ..lib import save_channels_config, get_channels_config
apply_info_json, get_video_duration, generate_video_poster, is_valid_poster
from ..lib import save_channels_config, get_channels_config, import_channels_config
class TestCommon(TestAPI):
@ -134,113 +130,50 @@ def test_is_valid_poster(ext, expected, test_directory):
assert is_valid_poster(image_path) == expected, f'is_valid_poster({image_path}) should be {expected}'
@pytest.mark.parametrize(
'_structure,paths',
(
(
{'channel1': ['vid1.mp4']},
[
'channel1/vid1.mp4',
],
),
(
{'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4']},
[
'channel1/vid1.mp4',
'channel2/vid1.mp4',
],
),
(
{'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4', 'vid2.mp4', 'vid2.en.vtt']},
[
'channel1/vid1.mp4',
'channel2/vid1.mp4',
'channel2/vid2.mp4',
'channel2/vid2.en.vtt',
],
),
)
)
def test_create_db_structure(_structure, paths):
@create_channel_structure(_structure)
def test_func(_, tempdir):
assert isinstance(tempdir, pathlib.Path)
for path in paths:
path = (tempdir / path)
assert path.exists()
assert path.is_file()
with get_db_session() as session:
for channel_name in _structure:
channel = session.query(Channel).filter_by(name=channel_name).one()
assert (tempdir / channel_name).is_dir()
assert channel
assert channel.directory == tempdir / channel_name
assert len(channel.videos) == len([i for i in _structure[channel_name] if i.endswith('mp4')])
with tempfile.TemporaryDirectory() as tmp_dir, \
mock.patch('wrolpi.media_path.get_media_directory') as mock_get_directory:
mock_get_directory.return_value = pathlib.Path(tmp_dir)
test_ = MagicMock()
test_.tmp_dir.name = tmp_dir
test_func(test_)
@wrap_test_db
@create_channel_structure(
{
'channel1': ['vid1.mp4', 'vid1.jpg'],
'channel2': ['vid2.mp4'],
'channel3': ['vid3.mp4', 'vid4.mp4'],
}
)
def test_update_view_count(tempdir: pathlib.Path):
def test_update_view_count(test_session, channel_factory, video_factory):
def check_view_counts(view_counts):
with get_db_session() as session_:
for source_id, view_count in view_counts.items():
vid = session_.query(Video).filter_by(source_id=source_id).one()
assert vid.view_count == view_count
for source_id, view_count in view_counts.items():
video = test_session.query(Video).filter_by(source_id=source_id).one()
assert video.view_count == view_count
with get_db_session(commit=True) as session:
channel1, channel2, channel3 = session.query(Channel).order_by(Channel.id).all()
channel1.info_json = {'entries': [{'id': 'vid1.mp4', 'view_count': 10}]}
channel2.info_json = {'entries': [{'id': 'vid2.mp4', 'view_count': 11}, {'id': 'bad_id', 'view_count': 12}]}
channel3.info_json = {'entries': [{'id': 'vid3.mp4', 'view_count': 13}, {'id': 'vid4.mp4', 'view_count': 14}]}
# Use the video file name as a unique source_id
for v in session.query(Video).all():
v.source_id = str(v.video_path.path).split('/')[-1]
channel1, channel2, channel3 = channel_factory(), channel_factory(), channel_factory()
video_factory(channel_id=channel1.id, with_poster_ext='jpg', title='vid1')
video_factory(channel_id=channel2.id, title='vid2')
video_factory(channel_id=channel3.id, with_poster_ext='jpg', title='vid3')
video_factory(channel_id=channel3.id, title='vid4')
channel1.info_json = {'entries': [{'id': 'vid1', 'view_count': 10}]}
channel2.info_json = {'entries': [{'id': 'vid2', 'view_count': 11}, {'id': 'bad_id', 'view_count': 12}]}
channel3.info_json = {'entries': [{'id': 'vid3', 'view_count': 13}, {'id': 'vid4', 'view_count': 14}]}
test_session.commit()
# Check all videos are empty.
check_view_counts({'vid1.mp4': None, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None})
check_view_counts({'vid1': None, 'vid2': None, 'vid3': None, 'vid4': None})
# Channel 1 is updated, the other channels are left alone.
apply_info_json(channel1.id)
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None})
check_view_counts({'vid1': 10, 'vid2': None, 'vid3': None, 'vid4': None})
# Channel 2 is updated, the other channels are left alone. The 'bad_id' video is ignored.
apply_info_json(channel2.id)
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': None, 'vid4.mp4': None})
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': None, 'vid4': None})
# All videos are updated.
apply_info_json(channel3.id)
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14})
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})
# An outdated view count will be overwritten.
with get_db_session(commit=True) as session:
vid = session.query(Video).filter_by(id=1).one()
vid.view_count = 8
check_view_counts({'vid1.mp4': 8, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14})
vid = test_session.query(Video).filter_by(id=1).one()
vid.view_count = 8
check_view_counts({'vid1': 8, 'vid2': 11, 'vid3': 13, 'vid4': 14})
apply_info_json(channel1.id)
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14})
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})
def test_generate_video_poster(video_file):
"""
A poster can be generated from a video file.
"""
poster_path = replace_extension(video_file, '.jpg')
assert not poster_path.is_file(), f'{poster_path} already exists!'
poster_path = video_file.with_suffix('.jpg')
generate_video_poster(video_file)
assert poster_path.is_file(), f'{poster_path} was not created!'
assert poster_path.stat().st_size > 0
@ -326,7 +259,7 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan
vid2_video_path = vid2.video_path.path
favorites = {
simple_channel.link: {vid3.video_path.path.name: {'favorite': favorite}},
str(simple_channel.directory.relative): {vid3.video_path.path.name: {'favorite': favorite}},
'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}},
}
@ -336,8 +269,8 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan
assert config.favorites == favorites
def import_and_verify(favorited_ids: List[int]):
with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'):
import_videos_config()
with mock.patch('modules.videos.lib.get_channel_source_id', lambda i: 'foo'):
import_channels_config()
for video in test_session.query(Video).all():
if video.id in favorited_ids:
assert video.favorite
@ -371,10 +304,8 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan
def test_import_channel_downloads(test_session, channel_factory, test_channels_config):
"""Importing the Channels' config should create any missing download records"""
channel1 = channel_factory()
channel2 = channel_factory()
channel1.source_id = 'foo'
channel2.source_id = 'bar'
channel1 = channel_factory(source_id='foo')
channel2 = channel_factory(source_id='bar')
# channel2 has no url, but has a frequency. Import should not create a download record.
channel2.url = None
channel2.download_frequency = DownloadFrequency.biweekly
@ -383,20 +314,25 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
assert len(test_session.query(Channel).all()) == 2
assert test_session.query(Download).all() == []
def update_channel_config(conf, source_id, d):
for c in conf.channels:
if c['source_id'] == source_id:
c.update(d)
conf.save()
# Config has no channels with a download_frequency.
save_channels_config()
import_videos_config()
import_channels_config()
assert channel1.download_frequency is None
assert len(test_session.query(Channel).all()) == 2
assert test_session.query(Download).all() == []
# Add a frequency to the Channel.
channels_config = get_channels_config()
channels_config.channels[channel1.link]['download_frequency'] = DownloadFrequency.biweekly
channels_config.save()
update_channel_config(channels_config, 'foo', {'download_frequency': DownloadFrequency.biweekly})
# Download record is created on import.
import_videos_config()
import_channels_config()
assert channel1.download_frequency is not None
assert len(test_session.query(Channel).all()) == 2
download: Download = test_session.query(Download).one()
@ -404,9 +340,8 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
assert download.frequency == channel1.download_frequency
# Download frequency is adjusted when config file changes.
channels_config.channels[channel1.link]['download_frequency'] = DownloadFrequency.weekly
channels_config.save()
import_videos_config()
update_channel_config(channels_config, 'foo', {'download_frequency': DownloadFrequency.weekly})
import_channels_config()
assert len(test_session.query(Channel).all()) == 2
assert download.url == channel1.url
assert download.frequency == channel1.download_frequency
@ -414,6 +349,67 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
assert download.next_download
next_download = str(download.next_download)
import_videos_config()
import_channels_config()
download: Download = test_session.query(Download).one()
assert next_download == str(download.next_download)
def test_import_channels_config_outdated(test_session, test_directory, channel_factory, test_channels_config,
video_factory):
"""The Channels' config used to use a "link" to differentiate channels, test that old configs can be imported."""
channel1 = channel_factory(name='Channel1', download_frequency=DownloadFrequency.weekly)
channel2 = channel_factory(name='Channel2', download_frequency=DownloadFrequency.weekly)
test_session.commit()
vid1 = video_factory(title='vid1', channel_id=channel1.id)
vid2 = video_factory(title='vid2', channel_id=channel2.id)
vid3 = video_factory(title='vid3')
vid4 = video_factory(title='vid4')
test_session.commit()
channel1_name, channel_1_directory = channel1.name, channel1.directory.path
channel2_name, channel_2_directory = channel2.name, channel2.directory.path
assert channel1.directory.path.exists() and channel1.directory.path.is_absolute()
assert channel2.directory.path.exists() and channel2.directory.path.is_absolute()
# Channel's used to have a relative directory in the DB.
test_session.execute('UPDATE channel SET directory=:directory WHERE id=2',
dict(directory=str(channel_2_directory.relative_to(test_directory))),
)
save_channels_config()
# The config has a list of Channels, use the old method of a "dict" with the sanitized name as the key.
channels_config = get_channels_config()
channels_config.channels = {sanitize_link(i['name']): i for i in channels_config.channels}
# Change frequency to verify that the channels are updated.
for link, channel in channels_config.channels.items():
channel['download_frequency'] = DownloadFrequency.biweekly
# Change the favorites to the old "link" method as well.
channels_config.favorites = {
sanitize_link(channel1.name): {str(vid1.video_path.path): {'favorite': now()}},
sanitize_link(channel2.name): {str(vid2.video_path.path): {'favorite': now()}},
'NO CHANNEL': {str(vid3.video_path.path): {'favorite': now()}},
}
channels_config.save()
assert not vid1.favorite and not vid2.favorite and not vid3.favorite and not vid3.favorite
assert test_session.query(Channel).count() == 2
with mock.patch('modules.videos.lib.get_channel_source_id') as mock_get_channel_source_id:
mock_get_channel_source_id.return_value = 'some source id'
import_channels_config()
# No new channels were created. Existing channels were updated.
assert test_session.query(Channel).count() == 2
channel1, channel2 = test_session.query(Channel).order_by(Channel.id)
assert channel1.name == channel1_name and str(channel1.directory.path) == str(channel_1_directory)
# Channel2's directory is now absolute.
assert channel2.name == channel2_name and str(channel2.directory.path) == str(channel_2_directory)
assert channel1.download_frequency == DownloadFrequency.biweekly
assert channel2.download_frequency == DownloadFrequency.biweekly
# Videos were favorited by their channel "link".
assert vid1.favorite and vid2.favorite and vid3.favorite
assert not vid4.favorite

View file

@ -1,4 +1,5 @@
import pathlib
import shutil
from copy import copy
from unittest import mock
@ -9,12 +10,12 @@ from modules.videos.channel.lib import download_channel
from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \
ChannelDownloader, get_or_create_channel
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
from wrolpi.db import get_db_session, get_db_context
from wrolpi.db import get_db_context
from wrolpi.downloader import DownloadManager, Download
from wrolpi.errors import InvalidDownload
from wrolpi.test.common import wrap_test_db, TestAPI
from wrolpi.test.test_downloader import HTTPDownloader
from wrolpi.vars import PROJECT_DIR
class FakeYDL:
@ -28,37 +29,32 @@ class FakeYDL:
}
class TestDownloader(TestAPI):
@wrap_test_db
@create_channel_structure(
{
'channel1': ['vid1.mp4'],
'channel2': ['vid2.mp4']
}
)
def test_find_all_missing_videos(self, tempdir):
with get_db_session(commit=True) as session:
channel1, channel2 = session.query(Channel).order_by(Channel.id).all()
channel1.url = channel2.url = 'some url'
channel1.info_json = {'entries': [{'id': 'foo'}]}
def test_find_all_missing_videos(test_session, channel_factory, video_factory):
channel1, channel2 = channel_factory(url='some url'), channel_factory(url='some other url')
channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]}
test_session.commit()
# No missing videos
self.assertEqual([], list(find_all_missing_videos()))
# Two videos are already downloaded.
video_factory()
video_factory()
# Create a video that has no video file.
with get_db_session(commit=True) as session:
video = Video(title='needs to be downloaded', channel_id=channel1.id, source_id='foo')
session.add(video)
# No missing videos
assert [] == list(find_all_missing_videos())
missing = list(find_all_missing_videos())
self.assertEqual(len(missing), 1)
# Create a video that has no video file.
video = Video(title='needs to be downloaded', channel_id=channel1.id, source_id='foo')
test_session.add(video)
test_session.commit()
# The video returned is the one we faked.
id_, source_id, entry = missing[0]
# Two videos were created for this test already.
self.assertEqual(id_, 3)
# The fake entry we added is regurgitated back.
self.assertEqual(entry, channel1.info_json['entries'][0])
missing = list(find_all_missing_videos())
assert len(missing) == 1
# The video returned is the one we faked.
id_, source_id, entry = missing[0]
# Two videos were created for this test already.
assert id_ == 3
# The fake entry we added is regurgitated back.
assert entry == channel1.info_json['entries'][0]
example_video_json = {
@ -171,43 +167,6 @@ class TestVideosDownloaders(TestAPI):
info_json_path.touch()
return video_path, poster_path, description_path, info_json_path
@wrap_test_db
def test_video_download(self):
channel_dir = self.videos_dir / 'channel name'
channel_dir.mkdir(parents=True)
# Video paths
video_path, poster_path, description_path, info_json_path = self._make_video_files(channel_dir)
with get_db_session() as session:
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
mock_extract_info.return_value = example_video_json
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
mock_process_runner.return_value = 0
self.mgr.create_download(url, session)
mock_process_runner.assert_called_once()
video_url, _, out_dir = mock_process_runner.call_args[0]
download: Download = session.query(Download).one()
self.assertEqual(video_url, download.url)
channel: Channel = session.query(Channel).one()
self.assertEqual(channel.directory, channel_dir)
video: Video = session.query(Video).one()
self.assertEqual(video.video_path, video_path)
self.assertEqual(video.description_path, description_path)
self.assertEqual(video.poster_path, poster_path)
self.assertEqual(video.info_json_path, info_json_path)
self.assertTrue(video.video_path.path.is_absolute())
self.assertTrue(video.poster_path.path.is_absolute())
self.assertTrue(video.description_path.path.is_absolute())
self.assertTrue(video.info_json_path.path.is_absolute())
@wrap_test_db
def test_video_download_no_channel(self):
_, session = get_db_context()
@ -239,73 +198,65 @@ class TestVideosDownloaders(TestAPI):
video: Video = session.query(Video).one()
self.assertEqual(str(video.video_path.path), f'{channel_dir}/video.mp4')
@wrap_test_db
def test_channel_download(self):
_, session = get_db_context()
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
session.add(Channel(url=url, link='foo'))
def test_channel_download(test_session, simple_channel, video_download_manager, video_file):
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
channel_dir = self.videos_dir / 'NO CHANNEL'
channel_dir.mkdir(parents=True)
video_path, _, _, _ = self._make_video_files(channel_dir)
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
mock_extract_info.return_value = example_channel_json
mock_prepare_filename.return_value = (video_file, {'id': 'foo'})
mock_process_runner.return_value = 0
video_download_manager.create_download(url, test_session)
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
mock_extract_info.return_value = example_channel_json
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
mock_process_runner.return_value = 0
self.mgr.create_download(url, session)
# Two videos are in the example channel.
downloads = video_download_manager.get_new_downloads(test_session)
expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o']
for download, expected in zip(downloads, expected):
assert download.url == expected
# Download is skipped for test.
assert download.status == 'new'
# Two videos are in the example channel.
downloads = self.mgr.get_new_downloads(session)
expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o']
for download, expected in zip(downloads, expected):
self.assertEqual(download.url, expected)
# Download is skipped for test.
self.assertEqual(download.status, 'new')
@wrap_test_db
def test_get_or_create_channel(self):
"""
A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can
match it.
"""
with get_db_session(commit=True) as session:
c1 = Channel(name='foo', link='foo', source_id='foo', url='foo')
c2 = Channel(name='bar', link='bar', source_id='bar')
c3 = Channel(name='baz', link='baz', source_id='baz', url='baz')
c4 = Channel(name='qux', link='qux')
session.add_all([c1, c2, c3, c4])
def test_get_or_create_channel(test_session):
"""
A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can
match it.
"""
c1 = Channel(name='foo', source_id='foo', url='foo')
c2 = Channel(name='bar', source_id='bar')
c3 = Channel(name='baz', source_id='baz', url='baz')
c4 = Channel(name='qux')
test_session.add_all([c1, c2, c3, c4])
test_session.commit()
# All existing channels should be used.
tests = [
(dict(source_id='foo'), c1),
(dict(link='foo'), c1),
(dict(url='foo'), c1),
(dict(url='foo', source_id='bar'), c2), # source_id is preferred.
(dict(name='foo', source_id='bar'), c2),
(dict(source_id='bar'), c2),
(dict(source_id='baz'), c3),
(dict(name='qux'), c4),
(dict(link='qux'), c4),
]
for kwargs, expected in tests:
# All existing channels should be used.
tests = [
(dict(source_id='foo'), c1),
(dict(url='foo'), c1),
(dict(url='foo', source_id='bar'), c2), # source_id has priority.
(dict(name='foo', source_id='bar'), c2),
(dict(source_id='bar'), c2),
(dict(source_id='baz'), c3),
(dict(name='qux'), c4),
]
for kwargs, expected in tests:
try:
channel = get_or_create_channel(**kwargs)
self.assertEqual(expected.id, channel.id, f'Expected {expected} got {channel}')
except Exception as e:
raise Exception(f'get_or_create_channel failed with {kwargs=}') from e
assert expected.id == channel.id, f'Expected {expected} for {kwargs} but got {channel}'
# A new channel is created. It will not be automatically downloaded.
channel = get_or_create_channel(source_id='quux', name='quux', url='quux')
self.assertEqual(channel.id, 5)
self.assertEqual(channel.source_id, 'quux')
self.assertEqual(channel.name, 'quux')
self.assertEqual(channel.link, 'quux')
self.assertEqual(channel.url, 'quux')
self.assertIsNone(channel.download_frequency)
# A new channel is created. It will not be automatically downloaded.
channel = get_or_create_channel(source_id='quux', name='quux', url='quux')
assert channel.id == 5
assert channel.source_id == 'quux'
assert channel.name == 'quux'
assert channel.url == 'quux'
# New channel can be retrieved.
self.assertEqual(get_or_create_channel(source_id='quux'), channel)
# New channel can be retrieved.
assert get_or_create_channel(source_id='quux') == channel
def test_download_info_save(test_session, video_download_manager):
@ -397,7 +348,7 @@ def test_channel_download_no_download(test_session, video_download_manager, simp
assert video_download_manager.get_downloads(test_session) == []
with pytest.raises(InvalidDownload):
download_channel(simple_channel.link)
download_channel(simple_channel.id)
def test_invalid_download_url(test_session, test_download_manager, test_downloader_config):
@ -413,3 +364,32 @@ def test_invalid_download_url(test_session, test_download_manager, test_download
download = test_download_manager.get_downloads(test_session)[0]
assert download.status == 'failed'
def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager):
simple_channel.source_id = example_video_json['channel_id']
simple_channel.directory = test_directory / 'videos/channel name'
simple_channel.directory.mkdir(parents=True)
video_file = simple_channel.directory / 'a video.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file)
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
mock_extract_info.return_value = example_video_json
mock_prepare_filename.return_value = (video_file, {'id': 'foo'})
mock_process_runner.return_value = 0
video_download_manager.create_download(url, test_session)
mock_process_runner.assert_called_once()
video_url, _, out_dir = mock_process_runner.call_args[0]
download: Download = test_session.query(Download).one()
assert video_url == download.url
assert test_session.query(Channel).one()
video: Video = test_session.query(Video).one()
assert video.video_path.path.is_absolute()

View file

@ -44,7 +44,7 @@ async def search(_: Request, body: schema.VideoSearchRequest):
body.search_str,
body.offset,
body.limit,
body.channel_link,
body.channel_id,
body.order_by,
body.filters,
)

View file

@ -80,16 +80,16 @@ def video_search(
search_str: str = None,
offset: int = None,
limit: int = VIDEO_QUERY_LIMIT,
channel_link: str = None,
channel_id: int = None,
order_by: str = None,
filters: List[str] = None,
) -> Tuple[List[dict], int]:
with get_db_curs() as curs:
params = dict(search_str=search_str, offset=offset)
channel_where = ''
if channel_link:
channel_where = 'AND channel_id = (select id from channel where link=%(channel_link)s)'
params['channel_link'] = channel_link
if channel_id:
channel_where = 'AND channel_id = %(channel_id)s'
params['channel_id'] = channel_id
# Filter for/against favorites, if it was provided
favorites_where = ''

View file

@ -1,163 +1,141 @@
import pathlib
from datetime import timedelta
from http import HTTPStatus
from json import dumps
from unittest import mock
from uuid import uuid4
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
from modules.videos.models import Video
from modules.videos.video.lib import get_video_for_app
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.errors import API_ERRORS, WROLModeEnabled
from wrolpi.root_api import api_app
from wrolpi.test.common import wrap_test_db, TestAPI
class TestVideoFunctions(TestAPI):
def test_get_video_prev_next(test_session, channel_factory, video_factory):
"""
Test that the previous and next videos will be retrieved when fetching a video.
"""
channel1, channel2, channel3, channel4 = channel_factory(), channel_factory(), channel_factory(), channel_factory()
@wrap_test_db
def test_get_video_prev_next(self):
"""
Test that the previous and next videos will be retrieved when fetching a video.
"""
now_ = now()
second = timedelta(seconds=1)
with get_db_session(commit=True) as session:
for _ in range(4):
session.add(Channel(link=str(uuid4())))
channel1, channel2, channel3, channel4 = session.query(Channel).all()
# The upload_date decides the order of the prev/next videos.
video_factory(title='vid1', channel_id=channel1.id, upload_date=now_)
video_factory(title='vid2', channel_id=channel1.id, upload_date=now_ + second)
video_factory(title='vid3', channel_id=channel2.id, upload_date=now_ + (second * 4))
video_factory(title='vid4', channel_id=channel1.id, upload_date=now_ + (second * 3))
video_factory(title='vid5', channel_id=channel2.id, upload_date=now_ + (second * 2))
video_factory(title='vid6', channel_id=channel2.id, upload_date=now_ + (second * 5))
video_factory(title='vid7', channel_id=channel1.id)
video_factory(title='vid8', channel_id=channel2.id, upload_date=now_ + (second * 7))
video_factory(title='vid9', channel_id=channel3.id, upload_date=now_ + (second * 8))
# Videos without an upload date can't be placed.
video_factory(title='vid10', channel_id=channel4.id)
video_factory(title='vid11', channel_id=channel4.id)
now_ = now()
second = timedelta(seconds=1)
test_session.commit()
# The upload_date decides the order of the prev/next videos.
session.add(Video(title=f'vid1', channel_id=channel1.id, upload_date=now_))
session.add(Video(title=f'vid2', channel_id=channel1.id, upload_date=now_ + second))
session.add(Video(title=f'vid3', channel_id=channel2.id, upload_date=now_ + (second * 4)))
session.add(Video(title=f'vid4', channel_id=channel1.id, upload_date=now_ + (second * 3)))
session.add(Video(title=f'vid5', channel_id=channel2.id, upload_date=now_ + (second * 2)))
session.add(Video(title=f'vid6', channel_id=channel2.id, upload_date=now_ + (second * 5)))
session.add(Video(title=f'vid7', channel_id=channel1.id))
session.add(Video(title=f'vid8', channel_id=channel2.id, upload_date=now_ + (second * 7)))
session.add(Video(title=f'vid9', channel_id=channel3.id, upload_date=now_ + (second * 8)))
session.add(Video(title=f'vid10', channel_id=channel4.id))
session.add(Video(title=f'vid11', channel_id=channel4.id))
tests = [
# Channel 1's videos were inserted in upload_date order.
(1, (None, 'vid2')),
(2, ('vid1', 'vid4')),
(4, ('vid2', None)), # 7 has no upload_date, so it doesn't come after 4.
(7, (None, None)), # 7 has no upload_date, so we don't know the order of it.
# Channel 3 has only one video.
(9, (None, None)),
# Channel 2 was inserted out of order.
(5, (None, 'vid3')),
(3, ('vid5', 'vid6')),
(8, ('vid6', None)),
# Channel 4's videos have no upload date, so we don't know what is previous/next.
(10, (None, None)),
]
tests = [
# Channel 1's videos were inserted in upload_date order.
(1, (None, 'vid2')),
(2, ('vid1', 'vid4')),
(4, ('vid2', None)), # 7 has no upload_date, so it doesn't come after 4.
(7, (None, None)), # 7 has no upload_date, so we don't know the order of it.
# Channel 3 has only one video.
(9, (None, None)),
# Channel 2 was inserted out of order.
(5, (None, 'vid3')),
(3, ('vid5', 'vid6')),
(8, ('vid6', None)),
# Channel 4's videos have no upload date, so we don't know what is previous/next.
(10, (None, None)),
]
for id_, (prev_title, next_title) in tests:
video = session.query(Video).filter_by(id=id_).one()
prev_video, next_video = video.get_surrounding_videos()
for id_, (prev_title, next_title) in tests:
video = test_session.query(Video).filter_by(id=id_).one()
prev_video, next_video = video.get_surrounding_videos()
try:
if prev_title is None:
self.assertIsNone(prev_video)
assert prev_video is None
else:
self.assertDictContains(prev_video, {'title': prev_title})
assert prev_video and prev_video.title == prev_title
if next_title is None:
self.assertIsNone(next_video)
assert next_video is None
else:
self.assertDictContains(next_video, {'title': next_title})
assert next_video and next_video.title == next_title
except AssertionError as e:
raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e
@wrap_test_db
@create_channel_structure({
'channel1': ['vid1.mp4']
})
def test_wrol_mode(self, tempdir):
"""
Many methods are blocked when WROL Mode is enabled.
"""
channel = dumps(dict(name='foo', directory='foo'))
favorite = dumps(dict(video_id=1, favorite=True))
with mock.patch('wrolpi.common.wrol_mode_enabled', lambda: True):
# Can't create, update, or delete a channel.
_, resp = api_app.test_client.post('/api/videos/channels', content=channel)
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
_, resp = api_app.test_client.put('/api/videos/channels/channel1', content=channel)
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
_, resp = api_app.test_client.delete('/api/videos/channels/channel1')
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
def test_get_video_for_app(test_session, simple_channel, simple_video):
vid, prev, next_ = get_video_for_app(simple_video.id)
assert vid['id'] == simple_video.id
# Can't delete a video
_, resp = api_app.test_client.delete('/api/videos/video/1')
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
# Can't refresh or download
_, resp = api_app.test_client.post('/api/videos/refresh')
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
_, resp = api_app.test_client.post('/api/videos/download')
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
def test_video_delete(test_session, test_directory, channel_factory, video_factory):
"""Video.delete() removes the video's files, but leave the DB record."""
channel1, channel2 = channel_factory(), channel_factory()
vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True)
vid2 = video_factory(channel_id=channel2.id, with_video_file=True, with_info_json=True)
test_session.commit()
# THE REST OF THESE METHODS ARE ALLOWED
_, resp = api_app.test_client.post('/api/videos/favorite', content=favorite)
self.assertEqual(resp.status_code, HTTPStatus.OK)
vid1_video_path, vid1_caption_path = vid1.video_path.path, vid1.caption_path.path
vid2_video_path, vid2_info_json_path = vid2.video_path.path, vid2.info_json_path.path
@wrap_test_db
@create_channel_structure({
'channel1': [
'vid1.mp4',
'vid1.en.vtt',
],
})
def test_get_video_for_app(self, tempdir):
with get_db_session(commit=True) as session:
vid1 = session.query(Video).one()
# No videos have been deleted yet.
assert vid1_video_path.is_file() and vid1_caption_path.is_file()
assert vid2_video_path.is_file() and vid2_info_json_path.is_file()
assert channel1.skip_download_videos is None
assert channel2.skip_download_videos is None
vid, prev, next_ = get_video_for_app(vid1.id)
self.assertEqual(vid['id'], vid1.id)
vid1.delete()
@wrap_test_db
@create_channel_structure({
'channel1': [
'vid1.mp4',
'vid1.en.vtt',
],
'channel2': [
'vid2.mp4',
'vid2.info.json',
],
})
def test_delete_video(self, tempdir: pathlib.Path):
with get_db_session(commit=True) as session:
channel1 = session.query(Channel).filter_by(name='channel1').one()
channel2 = session.query(Channel).filter_by(name='channel2').one()
vid1, vid2 = session.query(Video).order_by(Video.video_path).all()
vid1.source_id = 'foo'
vid2.source_id = 'bar'
# Video was added to skip list.
assert len(channel1.skip_download_videos) == 1
# Deleting a video leaves it's entry in the DB, but its files are deleted.
assert test_session.query(Video).count() == 2
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
assert vid2_video_path.is_file() and vid2_info_json_path.is_file()
# No videos have been deleted yet.
self.assertIsNone(channel1.skip_download_videos)
self.assertIsNone(channel2.skip_download_videos)
self.assertTrue((tempdir / 'channel1/vid1.mp4').is_file())
vid2.delete()
vid1.delete()
assert test_session.query(Video).count() == 2
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False
channel1 = session.query(Channel).filter_by(name='channel1').one()
# Video was added to skip list.
self.assertEqual(len(channel1.skip_download_videos), 1)
# Deleting a video leaves it's entry in the DB, but its files are deleted.
self.assertEqual(session.query(Video).count(), 2)
self.assertFalse((tempdir / 'channel1/vid1.mp4').is_file())
self.assertTrue((tempdir / 'channel2/vid2.mp4').is_file())
vid2.delete()
def test_wrol_mode(test_directory, simple_channel, simple_video):
"""Many methods are blocked when WROL Mode is enabled."""
channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory)))
favorite = dumps(dict(video_id=simple_video.id, favorite=True))
self.assertEqual(session.query(Video).count(), 2)
self.assertFalse((tempdir / 'channel1/vid1.mp4').is_file())
self.assertFalse((tempdir / 'channel2/vid2.mp4').is_file())
with mock.patch('wrolpi.common.wrol_mode_enabled', lambda: True):
# Can't create, update, or delete a channel.
_, resp = api_app.test_client.post('/api/videos/channels', content=channel)
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
_, resp = api_app.test_client.put(f'/api/videos/channels/{simple_channel.id}', content=channel)
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
_, resp = api_app.test_client.delete(f'/api/videos/channels/{simple_channel.id}')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
# Can't delete a video
_, resp = api_app.test_client.delete(f'/api/videos/video/{simple_video.id}')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
# Can't refresh or download
_, resp = api_app.test_client.post('/api/videos/refresh')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
_, resp = api_app.test_client.post(f'/api/videos/download/{simple_channel.id}')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
# THE REST OF THESE METHODS ARE ALLOWED
_, resp = api_app.test_client.post('/api/videos/favorite', content=favorite)
assert resp.status_code == HTTPStatus.OK

View file

@ -1,5 +1,6 @@
import inspect
import multiprocessing
from functools import wraps
BEFORE_STARTUP_FUNCTIONS = []
@ -29,6 +30,7 @@ def limit_concurrent(limit: int):
def wrapper(func: callable):
if inspect.iscoroutinefunction(func):
@wraps(func)
async def wrapped(*a, **kw):
acquired = sema.acquire(block=False)
if not acquired:
@ -37,6 +39,7 @@ def limit_concurrent(limit: int):
return wrapped
else:
@wraps(func)
def wrapped(*a, **kw):
acquired = sema.acquire(block=False)
if not acquired:

View file

@ -20,7 +20,7 @@ def which(*possible_paths: str, warn: bool = False) -> Path:
Returns None if no executable can be found.
"""
possible_paths = map(Path, possible_paths)
possible_paths = list(map(Path, possible_paths))
for path in possible_paths:
if is_executable(path):
# `path` was an absolute file which is executable.

View file

@ -44,7 +44,7 @@ Base = declarative_base()
class ModelHelper:
def dict(self) -> dict:
def dict(self, *_, **__) -> dict:
d = {i.name: getattr(self, i.name) for i in self.__table__.columns} # noqa
return d
@ -732,11 +732,6 @@ def native_only(func: callable):
return wrapped
def read_yaml(path: Path):
with path.open('rt') as fh:
return yaml.load(fh, Loader=yaml.Loader)
def recursive_map(obj: Any, func: callable):
"""
Apply `func` to all values of any dictionaries; or items in any list/set/tuple.
@ -755,3 +750,11 @@ def recursive_map(obj: Any, func: callable):
type_ = type(obj)
return type_(map(lambda i: recursive_map(i, func), obj))
return func(obj)
def any_extensions(filename: str, extensions: Iterable = ()):
"""Return True only if the file name ends with any of the possible extensions.
Matches lower or upper case of the extension.
"""
return any(filename.lower().endswith(ext) for ext in extensions)

View file

@ -1,7 +1,6 @@
"""
Fixtures for Pytest tests.
"""
import os
import pathlib
import tempfile
from typing import Tuple

View file

@ -158,7 +158,7 @@ API_ERRORS = {
},
UnknownChannel: {
'code': next(error_codes),
'message': 'The channel could not be found by its link.',
'message': 'The channel could not be found.',
'status': HTTPStatus.NOT_FOUND,
},
UnknownDirectory: {

View file

@ -18,10 +18,10 @@ class MediaPath:
self._path = path.absolute()
def __repr__(self):
return f'<MediaPath path={self.path.relative_to(get_media_directory())}>'
return f'<MediaPath path={self.relative}>'
def __str__(self):
return str(self.path.relative_to(get_media_directory()))
return str(self.relative)
@classmethod
def _validate_path(cls, path: PATH_TYPE):
@ -40,7 +40,7 @@ class MediaPath:
def __json__(self):
# Always return a relative path to frontend.
return str(self.path.relative_to(get_media_directory()))
return str(self.relative)
@property
def path(self) -> pathlib.Path:
@ -60,6 +60,11 @@ class MediaPath:
def startswith(self, other):
return str(self._path).startswith(other)
@property
def relative(self):
"""Get the path relative to the media directory."""
return self._path.relative_to(get_media_directory())
class MediaPathType(types.TypeDecorator): # noqa
impl = types.String

View file

@ -1,5 +1,5 @@
from dataclasses import dataclass
from typing import List, Optional
from typing import Optional
@dataclass

View file

@ -1,4 +1,4 @@
__version__ = '0.4.3-beta'
__version__ = '0.4.4-beta'
import subprocess
@ -25,7 +25,5 @@ def git_branch():
def get_version_string():
"""
Return a string containing the WROLPi version, git branch and git revision hash.
"""
"""Return a string containing the WROLPi version, git branch and git revision hash."""
return f'{__version__} (git: {git_branch()} {git_revision()})'