mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Merge pull request #118 from lrnselfreliance/feature/remove-channel-link
Feature/remove channel link
This commit is contained in:
commit
c8810fbeba
36 changed files with 1230 additions and 1329 deletions
37
alembic/versions/d8fcdb7773c0_drop_channel_link.py
Normal file
37
alembic/versions/d8fcdb7773c0_drop_channel_link.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
"""Drop channel.link.
|
||||
|
||||
Revision ID: d8fcdb7773c0
|
||||
Revises: 2fb39c1861c7
|
||||
Create Date: 2022-04-08 12:52:02.877449
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'd8fcdb7773c0'
|
||||
down_revision = '2fb39c1861c7'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
session.execute('ALTER TABLE channel DROP CONSTRAINT IF EXISTS channel_link_key')
|
||||
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS link')
|
||||
session.execute('DROP INDEX IF EXISTS channel_url_key')
|
||||
session.execute("UPDATE channel SET url=null WHERE url=''")
|
||||
session.execute('CREATE UNIQUE INDEX channel_url_key ON channel (url) WHERE url IS NOT NULL')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
session.execute('ALTER TABLE channel ADD CONSTRAINT channel_link_key UNIQUE (link)')
|
||||
session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS link TEXT')
|
||||
# Notice we are not removing the channel_url_key constraint.
|
||||
|
|
@ -82,7 +82,7 @@ export default function App() {
|
|||
<>
|
||||
<Switch>
|
||||
<Route path='/videos/video/:video_id' exact component={VideoWrapper}/>
|
||||
<Route path='/videos/channel/:channel_link/video/:video_id' exact component={VideoWrapper}/>
|
||||
<Route path='/videos/channel/:channel_id/video/:video_id' exact component={VideoWrapper}/>
|
||||
<Route path="/" exact={true} component={Welcome}/>
|
||||
<Route path="/videos" component={VideosRoute}/>
|
||||
<Route path="/admin" component={Admin}/>
|
||||
|
|
|
|||
|
|
@ -95,9 +95,9 @@ let apiPatch = async (url, body) => {
|
|||
return await apiCall(url, 'PATCH', body)
|
||||
};
|
||||
|
||||
export async function updateChannel(link, channel) {
|
||||
export async function updateChannel(id, channel) {
|
||||
channel = emptyToNull(channel);
|
||||
return await apiPut(`${VIDEOS_API}/channels/${link}`, channel);
|
||||
return await apiPut(`${VIDEOS_API}/channels/${id}`, channel);
|
||||
}
|
||||
|
||||
export async function createChannel(channel) {
|
||||
|
|
@ -105,8 +105,8 @@ export async function createChannel(channel) {
|
|||
return await apiPost(`${VIDEOS_API}/channels`, channel);
|
||||
}
|
||||
|
||||
export async function deleteChannel(channel_link) {
|
||||
return apiDelete(`${VIDEOS_API}/channels/${channel_link}`);
|
||||
export async function deleteChannel(id) {
|
||||
return apiDelete(`${VIDEOS_API}/channels/${id}`);
|
||||
}
|
||||
|
||||
export async function getChannels() {
|
||||
|
|
@ -114,12 +114,12 @@ export async function getChannels() {
|
|||
return (await response.json())['channels'];
|
||||
}
|
||||
|
||||
export async function getChannel(link) {
|
||||
let response = await apiGet(`${VIDEOS_API}/channels/${link}`);
|
||||
export async function getChannel(id) {
|
||||
let response = await apiGet(`${VIDEOS_API}/channels/${id}`);
|
||||
return (await response.json())['channel'];
|
||||
}
|
||||
|
||||
export async function searchVideos(offset, limit, channel_link, searchStr, order_by, filters) {
|
||||
export async function searchVideos(offset, limit, channel_id, searchStr, order_by, filters) {
|
||||
// Build a search query to retrieve a list of videos from the API
|
||||
offset = offset || 0;
|
||||
limit = limit || DEFAULT_LIMIT;
|
||||
|
|
@ -128,8 +128,8 @@ export async function searchVideos(offset, limit, channel_link, searchStr, order
|
|||
if (searchStr) {
|
||||
body.search_str = searchStr;
|
||||
}
|
||||
if (channel_link) {
|
||||
body.channel_link = channel_link;
|
||||
if (channel_id) {
|
||||
body.channel_id = channel_id;
|
||||
}
|
||||
body.order_by = order_by ? order_by : 'rank'
|
||||
|
||||
|
|
@ -211,8 +211,8 @@ export async function download() {
|
|||
return await response.json();
|
||||
}
|
||||
|
||||
export async function downloadChannel(link) {
|
||||
let url = `${VIDEOS_API}/download/${link}`;
|
||||
export async function downloadChannel(id) {
|
||||
let url = `${VIDEOS_API}/download/${id}`;
|
||||
let response = await apiPost(url);
|
||||
let json = await response.json();
|
||||
if (response.status === 400 && json['code'] === 30) {
|
||||
|
|
@ -225,8 +225,8 @@ export async function downloadChannel(link) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function refreshChannel(link) {
|
||||
let url = `${VIDEOS_API}/refresh/${link}`;
|
||||
export async function refreshChannel(id) {
|
||||
let url = `${VIDEOS_API}/refresh/${id}`;
|
||||
await fetch(url, {method: 'POST'});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -154,8 +154,8 @@ class ChannelPage extends APIForm {
|
|||
|
||||
async componentDidMount() {
|
||||
if (!this.props.create) {
|
||||
let channel_link = this.props.match.params.channel_link;
|
||||
let channel = await getChannel(channel_link);
|
||||
let channel_id = this.props.match.params.channel_id;
|
||||
let channel = await getChannel(channel_id);
|
||||
let statistics = channel.statistics;
|
||||
if (statistics) {
|
||||
this.setState({statistics: statistics || null});
|
||||
|
|
@ -173,7 +173,7 @@ class ChannelPage extends APIForm {
|
|||
|
||||
handleConfirm = async () => {
|
||||
this.setState({deleteOpen: false});
|
||||
let response = await deleteChannel(this.props.match.params.channel_link);
|
||||
let response = await deleteChannel(this.props.match.params.channel_id);
|
||||
if (response.status === 204) {
|
||||
this.props.history.push({
|
||||
pathname: '/videos/channel'
|
||||
|
|
@ -194,7 +194,7 @@ class ChannelPage extends APIForm {
|
|||
if (this.props.create) {
|
||||
response = await createChannel(inputs);
|
||||
} else {
|
||||
response = await updateChannel(this.props.match.params.channel_link, inputs);
|
||||
response = await updateChannel(this.props.match.params.channel_id, inputs);
|
||||
}
|
||||
|
||||
} finally {
|
||||
|
|
@ -215,7 +215,7 @@ class ChannelPage extends APIForm {
|
|||
this.setSuccess(
|
||||
'Channel created',
|
||||
<span>
|
||||
Your channel was created. View it <Link to={`/videos/channel/${channel.link}/edit`}>here</Link>
|
||||
Your channel was created. View it <Link to={`/videos/channel/${channel.id}/edit`}>here</Link>
|
||||
</span>
|
||||
);
|
||||
} else {
|
||||
|
|
@ -262,12 +262,12 @@ class ChannelPage extends APIForm {
|
|||
|
||||
downloadChannel = async (e) => {
|
||||
e.preventDefault();
|
||||
return await downloadChannel(this.props.match.params.channel_link);
|
||||
return await downloadChannel(this.props.match.params.channel_id);
|
||||
}
|
||||
|
||||
refreshChannel = async (e) => {
|
||||
e.preventDefault();
|
||||
return await refreshChannel(this.props.match.params.channel_link);
|
||||
return await refreshChannel(this.props.match.params.channel_id);
|
||||
}
|
||||
|
||||
handleDeleteButton = (e) => {
|
||||
|
|
@ -470,8 +470,8 @@ export function NewChannel(props) {
|
|||
class ChannelRow extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.editTo = `/videos/channel/${props.channel.link}/edit`;
|
||||
this.videosTo = `/videos/channel/${props.channel.link}/video`;
|
||||
this.editTo = `/videos/channel/${props.channel.id}/edit`;
|
||||
this.videosTo = `/videos/channel/${props.channel.id}/video`;
|
||||
}
|
||||
|
||||
render() {
|
||||
|
|
@ -646,7 +646,7 @@ export class Channels extends React.Component {
|
|||
<Table striped basic size='large'>
|
||||
{tableHeader}
|
||||
<Table.Body>
|
||||
{this.state.results.map((channel) => <ChannelRow key={channel.link}
|
||||
{this.state.results.map((channel) => <ChannelRow key={channel.id}
|
||||
channel={channel}/>)}
|
||||
</Table.Body>
|
||||
</Table>
|
||||
|
|
@ -655,7 +655,7 @@ export class Channels extends React.Component {
|
|||
<Table striped basic unstackable size='small'>
|
||||
<Table.Body>
|
||||
{this.state.results.map((channel) =>
|
||||
<MobileChannelRow key={channel.link} channel={channel}/>)}
|
||||
<MobileChannelRow key={channel.id} channel={channel}/>)}
|
||||
</Table.Body>
|
||||
</Table>
|
||||
</Responsive>
|
||||
|
|
|
|||
|
|
@ -135,8 +135,8 @@ export function VideoCard({video}) {
|
|||
let channel = video.channel ? video.channel : null;
|
||||
let channel_url = null;
|
||||
if (channel) {
|
||||
channel_url = `/videos/channel/${channel.link}/video`;
|
||||
video_url = `/videos/channel/${channel.link}/video/${video.id}`;
|
||||
channel_url = `/videos/channel/${channel.id}/video`;
|
||||
video_url = `/videos/channel/${channel.id}/video/${video.id}`;
|
||||
}
|
||||
let poster_url = video.poster_path ? `/media/${encodeURIComponent(video.poster_path)}` : null;
|
||||
|
||||
|
|
|
|||
|
|
@ -76,8 +76,8 @@ function VideoPage(props) {
|
|||
</Tab.Pane>
|
||||
};
|
||||
|
||||
let statisticsPane = {
|
||||
menuItem: 'Statistics', render: () => <Tab.Pane>
|
||||
let aboutPane = {
|
||||
menuItem: 'About', render: () => <Tab.Pane>
|
||||
<h3>Size</h3>
|
||||
<p>{video.size ? humanFileSize(video.size) : '???'}</p>
|
||||
|
||||
|
|
@ -104,7 +104,7 @@ function VideoPage(props) {
|
|||
</Tab.Pane>
|
||||
};
|
||||
|
||||
let tabPanes = [descriptionPane, statisticsPane, captionsPane];
|
||||
let tabPanes = [descriptionPane, aboutPane, captionsPane];
|
||||
|
||||
return (
|
||||
<>
|
||||
|
|
@ -135,7 +135,7 @@ function VideoPage(props) {
|
|||
<h2>{video.title || video.video_path}</h2>
|
||||
{video.upload_date && <h3>{uploadDate(video.upload_date)}</h3>}
|
||||
<h3>
|
||||
{channel && <Link to={`/videos/channel/${channel.link}/video`}>
|
||||
{channel && <Link to={`/videos/channel/${channel.id}/video`}>
|
||||
{channel.name}
|
||||
</Link>}
|
||||
</h3>
|
||||
|
|
|
|||
|
|
@ -103,8 +103,8 @@ export class VideoWrapper extends React.Component {
|
|||
async fetchVideo() {
|
||||
// Get and display the Video specified in the Router match
|
||||
let [video, prev, next] = await getVideo(this.props.match.params.video_id);
|
||||
let channel_link = this.props.match.params.channel_link;
|
||||
let channel = channel_link ? await getChannel(channel_link) : null;
|
||||
let channel_id = this.props.match.params.channel_id;
|
||||
let channel = channel_id ? await getChannel(channel_id) : null;
|
||||
let no_channel = false;
|
||||
if (!channel) {
|
||||
no_channel = true;
|
||||
|
|
@ -222,7 +222,7 @@ class Videos extends React.Component {
|
|||
}
|
||||
|
||||
async componentDidMount() {
|
||||
if (this.props.match.params.channel_link) {
|
||||
if (this.props.match.params.channel_id) {
|
||||
await this.fetchChannel();
|
||||
} else {
|
||||
await this.fetchVideos();
|
||||
|
|
@ -233,7 +233,7 @@ class Videos extends React.Component {
|
|||
async componentDidUpdate(prevProps, prevState, snapshot) {
|
||||
let params = this.props.match.params;
|
||||
|
||||
let channelChanged = params.channel_link !== prevProps.match.params.channel_link;
|
||||
let channelChanged = params.channel_id !== prevProps.match.params.channel_id;
|
||||
let pageChanged = (
|
||||
prevState.activePage !== this.state.activePage ||
|
||||
prevState.searchOrder !== this.state.searchOrder ||
|
||||
|
|
@ -269,10 +269,10 @@ class Videos extends React.Component {
|
|||
|
||||
async fetchChannel() {
|
||||
// Get and display the channel specified in the Router match
|
||||
let channel_link = this.props.match.params.channel_link;
|
||||
let channel_id = this.props.match.params.channel_id;
|
||||
let channel = null;
|
||||
if (channel_link) {
|
||||
channel = await getChannel(channel_link);
|
||||
if (channel_id) {
|
||||
channel = await getChannel(channel_id);
|
||||
}
|
||||
this.setState({
|
||||
channel,
|
||||
|
|
@ -300,13 +300,13 @@ class Videos extends React.Component {
|
|||
async fetchVideos() {
|
||||
this.setState({videos: null});
|
||||
let offset = this.state.limit * this.state.activePage - this.state.limit;
|
||||
let channel_link = this.state.channel ? this.state.channel.link : null;
|
||||
let channel_id = this.state.channel ? this.state.channel.id : null;
|
||||
let {queryStr, searchOrder, limit} = this.state;
|
||||
|
||||
// Pass any enabled filters to the search.
|
||||
let filtersArr = this.enabledFilters();
|
||||
|
||||
let [videos, total] = await searchVideos(offset, limit, channel_link, queryStr, searchOrder, filtersArr);
|
||||
let [videos, total] = await searchVideos(offset, limit, channel_id, queryStr, searchOrder, filtersArr);
|
||||
|
||||
let totalPages = Math.round(total / this.state.limit) || 1;
|
||||
this.setState({videos, totalPages});
|
||||
|
|
@ -428,7 +428,7 @@ class Videos extends React.Component {
|
|||
<>
|
||||
|
||||
|
||||
<Link to={`/videos/channel/${channel.link}/edit`}>
|
||||
<Link to={`/videos/channel/${channel.id}/edit`}>
|
||||
<Icon name="edit"/>
|
||||
</Link>
|
||||
</>
|
||||
|
|
@ -527,7 +527,7 @@ export function VideosRoute(props) {
|
|||
<Route path='/videos/channel' exact component={Channels}/>
|
||||
<Route path='/videos/manage' exact component={ManageTab}/>
|
||||
<Route path='/videos/channel/new' exact component={NewChannel}/>
|
||||
<Route path='/videos/channel/:channel_link/edit' exact
|
||||
<Route path='/videos/channel/:channel_id/edit' exact
|
||||
component={(i) =>
|
||||
<EditChannel
|
||||
match={i.match}
|
||||
|
|
@ -535,7 +535,7 @@ export function VideosRoute(props) {
|
|||
/>
|
||||
}
|
||||
/>
|
||||
<Route path='/videos/channel/:channel_link/video' exact component={Videos}/>
|
||||
<Route path='/videos/channel/:channel_id/video' exact component={Videos}/>
|
||||
</PageContainer>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -609,7 +609,6 @@ def search(search_str: str, domain: str, limit: int, offset: int) -> Tuple[List[
|
|||
OFFSET %(offset)s
|
||||
LIMIT %(limit)s
|
||||
'''
|
||||
print(stmt, offset, limit)
|
||||
curs.execute(stmt, params)
|
||||
results = [dict(i) for i in curs.fetchall()]
|
||||
total = results[0]['total'] if results else 0
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from wrolpi.vars import PYTEST
|
|||
|
||||
try:
|
||||
import magic
|
||||
|
||||
have_magic = True
|
||||
except ImportError:
|
||||
logger.error('Could not import magic. Is libmagic installed?')
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from unittest import mock
|
|||
import pytest
|
||||
|
||||
from wrolpi.errors import API_ERRORS, WROLModeEnabled
|
||||
from wrolpi.test.common import assert_dict_contains, skip_circleci
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
|
|
@ -128,8 +128,6 @@ def do_search(test_client, search_str, total, expected):
|
|||
assert file['path'] == file['key']
|
||||
|
||||
|
||||
# TODO this fails on CircleCI. The order is wrong. Why?
|
||||
@skip_circleci
|
||||
def test_files_search(test_session, test_client, make_files_structure):
|
||||
# You can search an empty directory.
|
||||
do_search(test_client, 'nothing', 0, [])
|
||||
|
|
@ -154,8 +152,8 @@ def test_files_search(test_session, test_client, make_files_structure):
|
|||
do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)])
|
||||
do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)])
|
||||
do_search(test_client, 'baz', 2, [
|
||||
dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736),
|
||||
dict(path='baz.mp4', mimetype='video/mp4', size=1055736),
|
||||
dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736),
|
||||
])
|
||||
do_search(test_client, 'two', 1, [dict(path='baz baz two.mp4', mimetype='video/mp4', size=1055736)])
|
||||
do_search(test_client, 'nothing', 0, [])
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
from functools import wraps
|
||||
from http import HTTPStatus
|
||||
from typing import List
|
||||
|
||||
from sanic import Blueprint, response
|
||||
from sanic.request import Request
|
||||
|
|
@ -32,12 +33,12 @@ refresh_queue, refresh_event = create_websocket_feed('refresh', '/feeds/refresh'
|
|||
|
||||
|
||||
@content_bp.post('/refresh')
|
||||
@content_bp.post('/refresh/<link:str>')
|
||||
@content_bp.post('/refresh/<channel_id:str>')
|
||||
@openapi.description('Search for videos that have previously been downloaded and stored.')
|
||||
@openapi.response(HTTPStatus.OK, schema.StreamResponse)
|
||||
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
|
||||
@wrol_mode_check
|
||||
async def refresh(_, link: str = None):
|
||||
async def refresh(_, channel_id: int = None):
|
||||
refresh_logger = logger.getChild('refresh')
|
||||
stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/refresh')
|
||||
|
||||
|
|
@ -51,8 +52,8 @@ async def refresh(_, link: str = None):
|
|||
try:
|
||||
refresh_logger.info('refresh started')
|
||||
|
||||
channel_links = [link] if link else None
|
||||
await refresh_videos(channel_links)
|
||||
channel_ids = [channel_id] if channel_id else None
|
||||
await refresh_videos(channel_ids)
|
||||
|
||||
refresh_logger.info('refresh complete')
|
||||
except Exception:
|
||||
|
|
@ -66,34 +67,19 @@ async def refresh(_, link: str = None):
|
|||
return response.json({'code': 'stream-started', 'stream_url': stream_url})
|
||||
|
||||
|
||||
download_queue, download_event = create_websocket_feed('download', '/feeds/download', content_bp)
|
||||
|
||||
|
||||
@content_bp.post('/download')
|
||||
@content_bp.post('/download/<link:str>')
|
||||
@openapi.description('Update channel catalogs, download any missing videos')
|
||||
@content_bp.post('/download/<channel_id:int>')
|
||||
@openapi.description('Update a channel catalog, download any missing videos')
|
||||
@openapi.response(HTTPStatus.OK, schema.StreamResponse)
|
||||
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
|
||||
@wrol_mode_check
|
||||
def download(_, link: str = None):
|
||||
download_logger = logger.getChild('download')
|
||||
|
||||
stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/download')
|
||||
# Only one download can run at a time
|
||||
if download_event.is_set():
|
||||
return response.json({'error': 'download already running', 'stream_url': stream_url}, HTTPStatus.CONFLICT)
|
||||
if refresh_event.is_set():
|
||||
return response.json({'error': 'Refresh is running. Cannot download.'})
|
||||
|
||||
channel_lib.download_channel(link)
|
||||
|
||||
download_logger.debug('do_download scheduled')
|
||||
return response.json({'code': 'stream-started', 'stream_url': stream_url})
|
||||
def download(_, channel_id: int = None):
|
||||
channel_lib.download_channel(channel_id)
|
||||
return response.empty()
|
||||
|
||||
|
||||
@wraps(lib.refresh_videos)
|
||||
async def refresh_videos(channel_links: list = None):
|
||||
return lib.refresh_videos(channel_links=channel_links)
|
||||
async def refresh_videos(channel_ids: List[int] = None):
|
||||
return lib.refresh_videos(channel_ids=channel_ids)
|
||||
|
||||
|
||||
@content_bp.post('/favorite')
|
||||
|
|
|
|||
|
|
@ -7,11 +7,13 @@ from sanic_ext import validate
|
|||
from sanic_ext.extensions.openapi import openapi
|
||||
|
||||
from wrolpi.common import logger, wrol_mode_check, make_media_directory, \
|
||||
get_media_directory
|
||||
get_media_directory, run_after
|
||||
from wrolpi.root_api import json_response
|
||||
from wrolpi.schema import JSONErrorResponse
|
||||
from wrolpi.vars import PYTEST
|
||||
from . import lib
|
||||
from .. import schema
|
||||
from ..lib import save_channels_config
|
||||
|
||||
channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels')
|
||||
|
||||
|
|
@ -26,11 +28,11 @@ async def get_channels(_: Request):
|
|||
return json_response({'channels': channels})
|
||||
|
||||
|
||||
@channel_bp.get('/<link:str>')
|
||||
@channel_bp.get('/<channel_id:int>')
|
||||
@openapi.description('Get a Channel')
|
||||
@openapi.response(HTTPStatus.OK, schema.ChannelResponse)
|
||||
def channel_get(_: Request, link: str):
|
||||
channel = lib.get_channel(link=link)
|
||||
def channel_get(_: Request, channel_id: int = None):
|
||||
channel = lib.get_channel(channel_id=channel_id)
|
||||
channel.pop('info_json')
|
||||
return json_response({'channel': channel})
|
||||
|
||||
|
|
@ -49,20 +51,19 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
|
|||
if not body.directory.is_dir() and body.mkdir:
|
||||
make_media_directory(body.directory)
|
||||
|
||||
bd_edit = body.__dict__
|
||||
bd_edit['name'] = bd_edit['name'].strip()
|
||||
channel = lib.create_channel(bd_edit) # TODO don't use the dataclass as a dict.
|
||||
channel = lib.create_channel(data=body, return_dict=False)
|
||||
|
||||
# Refresh the videos asynchronously
|
||||
from ..api import refresh_videos
|
||||
coro = refresh_videos([channel['link']])
|
||||
asyncio.ensure_future(coro)
|
||||
if not PYTEST:
|
||||
from ..api import refresh_videos
|
||||
coro = refresh_videos([channel.id])
|
||||
asyncio.ensure_future(coro)
|
||||
|
||||
return response.json({'success': 'Channel created successfully'}, HTTPStatus.CREATED,
|
||||
{'Location': f'/api/videos/channels/{channel["link"]}'})
|
||||
{'Location': f'/api/videos/channels/{channel.id}'})
|
||||
|
||||
|
||||
@channel_bp.put('/<link:str>')
|
||||
@channel_bp.put('/<channel_id:int>')
|
||||
@openapi.definition(
|
||||
description='Update a Channel',
|
||||
body=schema.ChannelPutRequest,
|
||||
|
|
@ -70,17 +71,18 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
|
|||
@openapi.response(HTTPStatus.NO_CONTENT)
|
||||
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
|
||||
@validate(schema.ChannelPutRequest)
|
||||
@run_after(save_channels_config)
|
||||
@wrol_mode_check
|
||||
def channel_update(_: Request, link: str, body: schema.ChannelPutRequest):
|
||||
channel = lib.update_channel(body.__dict__, link)
|
||||
def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest):
|
||||
channel = lib.update_channel(data=body, channel_id=channel_id)
|
||||
return response.raw('', HTTPStatus.NO_CONTENT,
|
||||
headers={'Location': f'/api/videos/channels/{channel.link}'})
|
||||
headers={'Location': f'/api/videos/channels/{channel.id}'})
|
||||
|
||||
|
||||
@channel_bp.delete('/<link:str>')
|
||||
@channel_bp.delete('/<channel_id:int>')
|
||||
@openapi.description('Delete a Channel')
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
@wrol_mode_check
|
||||
def channel_delete(_: Request, link: str):
|
||||
lib.delete_channel(link)
|
||||
def channel_delete(_: Request, channel_id: int):
|
||||
lib.delete_channel(channel_id=channel_id)
|
||||
return response.raw('', HTTPStatus.NO_CONTENT)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
from collections import defaultdict
|
||||
from datetime import timedelta
|
||||
from typing import List, Dict, Union, Generator
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Union
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from wrolpi.common import sanitize_link, run_after, get_relative_to_media_directory, make_media_directory, logger
|
||||
from wrolpi.dates import today
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.common import run_after, logger, \
|
||||
get_media_directory
|
||||
from wrolpi.db import get_db_curs, optional_session
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.errors import UnknownChannel, UnknownDirectory, APIError, ValidationError, InvalidDownload
|
||||
from .. import schema
|
||||
from ..common import check_for_channel_conflicts
|
||||
from ..lib import save_channels_config
|
||||
from ..models import Channel
|
||||
|
|
@ -25,7 +25,7 @@ async def get_minimal_channels() -> List[dict]:
|
|||
# Get all channels, even if they don't have videos.
|
||||
stmt = '''
|
||||
SELECT
|
||||
c.id, name, link, directory, url, download_frequency, info_date,
|
||||
c.id, name, directory, url, download_frequency, info_date,
|
||||
(select next_download from download d where d.url=c.url) AS next_download
|
||||
FROM
|
||||
channel AS c
|
||||
|
|
@ -72,148 +72,126 @@ async def get_channels_video_count() -> Dict[int, int]:
|
|||
COD = Union[dict, Channel]
|
||||
|
||||
|
||||
def get_channel(source_id: str = None, link: str = None, url: str = None, return_dict: bool = True) -> COD:
|
||||
@optional_session
|
||||
def get_channel(session: Session, *, channel_id: int = None, source_id: str = None, url: str = None,
|
||||
directory: str = None, name: str = None, return_dict: bool = True) -> COD:
|
||||
"""
|
||||
Attempt to find a Channel using the provided params. The params are in order of reliability.
|
||||
|
||||
Raises UnknownChannel if no channel is found.
|
||||
"""
|
||||
with get_db_session() as session:
|
||||
channel: COD = None # noqa
|
||||
# Try by source_id first.
|
||||
if source_id:
|
||||
channel = session.query(Channel).filter_by(source_id=source_id).one_or_none()
|
||||
if not channel and link:
|
||||
channel = session.query(Channel).filter_by(link=link).one_or_none()
|
||||
if not channel and url:
|
||||
channel = session.query(Channel).filter_by(url=url).one_or_none()
|
||||
channel: COD = None # noqa
|
||||
# Try to find the channel by the most reliable methods first.
|
||||
if channel_id:
|
||||
channel = session.query(Channel).filter_by(id=channel_id).one_or_none()
|
||||
if not channel and source_id:
|
||||
channel = session.query(Channel).filter_by(source_id=source_id).one_or_none()
|
||||
if not channel and url:
|
||||
channel = session.query(Channel).filter_by(url=url).one_or_none()
|
||||
if not channel and directory:
|
||||
directory = Path(directory)
|
||||
if not directory.is_absolute():
|
||||
directory = get_media_directory() / directory
|
||||
channel = session.query(Channel).filter_by(directory=directory).one_or_none()
|
||||
if not channel and name:
|
||||
channel = session.query(Channel).filter_by(name=name).one_or_none()
|
||||
|
||||
if not channel:
|
||||
raise UnknownChannel(f'No channel matches {link=} {source_id=} {url=}')
|
||||
if not channel:
|
||||
raise UnknownChannel(f'No channel matches {channel_id=} {source_id=} {url=} {directory=}')
|
||||
|
||||
logger.debug(f'Found {channel=} using {source_id=} {link=} {url=}')
|
||||
if return_dict:
|
||||
statistics = channel.get_statistics()
|
||||
channel = channel.dict()
|
||||
channel['statistics'] = statistics
|
||||
return channel
|
||||
|
||||
|
||||
def _spread_by_frequency(channels: List[Channel]) -> Generator[Dict, None, None]:
|
||||
channels_by_frequency = defaultdict(lambda: [])
|
||||
for channel in channels:
|
||||
channels_by_frequency[channel.download_frequency].append(channel)
|
||||
|
||||
for frequency, channels in channels_by_frequency.items():
|
||||
# The seconds between each download.
|
||||
chunk = frequency // len(channels)
|
||||
for channel in channels:
|
||||
# My position in the channel group.
|
||||
index = channels.index(channel)
|
||||
# My next download will be distributed by my frequency and my position.
|
||||
position = chunk * index
|
||||
next_download = today() + timedelta(seconds=frequency + position)
|
||||
yield dict(url=channel.url, frequency=frequency, next_download=next_download)
|
||||
logger.debug(f'Found {channel=} using {channel_id=} {source_id=} {url=} {directory=}')
|
||||
if return_dict:
|
||||
statistics = channel.get_statistics()
|
||||
channel = channel.dict()
|
||||
channel['statistics'] = statistics
|
||||
return channel
|
||||
|
||||
|
||||
@run_after(save_channels_config)
|
||||
def update_channel(data: dict, link: str) -> Channel:
|
||||
@optional_session
|
||||
def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
|
||||
"""Update a Channel's DB record"""
|
||||
with get_db_session(commit=True) as session:
|
||||
try:
|
||||
channel: Channel = session.query(Channel).filter_by(link=link).one()
|
||||
except NoResultFound:
|
||||
raise UnknownChannel()
|
||||
try:
|
||||
channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
|
||||
except NoResultFound:
|
||||
raise UnknownChannel()
|
||||
|
||||
# Only update directory if it was empty
|
||||
if data.get('directory') and not channel.directory:
|
||||
try:
|
||||
data['directory'] = get_relative_to_media_directory(data['directory'])
|
||||
except UnknownDirectory:
|
||||
if data['mkdir']:
|
||||
make_media_directory(data['directory'])
|
||||
data['directory'] = get_relative_to_media_directory(data['directory'])
|
||||
else:
|
||||
raise
|
||||
# Only update directory if it was empty
|
||||
if data.directory and not channel.directory:
|
||||
data.directory = get_media_directory() / data.directory
|
||||
if not data.directory.is_dir():
|
||||
if data.mkdir:
|
||||
data.directory.mkdir()
|
||||
else:
|
||||
raise UnknownDirectory()
|
||||
|
||||
if 'directory' in data:
|
||||
data['directory'] = str(data['directory'])
|
||||
# Verify that the URL/Name/directory aren't taken
|
||||
check_for_channel_conflicts(
|
||||
session,
|
||||
id_=channel.id,
|
||||
url=data.url,
|
||||
name=data.name,
|
||||
directory=data.directory,
|
||||
)
|
||||
|
||||
if data.get('match_regex') in ('None', 'null'):
|
||||
data['match_regex'] = None
|
||||
# Apply the changes now that we've OK'd them
|
||||
channel.update(data.__dict__)
|
||||
|
||||
data['link'] = data.get('link') or link
|
||||
|
||||
# Verify that the URL/Name/Link aren't taken
|
||||
check_for_channel_conflicts(
|
||||
session,
|
||||
id_=channel.id,
|
||||
url=data.get('url'),
|
||||
name=data.get('name'),
|
||||
link=data['link'],
|
||||
directory=data.get('directory'),
|
||||
)
|
||||
|
||||
# Apply the changes now that we've OK'd them
|
||||
channel.update(data)
|
||||
session.commit()
|
||||
|
||||
return channel
|
||||
|
||||
|
||||
@run_after(save_channels_config)
|
||||
def create_channel(data: dict, return_dict: bool = True) -> Union[Channel, dict]:
|
||||
@optional_session
|
||||
def create_channel(session: Session, data: schema.ChannelPostRequest, return_dict: bool = True) -> Union[Channel, dict]:
|
||||
"""
|
||||
Create a new Channel. Check for conflicts with existing Channels.
|
||||
"""
|
||||
with get_db_session(commit=True) as session:
|
||||
link = sanitize_link(data['name'])
|
||||
try:
|
||||
# Verify that the URL/Name/Link aren't taken
|
||||
check_for_channel_conflicts(
|
||||
session,
|
||||
url=data.get('url'),
|
||||
name=data['name'],
|
||||
link=link,
|
||||
directory=str(data['directory']),
|
||||
source_id=data.get('source_id'),
|
||||
)
|
||||
except APIError as e:
|
||||
raise ValidationError() from e
|
||||
|
||||
channel = Channel(
|
||||
name=data['name'],
|
||||
url=data.get('url'),
|
||||
match_regex=data.get('match_regex'),
|
||||
link=link,
|
||||
directory=str(data['directory']),
|
||||
download_frequency=data.get('download_frequency'),
|
||||
source_id=data.get('source_id'),
|
||||
try:
|
||||
# Verify that the URL/Name/directory aren't taken
|
||||
check_for_channel_conflicts(
|
||||
session,
|
||||
url=data.url,
|
||||
name=data.name,
|
||||
directory=data.directory,
|
||||
source_id=data.source_id,
|
||||
)
|
||||
session.add(channel)
|
||||
session.commit()
|
||||
session.flush()
|
||||
session.refresh(channel)
|
||||
except APIError as e:
|
||||
raise ValidationError() from e
|
||||
|
||||
if return_dict:
|
||||
return channel.dict()
|
||||
else:
|
||||
return channel
|
||||
channel = Channel(
|
||||
name=data.name,
|
||||
url=data.url,
|
||||
match_regex=data.match_regex,
|
||||
directory=data.directory, # noqa
|
||||
download_frequency=data.download_frequency,
|
||||
source_id=data.source_id,
|
||||
)
|
||||
session.add(channel)
|
||||
session.commit()
|
||||
session.flush()
|
||||
session.refresh(channel)
|
||||
|
||||
return channel.dict() if return_dict else channel
|
||||
|
||||
|
||||
@run_after(save_channels_config)
|
||||
def delete_channel(link):
|
||||
with get_db_session(commit=True) as session:
|
||||
try:
|
||||
channel = session.query(Channel).filter_by(link=link).one()
|
||||
except NoResultFound:
|
||||
raise UnknownChannel()
|
||||
@optional_session
|
||||
def delete_channel(session: Session, *, channel_id: int):
|
||||
try:
|
||||
channel = session.query(Channel).filter_by(id=channel_id).one()
|
||||
except NoResultFound:
|
||||
raise UnknownChannel()
|
||||
|
||||
channel.delete_with_videos()
|
||||
channel.delete_with_videos()
|
||||
|
||||
session.commit()
|
||||
|
||||
|
||||
def download_channel(link: str):
|
||||
def download_channel(id_: int):
|
||||
"""Create a Download record for a Channel's entire catalog. Start downloading."""
|
||||
channel = get_channel(link=link, return_dict=False)
|
||||
channel = get_channel(channel_id=id_, return_dict=False)
|
||||
session = Session.object_session(channel)
|
||||
download = channel.get_download()
|
||||
if not download:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
|
|
@ -11,301 +9,116 @@ import mock
|
|||
from modules.videos.channel.lib import delete_channel
|
||||
from modules.videos.common import get_no_channel_directory
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.test.common import create_channel_structure
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.downloader import download_manager, Download
|
||||
from wrolpi.errors import UnknownFile
|
||||
from wrolpi.root_api import api_app
|
||||
from wrolpi.test.common import wrap_test_db, TestAPI
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
||||
class TestVideoAPI(TestAPI):
|
||||
def test_get_channels(test_directory, channel_factory, test_client):
|
||||
channel_factory()
|
||||
channel_factory()
|
||||
channel_factory()
|
||||
request, response = test_client.get('/api/videos/channels')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['channels']) == 3
|
||||
|
||||
@wrap_test_db
|
||||
@create_channel_structure(
|
||||
{
|
||||
'Foo': ['vid1.mp4'],
|
||||
'Bar': ['vid2.mp4'],
|
||||
'Baz': [],
|
||||
}
|
||||
|
||||
def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel):
|
||||
# A regular user-defined channel
|
||||
vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
|
||||
vid1.touch()
|
||||
vid2 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title2.mp4'
|
||||
vid2.touch()
|
||||
|
||||
# The special NO CHANNEL directory.
|
||||
no_channel_path = get_no_channel_directory()
|
||||
vid3 = no_channel_path / 'some video.mp4'
|
||||
vid3.touch()
|
||||
no_channel_subdir = no_channel_path / 'subdir'
|
||||
no_channel_subdir.mkdir()
|
||||
vid4 = no_channel_subdir / 'other video.mp4'
|
||||
vid4.touch()
|
||||
|
||||
test_client.post('/api/videos/refresh')
|
||||
|
||||
videos = test_session.query(Video).order_by(Video.title).all()
|
||||
assert len(videos) == 4
|
||||
|
||||
# Add some fake videos, they should be removed.
|
||||
test_session.add(Video(title='vid5'))
|
||||
test_session.add(Video(title='vid6', channel_id=simple_channel.id))
|
||||
test_session.commit()
|
||||
|
||||
videos = test_session.query(Video).order_by(Video.title).all()
|
||||
assert len(videos) == 6
|
||||
|
||||
test_client.post('/api/videos/refresh')
|
||||
|
||||
videos = test_session.query(Video).order_by(Video.title).all()
|
||||
assert len(videos) == 4
|
||||
|
||||
|
||||
def test_get_video(test_client, test_session, simple_channel, video_factory):
|
||||
"""
|
||||
Test that you get can information about a video. Test that video file can be gotten.
|
||||
"""
|
||||
now_ = now()
|
||||
video1 = video_factory(channel_id=simple_channel.id)
|
||||
video1.upload_date, video1.title = now_, 'vid1'
|
||||
video2 = video_factory(channel_id=simple_channel.id)
|
||||
video2.upload_date, video2.title = now_ + timedelta(seconds=1), 'vid2'
|
||||
test_session.commit()
|
||||
|
||||
# Test that a 404 is returned when no video exists
|
||||
_, response = test_client.get('/api/videos/video/10')
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
|
||||
assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''}
|
||||
|
||||
# Get the video info we inserted
|
||||
_, response = test_client.get('/api/videos/video/1')
|
||||
assert response.status_code == HTTPStatus.OK, response.json
|
||||
assert_dict_contains(response.json['video'], {'title': 'vid1'})
|
||||
|
||||
# The next video is included.
|
||||
assert response.json['prev'] is None
|
||||
assert_dict_contains(response.json['next'], {'title': 'vid2'})
|
||||
|
||||
|
||||
def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel):
|
||||
"""A channel does not require a download frequency."""
|
||||
# No downloads are scheduled.
|
||||
assert len(download_manager.get_downloads(test_session)) == 0
|
||||
|
||||
# Get the Channel
|
||||
request, response = test_client.get('/api/videos/channels/1')
|
||||
channel = response.json['channel']
|
||||
assert channel['download_frequency'] is None
|
||||
|
||||
# Update the Channel with a frequency.
|
||||
new_channel = dict(
|
||||
directory=channel['directory'],
|
||||
name=channel['name'],
|
||||
url=channel['url'],
|
||||
download_frequency=10,
|
||||
)
|
||||
def test_get_channels(self, tempdir):
|
||||
request, response = api_app.test_client.get('/api/videos/channels')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
# Channels are sorted by name
|
||||
self.assertDictContains(response.json['channels'][0], {'id': 2, 'name': 'Bar', 'video_count': 1})
|
||||
self.assertDictContains(response.json['channels'][1], {'id': 3, 'name': 'Baz', 'video_count': 0})
|
||||
self.assertDictContains(response.json['channels'][2], {'id': 1, 'name': 'Foo', 'video_count': 1})
|
||||
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
|
||||
content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
||||
|
||||
@wrap_test_db
|
||||
def test_channel(self):
|
||||
channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name
|
||||
channel_directory_name = channel_directory.split('/')[-1]
|
||||
pathlib.Path(channel_directory).mkdir()
|
||||
# Download is scheduled.
|
||||
assert len(download_manager.get_downloads(test_session)) == 1
|
||||
|
||||
new_channel = dict(
|
||||
directory=channel_directory,
|
||||
match_regex='asdf',
|
||||
name=' Example Channel 1 ',
|
||||
url='https://example.com/channel1',
|
||||
)
|
||||
|
||||
# Channel doesn't exist
|
||||
request, response = api_app.test_client.get('/api/videos/channels/examplechannel1')
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
|
||||
|
||||
# Create it
|
||||
request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.CREATED, response.json
|
||||
location = response.headers['Location']
|
||||
request, response = api_app.test_client.get(location)
|
||||
created = response.json['channel']
|
||||
self.assertIsNotNone(created)
|
||||
self.assertIsNotNone(created['id'])
|
||||
|
||||
# Channel name leading/trailing whitespace should be stripped
|
||||
assert created['name'] == 'Example Channel 1'
|
||||
|
||||
# Channel directory should be relative to the media directory
|
||||
assert not pathlib.Path(created['directory']).is_absolute(), \
|
||||
f'Channel directory is absolute: {created["directory"]}'
|
||||
|
||||
# Get the link that was computed
|
||||
new_channel['link'] = response.json['channel']['link']
|
||||
assert new_channel['link']
|
||||
|
||||
# Can't create it again
|
||||
request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
# Update it
|
||||
new_channel['name'] = 'Example Channel 2'
|
||||
new_channel['directory'] = str(new_channel['directory'])
|
||||
request, response = api_app.test_client.put(location, content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
|
||||
request, response = api_app.test_client.get(location)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
self.assertDictContains(response.json['channel'], {
|
||||
'id': 1,
|
||||
'name': 'Example Channel 2',
|
||||
'directory': channel_directory_name,
|
||||
'match_regex': 'asdf',
|
||||
'url': 'https://example.com/channel1',
|
||||
})
|
||||
|
||||
# Can't update channel that doesn't exist
|
||||
request, response = api_app.test_client.put('/api/videos/channels/doesnt_exist',
|
||||
content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
# Delete the new channel
|
||||
request, response = api_app.test_client.delete(location)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Cant delete it again
|
||||
request, response = api_app.test_client.delete(location)
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
@wrap_test_db
|
||||
def test_refresh_no_channel(self):
|
||||
with get_db_session() as session:
|
||||
videos_path = pathlib.Path(self.tmp_dir.name) / 'videos'
|
||||
videos_path.mkdir()
|
||||
|
||||
# A regular user-defined channel
|
||||
channel1_path = videos_path / 'channel1'
|
||||
channel1_path.mkdir()
|
||||
vid1 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
|
||||
vid1.touch()
|
||||
vid2 = channel1_path / 'MyChannelName_20000101_abcdefghijk_title2.mp4'
|
||||
vid2.touch()
|
||||
|
||||
channel1 = Channel(name='channel1', link='channel1', directory=channel1_path)
|
||||
session.add(channel1)
|
||||
session.commit()
|
||||
|
||||
# The special NO CHANNEL directory.
|
||||
no_channel_path = get_no_channel_directory()
|
||||
vid3 = no_channel_path / 'some video.mp4'
|
||||
vid3.touch()
|
||||
no_channel_subdir = no_channel_path / 'subdir'
|
||||
no_channel_subdir.mkdir()
|
||||
vid4 = no_channel_subdir / 'other video.mp4'
|
||||
vid4.touch()
|
||||
|
||||
api_app.test_client.post('/api/videos/refresh')
|
||||
|
||||
videos = session.query(Video).order_by(Video.title).all()
|
||||
self.assertEqual(len(videos), 4)
|
||||
|
||||
# Add some fake videos, they should be removed.
|
||||
session.add(Video(title='vid5'))
|
||||
session.add(Video(title='vid6', channel_id=channel1.id))
|
||||
session.commit()
|
||||
|
||||
videos = session.query(Video).order_by(Video.title).all()
|
||||
self.assertEqual(len(videos), 6)
|
||||
|
||||
api_app.test_client.post('/api/videos/refresh')
|
||||
|
||||
videos = session.query(Video).order_by(Video.title).all()
|
||||
self.assertEqual(len(videos), 4)
|
||||
|
||||
@wrap_test_db
|
||||
def test_get_channel_videos(self):
|
||||
with get_db_session(commit=True) as session:
|
||||
channel1 = Channel(name='Foo', link='foo')
|
||||
channel2 = Channel(name='Bar', link='bar')
|
||||
session.add(channel1)
|
||||
session.add(channel2)
|
||||
session.flush()
|
||||
session.refresh(channel1)
|
||||
session.refresh(channel2)
|
||||
|
||||
# Channels don't have videos yet
|
||||
d = dict(channel_link=channel1.link)
|
||||
request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['videos']) == 0
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo')
|
||||
vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo')
|
||||
session.add(vid1)
|
||||
session.add(vid2)
|
||||
|
||||
# Videos are gotten by their respective channels
|
||||
request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['videos']) == 1
|
||||
assert response.json['totals']['videos'] == 1
|
||||
self.assertDictContains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id))
|
||||
|
||||
d = dict(channel_link=channel2.link)
|
||||
request, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['videos']) == 1
|
||||
self.assertDictContains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id))
|
||||
|
||||
@wrap_test_db
|
||||
def test_get_video(self):
|
||||
"""
|
||||
Test that you get can information about a video. Test that video file can be gotten.
|
||||
"""
|
||||
|
||||
def raise_unknown_file(_):
|
||||
raise UnknownFile()
|
||||
|
||||
with get_db_session(commit=True) as session, \
|
||||
mock.patch('modules.videos.common.get_absolute_video_info_json', raise_unknown_file):
|
||||
channel = Channel(name='Foo', link='foo')
|
||||
session.add(channel)
|
||||
session.flush()
|
||||
session.refresh(channel)
|
||||
now_ = now()
|
||||
session.add(Video(title='vid1', channel_id=channel.id, upload_date=now_))
|
||||
session.add(Video(title='vid2', channel_id=channel.id, upload_date=now_ + timedelta(seconds=1)))
|
||||
|
||||
# Test that a 404 is returned when no video exists
|
||||
_, response = api_app.test_client.get('/api/videos/video/10')
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
|
||||
assert response.json == {'code': 1, 'api_error': 'The video could not be found.', 'message': ''}
|
||||
|
||||
# Get the video info we inserted
|
||||
_, response = api_app.test_client.get('/api/videos/video/1')
|
||||
assert response.status_code == HTTPStatus.OK, response.json
|
||||
self.assertDictContains(response.json['video'], {'title': 'vid1'})
|
||||
|
||||
# The next video is included.
|
||||
self.assertIsNone(response.json['prev'])
|
||||
self.assertDictContains(response.json['next'], {'title': 'vid2'})
|
||||
|
||||
@wrap_test_db
|
||||
def test_get_channel_videos_pagination(self):
|
||||
with get_db_session(commit=True) as session:
|
||||
channel1 = Channel(name='Foo', link='foo')
|
||||
session.add(channel1)
|
||||
session.flush()
|
||||
session.refresh(channel1)
|
||||
|
||||
for i in range(50):
|
||||
session.add(Video(title=f'Foo.Video{i}', channel_id=channel1.id, video_path='foo'))
|
||||
|
||||
channel2 = Channel(name='Bar', link='bar')
|
||||
session.add(channel2)
|
||||
session.flush()
|
||||
session.refresh(channel2)
|
||||
session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo'))
|
||||
|
||||
# Get first, second, third, and empty pages of videos.
|
||||
tests = [
|
||||
# (offset, video_count)
|
||||
(0, 20),
|
||||
(20, 20),
|
||||
(40, 10),
|
||||
(50, 0),
|
||||
]
|
||||
last_ids = []
|
||||
for offset, video_count in tests:
|
||||
d = dict(channel_link=channel1.link, order_by='id', offset=offset)
|
||||
_, response = api_app.test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['videos']) == video_count
|
||||
current_ids = [i['id'] for i in response.json['videos']]
|
||||
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
||||
last_ids = current_ids
|
||||
|
||||
@wrap_test_db
|
||||
def test_channel_no_download_frequency(self):
|
||||
"""A channel does not require a download frequency."""
|
||||
channel_directory = tempfile.TemporaryDirectory(dir=self.tmp_dir.name).name
|
||||
os.mkdir(channel_directory)
|
||||
|
||||
new_channel = dict(
|
||||
directory=channel_directory,
|
||||
name='Example Channel 1',
|
||||
url='https://example.com/channel1',
|
||||
download_frequency=None,
|
||||
)
|
||||
|
||||
# Create the Channel
|
||||
request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.CREATED
|
||||
location = response.headers['Location']
|
||||
|
||||
# No downloads are scheduled.
|
||||
with get_db_session() as session:
|
||||
downloads = list(download_manager.get_new_downloads(session))
|
||||
self.assertEqual(len(downloads), 0)
|
||||
|
||||
# Get the Channel
|
||||
request, response = api_app.test_client.get(location)
|
||||
channel = response.json['channel']
|
||||
self.assertEqual(channel['download_frequency'], None)
|
||||
|
||||
# Update the Channel with a frequency.
|
||||
new_channel = dict(
|
||||
directory=channel['directory'],
|
||||
name=channel['name'],
|
||||
url=channel['url'],
|
||||
download_frequency=10,
|
||||
)
|
||||
request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}',
|
||||
content=json.dumps(new_channel))
|
||||
self.assertEqual(response.status_code, 204, response.json)
|
||||
|
||||
# Remove the frequency.
|
||||
new_channel = dict(
|
||||
directory=channel['directory'],
|
||||
name=channel['name'],
|
||||
url=channel['url'],
|
||||
download_frequency=None,
|
||||
)
|
||||
request, response = api_app.test_client.put(f'/api/videos/channels/{channel["link"]}',
|
||||
content=json.dumps(new_channel))
|
||||
self.assertEqual(response.status_code, 204, response.json)
|
||||
# Remove the frequency.
|
||||
new_channel = dict(
|
||||
directory=channel['directory'],
|
||||
name=channel['name'],
|
||||
url=channel['url'],
|
||||
download_frequency=None,
|
||||
)
|
||||
request, response = test_client.put(f'/api/videos/channels/{simple_channel.id}',
|
||||
content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
||||
|
||||
|
||||
def test_channel_frequency_update(download_channel, test_client, test_session):
|
||||
|
|
@ -321,7 +134,7 @@ def test_channel_frequency_update(download_channel, test_client, test_session):
|
|||
url=download_channel.url,
|
||||
download_frequency=100,
|
||||
)
|
||||
request, response = test_client.put(f'/api/videos/channels/{download_channel.link}', json=data)
|
||||
request, response = test_client.put(f'/api/videos/channels/{download_channel.id}', json=data)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
||||
|
||||
download = download_channel.get_download()
|
||||
|
|
@ -332,20 +145,59 @@ def test_channel_frequency_update(download_channel, test_client, test_session):
|
|||
assert len(list(downloads)) == 1
|
||||
|
||||
|
||||
def test_delete_channel_delete_download(download_channel, test_session):
|
||||
"""
|
||||
Deleting a Channel deletes it's Download.
|
||||
"""
|
||||
downloads = test_session.query(Download).all()
|
||||
assert all(i.url == download_channel.url for i in downloads)
|
||||
def test_channel_download_crud(test_session, simple_channel):
|
||||
"""Modifying a Channel modifies it's Download."""
|
||||
assert simple_channel.url
|
||||
assert simple_channel.download_frequency is None
|
||||
|
||||
delete_channel(download_channel.link)
|
||||
simple_channel.update({'download_frequency': 1000})
|
||||
test_session.commit()
|
||||
|
||||
download: Download = test_session.query(Download).one()
|
||||
assert simple_channel.get_download() == download
|
||||
assert simple_channel.url == download.url
|
||||
assert simple_channel.download_frequency == download.frequency
|
||||
|
||||
# Increase the frequency to 1 second.
|
||||
simple_channel.update({'download_frequency': 1})
|
||||
test_session.commit()
|
||||
assert simple_channel.url == download.url
|
||||
assert simple_channel.download_frequency == download.frequency == 1
|
||||
assert test_session.query(Download).count() == 1
|
||||
|
||||
# Changing the Channel's URL changes the download's URL.
|
||||
simple_channel.update({'url': 'https://example.com/new url'})
|
||||
test_session.commit()
|
||||
assert simple_channel.url == 'https://example.com/new url'
|
||||
assert simple_channel.download_frequency == download.frequency == 1
|
||||
assert test_session.query(Download).count() == 1
|
||||
|
||||
# Removing the Channel's frequency removes the download.
|
||||
simple_channel.update({'download_frequency': None})
|
||||
test_session.commit()
|
||||
assert test_session.query(Download).count() == 0
|
||||
|
||||
# Removing the URL is supported.
|
||||
simple_channel.update({'url': None})
|
||||
test_session.commit()
|
||||
|
||||
# Adding a URL is supported.
|
||||
simple_channel.update({'url': 'https://example.com'})
|
||||
test_session.commit()
|
||||
|
||||
|
||||
def test_delete_channel_delete_download(download_channel, test_session):
|
||||
"""Deleting a Channel deletes it's Download."""
|
||||
downloads = test_session.query(Download).all()
|
||||
assert downloads and all(i.url == download_channel.url for i in downloads)
|
||||
|
||||
delete_channel(channel_id=download_channel.id)
|
||||
|
||||
downloads = test_session.query(Download).all()
|
||||
assert not list(downloads)
|
||||
|
||||
|
||||
def test_video_search(test_session):
|
||||
def test_video_search(test_client, test_session):
|
||||
"""
|
||||
Test that videos can be searched and that their order is by their textsearch rank.
|
||||
"""
|
||||
|
|
@ -363,7 +215,7 @@ def test_video_search(test_session):
|
|||
|
||||
def do_search(search_str, limit=20):
|
||||
d = json.dumps({'search_str': search_str, 'limit': limit})
|
||||
_, resp = api_app.test_client.post('/api/videos/search', content=d)
|
||||
_, resp = test_client.post('/api/videos/search', content=d)
|
||||
return resp
|
||||
|
||||
def search_is_as_expected(resp, expected):
|
||||
|
|
@ -414,7 +266,7 @@ def test_video_file_name(test_session, simple_video, test_client):
|
|||
assert resp.json['video'].get('stem') == 'simple_video'
|
||||
|
||||
|
||||
def test_channel_conflicts(test_session, test_directory):
|
||||
def test_channel_conflicts(test_client, test_session, test_directory):
|
||||
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
||||
pathlib.Path(channel_directory).mkdir()
|
||||
new_channel = dict(
|
||||
|
|
@ -425,7 +277,7 @@ def test_channel_conflicts(test_session, test_directory):
|
|||
)
|
||||
|
||||
def _post_channel(channel):
|
||||
return api_app.test_client.post('/api/videos/channels', content=json.dumps(channel))
|
||||
return test_client.post('/api/videos/channels', content=json.dumps(channel))
|
||||
|
||||
# Create it
|
||||
request, response = _post_channel(new_channel)
|
||||
|
|
@ -443,18 +295,6 @@ def test_channel_conflicts(test_session, test_directory):
|
|||
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
||||
'cause': {'error': 'The channel name is already taken.', 'code': 5}}
|
||||
|
||||
# Name matches when converted to link
|
||||
channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
|
||||
pathlib.Path(channel_directory2).mkdir()
|
||||
new_channel = dict(
|
||||
directory=channel_directory2,
|
||||
name='Example channel 1',
|
||||
)
|
||||
request, response = _post_channel(new_channel)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json == {'error': 'Could not validate the contents of the request', 'code': 10,
|
||||
'cause': {'code': 11, 'error': 'Channel link already used by another channel'}}
|
||||
|
||||
# Directory was already used
|
||||
new_channel = dict(
|
||||
directory=channel_directory,
|
||||
|
|
@ -479,7 +319,7 @@ def test_channel_conflicts(test_session, test_directory):
|
|||
'cause': {'code': 6, 'error': 'The URL is already used by another channel.'}}
|
||||
|
||||
|
||||
def test_channel_empty_url_doesnt_conflict(test_session, test_directory):
|
||||
def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_directory):
|
||||
"""Two channels with empty URLs shouldn't conflict"""
|
||||
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
|
||||
pathlib.Path(channel_directory).mkdir()
|
||||
|
|
@ -488,7 +328,7 @@ def test_channel_empty_url_doesnt_conflict(test_session, test_directory):
|
|||
'name': 'Fooz',
|
||||
'directory': channel_directory,
|
||||
}
|
||||
request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.CREATED, response.json
|
||||
location = response.headers['Location']
|
||||
|
||||
|
|
@ -498,7 +338,7 @@ def test_channel_empty_url_doesnt_conflict(test_session, test_directory):
|
|||
'name': 'Barz',
|
||||
'directory': channel_directory2,
|
||||
}
|
||||
request, response = api_app.test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.CREATED, response.json
|
||||
assert location != response.headers['Location']
|
||||
|
||||
|
|
@ -524,11 +364,11 @@ def test_download_channel_no_refresh(test_session, download_channel, video_downl
|
|||
|
||||
def test_channel_post_directory(test_session, test_client, test_directory):
|
||||
"""A Channel can be created with or without an existing directory."""
|
||||
# Channel can be created with a missing directory.
|
||||
# Channel can be created with a directory which is not on disk.
|
||||
data = dict(name='foo', directory='foo')
|
||||
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.CREATED
|
||||
directory = test_session.query(Channel).filter_by(link='foo').one().directory.path
|
||||
directory = test_session.query(Channel).filter_by(id=1).one().directory.path
|
||||
assert (test_directory / 'foo') == directory
|
||||
assert not directory.is_dir()
|
||||
assert directory.is_absolute()
|
||||
|
|
@ -537,56 +377,147 @@ def test_channel_post_directory(test_session, test_client, test_directory):
|
|||
data = dict(name='bar', directory='bar', mkdir=True)
|
||||
request, response = test_client.post('/api/videos/channels', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.CREATED
|
||||
directory = test_session.query(Channel).filter_by(link='bar').one().directory.path
|
||||
directory = test_session.query(Channel).filter_by(id=2).one().directory.path
|
||||
assert (test_directory / 'bar') == directory
|
||||
assert directory.is_dir()
|
||||
assert directory.is_absolute()
|
||||
|
||||
|
||||
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory):
|
||||
subdir = test_directory / 'subdir'
|
||||
subdir.mkdir()
|
||||
def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
|
||||
request, response = test_client.get(f'/api/videos/channels/{simple_channel.id}')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
# video1 is in a subdirectory.
|
||||
video1 = video_factory(simple_channel.id, True, True, 'jpg')
|
||||
test_session.commit()
|
||||
shutil.move(video1.video_path.path, subdir / video1.video_path.path.name)
|
||||
shutil.move(video1.poster_path.path, subdir / video1.poster_path.path.name)
|
||||
video1.video_path = subdir / video1.video_path.path.name
|
||||
video1.poster_path = None
|
||||
# video2 is in the test directory.
|
||||
video2 = video_factory(simple_channel.id, True, True, 'jpg')
|
||||
video2.poster_path = video1.poster_path = None
|
||||
|
||||
def test_channel_crud(test_session, test_client, test_directory):
|
||||
channel_directory = test_directory / 'channel directory'
|
||||
channel_directory.mkdir()
|
||||
|
||||
new_channel = dict(
|
||||
directory=str(channel_directory),
|
||||
match_regex='asdf',
|
||||
name=' Example Channel 1 ',
|
||||
url='https://example.com/channel1',
|
||||
)
|
||||
|
||||
# Channel doesn't exist
|
||||
request, response = test_client.get('/api/videos/channels/examplechannel1')
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
|
||||
|
||||
# Create it
|
||||
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.CREATED, response.json
|
||||
location = response.headers['Location']
|
||||
|
||||
request, response = test_client.get(location)
|
||||
assert response.status_code == HTTPStatus.OK, response.json
|
||||
created = response.json['channel']
|
||||
assert created
|
||||
assert created['id']
|
||||
# No frequency was provided. No download exists.
|
||||
assert not created['download_frequency']
|
||||
assert test_session.query(Download).count() == 0
|
||||
|
||||
# Channel name leading/trailing whitespace should be stripped
|
||||
assert created['name'] == 'Example Channel 1'
|
||||
|
||||
# Channel directory should be relative to the media directory
|
||||
assert not pathlib.Path(created['directory']).is_absolute(), \
|
||||
f'Channel directory is absolute: {created["directory"]}'
|
||||
|
||||
# Can't create it again
|
||||
request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
# Update it
|
||||
new_channel['name'] = 'Example Channel 2'
|
||||
new_channel['directory'] = str(new_channel['directory']) # noqa
|
||||
request, response = test_client.put(location, content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
|
||||
request, response = test_client.get(location)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
channel = response.json['channel']
|
||||
assert channel['id'] == 1
|
||||
assert channel['name'] == 'Example Channel 2'
|
||||
assert channel['directory'] == channel_directory.name
|
||||
assert channel['match_regex'] == 'asdf'
|
||||
assert channel['url'] == 'https://example.com/channel1'
|
||||
|
||||
# Can't update channel that doesn't exist
|
||||
request, response = test_client.put('/api/videos/channels/doesnt_exist',
|
||||
content=json.dumps(new_channel))
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
# Delete the new channel
|
||||
request, response = test_client.delete(location)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Cant delete it again
|
||||
request, response = test_client.delete(location)
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
def test_search_channel_videos(test_client, test_session):
|
||||
with get_db_session(commit=True) as session:
|
||||
channel1 = Channel(name='Foo')
|
||||
channel2 = Channel(name='Bar')
|
||||
session.add(channel1)
|
||||
session.add(channel2)
|
||||
session.flush()
|
||||
session.refresh(channel1)
|
||||
session.refresh(channel2)
|
||||
|
||||
# Channels don't have videos yet
|
||||
d = dict(channel_id=channel1.id)
|
||||
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['videos']) == 0
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo')
|
||||
vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo')
|
||||
session.add(vid1)
|
||||
session.add(vid2)
|
||||
|
||||
# Videos are gotten by their respective channels
|
||||
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['videos']) == 1
|
||||
assert response.json['totals']['videos'] == 1
|
||||
assert_dict_contains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id))
|
||||
|
||||
d = dict(channel_id=channel2.id)
|
||||
request, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['videos']) == 1
|
||||
assert_dict_contains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id))
|
||||
|
||||
|
||||
def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory):
|
||||
for i in range(50):
|
||||
video_factory(channel_id=simple_channel.id)
|
||||
|
||||
channel2 = Channel(name='Bar')
|
||||
test_session.add(channel2)
|
||||
test_session.flush()
|
||||
test_session.refresh(channel2)
|
||||
video_factory(channel_id=channel2.id)
|
||||
test_session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo'))
|
||||
test_session.commit()
|
||||
|
||||
assert not video1.size, 'video1 should not have size during creation'
|
||||
|
||||
# Create a video not in the DB.
|
||||
vid3 = test_directory / 'vid3.mp4'
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', vid3)
|
||||
|
||||
# Orphan poster should be ignored.
|
||||
orphan_poster = pathlib.Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg')
|
||||
orphan_poster.touch()
|
||||
|
||||
# Create a bogus video in the channel.
|
||||
bogus = Video(video_path='foo', channel_id=simple_channel.id)
|
||||
test_session.add(bogus)
|
||||
|
||||
test_client.post('/api/videos/refresh')
|
||||
|
||||
# Posters were found during refresh.
|
||||
assert video1.poster_path
|
||||
assert 'subdir' in str(video1.poster_path.path)
|
||||
assert video2.poster_path
|
||||
# Missing video3 was found
|
||||
video3: Video = test_session.query(Video).filter_by(id=4).one()
|
||||
assert video3.video_path.path == vid3
|
||||
assert video3.video_path != video1.video_path
|
||||
assert video3.video_path != video2.video_path
|
||||
# Bogus video was removed.
|
||||
assert not any('foo' in str(i.video_path.path) for i in test_session.query(Video).all())
|
||||
# Orphan file was not deleted.
|
||||
assert orphan_poster.is_file(), 'Orphan poster was removed!'
|
||||
|
||||
assert video1.size, 'video1 size was not found'
|
||||
# Get first, second, third, and empty pages of videos.
|
||||
tests = [
|
||||
# (offset, video_count)
|
||||
(0, 20),
|
||||
(20, 20),
|
||||
(40, 10),
|
||||
(50, 0),
|
||||
]
|
||||
last_ids = []
|
||||
for offset, video_count in tests:
|
||||
d = dict(channel_id=simple_channel.id, order_by='id', offset=offset)
|
||||
_, response = test_client.post(f'/api/videos/search', content=json.dumps(d))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['videos']) == video_count
|
||||
current_ids = [i['id'] for i in response.json['videos']]
|
||||
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
||||
last_ids = current_ids
|
||||
|
|
|
|||
59
modules/videos/channel/test/test_lib.py
Normal file
59
modules/videos/channel/test/test_lib.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import pytest
|
||||
|
||||
from modules.videos.channel import lib
|
||||
from wrolpi.errors import UnknownChannel
|
||||
|
||||
|
||||
@pytest.mark.parametrize('params', [
|
||||
dict(),
|
||||
dict(channel_id=None),
|
||||
dict(source_id=None),
|
||||
dict(url=None),
|
||||
dict(directory=None),
|
||||
])
|
||||
def test_get_bad_channel(test_session, test_directory, params):
|
||||
"""A Channel cannot be found without data."""
|
||||
with pytest.raises(UnknownChannel):
|
||||
lib.get_channel(**params)
|
||||
|
||||
|
||||
def test_get_channel(test_session, test_directory, channel_factory):
|
||||
"""`get_channel` can find a Channel in multiple ways."""
|
||||
channel_id_channel = channel_factory()
|
||||
|
||||
source_id_channel = channel_factory()
|
||||
source_id_channel.source_id = 'the source id'
|
||||
|
||||
url_channel = channel_factory()
|
||||
url_channel.url = 'https://example.com/channel'
|
||||
|
||||
directory_channel = channel_factory()
|
||||
directory_channel.directory = test_directory / 'some directory'
|
||||
|
||||
test_session.commit()
|
||||
|
||||
channel = lib.get_channel(channel_id=channel_id_channel.id, return_dict=False)
|
||||
assert channel == channel_id_channel
|
||||
|
||||
channel = lib.get_channel(source_id='the source id', return_dict=False)
|
||||
assert channel == source_id_channel
|
||||
|
||||
channel = lib.get_channel(url='https://example.com/channel', return_dict=False)
|
||||
assert channel == url_channel
|
||||
|
||||
# Directory can be relative, or absolute.
|
||||
channel = lib.get_channel(directory='some directory', return_dict=False)
|
||||
assert channel == directory_channel
|
||||
channel = lib.get_channel(directory=f'{test_directory}/some directory', return_dict=False)
|
||||
assert channel == directory_channel
|
||||
|
||||
# Channel is found using the priority of params, incorrect data is ignored.
|
||||
params = [
|
||||
dict(channel_id=channel_id_channel.id, directory='bad directory'),
|
||||
dict(channel_id=channel_id_channel.id, directory='some directory'),
|
||||
dict(channel_id=channel_id_channel.id, source_id='bad source id'),
|
||||
dict(channel_id=channel_id_channel.id, source_id='the source id'),
|
||||
]
|
||||
for p in params:
|
||||
assert lib.get_channel(**p, return_dict=False) == channel_id_channel
|
||||
assert lib.get_channel(**p)
|
||||
|
|
@ -10,16 +10,13 @@ from typing import Union, Tuple, List, Set, Iterable, Optional
|
|||
import PIL
|
||||
from PIL import Image
|
||||
from sqlalchemy.orm import Session
|
||||
from yt_dlp import YoutubeDL
|
||||
|
||||
from wrolpi import before_startup
|
||||
from wrolpi.cmd import which
|
||||
from wrolpi.common import logger, iterify, get_media_directory, \
|
||||
minimize_dict, api_param_limiter
|
||||
minimize_dict, any_extensions
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \
|
||||
ChannelLinkConflict, ChannelDirectoryConflict, ChannelSourceIdConflict
|
||||
from wrolpi.errors import UnknownFile, ChannelNameConflict, ChannelURLConflict, \
|
||||
ChannelDirectoryConflict, ChannelSourceIdConflict
|
||||
from wrolpi.media_path import MediaPath
|
||||
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS
|
||||
from .models import Channel, Video
|
||||
|
|
@ -27,27 +24,21 @@ from .models import Channel, Video
|
|||
logger = logger.getChild(__name__)
|
||||
|
||||
REQUIRED_OPTIONS = ['name', 'directory']
|
||||
MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count', 'link'}
|
||||
MINIMUM_CHANNEL_KEYS = {'id', 'name', 'directory', 'url', 'video_count'}
|
||||
MINIMUM_INFO_JSON_KEYS = {'description', 'view_count', 'webpage_url'}
|
||||
MINIMUM_VIDEO_KEYS = {'id', 'title', 'upload_date', 'duration', 'channel', 'channel_id', 'favorite', 'size',
|
||||
'poster_path', 'caption_path', 'video_path', 'info_json', 'channel', 'viewed', 'source_id',
|
||||
'view_count'}
|
||||
VIDEO_QUERY_LIMIT = 20
|
||||
VIDEO_QUERY_MAX_LIMIT = 100
|
||||
# These are the supported video formats. These are in order of their preference.
|
||||
VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv')
|
||||
|
||||
video_limiter = api_param_limiter(100)
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_videos_directory() -> pathlib.Path:
|
||||
"""
|
||||
Get the "videos" directory in the media directory. Make it if it does not exist.
|
||||
"""
|
||||
"""Get the "videos" directory in the media directory. Make it if it does not exist."""
|
||||
directory = get_media_directory() / 'videos'
|
||||
if not directory.is_dir():
|
||||
directory.mkdir(parents=True)
|
||||
|
|
@ -55,89 +46,13 @@ def get_videos_directory() -> pathlib.Path:
|
|||
|
||||
|
||||
def get_no_channel_directory() -> pathlib.Path:
|
||||
"""
|
||||
Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist.
|
||||
"""
|
||||
"""Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist."""
|
||||
directory = get_videos_directory() / 'NO CHANNEL'
|
||||
if not directory.is_dir():
|
||||
directory.mkdir(parents=True)
|
||||
return directory
|
||||
|
||||
|
||||
@before_startup
|
||||
def import_videos_config():
|
||||
"""Import channel settings to the DB. Existing channels will be updated."""
|
||||
logger.info('Importing videos config')
|
||||
try:
|
||||
from .lib import get_channels_config
|
||||
from .downloader import ChannelDownloader
|
||||
config = get_channels_config()
|
||||
config, favorites = config.channels, config.favorites
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
for link in config:
|
||||
for option in (i for i in REQUIRED_OPTIONS if i not in config[link]):
|
||||
raise ConfigError(f'Channel "{link}" is required to have "{option}"')
|
||||
|
||||
name, directory = config[link]['name'], config[link]['directory']
|
||||
|
||||
channel = session.query(Channel).filter(Channel.link == link).one_or_none()
|
||||
if not channel:
|
||||
# Channel not yet in the DB, add it.
|
||||
channel = Channel(link=link)
|
||||
|
||||
# Only name and directory are required
|
||||
channel.name = name
|
||||
channel.directory = str(directory)
|
||||
|
||||
channel.calculate_duration = config[link].get('calculate_duration') or channel.calculate_duration
|
||||
channel.download_frequency = config[link].get('download_frequency') or channel.download_frequency
|
||||
channel.generate_posters = config[link].get('generate_posters') or channel.generate_posters
|
||||
channel.match_regex = config[link].get('match_regex') or channel.match_regex
|
||||
channel.skip_download_videos = list(set(config[link].get('skip_download_videos', {})))
|
||||
channel.source_id = config[link].get('source_id') or channel.source_id
|
||||
channel.url = config[link].get('url') or channel.url
|
||||
|
||||
if not channel.source_id and channel.url:
|
||||
# If we can download from a channel, we must have its source_id.
|
||||
channel.source_id = get_channel_source_id(channel.url)
|
||||
|
||||
if channel.download_frequency and channel.url:
|
||||
download = download_manager.get_or_create_download(channel.url, session=session)
|
||||
download.frequency = channel.download_frequency
|
||||
download.downloader = ChannelDownloader.name
|
||||
download.next_download = download.next_download or \
|
||||
download_manager.get_next_download(download, session=session)
|
||||
|
||||
session.add(channel)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
for link, favorites in favorites.items():
|
||||
if link != 'NO CHANNEL':
|
||||
channel: Channel = session.query(Channel).filter_by(link=link).one_or_none()
|
||||
if not channel:
|
||||
logger.warning(f'Cannot find channel {link=} for favorites!')
|
||||
continue
|
||||
channel_dir = channel.directory.path
|
||||
else:
|
||||
from .downloader import get_no_channel_directory
|
||||
channel_dir = get_no_channel_directory()
|
||||
|
||||
# Set favorite Videos of this Channel.
|
||||
for video_path, data in favorites.items():
|
||||
# Favorite in the config is the name of the video_path. Add the channel directory onto this
|
||||
# video_path, so we can match the complete path for the Video.
|
||||
video_path = channel_dir / video_path
|
||||
video = session.query(Video).filter_by(video_path=video_path).one_or_none()
|
||||
# If no Video is found, it may be that we need to refresh.
|
||||
if video:
|
||||
video.favorite = data['favorite']
|
||||
else:
|
||||
logger.warning(f'Cannot find video to favorite: {video_path}')
|
||||
except Exception as e:
|
||||
logger.warning('Failed to load channels config!', exc_info=e)
|
||||
|
||||
|
||||
VALID_VIDEO_KINDS = {'video', 'caption', 'poster', 'description', 'info_json'}
|
||||
|
||||
|
||||
|
|
@ -150,66 +65,11 @@ def get_absolute_video_path(video: Video, kind: str = 'video') -> MediaPath:
|
|||
raise UnknownFile()
|
||||
|
||||
|
||||
get_absolute_video_caption = partial(get_absolute_video_path, kind='caption')
|
||||
get_absolute_video_poster = partial(get_absolute_video_path, kind='poster')
|
||||
get_absolute_video_description = partial(get_absolute_video_path, kind='description')
|
||||
get_absolute_video_info_json = partial(get_absolute_video_path, kind='info_json')
|
||||
|
||||
|
||||
def get_absolute_video_files(video: Video) -> List[MediaPath]:
|
||||
"""
|
||||
Get all video files that exist.
|
||||
"""
|
||||
getters = [
|
||||
get_absolute_video_description,
|
||||
get_absolute_video_caption,
|
||||
get_absolute_video_poster,
|
||||
get_absolute_video_info_json,
|
||||
get_absolute_video_path,
|
||||
]
|
||||
|
||||
def _get():
|
||||
for getter in getters:
|
||||
try:
|
||||
yield getter(video)
|
||||
except UnknownFile:
|
||||
pass
|
||||
|
||||
return list(_get())
|
||||
|
||||
|
||||
def get_video_info_json(video) -> Optional[dict]:
|
||||
"""Get the info_json object from a video's meta-file. Return an empty dict if not possible."""
|
||||
if not video.channel or not video.channel.directory:
|
||||
return
|
||||
|
||||
try:
|
||||
path = get_absolute_video_info_json(video)
|
||||
if path.exists():
|
||||
with open(str(path), 'rb') as fh:
|
||||
contents = json.load(fh)
|
||||
return contents
|
||||
except UnknownFile:
|
||||
return
|
||||
except UnknownDirectory:
|
||||
return
|
||||
|
||||
|
||||
def any_extensions(filename: str, extensions: Iterable = ()):
|
||||
"""
|
||||
Return True only if the file name ends with any of the possible extensions.
|
||||
Matches lower or upper case of the extension.
|
||||
"""
|
||||
return any(filename.lower().endswith(ext) for ext in extensions)
|
||||
|
||||
|
||||
match_video_extensions = partial(any_extensions, extensions=VIDEO_EXTENSIONS)
|
||||
|
||||
|
||||
def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, pathlib.Path]:
|
||||
"""
|
||||
Generate a list of video paths in the provided directory.
|
||||
"""
|
||||
"""Generate a list of video paths in the provided directory."""
|
||||
directory = pathlib.Path(directory)
|
||||
|
||||
for child in directory.iterdir():
|
||||
|
|
@ -223,9 +83,11 @@ def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, path
|
|||
@iterify(set)
|
||||
def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]:
|
||||
"""
|
||||
Remove any duplicate paths from a given list. Duplicate is defined as any file that shares the EXACT same
|
||||
name as another file, but with a different extension. If a duplicate is found, only yield one of the paths. The
|
||||
path that will be yielded will be whichever is first in the VIDEO_EXTESIONS tuple.
|
||||
Remove any duplicate paths from a given list.
|
||||
|
||||
Duplicate is defined as any file that shares the EXACT same name as another file, but with a different extension.
|
||||
If a duplicate is found, only yield one of the paths. The path that will be yielded will be whichever is first in
|
||||
the VIDEO_EXTESIONS tuple.
|
||||
|
||||
i.e.
|
||||
>>> remove_duplicate_video_paths([Path('one.mp4'), Path('two.mp4'), Path('one.ogg')])
|
||||
|
|
@ -262,15 +124,15 @@ def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]:
|
|||
yield sorted(paths)[0]
|
||||
|
||||
|
||||
def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, link=None, directory=None,
|
||||
def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None, directory=None,
|
||||
source_id=None):
|
||||
"""
|
||||
Search for any channels that conflict with the provided args, raise a relevant exception if any conflicts are found.
|
||||
"""
|
||||
if not any([id_, url, name, link, directory]):
|
||||
if not any([id_, url, name, directory]):
|
||||
raise Exception('Cannot search for channel with no arguments')
|
||||
|
||||
logger.debug(f'Checking for channel conflicts: id={id_} url={url} name={name} link={link} directory={directory}')
|
||||
logger.debug(f'Checking for channel conflicts: id={id_} url={url} name={name} directory={directory}')
|
||||
|
||||
# A channel can't conflict with itself
|
||||
if id_:
|
||||
|
|
@ -286,10 +148,6 @@ def check_for_channel_conflicts(session: Session, id_=None, url=None, name=None,
|
|||
conflicts = base_where.filter(Channel.name == name)
|
||||
if list(conflicts):
|
||||
raise ChannelNameConflict()
|
||||
if link:
|
||||
conflicts = base_where.filter(Channel.link == link)
|
||||
if list(conflicts):
|
||||
raise ChannelLinkConflict()
|
||||
if directory:
|
||||
conflicts = base_where.filter(Channel.directory == directory)
|
||||
if list(conflicts):
|
||||
|
|
@ -329,29 +187,13 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]:
|
|||
return paths
|
||||
|
||||
|
||||
def replace_extension(path: pathlib.Path, new_ext) -> pathlib.Path:
|
||||
"""Swap the extension of a file's path.
|
||||
|
||||
Example:
|
||||
>>> foo = pathlib.Path('foo.bar')
|
||||
>>> replace_extension(foo, 'baz')
|
||||
'foo.baz'
|
||||
"""
|
||||
parent = path.parent
|
||||
existing_ext = path.suffix
|
||||
path = str(path)
|
||||
name, _, _ = path.rpartition(existing_ext)
|
||||
path = pathlib.Path(str(parent / name) + new_ext)
|
||||
return path
|
||||
|
||||
|
||||
def generate_video_poster(video_path: Path) -> Path:
|
||||
"""
|
||||
Create a poster (aka thumbnail) next to the provided video_path.
|
||||
"""
|
||||
poster_path = replace_extension(video_path, '.jpg')
|
||||
cmd = ['/usr/bin/ffmpeg', '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000',
|
||||
str(poster_path)]
|
||||
poster_path = video_path.with_suffix('.jpg')
|
||||
cmd = ('/usr/bin/ffmpeg', '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000',
|
||||
str(poster_path))
|
||||
try:
|
||||
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
logger.info(f'Generated poster at {poster_path}')
|
||||
|
|
@ -474,13 +316,3 @@ def minimize_video(video: dict) -> dict:
|
|||
video['info_json'] = minimize_video_info_json(video['info_json'])
|
||||
|
||||
return video
|
||||
|
||||
|
||||
YDL = YoutubeDL()
|
||||
YDL.params['logger'] = logger.getChild('youtube-dl')
|
||||
YDL.add_default_info_extractors()
|
||||
|
||||
|
||||
def get_channel_source_id(url: str) -> str:
|
||||
channel_info = YDL.extract_info(url, download=False, process=False)
|
||||
return channel_info.get('uploader_id') or channel_info['id']
|
||||
|
|
|
|||
|
|
@ -10,22 +10,18 @@ from PIL import Image
|
|||
from modules.videos.downloader import VideoDownloader, ChannelDownloader
|
||||
from modules.videos.lib import set_test_channels_config, set_test_downloader_config
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.common import sanitize_link
|
||||
from wrolpi.downloader import DownloadFrequency, DownloadManager, Download
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def simple_channel(test_session, test_directory) -> Channel:
|
||||
"""
|
||||
Get a Channel with the minimum properties. This Channel has no download!
|
||||
"""
|
||||
"""Get a Channel with the minimum properties. This Channel has no download!"""
|
||||
channel = Channel(
|
||||
directory=test_directory,
|
||||
name='Simple Channel',
|
||||
url='https://example.com/channel1',
|
||||
download_frequency=None, # noqa
|
||||
link='simplechannel',
|
||||
)
|
||||
test_session.add(channel)
|
||||
test_session.commit()
|
||||
|
|
@ -34,20 +30,18 @@ def simple_channel(test_session, test_directory) -> Channel:
|
|||
|
||||
@pytest.fixture
|
||||
def channel_factory(test_session, test_directory):
|
||||
"""
|
||||
Create a random Channel with a directory, but no frequency.
|
||||
"""
|
||||
"""Create a random Channel with a directory, but no frequency."""
|
||||
|
||||
def _():
|
||||
name = str(uuid4())
|
||||
def _(source_id: str = None, download_frequency: DownloadFrequency = None, url: str = None, name: str = None):
|
||||
name = name or str(uuid4())
|
||||
directory = test_directory / name
|
||||
directory.mkdir()
|
||||
channel = Channel(
|
||||
directory=directory, # noqa
|
||||
name=name,
|
||||
url=f'https://example.com/{name}',
|
||||
download_frequency=None, # noqa
|
||||
link=sanitize_link(name),
|
||||
url=url or f'https://example.com/{name}',
|
||||
download_frequency=download_frequency,
|
||||
source_id=source_id,
|
||||
)
|
||||
test_session.add(channel)
|
||||
test_session.commit()
|
||||
|
|
@ -67,7 +61,6 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch
|
|||
name='Download Channel',
|
||||
url='https://example.com/channel1',
|
||||
download_frequency=DownloadFrequency.weekly,
|
||||
link='downloadchannel',
|
||||
)
|
||||
download = Download(url=channel.url, downloader='video_channel', frequency=channel.download_frequency)
|
||||
test_session.add_all([channel, download])
|
||||
|
|
@ -77,9 +70,7 @@ def download_channel(test_session, test_directory, video_download_manager) -> Ch
|
|||
|
||||
@pytest.fixture
|
||||
def simple_video(test_session, test_directory, simple_channel) -> Video:
|
||||
"""
|
||||
A Video with a video file whose channel is the Simple Channel.
|
||||
"""
|
||||
"""A Video with an empty video file whose channel is the Simple Channel."""
|
||||
video_path = test_directory / 'simple_video.mp4'
|
||||
video_path.touch()
|
||||
video = Video(video_path=video_path, channel_id=simple_channel.id)
|
||||
|
|
@ -103,9 +94,9 @@ def video_file(test_directory) -> pathlib.Path:
|
|||
def video_factory(test_session, test_directory):
|
||||
"""Creates Videos for testing."""
|
||||
|
||||
def _(channel_id: int = None, with_video_file: bool = False, with_info_json: dict = None,
|
||||
with_poster_ext: str = None):
|
||||
title = str(uuid4())
|
||||
def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file: bool = False,
|
||||
with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False):
|
||||
title = title or str(uuid4())
|
||||
if channel_id:
|
||||
path = test_directory / f'{title}.mp4'
|
||||
else:
|
||||
|
|
@ -129,12 +120,18 @@ def video_factory(test_session, test_directory):
|
|||
poster_path = path.with_suffix(f'.{with_poster_ext}')
|
||||
Image.new('RGB', (25, 25)).save(poster_path)
|
||||
|
||||
caption_path = None
|
||||
if with_caption_file:
|
||||
caption_path = path.with_suffix('.en.vtt')
|
||||
caption_path.touch()
|
||||
|
||||
video = Video(video_path=str(path), title=title, channel_id=channel_id, source_id=title,
|
||||
info_json_path=info_json_path, poster_path=poster_path)
|
||||
info_json_path=info_json_path, poster_path=poster_path, caption_path=caption_path,
|
||||
upload_date=upload_date)
|
||||
test_session.add(video)
|
||||
return video
|
||||
|
||||
return _
|
||||
return factory
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
|
|||
|
|
@ -6,23 +6,23 @@ from abc import ABC
|
|||
from typing import Tuple, List, Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
from yt_dlp import YoutubeDL
|
||||
from yt_dlp.extractor import YoutubeTabIE
|
||||
from yt_dlp.utils import UnsupportedError, DownloadError
|
||||
|
||||
from wrolpi.cmd import which
|
||||
from wrolpi.common import logger, sanitize_link, extract_domain
|
||||
from wrolpi.common import logger, extract_domain, get_media_directory
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG
|
||||
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
from .common import apply_info_json, get_channel_source_id, get_no_channel_directory, \
|
||||
from .common import apply_info_json, get_no_channel_directory, \
|
||||
get_videos_directory
|
||||
from .lib import upsert_video, refresh_channel_videos, get_downloader_config
|
||||
from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id
|
||||
from .models import Video, Channel
|
||||
from .schema import ChannelPostRequest
|
||||
from .video_url_resolver import video_url_resolver
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -70,12 +70,12 @@ class ChannelDownloader(Downloader, ABC):
|
|||
if not channel:
|
||||
# Couldn't get channel by URL, this is probably a playlist. Find the channel for the playlist.
|
||||
channel_source_id = get_channel_source_id(download.url)
|
||||
channel = get_channel(channel_source_id, return_dict=False)
|
||||
channel = get_channel(source_id=channel_source_id, return_dict=False)
|
||||
|
||||
update_channel(channel)
|
||||
|
||||
domain = extract_domain(download.url)
|
||||
missing_videos = find_all_missing_videos(channel.link)
|
||||
missing_videos = find_all_missing_videos(channel.id)
|
||||
with get_db_session(commit=True) as session:
|
||||
for video_id, source_id, missing_video in missing_videos:
|
||||
url = video_url_resolver(domain, missing_video)
|
||||
|
|
@ -152,7 +152,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
channel = None
|
||||
if channel_name or channel_id:
|
||||
channel_url = info.get('channel_url')
|
||||
channel = get_or_create_channel(channel_id, None, channel_url, channel_name)
|
||||
channel = get_or_create_channel(source_id=channel_id, url=channel_url, name=channel_name)
|
||||
|
||||
# Use the default directory if this video has no channel.
|
||||
out_dir = get_no_channel_directory()
|
||||
|
|
@ -245,34 +245,32 @@ channel_downloader = ChannelDownloader()
|
|||
video_downloader = VideoDownloader(40)
|
||||
|
||||
|
||||
def get_or_create_channel(source_id: str = None, link: str = None, url: str = None, name: str = None) -> Channel:
|
||||
def get_or_create_channel(source_id: str = None, url: str = None, name: str = None) -> Channel:
|
||||
"""
|
||||
Attempt to find a Channel using the provided params. The params are in order of reliability.
|
||||
|
||||
Creates a new Channel if one cannot be found.
|
||||
"""
|
||||
if not link and name:
|
||||
link = sanitize_link(name)
|
||||
|
||||
try:
|
||||
channel = get_channel(link=link, source_id=source_id, url=url, return_dict=False)
|
||||
channel = get_channel(source_id=source_id, url=url, name=name, return_dict=False)
|
||||
return channel
|
||||
except UnknownChannel:
|
||||
pass
|
||||
|
||||
if not name:
|
||||
raise ValueError(f'Cannot create channel without a name')
|
||||
|
||||
# Channel does not exist. Create one in the video directory.
|
||||
channel_directory = get_videos_directory() / name
|
||||
if not channel_directory.is_dir():
|
||||
channel_directory.mkdir(parents=True)
|
||||
data = dict(
|
||||
data = ChannelPostRequest(
|
||||
source_id=source_id,
|
||||
name=name,
|
||||
url=url,
|
||||
link=link,
|
||||
domain=extract_domain(url),
|
||||
directory=str(channel_directory),
|
||||
source_id=source_id,
|
||||
directory=str(channel_directory.relative_to(get_media_directory())),
|
||||
)
|
||||
channel = create_channel(data, return_dict=False)
|
||||
channel = create_channel(data=data, return_dict=False)
|
||||
# Create the directory now that the channel is approved.
|
||||
channel_directory.mkdir(exist_ok=True)
|
||||
|
||||
|
|
@ -348,21 +346,17 @@ def update_channel(channel: Channel):
|
|||
apply_info_json(channel_id)
|
||||
|
||||
|
||||
def _find_all_missing_videos(link: str = None) -> List[Tuple]:
|
||||
def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]:
|
||||
"""Get all Video entries which don't have the required media files (i.e. hasn't been downloaded).
|
||||
|
||||
Restrict to a single channel if "link" is provided.
|
||||
Restrict to a single channel if `channel_id` is provided.
|
||||
"""
|
||||
with get_db_curs() as curs:
|
||||
# Get all channels by default.
|
||||
where = ''
|
||||
params = ()
|
||||
|
||||
if link:
|
||||
# Restrict by channel when link is provided.
|
||||
query = 'SELECT id FROM channel WHERE link = %s'
|
||||
curs.execute(query, (link,))
|
||||
channel_id = curs.fetchall()[0][0]
|
||||
if channel_id:
|
||||
where = 'AND channel_id = %s'
|
||||
params = (channel_id,)
|
||||
|
||||
|
|
@ -386,18 +380,15 @@ def _find_all_missing_videos(link: str = None) -> List[Tuple]:
|
|||
return missing_videos
|
||||
|
||||
|
||||
def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
|
||||
def find_all_missing_videos(channel_id: str = None) -> Tuple[dict, dict]:
|
||||
"""
|
||||
Find all videos that don't have a video file, but are found in the DB (taken from the channel's info_json).
|
||||
|
||||
Yields a Channel Dict object, our Video id, and the "entry" of the video from the channel's info_json['entries'].
|
||||
"""
|
||||
with get_db_session() as session:
|
||||
if link:
|
||||
try:
|
||||
channel = session.query(Channel).filter_by(link=link).one()
|
||||
except NoResultFound:
|
||||
raise UnknownChannel(f'No channel with link: {link}')
|
||||
if channel_id:
|
||||
channel = get_channel(channel_id=channel_id, return_dict=False)
|
||||
if not channel.url:
|
||||
raise ChannelURLEmpty('No URL for this channel')
|
||||
channels = [channel, ]
|
||||
|
|
@ -420,12 +411,12 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
|
|||
# Convert the channel video entries into a form that allows them to be quickly retrieved without searching through
|
||||
# the entire entries list.
|
||||
channels_entries = {}
|
||||
for id_, channel in channels.items():
|
||||
channels_entries[id_] = {i['id']: i for i in channel.info_json['entries']}
|
||||
for channel_id_, channel in channels.items():
|
||||
channels_entries[channel_id_] = {i['id']: i for i in channel.info_json['entries']}
|
||||
|
||||
missing_videos = _find_all_missing_videos(link)
|
||||
missing_videos = _find_all_missing_videos(channel_id)
|
||||
|
||||
for id_, source_id, channel_id in missing_videos:
|
||||
for video_id, source_id, channel_id in missing_videos:
|
||||
channel = channels[channel_id]
|
||||
|
||||
if channel.skip_download_videos and source_id in channel.skip_download_videos:
|
||||
|
|
@ -435,13 +426,13 @@ def find_all_missing_videos(link: str = None) -> Tuple[dict, dict]:
|
|||
try:
|
||||
missing_video = channels_entries[channel_id][source_id]
|
||||
except KeyError:
|
||||
logger.warning(f'Video {id_} / {source_id} is not in {channel.name} info_json')
|
||||
logger.warning(f'Video {channel.name} / {source_id} is not in {channel.name} info_json')
|
||||
continue
|
||||
|
||||
match_regex: re.compile = match_regexen.get(channel_id)
|
||||
if not match_regex or (match_regex and missing_video['title'] and match_regex.match(missing_video['title'])):
|
||||
# No title match regex, or the title matches the regex.
|
||||
yield id_, source_id, missing_video
|
||||
yield video_id, source_id, missing_video
|
||||
|
||||
|
||||
UNRECOVERABLE_ERRORS = {
|
||||
|
|
|
|||
|
|
@ -2,20 +2,23 @@ import html
|
|||
import pathlib
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import Tuple, Optional, List
|
||||
from typing import Tuple, Optional, List, Union
|
||||
from uuid import uuid1
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.exc import InvalidRequestError
|
||||
from sqlalchemy.orm import Session
|
||||
from yt_dlp import YoutubeDL
|
||||
|
||||
from wrolpi.common import logger, chunks, ConfigFile
|
||||
from wrolpi import before_startup
|
||||
from wrolpi.common import chunks, ConfigFile, get_media_directory, sanitize_link
|
||||
from wrolpi.dates import from_timestamp, Seconds
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.media_path import MediaPath
|
||||
from wrolpi.vars import PYTEST
|
||||
from .captions import get_captions
|
||||
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, import_videos_config, \
|
||||
get_video_duration, is_valid_poster, convert_image, generate_video_poster
|
||||
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \
|
||||
is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError
|
||||
from .models import Channel, Video
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -66,8 +69,9 @@ def refresh_channel_videos(channel: Channel):
|
|||
|
||||
logger.info(f'{channel.name}: {len(new_videos)} new videos, {len(existing_paths)} already existed. ')
|
||||
|
||||
channel.refreshed = True
|
||||
Session.object_session(channel).commit()
|
||||
with get_db_session(commit=True) as session:
|
||||
channel = session.query(Channel).filter_by(id=channel.id).one()
|
||||
channel.refreshed = True
|
||||
|
||||
apply_info_json(channel.id)
|
||||
|
||||
|
|
@ -155,7 +159,7 @@ def validate_videos():
|
|||
video.validated = True
|
||||
except Exception as e:
|
||||
# This video failed to validate, continue validation for the rest of the videos.
|
||||
logger.warning(f'Failed to validate {video=}', exc_info=e)
|
||||
logger.warning(f'Failed to validate {video}', exc_info=e)
|
||||
|
||||
|
||||
def validate_video(video: Video, channel_generate_poster: bool):
|
||||
|
|
@ -246,7 +250,7 @@ def convert_or_generate_poster(video: Video) -> Optional[pathlib.Path]:
|
|||
logger.error(f'Failed to generate poster for {video}', exc_info=e)
|
||||
|
||||
|
||||
def refresh_videos(channel_links: list = None):
|
||||
def refresh_videos(channel_ids: List[int] = None):
|
||||
"""
|
||||
Find any videos in the channel directories and add them to the DB. Delete DB records of any videos not in the
|
||||
file system.
|
||||
|
|
@ -257,15 +261,15 @@ def refresh_videos(channel_links: list = None):
|
|||
"""
|
||||
logger.info('Refreshing video files')
|
||||
with get_db_session() as session:
|
||||
if channel_links:
|
||||
channels = session.query(Channel).filter(Channel.link.in_(channel_links))
|
||||
if channel_ids:
|
||||
channels = session.query(Channel).filter(Channel.id.in_(channel_ids))
|
||||
else:
|
||||
channels = session.query(Channel).all()
|
||||
|
||||
channels = list(channels)
|
||||
|
||||
if not channels and channel_links:
|
||||
raise Exception(f'No channels match links(s): {channel_links}')
|
||||
if not channels and channel_ids:
|
||||
raise Exception(f'No channels match id(s): {channel_ids}')
|
||||
elif not channels:
|
||||
raise Exception(f'No channels in DB. Have you created any?')
|
||||
|
||||
|
|
@ -276,13 +280,13 @@ def refresh_videos(channel_links: list = None):
|
|||
logger.fatal(f'Failed to refresh videos for channel {channel.name}!', exc_info=e)
|
||||
pass
|
||||
|
||||
if not channel_links:
|
||||
if not channel_ids:
|
||||
# Refresh NO CHANNEL videos when not refreshing a specific channel.
|
||||
refresh_no_channel_videos()
|
||||
|
||||
# Fill in any missing data for all videos.
|
||||
if not PYTEST:
|
||||
import_videos_config()
|
||||
import_channels_config()
|
||||
|
||||
validate_videos()
|
||||
|
||||
|
|
@ -460,8 +464,8 @@ def set_test_downloader_config(enabled: bool):
|
|||
|
||||
def get_channels_config_from_db(session: Session) -> dict:
|
||||
"""Create a dictionary that contains all the Channels from the DB."""
|
||||
channels = session.query(Channel).order_by(Channel.link).all()
|
||||
channels = {i.link: i.config_view() for i in channels}
|
||||
channels = session.query(Channel).order_by(Channel.directory).all()
|
||||
channels = sorted((i.config_view() for i in channels), key=lambda i: i['directory'])
|
||||
|
||||
# Get all Videos that are favorites. Store them in their own config section, so they can be preserved if a channel
|
||||
# is deleted or the DB is wiped.
|
||||
|
|
@ -469,7 +473,7 @@ def get_channels_config_from_db(session: Session) -> dict:
|
|||
favorites = defaultdict(lambda: {})
|
||||
for video in favorite_videos:
|
||||
if video.channel:
|
||||
favorites[video.channel.link][video.video_path.path.name] = dict(favorite=video.favorite)
|
||||
favorites[str(video.channel.directory.relative)][video.video_path.path.name] = dict(favorite=video.favorite)
|
||||
else:
|
||||
favorites['NO CHANNEL'][video.video_path.path.name] = dict(favorite=video.favorite)
|
||||
favorites = dict(favorites)
|
||||
|
|
@ -477,18 +481,128 @@ def get_channels_config_from_db(session: Session) -> dict:
|
|||
return dict(channels=channels, favorites=favorites)
|
||||
|
||||
|
||||
def _detect_old_favorites(config: Union[ChannelsConfig, dict]) -> bool:
|
||||
# TODO remove these old favorites after beta.
|
||||
media_directory = get_media_directory()
|
||||
favorites = config.favorites if hasattr(config, 'favorites') else config['favorites']
|
||||
for channel, favorites in favorites.items():
|
||||
if channel == 'NO CHANNEL':
|
||||
continue
|
||||
if not (media_directory / channel).is_dir():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@optional_session()
|
||||
def save_channels_config(session=None, preserve_favorites: bool = True):
|
||||
"""
|
||||
Pull the Channel information from the DB, save it to the config.
|
||||
"""
|
||||
"""Get the Channel information from the DB, save it to the config."""
|
||||
config = get_channels_config_from_db(session)
|
||||
channels_config = get_channels_config()
|
||||
# TODO remove these old favorites after beta.
|
||||
old_favorites = _detect_old_favorites(channels_config)
|
||||
if old_favorites and config['favorites']:
|
||||
# There are favorites in the database, remove the old favorites.
|
||||
channels_config.favorites = {}
|
||||
if preserve_favorites:
|
||||
config['favorites'].update(channels_config.favorites or {})
|
||||
channels_config.update(config)
|
||||
|
||||
|
||||
@before_startup
|
||||
def import_channels_config():
|
||||
"""Import channel settings to the DB. Existing channels will be updated."""
|
||||
logger.info('Importing videos config')
|
||||
try:
|
||||
from .downloader import ChannelDownloader
|
||||
config = get_channels_config()
|
||||
channels, favorites = config.channels, config.favorites
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
for data in channels:
|
||||
if isinstance(data, str):
|
||||
# Outdated style config.
|
||||
# TODO remove this after beta.
|
||||
data = channels[data]
|
||||
# A Channel's directory is saved (in config) relative to the media directory.
|
||||
directory = get_media_directory() / data['directory']
|
||||
for option in (i for i in REQUIRED_OPTIONS if i not in data):
|
||||
raise ConfigError(f'Channel "{directory}" is required to have "{option}"')
|
||||
|
||||
channel = session.query(Channel).filter(
|
||||
or_(
|
||||
Channel.directory == str(directory),
|
||||
Channel.directory == str(data['directory']),
|
||||
Channel.directory == str(directory.relative_to(get_media_directory())),
|
||||
),
|
||||
).one_or_none()
|
||||
if not channel:
|
||||
# Channel not yet in the DB, add it.
|
||||
channel = Channel(directory=directory)
|
||||
session.add(channel)
|
||||
|
||||
# Only name and directory are required
|
||||
channel.name = data['name']
|
||||
channel.directory = directory
|
||||
|
||||
# Copy existing channel data, update all values from the config. This is necessary to clear out
|
||||
# values not in the config.
|
||||
full_data = channel.dict()
|
||||
full_data.update(data)
|
||||
print(f'{full_data=}')
|
||||
full_data['skip_download_videos'] = list(set(data.get('skip_download_videos', {})))
|
||||
channel.update(full_data)
|
||||
|
||||
if not channel.source_id and channel.url:
|
||||
# If we can download from a channel, we must have its source_id.
|
||||
channel.source_id = get_channel_source_id(channel.url)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
channels_by_link = {sanitize_link(i.name): i for i in session.query(Channel)}
|
||||
for directory_, favorites in favorites.items():
|
||||
if directory_ != 'NO CHANNEL':
|
||||
# A Channel's directory is saved (in the config) relative to the media directory.
|
||||
directory = get_media_directory() / directory_
|
||||
channel: Channel = session.query(Channel).filter_by(directory=directory).one_or_none()
|
||||
if not channel:
|
||||
# Directory may be the outdated "link".
|
||||
# TODO remove these old favorites after beta.
|
||||
channel = channels_by_link.get(directory_)
|
||||
if not channel:
|
||||
logger.warning(f'Cannot find channel {directory=} for favorites!')
|
||||
continue
|
||||
channel_dir = channel.directory.path
|
||||
else:
|
||||
from .downloader import get_no_channel_directory
|
||||
channel_dir = get_no_channel_directory()
|
||||
|
||||
# Set favorite Videos of this Channel.
|
||||
for video_path, data in favorites.items():
|
||||
# Favorite in the config is the name of the video_path. Add the channel directory onto this
|
||||
# video_path, so we can match the complete path for the Video.
|
||||
video_path = channel_dir / video_path
|
||||
video = session.query(Video).filter_by(video_path=video_path).one_or_none()
|
||||
# If no Video is found, it may be that we need to refresh.
|
||||
if video:
|
||||
video.favorite = data['favorite']
|
||||
else:
|
||||
logger.warning(f'Cannot find video to favorite: {video_path}')
|
||||
except Exception as e:
|
||||
logger.warning('Failed to load channels config!', exc_info=e)
|
||||
if PYTEST:
|
||||
# Do not interrupt startup, only raise during testing.
|
||||
raise
|
||||
|
||||
|
||||
YDL = YoutubeDL()
|
||||
YDL.params['logger'] = logger.getChild('youtube-dl')
|
||||
YDL.add_default_info_extractors()
|
||||
|
||||
|
||||
def get_channel_source_id(url: str) -> str:
|
||||
channel_info = YDL.extract_info(url, download=False, process=False)
|
||||
return channel_info.get('uploader_id') or channel_info['id']
|
||||
|
||||
|
||||
async def get_statistics():
|
||||
with get_db_curs() as curs:
|
||||
curs.execute('''
|
||||
|
|
@ -608,7 +722,6 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
|
|||
video.info_json_path = info_json_path
|
||||
video.poster_path = poster_path
|
||||
video.video_path = video_path
|
||||
|
||||
if channel and not str(video_path).startswith(str(channel.directory.path)):
|
||||
raise ValueError(f'Video path is not within its channel {video_path=} not in {channel.directory=}')
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import json
|
||||
import pathlib
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Generator
|
||||
|
|
@ -12,7 +11,7 @@ from wrolpi.common import Base, tsvector, ModelHelper, logger, get_media_directo
|
|||
from wrolpi.dates import now, TZDateTime
|
||||
from wrolpi.db import get_db_curs
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory, InvalidDownload
|
||||
from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory
|
||||
from wrolpi.media_path import MediaPathType, MediaPath
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -116,9 +115,7 @@ class Video(ModelHelper, Base):
|
|||
save_channels_config(preserve_favorites=False)
|
||||
|
||||
def add_to_skip_list(self):
|
||||
"""
|
||||
Add this video to it's Channel's skip list.
|
||||
"""
|
||||
"""Add this video to it's Channel's skip list."""
|
||||
if self.channel and self.source_id:
|
||||
self.channel.add_video_to_skip_list(self.source_id)
|
||||
|
||||
|
|
@ -130,18 +127,16 @@ class Video(ModelHelper, Base):
|
|||
self.viewed = now()
|
||||
|
||||
def get_info_json(self) -> Optional[JSON]:
|
||||
"""
|
||||
If this Video has an info_json file, return it's contents. Otherwise, return None.
|
||||
"""
|
||||
"""If this Video has an info_json file, return it's contents. Otherwise, return None."""
|
||||
if not self.info_json_path:
|
||||
return
|
||||
|
||||
try:
|
||||
if isinstance(self.info_json_path, pathlib.Path):
|
||||
with open(self.info_json_path, 'rb') as fh:
|
||||
contents = json.load(fh)
|
||||
return contents
|
||||
elif self.info_json_path:
|
||||
with open(self.info_json_path.path, 'rb') as fh:
|
||||
contents = json.load(fh)
|
||||
return contents
|
||||
info_json_path = self.info_json_path.path if \
|
||||
isinstance(self.info_json_path, MediaPath) else self.info_json_path
|
||||
with open(info_json_path, 'rb') as fh:
|
||||
contents = json.load(fh)
|
||||
return contents
|
||||
except UnknownFile:
|
||||
pass
|
||||
except UnknownDirectory:
|
||||
|
|
@ -265,9 +260,8 @@ class Channel(ModelHelper, Base):
|
|||
__tablename__ = 'channel'
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String)
|
||||
link = Column(String, nullable=False)
|
||||
idempotency = Column(String)
|
||||
url = Column(String)
|
||||
url = Column(String, unique=True)
|
||||
match_regex = Column(String)
|
||||
directory: MediaPath = Column(MediaPathType)
|
||||
skip_download_videos = Column(ARRAY(String))
|
||||
|
|
@ -307,18 +301,28 @@ class Channel(ModelHelper, Base):
|
|||
|
||||
def update(self, data: dict):
|
||||
"""
|
||||
Update the values of this Channel. Will also update the Channel's Download, if it has one.
|
||||
Update the attributes of this Channel. Will also update the Channel's Download, if it has one.
|
||||
"""
|
||||
# Get the download before we change the URL.
|
||||
download = self.get_download()
|
||||
|
||||
for key, value in data.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
if not self.url:
|
||||
return
|
||||
download = self.get_download()
|
||||
if download:
|
||||
session = Session.object_session(self)
|
||||
# All channels with a URL and download_frequency should have a download.
|
||||
session = Session.object_session(self)
|
||||
if download and not self.download_frequency:
|
||||
download_manager.delete_download(download.id, session)
|
||||
elif download and self.download_frequency:
|
||||
download.frequency = self.download_frequency
|
||||
download.url = self.url
|
||||
download.next_download = download_manager.get_next_download(download, session)
|
||||
elif not download and self.download_frequency and self.url:
|
||||
download = Download(frequency=self.download_frequency, url=self.url, downloader='video_channel')
|
||||
session.add(download)
|
||||
session.flush()
|
||||
download.next_download = download_manager.get_next_download(download, session)
|
||||
session.flush()
|
||||
|
||||
def config_view(self) -> dict:
|
||||
"""
|
||||
|
|
@ -349,7 +353,7 @@ class Channel(ModelHelper, Base):
|
|||
Get the Download row for this Channel. If there isn't a Download, return None.
|
||||
"""
|
||||
if not self.url:
|
||||
raise InvalidDownload(f'Channel {self.name} does not have a URL to download!')
|
||||
return None
|
||||
|
||||
session = Session.object_session(self)
|
||||
download = session.query(Download).filter_by(url=self.url).one_or_none()
|
||||
|
|
@ -361,10 +365,15 @@ class Channel(ModelHelper, Base):
|
|||
name=self.name,
|
||||
directory=self.directory,
|
||||
url=self.url,
|
||||
link=self.link,
|
||||
)
|
||||
return d
|
||||
|
||||
def dict(self, with_statistics: bool = False):
|
||||
d = super(Channel, self).dict()
|
||||
if with_statistics:
|
||||
d['statistics'] = self.get_statistics()
|
||||
return d
|
||||
|
||||
def get_statistics(self):
|
||||
"""
|
||||
Get statistics about this channel.
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from dataclasses import dataclass, field
|
|||
from datetime import date, datetime
|
||||
from typing import List, Optional
|
||||
|
||||
from modules.videos.lib import DEFAULT_DOWNLOAD_FREQUENCY
|
||||
from modules.videos.video.lib import DEFAULT_VIDEO_ORDER, VIDEO_QUERY_LIMIT
|
||||
|
||||
|
||||
|
|
@ -11,25 +10,37 @@ class ChannelPostRequest:
|
|||
name: str
|
||||
directory: str
|
||||
calculate_duration: Optional[bool] = None
|
||||
download_frequency: Optional[int] = DEFAULT_DOWNLOAD_FREQUENCY
|
||||
download_frequency: Optional[int] = None
|
||||
generate_posters: Optional[bool] = None
|
||||
match_regex: Optional[str] = None
|
||||
mkdir: Optional[bool] = None
|
||||
url: Optional[str] = None
|
||||
source_id: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.name = self.name.strip() or None
|
||||
self.directory = self.directory.strip() or None
|
||||
self.url = self.url.strip() if self.url else None
|
||||
self.source_id = self.source_id.strip() if self.source_id else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChannelPutRequest:
|
||||
calculate_duration: Optional[bool] = None
|
||||
directory: Optional[str] = None
|
||||
download_frequency: Optional[int] = DEFAULT_DOWNLOAD_FREQUENCY
|
||||
download_frequency: Optional[int] = None
|
||||
generate_posters: Optional[bool] = None
|
||||
link: Optional[str] = None
|
||||
match_regex: Optional[str] = None
|
||||
mkdir: Optional[bool] = None
|
||||
name: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.name = self.name.strip() or None
|
||||
self.directory = self.directory.strip() or None
|
||||
self.url = self.url.strip() if self.url else None
|
||||
self.match_regex = None if self.match_regex in ('None', '') else self.match_regex
|
||||
|
||||
|
||||
@dataclass
|
||||
class RefreshRequest:
|
||||
|
|
@ -42,7 +53,6 @@ class ChannelModel:
|
|||
url: str
|
||||
name: str
|
||||
match_regex: str
|
||||
link: str
|
||||
directory: str
|
||||
|
||||
|
||||
|
|
@ -60,7 +70,6 @@ class ChannelPostResponse:
|
|||
class ChannelsChannelModel:
|
||||
id: int
|
||||
name: str
|
||||
link: str
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -132,7 +141,7 @@ class VideoSearchRequest:
|
|||
offset: Optional[int] = None
|
||||
limit: Optional[int] = VIDEO_QUERY_LIMIT
|
||||
order_by: Optional[str] = DEFAULT_VIDEO_ORDER
|
||||
channel_link: Optional[str] = None
|
||||
channel_id: Optional[int] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -1,75 +0,0 @@
|
|||
import pathlib
|
||||
import tempfile
|
||||
from functools import wraps
|
||||
|
||||
from modules.videos.lib import refresh_channel_videos
|
||||
from modules.videos.models import Channel
|
||||
from wrolpi.common import insert_parameter, set_test_media_directory
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.test.common import wrap_test_db, build_test_directories
|
||||
|
||||
|
||||
def create_channel_structure(structure):
|
||||
"""
|
||||
Create a directory containing the specified structure of channels and videos. Create DB entries for these
|
||||
channels and videos.
|
||||
|
||||
Example:
|
||||
>>> s = {'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4', 'vid2.mp4', 'vid2.en.vtt']}
|
||||
>>> create_channel_structure(s)
|
||||
|
||||
Creates directories like so:
|
||||
channel1/vid1.mp4
|
||||
channel2/vid1.mp4
|
||||
channel2/vid2.mp4
|
||||
channel2/vid2.en.vtt
|
||||
|
||||
Channels like so:
|
||||
Channel(name='channel1', directory='channel1')
|
||||
Channel(name='channel2', directory='channel2')
|
||||
|
||||
And, Videos like so:
|
||||
Video(channel_id=1, video_path='vid1.mp4')
|
||||
Video(channel_id=2, video_path='vid1.mp4')
|
||||
Video(channel_id=2, video_path='vid2.mp4', caption_path='vid2.en.vtt')
|
||||
"""
|
||||
|
||||
def wrapper(func):
|
||||
@wraps(func)
|
||||
@wrap_test_db
|
||||
def wrapped(*args, **kwargs):
|
||||
if args and (test := args[0]) and hasattr(test, 'tmp_dir'):
|
||||
tmp_dir = pathlib.Path(test.tmp_dir.name)
|
||||
else:
|
||||
tmp_dir = pathlib.Path(tempfile.TemporaryDirectory().name)
|
||||
set_test_media_directory(tmp_dir)
|
||||
|
||||
# Convert the channel/video structure to a file structure for the test.
|
||||
file_structure = []
|
||||
for channel, paths in structure.items():
|
||||
for path in paths:
|
||||
file_structure.append(f'{channel}/{path}')
|
||||
file_structure.append(f'{channel}/')
|
||||
|
||||
with build_test_directories(file_structure, tmp_dir) as tempdir:
|
||||
args, kwargs = insert_parameter(func, 'tempdir', tempdir, args, kwargs)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
for channel in structure:
|
||||
channel_ = Channel(directory=str(tempdir / channel), name=channel, link=channel)
|
||||
session.add(channel_)
|
||||
session.flush()
|
||||
session.refresh(channel_)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
for channel_ in session.query(Channel).all():
|
||||
refresh_channel_videos(channel_)
|
||||
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
finally:
|
||||
set_test_media_directory(None)
|
||||
|
||||
return wrapped
|
||||
|
||||
return wrapper
|
||||
72
modules/videos/test/test_api.py
Normal file
72
modules/videos/test/test_api.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import shutil
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
|
||||
from modules.videos.models import Video
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
def test_refresh_videos(test_client, test_session, test_directory, simple_channel, video_factory):
|
||||
subdir = test_directory / 'subdir'
|
||||
subdir.mkdir()
|
||||
|
||||
# video1 is in a subdirectory.
|
||||
video1 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True,
|
||||
with_poster_ext='jpg')
|
||||
test_session.commit()
|
||||
shutil.move(video1.video_path.path, subdir / video1.video_path.path.name)
|
||||
shutil.move(video1.poster_path.path, subdir / video1.poster_path.path.name)
|
||||
video1.video_path = subdir / video1.video_path.path.name
|
||||
video1.poster_path = None
|
||||
# video2 is in the test directory.
|
||||
video2 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True,
|
||||
with_poster_ext='jpg')
|
||||
video2.poster_path = video1.poster_path = None
|
||||
test_session.commit()
|
||||
|
||||
assert not video1.size, 'video1 should not have size during creation'
|
||||
|
||||
# Create a video not in the DB.
|
||||
vid3 = test_directory / 'vid3.mp4'
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', vid3)
|
||||
|
||||
# Orphan poster should be ignored.
|
||||
orphan_poster = Path(test_directory / 'channel name_20000104_defghijklmn_title.jpg')
|
||||
orphan_poster.touch()
|
||||
|
||||
# Create a bogus video in the channel.
|
||||
bogus = Video(video_path='foo', channel_id=simple_channel.id)
|
||||
test_session.add(bogus)
|
||||
|
||||
test_client.post('/api/videos/refresh')
|
||||
|
||||
# Posters were found during refresh.
|
||||
assert video1.poster_path
|
||||
assert 'subdir' in str(video1.poster_path.path)
|
||||
assert video2.poster_path
|
||||
# Missing video3 was found
|
||||
video3: Video = test_session.query(Video).filter_by(id=4).one()
|
||||
assert video3.video_path.path == vid3
|
||||
assert video3.video_path != video1.video_path
|
||||
assert video3.video_path != video2.video_path
|
||||
# Bogus video was removed.
|
||||
assert not any('foo' in str(i.video_path.path) for i in test_session.query(Video).all())
|
||||
# Orphan file was not deleted.
|
||||
assert orphan_poster.is_file(), 'Orphan poster was removed!'
|
||||
|
||||
assert video1.size, 'video1 size was not found'
|
||||
|
||||
|
||||
def test_api_download_channel(test_session, test_client, simple_channel):
|
||||
"""A Channel download (a catalog update) can be triggered via the API."""
|
||||
# simple_channel does not have a download record.
|
||||
request, response = test_client.post(f'/api/videos/download/{simple_channel.id}')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.json
|
||||
assert 'not have a download' in response.json['message']
|
||||
|
||||
# Add a download frequency to the channel, this should also create a download.
|
||||
simple_channel.update(dict(download_frequency=DownloadFrequency.daily))
|
||||
test_session.commit()
|
||||
request, response = test_client.post(f'/api/videos/download/{simple_channel.id}')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
|
@ -4,23 +4,19 @@ import tempfile
|
|||
from datetime import datetime
|
||||
from typing import List
|
||||
from unittest import mock
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.test.common import create_channel_structure
|
||||
from wrolpi.common import get_absolute_media_path
|
||||
from wrolpi.dates import local_timezone
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.common import get_absolute_media_path, sanitize_link
|
||||
from wrolpi.dates import local_timezone, now
|
||||
from wrolpi.downloader import Download, DownloadFrequency
|
||||
from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI
|
||||
from wrolpi.test.common import build_test_directories, TestAPI
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \
|
||||
apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \
|
||||
import_videos_config
|
||||
from ..lib import save_channels_config, get_channels_config
|
||||
apply_info_json, get_video_duration, generate_video_poster, is_valid_poster
|
||||
from ..lib import save_channels_config, get_channels_config, import_channels_config
|
||||
|
||||
|
||||
class TestCommon(TestAPI):
|
||||
|
|
@ -134,113 +130,50 @@ def test_is_valid_poster(ext, expected, test_directory):
|
|||
assert is_valid_poster(image_path) == expected, f'is_valid_poster({image_path}) should be {expected}'
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'_structure,paths',
|
||||
(
|
||||
(
|
||||
{'channel1': ['vid1.mp4']},
|
||||
[
|
||||
'channel1/vid1.mp4',
|
||||
],
|
||||
),
|
||||
(
|
||||
{'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4']},
|
||||
[
|
||||
'channel1/vid1.mp4',
|
||||
'channel2/vid1.mp4',
|
||||
],
|
||||
),
|
||||
(
|
||||
{'channel1': ['vid1.mp4'], 'channel2': ['vid1.mp4', 'vid2.mp4', 'vid2.en.vtt']},
|
||||
[
|
||||
'channel1/vid1.mp4',
|
||||
'channel2/vid1.mp4',
|
||||
'channel2/vid2.mp4',
|
||||
'channel2/vid2.en.vtt',
|
||||
],
|
||||
),
|
||||
)
|
||||
)
|
||||
def test_create_db_structure(_structure, paths):
|
||||
@create_channel_structure(_structure)
|
||||
def test_func(_, tempdir):
|
||||
assert isinstance(tempdir, pathlib.Path)
|
||||
for path in paths:
|
||||
path = (tempdir / path)
|
||||
assert path.exists()
|
||||
assert path.is_file()
|
||||
|
||||
with get_db_session() as session:
|
||||
for channel_name in _structure:
|
||||
channel = session.query(Channel).filter_by(name=channel_name).one()
|
||||
assert (tempdir / channel_name).is_dir()
|
||||
assert channel
|
||||
assert channel.directory == tempdir / channel_name
|
||||
assert len(channel.videos) == len([i for i in _structure[channel_name] if i.endswith('mp4')])
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir, \
|
||||
mock.patch('wrolpi.media_path.get_media_directory') as mock_get_directory:
|
||||
mock_get_directory.return_value = pathlib.Path(tmp_dir)
|
||||
test_ = MagicMock()
|
||||
test_.tmp_dir.name = tmp_dir
|
||||
test_func(test_)
|
||||
|
||||
|
||||
@wrap_test_db
|
||||
@create_channel_structure(
|
||||
{
|
||||
'channel1': ['vid1.mp4', 'vid1.jpg'],
|
||||
'channel2': ['vid2.mp4'],
|
||||
'channel3': ['vid3.mp4', 'vid4.mp4'],
|
||||
}
|
||||
)
|
||||
def test_update_view_count(tempdir: pathlib.Path):
|
||||
def test_update_view_count(test_session, channel_factory, video_factory):
|
||||
def check_view_counts(view_counts):
|
||||
with get_db_session() as session_:
|
||||
for source_id, view_count in view_counts.items():
|
||||
vid = session_.query(Video).filter_by(source_id=source_id).one()
|
||||
assert vid.view_count == view_count
|
||||
for source_id, view_count in view_counts.items():
|
||||
video = test_session.query(Video).filter_by(source_id=source_id).one()
|
||||
assert video.view_count == view_count
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
channel1, channel2, channel3 = session.query(Channel).order_by(Channel.id).all()
|
||||
channel1.info_json = {'entries': [{'id': 'vid1.mp4', 'view_count': 10}]}
|
||||
channel2.info_json = {'entries': [{'id': 'vid2.mp4', 'view_count': 11}, {'id': 'bad_id', 'view_count': 12}]}
|
||||
channel3.info_json = {'entries': [{'id': 'vid3.mp4', 'view_count': 13}, {'id': 'vid4.mp4', 'view_count': 14}]}
|
||||
|
||||
# Use the video file name as a unique source_id
|
||||
for v in session.query(Video).all():
|
||||
v.source_id = str(v.video_path.path).split('/')[-1]
|
||||
channel1, channel2, channel3 = channel_factory(), channel_factory(), channel_factory()
|
||||
video_factory(channel_id=channel1.id, with_poster_ext='jpg', title='vid1')
|
||||
video_factory(channel_id=channel2.id, title='vid2')
|
||||
video_factory(channel_id=channel3.id, with_poster_ext='jpg', title='vid3')
|
||||
video_factory(channel_id=channel3.id, title='vid4')
|
||||
channel1.info_json = {'entries': [{'id': 'vid1', 'view_count': 10}]}
|
||||
channel2.info_json = {'entries': [{'id': 'vid2', 'view_count': 11}, {'id': 'bad_id', 'view_count': 12}]}
|
||||
channel3.info_json = {'entries': [{'id': 'vid3', 'view_count': 13}, {'id': 'vid4', 'view_count': 14}]}
|
||||
test_session.commit()
|
||||
|
||||
# Check all videos are empty.
|
||||
check_view_counts({'vid1.mp4': None, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None})
|
||||
check_view_counts({'vid1': None, 'vid2': None, 'vid3': None, 'vid4': None})
|
||||
|
||||
# Channel 1 is updated, the other channels are left alone.
|
||||
apply_info_json(channel1.id)
|
||||
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None})
|
||||
check_view_counts({'vid1': 10, 'vid2': None, 'vid3': None, 'vid4': None})
|
||||
|
||||
# Channel 2 is updated, the other channels are left alone. The 'bad_id' video is ignored.
|
||||
apply_info_json(channel2.id)
|
||||
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': None, 'vid4.mp4': None})
|
||||
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': None, 'vid4': None})
|
||||
|
||||
# All videos are updated.
|
||||
apply_info_json(channel3.id)
|
||||
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14})
|
||||
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})
|
||||
|
||||
# An outdated view count will be overwritten.
|
||||
with get_db_session(commit=True) as session:
|
||||
vid = session.query(Video).filter_by(id=1).one()
|
||||
vid.view_count = 8
|
||||
check_view_counts({'vid1.mp4': 8, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14})
|
||||
vid = test_session.query(Video).filter_by(id=1).one()
|
||||
vid.view_count = 8
|
||||
check_view_counts({'vid1': 8, 'vid2': 11, 'vid3': 13, 'vid4': 14})
|
||||
apply_info_json(channel1.id)
|
||||
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14})
|
||||
check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})
|
||||
|
||||
|
||||
def test_generate_video_poster(video_file):
|
||||
"""
|
||||
A poster can be generated from a video file.
|
||||
"""
|
||||
poster_path = replace_extension(video_file, '.jpg')
|
||||
assert not poster_path.is_file(), f'{poster_path} already exists!'
|
||||
poster_path = video_file.with_suffix('.jpg')
|
||||
generate_video_poster(video_file)
|
||||
assert poster_path.is_file(), f'{poster_path} was not created!'
|
||||
assert poster_path.stat().st_size > 0
|
||||
|
|
@ -326,7 +259,7 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan
|
|||
vid2_video_path = vid2.video_path.path
|
||||
|
||||
favorites = {
|
||||
simple_channel.link: {vid3.video_path.path.name: {'favorite': favorite}},
|
||||
str(simple_channel.directory.relative): {vid3.video_path.path.name: {'favorite': favorite}},
|
||||
'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}},
|
||||
}
|
||||
|
||||
|
|
@ -336,8 +269,8 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan
|
|||
assert config.favorites == favorites
|
||||
|
||||
def import_and_verify(favorited_ids: List[int]):
|
||||
with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'):
|
||||
import_videos_config()
|
||||
with mock.patch('modules.videos.lib.get_channel_source_id', lambda i: 'foo'):
|
||||
import_channels_config()
|
||||
for video in test_session.query(Video).all():
|
||||
if video.id in favorited_ids:
|
||||
assert video.favorite
|
||||
|
|
@ -371,10 +304,8 @@ def test_import_favorites(test_session, simple_channel, video_factory, test_chan
|
|||
|
||||
def test_import_channel_downloads(test_session, channel_factory, test_channels_config):
|
||||
"""Importing the Channels' config should create any missing download records"""
|
||||
channel1 = channel_factory()
|
||||
channel2 = channel_factory()
|
||||
channel1.source_id = 'foo'
|
||||
channel2.source_id = 'bar'
|
||||
channel1 = channel_factory(source_id='foo')
|
||||
channel2 = channel_factory(source_id='bar')
|
||||
# channel2 has no url, but has a frequency. Import should not create a download record.
|
||||
channel2.url = None
|
||||
channel2.download_frequency = DownloadFrequency.biweekly
|
||||
|
|
@ -383,20 +314,25 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
|
|||
assert len(test_session.query(Channel).all()) == 2
|
||||
assert test_session.query(Download).all() == []
|
||||
|
||||
def update_channel_config(conf, source_id, d):
|
||||
for c in conf.channels:
|
||||
if c['source_id'] == source_id:
|
||||
c.update(d)
|
||||
conf.save()
|
||||
|
||||
# Config has no channels with a download_frequency.
|
||||
save_channels_config()
|
||||
import_videos_config()
|
||||
import_channels_config()
|
||||
assert channel1.download_frequency is None
|
||||
assert len(test_session.query(Channel).all()) == 2
|
||||
assert test_session.query(Download).all() == []
|
||||
|
||||
# Add a frequency to the Channel.
|
||||
channels_config = get_channels_config()
|
||||
channels_config.channels[channel1.link]['download_frequency'] = DownloadFrequency.biweekly
|
||||
channels_config.save()
|
||||
update_channel_config(channels_config, 'foo', {'download_frequency': DownloadFrequency.biweekly})
|
||||
|
||||
# Download record is created on import.
|
||||
import_videos_config()
|
||||
import_channels_config()
|
||||
assert channel1.download_frequency is not None
|
||||
assert len(test_session.query(Channel).all()) == 2
|
||||
download: Download = test_session.query(Download).one()
|
||||
|
|
@ -404,9 +340,8 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
|
|||
assert download.frequency == channel1.download_frequency
|
||||
|
||||
# Download frequency is adjusted when config file changes.
|
||||
channels_config.channels[channel1.link]['download_frequency'] = DownloadFrequency.weekly
|
||||
channels_config.save()
|
||||
import_videos_config()
|
||||
update_channel_config(channels_config, 'foo', {'download_frequency': DownloadFrequency.weekly})
|
||||
import_channels_config()
|
||||
assert len(test_session.query(Channel).all()) == 2
|
||||
assert download.url == channel1.url
|
||||
assert download.frequency == channel1.download_frequency
|
||||
|
|
@ -414,6 +349,67 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
|
|||
assert download.next_download
|
||||
|
||||
next_download = str(download.next_download)
|
||||
import_videos_config()
|
||||
import_channels_config()
|
||||
download: Download = test_session.query(Download).one()
|
||||
assert next_download == str(download.next_download)
|
||||
|
||||
|
||||
def test_import_channels_config_outdated(test_session, test_directory, channel_factory, test_channels_config,
|
||||
video_factory):
|
||||
"""The Channels' config used to use a "link" to differentiate channels, test that old configs can be imported."""
|
||||
channel1 = channel_factory(name='Channel1', download_frequency=DownloadFrequency.weekly)
|
||||
channel2 = channel_factory(name='Channel2', download_frequency=DownloadFrequency.weekly)
|
||||
test_session.commit()
|
||||
|
||||
vid1 = video_factory(title='vid1', channel_id=channel1.id)
|
||||
vid2 = video_factory(title='vid2', channel_id=channel2.id)
|
||||
vid3 = video_factory(title='vid3')
|
||||
vid4 = video_factory(title='vid4')
|
||||
test_session.commit()
|
||||
|
||||
channel1_name, channel_1_directory = channel1.name, channel1.directory.path
|
||||
channel2_name, channel_2_directory = channel2.name, channel2.directory.path
|
||||
|
||||
assert channel1.directory.path.exists() and channel1.directory.path.is_absolute()
|
||||
assert channel2.directory.path.exists() and channel2.directory.path.is_absolute()
|
||||
|
||||
# Channel's used to have a relative directory in the DB.
|
||||
test_session.execute('UPDATE channel SET directory=:directory WHERE id=2',
|
||||
dict(directory=str(channel_2_directory.relative_to(test_directory))),
|
||||
)
|
||||
|
||||
save_channels_config()
|
||||
|
||||
# The config has a list of Channels, use the old method of a "dict" with the sanitized name as the key.
|
||||
channels_config = get_channels_config()
|
||||
channels_config.channels = {sanitize_link(i['name']): i for i in channels_config.channels}
|
||||
# Change frequency to verify that the channels are updated.
|
||||
for link, channel in channels_config.channels.items():
|
||||
channel['download_frequency'] = DownloadFrequency.biweekly
|
||||
# Change the favorites to the old "link" method as well.
|
||||
channels_config.favorites = {
|
||||
sanitize_link(channel1.name): {str(vid1.video_path.path): {'favorite': now()}},
|
||||
sanitize_link(channel2.name): {str(vid2.video_path.path): {'favorite': now()}},
|
||||
'NO CHANNEL': {str(vid3.video_path.path): {'favorite': now()}},
|
||||
}
|
||||
channels_config.save()
|
||||
|
||||
assert not vid1.favorite and not vid2.favorite and not vid3.favorite and not vid3.favorite
|
||||
|
||||
assert test_session.query(Channel).count() == 2
|
||||
|
||||
with mock.patch('modules.videos.lib.get_channel_source_id') as mock_get_channel_source_id:
|
||||
mock_get_channel_source_id.return_value = 'some source id'
|
||||
import_channels_config()
|
||||
|
||||
# No new channels were created. Existing channels were updated.
|
||||
assert test_session.query(Channel).count() == 2
|
||||
channel1, channel2 = test_session.query(Channel).order_by(Channel.id)
|
||||
assert channel1.name == channel1_name and str(channel1.directory.path) == str(channel_1_directory)
|
||||
# Channel2's directory is now absolute.
|
||||
assert channel2.name == channel2_name and str(channel2.directory.path) == str(channel_2_directory)
|
||||
assert channel1.download_frequency == DownloadFrequency.biweekly
|
||||
assert channel2.download_frequency == DownloadFrequency.biweekly
|
||||
# Videos were favorited by their channel "link".
|
||||
assert vid1.favorite and vid2.favorite and vid3.favorite
|
||||
assert not vid4.favorite
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import pathlib
|
||||
import shutil
|
||||
from copy import copy
|
||||
from unittest import mock
|
||||
|
||||
|
|
@ -9,12 +10,12 @@ from modules.videos.channel.lib import download_channel
|
|||
from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \
|
||||
ChannelDownloader, get_or_create_channel
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.test.common import create_channel_structure
|
||||
from wrolpi.db import get_db_session, get_db_context
|
||||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import DownloadManager, Download
|
||||
from wrolpi.errors import InvalidDownload
|
||||
from wrolpi.test.common import wrap_test_db, TestAPI
|
||||
from wrolpi.test.test_downloader import HTTPDownloader
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
class FakeYDL:
|
||||
|
|
@ -28,37 +29,32 @@ class FakeYDL:
|
|||
}
|
||||
|
||||
|
||||
class TestDownloader(TestAPI):
|
||||
@wrap_test_db
|
||||
@create_channel_structure(
|
||||
{
|
||||
'channel1': ['vid1.mp4'],
|
||||
'channel2': ['vid2.mp4']
|
||||
}
|
||||
)
|
||||
def test_find_all_missing_videos(self, tempdir):
|
||||
with get_db_session(commit=True) as session:
|
||||
channel1, channel2 = session.query(Channel).order_by(Channel.id).all()
|
||||
channel1.url = channel2.url = 'some url'
|
||||
channel1.info_json = {'entries': [{'id': 'foo'}]}
|
||||
def test_find_all_missing_videos(test_session, channel_factory, video_factory):
|
||||
channel1, channel2 = channel_factory(url='some url'), channel_factory(url='some other url')
|
||||
channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]}
|
||||
test_session.commit()
|
||||
|
||||
# No missing videos
|
||||
self.assertEqual([], list(find_all_missing_videos()))
|
||||
# Two videos are already downloaded.
|
||||
video_factory()
|
||||
video_factory()
|
||||
|
||||
# Create a video that has no video file.
|
||||
with get_db_session(commit=True) as session:
|
||||
video = Video(title='needs to be downloaded', channel_id=channel1.id, source_id='foo')
|
||||
session.add(video)
|
||||
# No missing videos
|
||||
assert [] == list(find_all_missing_videos())
|
||||
|
||||
missing = list(find_all_missing_videos())
|
||||
self.assertEqual(len(missing), 1)
|
||||
# Create a video that has no video file.
|
||||
video = Video(title='needs to be downloaded', channel_id=channel1.id, source_id='foo')
|
||||
test_session.add(video)
|
||||
test_session.commit()
|
||||
|
||||
# The video returned is the one we faked.
|
||||
id_, source_id, entry = missing[0]
|
||||
# Two videos were created for this test already.
|
||||
self.assertEqual(id_, 3)
|
||||
# The fake entry we added is regurgitated back.
|
||||
self.assertEqual(entry, channel1.info_json['entries'][0])
|
||||
missing = list(find_all_missing_videos())
|
||||
assert len(missing) == 1
|
||||
|
||||
# The video returned is the one we faked.
|
||||
id_, source_id, entry = missing[0]
|
||||
# Two videos were created for this test already.
|
||||
assert id_ == 3
|
||||
# The fake entry we added is regurgitated back.
|
||||
assert entry == channel1.info_json['entries'][0]
|
||||
|
||||
|
||||
example_video_json = {
|
||||
|
|
@ -171,43 +167,6 @@ class TestVideosDownloaders(TestAPI):
|
|||
info_json_path.touch()
|
||||
return video_path, poster_path, description_path, info_json_path
|
||||
|
||||
@wrap_test_db
|
||||
def test_video_download(self):
|
||||
channel_dir = self.videos_dir / 'channel name'
|
||||
channel_dir.mkdir(parents=True)
|
||||
|
||||
# Video paths
|
||||
video_path, poster_path, description_path, info_json_path = self._make_video_files(channel_dir)
|
||||
|
||||
with get_db_session() as session:
|
||||
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
|
||||
mock_extract_info.return_value = example_video_json
|
||||
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
||||
mock_process_runner.return_value = 0
|
||||
|
||||
self.mgr.create_download(url, session)
|
||||
|
||||
mock_process_runner.assert_called_once()
|
||||
video_url, _, out_dir = mock_process_runner.call_args[0]
|
||||
|
||||
download: Download = session.query(Download).one()
|
||||
self.assertEqual(video_url, download.url)
|
||||
channel: Channel = session.query(Channel).one()
|
||||
self.assertEqual(channel.directory, channel_dir)
|
||||
|
||||
video: Video = session.query(Video).one()
|
||||
self.assertEqual(video.video_path, video_path)
|
||||
self.assertEqual(video.description_path, description_path)
|
||||
self.assertEqual(video.poster_path, poster_path)
|
||||
self.assertEqual(video.info_json_path, info_json_path)
|
||||
self.assertTrue(video.video_path.path.is_absolute())
|
||||
self.assertTrue(video.poster_path.path.is_absolute())
|
||||
self.assertTrue(video.description_path.path.is_absolute())
|
||||
self.assertTrue(video.info_json_path.path.is_absolute())
|
||||
|
||||
@wrap_test_db
|
||||
def test_video_download_no_channel(self):
|
||||
_, session = get_db_context()
|
||||
|
|
@ -239,73 +198,65 @@ class TestVideosDownloaders(TestAPI):
|
|||
video: Video = session.query(Video).one()
|
||||
self.assertEqual(str(video.video_path.path), f'{channel_dir}/video.mp4')
|
||||
|
||||
@wrap_test_db
|
||||
def test_channel_download(self):
|
||||
_, session = get_db_context()
|
||||
|
||||
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
|
||||
session.add(Channel(url=url, link='foo'))
|
||||
def test_channel_download(test_session, simple_channel, video_download_manager, video_file):
|
||||
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
|
||||
|
||||
channel_dir = self.videos_dir / 'NO CHANNEL'
|
||||
channel_dir.mkdir(parents=True)
|
||||
video_path, _, _, _ = self._make_video_files(channel_dir)
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
|
||||
mock_extract_info.return_value = example_channel_json
|
||||
mock_prepare_filename.return_value = (video_file, {'id': 'foo'})
|
||||
mock_process_runner.return_value = 0
|
||||
video_download_manager.create_download(url, test_session)
|
||||
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
|
||||
mock_extract_info.return_value = example_channel_json
|
||||
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
||||
mock_process_runner.return_value = 0
|
||||
self.mgr.create_download(url, session)
|
||||
# Two videos are in the example channel.
|
||||
downloads = video_download_manager.get_new_downloads(test_session)
|
||||
expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o']
|
||||
for download, expected in zip(downloads, expected):
|
||||
assert download.url == expected
|
||||
# Download is skipped for test.
|
||||
assert download.status == 'new'
|
||||
|
||||
# Two videos are in the example channel.
|
||||
downloads = self.mgr.get_new_downloads(session)
|
||||
expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o']
|
||||
for download, expected in zip(downloads, expected):
|
||||
self.assertEqual(download.url, expected)
|
||||
# Download is skipped for test.
|
||||
self.assertEqual(download.status, 'new')
|
||||
|
||||
@wrap_test_db
|
||||
def test_get_or_create_channel(self):
|
||||
"""
|
||||
A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can
|
||||
match it.
|
||||
"""
|
||||
with get_db_session(commit=True) as session:
|
||||
c1 = Channel(name='foo', link='foo', source_id='foo', url='foo')
|
||||
c2 = Channel(name='bar', link='bar', source_id='bar')
|
||||
c3 = Channel(name='baz', link='baz', source_id='baz', url='baz')
|
||||
c4 = Channel(name='qux', link='qux')
|
||||
session.add_all([c1, c2, c3, c4])
|
||||
def test_get_or_create_channel(test_session):
|
||||
"""
|
||||
A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can
|
||||
match it.
|
||||
"""
|
||||
c1 = Channel(name='foo', source_id='foo', url='foo')
|
||||
c2 = Channel(name='bar', source_id='bar')
|
||||
c3 = Channel(name='baz', source_id='baz', url='baz')
|
||||
c4 = Channel(name='qux')
|
||||
test_session.add_all([c1, c2, c3, c4])
|
||||
test_session.commit()
|
||||
|
||||
# All existing channels should be used.
|
||||
tests = [
|
||||
(dict(source_id='foo'), c1),
|
||||
(dict(link='foo'), c1),
|
||||
(dict(url='foo'), c1),
|
||||
(dict(url='foo', source_id='bar'), c2), # source_id is preferred.
|
||||
(dict(name='foo', source_id='bar'), c2),
|
||||
(dict(source_id='bar'), c2),
|
||||
(dict(source_id='baz'), c3),
|
||||
(dict(name='qux'), c4),
|
||||
(dict(link='qux'), c4),
|
||||
]
|
||||
for kwargs, expected in tests:
|
||||
# All existing channels should be used.
|
||||
tests = [
|
||||
(dict(source_id='foo'), c1),
|
||||
(dict(url='foo'), c1),
|
||||
(dict(url='foo', source_id='bar'), c2), # source_id has priority.
|
||||
(dict(name='foo', source_id='bar'), c2),
|
||||
(dict(source_id='bar'), c2),
|
||||
(dict(source_id='baz'), c3),
|
||||
(dict(name='qux'), c4),
|
||||
]
|
||||
for kwargs, expected in tests:
|
||||
try:
|
||||
channel = get_or_create_channel(**kwargs)
|
||||
self.assertEqual(expected.id, channel.id, f'Expected {expected} got {channel}')
|
||||
except Exception as e:
|
||||
raise Exception(f'get_or_create_channel failed with {kwargs=}') from e
|
||||
assert expected.id == channel.id, f'Expected {expected} for {kwargs} but got {channel}'
|
||||
|
||||
# A new channel is created. It will not be automatically downloaded.
|
||||
channel = get_or_create_channel(source_id='quux', name='quux', url='quux')
|
||||
self.assertEqual(channel.id, 5)
|
||||
self.assertEqual(channel.source_id, 'quux')
|
||||
self.assertEqual(channel.name, 'quux')
|
||||
self.assertEqual(channel.link, 'quux')
|
||||
self.assertEqual(channel.url, 'quux')
|
||||
self.assertIsNone(channel.download_frequency)
|
||||
# A new channel is created. It will not be automatically downloaded.
|
||||
channel = get_or_create_channel(source_id='quux', name='quux', url='quux')
|
||||
assert channel.id == 5
|
||||
assert channel.source_id == 'quux'
|
||||
assert channel.name == 'quux'
|
||||
assert channel.url == 'quux'
|
||||
|
||||
# New channel can be retrieved.
|
||||
self.assertEqual(get_or_create_channel(source_id='quux'), channel)
|
||||
# New channel can be retrieved.
|
||||
assert get_or_create_channel(source_id='quux') == channel
|
||||
|
||||
|
||||
def test_download_info_save(test_session, video_download_manager):
|
||||
|
|
@ -397,7 +348,7 @@ def test_channel_download_no_download(test_session, video_download_manager, simp
|
|||
assert video_download_manager.get_downloads(test_session) == []
|
||||
|
||||
with pytest.raises(InvalidDownload):
|
||||
download_channel(simple_channel.link)
|
||||
download_channel(simple_channel.id)
|
||||
|
||||
|
||||
def test_invalid_download_url(test_session, test_download_manager, test_downloader_config):
|
||||
|
|
@ -413,3 +364,32 @@ def test_invalid_download_url(test_session, test_download_manager, test_download
|
|||
|
||||
download = test_download_manager.get_downloads(test_session)[0]
|
||||
assert download.status == 'failed'
|
||||
|
||||
|
||||
def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager):
|
||||
simple_channel.source_id = example_video_json['channel_id']
|
||||
simple_channel.directory = test_directory / 'videos/channel name'
|
||||
simple_channel.directory.mkdir(parents=True)
|
||||
|
||||
video_file = simple_channel.directory / 'a video.mp4'
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file)
|
||||
|
||||
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
|
||||
mock_extract_info.return_value = example_video_json
|
||||
mock_prepare_filename.return_value = (video_file, {'id': 'foo'})
|
||||
mock_process_runner.return_value = 0
|
||||
|
||||
video_download_manager.create_download(url, test_session)
|
||||
|
||||
mock_process_runner.assert_called_once()
|
||||
video_url, _, out_dir = mock_process_runner.call_args[0]
|
||||
|
||||
download: Download = test_session.query(Download).one()
|
||||
assert video_url == download.url
|
||||
assert test_session.query(Channel).one()
|
||||
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.video_path.path.is_absolute()
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ async def search(_: Request, body: schema.VideoSearchRequest):
|
|||
body.search_str,
|
||||
body.offset,
|
||||
body.limit,
|
||||
body.channel_link,
|
||||
body.channel_id,
|
||||
body.order_by,
|
||||
body.filters,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -80,16 +80,16 @@ def video_search(
|
|||
search_str: str = None,
|
||||
offset: int = None,
|
||||
limit: int = VIDEO_QUERY_LIMIT,
|
||||
channel_link: str = None,
|
||||
channel_id: int = None,
|
||||
order_by: str = None,
|
||||
filters: List[str] = None,
|
||||
) -> Tuple[List[dict], int]:
|
||||
with get_db_curs() as curs:
|
||||
params = dict(search_str=search_str, offset=offset)
|
||||
channel_where = ''
|
||||
if channel_link:
|
||||
channel_where = 'AND channel_id = (select id from channel where link=%(channel_link)s)'
|
||||
params['channel_link'] = channel_link
|
||||
if channel_id:
|
||||
channel_where = 'AND channel_id = %(channel_id)s'
|
||||
params['channel_id'] = channel_id
|
||||
|
||||
# Filter for/against favorites, if it was provided
|
||||
favorites_where = ''
|
||||
|
|
|
|||
|
|
@ -1,163 +1,141 @@
|
|||
import pathlib
|
||||
from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
from json import dumps
|
||||
from unittest import mock
|
||||
from uuid import uuid4
|
||||
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.test.common import create_channel_structure
|
||||
from modules.videos.models import Video
|
||||
from modules.videos.video.lib import get_video_for_app
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import API_ERRORS, WROLModeEnabled
|
||||
from wrolpi.root_api import api_app
|
||||
from wrolpi.test.common import wrap_test_db, TestAPI
|
||||
|
||||
|
||||
class TestVideoFunctions(TestAPI):
|
||||
def test_get_video_prev_next(test_session, channel_factory, video_factory):
|
||||
"""
|
||||
Test that the previous and next videos will be retrieved when fetching a video.
|
||||
"""
|
||||
channel1, channel2, channel3, channel4 = channel_factory(), channel_factory(), channel_factory(), channel_factory()
|
||||
|
||||
@wrap_test_db
|
||||
def test_get_video_prev_next(self):
|
||||
"""
|
||||
Test that the previous and next videos will be retrieved when fetching a video.
|
||||
"""
|
||||
now_ = now()
|
||||
second = timedelta(seconds=1)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
for _ in range(4):
|
||||
session.add(Channel(link=str(uuid4())))
|
||||
channel1, channel2, channel3, channel4 = session.query(Channel).all()
|
||||
# The upload_date decides the order of the prev/next videos.
|
||||
video_factory(title='vid1', channel_id=channel1.id, upload_date=now_)
|
||||
video_factory(title='vid2', channel_id=channel1.id, upload_date=now_ + second)
|
||||
video_factory(title='vid3', channel_id=channel2.id, upload_date=now_ + (second * 4))
|
||||
video_factory(title='vid4', channel_id=channel1.id, upload_date=now_ + (second * 3))
|
||||
video_factory(title='vid5', channel_id=channel2.id, upload_date=now_ + (second * 2))
|
||||
video_factory(title='vid6', channel_id=channel2.id, upload_date=now_ + (second * 5))
|
||||
video_factory(title='vid7', channel_id=channel1.id)
|
||||
video_factory(title='vid8', channel_id=channel2.id, upload_date=now_ + (second * 7))
|
||||
video_factory(title='vid9', channel_id=channel3.id, upload_date=now_ + (second * 8))
|
||||
# Videos without an upload date can't be placed.
|
||||
video_factory(title='vid10', channel_id=channel4.id)
|
||||
video_factory(title='vid11', channel_id=channel4.id)
|
||||
|
||||
now_ = now()
|
||||
second = timedelta(seconds=1)
|
||||
test_session.commit()
|
||||
|
||||
# The upload_date decides the order of the prev/next videos.
|
||||
session.add(Video(title=f'vid1', channel_id=channel1.id, upload_date=now_))
|
||||
session.add(Video(title=f'vid2', channel_id=channel1.id, upload_date=now_ + second))
|
||||
session.add(Video(title=f'vid3', channel_id=channel2.id, upload_date=now_ + (second * 4)))
|
||||
session.add(Video(title=f'vid4', channel_id=channel1.id, upload_date=now_ + (second * 3)))
|
||||
session.add(Video(title=f'vid5', channel_id=channel2.id, upload_date=now_ + (second * 2)))
|
||||
session.add(Video(title=f'vid6', channel_id=channel2.id, upload_date=now_ + (second * 5)))
|
||||
session.add(Video(title=f'vid7', channel_id=channel1.id))
|
||||
session.add(Video(title=f'vid8', channel_id=channel2.id, upload_date=now_ + (second * 7)))
|
||||
session.add(Video(title=f'vid9', channel_id=channel3.id, upload_date=now_ + (second * 8)))
|
||||
session.add(Video(title=f'vid10', channel_id=channel4.id))
|
||||
session.add(Video(title=f'vid11', channel_id=channel4.id))
|
||||
tests = [
|
||||
# Channel 1's videos were inserted in upload_date order.
|
||||
(1, (None, 'vid2')),
|
||||
(2, ('vid1', 'vid4')),
|
||||
(4, ('vid2', None)), # 7 has no upload_date, so it doesn't come after 4.
|
||||
(7, (None, None)), # 7 has no upload_date, so we don't know the order of it.
|
||||
# Channel 3 has only one video.
|
||||
(9, (None, None)),
|
||||
# Channel 2 was inserted out of order.
|
||||
(5, (None, 'vid3')),
|
||||
(3, ('vid5', 'vid6')),
|
||||
(8, ('vid6', None)),
|
||||
# Channel 4's videos have no upload date, so we don't know what is previous/next.
|
||||
(10, (None, None)),
|
||||
]
|
||||
|
||||
tests = [
|
||||
# Channel 1's videos were inserted in upload_date order.
|
||||
(1, (None, 'vid2')),
|
||||
(2, ('vid1', 'vid4')),
|
||||
(4, ('vid2', None)), # 7 has no upload_date, so it doesn't come after 4.
|
||||
(7, (None, None)), # 7 has no upload_date, so we don't know the order of it.
|
||||
# Channel 3 has only one video.
|
||||
(9, (None, None)),
|
||||
# Channel 2 was inserted out of order.
|
||||
(5, (None, 'vid3')),
|
||||
(3, ('vid5', 'vid6')),
|
||||
(8, ('vid6', None)),
|
||||
# Channel 4's videos have no upload date, so we don't know what is previous/next.
|
||||
(10, (None, None)),
|
||||
]
|
||||
|
||||
for id_, (prev_title, next_title) in tests:
|
||||
video = session.query(Video).filter_by(id=id_).one()
|
||||
prev_video, next_video = video.get_surrounding_videos()
|
||||
for id_, (prev_title, next_title) in tests:
|
||||
video = test_session.query(Video).filter_by(id=id_).one()
|
||||
prev_video, next_video = video.get_surrounding_videos()
|
||||
|
||||
try:
|
||||
if prev_title is None:
|
||||
self.assertIsNone(prev_video)
|
||||
assert prev_video is None
|
||||
else:
|
||||
self.assertDictContains(prev_video, {'title': prev_title})
|
||||
assert prev_video and prev_video.title == prev_title
|
||||
|
||||
if next_title is None:
|
||||
self.assertIsNone(next_video)
|
||||
assert next_video is None
|
||||
else:
|
||||
self.assertDictContains(next_video, {'title': next_title})
|
||||
assert next_video and next_video.title == next_title
|
||||
except AssertionError as e:
|
||||
raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e
|
||||
|
||||
@wrap_test_db
|
||||
@create_channel_structure({
|
||||
'channel1': ['vid1.mp4']
|
||||
})
|
||||
def test_wrol_mode(self, tempdir):
|
||||
"""
|
||||
Many methods are blocked when WROL Mode is enabled.
|
||||
"""
|
||||
channel = dumps(dict(name='foo', directory='foo'))
|
||||
favorite = dumps(dict(video_id=1, favorite=True))
|
||||
|
||||
with mock.patch('wrolpi.common.wrol_mode_enabled', lambda: True):
|
||||
# Can't create, update, or delete a channel.
|
||||
_, resp = api_app.test_client.post('/api/videos/channels', content=channel)
|
||||
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
|
||||
_, resp = api_app.test_client.put('/api/videos/channels/channel1', content=channel)
|
||||
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
|
||||
_, resp = api_app.test_client.delete('/api/videos/channels/channel1')
|
||||
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
|
||||
def test_get_video_for_app(test_session, simple_channel, simple_video):
|
||||
vid, prev, next_ = get_video_for_app(simple_video.id)
|
||||
assert vid['id'] == simple_video.id
|
||||
|
||||
# Can't delete a video
|
||||
_, resp = api_app.test_client.delete('/api/videos/video/1')
|
||||
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
|
||||
|
||||
# Can't refresh or download
|
||||
_, resp = api_app.test_client.post('/api/videos/refresh')
|
||||
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
|
||||
_, resp = api_app.test_client.post('/api/videos/download')
|
||||
self.assertError(resp, HTTPStatus.FORBIDDEN, API_ERRORS[WROLModeEnabled]['code'])
|
||||
def test_video_delete(test_session, test_directory, channel_factory, video_factory):
|
||||
"""Video.delete() removes the video's files, but leave the DB record."""
|
||||
channel1, channel2 = channel_factory(), channel_factory()
|
||||
vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True)
|
||||
vid2 = video_factory(channel_id=channel2.id, with_video_file=True, with_info_json=True)
|
||||
test_session.commit()
|
||||
|
||||
# THE REST OF THESE METHODS ARE ALLOWED
|
||||
_, resp = api_app.test_client.post('/api/videos/favorite', content=favorite)
|
||||
self.assertEqual(resp.status_code, HTTPStatus.OK)
|
||||
vid1_video_path, vid1_caption_path = vid1.video_path.path, vid1.caption_path.path
|
||||
vid2_video_path, vid2_info_json_path = vid2.video_path.path, vid2.info_json_path.path
|
||||
|
||||
@wrap_test_db
|
||||
@create_channel_structure({
|
||||
'channel1': [
|
||||
'vid1.mp4',
|
||||
'vid1.en.vtt',
|
||||
],
|
||||
})
|
||||
def test_get_video_for_app(self, tempdir):
|
||||
with get_db_session(commit=True) as session:
|
||||
vid1 = session.query(Video).one()
|
||||
# No videos have been deleted yet.
|
||||
assert vid1_video_path.is_file() and vid1_caption_path.is_file()
|
||||
assert vid2_video_path.is_file() and vid2_info_json_path.is_file()
|
||||
assert channel1.skip_download_videos is None
|
||||
assert channel2.skip_download_videos is None
|
||||
|
||||
vid, prev, next_ = get_video_for_app(vid1.id)
|
||||
self.assertEqual(vid['id'], vid1.id)
|
||||
vid1.delete()
|
||||
|
||||
@wrap_test_db
|
||||
@create_channel_structure({
|
||||
'channel1': [
|
||||
'vid1.mp4',
|
||||
'vid1.en.vtt',
|
||||
],
|
||||
'channel2': [
|
||||
'vid2.mp4',
|
||||
'vid2.info.json',
|
||||
],
|
||||
})
|
||||
def test_delete_video(self, tempdir: pathlib.Path):
|
||||
with get_db_session(commit=True) as session:
|
||||
channel1 = session.query(Channel).filter_by(name='channel1').one()
|
||||
channel2 = session.query(Channel).filter_by(name='channel2').one()
|
||||
vid1, vid2 = session.query(Video).order_by(Video.video_path).all()
|
||||
vid1.source_id = 'foo'
|
||||
vid2.source_id = 'bar'
|
||||
# Video was added to skip list.
|
||||
assert len(channel1.skip_download_videos) == 1
|
||||
# Deleting a video leaves it's entry in the DB, but its files are deleted.
|
||||
assert test_session.query(Video).count() == 2
|
||||
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
|
||||
assert vid2_video_path.is_file() and vid2_info_json_path.is_file()
|
||||
|
||||
# No videos have been deleted yet.
|
||||
self.assertIsNone(channel1.skip_download_videos)
|
||||
self.assertIsNone(channel2.skip_download_videos)
|
||||
self.assertTrue((tempdir / 'channel1/vid1.mp4').is_file())
|
||||
vid2.delete()
|
||||
|
||||
vid1.delete()
|
||||
assert test_session.query(Video).count() == 2
|
||||
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
|
||||
assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False
|
||||
|
||||
channel1 = session.query(Channel).filter_by(name='channel1').one()
|
||||
# Video was added to skip list.
|
||||
self.assertEqual(len(channel1.skip_download_videos), 1)
|
||||
# Deleting a video leaves it's entry in the DB, but its files are deleted.
|
||||
self.assertEqual(session.query(Video).count(), 2)
|
||||
self.assertFalse((tempdir / 'channel1/vid1.mp4').is_file())
|
||||
self.assertTrue((tempdir / 'channel2/vid2.mp4').is_file())
|
||||
|
||||
vid2.delete()
|
||||
def test_wrol_mode(test_directory, simple_channel, simple_video):
|
||||
"""Many methods are blocked when WROL Mode is enabled."""
|
||||
channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory)))
|
||||
favorite = dumps(dict(video_id=simple_video.id, favorite=True))
|
||||
|
||||
self.assertEqual(session.query(Video).count(), 2)
|
||||
self.assertFalse((tempdir / 'channel1/vid1.mp4').is_file())
|
||||
self.assertFalse((tempdir / 'channel2/vid2.mp4').is_file())
|
||||
with mock.patch('wrolpi.common.wrol_mode_enabled', lambda: True):
|
||||
# Can't create, update, or delete a channel.
|
||||
_, resp = api_app.test_client.post('/api/videos/channels', content=channel)
|
||||
assert resp.status_code == HTTPStatus.FORBIDDEN
|
||||
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
|
||||
_, resp = api_app.test_client.put(f'/api/videos/channels/{simple_channel.id}', content=channel)
|
||||
assert resp.status_code == HTTPStatus.FORBIDDEN
|
||||
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
|
||||
_, resp = api_app.test_client.delete(f'/api/videos/channels/{simple_channel.id}')
|
||||
assert resp.status_code == HTTPStatus.FORBIDDEN
|
||||
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
|
||||
|
||||
# Can't delete a video
|
||||
_, resp = api_app.test_client.delete(f'/api/videos/video/{simple_video.id}')
|
||||
assert resp.status_code == HTTPStatus.FORBIDDEN
|
||||
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
|
||||
|
||||
# Can't refresh or download
|
||||
_, resp = api_app.test_client.post('/api/videos/refresh')
|
||||
assert resp.status_code == HTTPStatus.FORBIDDEN
|
||||
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
|
||||
_, resp = api_app.test_client.post(f'/api/videos/download/{simple_channel.id}')
|
||||
assert resp.status_code == HTTPStatus.FORBIDDEN
|
||||
assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
|
||||
|
||||
# THE REST OF THESE METHODS ARE ALLOWED
|
||||
_, resp = api_app.test_client.post('/api/videos/favorite', content=favorite)
|
||||
assert resp.status_code == HTTPStatus.OK
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import inspect
|
||||
import multiprocessing
|
||||
from functools import wraps
|
||||
|
||||
BEFORE_STARTUP_FUNCTIONS = []
|
||||
|
||||
|
|
@ -29,6 +30,7 @@ def limit_concurrent(limit: int):
|
|||
|
||||
def wrapper(func: callable):
|
||||
if inspect.iscoroutinefunction(func):
|
||||
@wraps(func)
|
||||
async def wrapped(*a, **kw):
|
||||
acquired = sema.acquire(block=False)
|
||||
if not acquired:
|
||||
|
|
@ -37,6 +39,7 @@ def limit_concurrent(limit: int):
|
|||
|
||||
return wrapped
|
||||
else:
|
||||
@wraps(func)
|
||||
def wrapped(*a, **kw):
|
||||
acquired = sema.acquire(block=False)
|
||||
if not acquired:
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ def which(*possible_paths: str, warn: bool = False) -> Path:
|
|||
|
||||
Returns None if no executable can be found.
|
||||
"""
|
||||
possible_paths = map(Path, possible_paths)
|
||||
possible_paths = list(map(Path, possible_paths))
|
||||
for path in possible_paths:
|
||||
if is_executable(path):
|
||||
# `path` was an absolute file which is executable.
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ Base = declarative_base()
|
|||
|
||||
class ModelHelper:
|
||||
|
||||
def dict(self) -> dict:
|
||||
def dict(self, *_, **__) -> dict:
|
||||
d = {i.name: getattr(self, i.name) for i in self.__table__.columns} # noqa
|
||||
return d
|
||||
|
||||
|
|
@ -732,11 +732,6 @@ def native_only(func: callable):
|
|||
return wrapped
|
||||
|
||||
|
||||
def read_yaml(path: Path):
|
||||
with path.open('rt') as fh:
|
||||
return yaml.load(fh, Loader=yaml.Loader)
|
||||
|
||||
|
||||
def recursive_map(obj: Any, func: callable):
|
||||
"""
|
||||
Apply `func` to all values of any dictionaries; or items in any list/set/tuple.
|
||||
|
|
@ -755,3 +750,11 @@ def recursive_map(obj: Any, func: callable):
|
|||
type_ = type(obj)
|
||||
return type_(map(lambda i: recursive_map(i, func), obj))
|
||||
return func(obj)
|
||||
|
||||
|
||||
def any_extensions(filename: str, extensions: Iterable = ()):
|
||||
"""Return True only if the file name ends with any of the possible extensions.
|
||||
|
||||
Matches lower or upper case of the extension.
|
||||
"""
|
||||
return any(filename.lower().endswith(ext) for ext in extensions)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""
|
||||
Fixtures for Pytest tests.
|
||||
"""
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
from typing import Tuple
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ API_ERRORS = {
|
|||
},
|
||||
UnknownChannel: {
|
||||
'code': next(error_codes),
|
||||
'message': 'The channel could not be found by its link.',
|
||||
'message': 'The channel could not be found.',
|
||||
'status': HTTPStatus.NOT_FOUND,
|
||||
},
|
||||
UnknownDirectory: {
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@ class MediaPath:
|
|||
self._path = path.absolute()
|
||||
|
||||
def __repr__(self):
|
||||
return f'<MediaPath path={self.path.relative_to(get_media_directory())}>'
|
||||
return f'<MediaPath path={self.relative}>'
|
||||
|
||||
def __str__(self):
|
||||
return str(self.path.relative_to(get_media_directory()))
|
||||
return str(self.relative)
|
||||
|
||||
@classmethod
|
||||
def _validate_path(cls, path: PATH_TYPE):
|
||||
|
|
@ -40,7 +40,7 @@ class MediaPath:
|
|||
|
||||
def __json__(self):
|
||||
# Always return a relative path to frontend.
|
||||
return str(self.path.relative_to(get_media_directory()))
|
||||
return str(self.relative)
|
||||
|
||||
@property
|
||||
def path(self) -> pathlib.Path:
|
||||
|
|
@ -60,6 +60,11 @@ class MediaPath:
|
|||
def startswith(self, other):
|
||||
return str(self._path).startswith(other)
|
||||
|
||||
@property
|
||||
def relative(self):
|
||||
"""Get the path relative to the media directory."""
|
||||
return self._path.relative_to(get_media_directory())
|
||||
|
||||
|
||||
class MediaPathType(types.TypeDecorator): # noqa
|
||||
impl = types.String
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
__version__ = '0.4.3-beta'
|
||||
__version__ = '0.4.4-beta'
|
||||
|
||||
import subprocess
|
||||
|
||||
|
|
@ -25,7 +25,5 @@ def git_branch():
|
|||
|
||||
|
||||
def get_version_string():
|
||||
"""
|
||||
Return a string containing the WROLPi version, git branch and git revision hash.
|
||||
"""
|
||||
"""Return a string containing the WROLPi version, git branch and git revision hash."""
|
||||
return f'{__version__} (git: {git_branch()} {git_revision()})'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue