mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Feature/censored videos (#65)
* Creating api endpoint to find censored videos. * Videos can be filtered by censored status. * Not deleting when there has not been a successful download. * Search now has a different param order. * Displaying correct header for videos page. * Reworking FilterModal to be more universal. * Using merge_output_format for preparing filename. * Resetting attempts when user requests a download be started. * Displaying video stem as title if no title is available. Signed-off-by: Roland <roland@learningselfreliance.com>
This commit is contained in:
lrnselfreliance
Roland
parent
ea2bf8530b
commit
02116bf2bc
15 changed files with 451 additions and 121 deletions
|
|
@ -36,7 +36,7 @@ async function apiCall(url, method, body, ms = 10000) {
|
|||
if (response.status === 403 && (await response.json())['code'] === 17) {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'WROL Mode is enabled!',
|
||||
header: 'WROL Mode is enabled!',
|
||||
description: 'This functionality is not enabled while WROL Mode is enabled!',
|
||||
time: 5000,
|
||||
});
|
||||
|
|
@ -47,14 +47,14 @@ async function apiCall(url, method, body, ms = 10000) {
|
|||
if (e.message === 'promise timeout') {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Server failed to respond!',
|
||||
header: 'Server failed to respond!',
|
||||
description: 'Timeout while waiting for server response. See server logs.',
|
||||
time: 5000,
|
||||
});
|
||||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Error!',
|
||||
header: 'Error!',
|
||||
description: 'See client logs',
|
||||
time: 5000,
|
||||
});
|
||||
|
|
@ -103,11 +103,11 @@ export async function getChannel(link) {
|
|||
return (await response.json())['channel'];
|
||||
}
|
||||
|
||||
export async function searchVideos(offset, limit, channel_link, searchStr, favorites, order_by) {
|
||||
export async function searchVideos(offset, limit, channel_link, searchStr, order_by, filters) {
|
||||
// Build a search query to retrieve a list of videos from the API
|
||||
offset = offset || 0;
|
||||
limit = limit || DEFAULT_LIMIT;
|
||||
let body = {offset, limit, favorites: favorites};
|
||||
let body = {offset, limit, filters};
|
||||
|
||||
if (searchStr) {
|
||||
body.search_str = searchStr;
|
||||
|
|
@ -138,7 +138,7 @@ export async function deleteVideo(video_id) {
|
|||
if (data.code === 17) {
|
||||
toast({
|
||||
type: 'warning',
|
||||
title: 'WROL Mode Enabled',
|
||||
header: 'WROL Mode Enabled',
|
||||
description: 'This cannot be done while WROL Mode is enabled.',
|
||||
time: 5000,
|
||||
});
|
||||
|
|
@ -147,7 +147,7 @@ export async function deleteVideo(video_id) {
|
|||
if (response.status !== 204) {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Unexpected server response',
|
||||
header: 'Unexpected server response',
|
||||
description: 'Failed to delete video. See server logs.',
|
||||
time: 5000,
|
||||
});
|
||||
|
|
@ -223,7 +223,7 @@ export async function encryptOTP(otp, plaintext) {
|
|||
if (response.status !== 200) {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Error!',
|
||||
header: 'Error!',
|
||||
description: 'Failed to encrypt OTP',
|
||||
time: 5000,
|
||||
});
|
||||
|
|
@ -238,7 +238,7 @@ export async function decryptOTP(otp, ciphertext) {
|
|||
if (response.status !== 200) {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Error!',
|
||||
header: 'Error!',
|
||||
description: 'Failed to decrypt OTP',
|
||||
time: 5000,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ export function VideoCard({video}) {
|
|||
<Card.Header>
|
||||
<Container textAlign='left'>
|
||||
<Link to={video_url} className="no-link-underscore video-card-link">
|
||||
<p>{video.title || video.video_path}</p>
|
||||
<p>{video.title || video.stem || video.video_path}</p>
|
||||
</Link>
|
||||
</Container>
|
||||
</Card.Header>
|
||||
|
|
|
|||
|
|
@ -526,7 +526,7 @@ class EditInventory extends React.Component {
|
|||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Failed to save inventory!',
|
||||
header: 'Failed to save inventory!',
|
||||
description: 'Server responded with an error.',
|
||||
time: 5000,
|
||||
})
|
||||
|
|
@ -542,7 +542,7 @@ class EditInventory extends React.Component {
|
|||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Failed to delete inventory!',
|
||||
header: 'Failed to delete inventory!',
|
||||
description: 'Server responded with an error.',
|
||||
time: 5000,
|
||||
})
|
||||
|
|
@ -575,7 +575,7 @@ class NewInventory extends React.Component {
|
|||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Failed to save inventory!',
|
||||
header: 'Failed to save inventory!',
|
||||
description: 'Server responded with an error.',
|
||||
time: 5000,
|
||||
})
|
||||
|
|
@ -968,7 +968,7 @@ class InventoryAddList extends React.Component {
|
|||
} else {
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Failed to save item!',
|
||||
header: 'Failed to save item!',
|
||||
description: 'Server responded with an error.',
|
||||
time: 5000,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -15,7 +15,21 @@ import Paginator, {
|
|||
} from "./Common"
|
||||
import VideoPage from "./VideoPlayer";
|
||||
import {download, getChannel, getStatistics, getVideo, refresh, searchVideos} from "../api";
|
||||
import {Button, Dropdown, Form, Grid, Header, Icon, Input, Loader, Segment, Statistic, Tab} from "semantic-ui-react";
|
||||
import {
|
||||
Button,
|
||||
Dropdown,
|
||||
Form,
|
||||
Grid,
|
||||
Header,
|
||||
Icon,
|
||||
Input,
|
||||
Loader,
|
||||
Modal,
|
||||
Radio,
|
||||
Segment,
|
||||
Statistic,
|
||||
Tab
|
||||
} from "semantic-ui-react";
|
||||
import * as QueryString from 'query-string';
|
||||
import Container from "semantic-ui-react/dist/commonjs/elements/Container";
|
||||
import {Channels, EditChannel, NewChannel} from "./Channels";
|
||||
|
|
@ -160,7 +174,7 @@ export class FavoriteVideosPreview extends VideosPreview {
|
|||
|
||||
async fetchVideos() {
|
||||
let [videos] = await searchVideos(
|
||||
0, 3, null, null, true, '-favorite');
|
||||
0, 3, null, null, '-favorite', ['favorite']);
|
||||
this.setState({videos});
|
||||
}
|
||||
|
||||
|
|
@ -175,12 +189,47 @@ export class ViewedVideosPreview extends VideosPreview {
|
|||
|
||||
async fetchVideos() {
|
||||
let [videos] = await searchVideos(
|
||||
0, 3, null, null, null, '-viewed');
|
||||
0, 3, null, null, '-viewed');
|
||||
this.setState({videos});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function FilterModal(props) {
|
||||
/*
|
||||
Expecting props.filters = [
|
||||
{label: 'Favorites', name: 'favorite', value: false},
|
||||
{label: 'Censored', name: 'censored', value: false},
|
||||
];
|
||||
*/
|
||||
let inputs = props.filters.map((i) =>
|
||||
<Form.Input>
|
||||
<Radio toggle
|
||||
checked={i.value}
|
||||
label={i.label}
|
||||
onClick={() => props.toggleFilter(i.name)}
|
||||
/>
|
||||
</Form.Input>
|
||||
)
|
||||
|
||||
return (
|
||||
<Modal closeIcon open={props.open} onClose={props.close} onOpen={props.open} size='mini'>
|
||||
<Modal.Header>{props.header}</Modal.Header>
|
||||
<Modal.Content>
|
||||
<Modal.Description>
|
||||
<Form>
|
||||
{inputs}
|
||||
</Form>
|
||||
</Modal.Description>
|
||||
</Modal.Content>
|
||||
<Modal.Actions>
|
||||
{props.applyButton && <Button color='blue'>Apply</Button>}
|
||||
<Button color='black' onClick={props.close}>Close</Button>
|
||||
</Modal.Actions>
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
class Videos extends React.Component {
|
||||
|
||||
|
|
@ -191,6 +240,11 @@ class Videos extends React.Component {
|
|||
let searchStr = query.q || '';
|
||||
let searchOrder = query.o || defaultVideoOrder;
|
||||
|
||||
let filters = [
|
||||
{label: 'Favorites', name: 'favorite', value: this.props.filter === 'favorite'},
|
||||
{label: 'Censored', name: 'censored', value: this.props.filter === 'censored'},
|
||||
];
|
||||
|
||||
this.state = {
|
||||
channel: null,
|
||||
videos: null,
|
||||
|
|
@ -203,7 +257,10 @@ class Videos extends React.Component {
|
|||
next: null,
|
||||
videoOrders: searchStr === '' ? videoOrders : searchOrders,
|
||||
searchOrder: searchOrder,
|
||||
title: '',
|
||||
header: '',
|
||||
filtersOpen: false,
|
||||
filters,
|
||||
filterEnabled: false,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -213,7 +270,7 @@ class Videos extends React.Component {
|
|||
} else {
|
||||
await this.fetchVideos();
|
||||
}
|
||||
this.setTitle();
|
||||
this.setHeader();
|
||||
}
|
||||
|
||||
async componentDidUpdate(prevProps, prevState, snapshot) {
|
||||
|
|
@ -232,24 +289,29 @@ class Videos extends React.Component {
|
|||
this.applyStateToHistory();
|
||||
await this.fetchVideos();
|
||||
}
|
||||
|
||||
let filterEnabled = this.enabledFilters().length > 0;
|
||||
if (this.state.filterEnabled !== filterEnabled) {
|
||||
this.state.filterEnabled = filterEnabled;
|
||||
}
|
||||
}
|
||||
|
||||
setTitle() {
|
||||
let title = '';
|
||||
if (this.props.filter === 'favorites') {
|
||||
title = 'Favorite Videos';
|
||||
setHeader() {
|
||||
let header = '';
|
||||
if (this.props.header) {
|
||||
header = this.props.header;
|
||||
} else if (this.state.channel) {
|
||||
title = this.state.channel.name;
|
||||
header = this.state.channel.name;
|
||||
} else {
|
||||
// Find the matching title from the search orders.
|
||||
// Find the matching header from the search orders.
|
||||
for (let i = 0; i < this.state.videoOrders.length; i++) {
|
||||
let item = this.state.videoOrders[i];
|
||||
if (item.value === this.state.searchOrder) {
|
||||
title = item.title;
|
||||
header = item.title;
|
||||
}
|
||||
}
|
||||
}
|
||||
this.setState({title: title});
|
||||
this.setState({header: header});
|
||||
}
|
||||
|
||||
async fetchChannel() {
|
||||
|
|
@ -270,12 +332,27 @@ class Videos extends React.Component {
|
|||
this.fetchVideos);
|
||||
}
|
||||
|
||||
enabledFilters = () => {
|
||||
let filters = this.state.filters;
|
||||
let filtersArr = [];
|
||||
for (let i = 0; i < filters.length; i++) {
|
||||
let filter = filters[i];
|
||||
if (filter['value'] === true) {
|
||||
filtersArr = filtersArr.concat([filter['name']]);
|
||||
}
|
||||
}
|
||||
return filtersArr;
|
||||
}
|
||||
|
||||
async fetchVideos() {
|
||||
let offset = this.state.limit * this.state.activePage - this.state.limit;
|
||||
let channel_link = this.state.channel ? this.state.channel.link : null;
|
||||
let favorites = this.props.filter !== undefined ? this.props.filter === 'favorites' : null;
|
||||
let [videos, total] = await searchVideos(
|
||||
offset, this.state.limit, channel_link, this.state.queryStr, favorites, this.state.searchOrder);
|
||||
let {queryStr, searchOrder, limit} = this.state;
|
||||
|
||||
// Pass any enabled filters to the search.
|
||||
let filtersArr = this.enabledFilters();
|
||||
|
||||
let [videos, total] = await searchVideos(offset, limit, channel_link, queryStr, searchOrder, filtersArr);
|
||||
|
||||
let totalPages = Math.round(total / this.state.limit) || 1;
|
||||
this.setState({videos, totalPages});
|
||||
|
|
@ -309,6 +386,24 @@ class Videos extends React.Component {
|
|||
this.setState({searchOrder: value, activePage: 1}, this.applyStateToHistory);
|
||||
}
|
||||
|
||||
toggleFilter = (name) => {
|
||||
let filters = this.state.filters;
|
||||
for (let i = 0; i < filters.length; i++) {
|
||||
if (filters[i].name === name) {
|
||||
filters[i].value = !filters[i].value;
|
||||
}
|
||||
}
|
||||
this.setState({filters}, this.fetchVideos);
|
||||
}
|
||||
|
||||
closeFilters = () => {
|
||||
this.setState({filtersOpen: false});
|
||||
}
|
||||
|
||||
openFilters = () => {
|
||||
this.setState({filtersOpen: true});
|
||||
}
|
||||
|
||||
render() {
|
||||
let {
|
||||
activePage,
|
||||
|
|
@ -316,10 +411,11 @@ class Videos extends React.Component {
|
|||
queryStr,
|
||||
searchOrder,
|
||||
searchStr,
|
||||
title,
|
||||
header,
|
||||
totalPages,
|
||||
videoOrders,
|
||||
videos,
|
||||
filterEnabled,
|
||||
} = this.state;
|
||||
|
||||
let body = <VideoPlaceholder/>;
|
||||
|
|
@ -358,10 +454,10 @@ class Videos extends React.Component {
|
|||
|
||||
return (
|
||||
<Container textAlign='center'>
|
||||
<Grid columns={3} stackable>
|
||||
<Grid.Column textAlign='left'>
|
||||
<Grid columns={4} stackable>
|
||||
<Grid.Column textAlign='left' width={6}>
|
||||
<h1>
|
||||
{title}
|
||||
{header}
|
||||
{
|
||||
channel &&
|
||||
<>
|
||||
|
|
@ -375,7 +471,21 @@ class Videos extends React.Component {
|
|||
</h1>
|
||||
{queryStr && clearSearchButton}
|
||||
</Grid.Column>
|
||||
<Grid.Column textAlign='right'>
|
||||
<Grid.Column width={1}>
|
||||
<Button icon onClick={this.openFilters} color={filterEnabled ? 'black' : null}>
|
||||
<Icon name='filter'/>
|
||||
</Button>
|
||||
<FilterModal
|
||||
applyFilters={this.applyFilters}
|
||||
open={this.state.filtersOpen}
|
||||
close={this.closeFilters}
|
||||
header='Filter Videos'
|
||||
filters={this.state.filters}
|
||||
toggleFilter={this.toggleFilter}
|
||||
applyButton={null}
|
||||
/>
|
||||
</Grid.Column>
|
||||
<Grid.Column textAlign='right' width={5}>
|
||||
<Form onSubmit={this.handleSearch}>
|
||||
<Input
|
||||
fluid
|
||||
|
|
@ -386,7 +496,7 @@ class Videos extends React.Component {
|
|||
onChange={this.handleInputChange}/>
|
||||
</Form>
|
||||
</Grid.Column>
|
||||
<Grid.Column>
|
||||
<Grid.Column width={4}>
|
||||
<Dropdown
|
||||
size='large'
|
||||
placeholder='Sort by...'
|
||||
|
|
@ -443,7 +553,8 @@ export class VideosRoute extends React.Component {
|
|||
match={i.match}
|
||||
history={i.history}
|
||||
location={i.location}
|
||||
filter='favorites'
|
||||
filter='favorite'
|
||||
header='Favorite Videos'
|
||||
/>
|
||||
}/>
|
||||
<Route path='/videos/channel' exact component={Channels}/>
|
||||
|
|
|
|||
|
|
@ -11,13 +11,13 @@ from wrolpi.common import create_websocket_feed, get_sanic_url, \
|
|||
from wrolpi.common import logger
|
||||
from wrolpi.root_api import add_blueprint, json_response
|
||||
from wrolpi.schema import validate_doc, JSONErrorResponse
|
||||
from .channel import lib as channel_lib
|
||||
from .channel.api import channel_bp
|
||||
from .channel.lib import download_channel, spread_channel_downloads
|
||||
from .lib import _refresh_videos, get_statistics
|
||||
from .schema import StreamResponse, \
|
||||
FavoriteRequest, FavoriteResponse, VideosStatisticsResponse
|
||||
FavoriteRequest, FavoriteResponse, VideosStatisticsResponse, CensoredVideoResponse, CensoredVideoRequest
|
||||
from .video import lib as video_lib
|
||||
from .video.api import video_bp
|
||||
from .video.lib import set_video_favorite
|
||||
|
||||
content_bp = Blueprint('VideoContent', '/api/videos')
|
||||
bp = Blueprint('Videos', '/api/videos').group(
|
||||
|
|
@ -94,7 +94,7 @@ def download(_, link: str = None):
|
|||
if refresh_event.is_set():
|
||||
return response.json({'error': 'Refresh is running. Cannot download.'})
|
||||
|
||||
download_channel(link)
|
||||
channel_lib.download_channel(link)
|
||||
|
||||
download_logger.debug('do_download scheduled')
|
||||
return response.json({'code': 'stream-started', 'stream_url': stream_url})
|
||||
|
|
@ -115,7 +115,7 @@ async def refresh_videos(channel_links: list = None):
|
|||
]
|
||||
)
|
||||
async def favorite(_: Request, data: dict):
|
||||
_favorite = set_video_favorite(data['video_id'], data['favorite'])
|
||||
_favorite = video_lib.set_video_favorite(data['video_id'], data['favorite'])
|
||||
ret = {'video_id': data['video_id'], 'favorite': _favorite}
|
||||
return json_response(ret, HTTPStatus.OK)
|
||||
|
||||
|
|
@ -132,4 +132,20 @@ async def statistics(_: Request):
|
|||
|
||||
@before_startup
|
||||
def startup_spread_channel_downloads():
|
||||
spread_channel_downloads()
|
||||
channel_lib.spread_channel_downloads()
|
||||
|
||||
|
||||
@content_bp.post('/censored')
|
||||
@content_bp.post('/censored/<link:str>')
|
||||
@validate_doc(
|
||||
summary='Get all videos that were deleted on their source website',
|
||||
consumes=CensoredVideoRequest,
|
||||
produces=CensoredVideoResponse,
|
||||
)
|
||||
async def censored_videos(_: Request, data: dict, link: str = None):
|
||||
limit = data.get('limit', 20)
|
||||
offset = data.get('offset', 0)
|
||||
|
||||
videos = video_lib.censored_source_ids(link, limit, offset)
|
||||
ret = {'videos': videos}
|
||||
return json_response(ret)
|
||||
|
|
|
|||
|
|
@ -453,65 +453,6 @@ class TestVideoAPI(TestAPI):
|
|||
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
||||
last_ids = current_ids
|
||||
|
||||
@wrap_test_db
|
||||
def test_video_search(self):
|
||||
"""
|
||||
Test that videos can be searched and that their order is by their textsearch rank.
|
||||
"""
|
||||
# These captions have repeated letters so they will be higher in the ranking
|
||||
videos = [
|
||||
('1', 'b b b b e d d'),
|
||||
('2', '2 b b b d'),
|
||||
('3', 'b b'),
|
||||
('4', 'b e e'),
|
||||
('5', ''),
|
||||
]
|
||||
with get_db_session(commit=True) as session:
|
||||
for title, caption in videos:
|
||||
session.add(Video(title=title, caption=caption, video_path='foo'))
|
||||
|
||||
def do_search(search_str, limit=20):
|
||||
d = json.dumps({'search_str': search_str, 'limit': limit})
|
||||
_, resp = api_app.test_client.post('/api/videos/search', content=d)
|
||||
return resp
|
||||
|
||||
def search_is_as_expected(resp, expected):
|
||||
assert resp.status_code == HTTPStatus.OK
|
||||
response_ids = [i['id'] for i in resp.json['videos']]
|
||||
assert response_ids == expected
|
||||
assert resp.json['totals']['videos'] == len(expected)
|
||||
|
||||
# Repeated runs should return the same result
|
||||
for _ in range(2):
|
||||
# Only videos with a b are returned, ordered by the amount of b's
|
||||
response = do_search('b')
|
||||
search_is_as_expected(response, [1, 2, 3, 4])
|
||||
|
||||
# Only two captions have e
|
||||
response = do_search('e')
|
||||
search_is_as_expected(response, [4, 1])
|
||||
|
||||
# Only two captions have d
|
||||
response = do_search('d')
|
||||
search_is_as_expected(response, [1, 2])
|
||||
|
||||
# 5 can be gotten by it's title
|
||||
response = do_search('5')
|
||||
search_is_as_expected(response, [5])
|
||||
|
||||
# only video 1 has e and d
|
||||
response = do_search('e d')
|
||||
search_is_as_expected(response, [1])
|
||||
|
||||
# video 1 and 4 have b and e, but 1 has more
|
||||
response = do_search('b e')
|
||||
search_is_as_expected(response, [1, 4])
|
||||
|
||||
# Check totals are correct even with a limit
|
||||
response = do_search('b', 2)
|
||||
assert [i['id'] for i in response.json['videos']] == [1, 2]
|
||||
assert response.json['totals']['videos'] == 4
|
||||
|
||||
@wrap_test_db
|
||||
def test_channel_no_download_frequency(self):
|
||||
"""
|
||||
|
|
@ -601,3 +542,72 @@ def test_delete_channel_delete_download(download_channel, test_session):
|
|||
|
||||
downloads = test_session.query(Download).all()
|
||||
assert not list(downloads)
|
||||
|
||||
|
||||
def test_video_search(test_session):
|
||||
"""
|
||||
Test that videos can be searched and that their order is by their textsearch rank.
|
||||
"""
|
||||
# These captions have repeated letters, so they will be higher in the ranking.
|
||||
videos = [
|
||||
('1', 'b b b b e d d'),
|
||||
('2', '2 b b b d'),
|
||||
('3', 'b b'),
|
||||
('4', 'b e e'),
|
||||
('5', ''),
|
||||
]
|
||||
for title, caption in videos:
|
||||
test_session.add(Video(title=title, caption=caption, video_path='foo'))
|
||||
test_session.commit()
|
||||
|
||||
def do_search(search_str, limit=20):
|
||||
d = json.dumps({'search_str': search_str, 'limit': limit})
|
||||
_, resp = api_app.test_client.post('/api/videos/search', content=d)
|
||||
return resp
|
||||
|
||||
def search_is_as_expected(resp, expected):
|
||||
assert resp.status_code == HTTPStatus.OK
|
||||
response_ids = [i['id'] for i in resp.json['videos']]
|
||||
assert response_ids == expected
|
||||
assert resp.json['totals']['videos'] == len(expected)
|
||||
|
||||
# Repeated runs should return the same result
|
||||
for _ in range(2):
|
||||
# Only videos with a b are returned, ordered by the amount of b's
|
||||
response = do_search('b')
|
||||
search_is_as_expected(response, [1, 2, 3, 4])
|
||||
|
||||
# Only two captions have e
|
||||
response = do_search('e')
|
||||
search_is_as_expected(response, [4, 1])
|
||||
|
||||
# Only two captions have d
|
||||
response = do_search('d')
|
||||
search_is_as_expected(response, [1, 2])
|
||||
|
||||
# 5 can be gotten by it's title
|
||||
response = do_search('5')
|
||||
search_is_as_expected(response, [5])
|
||||
|
||||
# only video 1 has e and d
|
||||
response = do_search('e d')
|
||||
search_is_as_expected(response, [1])
|
||||
|
||||
# video 1 and 4 have b and e, but 1 has more
|
||||
response = do_search('b e')
|
||||
search_is_as_expected(response, [1, 4])
|
||||
|
||||
# Check totals are correct even with a limit
|
||||
response = do_search('b', 2)
|
||||
assert [i['id'] for i in response.json['videos']] == [1, 2]
|
||||
assert response.json['totals']['videos'] == 4
|
||||
|
||||
|
||||
def test_video_file_name(test_session, simple_video, test_client):
|
||||
"""
|
||||
If a Video has no title, the front-end can use the file name as the title.
|
||||
"""
|
||||
_, resp = test_client.get(f'/api/videos/video/{simple_video.id}')
|
||||
assert resp.status_code == HTTPStatus.OK
|
||||
assert resp.json['video']['video_path'] == 'simple_video.mp4'
|
||||
assert resp.json['video'].get('stem') == 'simple_video'
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import pytest
|
|||
|
||||
from modules.videos.channel.lib import spread_channel_downloads
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.common import sanitize_link
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
|
@ -27,6 +28,30 @@ def simple_channel(test_session, test_directory) -> Channel:
|
|||
return channel
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def channel_factory(test_session, test_directory):
|
||||
"""
|
||||
Create a random Channel with a directory, but no frequency.
|
||||
"""
|
||||
|
||||
def _():
|
||||
name = str(uuid4())
|
||||
directory = test_directory / name
|
||||
directory.mkdir()
|
||||
channel = Channel(
|
||||
directory=directory, # noqa
|
||||
name=name,
|
||||
url=f'https://example.com/{name}',
|
||||
download_frequency=None, # noqa
|
||||
link=sanitize_link(name),
|
||||
)
|
||||
test_session.add(channel)
|
||||
test_session.commit()
|
||||
return channel
|
||||
|
||||
return _
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def download_channel(test_session, test_directory) -> Channel:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -185,6 +185,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
config = load_downloader_config()
|
||||
options = config.copy()
|
||||
options['outtmpl'] = f'{out_dir}/{config["file_name_format"]}'
|
||||
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
|
||||
# We don't need format when getting the filename.
|
||||
del options['format']
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,8 @@ class Video(ModelHelper, Base):
|
|||
COALESCE(caption, ''::text)))'''))
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Video id={self.id}, title={self.title}, path={self.video_path.path}, channel={self.channel_id} ' \
|
||||
video_path = self.video_path.path if self.video_path else None
|
||||
return f'<Video id={self.id} title={self.title} path={video_path} channel={self.channel_id} ' \
|
||||
f'source_id={self.source_id}>'
|
||||
|
||||
def dict(self) -> dict:
|
||||
|
|
@ -210,6 +211,7 @@ class Video(ModelHelper, Base):
|
|||
def __json__(self):
|
||||
from modules.videos.common import minimize_video_info_json
|
||||
info_json = minimize_video_info_json(self.get_info_json()) if self.info_json_path else None
|
||||
stem = self.video_path.path.stem
|
||||
|
||||
d = dict(
|
||||
caption_path=self.caption_path,
|
||||
|
|
@ -222,6 +224,7 @@ class Video(ModelHelper, Base):
|
|||
poster_path=self.poster_path,
|
||||
size=self.size,
|
||||
source_id=self.source_id,
|
||||
stem=stem,
|
||||
title=self.title,
|
||||
upload_date=self.upload_date,
|
||||
url=self.url,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
from sanic_openapi import doc
|
||||
|
||||
from wrolpi.schema import Trinary
|
||||
|
||||
|
||||
class ChannelPostRequest:
|
||||
url = doc.String()
|
||||
|
|
@ -118,7 +116,7 @@ class VideoSearchRequest:
|
|||
order_by = doc.String()
|
||||
offset = doc.Integer()
|
||||
limit = doc.Integer()
|
||||
favorites = Trinary()
|
||||
filters = doc.List(doc.String())
|
||||
|
||||
|
||||
class VideoSearchResponse:
|
||||
|
|
@ -170,3 +168,12 @@ class ChannelStatistics:
|
|||
class VideosStatisticsResponse:
|
||||
videos = VideoStatistics
|
||||
channels = ChannelStatistics
|
||||
|
||||
|
||||
class CensoredVideoRequest:
|
||||
limit = doc.Integer()
|
||||
offset = doc.Integer()
|
||||
|
||||
|
||||
class CensoredVideoResponse:
|
||||
videos = doc.List(VideoWithChannel)
|
||||
|
|
|
|||
|
|
@ -2,11 +2,14 @@ from datetime import datetime
|
|||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import sqlalchemy
|
||||
|
||||
from modules.videos.channel import lib
|
||||
from modules.videos.models import Channel
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.video.lib import _censored_source_ids, video_search
|
||||
from wrolpi.dates import local_timezone
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.errors import UnknownChannel
|
||||
|
||||
daily, weekly, days30 = DownloadFrequency.daily, DownloadFrequency.weekly, DownloadFrequency.days30
|
||||
|
||||
|
|
@ -72,3 +75,99 @@ def test_spread_by_frequency(channels, expected):
|
|||
assert result['url'] == expected_url
|
||||
assert result['frequency'] == expected_frequency
|
||||
assert str(result['next_download']) == expected_dt
|
||||
|
||||
|
||||
def test_censored_videos(test_session, channel_factory):
|
||||
"""
|
||||
Censored videos can be found by comparing the videos we have versus what is listed in the Channel's catalog.
|
||||
"""
|
||||
channel1 = channel_factory()
|
||||
channel2 = channel_factory()
|
||||
|
||||
channel1.info_json = {'entries': [
|
||||
{'id': 'foo'},
|
||||
{'id': 'bar'},
|
||||
]}
|
||||
channel2.info_json = {'entries': [
|
||||
{'id': 'qux'},
|
||||
{'id': 'quux'},
|
||||
{'id': 'quuz'},
|
||||
]}
|
||||
vid1 = Video(source_id='foo', channel=channel1)
|
||||
vid2 = Video(source_id='bar', channel=channel1)
|
||||
vid3 = Video(source_id='baz') # Without a channel, this will never be returned.
|
||||
vid4 = Video(source_id='qux', channel=channel2)
|
||||
vid5 = Video(source_id='quux', channel=channel2)
|
||||
vid6 = Video(source_id='quuz', channel=channel2)
|
||||
test_session.add_all([vid1, vid2, vid3, vid4, vid5, vid6])
|
||||
test_session.commit()
|
||||
|
||||
def set_entries(channel1_entries, channel2_entries):
|
||||
channel1.info_json = {'entries': [{'id': i} for i in channel1_entries]} if channel1_entries else \
|
||||
sqlalchemy.null()
|
||||
channel2.info_json = {'entries': [{'id': i} for i in channel2_entries]} if channel2_entries else \
|
||||
sqlalchemy.null()
|
||||
test_session.commit()
|
||||
|
||||
# All source_id's are in "entries".
|
||||
set_entries(['foo', 'bar'], ['qux', 'quux', 'quuz'])
|
||||
assert _censored_source_ids(channel1.link) == set()
|
||||
|
||||
# "foo" is censored.
|
||||
set_entries(['bar'], ['qux', 'quux', 'quuz'])
|
||||
assert _censored_source_ids(channel1.link) == {'foo'}
|
||||
assert _censored_source_ids() == {'foo'}
|
||||
|
||||
# channel1 has no info_json.
|
||||
set_entries(None, ['qux', 'quux', 'quuz'])
|
||||
assert _censored_source_ids(channel1.link) == set()
|
||||
|
||||
# Censor channel2 and channel1.
|
||||
set_entries(['bar'], ['quuz'])
|
||||
assert _censored_source_ids() == {'foo', 'qux', 'quux'}
|
||||
assert _censored_source_ids(channel1.link) == {'foo', }
|
||||
assert _censored_source_ids(channel2.link) == {'qux', 'quux'}
|
||||
|
||||
# channel 1 has no info_json
|
||||
set_entries(None, ['quuz'])
|
||||
assert _censored_source_ids() == {'qux', 'quux'}
|
||||
assert _censored_source_ids(channel1.link) == set()
|
||||
assert _censored_source_ids(channel2.link) == {'qux', 'quux'}
|
||||
|
||||
# channel 2 has no entries
|
||||
set_entries(None, [])
|
||||
assert _censored_source_ids() == set()
|
||||
assert _censored_source_ids(channel2.link) == set()
|
||||
|
||||
|
||||
def test_search_censored_videos(test_session, simple_channel):
|
||||
for i in map(str, range(50)):
|
||||
test_session.add(Video(source_id=i, channel=simple_channel, video_path='foo'))
|
||||
test_session.commit()
|
||||
|
||||
def set_entries(entries):
|
||||
simple_channel.info_json = {'entries': [{'id': j} for j in entries]} if entries else sqlalchemy.null()
|
||||
test_session.commit()
|
||||
|
||||
# All source_ids are in the entries.
|
||||
set_entries(map(str, range(50)))
|
||||
videos, total = video_search(filters=['censored'])
|
||||
assert [i['source_id'] for i in videos] == []
|
||||
assert total == 0
|
||||
|
||||
# First 5 are censored.
|
||||
set_entries(map(str, range(5, 50)))
|
||||
videos, total = video_search(filters=['censored'])
|
||||
assert [i['source_id'] for i in videos] == [str(i) for i in range(5)]
|
||||
assert total == 5
|
||||
|
||||
# First 25 are censored.
|
||||
set_entries(map(str, range(25, 50)))
|
||||
videos, total = video_search(filters=['censored'])
|
||||
assert [i['source_id'] for i in videos] == [str(i) for i in range(20)]
|
||||
assert total == 25
|
||||
|
||||
|
||||
def test_censored_videos_no_channel(test_session):
|
||||
with pytest.raises(UnknownChannel):
|
||||
_censored_source_ids('foo')
|
||||
|
|
|
|||
|
|
@ -46,14 +46,14 @@ async def search(_: Request, data: dict):
|
|||
order_by = data.get('order_by', DEFAULT_VIDEO_ORDER)
|
||||
offset = int(data.get('offset', 0))
|
||||
limit = get_allowed_limit(data.get('limit'))
|
||||
favorites = data.get('favorites', None)
|
||||
filters = data.get('filters', None)
|
||||
except Exception as e:
|
||||
raise ValidationError('Unable to validate search queries') from e
|
||||
|
||||
if order_by not in VIDEO_ORDERS:
|
||||
raise InvalidOrderBy('Invalid order by')
|
||||
|
||||
videos, videos_total = video_search(search_str, offset, limit, channel_link, order_by, favorites)
|
||||
videos, videos_total = video_search(search_str, offset, limit, channel_link, order_by, filters)
|
||||
|
||||
ret = {'videos': list(videos), 'totals': {'videos': videos_total}}
|
||||
return json_response(ret)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
from datetime import datetime
|
||||
from typing import Tuple, Optional, List
|
||||
from typing import Tuple, Optional, List, Set
|
||||
|
||||
import cachetools
|
||||
import psycopg2
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from wrolpi.common import run_after, logger
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.errors import UnknownVideo
|
||||
from wrolpi.errors import UnknownVideo, UnknownChannel
|
||||
from wrolpi.vars import PYTEST
|
||||
from ..lib import save_channels_config
|
||||
from ..models import Video
|
||||
|
||||
|
|
@ -76,31 +78,41 @@ def video_search(
|
|||
limit: int = VIDEO_QUERY_LIMIT,
|
||||
channel_link: str = None,
|
||||
order_by: str = None,
|
||||
favorites: bool = None,
|
||||
filters: List[str] = None,
|
||||
) -> Tuple[List[dict], int]:
|
||||
with get_db_curs() as curs:
|
||||
args = dict(search_str=search_str, offset=offset)
|
||||
params = dict(search_str=search_str, offset=offset)
|
||||
channel_where = ''
|
||||
if channel_link:
|
||||
channel_where = 'AND channel_id = (select id from channel where link=%(channel_link)s)'
|
||||
args['channel_link'] = channel_link
|
||||
params['channel_link'] = channel_link
|
||||
|
||||
# Filter for/against favorites, if it was provided
|
||||
favorites_where = ''
|
||||
if favorites is not None:
|
||||
favorites_where = f'AND favorite IS {"NOT" if favorites else ""} NULL'
|
||||
if isinstance(filters, list) and 'favorite' in filters:
|
||||
favorites_where = 'AND favorite IS NOT NULL'
|
||||
|
||||
censored_where = ''
|
||||
if isinstance(filters, list) and 'censored' in filters:
|
||||
if PYTEST:
|
||||
# Bypass cache for testing only.
|
||||
params['censored_source_ids'] = list(_censored_source_ids(channel_link))
|
||||
else:
|
||||
params['censored_source_ids'] = list(censored_source_ids(channel_link))
|
||||
censored_where = 'AND source_id = ANY(%(censored_source_ids)s)'
|
||||
|
||||
where = ''
|
||||
if search_str:
|
||||
# A search_str was provided by the user, modify the query to filter by it.
|
||||
columns = 'id, ts_rank_cd(textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
|
||||
where = 'AND textsearch @@ websearch_to_tsquery(%(search_str)s)'
|
||||
args['search_str'] = search_str
|
||||
params['search_str'] = search_str
|
||||
else:
|
||||
# No search_str provided. Get id and total only.
|
||||
columns = 'id, COUNT(*) OVER() AS total'
|
||||
|
||||
# Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the whitelist.
|
||||
# Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the
|
||||
# whitelist.
|
||||
order = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER]
|
||||
if order_by:
|
||||
try:
|
||||
|
|
@ -119,12 +131,13 @@ def video_search(
|
|||
{where}
|
||||
{channel_where}
|
||||
{favorites_where}
|
||||
{censored_where}
|
||||
ORDER BY {order}
|
||||
OFFSET %(offset)s LIMIT {int(limit)}
|
||||
'''.strip()
|
||||
logger.debug(query)
|
||||
|
||||
curs.execute(query, args)
|
||||
curs.execute(query, params)
|
||||
try:
|
||||
results = [dict(i) for i in curs.fetchall()]
|
||||
except psycopg2.ProgrammingError:
|
||||
|
|
@ -154,3 +167,47 @@ def set_video_favorite(video_id: int, favorite: bool) -> Optional[datetime]:
|
|||
favorite = video.set_favorite(favorite)
|
||||
|
||||
return favorite
|
||||
|
||||
|
||||
def _censored_source_ids(link: str = None) -> Set[str]:
|
||||
with get_db_curs() as curs:
|
||||
if link:
|
||||
curs.execute('SELECT id FROM channel WHERE link=%s', (link,))
|
||||
try:
|
||||
id_ = curs.fetchone()[0]
|
||||
except TypeError:
|
||||
raise UnknownChannel(f'No channel {link}')
|
||||
|
||||
curs.execute("SELECT info_json->'entries' FROM channel WHERE info_json IS NOT NULL AND link=%s", (link,))
|
||||
try:
|
||||
entries = curs.fetchone()[0]
|
||||
except TypeError:
|
||||
# Channel has no info_json entries, so we can't find censored.
|
||||
return set()
|
||||
|
||||
source_ids = {i['id'] for i in entries if i}
|
||||
curs.execute('SELECT source_id FROM video WHERE channel_id =%s', (id_,))
|
||||
our_source_ids = {i[0] for i in curs.fetchall()}
|
||||
else:
|
||||
curs.execute("SELECT info_json->'entries' FROM channel WHERE info_json IS NOT NULL")
|
||||
source_ids = {j['id'] for i in curs.fetchall() if i and i[0] for j in i[0]}
|
||||
stmt = '''
|
||||
SELECT v.source_id
|
||||
FROM video v
|
||||
WHERE v.channel_id IN (select id from channel c where info_json is not null)
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
result = curs.fetchall()
|
||||
our_source_ids = {i[0] for i in result}
|
||||
|
||||
censored_ids = our_source_ids - source_ids
|
||||
return censored_ids
|
||||
|
||||
|
||||
@cachetools.cached(cachetools.TTLCache(maxsize=100, ttl=600))
|
||||
def censored_source_ids(link: str = None) -> Set[str]:
|
||||
"""
|
||||
Get all Videos that are downloaded, but are not in its Channel's catalog. Videos without a channel will not be
|
||||
returned.
|
||||
"""
|
||||
return _censored_source_ids(link)
|
||||
|
|
|
|||
|
|
@ -519,7 +519,8 @@ class DownloadManager:
|
|||
downloads = self.get_once_downloads(session)
|
||||
one_month = now() - timedelta(days=30)
|
||||
for download in downloads:
|
||||
if download.status in self.FINISHED_STATUSES and download.last_successful_download < one_month:
|
||||
if download.status in self.FINISHED_STATUSES and download.last_successful_download and \
|
||||
download.last_successful_download < one_month:
|
||||
session.delete(download)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ def valid_regex(_: Request, data: dict):
|
|||
async def post_download(request: Request, data: dict):
|
||||
# URLs are provided in a textarea, lets split all lines.
|
||||
urls = [i.strip() for i in str(data['urls']).strip().splitlines()]
|
||||
download_manager.create_downloads(urls)
|
||||
download_manager.create_downloads(urls, reset_attempts=True)
|
||||
return response.empty()
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue