From 02116bf2bcd967f891551bf2e03a2fa8736738d8 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Thu, 23 Dec 2021 09:38:12 -0700 Subject: [PATCH] Feature/censored videos (#65) * Creating api endpoint to find censored videos. * Videos can be filtered by censored status. * Not deleting when there has not been a successful download. * Search now has a different param order. * Displaying correct header for videos page. * Reworking FilterModal to be more universal. * Using merge_output_format for preparing filename. * Resetting attempts when user requests a download be started. * Displaying video stem as title if no title is available. Signed-off-by: Roland --- app/src/api.js | 18 +-- app/src/components/Common.js | 2 +- app/src/components/Inventory.js | 8 +- app/src/components/Videos.js | 157 ++++++++++++++++++++---- modules/videos/api.py | 28 ++++- modules/videos/channel/test/test_api.py | 128 ++++++++++--------- modules/videos/conftest.py | 25 ++++ modules/videos/downloader.py | 1 + modules/videos/models.py | 5 +- modules/videos/schema.py | 13 +- modules/videos/test/test_lib.py | 101 ++++++++++++++- modules/videos/video/api.py | 4 +- modules/videos/video/lib.py | 77 ++++++++++-- wrolpi/downloader.py | 3 +- wrolpi/root_api.py | 2 +- 15 files changed, 451 insertions(+), 121 deletions(-) diff --git a/app/src/api.js b/app/src/api.js index 0c207a76..f261a88d 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -36,7 +36,7 @@ async function apiCall(url, method, body, ms = 10000) { if (response.status === 403 && (await response.json())['code'] === 17) { toast({ type: 'error', - title: 'WROL Mode is enabled!', + header: 'WROL Mode is enabled!', description: 'This functionality is not enabled while WROL Mode is enabled!', time: 5000, }); @@ -47,14 +47,14 @@ async function apiCall(url, method, body, ms = 10000) { if (e.message === 'promise timeout') { toast({ type: 'error', - title: 'Server failed to respond!', + header: 'Server failed to respond!', description: 'Timeout while waiting for server response. See server logs.', time: 5000, }); } else { toast({ type: 'error', - title: 'Error!', + header: 'Error!', description: 'See client logs', time: 5000, }); @@ -103,11 +103,11 @@ export async function getChannel(link) { return (await response.json())['channel']; } -export async function searchVideos(offset, limit, channel_link, searchStr, favorites, order_by) { +export async function searchVideos(offset, limit, channel_link, searchStr, order_by, filters) { // Build a search query to retrieve a list of videos from the API offset = offset || 0; limit = limit || DEFAULT_LIMIT; - let body = {offset, limit, favorites: favorites}; + let body = {offset, limit, filters}; if (searchStr) { body.search_str = searchStr; @@ -138,7 +138,7 @@ export async function deleteVideo(video_id) { if (data.code === 17) { toast({ type: 'warning', - title: 'WROL Mode Enabled', + header: 'WROL Mode Enabled', description: 'This cannot be done while WROL Mode is enabled.', time: 5000, }); @@ -147,7 +147,7 @@ export async function deleteVideo(video_id) { if (response.status !== 204) { toast({ type: 'error', - title: 'Unexpected server response', + header: 'Unexpected server response', description: 'Failed to delete video. See server logs.', time: 5000, }); @@ -223,7 +223,7 @@ export async function encryptOTP(otp, plaintext) { if (response.status !== 200) { toast({ type: 'error', - title: 'Error!', + header: 'Error!', description: 'Failed to encrypt OTP', time: 5000, }); @@ -238,7 +238,7 @@ export async function decryptOTP(otp, ciphertext) { if (response.status !== 200) { toast({ type: 'error', - title: 'Error!', + header: 'Error!', description: 'Failed to decrypt OTP', time: 5000, }); diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 925c75d7..8c2f368d 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -108,7 +108,7 @@ export function VideoCard({video}) { -

{video.title || video.video_path}

+

{video.title || video.stem || video.video_path}

 diff --git a/app/src/components/Inventory.js b/app/src/components/Inventory.js index 40852d06..b676cada 100644 --- a/app/src/components/Inventory.js +++ b/app/src/components/Inventory.js @@ -526,7 +526,7 @@ class EditInventory extends React.Component { } else { toast({ type: 'error', - title: 'Failed to save inventory!', + header: 'Failed to save inventory!', description: 'Server responded with an error.', time: 5000, }) @@ -542,7 +542,7 @@ class EditInventory extends React.Component { } else { toast({ type: 'error', - title: 'Failed to delete inventory!', + header: 'Failed to delete inventory!', description: 'Server responded with an error.', time: 5000, }) @@ -575,7 +575,7 @@ class NewInventory extends React.Component { } else { toast({ type: 'error', - title: 'Failed to save inventory!', + header: 'Failed to save inventory!', description: 'Server responded with an error.', time: 5000, }) @@ -968,7 +968,7 @@ class InventoryAddList extends React.Component { } else { toast({ type: 'error', - title: 'Failed to save item!', + header: 'Failed to save item!', description: 'Server responded with an error.', time: 5000, }) diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 282184ae..a6167b2e 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -15,7 +15,21 @@ import Paginator, { } from "./Common" import VideoPage from "./VideoPlayer"; import {download, getChannel, getStatistics, getVideo, refresh, searchVideos} from "../api"; -import {Button, Dropdown, Form, Grid, Header, Icon, Input, Loader, Segment, Statistic, Tab} from "semantic-ui-react"; +import { + Button, + Dropdown, + Form, + Grid, + Header, + Icon, + Input, + Loader, + Modal, + Radio, + Segment, + Statistic, + Tab +} from "semantic-ui-react"; import * as QueryString from 'query-string'; import Container from "semantic-ui-react/dist/commonjs/elements/Container"; import {Channels, EditChannel, NewChannel} from "./Channels"; @@ -160,7 +174,7 @@ export class FavoriteVideosPreview extends VideosPreview { async fetchVideos() { let [videos] = await searchVideos( - 0, 3, null, null, true, '-favorite'); + 0, 3, null, null, '-favorite', ['favorite']); this.setState({videos}); } @@ -175,12 +189,47 @@ export class ViewedVideosPreview extends VideosPreview { async fetchVideos() { let [videos] = await searchVideos( - 0, 3, null, null, null, '-viewed'); + 0, 3, null, null, '-viewed'); this.setState({videos}); } } +function FilterModal(props) { + /* + Expecting props.filters = [ + {label: 'Favorites', name: 'favorite', value: false}, + {label: 'Censored', name: 'censored', value: false}, + ]; + */ + let inputs = props.filters.map((i) => + + props.toggleFilter(i.name)} + /> + + ) + + return ( + + {props.header} + + +
+ {inputs} +
+ + + + {props.applyButton && } + + + + ) +} + class Videos extends React.Component { @@ -191,6 +240,11 @@ class Videos extends React.Component { let searchStr = query.q || ''; let searchOrder = query.o || defaultVideoOrder; + let filters = [ + {label: 'Favorites', name: 'favorite', value: this.props.filter === 'favorite'}, + {label: 'Censored', name: 'censored', value: this.props.filter === 'censored'}, + ]; + this.state = { channel: null, videos: null, @@ -203,7 +257,10 @@ class Videos extends React.Component { next: null, videoOrders: searchStr === '' ? videoOrders : searchOrders, searchOrder: searchOrder, - title: '', + header: '', + filtersOpen: false, + filters, + filterEnabled: false, }; } @@ -213,7 +270,7 @@ class Videos extends React.Component { } else { await this.fetchVideos(); } - this.setTitle(); + this.setHeader(); } async componentDidUpdate(prevProps, prevState, snapshot) { @@ -232,24 +289,29 @@ class Videos extends React.Component { this.applyStateToHistory(); await this.fetchVideos(); } + + let filterEnabled = this.enabledFilters().length > 0; + if (this.state.filterEnabled !== filterEnabled) { + this.state.filterEnabled = filterEnabled; + } } - setTitle() { - let title = ''; - if (this.props.filter === 'favorites') { - title = 'Favorite Videos'; + setHeader() { + let header = ''; + if (this.props.header) { + header = this.props.header; } else if (this.state.channel) { - title = this.state.channel.name; + header = this.state.channel.name; } else { - // Find the matching title from the search orders. + // Find the matching header from the search orders. for (let i = 0; i < this.state.videoOrders.length; i++) { let item = this.state.videoOrders[i]; if (item.value === this.state.searchOrder) { - title = item.title; + header = item.title; } } } - this.setState({title: title}); + this.setState({header: header}); } async fetchChannel() { @@ -270,12 +332,27 @@ class Videos extends React.Component { this.fetchVideos); } + enabledFilters = () => { + let filters = this.state.filters; + let filtersArr = []; + for (let i = 0; i < filters.length; i++) { + let filter = filters[i]; + if (filter['value'] === true) { + filtersArr = filtersArr.concat([filter['name']]); + } + } + return filtersArr; + } + async fetchVideos() { let offset = this.state.limit * this.state.activePage - this.state.limit; let channel_link = this.state.channel ? this.state.channel.link : null; - let favorites = this.props.filter !== undefined ? this.props.filter === 'favorites' : null; - let [videos, total] = await searchVideos( - offset, this.state.limit, channel_link, this.state.queryStr, favorites, this.state.searchOrder); + let {queryStr, searchOrder, limit} = this.state; + + // Pass any enabled filters to the search. + let filtersArr = this.enabledFilters(); + + let [videos, total] = await searchVideos(offset, limit, channel_link, queryStr, searchOrder, filtersArr); let totalPages = Math.round(total / this.state.limit) || 1; this.setState({videos, totalPages}); @@ -309,6 +386,24 @@ class Videos extends React.Component { this.setState({searchOrder: value, activePage: 1}, this.applyStateToHistory); } + toggleFilter = (name) => { + let filters = this.state.filters; + for (let i = 0; i < filters.length; i++) { + if (filters[i].name === name) { + filters[i].value = !filters[i].value; + } + } + this.setState({filters}, this.fetchVideos); + } + + closeFilters = () => { + this.setState({filtersOpen: false}); + } + + openFilters = () => { + this.setState({filtersOpen: true}); + } + render() { let { activePage, @@ -316,10 +411,11 @@ class Videos extends React.Component { queryStr, searchOrder, searchStr, - title, + header, totalPages, videoOrders, videos, + filterEnabled, } = this.state; let body = ; @@ -358,10 +454,10 @@ class Videos extends React.Component { return ( - - + +

- {title} + {header} { channel && <> @@ -375,7 +471,21 @@ class Videos extends React.Component {

{queryStr && clearSearchButton} - + + + + +
- + }/> diff --git a/modules/videos/api.py b/modules/videos/api.py index dd629c39..99cd65d6 100644 --- a/modules/videos/api.py +++ b/modules/videos/api.py @@ -11,13 +11,13 @@ from wrolpi.common import create_websocket_feed, get_sanic_url, \ from wrolpi.common import logger from wrolpi.root_api import add_blueprint, json_response from wrolpi.schema import validate_doc, JSONErrorResponse +from .channel import lib as channel_lib from .channel.api import channel_bp -from .channel.lib import download_channel, spread_channel_downloads from .lib import _refresh_videos, get_statistics from .schema import StreamResponse, \ - FavoriteRequest, FavoriteResponse, VideosStatisticsResponse + FavoriteRequest, FavoriteResponse, VideosStatisticsResponse, CensoredVideoResponse, CensoredVideoRequest +from .video import lib as video_lib from .video.api import video_bp -from .video.lib import set_video_favorite content_bp = Blueprint('VideoContent', '/api/videos') bp = Blueprint('Videos', '/api/videos').group( @@ -94,7 +94,7 @@ def download(_, link: str = None): if refresh_event.is_set(): return response.json({'error': 'Refresh is running. Cannot download.'}) - download_channel(link) + channel_lib.download_channel(link) download_logger.debug('do_download scheduled') return response.json({'code': 'stream-started', 'stream_url': stream_url}) @@ -115,7 +115,7 @@ async def refresh_videos(channel_links: list = None): ] ) async def favorite(_: Request, data: dict): - _favorite = set_video_favorite(data['video_id'], data['favorite']) + _favorite = video_lib.set_video_favorite(data['video_id'], data['favorite']) ret = {'video_id': data['video_id'], 'favorite': _favorite} return json_response(ret, HTTPStatus.OK) @@ -132,4 +132,20 @@ async def statistics(_: Request): @before_startup def startup_spread_channel_downloads(): - spread_channel_downloads() + channel_lib.spread_channel_downloads() + + +@content_bp.post('/censored') +@content_bp.post('/censored/') +@validate_doc( + summary='Get all videos that were deleted on their source website', + consumes=CensoredVideoRequest, + produces=CensoredVideoResponse, +) +async def censored_videos(_: Request, data: dict, link: str = None): + limit = data.get('limit', 20) + offset = data.get('offset', 0) + + videos = video_lib.censored_source_ids(link, limit, offset) + ret = {'videos': videos} + return json_response(ret) diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index 14beac8f..5fa5bd1d 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -453,65 +453,6 @@ class TestVideoAPI(TestAPI): assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}' last_ids = current_ids - @wrap_test_db - def test_video_search(self): - """ - Test that videos can be searched and that their order is by their textsearch rank. - """ - # These captions have repeated letters so they will be higher in the ranking - videos = [ - ('1', 'b b b b e d d'), - ('2', '2 b b b d'), - ('3', 'b b'), - ('4', 'b e e'), - ('5', ''), - ] - with get_db_session(commit=True) as session: - for title, caption in videos: - session.add(Video(title=title, caption=caption, video_path='foo')) - - def do_search(search_str, limit=20): - d = json.dumps({'search_str': search_str, 'limit': limit}) - _, resp = api_app.test_client.post('/api/videos/search', content=d) - return resp - - def search_is_as_expected(resp, expected): - assert resp.status_code == HTTPStatus.OK - response_ids = [i['id'] for i in resp.json['videos']] - assert response_ids == expected - assert resp.json['totals']['videos'] == len(expected) - - # Repeated runs should return the same result - for _ in range(2): - # Only videos with a b are returned, ordered by the amount of b's - response = do_search('b') - search_is_as_expected(response, [1, 2, 3, 4]) - - # Only two captions have e - response = do_search('e') - search_is_as_expected(response, [4, 1]) - - # Only two captions have d - response = do_search('d') - search_is_as_expected(response, [1, 2]) - - # 5 can be gotten by it's title - response = do_search('5') - search_is_as_expected(response, [5]) - - # only video 1 has e and d - response = do_search('e d') - search_is_as_expected(response, [1]) - - # video 1 and 4 have b and e, but 1 has more - response = do_search('b e') - search_is_as_expected(response, [1, 4]) - - # Check totals are correct even with a limit - response = do_search('b', 2) - assert [i['id'] for i in response.json['videos']] == [1, 2] - assert response.json['totals']['videos'] == 4 - @wrap_test_db def test_channel_no_download_frequency(self): """ @@ -601,3 +542,72 @@ def test_delete_channel_delete_download(download_channel, test_session): downloads = test_session.query(Download).all() assert not list(downloads) + + +def test_video_search(test_session): + """ + Test that videos can be searched and that their order is by their textsearch rank. + """ + # These captions have repeated letters, so they will be higher in the ranking. + videos = [ + ('1', 'b b b b e d d'), + ('2', '2 b b b d'), + ('3', 'b b'), + ('4', 'b e e'), + ('5', ''), + ] + for title, caption in videos: + test_session.add(Video(title=title, caption=caption, video_path='foo')) + test_session.commit() + + def do_search(search_str, limit=20): + d = json.dumps({'search_str': search_str, 'limit': limit}) + _, resp = api_app.test_client.post('/api/videos/search', content=d) + return resp + + def search_is_as_expected(resp, expected): + assert resp.status_code == HTTPStatus.OK + response_ids = [i['id'] for i in resp.json['videos']] + assert response_ids == expected + assert resp.json['totals']['videos'] == len(expected) + + # Repeated runs should return the same result + for _ in range(2): + # Only videos with a b are returned, ordered by the amount of b's + response = do_search('b') + search_is_as_expected(response, [1, 2, 3, 4]) + + # Only two captions have e + response = do_search('e') + search_is_as_expected(response, [4, 1]) + + # Only two captions have d + response = do_search('d') + search_is_as_expected(response, [1, 2]) + + # 5 can be gotten by it's title + response = do_search('5') + search_is_as_expected(response, [5]) + + # only video 1 has e and d + response = do_search('e d') + search_is_as_expected(response, [1]) + + # video 1 and 4 have b and e, but 1 has more + response = do_search('b e') + search_is_as_expected(response, [1, 4]) + + # Check totals are correct even with a limit + response = do_search('b', 2) + assert [i['id'] for i in response.json['videos']] == [1, 2] + assert response.json['totals']['videos'] == 4 + + +def test_video_file_name(test_session, simple_video, test_client): + """ + If a Video has no title, the front-end can use the file name as the title. + """ + _, resp = test_client.get(f'/api/videos/video/{simple_video.id}') + assert resp.status_code == HTTPStatus.OK + assert resp.json['video']['video_path'] == 'simple_video.mp4' + assert resp.json['video'].get('stem') == 'simple_video' diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index f3e3f93a..b1e830bc 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -6,6 +6,7 @@ import pytest from modules.videos.channel.lib import spread_channel_downloads from modules.videos.models import Channel, Video +from wrolpi.common import sanitize_link from wrolpi.downloader import DownloadFrequency from wrolpi.vars import PROJECT_DIR @@ -27,6 +28,30 @@ def simple_channel(test_session, test_directory) -> Channel: return channel +@pytest.fixture +def channel_factory(test_session, test_directory): + """ + Create a random Channel with a directory, but no frequency. + """ + + def _(): + name = str(uuid4()) + directory = test_directory / name + directory.mkdir() + channel = Channel( + directory=directory, # noqa + name=name, + url=f'https://example.com/{name}', + download_frequency=None, # noqa + link=sanitize_link(name), + ) + test_session.add(channel) + test_session.commit() + return channel + + return _ + + @pytest.fixture def download_channel(test_session, test_directory) -> Channel: """ diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 4663ff53..debbf25d 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -185,6 +185,7 @@ class VideoDownloader(Downloader, ABC): config = load_downloader_config() options = config.copy() options['outtmpl'] = f'{out_dir}/{config["file_name_format"]}' + options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION # We don't need format when getting the filename. del options['format'] diff --git a/modules/videos/models.py b/modules/videos/models.py index 482e93ed..fb5f675a 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -48,7 +48,8 @@ class Video(ModelHelper, Base): COALESCE(caption, ''::text)))''')) def __repr__(self): - return f'