mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
Feature/censored videos (#65)
* Creating api endpoint to find censored videos. * Videos can be filtered by censored status. * Not deleting when there has not been a successful download. * Search now has a different param order. * Displaying correct header for videos page. * Reworking FilterModal to be more universal. * Using merge_output_format for preparing filename. * Resetting attempts when user requests a download be started. * Displaying video stem as title if no title is available. Signed-off-by: Roland <roland@learningselfreliance.com>
This commit is contained in:
parent
ea2bf8530b
commit
02116bf2bc
15 changed files with 451 additions and 121 deletions
|
|
@ -11,13 +11,13 @@ from wrolpi.common import create_websocket_feed, get_sanic_url, \
|
|||
from wrolpi.common import logger
|
||||
from wrolpi.root_api import add_blueprint, json_response
|
||||
from wrolpi.schema import validate_doc, JSONErrorResponse
|
||||
from .channel import lib as channel_lib
|
||||
from .channel.api import channel_bp
|
||||
from .channel.lib import download_channel, spread_channel_downloads
|
||||
from .lib import _refresh_videos, get_statistics
|
||||
from .schema import StreamResponse, \
|
||||
FavoriteRequest, FavoriteResponse, VideosStatisticsResponse
|
||||
FavoriteRequest, FavoriteResponse, VideosStatisticsResponse, CensoredVideoResponse, CensoredVideoRequest
|
||||
from .video import lib as video_lib
|
||||
from .video.api import video_bp
|
||||
from .video.lib import set_video_favorite
|
||||
|
||||
content_bp = Blueprint('VideoContent', '/api/videos')
|
||||
bp = Blueprint('Videos', '/api/videos').group(
|
||||
|
|
@ -94,7 +94,7 @@ def download(_, link: str = None):
|
|||
if refresh_event.is_set():
|
||||
return response.json({'error': 'Refresh is running. Cannot download.'})
|
||||
|
||||
download_channel(link)
|
||||
channel_lib.download_channel(link)
|
||||
|
||||
download_logger.debug('do_download scheduled')
|
||||
return response.json({'code': 'stream-started', 'stream_url': stream_url})
|
||||
|
|
@ -115,7 +115,7 @@ async def refresh_videos(channel_links: list = None):
|
|||
]
|
||||
)
|
||||
async def favorite(_: Request, data: dict):
|
||||
_favorite = set_video_favorite(data['video_id'], data['favorite'])
|
||||
_favorite = video_lib.set_video_favorite(data['video_id'], data['favorite'])
|
||||
ret = {'video_id': data['video_id'], 'favorite': _favorite}
|
||||
return json_response(ret, HTTPStatus.OK)
|
||||
|
||||
|
|
@ -132,4 +132,20 @@ async def statistics(_: Request):
|
|||
|
||||
@before_startup
|
||||
def startup_spread_channel_downloads():
|
||||
spread_channel_downloads()
|
||||
channel_lib.spread_channel_downloads()
|
||||
|
||||
|
||||
@content_bp.post('/censored')
|
||||
@content_bp.post('/censored/<link:str>')
|
||||
@validate_doc(
|
||||
summary='Get all videos that were deleted on their source website',
|
||||
consumes=CensoredVideoRequest,
|
||||
produces=CensoredVideoResponse,
|
||||
)
|
||||
async def censored_videos(_: Request, data: dict, link: str = None):
|
||||
limit = data.get('limit', 20)
|
||||
offset = data.get('offset', 0)
|
||||
|
||||
videos = video_lib.censored_source_ids(link, limit, offset)
|
||||
ret = {'videos': videos}
|
||||
return json_response(ret)
|
||||
|
|
|
|||
|
|
@ -453,65 +453,6 @@ class TestVideoAPI(TestAPI):
|
|||
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
||||
last_ids = current_ids
|
||||
|
||||
@wrap_test_db
|
||||
def test_video_search(self):
|
||||
"""
|
||||
Test that videos can be searched and that their order is by their textsearch rank.
|
||||
"""
|
||||
# These captions have repeated letters so they will be higher in the ranking
|
||||
videos = [
|
||||
('1', 'b b b b e d d'),
|
||||
('2', '2 b b b d'),
|
||||
('3', 'b b'),
|
||||
('4', 'b e e'),
|
||||
('5', ''),
|
||||
]
|
||||
with get_db_session(commit=True) as session:
|
||||
for title, caption in videos:
|
||||
session.add(Video(title=title, caption=caption, video_path='foo'))
|
||||
|
||||
def do_search(search_str, limit=20):
|
||||
d = json.dumps({'search_str': search_str, 'limit': limit})
|
||||
_, resp = api_app.test_client.post('/api/videos/search', content=d)
|
||||
return resp
|
||||
|
||||
def search_is_as_expected(resp, expected):
|
||||
assert resp.status_code == HTTPStatus.OK
|
||||
response_ids = [i['id'] for i in resp.json['videos']]
|
||||
assert response_ids == expected
|
||||
assert resp.json['totals']['videos'] == len(expected)
|
||||
|
||||
# Repeated runs should return the same result
|
||||
for _ in range(2):
|
||||
# Only videos with a b are returned, ordered by the amount of b's
|
||||
response = do_search('b')
|
||||
search_is_as_expected(response, [1, 2, 3, 4])
|
||||
|
||||
# Only two captions have e
|
||||
response = do_search('e')
|
||||
search_is_as_expected(response, [4, 1])
|
||||
|
||||
# Only two captions have d
|
||||
response = do_search('d')
|
||||
search_is_as_expected(response, [1, 2])
|
||||
|
||||
# 5 can be gotten by it's title
|
||||
response = do_search('5')
|
||||
search_is_as_expected(response, [5])
|
||||
|
||||
# only video 1 has e and d
|
||||
response = do_search('e d')
|
||||
search_is_as_expected(response, [1])
|
||||
|
||||
# video 1 and 4 have b and e, but 1 has more
|
||||
response = do_search('b e')
|
||||
search_is_as_expected(response, [1, 4])
|
||||
|
||||
# Check totals are correct even with a limit
|
||||
response = do_search('b', 2)
|
||||
assert [i['id'] for i in response.json['videos']] == [1, 2]
|
||||
assert response.json['totals']['videos'] == 4
|
||||
|
||||
@wrap_test_db
|
||||
def test_channel_no_download_frequency(self):
|
||||
"""
|
||||
|
|
@ -601,3 +542,72 @@ def test_delete_channel_delete_download(download_channel, test_session):
|
|||
|
||||
downloads = test_session.query(Download).all()
|
||||
assert not list(downloads)
|
||||
|
||||
|
||||
def test_video_search(test_session):
|
||||
"""
|
||||
Test that videos can be searched and that their order is by their textsearch rank.
|
||||
"""
|
||||
# These captions have repeated letters, so they will be higher in the ranking.
|
||||
videos = [
|
||||
('1', 'b b b b e d d'),
|
||||
('2', '2 b b b d'),
|
||||
('3', 'b b'),
|
||||
('4', 'b e e'),
|
||||
('5', ''),
|
||||
]
|
||||
for title, caption in videos:
|
||||
test_session.add(Video(title=title, caption=caption, video_path='foo'))
|
||||
test_session.commit()
|
||||
|
||||
def do_search(search_str, limit=20):
|
||||
d = json.dumps({'search_str': search_str, 'limit': limit})
|
||||
_, resp = api_app.test_client.post('/api/videos/search', content=d)
|
||||
return resp
|
||||
|
||||
def search_is_as_expected(resp, expected):
|
||||
assert resp.status_code == HTTPStatus.OK
|
||||
response_ids = [i['id'] for i in resp.json['videos']]
|
||||
assert response_ids == expected
|
||||
assert resp.json['totals']['videos'] == len(expected)
|
||||
|
||||
# Repeated runs should return the same result
|
||||
for _ in range(2):
|
||||
# Only videos with a b are returned, ordered by the amount of b's
|
||||
response = do_search('b')
|
||||
search_is_as_expected(response, [1, 2, 3, 4])
|
||||
|
||||
# Only two captions have e
|
||||
response = do_search('e')
|
||||
search_is_as_expected(response, [4, 1])
|
||||
|
||||
# Only two captions have d
|
||||
response = do_search('d')
|
||||
search_is_as_expected(response, [1, 2])
|
||||
|
||||
# 5 can be gotten by it's title
|
||||
response = do_search('5')
|
||||
search_is_as_expected(response, [5])
|
||||
|
||||
# only video 1 has e and d
|
||||
response = do_search('e d')
|
||||
search_is_as_expected(response, [1])
|
||||
|
||||
# video 1 and 4 have b and e, but 1 has more
|
||||
response = do_search('b e')
|
||||
search_is_as_expected(response, [1, 4])
|
||||
|
||||
# Check totals are correct even with a limit
|
||||
response = do_search('b', 2)
|
||||
assert [i['id'] for i in response.json['videos']] == [1, 2]
|
||||
assert response.json['totals']['videos'] == 4
|
||||
|
||||
|
||||
def test_video_file_name(test_session, simple_video, test_client):
|
||||
"""
|
||||
If a Video has no title, the front-end can use the file name as the title.
|
||||
"""
|
||||
_, resp = test_client.get(f'/api/videos/video/{simple_video.id}')
|
||||
assert resp.status_code == HTTPStatus.OK
|
||||
assert resp.json['video']['video_path'] == 'simple_video.mp4'
|
||||
assert resp.json['video'].get('stem') == 'simple_video'
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import pytest
|
|||
|
||||
from modules.videos.channel.lib import spread_channel_downloads
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.common import sanitize_link
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
|
@ -27,6 +28,30 @@ def simple_channel(test_session, test_directory) -> Channel:
|
|||
return channel
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def channel_factory(test_session, test_directory):
|
||||
"""
|
||||
Create a random Channel with a directory, but no frequency.
|
||||
"""
|
||||
|
||||
def _():
|
||||
name = str(uuid4())
|
||||
directory = test_directory / name
|
||||
directory.mkdir()
|
||||
channel = Channel(
|
||||
directory=directory, # noqa
|
||||
name=name,
|
||||
url=f'https://example.com/{name}',
|
||||
download_frequency=None, # noqa
|
||||
link=sanitize_link(name),
|
||||
)
|
||||
test_session.add(channel)
|
||||
test_session.commit()
|
||||
return channel
|
||||
|
||||
return _
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def download_channel(test_session, test_directory) -> Channel:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -185,6 +185,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
config = load_downloader_config()
|
||||
options = config.copy()
|
||||
options['outtmpl'] = f'{out_dir}/{config["file_name_format"]}'
|
||||
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
|
||||
# We don't need format when getting the filename.
|
||||
del options['format']
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,8 @@ class Video(ModelHelper, Base):
|
|||
COALESCE(caption, ''::text)))'''))
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Video id={self.id}, title={self.title}, path={self.video_path.path}, channel={self.channel_id} ' \
|
||||
video_path = self.video_path.path if self.video_path else None
|
||||
return f'<Video id={self.id} title={self.title} path={video_path} channel={self.channel_id} ' \
|
||||
f'source_id={self.source_id}>'
|
||||
|
||||
def dict(self) -> dict:
|
||||
|
|
@ -210,6 +211,7 @@ class Video(ModelHelper, Base):
|
|||
def __json__(self):
|
||||
from modules.videos.common import minimize_video_info_json
|
||||
info_json = minimize_video_info_json(self.get_info_json()) if self.info_json_path else None
|
||||
stem = self.video_path.path.stem
|
||||
|
||||
d = dict(
|
||||
caption_path=self.caption_path,
|
||||
|
|
@ -222,6 +224,7 @@ class Video(ModelHelper, Base):
|
|||
poster_path=self.poster_path,
|
||||
size=self.size,
|
||||
source_id=self.source_id,
|
||||
stem=stem,
|
||||
title=self.title,
|
||||
upload_date=self.upload_date,
|
||||
url=self.url,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
from sanic_openapi import doc
|
||||
|
||||
from wrolpi.schema import Trinary
|
||||
|
||||
|
||||
class ChannelPostRequest:
|
||||
url = doc.String()
|
||||
|
|
@ -118,7 +116,7 @@ class VideoSearchRequest:
|
|||
order_by = doc.String()
|
||||
offset = doc.Integer()
|
||||
limit = doc.Integer()
|
||||
favorites = Trinary()
|
||||
filters = doc.List(doc.String())
|
||||
|
||||
|
||||
class VideoSearchResponse:
|
||||
|
|
@ -170,3 +168,12 @@ class ChannelStatistics:
|
|||
class VideosStatisticsResponse:
|
||||
videos = VideoStatistics
|
||||
channels = ChannelStatistics
|
||||
|
||||
|
||||
class CensoredVideoRequest:
|
||||
limit = doc.Integer()
|
||||
offset = doc.Integer()
|
||||
|
||||
|
||||
class CensoredVideoResponse:
|
||||
videos = doc.List(VideoWithChannel)
|
||||
|
|
|
|||
|
|
@ -2,11 +2,14 @@ from datetime import datetime
|
|||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import sqlalchemy
|
||||
|
||||
from modules.videos.channel import lib
|
||||
from modules.videos.models import Channel
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.video.lib import _censored_source_ids, video_search
|
||||
from wrolpi.dates import local_timezone
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.errors import UnknownChannel
|
||||
|
||||
daily, weekly, days30 = DownloadFrequency.daily, DownloadFrequency.weekly, DownloadFrequency.days30
|
||||
|
||||
|
|
@ -72,3 +75,99 @@ def test_spread_by_frequency(channels, expected):
|
|||
assert result['url'] == expected_url
|
||||
assert result['frequency'] == expected_frequency
|
||||
assert str(result['next_download']) == expected_dt
|
||||
|
||||
|
||||
def test_censored_videos(test_session, channel_factory):
|
||||
"""
|
||||
Censored videos can be found by comparing the videos we have versus what is listed in the Channel's catalog.
|
||||
"""
|
||||
channel1 = channel_factory()
|
||||
channel2 = channel_factory()
|
||||
|
||||
channel1.info_json = {'entries': [
|
||||
{'id': 'foo'},
|
||||
{'id': 'bar'},
|
||||
]}
|
||||
channel2.info_json = {'entries': [
|
||||
{'id': 'qux'},
|
||||
{'id': 'quux'},
|
||||
{'id': 'quuz'},
|
||||
]}
|
||||
vid1 = Video(source_id='foo', channel=channel1)
|
||||
vid2 = Video(source_id='bar', channel=channel1)
|
||||
vid3 = Video(source_id='baz') # Without a channel, this will never be returned.
|
||||
vid4 = Video(source_id='qux', channel=channel2)
|
||||
vid5 = Video(source_id='quux', channel=channel2)
|
||||
vid6 = Video(source_id='quuz', channel=channel2)
|
||||
test_session.add_all([vid1, vid2, vid3, vid4, vid5, vid6])
|
||||
test_session.commit()
|
||||
|
||||
def set_entries(channel1_entries, channel2_entries):
|
||||
channel1.info_json = {'entries': [{'id': i} for i in channel1_entries]} if channel1_entries else \
|
||||
sqlalchemy.null()
|
||||
channel2.info_json = {'entries': [{'id': i} for i in channel2_entries]} if channel2_entries else \
|
||||
sqlalchemy.null()
|
||||
test_session.commit()
|
||||
|
||||
# All source_id's are in "entries".
|
||||
set_entries(['foo', 'bar'], ['qux', 'quux', 'quuz'])
|
||||
assert _censored_source_ids(channel1.link) == set()
|
||||
|
||||
# "foo" is censored.
|
||||
set_entries(['bar'], ['qux', 'quux', 'quuz'])
|
||||
assert _censored_source_ids(channel1.link) == {'foo'}
|
||||
assert _censored_source_ids() == {'foo'}
|
||||
|
||||
# channel1 has no info_json.
|
||||
set_entries(None, ['qux', 'quux', 'quuz'])
|
||||
assert _censored_source_ids(channel1.link) == set()
|
||||
|
||||
# Censor channel2 and channel1.
|
||||
set_entries(['bar'], ['quuz'])
|
||||
assert _censored_source_ids() == {'foo', 'qux', 'quux'}
|
||||
assert _censored_source_ids(channel1.link) == {'foo', }
|
||||
assert _censored_source_ids(channel2.link) == {'qux', 'quux'}
|
||||
|
||||
# channel 1 has no info_json
|
||||
set_entries(None, ['quuz'])
|
||||
assert _censored_source_ids() == {'qux', 'quux'}
|
||||
assert _censored_source_ids(channel1.link) == set()
|
||||
assert _censored_source_ids(channel2.link) == {'qux', 'quux'}
|
||||
|
||||
# channel 2 has no entries
|
||||
set_entries(None, [])
|
||||
assert _censored_source_ids() == set()
|
||||
assert _censored_source_ids(channel2.link) == set()
|
||||
|
||||
|
||||
def test_search_censored_videos(test_session, simple_channel):
|
||||
for i in map(str, range(50)):
|
||||
test_session.add(Video(source_id=i, channel=simple_channel, video_path='foo'))
|
||||
test_session.commit()
|
||||
|
||||
def set_entries(entries):
|
||||
simple_channel.info_json = {'entries': [{'id': j} for j in entries]} if entries else sqlalchemy.null()
|
||||
test_session.commit()
|
||||
|
||||
# All source_ids are in the entries.
|
||||
set_entries(map(str, range(50)))
|
||||
videos, total = video_search(filters=['censored'])
|
||||
assert [i['source_id'] for i in videos] == []
|
||||
assert total == 0
|
||||
|
||||
# First 5 are censored.
|
||||
set_entries(map(str, range(5, 50)))
|
||||
videos, total = video_search(filters=['censored'])
|
||||
assert [i['source_id'] for i in videos] == [str(i) for i in range(5)]
|
||||
assert total == 5
|
||||
|
||||
# First 25 are censored.
|
||||
set_entries(map(str, range(25, 50)))
|
||||
videos, total = video_search(filters=['censored'])
|
||||
assert [i['source_id'] for i in videos] == [str(i) for i in range(20)]
|
||||
assert total == 25
|
||||
|
||||
|
||||
def test_censored_videos_no_channel(test_session):
|
||||
with pytest.raises(UnknownChannel):
|
||||
_censored_source_ids('foo')
|
||||
|
|
|
|||
|
|
@ -46,14 +46,14 @@ async def search(_: Request, data: dict):
|
|||
order_by = data.get('order_by', DEFAULT_VIDEO_ORDER)
|
||||
offset = int(data.get('offset', 0))
|
||||
limit = get_allowed_limit(data.get('limit'))
|
||||
favorites = data.get('favorites', None)
|
||||
filters = data.get('filters', None)
|
||||
except Exception as e:
|
||||
raise ValidationError('Unable to validate search queries') from e
|
||||
|
||||
if order_by not in VIDEO_ORDERS:
|
||||
raise InvalidOrderBy('Invalid order by')
|
||||
|
||||
videos, videos_total = video_search(search_str, offset, limit, channel_link, order_by, favorites)
|
||||
videos, videos_total = video_search(search_str, offset, limit, channel_link, order_by, filters)
|
||||
|
||||
ret = {'videos': list(videos), 'totals': {'videos': videos_total}}
|
||||
return json_response(ret)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
from datetime import datetime
|
||||
from typing import Tuple, Optional, List
|
||||
from typing import Tuple, Optional, List, Set
|
||||
|
||||
import cachetools
|
||||
import psycopg2
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from wrolpi.common import run_after, logger
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.errors import UnknownVideo
|
||||
from wrolpi.errors import UnknownVideo, UnknownChannel
|
||||
from wrolpi.vars import PYTEST
|
||||
from ..lib import save_channels_config
|
||||
from ..models import Video
|
||||
|
||||
|
|
@ -76,31 +78,41 @@ def video_search(
|
|||
limit: int = VIDEO_QUERY_LIMIT,
|
||||
channel_link: str = None,
|
||||
order_by: str = None,
|
||||
favorites: bool = None,
|
||||
filters: List[str] = None,
|
||||
) -> Tuple[List[dict], int]:
|
||||
with get_db_curs() as curs:
|
||||
args = dict(search_str=search_str, offset=offset)
|
||||
params = dict(search_str=search_str, offset=offset)
|
||||
channel_where = ''
|
||||
if channel_link:
|
||||
channel_where = 'AND channel_id = (select id from channel where link=%(channel_link)s)'
|
||||
args['channel_link'] = channel_link
|
||||
params['channel_link'] = channel_link
|
||||
|
||||
# Filter for/against favorites, if it was provided
|
||||
favorites_where = ''
|
||||
if favorites is not None:
|
||||
favorites_where = f'AND favorite IS {"NOT" if favorites else ""} NULL'
|
||||
if isinstance(filters, list) and 'favorite' in filters:
|
||||
favorites_where = 'AND favorite IS NOT NULL'
|
||||
|
||||
censored_where = ''
|
||||
if isinstance(filters, list) and 'censored' in filters:
|
||||
if PYTEST:
|
||||
# Bypass cache for testing only.
|
||||
params['censored_source_ids'] = list(_censored_source_ids(channel_link))
|
||||
else:
|
||||
params['censored_source_ids'] = list(censored_source_ids(channel_link))
|
||||
censored_where = 'AND source_id = ANY(%(censored_source_ids)s)'
|
||||
|
||||
where = ''
|
||||
if search_str:
|
||||
# A search_str was provided by the user, modify the query to filter by it.
|
||||
columns = 'id, ts_rank_cd(textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total'
|
||||
where = 'AND textsearch @@ websearch_to_tsquery(%(search_str)s)'
|
||||
args['search_str'] = search_str
|
||||
params['search_str'] = search_str
|
||||
else:
|
||||
# No search_str provided. Get id and total only.
|
||||
columns = 'id, COUNT(*) OVER() AS total'
|
||||
|
||||
# Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the whitelist.
|
||||
# Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the
|
||||
# whitelist.
|
||||
order = VIDEO_ORDERS[DEFAULT_VIDEO_ORDER]
|
||||
if order_by:
|
||||
try:
|
||||
|
|
@ -119,12 +131,13 @@ def video_search(
|
|||
{where}
|
||||
{channel_where}
|
||||
{favorites_where}
|
||||
{censored_where}
|
||||
ORDER BY {order}
|
||||
OFFSET %(offset)s LIMIT {int(limit)}
|
||||
'''.strip()
|
||||
logger.debug(query)
|
||||
|
||||
curs.execute(query, args)
|
||||
curs.execute(query, params)
|
||||
try:
|
||||
results = [dict(i) for i in curs.fetchall()]
|
||||
except psycopg2.ProgrammingError:
|
||||
|
|
@ -154,3 +167,47 @@ def set_video_favorite(video_id: int, favorite: bool) -> Optional[datetime]:
|
|||
favorite = video.set_favorite(favorite)
|
||||
|
||||
return favorite
|
||||
|
||||
|
||||
def _censored_source_ids(link: str = None) -> Set[str]:
|
||||
with get_db_curs() as curs:
|
||||
if link:
|
||||
curs.execute('SELECT id FROM channel WHERE link=%s', (link,))
|
||||
try:
|
||||
id_ = curs.fetchone()[0]
|
||||
except TypeError:
|
||||
raise UnknownChannel(f'No channel {link}')
|
||||
|
||||
curs.execute("SELECT info_json->'entries' FROM channel WHERE info_json IS NOT NULL AND link=%s", (link,))
|
||||
try:
|
||||
entries = curs.fetchone()[0]
|
||||
except TypeError:
|
||||
# Channel has no info_json entries, so we can't find censored.
|
||||
return set()
|
||||
|
||||
source_ids = {i['id'] for i in entries if i}
|
||||
curs.execute('SELECT source_id FROM video WHERE channel_id =%s', (id_,))
|
||||
our_source_ids = {i[0] for i in curs.fetchall()}
|
||||
else:
|
||||
curs.execute("SELECT info_json->'entries' FROM channel WHERE info_json IS NOT NULL")
|
||||
source_ids = {j['id'] for i in curs.fetchall() if i and i[0] for j in i[0]}
|
||||
stmt = '''
|
||||
SELECT v.source_id
|
||||
FROM video v
|
||||
WHERE v.channel_id IN (select id from channel c where info_json is not null)
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
result = curs.fetchall()
|
||||
our_source_ids = {i[0] for i in result}
|
||||
|
||||
censored_ids = our_source_ids - source_ids
|
||||
return censored_ids
|
||||
|
||||
|
||||
@cachetools.cached(cachetools.TTLCache(maxsize=100, ttl=600))
|
||||
def censored_source_ids(link: str = None) -> Set[str]:
|
||||
"""
|
||||
Get all Videos that are downloaded, but are not in its Channel's catalog. Videos without a channel will not be
|
||||
returned.
|
||||
"""
|
||||
return _censored_source_ids(link)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue