mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 02:48:38 -04:00
Creating video.censored column so videos can be quickly filtered.
This commit is contained in:
Roland
parent
02116bf2bc
commit
944782fd7c
10 changed files with 150 additions and 157 deletions
|
|
@ -15,7 +15,7 @@ from .channel import lib as channel_lib
|
|||
from .channel.api import channel_bp
|
||||
from .lib import _refresh_videos, get_statistics
|
||||
from .schema import StreamResponse, \
|
||||
FavoriteRequest, FavoriteResponse, VideosStatisticsResponse, CensoredVideoResponse, CensoredVideoRequest
|
||||
FavoriteRequest, FavoriteResponse, VideosStatisticsResponse
|
||||
from .video import lib as video_lib
|
||||
from .video.api import video_bp
|
||||
|
||||
|
|
@ -133,19 +133,3 @@ async def statistics(_: Request):
|
|||
@before_startup
|
||||
def startup_spread_channel_downloads():
|
||||
channel_lib.spread_channel_downloads()
|
||||
|
||||
|
||||
@content_bp.post('/censored')
|
||||
@content_bp.post('/censored/<link:str>')
|
||||
@validate_doc(
|
||||
summary='Get all videos that were deleted on their source website',
|
||||
consumes=CensoredVideoRequest,
|
||||
produces=CensoredVideoResponse,
|
||||
)
|
||||
async def censored_videos(_: Request, data: dict, link: str = None):
|
||||
limit = data.get('limit', 20)
|
||||
offset = data.get('offset', 0)
|
||||
|
||||
videos = video_lib.censored_source_ids(link, limit, offset)
|
||||
ret = {'videos': videos}
|
||||
return json_response(ret)
|
||||
|
|
|
|||
|
|
@ -485,12 +485,14 @@ async def get_bulk_video_size(video_ids: List[int]):
|
|||
logger.info('Done getting video sizes')
|
||||
|
||||
|
||||
def update_view_count(channel_id: int):
|
||||
def apply_info_json(channel_id: int):
|
||||
"""
|
||||
Update view_count for all Videos in a channel using it's info_json file.
|
||||
Update view_count for all Videos in a channel using its info_json file. Mark any videos not in the info_json as
|
||||
"censored".
|
||||
"""
|
||||
with get_db_session() as session:
|
||||
channel = session.query(Channel).filter_by(id=channel_id).one()
|
||||
channel_name = channel.name
|
||||
info = channel.info_json
|
||||
|
||||
if not info:
|
||||
|
|
@ -502,6 +504,7 @@ def update_view_count(channel_id: int):
|
|||
view_counts_str = json.dumps(view_counts)
|
||||
|
||||
with get_db_curs(commit=True) as curs:
|
||||
# Update the view_count for each video.
|
||||
stmt = '''
|
||||
WITH source AS (select * from json_to_recordset(%s::json) as (id text, view_count int))
|
||||
UPDATE video
|
||||
|
|
@ -512,7 +515,15 @@ def update_view_count(channel_id: int):
|
|||
'''
|
||||
curs.execute(stmt, (view_counts_str, channel_id))
|
||||
count = len(curs.fetchall())
|
||||
logger.debug(f'Updated {count} view counts in DB.')
|
||||
logger.debug(f'Updated {count} view counts in DB for {channel_name}.')
|
||||
|
||||
# Mark any video not in the info_json entries as censored.
|
||||
source_ids = [i['id'] for i in info['entries']]
|
||||
stmt = '''
|
||||
UPDATE video SET censored=(source_id != ALL(%s))
|
||||
WHERE channel_id=%s
|
||||
'''
|
||||
curs.execute(stmt, (source_ids, channel_id))
|
||||
|
||||
|
||||
minimize_channel = partial(minimize_dict, keys=MINIMUM_CHANNEL_KEYS)
|
||||
|
|
|
|||
|
|
@ -93,3 +93,15 @@ def video_file(test_directory) -> pathlib.Path:
|
|||
yield destination
|
||||
|
||||
destination.unlink()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def video_factory(test_session, test_directory):
|
||||
def _(channel_id: int = None):
|
||||
title = str(uuid4())
|
||||
path = str(test_directory / f'{title}.mp4')
|
||||
video = Video(video_path=path, title=title, channel_id=channel_id, source_id=title)
|
||||
test_session.add(video)
|
||||
return video
|
||||
|
||||
return _
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from wrolpi.downloader import Downloader, Download
|
|||
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
from .common import load_downloader_config, update_view_count
|
||||
from .common import load_downloader_config, apply_info_json
|
||||
from .lib import upsert_video, _refresh_videos
|
||||
from .models import Video, Channel
|
||||
from .video_url_resolver import video_url_resolver
|
||||
|
|
@ -304,7 +304,7 @@ def update_channel(channel: Channel):
|
|||
json.dump(info, fh, indent=2)
|
||||
|
||||
# Update all view counts using the latest from the Channel's info_json.
|
||||
update_view_count(channel_id)
|
||||
apply_info_json(channel_id)
|
||||
|
||||
|
||||
def _find_all_missing_videos(link: str = None) -> List[Tuple]:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from wrolpi.vars import PYTEST
|
|||
from .captions import insert_bulk_captions, process_captions
|
||||
from .common import generate_bulk_posters, get_bulk_video_info_json, get_bulk_video_size, \
|
||||
generate_video_paths, remove_duplicate_video_paths, bulk_validate_posters, \
|
||||
update_view_count
|
||||
apply_info_json
|
||||
from .models import Channel, Video
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -174,7 +174,7 @@ def refresh_channel_videos(channel: Channel):
|
|||
|
||||
logger.info(f'{channel.name}: {len(new_videos)} new videos, {len(existing_paths)} already existed. ')
|
||||
|
||||
update_view_count(channel.id)
|
||||
apply_info_json(channel.id)
|
||||
|
||||
|
||||
def refresh_no_channel_videos():
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ class Video(ModelHelper, Base):
|
|||
viewed = Column(TZDateTime)
|
||||
view_count = Column(Integer)
|
||||
url = Column(String)
|
||||
censored = Column(Boolean, default=False)
|
||||
|
||||
textsearch = Column(tsvector, Computed('''to_tsvector('english'::regconfig,
|
||||
((COALESCE(title, ''::text) || ' '::text) ||
|
||||
COALESCE(caption, ''::text)))'''))
|
||||
|
|
@ -211,7 +213,7 @@ class Video(ModelHelper, Base):
|
|||
def __json__(self):
|
||||
from modules.videos.common import minimize_video_info_json
|
||||
info_json = minimize_video_info_json(self.get_info_json()) if self.info_json_path else None
|
||||
stem = self.video_path.path.stem
|
||||
stem = self.video_path.path.stem if self.video_path else None
|
||||
|
||||
d = dict(
|
||||
caption_path=self.caption_path,
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from wrolpi.db import get_db_session
|
|||
from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
from ..common import get_matching_directories, convert_image, bulk_validate_posters, remove_duplicate_video_paths, \
|
||||
update_view_count, get_video_duration, generate_video_poster, replace_extension, is_valid_poster
|
||||
apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster
|
||||
|
||||
|
||||
class TestCommon(TestAPI):
|
||||
|
|
@ -282,15 +282,15 @@ def test_update_view_count(tempdir: pathlib.Path):
|
|||
check_view_counts({'vid1.mp4': None, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None})
|
||||
|
||||
# Channel 1 is updated, the other channels are left alone.
|
||||
update_view_count(channel1.id)
|
||||
apply_info_json(channel1.id)
|
||||
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None})
|
||||
|
||||
# Channel 2 is updated, the other channels are left alone. The 'bad_id' video is ignored.
|
||||
update_view_count(channel2.id)
|
||||
apply_info_json(channel2.id)
|
||||
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': None, 'vid4.mp4': None})
|
||||
|
||||
# All videos are updated.
|
||||
update_view_count(channel3.id)
|
||||
apply_info_json(channel3.id)
|
||||
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14})
|
||||
|
||||
# An outdated view count will be overwritten.
|
||||
|
|
@ -298,7 +298,7 @@ def test_update_view_count(tempdir: pathlib.Path):
|
|||
vid = session.query(Video).filter_by(id=1).one()
|
||||
vid.view_count = 8
|
||||
check_view_counts({'vid1.mp4': 8, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14})
|
||||
update_view_count(channel1.id)
|
||||
apply_info_json(channel1.id)
|
||||
check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14})
|
||||
|
||||
|
||||
|
|
@ -311,3 +311,67 @@ def test_generate_video_poster(video_file):
|
|||
generate_video_poster(video_file)
|
||||
assert poster_path.is_file(), f'{poster_path} was not created!'
|
||||
assert poster_path.stat().st_size > 0
|
||||
|
||||
|
||||
def test_update_censored_videos(test_session, video_factory, simple_channel):
|
||||
vid1 = video_factory(channel_id=simple_channel.id)
|
||||
vid2 = video_factory(channel_id=simple_channel.id)
|
||||
vid3 = video_factory(channel_id=simple_channel.id)
|
||||
vid4 = video_factory() # should not be censored because it has no channel.
|
||||
test_session.commit()
|
||||
|
||||
def check_censored(expected):
|
||||
for video_id, censored in expected:
|
||||
video = test_session.query(Video).filter_by(id=video_id).one()
|
||||
assert video.censored == censored
|
||||
|
||||
# Videos are not censored by default.
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, False), (vid4.id, False)])
|
||||
|
||||
# All videos are in the info_json.
|
||||
simple_channel.info_json = {
|
||||
'entries': [
|
||||
dict(id=vid1.source_id, view_count=0),
|
||||
dict(id=vid2.source_id, view_count=0),
|
||||
dict(id=vid3.source_id, view_count=0),
|
||||
]
|
||||
}
|
||||
test_session.commit()
|
||||
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, False), (vid4.id, False)])
|
||||
|
||||
simple_channel.info_json = {
|
||||
'entries': [
|
||||
dict(id=vid1.source_id, view_count=0), # vid2 is missing
|
||||
dict(id=vid3.source_id, view_count=0),
|
||||
]
|
||||
}
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, False), (vid2.id, True), (vid3.id, False), (vid4.id, False)])
|
||||
|
||||
simple_channel.info_json = {
|
||||
'entries': [
|
||||
dict(id=vid1.source_id, view_count=0), # vid2 is back, vid3 is missing.
|
||||
dict(id=vid2.source_id, view_count=0),
|
||||
]
|
||||
}
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, True), (vid4.id, False)])
|
||||
|
||||
simple_channel.info_json = {
|
||||
'entries': [] # all videos gone
|
||||
}
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)])
|
||||
|
||||
# Channels without info json preserve their last censored.
|
||||
simple_channel.info_json = None
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)])
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ import pytest
|
|||
import sqlalchemy
|
||||
|
||||
from modules.videos.channel import lib
|
||||
from modules.videos.common import apply_info_json
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.video.lib import _censored_source_ids, video_search
|
||||
from modules.videos.video.lib import video_search
|
||||
from wrolpi.dates import local_timezone
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.errors import UnknownChannel
|
||||
|
||||
daily, weekly, days30 = DownloadFrequency.daily, DownloadFrequency.weekly, DownloadFrequency.days30
|
||||
|
||||
|
|
@ -77,97 +77,34 @@ def test_spread_by_frequency(channels, expected):
|
|||
assert str(result['next_download']) == expected_dt
|
||||
|
||||
|
||||
def test_censored_videos(test_session, channel_factory):
|
||||
"""
|
||||
Censored videos can be found by comparing the videos we have versus what is listed in the Channel's catalog.
|
||||
"""
|
||||
channel1 = channel_factory()
|
||||
channel2 = channel_factory()
|
||||
|
||||
channel1.info_json = {'entries': [
|
||||
{'id': 'foo'},
|
||||
{'id': 'bar'},
|
||||
]}
|
||||
channel2.info_json = {'entries': [
|
||||
{'id': 'qux'},
|
||||
{'id': 'quux'},
|
||||
{'id': 'quuz'},
|
||||
]}
|
||||
vid1 = Video(source_id='foo', channel=channel1)
|
||||
vid2 = Video(source_id='bar', channel=channel1)
|
||||
vid3 = Video(source_id='baz') # Without a channel, this will never be returned.
|
||||
vid4 = Video(source_id='qux', channel=channel2)
|
||||
vid5 = Video(source_id='quux', channel=channel2)
|
||||
vid6 = Video(source_id='quuz', channel=channel2)
|
||||
test_session.add_all([vid1, vid2, vid3, vid4, vid5, vid6])
|
||||
test_session.commit()
|
||||
|
||||
def set_entries(channel1_entries, channel2_entries):
|
||||
channel1.info_json = {'entries': [{'id': i} for i in channel1_entries]} if channel1_entries else \
|
||||
sqlalchemy.null()
|
||||
channel2.info_json = {'entries': [{'id': i} for i in channel2_entries]} if channel2_entries else \
|
||||
sqlalchemy.null()
|
||||
test_session.commit()
|
||||
|
||||
# All source_id's are in "entries".
|
||||
set_entries(['foo', 'bar'], ['qux', 'quux', 'quuz'])
|
||||
assert _censored_source_ids(channel1.link) == set()
|
||||
|
||||
# "foo" is censored.
|
||||
set_entries(['bar'], ['qux', 'quux', 'quuz'])
|
||||
assert _censored_source_ids(channel1.link) == {'foo'}
|
||||
assert _censored_source_ids() == {'foo'}
|
||||
|
||||
# channel1 has no info_json.
|
||||
set_entries(None, ['qux', 'quux', 'quuz'])
|
||||
assert _censored_source_ids(channel1.link) == set()
|
||||
|
||||
# Censor channel2 and channel1.
|
||||
set_entries(['bar'], ['quuz'])
|
||||
assert _censored_source_ids() == {'foo', 'qux', 'quux'}
|
||||
assert _censored_source_ids(channel1.link) == {'foo', }
|
||||
assert _censored_source_ids(channel2.link) == {'qux', 'quux'}
|
||||
|
||||
# channel 1 has no info_json
|
||||
set_entries(None, ['quuz'])
|
||||
assert _censored_source_ids() == {'qux', 'quux'}
|
||||
assert _censored_source_ids(channel1.link) == set()
|
||||
assert _censored_source_ids(channel2.link) == {'qux', 'quux'}
|
||||
|
||||
# channel 2 has no entries
|
||||
set_entries(None, [])
|
||||
assert _censored_source_ids() == set()
|
||||
assert _censored_source_ids(channel2.link) == set()
|
||||
|
||||
|
||||
def test_search_censored_videos(test_session, simple_channel):
|
||||
for i in map(str, range(50)):
|
||||
test_session.add(Video(source_id=i, channel=simple_channel, video_path='foo'))
|
||||
vid = Video(source_id='51', video_path='bar') # this should never be modified because it has no channel
|
||||
test_session.add(vid)
|
||||
test_session.commit()
|
||||
|
||||
def set_entries(entries):
|
||||
simple_channel.info_json = {'entries': [{'id': j} for j in entries]} if entries else sqlalchemy.null()
|
||||
simple_channel.info_json = {
|
||||
'entries': [{'id': j, 'view_count': 0} for j in entries]} if entries else sqlalchemy.null()
|
||||
test_session.commit()
|
||||
apply_info_json(simple_channel.id)
|
||||
test_session.commit()
|
||||
|
||||
# All source_ids are in the entries.
|
||||
set_entries(map(str, range(50)))
|
||||
videos, total = video_search(filters=['censored'])
|
||||
videos, total = video_search(filters=['censored'], order_by='id')
|
||||
assert [i['source_id'] for i in videos] == []
|
||||
assert total == 0
|
||||
|
||||
# First 5 are censored.
|
||||
set_entries(map(str, range(5, 50)))
|
||||
videos, total = video_search(filters=['censored'])
|
||||
videos, total = video_search(filters=['censored'], order_by='id')
|
||||
assert [i['source_id'] for i in videos] == [str(i) for i in range(5)]
|
||||
assert total == 5
|
||||
|
||||
# First 25 are censored.
|
||||
set_entries(map(str, range(25, 50)))
|
||||
videos, total = video_search(filters=['censored'])
|
||||
videos, total = video_search(filters=['censored'], order_by='id')
|
||||
assert [i['source_id'] for i in videos] == [str(i) for i in range(20)]
|
||||
assert total == 25
|
||||
|
||||
|
||||
def test_censored_videos_no_channel(test_session):
|
||||
with pytest.raises(UnknownChannel):
|
||||
_censored_source_ids('foo')
|
||||
|
|
|
|||
|
|
@ -1,14 +1,13 @@
|
|||
from datetime import datetime
|
||||
from typing import Tuple, Optional, List, Set
|
||||
from typing import Tuple, Optional, List
|
||||
|
||||
import cachetools
|
||||
import psycopg2
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from wrolpi.common import run_after, logger
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.errors import UnknownVideo, UnknownChannel
|
||||
from wrolpi.errors import UnknownVideo
|
||||
from wrolpi.vars import PYTEST
|
||||
from ..lib import save_channels_config
|
||||
from ..models import Video
|
||||
|
|
@ -94,12 +93,7 @@ def video_search(
|
|||
|
||||
censored_where = ''
|
||||
if isinstance(filters, list) and 'censored' in filters:
|
||||
if PYTEST:
|
||||
# Bypass cache for testing only.
|
||||
params['censored_source_ids'] = list(_censored_source_ids(channel_link))
|
||||
else:
|
||||
params['censored_source_ids'] = list(censored_source_ids(channel_link))
|
||||
censored_where = 'AND source_id = ANY(%(censored_source_ids)s)'
|
||||
censored_where = 'AND censored = true'
|
||||
|
||||
where = ''
|
||||
if search_str:
|
||||
|
|
@ -167,47 +161,3 @@ def set_video_favorite(video_id: int, favorite: bool) -> Optional[datetime]:
|
|||
favorite = video.set_favorite(favorite)
|
||||
|
||||
return favorite
|
||||
|
||||
|
||||
def _censored_source_ids(link: str = None) -> Set[str]:
|
||||
with get_db_curs() as curs:
|
||||
if link:
|
||||
curs.execute('SELECT id FROM channel WHERE link=%s', (link,))
|
||||
try:
|
||||
id_ = curs.fetchone()[0]
|
||||
except TypeError:
|
||||
raise UnknownChannel(f'No channel {link}')
|
||||
|
||||
curs.execute("SELECT info_json->'entries' FROM channel WHERE info_json IS NOT NULL AND link=%s", (link,))
|
||||
try:
|
||||
entries = curs.fetchone()[0]
|
||||
except TypeError:
|
||||
# Channel has no info_json entries, so we can't find censored.
|
||||
return set()
|
||||
|
||||
source_ids = {i['id'] for i in entries if i}
|
||||
curs.execute('SELECT source_id FROM video WHERE channel_id =%s', (id_,))
|
||||
our_source_ids = {i[0] for i in curs.fetchall()}
|
||||
else:
|
||||
curs.execute("SELECT info_json->'entries' FROM channel WHERE info_json IS NOT NULL")
|
||||
source_ids = {j['id'] for i in curs.fetchall() if i and i[0] for j in i[0]}
|
||||
stmt = '''
|
||||
SELECT v.source_id
|
||||
FROM video v
|
||||
WHERE v.channel_id IN (select id from channel c where info_json is not null)
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
result = curs.fetchall()
|
||||
our_source_ids = {i[0] for i in result}
|
||||
|
||||
censored_ids = our_source_ids - source_ids
|
||||
return censored_ids
|
||||
|
||||
|
||||
@cachetools.cached(cachetools.TTLCache(maxsize=100, ttl=600))
|
||||
def censored_source_ids(link: str = None) -> Set[str]:
|
||||
"""
|
||||
Get all Videos that are downloaded, but are not in its Channel's catalog. Videos without a channel will not be
|
||||
returned.
|
||||
"""
|
||||
return _censored_source_ids(link)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue