From 944782fd7c8c1a2c6bf3c072c2e9cfa8d5c2d6ca Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 23 Dec 2021 14:09:31 -0700 Subject: [PATCH] Creating video.censored column so videos can be quickly filtered. --- .../versions/32d96eb9122d_video_censored.py | 33 +++++++ modules/videos/api.py | 18 +--- modules/videos/common.py | 17 +++- modules/videos/conftest.py | 12 +++ modules/videos/downloader.py | 4 +- modules/videos/lib.py | 4 +- modules/videos/models.py | 4 +- modules/videos/test/test_common.py | 74 ++++++++++++++-- modules/videos/test/test_lib.py | 85 +++---------------- modules/videos/video/lib.py | 56 +----------- 10 files changed, 150 insertions(+), 157 deletions(-) create mode 100644 alembic/versions/32d96eb9122d_video_censored.py diff --git a/alembic/versions/32d96eb9122d_video_censored.py b/alembic/versions/32d96eb9122d_video_censored.py new file mode 100644 index 00000000..9ce4355e --- /dev/null +++ b/alembic/versions/32d96eb9122d_video_censored.py @@ -0,0 +1,33 @@ +"""video censored + +Revision ID: 32d96eb9122d +Revises: 184329ed5137 +Create Date: 2021-12-23 14:03:53.677547 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '32d96eb9122d' +down_revision = '184329ed5137' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE video ADD COLUMN censored BOOL DEFAULT false') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS censored') diff --git a/modules/videos/api.py b/modules/videos/api.py index 99cd65d6..82af9f0b 100644 --- a/modules/videos/api.py +++ b/modules/videos/api.py @@ -15,7 +15,7 @@ from .channel import lib as channel_lib from .channel.api import channel_bp from .lib import _refresh_videos, get_statistics from .schema import StreamResponse, \ - FavoriteRequest, FavoriteResponse, VideosStatisticsResponse, CensoredVideoResponse, CensoredVideoRequest + FavoriteRequest, FavoriteResponse, VideosStatisticsResponse from .video import lib as video_lib from .video.api import video_bp @@ -133,19 +133,3 @@ async def statistics(_: Request): @before_startup def startup_spread_channel_downloads(): channel_lib.spread_channel_downloads() - - -@content_bp.post('/censored') -@content_bp.post('/censored/') -@validate_doc( - summary='Get all videos that were deleted on their source website', - consumes=CensoredVideoRequest, - produces=CensoredVideoResponse, -) -async def censored_videos(_: Request, data: dict, link: str = None): - limit = data.get('limit', 20) - offset = data.get('offset', 0) - - videos = video_lib.censored_source_ids(link, limit, offset) - ret = {'videos': videos} - return json_response(ret) diff --git a/modules/videos/common.py b/modules/videos/common.py index b01b3fe1..ca2ed038 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -485,12 +485,14 @@ async def get_bulk_video_size(video_ids: List[int]): logger.info('Done getting video sizes') -def update_view_count(channel_id: int): +def apply_info_json(channel_id: int): """ - Update view_count for all Videos in a channel using it's info_json file. + Update view_count for all Videos in a channel using its info_json file. Mark any videos not in the info_json as + "censored". """ with get_db_session() as session: channel = session.query(Channel).filter_by(id=channel_id).one() + channel_name = channel.name info = channel.info_json if not info: @@ -502,6 +504,7 @@ def update_view_count(channel_id: int): view_counts_str = json.dumps(view_counts) with get_db_curs(commit=True) as curs: + # Update the view_count for each video. stmt = ''' WITH source AS (select * from json_to_recordset(%s::json) as (id text, view_count int)) UPDATE video @@ -512,7 +515,15 @@ def update_view_count(channel_id: int): ''' curs.execute(stmt, (view_counts_str, channel_id)) count = len(curs.fetchall()) - logger.debug(f'Updated {count} view counts in DB.') + logger.debug(f'Updated {count} view counts in DB for {channel_name}.') + + # Mark any video not in the info_json entries as censored. + source_ids = [i['id'] for i in info['entries']] + stmt = ''' + UPDATE video SET censored=(source_id != ALL(%s)) + WHERE channel_id=%s + ''' + curs.execute(stmt, (source_ids, channel_id)) minimize_channel = partial(minimize_dict, keys=MINIMUM_CHANNEL_KEYS) diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index b1e830bc..230b6f7c 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -93,3 +93,15 @@ def video_file(test_directory) -> pathlib.Path: yield destination destination.unlink() + + +@pytest.fixture +def video_factory(test_session, test_directory): + def _(channel_id: int = None): + title = str(uuid4()) + path = str(test_directory / f'{title}.mp4') + video = Video(video_path=path, title=title, channel_id=channel_id, source_id=title) + test_session.add(video) + return video + + return _ diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index debbf25d..6cb07f76 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -18,7 +18,7 @@ from wrolpi.downloader import Downloader, Download from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel -from .common import load_downloader_config, update_view_count +from .common import load_downloader_config, apply_info_json from .lib import upsert_video, _refresh_videos from .models import Video, Channel from .video_url_resolver import video_url_resolver @@ -304,7 +304,7 @@ def update_channel(channel: Channel): json.dump(info, fh, indent=2) # Update all view counts using the latest from the Channel's info_json. - update_view_count(channel_id) + apply_info_json(channel_id) def _find_all_missing_videos(link: str = None) -> List[Tuple]: diff --git a/modules/videos/lib.py b/modules/videos/lib.py index d749644b..94d1484e 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -14,7 +14,7 @@ from wrolpi.vars import PYTEST from .captions import insert_bulk_captions, process_captions from .common import generate_bulk_posters, get_bulk_video_info_json, get_bulk_video_size, \ generate_video_paths, remove_duplicate_video_paths, bulk_validate_posters, \ - update_view_count + apply_info_json from .models import Channel, Video logger = logger.getChild(__name__) @@ -174,7 +174,7 @@ def refresh_channel_videos(channel: Channel): logger.info(f'{channel.name}: {len(new_videos)} new videos, {len(existing_paths)} already existed. ') - update_view_count(channel.id) + apply_info_json(channel.id) def refresh_no_channel_videos(): diff --git a/modules/videos/models.py b/modules/videos/models.py index fb5f675a..c7aa50cb 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -43,6 +43,8 @@ class Video(ModelHelper, Base): viewed = Column(TZDateTime) view_count = Column(Integer) url = Column(String) + censored = Column(Boolean, default=False) + textsearch = Column(tsvector, Computed('''to_tsvector('english'::regconfig, ((COALESCE(title, ''::text) || ' '::text) || COALESCE(caption, ''::text)))''')) @@ -211,7 +213,7 @@ class Video(ModelHelper, Base): def __json__(self): from modules.videos.common import minimize_video_info_json info_json = minimize_video_info_json(self.get_info_json()) if self.info_json_path else None - stem = self.video_path.path.stem + stem = self.video_path.path.stem if self.video_path else None d = dict( caption_path=self.caption_path, diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index eae56238..17633763 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -14,7 +14,7 @@ from wrolpi.db import get_db_session from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI from wrolpi.vars import PROJECT_DIR from ..common import get_matching_directories, convert_image, bulk_validate_posters, remove_duplicate_video_paths, \ - update_view_count, get_video_duration, generate_video_poster, replace_extension, is_valid_poster + apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster class TestCommon(TestAPI): @@ -282,15 +282,15 @@ def test_update_view_count(tempdir: pathlib.Path): check_view_counts({'vid1.mp4': None, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None}) # Channel 1 is updated, the other channels are left alone. - update_view_count(channel1.id) + apply_info_json(channel1.id) check_view_counts({'vid1.mp4': 10, 'vid2.mp4': None, 'vid3.mp4': None, 'vid4.mp4': None}) # Channel 2 is updated, the other channels are left alone. The 'bad_id' video is ignored. - update_view_count(channel2.id) + apply_info_json(channel2.id) check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': None, 'vid4.mp4': None}) # All videos are updated. - update_view_count(channel3.id) + apply_info_json(channel3.id) check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14}) # An outdated view count will be overwritten. @@ -298,7 +298,7 @@ def test_update_view_count(tempdir: pathlib.Path): vid = session.query(Video).filter_by(id=1).one() vid.view_count = 8 check_view_counts({'vid1.mp4': 8, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14}) - update_view_count(channel1.id) + apply_info_json(channel1.id) check_view_counts({'vid1.mp4': 10, 'vid2.mp4': 11, 'vid3.mp4': 13, 'vid4.mp4': 14}) @@ -311,3 +311,67 @@ def test_generate_video_poster(video_file): generate_video_poster(video_file) assert poster_path.is_file(), f'{poster_path} was not created!' assert poster_path.stat().st_size > 0 + + +def test_update_censored_videos(test_session, video_factory, simple_channel): + vid1 = video_factory(channel_id=simple_channel.id) + vid2 = video_factory(channel_id=simple_channel.id) + vid3 = video_factory(channel_id=simple_channel.id) + vid4 = video_factory() # should not be censored because it has no channel. + test_session.commit() + + def check_censored(expected): + for video_id, censored in expected: + video = test_session.query(Video).filter_by(id=video_id).one() + assert video.censored == censored + + # Videos are not censored by default. + test_session.commit() + apply_info_json(simple_channel.id) + check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, False), (vid4.id, False)]) + + # All videos are in the info_json. + simple_channel.info_json = { + 'entries': [ + dict(id=vid1.source_id, view_count=0), + dict(id=vid2.source_id, view_count=0), + dict(id=vid3.source_id, view_count=0), + ] + } + test_session.commit() + + apply_info_json(simple_channel.id) + check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, False), (vid4.id, False)]) + + simple_channel.info_json = { + 'entries': [ + dict(id=vid1.source_id, view_count=0), # vid2 is missing + dict(id=vid3.source_id, view_count=0), + ] + } + test_session.commit() + apply_info_json(simple_channel.id) + check_censored([(vid1.id, False), (vid2.id, True), (vid3.id, False), (vid4.id, False)]) + + simple_channel.info_json = { + 'entries': [ + dict(id=vid1.source_id, view_count=0), # vid2 is back, vid3 is missing. + dict(id=vid2.source_id, view_count=0), + ] + } + test_session.commit() + apply_info_json(simple_channel.id) + check_censored([(vid1.id, False), (vid2.id, False), (vid3.id, True), (vid4.id, False)]) + + simple_channel.info_json = { + 'entries': [] # all videos gone + } + test_session.commit() + apply_info_json(simple_channel.id) + check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)]) + + # Channels without info json preserve their last censored. + simple_channel.info_json = None + test_session.commit() + apply_info_json(simple_channel.id) + check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)]) diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 401c9507..6a7b9da6 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -5,11 +5,11 @@ import pytest import sqlalchemy from modules.videos.channel import lib +from modules.videos.common import apply_info_json from modules.videos.models import Channel, Video -from modules.videos.video.lib import _censored_source_ids, video_search +from modules.videos.video.lib import video_search from wrolpi.dates import local_timezone from wrolpi.downloader import DownloadFrequency -from wrolpi.errors import UnknownChannel daily, weekly, days30 = DownloadFrequency.daily, DownloadFrequency.weekly, DownloadFrequency.days30 @@ -77,97 +77,34 @@ def test_spread_by_frequency(channels, expected): assert str(result['next_download']) == expected_dt -def test_censored_videos(test_session, channel_factory): - """ - Censored videos can be found by comparing the videos we have versus what is listed in the Channel's catalog. - """ - channel1 = channel_factory() - channel2 = channel_factory() - - channel1.info_json = {'entries': [ - {'id': 'foo'}, - {'id': 'bar'}, - ]} - channel2.info_json = {'entries': [ - {'id': 'qux'}, - {'id': 'quux'}, - {'id': 'quuz'}, - ]} - vid1 = Video(source_id='foo', channel=channel1) - vid2 = Video(source_id='bar', channel=channel1) - vid3 = Video(source_id='baz') # Without a channel, this will never be returned. - vid4 = Video(source_id='qux', channel=channel2) - vid5 = Video(source_id='quux', channel=channel2) - vid6 = Video(source_id='quuz', channel=channel2) - test_session.add_all([vid1, vid2, vid3, vid4, vid5, vid6]) - test_session.commit() - - def set_entries(channel1_entries, channel2_entries): - channel1.info_json = {'entries': [{'id': i} for i in channel1_entries]} if channel1_entries else \ - sqlalchemy.null() - channel2.info_json = {'entries': [{'id': i} for i in channel2_entries]} if channel2_entries else \ - sqlalchemy.null() - test_session.commit() - - # All source_id's are in "entries". - set_entries(['foo', 'bar'], ['qux', 'quux', 'quuz']) - assert _censored_source_ids(channel1.link) == set() - - # "foo" is censored. - set_entries(['bar'], ['qux', 'quux', 'quuz']) - assert _censored_source_ids(channel1.link) == {'foo'} - assert _censored_source_ids() == {'foo'} - - # channel1 has no info_json. - set_entries(None, ['qux', 'quux', 'quuz']) - assert _censored_source_ids(channel1.link) == set() - - # Censor channel2 and channel1. - set_entries(['bar'], ['quuz']) - assert _censored_source_ids() == {'foo', 'qux', 'quux'} - assert _censored_source_ids(channel1.link) == {'foo', } - assert _censored_source_ids(channel2.link) == {'qux', 'quux'} - - # channel 1 has no info_json - set_entries(None, ['quuz']) - assert _censored_source_ids() == {'qux', 'quux'} - assert _censored_source_ids(channel1.link) == set() - assert _censored_source_ids(channel2.link) == {'qux', 'quux'} - - # channel 2 has no entries - set_entries(None, []) - assert _censored_source_ids() == set() - assert _censored_source_ids(channel2.link) == set() - - def test_search_censored_videos(test_session, simple_channel): for i in map(str, range(50)): test_session.add(Video(source_id=i, channel=simple_channel, video_path='foo')) + vid = Video(source_id='51', video_path='bar') # this should never be modified because it has no channel + test_session.add(vid) test_session.commit() def set_entries(entries): - simple_channel.info_json = {'entries': [{'id': j} for j in entries]} if entries else sqlalchemy.null() + simple_channel.info_json = { + 'entries': [{'id': j, 'view_count': 0} for j in entries]} if entries else sqlalchemy.null() + test_session.commit() + apply_info_json(simple_channel.id) test_session.commit() # All source_ids are in the entries. set_entries(map(str, range(50))) - videos, total = video_search(filters=['censored']) + videos, total = video_search(filters=['censored'], order_by='id') assert [i['source_id'] for i in videos] == [] assert total == 0 # First 5 are censored. set_entries(map(str, range(5, 50))) - videos, total = video_search(filters=['censored']) + videos, total = video_search(filters=['censored'], order_by='id') assert [i['source_id'] for i in videos] == [str(i) for i in range(5)] assert total == 5 # First 25 are censored. set_entries(map(str, range(25, 50))) - videos, total = video_search(filters=['censored']) + videos, total = video_search(filters=['censored'], order_by='id') assert [i['source_id'] for i in videos] == [str(i) for i in range(20)] assert total == 25 - - -def test_censored_videos_no_channel(test_session): - with pytest.raises(UnknownChannel): - _censored_source_ids('foo') diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index 432832a1..3dbc8212 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -1,14 +1,13 @@ from datetime import datetime -from typing import Tuple, Optional, List, Set +from typing import Tuple, Optional, List -import cachetools import psycopg2 from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from wrolpi.common import run_after, logger from wrolpi.db import get_db_session, get_db_curs -from wrolpi.errors import UnknownVideo, UnknownChannel +from wrolpi.errors import UnknownVideo from wrolpi.vars import PYTEST from ..lib import save_channels_config from ..models import Video @@ -94,12 +93,7 @@ def video_search( censored_where = '' if isinstance(filters, list) and 'censored' in filters: - if PYTEST: - # Bypass cache for testing only. - params['censored_source_ids'] = list(_censored_source_ids(channel_link)) - else: - params['censored_source_ids'] = list(censored_source_ids(channel_link)) - censored_where = 'AND source_id = ANY(%(censored_source_ids)s)' + censored_where = 'AND censored = true' where = '' if search_str: @@ -167,47 +161,3 @@ def set_video_favorite(video_id: int, favorite: bool) -> Optional[datetime]: favorite = video.set_favorite(favorite) return favorite - - -def _censored_source_ids(link: str = None) -> Set[str]: - with get_db_curs() as curs: - if link: - curs.execute('SELECT id FROM channel WHERE link=%s', (link,)) - try: - id_ = curs.fetchone()[0] - except TypeError: - raise UnknownChannel(f'No channel {link}') - - curs.execute("SELECT info_json->'entries' FROM channel WHERE info_json IS NOT NULL AND link=%s", (link,)) - try: - entries = curs.fetchone()[0] - except TypeError: - # Channel has no info_json entries, so we can't find censored. - return set() - - source_ids = {i['id'] for i in entries if i} - curs.execute('SELECT source_id FROM video WHERE channel_id =%s', (id_,)) - our_source_ids = {i[0] for i in curs.fetchall()} - else: - curs.execute("SELECT info_json->'entries' FROM channel WHERE info_json IS NOT NULL") - source_ids = {j['id'] for i in curs.fetchall() if i and i[0] for j in i[0]} - stmt = ''' - SELECT v.source_id - FROM video v - WHERE v.channel_id IN (select id from channel c where info_json is not null) - ''' - curs.execute(stmt) - result = curs.fetchall() - our_source_ids = {i[0] for i in result} - - censored_ids = our_source_ids - source_ids - return censored_ids - - -@cachetools.cached(cachetools.TTLCache(maxsize=100, ttl=600)) -def censored_source_ids(link: str = None) -> Set[str]: - """ - Get all Videos that are downloaded, but are not in its Channel's catalog. Videos without a channel will not be - returned. - """ - return _censored_source_ids(link)