From 0fd545d2e83e7dd55b8a42e08510f5c175b7966f Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 17 Jun 2024 14:48:41 -0600 Subject: [PATCH] Setting FileGroup.censored when refreshing channel files. --- .../00f11f309c53_filegroup_censored.py | 33 +++++++++++++++++++ modules/videos/common.py | 22 +++++++++++-- modules/videos/downloader.py | 4 +-- modules/videos/models.py | 6 ++-- modules/videos/test/test_common.py | 10 +++--- 5 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 alembic/versions/00f11f309c53_filegroup_censored.py diff --git a/alembic/versions/00f11f309c53_filegroup_censored.py b/alembic/versions/00f11f309c53_filegroup_censored.py new file mode 100644 index 00000000..82ebea74 --- /dev/null +++ b/alembic/versions/00f11f309c53_filegroup_censored.py @@ -0,0 +1,33 @@ +"""FileGroup.censored + +Revision ID: 00f11f309c53 +Revises: 1c041aeee0fc +Create Date: 2024-06-17 14:02:09.881477 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '00f11f309c53' +down_revision = '1c041aeee0fc' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE file_group ADD COLUMN censored BOOLEAN DEFAULT FALSE') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS censored') diff --git a/modules/videos/common.py b/modules/videos/common.py index c25bf211..c63e0745 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -302,8 +302,9 @@ def check_for_video_corruption(video_path: Path) -> bool: return corrupt -async def update_view_counts(channel_id: int): - """Update view_count for all Videos in a channel using its info_json file.""" +async def update_view_counts_and_censored(channel_id: int): + """Update view_count for all Videos in a channel using its info_json file. Also sets FileGroup.censored + if Video is no longer available on the Channel.""" with get_db_session() as session: channel: Channel = session.query(Channel).filter_by(id=channel_id).one() channel_name = channel.name @@ -328,4 +329,19 @@ async def update_view_counts(channel_id: int): ''' curs.execute(stmt, (view_counts_str, channel_id)) count = len(curs.fetchall()) - logger.debug(f'Updated {count} view counts in DB for {channel_name}.') + logger.info(f'Updated {count} view counts in DB for {channel_name}.') + + source_ids = [i['id'] for i in info['entries']] + with get_db_curs(commit=True) as curs: + # Set FileGroup.censored if the video is no longer on the Channel. + stmt = ''' + UPDATE file_group fg + SET censored = NOT (v.source_id = ANY(%(source_ids)s)) + FROM video v + WHERE v.file_group_id = fg.id + AND v.channel_id = %(channel_id)s + RETURNING fg.id, fg.censored + ''' + curs.execute(stmt, {'channel_id': channel_id, 'source_ids': source_ids}) + censored = len([i for i in curs.fetchall() if i['censored']]) + logger.info(f'Set {censored} censored videos for {channel_name}.') diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index d4b212e4..39f9d1f7 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -23,7 +23,7 @@ from wrolpi.files.lib import glob_shared_stem from wrolpi.files.models import FileGroup from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR from .channel.lib import create_channel, get_channel -from .common import get_no_channel_directory, get_videos_directory, update_view_counts, ffmpeg_video_complete +from .common import get_no_channel_directory, get_videos_directory, update_view_counts_and_censored, ffmpeg_video_complete from .errors import UnknownChannel from .lib import get_downloader_config, YDL, ydl_logger from .models import Video, Channel @@ -592,7 +592,7 @@ def update_channel_catalog(channel: Channel, info: dict): logger.info(f'Finished downloading video list for {channel} found {len(entries)} videos') # Update all view counts using the latest from the Channel's info_json. - background_task(update_view_counts(channel_id)) + background_task(update_view_counts_and_censored(channel_id)) UNRECOVERABLE_ERRORS = { diff --git a/modules/videos/models.py b/modules/videos/models.py index 246c442b..89bdf80a 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -657,15 +657,15 @@ class Channel(ModelHelper, Base): await refresh_files([self.directory], send_events=send_events) # Apply any info_json (update view counts) second. - from modules.videos.common import update_view_counts + from modules.videos.common import update_view_counts_and_censored if PYTEST: - await update_view_counts(self_id) + await update_view_counts_and_censored(self_id) self.refreshed = True else: # Perform info_json in background task. Channel will be marked as refreshed after this completes. async def _(): - await update_view_counts(self_id) + await update_view_counts_and_censored(self_id) with get_db_session(commit=True) as session: channel: Channel = session.query(Channel).filter(Channel.id == self_id).one() channel.refreshed = True diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 6fc47e1b..ba2e565e 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -12,7 +12,7 @@ from wrolpi.downloader import Download, DownloadFrequency from wrolpi.files import lib as files_lib from wrolpi.vars import PROJECT_DIR from .. import common -from ..common import convert_image, update_view_counts, get_video_duration, generate_video_poster, is_valid_poster +from ..common import convert_image, update_view_counts_and_censored, get_video_duration, generate_video_poster, is_valid_poster from ..lib import save_channels_config, get_channels_config, import_channels_config, ChannelsConfig @@ -102,22 +102,22 @@ async def test_update_view_count(test_session, channel_factory, video_factory): check_view_counts({'vid1': None, 'vid2': None, 'vid3': None, 'vid4': None}) # Channel 1 is updated, the other channels are left alone. - await update_view_counts(channel1.id) + await update_view_counts_and_censored(channel1.id) check_view_counts({'vid1': 10, 'vid2': None, 'vid3': None, 'vid4': None}) # Channel 2 is updated, the other channels are left alone. The 'bad_id' video is ignored. - await update_view_counts(channel2.id) + await update_view_counts_and_censored(channel2.id) check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': None, 'vid4': None}) # All videos are updated. - await update_view_counts(channel3.id) + await update_view_counts_and_censored(channel3.id) check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14}) # An outdated view count will be overwritten. vid = test_session.query(Video).filter_by(id=1).one() vid.view_count = 8 check_view_counts({'vid1': 8, 'vid2': 11, 'vid3': 13, 'vid4': 14}) - await update_view_counts(channel1.id) + await update_view_counts_and_censored(channel1.id) check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})