Setting FileGroup.censored when refreshing channel files.

alembic/versions/00f11f309c53_filegroup_censored.py

@@ -0,0 +1,33 @@
+"""FileGroup.censored
+
+Revision ID: 00f11f309c53
+Revises: 1c041aeee0fc
+Create Date: 2024-06-17 14:02:09.881477
+
+"""
+import os
+
+from alembic import op
+from sqlalchemy.orm import Session
+
+# revision identifiers, used by Alembic.
+revision = '00f11f309c53'
+down_revision = '1c041aeee0fc'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    session.execute('ALTER TABLE file_group ADD COLUMN censored BOOLEAN DEFAULT FALSE')
+
+
+def downgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS censored')

modules/videos/common.py

@@ -302,8 +302,9 @@ def check_for_video_corruption(video_path: Path) -> bool:
     return corrupt
 
 
-async def update_view_counts(channel_id: int):
-    """Update view_count for all Videos in a channel using its info_json file."""
+async def update_view_counts_and_censored(channel_id: int):
+    """Update view_count for all Videos in a channel using its info_json file.  Also sets FileGroup.censored
+    if Video is no longer available on the Channel."""
     with get_db_session() as session:
         channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
         channel_name = channel.name
@@ -328,4 +329,19 @@ async def update_view_counts(channel_id: int):
         '''
         curs.execute(stmt, (view_counts_str, channel_id))
         count = len(curs.fetchall())
-        logger.debug(f'Updated {count} view counts in DB for {channel_name}.')
+        logger.info(f'Updated {count} view counts in DB for {channel_name}.')
+
+    source_ids = [i['id'] for i in info['entries']]
+    with get_db_curs(commit=True) as curs:
+        # Set FileGroup.censored if the video is no longer on the Channel.
+        stmt = '''
+            UPDATE file_group fg
+            SET censored = NOT (v.source_id = ANY(%(source_ids)s))
+            FROM video v
+            WHERE v.file_group_id = fg.id
+                AND v.channel_id = %(channel_id)s
+            RETURNING fg.id, fg.censored
+        '''
+        curs.execute(stmt, {'channel_id': channel_id, 'source_ids': source_ids})
+        censored = len([i for i in curs.fetchall() if i['censored']])
+        logger.info(f'Set {censored} censored videos for {channel_name}.')

modules/videos/downloader.py

@@ -23,7 +23,7 @@ from wrolpi.files.lib import glob_shared_stem
 from wrolpi.files.models import FileGroup
 from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
 from .channel.lib import create_channel, get_channel
-from .common import get_no_channel_directory, get_videos_directory, update_view_counts, ffmpeg_video_complete
+from .common import get_no_channel_directory, get_videos_directory, update_view_counts_and_censored, ffmpeg_video_complete
 from .errors import UnknownChannel
 from .lib import get_downloader_config, YDL, ydl_logger
 from .models import Video, Channel
@@ -592,7 +592,7 @@ def update_channel_catalog(channel: Channel, info: dict):
     logger.info(f'Finished downloading video list for {channel} found {len(entries)} videos')
 
     # Update all view counts using the latest from the Channel's info_json.
-    background_task(update_view_counts(channel_id))
+    background_task(update_view_counts_and_censored(channel_id))
 
 
 UNRECOVERABLE_ERRORS = {

modules/videos/models.py

@@ -657,15 +657,15 @@ class Channel(ModelHelper, Base):
         await refresh_files([self.directory], send_events=send_events)
 
         # Apply any info_json (update view counts) second.
-        from modules.videos.common import update_view_counts
+        from modules.videos.common import update_view_counts_and_censored
         if PYTEST:
-            await update_view_counts(self_id)
+            await update_view_counts_and_censored(self_id)
             self.refreshed = True
         else:
 
             # Perform info_json in background task.  Channel will be marked as refreshed after this completes.
             async def _():
-                await update_view_counts(self_id)
+                await update_view_counts_and_censored(self_id)
                 with get_db_session(commit=True) as session:
                     channel: Channel = session.query(Channel).filter(Channel.id == self_id).one()
                     channel.refreshed = True

modules/videos/test/test_common.py

@@ -12,7 +12,7 @@ from wrolpi.downloader import Download, DownloadFrequency
 from wrolpi.files import lib as files_lib
 from wrolpi.vars import PROJECT_DIR
 from .. import common
-from ..common import convert_image, update_view_counts, get_video_duration, generate_video_poster, is_valid_poster
+from ..common import convert_image, update_view_counts_and_censored, get_video_duration, generate_video_poster, is_valid_poster
 from ..lib import save_channels_config, get_channels_config, import_channels_config, ChannelsConfig
 
 
@@ -102,22 +102,22 @@ async def test_update_view_count(test_session, channel_factory, video_factory):
     check_view_counts({'vid1': None, 'vid2': None, 'vid3': None, 'vid4': None})
 
     # Channel 1 is updated, the other channels are left alone.
-    await update_view_counts(channel1.id)
+    await update_view_counts_and_censored(channel1.id)
     check_view_counts({'vid1': 10, 'vid2': None, 'vid3': None, 'vid4': None})
 
     # Channel 2 is updated, the other channels are left alone.  The 'bad_id' video is ignored.
-    await update_view_counts(channel2.id)
+    await update_view_counts_and_censored(channel2.id)
     check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': None, 'vid4': None})
 
     # All videos are updated.
-    await update_view_counts(channel3.id)
+    await update_view_counts_and_censored(channel3.id)
     check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})
 
     # An outdated view count will be overwritten.
     vid = test_session.query(Video).filter_by(id=1).one()
     vid.view_count = 8
     check_view_counts({'vid1': 8, 'vid2': 11, 'vid3': 13, 'vid4': 14})
-    await update_view_counts(channel1.id)
+    await update_view_counts_and_censored(channel1.id)
     check_view_counts({'vid1': 10, 'vid2': 11, 'vid3': 13, 'vid4': 14})