From 25f9568154192c3e6a8edc459d704aee623a5534 Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 16 Aug 2025 12:03:58 -0600 Subject: [PATCH] Adding summary columns to Channel table. Adding triggers to keep columns updated. --- .../54c19aa847da_channel_summary_indexes.py | 145 ++++++++++++++++++ modules/videos/channel/lib.py | 52 ++----- modules/videos/models.py | 61 ++++---- 3 files changed, 187 insertions(+), 71 deletions(-) create mode 100644 alembic/versions/54c19aa847da_channel_summary_indexes.py diff --git a/alembic/versions/54c19aa847da_channel_summary_indexes.py b/alembic/versions/54c19aa847da_channel_summary_indexes.py new file mode 100644 index 00000000..870905b2 --- /dev/null +++ b/alembic/versions/54c19aa847da_channel_summary_indexes.py @@ -0,0 +1,145 @@ +"""Channel summary columns, triggers and indexes. + +Revision ID: 54c19aa847da +Revises: 158973fda20f +Create Date: 2025-08-16 11:45:48.190427 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '54c19aa847da' +down_revision = '158973fda20f' +branch_labels = None +depends_on = None + + +def upgrade(): + # Add columns + op.execute(""" + ALTER TABLE channel + ADD COLUMN video_count INTEGER DEFAULT 0 NOT NULL, + ADD COLUMN total_size BIGINT DEFAULT 0 NOT NULL, + ADD COLUMN minimum_frequency INTEGER; + """) + + # Populate columns + op.execute(""" + UPDATE channel + SET video_count = (SELECT COUNT(*) + FROM video + WHERE channel_id = channel.id), + total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT + FROM video v + LEFT JOIN file_group fg ON v.file_group_id = fg.id + WHERE v.channel_id = channel.id), + minimum_frequency = (SELECT MIN(frequency) + FROM download + WHERE channel_id = channel.id); + """) + + # Create trigger functions + op.execute(""" + CREATE OR REPLACE FUNCTION update_channel_video_count() + RETURNS TRIGGER AS + $$ + BEGIN + UPDATE channel + SET video_count = (SELECT COUNT(*) FROM video WHERE channel_id = NEW.channel_id) + WHERE id = NEW.channel_id; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """) + op.execute(""" + CREATE OR REPLACE FUNCTION update_channel_size() + RETURNS TRIGGER AS + $$ + BEGIN + UPDATE channel + SET total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT + FROM video v + LEFT JOIN file_group fg ON v.file_group_id = fg.id + WHERE v.channel_id = NEW.channel_id) + WHERE id = NEW.channel_id; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """) + op.execute(""" + CREATE OR REPLACE FUNCTION update_channel_minimum_frequency() + RETURNS TRIGGER AS + $$ + BEGIN + UPDATE channel + SET minimum_frequency = (SELECT MIN(frequency) FROM download WHERE channel_id = NEW.channel_id) + WHERE id = NEW.channel_id; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """) + + # Create triggers + op.execute(""" + CREATE TRIGGER video_count_trigger + AFTER INSERT OR DELETE + ON video + FOR EACH ROW + EXECUTE FUNCTION update_channel_video_count(); + """) + op.execute(""" + CREATE TRIGGER video_size_trigger + AFTER INSERT OR DELETE + ON video + FOR EACH ROW + EXECUTE FUNCTION update_channel_size(); + """) + op.execute(""" + CREATE TRIGGER file_group_size_trigger + AFTER UPDATE OF size + ON file_group + FOR EACH ROW + WHEN (OLD.size IS DISTINCT FROM NEW.size) + EXECUTE FUNCTION update_channel_size(); + """) + op.execute(""" + CREATE TRIGGER download_frequency_trigger + AFTER INSERT OR UPDATE OF frequency OR DELETE + ON download + FOR EACH ROW + EXECUTE FUNCTION update_channel_minimum_frequency(); + """) + + # Create indexes + op.execute(""" + CREATE INDEX channel_video_count_idx ON channel (video_count); + CREATE INDEX channel_total_size_idx ON channel (total_size); + CREATE INDEX channel_minimum_frequency_idx ON channel (minimum_frequency); + """) + + +def downgrade(): + # Drop triggers + op.execute("DROP TRIGGER IF EXISTS video_count_trigger ON video;") + op.execute("DROP TRIGGER IF EXISTS video_size_trigger ON video;") + op.execute("DROP TRIGGER IF EXISTS file_group_size_trigger ON file_group;") + op.execute("DROP TRIGGER IF EXISTS download_frequency_trigger ON download;") + + # Drop trigger functions + op.execute("DROP FUNCTION IF EXISTS update_channel_video_count;") + op.execute("DROP FUNCTION IF EXISTS update_channel_size;") + op.execute("DROP FUNCTION IF EXISTS update_channel_minimum_frequency;") + + # Drop indexes + op.execute("DROP INDEX IF EXISTS channel_minimum_frequency_idx;") + op.execute("DROP INDEX IF EXISTS channel_total_size_idx;") + op.execute("DROP INDEX IF EXISTS channel_video_count_idx;") + + # Drop columns + op.execute(""" + ALTER TABLE channel + DROP COLUMN IF EXISTS minimum_frequency, + DROP COLUMN IF EXISTS total_size, + DROP COLUMN IF EXISTS video_count; + """) diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index f991b4b4..36d52f8c 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -29,52 +29,24 @@ async def get_minimal_channels() -> List[dict]: # Get all channels, even if they don't have videos. Also get the minimum frequency download because this is the # one that will consume the most resources. stmt = ''' - SELECT - c.id, c.name AS "name", directory, c.url, t.name AS "tag_name", - COUNT(v.id) as video_count, - SUM(fg.size)::BIGINT AS size, - ( - select min(d.frequency) - from download d - where d.channel_id = c.id - ) AS minimum_frequency - FROM - channel AS c - LEFT JOIN video v on c.id = v.channel_id - LEFT JOIN file_group fg on fg.id = v.file_group_id - LEFT JOIN tag t ON t.id = c.tag_id - GROUP BY 1, 2, 3, 4, 5 - ''' + SELECT c.id, + c.name AS "name", + c.directory, + c.url, + t.name AS "tag_name", + c.video_count, + c.total_size, + c.minimum_frequency + FROM channel AS c + LEFT JOIN tag t ON t.id = c.tag_id + ''' curs.execute(stmt) + logger.debug(stmt) channels = sorted([dict(i) for i in curs.fetchall()], key=lambda i: i['name'].lower()) return channels -async def get_channels_video_count() -> Dict[int, int]: - """ - Add video counts to all channels - """ - with get_db_curs() as curs: - stmt = 'SELECT id FROM channel' - curs.execute(stmt) - # Get all channel IDs, start them with a count of 0. - video_counts = {int(i['id']): 0 for i in curs.fetchall()} - - stmt = ''' - SELECT - c.id, COUNT(v.id) AS video_count - FROM - channel AS c - LEFT JOIN video AS v ON v.channel_id = c.id - GROUP BY 1 - ''' - curs.execute(stmt) - # Replace all the counts of those channels with videos. - video_counts.update({int(i['id']): int(i['video_count']) for i in curs.fetchall()}) - return video_counts - - COD = Union[dict, Channel] diff --git a/modules/videos/models.py b/modules/videos/models.py index ed72b3bc..f739663e 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -152,25 +152,20 @@ class Video(ModelHelper, Base): if self.file_group.published_datetime: # Get videos next to this Video's upload date. stmt = ''' - WITH numbered_videos AS ( - SELECT fg.id AS fg_id, v.id AS v_id, - ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number - FROM file_group fg - LEFT OUTER JOIN video v on fg.id = v.file_group_id - WHERE - v.channel_id = %(channel_id)s - AND fg.published_datetime IS NOT NULL - ) - SELECT v_id - FROM numbered_videos - WHERE row_number IN ( - SELECT row_number+i - FROM numbered_videos - CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n - WHERE - fg_id = %(fg_id)s - ) - ''' + WITH numbered_videos AS (SELECT fg.id AS fg_id, + v.id AS v_id, + ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number + FROM file_group fg + LEFT OUTER JOIN video v on fg.id = v.file_group_id + WHERE v.channel_id = %(channel_id)s + AND fg.published_datetime IS NOT NULL) + SELECT v_id + FROM numbered_videos + WHERE row_number IN (SELECT row_number + i + FROM numbered_videos + CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n + WHERE fg_id = %(fg_id)s) \ + ''' else: # No videos near this Video with upload dates, recommend the files next to this Video. # Only recommend videos in the same Channel (or similarly without a Channel). @@ -532,6 +527,11 @@ class Channel(ModelHelper, Base): source_id = Column(String) # the ID from the source website. refreshed = Column(Boolean, default=False) # The files in the Channel have been refreshed. + # Columns updated by triggers + video_count = Column(Integer) # update_channel_video_count + total_size = Column(Integer) # update_channel_size + minimum_frequency = Column(Integer) # update_channel_minimum_frequency + info_json = deferred(Column(JSON)) info_date = Column(Date) @@ -680,18 +680,17 @@ class Channel(ModelHelper, Base): """Get statistics about this channel.""" with get_db_curs() as curs: stmt = ''' - SELECT - SUM(size) AS "size", - MAX(size) AS "largest_video", - COUNT(video.id) AS "video_count", - SUM(fg.length) AS "length", - -- Videos may use multiple tags. - COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags" - FROM video - LEFT JOIN file_group fg on fg.id = video.file_group_id - LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id - WHERE channel_id = %(id)s - ''' + SELECT SUM(size) AS "size", + MAX(size) AS "largest_video", + COUNT(video.id) AS "video_count", + SUM(fg.length) AS "length", + -- Videos may use multiple tags. + COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags" + FROM video + LEFT JOIN file_group fg on fg.id = video.file_group_id + LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id + WHERE channel_id = %(id)s \ + ''' curs.execute(stmt, dict(id=self.id)) return dict(curs.fetchone())