Adding summary columns to Channel table. Adding triggers to keep columns updated.

This commit is contained in:
Roland 2025-08-16 12:03:58 -06:00 • committed by lrnselfreliance
parent e3ad88bfc3
commit 25f9568154
3 changed files with 187 additions and 71 deletions

View file

@ -0,0 +1,145 @@
"""Channel summary columns, triggers and indexes.
Revision ID: 54c19aa847da
Revises: 158973fda20f
Create Date: 2025-08-16 11:45:48.190427
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '54c19aa847da'
down_revision = '158973fda20f'
branch_labels = None
depends_on = None
def upgrade():
# Add columns
op.execute("""
ALTER TABLE channel
ADD COLUMN video_count INTEGER DEFAULT 0 NOT NULL,
ADD COLUMN total_size BIGINT DEFAULT 0 NOT NULL,
ADD COLUMN minimum_frequency INTEGER;
""")
# Populate columns
op.execute("""
UPDATE channel
SET video_count = (SELECT COUNT(*)
FROM video
WHERE channel_id = channel.id),
total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT
FROM video v
LEFT JOIN file_group fg ON v.file_group_id = fg.id
WHERE v.channel_id = channel.id),
minimum_frequency = (SELECT MIN(frequency)
FROM download
WHERE channel_id = channel.id);
""")
# Create trigger functions
op.execute("""
CREATE OR REPLACE FUNCTION update_channel_video_count()
RETURNS TRIGGER AS
$$
BEGIN
UPDATE channel
SET video_count = (SELECT COUNT(*) FROM video WHERE channel_id = NEW.channel_id)
WHERE id = NEW.channel_id;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
""")
op.execute("""
CREATE OR REPLACE FUNCTION update_channel_size()
RETURNS TRIGGER AS
$$
BEGIN
UPDATE channel
SET total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT
FROM video v
LEFT JOIN file_group fg ON v.file_group_id = fg.id
WHERE v.channel_id = NEW.channel_id)
WHERE id = NEW.channel_id;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
""")
op.execute("""
CREATE OR REPLACE FUNCTION update_channel_minimum_frequency()
RETURNS TRIGGER AS
$$
BEGIN
UPDATE channel
SET minimum_frequency = (SELECT MIN(frequency) FROM download WHERE channel_id = NEW.channel_id)
WHERE id = NEW.channel_id;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
""")
# Create triggers
op.execute("""
CREATE TRIGGER video_count_trigger
AFTER INSERT OR DELETE
ON video
FOR EACH ROW
EXECUTE FUNCTION update_channel_video_count();
""")
op.execute("""
CREATE TRIGGER video_size_trigger
AFTER INSERT OR DELETE
ON video
FOR EACH ROW
EXECUTE FUNCTION update_channel_size();
""")
op.execute("""
CREATE TRIGGER file_group_size_trigger
AFTER UPDATE OF size
ON file_group
FOR EACH ROW
WHEN (OLD.size IS DISTINCT FROM NEW.size)
EXECUTE FUNCTION update_channel_size();
""")
op.execute("""
CREATE TRIGGER download_frequency_trigger
AFTER INSERT OR UPDATE OF frequency OR DELETE
ON download
FOR EACH ROW
EXECUTE FUNCTION update_channel_minimum_frequency();
""")
# Create indexes
op.execute("""
CREATE INDEX channel_video_count_idx ON channel (video_count);
CREATE INDEX channel_total_size_idx ON channel (total_size);
CREATE INDEX channel_minimum_frequency_idx ON channel (minimum_frequency);
""")
def downgrade():
# Drop triggers
op.execute("DROP TRIGGER IF EXISTS video_count_trigger ON video;")
op.execute("DROP TRIGGER IF EXISTS video_size_trigger ON video;")
op.execute("DROP TRIGGER IF EXISTS file_group_size_trigger ON file_group;")
op.execute("DROP TRIGGER IF EXISTS download_frequency_trigger ON download;")
# Drop trigger functions
op.execute("DROP FUNCTION IF EXISTS update_channel_video_count;")
op.execute("DROP FUNCTION IF EXISTS update_channel_size;")
op.execute("DROP FUNCTION IF EXISTS update_channel_minimum_frequency;")
# Drop indexes
op.execute("DROP INDEX IF EXISTS channel_minimum_frequency_idx;")
op.execute("DROP INDEX IF EXISTS channel_total_size_idx;")
op.execute("DROP INDEX IF EXISTS channel_video_count_idx;")
# Drop columns
op.execute("""
ALTER TABLE channel
DROP COLUMN IF EXISTS minimum_frequency,
DROP COLUMN IF EXISTS total_size,
DROP COLUMN IF EXISTS video_count;
""")

View file

@ -29,52 +29,24 @@ async def get_minimal_channels() -> List[dict]:
# Get all channels, even if they don't have videos. Also get the minimum frequency download because this is the
# one that will consume the most resources.
stmt = '''
SELECT
c.id, c.name AS "name", directory, c.url, t.name AS "tag_name",
COUNT(v.id) as video_count,
SUM(fg.size)::BIGINT AS size,
(
select min(d.frequency)
from download d
where d.channel_id = c.id
) AS minimum_frequency
FROM
channel AS c
LEFT JOIN video v on c.id = v.channel_id
LEFT JOIN file_group fg on fg.id = v.file_group_id
LEFT JOIN tag t ON t.id = c.tag_id
GROUP BY 1, 2, 3, 4, 5
'''
SELECT c.id,
c.name AS "name",
c.directory,
c.url,
t.name AS "tag_name",
c.video_count,
c.total_size,
c.minimum_frequency
FROM channel AS c
LEFT JOIN tag t ON t.id = c.tag_id
'''
curs.execute(stmt)
logger.debug(stmt)
channels = sorted([dict(i) for i in curs.fetchall()], key=lambda i: i['name'].lower())
return channels
async def get_channels_video_count() -> Dict[int, int]:
"""
Add video counts to all channels
"""
with get_db_curs() as curs:
stmt = 'SELECT id FROM channel'
curs.execute(stmt)
# Get all channel IDs, start them with a count of 0.
video_counts = {int(i['id']): 0 for i in curs.fetchall()}
stmt = '''
SELECT
c.id, COUNT(v.id) AS video_count
FROM
channel AS c
LEFT JOIN video AS v ON v.channel_id = c.id
GROUP BY 1
'''
curs.execute(stmt)
# Replace all the counts of those channels with videos.
video_counts.update({int(i['id']): int(i['video_count']) for i in curs.fetchall()})
return video_counts
COD = Union[dict, Channel]

View file

@ -152,25 +152,20 @@ class Video(ModelHelper, Base):
if self.file_group.published_datetime:
# Get videos next to this Video's upload date.
stmt = '''
WITH numbered_videos AS (
SELECT fg.id AS fg_id, v.id AS v_id,
ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number
FROM file_group fg
LEFT OUTER JOIN video v on fg.id = v.file_group_id
WHERE
v.channel_id = %(channel_id)s
AND fg.published_datetime IS NOT NULL
)
SELECT v_id
FROM numbered_videos
WHERE row_number IN (
SELECT row_number+i
FROM numbered_videos
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
WHERE
fg_id = %(fg_id)s
)
'''
WITH numbered_videos AS (SELECT fg.id AS fg_id,
v.id AS v_id,
ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number
FROM file_group fg
LEFT OUTER JOIN video v on fg.id = v.file_group_id
WHERE v.channel_id = %(channel_id)s
AND fg.published_datetime IS NOT NULL)
SELECT v_id
FROM numbered_videos
WHERE row_number IN (SELECT row_number + i
FROM numbered_videos
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
WHERE fg_id = %(fg_id)s) \
'''
else:
# No videos near this Video with upload dates, recommend the files next to this Video.
# Only recommend videos in the same Channel (or similarly without a Channel).
@ -532,6 +527,11 @@ class Channel(ModelHelper, Base):
source_id = Column(String) # the ID from the source website.
refreshed = Column(Boolean, default=False) # The files in the Channel have been refreshed.
# Columns updated by triggers
video_count = Column(Integer) # update_channel_video_count
total_size = Column(Integer) # update_channel_size
minimum_frequency = Column(Integer) # update_channel_minimum_frequency
info_json = deferred(Column(JSON))
info_date = Column(Date)
@ -680,18 +680,17 @@ class Channel(ModelHelper, Base):
"""Get statistics about this channel."""
with get_db_curs() as curs:
stmt = '''
SELECT
SUM(size) AS "size",
MAX(size) AS "largest_video",
COUNT(video.id) AS "video_count",
SUM(fg.length) AS "length",
-- Videos may use multiple tags.
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
FROM video
LEFT JOIN file_group fg on fg.id = video.file_group_id
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
WHERE channel_id = %(id)s
'''
SELECT SUM(size) AS "size",
MAX(size) AS "largest_video",
COUNT(video.id) AS "video_count",
SUM(fg.length) AS "length",
-- Videos may use multiple tags.
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
FROM video
LEFT JOIN file_group fg on fg.id = video.file_group_id
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
WHERE channel_id = %(id)s \
'''
curs.execute(stmt, dict(id=self.id))
return dict(curs.fetchone())