mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 23:16:17 -05:00
Adding summary columns to Channel table. Adding triggers to keep columns updated.
This commit is contained in:
parent
e3ad88bfc3
commit
25f9568154
3 changed files with 187 additions and 71 deletions
145
alembic/versions/54c19aa847da_channel_summary_indexes.py
Normal file
145
alembic/versions/54c19aa847da_channel_summary_indexes.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
"""Channel summary columns, triggers and indexes.
|
||||
|
||||
Revision ID: 54c19aa847da
|
||||
Revises: 158973fda20f
|
||||
Create Date: 2025-08-16 11:45:48.190427
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '54c19aa847da'
|
||||
down_revision = '158973fda20f'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# Add columns
|
||||
op.execute("""
|
||||
ALTER TABLE channel
|
||||
ADD COLUMN video_count INTEGER DEFAULT 0 NOT NULL,
|
||||
ADD COLUMN total_size BIGINT DEFAULT 0 NOT NULL,
|
||||
ADD COLUMN minimum_frequency INTEGER;
|
||||
""")
|
||||
|
||||
# Populate columns
|
||||
op.execute("""
|
||||
UPDATE channel
|
||||
SET video_count = (SELECT COUNT(*)
|
||||
FROM video
|
||||
WHERE channel_id = channel.id),
|
||||
total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT
|
||||
FROM video v
|
||||
LEFT JOIN file_group fg ON v.file_group_id = fg.id
|
||||
WHERE v.channel_id = channel.id),
|
||||
minimum_frequency = (SELECT MIN(frequency)
|
||||
FROM download
|
||||
WHERE channel_id = channel.id);
|
||||
""")
|
||||
|
||||
# Create trigger functions
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION update_channel_video_count()
|
||||
RETURNS TRIGGER AS
|
||||
$$
|
||||
BEGIN
|
||||
UPDATE channel
|
||||
SET video_count = (SELECT COUNT(*) FROM video WHERE channel_id = NEW.channel_id)
|
||||
WHERE id = NEW.channel_id;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION update_channel_size()
|
||||
RETURNS TRIGGER AS
|
||||
$$
|
||||
BEGIN
|
||||
UPDATE channel
|
||||
SET total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT
|
||||
FROM video v
|
||||
LEFT JOIN file_group fg ON v.file_group_id = fg.id
|
||||
WHERE v.channel_id = NEW.channel_id)
|
||||
WHERE id = NEW.channel_id;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION update_channel_minimum_frequency()
|
||||
RETURNS TRIGGER AS
|
||||
$$
|
||||
BEGIN
|
||||
UPDATE channel
|
||||
SET minimum_frequency = (SELECT MIN(frequency) FROM download WHERE channel_id = NEW.channel_id)
|
||||
WHERE id = NEW.channel_id;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
|
||||
# Create triggers
|
||||
op.execute("""
|
||||
CREATE TRIGGER video_count_trigger
|
||||
AFTER INSERT OR DELETE
|
||||
ON video
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_channel_video_count();
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE TRIGGER video_size_trigger
|
||||
AFTER INSERT OR DELETE
|
||||
ON video
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_channel_size();
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE TRIGGER file_group_size_trigger
|
||||
AFTER UPDATE OF size
|
||||
ON file_group
|
||||
FOR EACH ROW
|
||||
WHEN (OLD.size IS DISTINCT FROM NEW.size)
|
||||
EXECUTE FUNCTION update_channel_size();
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE TRIGGER download_frequency_trigger
|
||||
AFTER INSERT OR UPDATE OF frequency OR DELETE
|
||||
ON download
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_channel_minimum_frequency();
|
||||
""")
|
||||
|
||||
# Create indexes
|
||||
op.execute("""
|
||||
CREATE INDEX channel_video_count_idx ON channel (video_count);
|
||||
CREATE INDEX channel_total_size_idx ON channel (total_size);
|
||||
CREATE INDEX channel_minimum_frequency_idx ON channel (minimum_frequency);
|
||||
""")
|
||||
|
||||
|
||||
def downgrade():
|
||||
# Drop triggers
|
||||
op.execute("DROP TRIGGER IF EXISTS video_count_trigger ON video;")
|
||||
op.execute("DROP TRIGGER IF EXISTS video_size_trigger ON video;")
|
||||
op.execute("DROP TRIGGER IF EXISTS file_group_size_trigger ON file_group;")
|
||||
op.execute("DROP TRIGGER IF EXISTS download_frequency_trigger ON download;")
|
||||
|
||||
# Drop trigger functions
|
||||
op.execute("DROP FUNCTION IF EXISTS update_channel_video_count;")
|
||||
op.execute("DROP FUNCTION IF EXISTS update_channel_size;")
|
||||
op.execute("DROP FUNCTION IF EXISTS update_channel_minimum_frequency;")
|
||||
|
||||
# Drop indexes
|
||||
op.execute("DROP INDEX IF EXISTS channel_minimum_frequency_idx;")
|
||||
op.execute("DROP INDEX IF EXISTS channel_total_size_idx;")
|
||||
op.execute("DROP INDEX IF EXISTS channel_video_count_idx;")
|
||||
|
||||
# Drop columns
|
||||
op.execute("""
|
||||
ALTER TABLE channel
|
||||
DROP COLUMN IF EXISTS minimum_frequency,
|
||||
DROP COLUMN IF EXISTS total_size,
|
||||
DROP COLUMN IF EXISTS video_count;
|
||||
""")
|
||||
|
|
@ -29,52 +29,24 @@ async def get_minimal_channels() -> List[dict]:
|
|||
# Get all channels, even if they don't have videos. Also get the minimum frequency download because this is the
|
||||
# one that will consume the most resources.
|
||||
stmt = '''
|
||||
SELECT
|
||||
c.id, c.name AS "name", directory, c.url, t.name AS "tag_name",
|
||||
COUNT(v.id) as video_count,
|
||||
SUM(fg.size)::BIGINT AS size,
|
||||
(
|
||||
select min(d.frequency)
|
||||
from download d
|
||||
where d.channel_id = c.id
|
||||
) AS minimum_frequency
|
||||
FROM
|
||||
channel AS c
|
||||
LEFT JOIN video v on c.id = v.channel_id
|
||||
LEFT JOIN file_group fg on fg.id = v.file_group_id
|
||||
LEFT JOIN tag t ON t.id = c.tag_id
|
||||
GROUP BY 1, 2, 3, 4, 5
|
||||
'''
|
||||
SELECT c.id,
|
||||
c.name AS "name",
|
||||
c.directory,
|
||||
c.url,
|
||||
t.name AS "tag_name",
|
||||
c.video_count,
|
||||
c.total_size,
|
||||
c.minimum_frequency
|
||||
FROM channel AS c
|
||||
LEFT JOIN tag t ON t.id = c.tag_id
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
logger.debug(stmt)
|
||||
channels = sorted([dict(i) for i in curs.fetchall()], key=lambda i: i['name'].lower())
|
||||
|
||||
return channels
|
||||
|
||||
|
||||
async def get_channels_video_count() -> Dict[int, int]:
|
||||
"""
|
||||
Add video counts to all channels
|
||||
"""
|
||||
with get_db_curs() as curs:
|
||||
stmt = 'SELECT id FROM channel'
|
||||
curs.execute(stmt)
|
||||
# Get all channel IDs, start them with a count of 0.
|
||||
video_counts = {int(i['id']): 0 for i in curs.fetchall()}
|
||||
|
||||
stmt = '''
|
||||
SELECT
|
||||
c.id, COUNT(v.id) AS video_count
|
||||
FROM
|
||||
channel AS c
|
||||
LEFT JOIN video AS v ON v.channel_id = c.id
|
||||
GROUP BY 1
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
# Replace all the counts of those channels with videos.
|
||||
video_counts.update({int(i['id']): int(i['video_count']) for i in curs.fetchall()})
|
||||
return video_counts
|
||||
|
||||
|
||||
COD = Union[dict, Channel]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -152,25 +152,20 @@ class Video(ModelHelper, Base):
|
|||
if self.file_group.published_datetime:
|
||||
# Get videos next to this Video's upload date.
|
||||
stmt = '''
|
||||
WITH numbered_videos AS (
|
||||
SELECT fg.id AS fg_id, v.id AS v_id,
|
||||
ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number
|
||||
FROM file_group fg
|
||||
LEFT OUTER JOIN video v on fg.id = v.file_group_id
|
||||
WHERE
|
||||
v.channel_id = %(channel_id)s
|
||||
AND fg.published_datetime IS NOT NULL
|
||||
)
|
||||
SELECT v_id
|
||||
FROM numbered_videos
|
||||
WHERE row_number IN (
|
||||
SELECT row_number+i
|
||||
FROM numbered_videos
|
||||
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
|
||||
WHERE
|
||||
fg_id = %(fg_id)s
|
||||
)
|
||||
'''
|
||||
WITH numbered_videos AS (SELECT fg.id AS fg_id,
|
||||
v.id AS v_id,
|
||||
ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number
|
||||
FROM file_group fg
|
||||
LEFT OUTER JOIN video v on fg.id = v.file_group_id
|
||||
WHERE v.channel_id = %(channel_id)s
|
||||
AND fg.published_datetime IS NOT NULL)
|
||||
SELECT v_id
|
||||
FROM numbered_videos
|
||||
WHERE row_number IN (SELECT row_number + i
|
||||
FROM numbered_videos
|
||||
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
|
||||
WHERE fg_id = %(fg_id)s) \
|
||||
'''
|
||||
else:
|
||||
# No videos near this Video with upload dates, recommend the files next to this Video.
|
||||
# Only recommend videos in the same Channel (or similarly without a Channel).
|
||||
|
|
@ -532,6 +527,11 @@ class Channel(ModelHelper, Base):
|
|||
source_id = Column(String) # the ID from the source website.
|
||||
refreshed = Column(Boolean, default=False) # The files in the Channel have been refreshed.
|
||||
|
||||
# Columns updated by triggers
|
||||
video_count = Column(Integer) # update_channel_video_count
|
||||
total_size = Column(Integer) # update_channel_size
|
||||
minimum_frequency = Column(Integer) # update_channel_minimum_frequency
|
||||
|
||||
info_json = deferred(Column(JSON))
|
||||
info_date = Column(Date)
|
||||
|
||||
|
|
@ -680,18 +680,17 @@ class Channel(ModelHelper, Base):
|
|||
"""Get statistics about this channel."""
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
SELECT
|
||||
SUM(size) AS "size",
|
||||
MAX(size) AS "largest_video",
|
||||
COUNT(video.id) AS "video_count",
|
||||
SUM(fg.length) AS "length",
|
||||
-- Videos may use multiple tags.
|
||||
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
|
||||
FROM video
|
||||
LEFT JOIN file_group fg on fg.id = video.file_group_id
|
||||
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
|
||||
WHERE channel_id = %(id)s
|
||||
'''
|
||||
SELECT SUM(size) AS "size",
|
||||
MAX(size) AS "largest_video",
|
||||
COUNT(video.id) AS "video_count",
|
||||
SUM(fg.length) AS "length",
|
||||
-- Videos may use multiple tags.
|
||||
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
|
||||
FROM video
|
||||
LEFT JOIN file_group fg on fg.id = video.file_group_id
|
||||
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
|
||||
WHERE channel_id = %(id)s \
|
||||
'''
|
||||
curs.execute(stmt, dict(id=self.id))
|
||||
return dict(curs.fetchone())
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue