collections: channel and domain listings read trigger-maintained summary columns (#739)

* collections: channel listing reads trigger columns and batch-loads tag/downloads

Listing channel collections summed FileGroup sizes through a video join, which reads a
random page of the file_group table per video, and lazy-loaded every collection's tag
and downloads one query at a time. On a large library with a cold disk cache the
listing took over two minutes, past the browser's 60 s timeout, so the Channels page
showed an error.

channel.video_count and channel.total_size are already maintained by triggers
(wrolpi/schema_ddl.py), so read those. Tag and downloads are selectin-loaded, so the
query count no longer grows with the number of collections.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* collections: domain listing reads trigger-maintained summary columns

Domain Collections summed file_group.size through an archive join on every listing,
one random page read per archive (about 9 s from a cold disk cache on a large
library). New triggers keep collection.item_count and collection.total_size current
for Domain Collections, as the channel triggers already do for Channels, and the
listing reads those columns. A migration installs the triggers and backfills
existing Domain Collections.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-09-27 11:25:12 -06:00 • committed by Roland
parent 4fea50943b
commit 20be5dfc27
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
5 changed files with 281 additions and 34 deletions

View file

@ -0,0 +1,45 @@
"""Maintain collection.item_count/total_size for Domain Collections with triggers.
Listing Domain Collections summed file_group.size through an archive join on every request,
one random page read per archive. The new triggers keep the summary columns current
(as the channel triggers already do for Channels); this migration installs them and
backfills existing Domain Collections.
Revision ID: 2026_09_27_1200
Revises: 2026_09_20_1200
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = '2026_09_27_1200'
down_revision = '2026_09_20_1200'
branch_labels = None
depends_on = None
TRIGGERS = (
'archive_insert_collection_summary',
'archive_delete_collection_summary',
'archive_update_collection_summary',
'file_group_size_collection_summary',
)
def upgrade():
from wrolpi.schema_ddl import install_raw_ddl
# Idempotent: only the new triggers are created.
install_raw_ddl(op.get_bind())
op.execute('''
UPDATE collection SET
item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id),
total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a
LEFT JOIN file_group fg ON a.file_group_id = fg.id
WHERE a.collection_id = collection.id)
WHERE kind = 'domain'
''')
def downgrade():
for name in TRIGGERS:
op.execute(f'DROP TRIGGER IF EXISTS {name}')

View file

@ -8,7 +8,7 @@ import pathlib
from typing import List, Optional, Dict
from sqlalchemy import func
from sqlalchemy.orm import Session
from sqlalchemy.orm import Session, selectinload
from wrolpi import flags
from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
@ -70,12 +70,12 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
Returns:
List of collection dicts with statistics for each collection type
"""
# Local imports to avoid circular import: collections -> archive/videos -> collections
from modules.archive import Archive
from modules.videos.models import Video, Channel
from wrolpi.files.models import FileGroup
# Local imports to avoid circular import: collections -> videos -> collections
from modules.videos.models import Channel
query = session.query(Collection)
# Serializing a Collection reads its tag and downloads; load them in two batch queries rather
# than one lazy query per Collection (thousands of Collections on a large library).
query = session.query(Collection).options(selectinload(Collection.tag), selectinload(Collection.downloads))
if kind:
query = query.filter(Collection.kind == kind)
@ -86,40 +86,23 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
return []
# Separate collections by kind for batch processing
domain_ids = [c.id for c in collections if c.kind == 'domain']
channel_ids = [c.id for c in collections if c.kind == 'channel']
generic_ids = [c.id for c in collections if c.kind not in ('domain', 'channel')]
# Batch query: Get archive stats for all domain collections at once
domain_stats_map = {}
if domain_ids:
domain_stats = session.query(
Archive.collection_id,
func.count(Archive.id).label('archive_count'),
func.coalesce(func.sum(FileGroup.size), 0).label('size')
).outerjoin(
FileGroup, FileGroup.id == Archive.file_group_id
).filter(
Archive.collection_id.in_(domain_ids)
).group_by(Archive.collection_id).all()
domain_stats_map = {s.collection_id: s for s in domain_stats}
# Batch query: Get channel info and video stats for all channel collections at once
# Batch query: Get channel info and video stats for all channel collections at once.
# `channel.video_count` and `channel.total_size` are maintained by triggers (see
# wrolpi/schema_ddl.py); summing FileGroup sizes here instead read a random page of the
# file_group table per video, which took minutes from a cold disk cache on a large library.
channel_stats_map = {}
if channel_ids:
channel_stats = session.query(
Channel.collection_id,
Channel.id.label('channel_id'),
func.count(Video.id).label('video_count'),
func.coalesce(func.sum(FileGroup.size), 0).label('size')
).outerjoin(
Video, Video.channel_id == Channel.id
).outerjoin(
FileGroup, FileGroup.id == Video.file_group_id
Channel.video_count,
Channel.total_size.label('size'),
).filter(
Channel.collection_id.in_(channel_ids)
).group_by(Channel.collection_id, Channel.id).all()
).all()
channel_stats_map = {s.collection_id: s for s in channel_stats}
@ -151,9 +134,10 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
# Add type-specific statistics from batch query results
if collection.kind == 'domain':
stats = domain_stats_map.get(collection.id)
data['archive_count'] = stats.archive_count if stats else 0
data['size'] = int(stats.size) if stats else 0
# `collection.item_count` and `collection.total_size` are maintained by the archive
# triggers (see wrolpi/schema_ddl.py); no join through file_group needed.
data['archive_count'] = collection.item_count
data['size'] = collection.total_size
data['domain'] = data['name'] # Alias for backward compatibility
elif collection.kind == 'channel':

View file

@ -2,6 +2,7 @@
from http import HTTPStatus
import pytest
from sqlalchemy import text as sa_text
from wrolpi.collections.models import Collection
@ -577,3 +578,117 @@ class TestCollectionTagging:
# No conflict checking needed without a directory
assert data['conflict'] is False
assert data['conflict_message'] is None
@pytest.mark.asyncio
async def test_get_channel_collections_uses_summary_columns_and_batches_queries(
async_client, test_session, test_directory, channel_factory, video_factory, tag_factory):
"""Listing channel collections reads the trigger-maintained channel summary columns and
loads downloads and tags in batches, so the query count does not grow with the channel count.
A production DB was measured at 143 s cold for this listing: summing FileGroup sizes touched a
random page of the 2 GB file_group table per video, and each collection lazy-loaded its
downloads and tag one query at a time.
"""
from sqlalchemy import event
from wrolpi.collections import lib
from wrolpi.downloader import DownloadFrequency
await tag_factory('one')
await tag_factory('two')
channels = []
for i in range(6):
tag_name = ('one', 'two', None)[i % 3]
channel = channel_factory(name=f'Channel {i}', download_frequency=DownloadFrequency.weekly, tag_name=tag_name)
video_factory(channel_id=channel.id, with_video_file=True)
video_factory(channel_id=channel.id, with_video_file=True)
channels.append(channel)
test_session.commit()
test_session.expire_all()
statements = []
engine = test_session.get_bind()
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
statements.append(statement)
event.listen(engine, 'before_cursor_execute', before_cursor_execute)
try:
collections = lib.get_collections(test_session, kind='channel')
finally:
event.remove(engine, 'before_cursor_execute', before_cursor_execute)
assert len(collections) == 6
for channel, data in zip(channels, sorted(collections, key=lambda i: i['name'])):
assert data['channel_id'] == channel.id
assert data['video_count'] == channel.video_count == 2
assert data['total_size'] == channel.total_size > 0
assert data['min_download_frequency'] == DownloadFrequency.weekly
assert data['tag_name'] == channel.tag_name
# BEGIN, collections, channel summaries, tags, downloads: fixed count regardless of channel count.
assert len(statements) <= 5, '\n'.join(statements)
assert not any('file_group' in s for s in statements), 'Channel sizes must come from channel.total_size'
@pytest.mark.asyncio
async def test_domain_collection_summary_columns_follow_archives(
async_client, test_session, test_directory, archive_factory):
"""`collection.item_count` and `collection.total_size` of a Domain Collection are kept current
by triggers as Archives are added, resized, re-assigned and deleted, and the listing reads
them without joining file_group."""
from sqlalchemy import event
from wrolpi.collections import lib
from wrolpi.files.models import FileGroup
archive1 = archive_factory(domain='example.com', title='One', url='https://example.com/1')
archive2 = archive_factory(domain='example.com', title='Two', url='https://example.com/2')
archive3 = archive_factory(domain='other.org', title='Three', url='https://other.org/3')
test_session.commit()
def summaries() -> dict:
test_session.expire_all()
return {c.name: (c.item_count, c.total_size) for c in
test_session.query(Collection).filter_by(kind='domain')}
sizes = {i.file_group.id: i.file_group.size for i in (archive1, archive2, archive3)}
assert all(sizes.values()), 'archives must have a size for this test to mean anything'
assert summaries() == {
'example.com': (2, sizes[archive1.file_group.id] + sizes[archive2.file_group.id]),
'other.org': (1, sizes[archive3.file_group.id]),
}
# A FileGroup's size changes (refresh found the files grew).
test_session.query(FileGroup).filter_by(id=archive1.file_group.id).update({'size': 1_000_000})
test_session.commit()
assert summaries()['example.com'] == (2, 1_000_000 + sizes[archive2.file_group.id])
# An Archive moves to another Domain.
other = test_session.query(Collection).filter_by(name='other.org').one()
test_session.execute(
sa_text('UPDATE archive SET collection_id = :cid WHERE id = :aid'),
{'cid': other.id, 'aid': archive2.id})
test_session.commit()
assert summaries() == {
'example.com': (1, 1_000_000),
'other.org': (2, sizes[archive3.file_group.id] + sizes[archive2.file_group.id]),
}
# An Archive is deleted.
test_session.execute(sa_text('DELETE FROM archive WHERE id = :aid'), {'aid': archive1.id})
test_session.commit()
assert summaries()['example.com'] == (0, 0)
# The listing reads the columns: no file_group in any statement.
statements = []
engine = test_session.get_bind()
listener = lambda conn, cursor, statement, *_: statements.append(statement) # noqa: E731
event.listen(engine, 'before_cursor_execute', listener)
try:
collections = lib.get_collections(test_session, kind='domain')
finally:
event.remove(engine, 'before_cursor_execute', listener)
by_name = {c['name']: c for c in collections}
assert (by_name['example.com']['archive_count'], by_name['example.com']['size']) == (0, 0)
assert (by_name['other.org']['archive_count'], by_name['other.org']['size']) == \
(2, sizes[archive3.file_group.id] + sizes[archive2.file_group.id])
assert not any('file_group' in s for s in statements), '\n'.join(statements)

View file

@ -12,7 +12,8 @@ All statements are idempotent (IF NOT EXISTS).
"""
# Triggers maintaining the summary columns `channel.video_count`, `channel.total_size`,
# `channel.minimum_frequency` and `file_group.effective_datetime`.
# `channel.minimum_frequency`, `collection.item_count`/`collection.total_size` (domains) and
# `file_group.effective_datetime`.
#
# These are re-authored from the old PL/pgSQL triggers (see git history of alembic/versions/).
# The old triggers had known bugs which are fixed here:
@ -70,6 +71,55 @@ TRIGGER_DDL = [
WHERE id = (SELECT channel_id FROM video WHERE file_group_id = new.id);
END
''',
# collection.item_count + collection.total_size follow the Archives of a Domain Collection.
# (Channels keep their summary on the channel table; generic Collections count collection_item.)
'''
CREATE TRIGGER IF NOT EXISTS archive_insert_collection_summary
AFTER INSERT ON archive WHEN new.collection_id IS NOT NULL
BEGIN
UPDATE collection SET
item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id),
total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a
LEFT JOIN file_group fg ON a.file_group_id = fg.id
WHERE a.collection_id = collection.id)
WHERE id = new.collection_id;
END
''',
'''
CREATE TRIGGER IF NOT EXISTS archive_delete_collection_summary
AFTER DELETE ON archive WHEN old.collection_id IS NOT NULL
BEGIN
UPDATE collection SET
item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id),
total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a
LEFT JOIN file_group fg ON a.file_group_id = fg.id
WHERE a.collection_id = collection.id)
WHERE id = old.collection_id;
END
''',
'''
CREATE TRIGGER IF NOT EXISTS archive_update_collection_summary
AFTER UPDATE OF collection_id, file_group_id ON archive
BEGIN
UPDATE collection SET
item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id),
total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a
LEFT JOIN file_group fg ON a.file_group_id = fg.id
WHERE a.collection_id = collection.id)
WHERE id IN (old.collection_id, new.collection_id);
END
''',
'''
CREATE TRIGGER IF NOT EXISTS file_group_size_collection_summary
AFTER UPDATE OF size ON file_group WHEN old.size IS NOT new.size
BEGIN
UPDATE collection SET
total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a
LEFT JOIN file_group fg ON a.file_group_id = fg.id
WHERE a.collection_id = collection.id)
WHERE id = (SELECT collection_id FROM archive WHERE file_group_id = new.id);
END
''',
# channel.minimum_frequency follows the Downloads of the Channel's Collection.
'''
CREATE TRIGGER IF NOT EXISTS download_insert_minimum_frequency

View file

@ -158,6 +158,59 @@ def test_drop_file_group_idempotency_preserves_fts(test_directory):
assert len(hits) == 1
def test_domain_collection_summary_migration_backfills(test_directory):
"""The domain-summary migration installs the archive triggers and backfills the summary
columns of existing Domain Collections from their Archives."""
import sqlite3
from alembic.config import Config
from alembic import command
from wrolpi.db import get_db_uri, get_db_file
from wrolpi.vars import PROJECT_DIR
(test_directory / 'config').mkdir(parents=True, exist_ok=True)
config = Config(str(PROJECT_DIR / 'alembic.ini'))
config.set_main_option('sqlalchemy.url', get_db_uri())
command.upgrade(config, '2026_09_20_1200')
db_file = get_db_file()
directory = str(test_directory)
triggers = {'archive_insert_collection_summary', 'archive_delete_collection_summary',
'archive_update_collection_summary', 'file_group_size_collection_summary'}
with sqlite3.connect(db_file) as conn:
conn.execute('PRAGMA foreign_keys=ON')
# The baseline installs the current trigger DDL, so a fresh DB already has the archive
# triggers. A production DB created before them does not; emulate that.
for name in triggers:
conn.execute(f'DROP TRIGGER {name}')
for id_, name, kind in ((1, 'example.com', 'domain'), (2, 'empty.org', 'domain'), (3, 'chan', 'channel')):
conn.execute('INSERT INTO collection (id, name, kind, item_count, total_size) VALUES (?, ?, ?, 0, 0)',
(id_, name, kind))
for fg_id, stem, size in ((10, 'one', 100), (11, 'two', 250)):
conn.execute(
'INSERT INTO file_group (id, directory, primary_path, files, stem, size, indexed)'
' VALUES (?,?,?,?,?,?,0)',
(fg_id, directory, str(test_directory / f'{stem}.html'), '[]', stem, size),
)
conn.execute('INSERT INTO archive (collection_id, file_group_id) VALUES (1, ?)', (fg_id,))
conn.commit()
# Nothing maintains the columns before the migration.
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (0, 0)
command.upgrade(config, 'head')
with sqlite3.connect(db_file) as conn:
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (2, 350)
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 2').fetchone() == (0, 0)
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 3').fetchone() == (0, 0)
installed = {r[0] for r in conn.execute(
"SELECT name FROM sqlite_master WHERE type='trigger' AND name LIKE '%collection_summary'")}
assert installed == triggers
# The triggers are live after the migration.
conn.execute('DELETE FROM archive WHERE file_group_id = 11')
conn.commit()
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (1, 100)
def test_bootstrap_lock(test_directory):
"""Only one process can hold the bootstrap lock."""
with db_bootstrap.bootstrap_lock() as acquired: