From 20be5dfc2724a2b62cad7a3abe935b6162e463dc Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:25:12 -0600 Subject: [PATCH] collections: channel and domain listings read trigger-maintained summary columns (#739) * collections: channel listing reads trigger columns and batch-loads tag/downloads Listing channel collections summed FileGroup sizes through a video join, which reads a random page of the file_group table per video, and lazy-loaded every collection's tag and downloads one query at a time. On a large library with a cold disk cache the listing took over two minutes, past the browser's 60 s timeout, so the Channels page showed an error. channel.video_count and channel.total_size are already maintained by triggers (wrolpi/schema_ddl.py), so read those. Tag and downloads are selectin-loaded, so the query count no longer grows with the number of collections. Co-Authored-By: Claude Fable 5.1 * collections: domain listing reads trigger-maintained summary columns Domain Collections summed file_group.size through an archive join on every listing, one random page read per archive (about 9 s from a cold disk cache on a large library). New triggers keep collection.item_count and collection.total_size current for Domain Collections, as the channel triggers already do for Channels, and the listing reads those columns. A migration installs the triggers and backfills existing Domain Collections. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- ...26_09_27_1200_domain_collection_summary.py | 45 +++++++ wrolpi/collections/lib.py | 50 +++----- wrolpi/collections/test/test_api.py | 115 ++++++++++++++++++ wrolpi/schema_ddl.py | 52 +++++++- wrolpi/test/test_db_bootstrap.py | 53 ++++++++ 5 files changed, 281 insertions(+), 34 deletions(-) create mode 100644 alembic/versions/2026_09_27_1200_domain_collection_summary.py diff --git a/alembic/versions/2026_09_27_1200_domain_collection_summary.py b/alembic/versions/2026_09_27_1200_domain_collection_summary.py new file mode 100644 index 00000000..405dbfe1 --- /dev/null +++ b/alembic/versions/2026_09_27_1200_domain_collection_summary.py @@ -0,0 +1,45 @@ +"""Maintain collection.item_count/total_size for Domain Collections with triggers. + +Listing Domain Collections summed file_group.size through an archive join on every request, +one random page read per archive. The new triggers keep the summary columns current +(as the channel triggers already do for Channels); this migration installs them and +backfills existing Domain Collections. + +Revision ID: 2026_09_27_1200 +Revises: 2026_09_20_1200 +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = '2026_09_27_1200' +down_revision = '2026_09_20_1200' +branch_labels = None +depends_on = None + +TRIGGERS = ( + 'archive_insert_collection_summary', + 'archive_delete_collection_summary', + 'archive_update_collection_summary', + 'file_group_size_collection_summary', +) + + +def upgrade(): + from wrolpi.schema_ddl import install_raw_ddl + + # Idempotent: only the new triggers are created. + install_raw_ddl(op.get_bind()) + + op.execute(''' + UPDATE collection SET + item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id), + total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a + LEFT JOIN file_group fg ON a.file_group_id = fg.id + WHERE a.collection_id = collection.id) + WHERE kind = 'domain' + ''') + + +def downgrade(): + for name in TRIGGERS: + op.execute(f'DROP TRIGGER IF EXISTS {name}') diff --git a/wrolpi/collections/lib.py b/wrolpi/collections/lib.py index 50fcd5c6..03082b42 100644 --- a/wrolpi/collections/lib.py +++ b/wrolpi/collections/lib.py @@ -8,7 +8,7 @@ import pathlib from typing import List, Optional, Dict from sqlalchemy import func -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, selectinload from wrolpi import flags from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task @@ -70,12 +70,12 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]: Returns: List of collection dicts with statistics for each collection type """ - # Local imports to avoid circular import: collections -> archive/videos -> collections - from modules.archive import Archive - from modules.videos.models import Video, Channel - from wrolpi.files.models import FileGroup + # Local imports to avoid circular import: collections -> videos -> collections + from modules.videos.models import Channel - query = session.query(Collection) + # Serializing a Collection reads its tag and downloads; load them in two batch queries rather + # than one lazy query per Collection (thousands of Collections on a large library). + query = session.query(Collection).options(selectinload(Collection.tag), selectinload(Collection.downloads)) if kind: query = query.filter(Collection.kind == kind) @@ -86,40 +86,23 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]: return [] # Separate collections by kind for batch processing - domain_ids = [c.id for c in collections if c.kind == 'domain'] channel_ids = [c.id for c in collections if c.kind == 'channel'] generic_ids = [c.id for c in collections if c.kind not in ('domain', 'channel')] - # Batch query: Get archive stats for all domain collections at once - domain_stats_map = {} - if domain_ids: - domain_stats = session.query( - Archive.collection_id, - func.count(Archive.id).label('archive_count'), - func.coalesce(func.sum(FileGroup.size), 0).label('size') - ).outerjoin( - FileGroup, FileGroup.id == Archive.file_group_id - ).filter( - Archive.collection_id.in_(domain_ids) - ).group_by(Archive.collection_id).all() - - domain_stats_map = {s.collection_id: s for s in domain_stats} - - # Batch query: Get channel info and video stats for all channel collections at once + # Batch query: Get channel info and video stats for all channel collections at once. + # `channel.video_count` and `channel.total_size` are maintained by triggers (see + # wrolpi/schema_ddl.py); summing FileGroup sizes here instead read a random page of the + # file_group table per video, which took minutes from a cold disk cache on a large library. channel_stats_map = {} if channel_ids: channel_stats = session.query( Channel.collection_id, Channel.id.label('channel_id'), - func.count(Video.id).label('video_count'), - func.coalesce(func.sum(FileGroup.size), 0).label('size') - ).outerjoin( - Video, Video.channel_id == Channel.id - ).outerjoin( - FileGroup, FileGroup.id == Video.file_group_id + Channel.video_count, + Channel.total_size.label('size'), ).filter( Channel.collection_id.in_(channel_ids) - ).group_by(Channel.collection_id, Channel.id).all() + ).all() channel_stats_map = {s.collection_id: s for s in channel_stats} @@ -151,9 +134,10 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]: # Add type-specific statistics from batch query results if collection.kind == 'domain': - stats = domain_stats_map.get(collection.id) - data['archive_count'] = stats.archive_count if stats else 0 - data['size'] = int(stats.size) if stats else 0 + # `collection.item_count` and `collection.total_size` are maintained by the archive + # triggers (see wrolpi/schema_ddl.py); no join through file_group needed. + data['archive_count'] = collection.item_count + data['size'] = collection.total_size data['domain'] = data['name'] # Alias for backward compatibility elif collection.kind == 'channel': diff --git a/wrolpi/collections/test/test_api.py b/wrolpi/collections/test/test_api.py index ea788706..7554d566 100644 --- a/wrolpi/collections/test/test_api.py +++ b/wrolpi/collections/test/test_api.py @@ -2,6 +2,7 @@ from http import HTTPStatus import pytest +from sqlalchemy import text as sa_text from wrolpi.collections.models import Collection @@ -577,3 +578,117 @@ class TestCollectionTagging: # No conflict checking needed without a directory assert data['conflict'] is False assert data['conflict_message'] is None + + +@pytest.mark.asyncio +async def test_get_channel_collections_uses_summary_columns_and_batches_queries( + async_client, test_session, test_directory, channel_factory, video_factory, tag_factory): + """Listing channel collections reads the trigger-maintained channel summary columns and + loads downloads and tags in batches, so the query count does not grow with the channel count. + + A production DB was measured at 143 s cold for this listing: summing FileGroup sizes touched a + random page of the 2 GB file_group table per video, and each collection lazy-loaded its + downloads and tag one query at a time. + """ + from sqlalchemy import event + from wrolpi.collections import lib + from wrolpi.downloader import DownloadFrequency + + await tag_factory('one') + await tag_factory('two') + channels = [] + for i in range(6): + tag_name = ('one', 'two', None)[i % 3] + channel = channel_factory(name=f'Channel {i}', download_frequency=DownloadFrequency.weekly, tag_name=tag_name) + video_factory(channel_id=channel.id, with_video_file=True) + video_factory(channel_id=channel.id, with_video_file=True) + channels.append(channel) + test_session.commit() + test_session.expire_all() + + statements = [] + engine = test_session.get_bind() + + def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + event.listen(engine, 'before_cursor_execute', before_cursor_execute) + try: + collections = lib.get_collections(test_session, kind='channel') + finally: + event.remove(engine, 'before_cursor_execute', before_cursor_execute) + + assert len(collections) == 6 + for channel, data in zip(channels, sorted(collections, key=lambda i: i['name'])): + assert data['channel_id'] == channel.id + assert data['video_count'] == channel.video_count == 2 + assert data['total_size'] == channel.total_size > 0 + assert data['min_download_frequency'] == DownloadFrequency.weekly + assert data['tag_name'] == channel.tag_name + # BEGIN, collections, channel summaries, tags, downloads: fixed count regardless of channel count. + assert len(statements) <= 5, '\n'.join(statements) + assert not any('file_group' in s for s in statements), 'Channel sizes must come from channel.total_size' + + +@pytest.mark.asyncio +async def test_domain_collection_summary_columns_follow_archives( + async_client, test_session, test_directory, archive_factory): + """`collection.item_count` and `collection.total_size` of a Domain Collection are kept current + by triggers as Archives are added, resized, re-assigned and deleted, and the listing reads + them without joining file_group.""" + from sqlalchemy import event + from wrolpi.collections import lib + from wrolpi.files.models import FileGroup + + archive1 = archive_factory(domain='example.com', title='One', url='https://example.com/1') + archive2 = archive_factory(domain='example.com', title='Two', url='https://example.com/2') + archive3 = archive_factory(domain='other.org', title='Three', url='https://other.org/3') + test_session.commit() + + def summaries() -> dict: + test_session.expire_all() + return {c.name: (c.item_count, c.total_size) for c in + test_session.query(Collection).filter_by(kind='domain')} + + sizes = {i.file_group.id: i.file_group.size for i in (archive1, archive2, archive3)} + assert all(sizes.values()), 'archives must have a size for this test to mean anything' + assert summaries() == { + 'example.com': (2, sizes[archive1.file_group.id] + sizes[archive2.file_group.id]), + 'other.org': (1, sizes[archive3.file_group.id]), + } + + # A FileGroup's size changes (refresh found the files grew). + test_session.query(FileGroup).filter_by(id=archive1.file_group.id).update({'size': 1_000_000}) + test_session.commit() + assert summaries()['example.com'] == (2, 1_000_000 + sizes[archive2.file_group.id]) + + # An Archive moves to another Domain. + other = test_session.query(Collection).filter_by(name='other.org').one() + test_session.execute( + sa_text('UPDATE archive SET collection_id = :cid WHERE id = :aid'), + {'cid': other.id, 'aid': archive2.id}) + test_session.commit() + assert summaries() == { + 'example.com': (1, 1_000_000), + 'other.org': (2, sizes[archive3.file_group.id] + sizes[archive2.file_group.id]), + } + + # An Archive is deleted. + test_session.execute(sa_text('DELETE FROM archive WHERE id = :aid'), {'aid': archive1.id}) + test_session.commit() + assert summaries()['example.com'] == (0, 0) + + # The listing reads the columns: no file_group in any statement. + statements = [] + engine = test_session.get_bind() + listener = lambda conn, cursor, statement, *_: statements.append(statement) # noqa: E731 + event.listen(engine, 'before_cursor_execute', listener) + try: + collections = lib.get_collections(test_session, kind='domain') + finally: + event.remove(engine, 'before_cursor_execute', listener) + by_name = {c['name']: c for c in collections} + assert (by_name['example.com']['archive_count'], by_name['example.com']['size']) == (0, 0) + assert (by_name['other.org']['archive_count'], by_name['other.org']['size']) == \ + (2, sizes[archive3.file_group.id] + sizes[archive2.file_group.id]) + assert not any('file_group' in s for s in statements), '\n'.join(statements) diff --git a/wrolpi/schema_ddl.py b/wrolpi/schema_ddl.py index b309b012..4fbf1fb1 100644 --- a/wrolpi/schema_ddl.py +++ b/wrolpi/schema_ddl.py @@ -12,7 +12,8 @@ All statements are idempotent (IF NOT EXISTS). """ # Triggers maintaining the summary columns `channel.video_count`, `channel.total_size`, -# `channel.minimum_frequency` and `file_group.effective_datetime`. +# `channel.minimum_frequency`, `collection.item_count`/`collection.total_size` (domains) and +# `file_group.effective_datetime`. # # These are re-authored from the old PL/pgSQL triggers (see git history of alembic/versions/). # The old triggers had known bugs which are fixed here: @@ -70,6 +71,55 @@ TRIGGER_DDL = [ WHERE id = (SELECT channel_id FROM video WHERE file_group_id = new.id); END ''', + # collection.item_count + collection.total_size follow the Archives of a Domain Collection. + # (Channels keep their summary on the channel table; generic Collections count collection_item.) + ''' + CREATE TRIGGER IF NOT EXISTS archive_insert_collection_summary + AFTER INSERT ON archive WHEN new.collection_id IS NOT NULL + BEGIN + UPDATE collection SET + item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id), + total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a + LEFT JOIN file_group fg ON a.file_group_id = fg.id + WHERE a.collection_id = collection.id) + WHERE id = new.collection_id; + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS archive_delete_collection_summary + AFTER DELETE ON archive WHEN old.collection_id IS NOT NULL + BEGIN + UPDATE collection SET + item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id), + total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a + LEFT JOIN file_group fg ON a.file_group_id = fg.id + WHERE a.collection_id = collection.id) + WHERE id = old.collection_id; + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS archive_update_collection_summary + AFTER UPDATE OF collection_id, file_group_id ON archive + BEGIN + UPDATE collection SET + item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id), + total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a + LEFT JOIN file_group fg ON a.file_group_id = fg.id + WHERE a.collection_id = collection.id) + WHERE id IN (old.collection_id, new.collection_id); + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS file_group_size_collection_summary + AFTER UPDATE OF size ON file_group WHEN old.size IS NOT new.size + BEGIN + UPDATE collection SET + total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a + LEFT JOIN file_group fg ON a.file_group_id = fg.id + WHERE a.collection_id = collection.id) + WHERE id = (SELECT collection_id FROM archive WHERE file_group_id = new.id); + END + ''', # channel.minimum_frequency follows the Downloads of the Channel's Collection. ''' CREATE TRIGGER IF NOT EXISTS download_insert_minimum_frequency diff --git a/wrolpi/test/test_db_bootstrap.py b/wrolpi/test/test_db_bootstrap.py index 1098b22b..eac08b5b 100644 --- a/wrolpi/test/test_db_bootstrap.py +++ b/wrolpi/test/test_db_bootstrap.py @@ -158,6 +158,59 @@ def test_drop_file_group_idempotency_preserves_fts(test_directory): assert len(hits) == 1 +def test_domain_collection_summary_migration_backfills(test_directory): + """The domain-summary migration installs the archive triggers and backfills the summary + columns of existing Domain Collections from their Archives.""" + import sqlite3 + from alembic.config import Config + from alembic import command + from wrolpi.db import get_db_uri, get_db_file + from wrolpi.vars import PROJECT_DIR + + (test_directory / 'config').mkdir(parents=True, exist_ok=True) + config = Config(str(PROJECT_DIR / 'alembic.ini')) + config.set_main_option('sqlalchemy.url', get_db_uri()) + command.upgrade(config, '2026_09_20_1200') + + db_file = get_db_file() + directory = str(test_directory) + triggers = {'archive_insert_collection_summary', 'archive_delete_collection_summary', + 'archive_update_collection_summary', 'file_group_size_collection_summary'} + with sqlite3.connect(db_file) as conn: + conn.execute('PRAGMA foreign_keys=ON') + # The baseline installs the current trigger DDL, so a fresh DB already has the archive + # triggers. A production DB created before them does not; emulate that. + for name in triggers: + conn.execute(f'DROP TRIGGER {name}') + for id_, name, kind in ((1, 'example.com', 'domain'), (2, 'empty.org', 'domain'), (3, 'chan', 'channel')): + conn.execute('INSERT INTO collection (id, name, kind, item_count, total_size) VALUES (?, ?, ?, 0, 0)', + (id_, name, kind)) + for fg_id, stem, size in ((10, 'one', 100), (11, 'two', 250)): + conn.execute( + 'INSERT INTO file_group (id, directory, primary_path, files, stem, size, indexed)' + ' VALUES (?,?,?,?,?,?,0)', + (fg_id, directory, str(test_directory / f'{stem}.html'), '[]', stem, size), + ) + conn.execute('INSERT INTO archive (collection_id, file_group_id) VALUES (1, ?)', (fg_id,)) + conn.commit() + # Nothing maintains the columns before the migration. + assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (0, 0) + + command.upgrade(config, 'head') + + with sqlite3.connect(db_file) as conn: + assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (2, 350) + assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 2').fetchone() == (0, 0) + assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 3').fetchone() == (0, 0) + installed = {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='trigger' AND name LIKE '%collection_summary'")} + assert installed == triggers + # The triggers are live after the migration. + conn.execute('DELETE FROM archive WHERE file_group_id = 11') + conn.commit() + assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (1, 100) + + def test_bootstrap_lock(test_directory): """Only one process can hold the bootstrap lock.""" with db_bootstrap.bootstrap_lock() as acquired: