From c95068e9b192dbda523b7927df43cab4dde97437 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 27 Sep 2026 10:48:42 -0600 Subject: [PATCH] collections: channel listing reads trigger columns and batch-loads tag/downloads Listing channel collections summed FileGroup sizes through a video join, which reads a random page of the file_group table per video, and lazy-loaded every collection's tag and downloads one query at a time. On a large library with a cold disk cache the listing took over two minutes, past the browser's 60 s timeout, so the Channels page showed an error. channel.video_count and channel.total_size are already maintained by triggers (wrolpi/schema_ddl.py), so read those. Tag and downloads are selectin-loaded, so the query count no longer grows with the number of collections. Co-Authored-By: Claude Fable 5.1 --- wrolpi/collections/lib.py | 23 ++++++------- wrolpi/collections/test/test_api.py | 50 +++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/wrolpi/collections/lib.py b/wrolpi/collections/lib.py index 50fcd5c6..65b7d66c 100644 --- a/wrolpi/collections/lib.py +++ b/wrolpi/collections/lib.py @@ -8,7 +8,7 @@ import pathlib from typing import List, Optional, Dict from sqlalchemy import func -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, selectinload from wrolpi import flags from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task @@ -72,10 +72,12 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]: """ # Local imports to avoid circular import: collections -> archive/videos -> collections from modules.archive import Archive - from modules.videos.models import Video, Channel + from modules.videos.models import Channel from wrolpi.files.models import FileGroup - query = session.query(Collection) + # Serializing a Collection reads its tag and downloads; load them in two batch queries rather + # than one lazy query per Collection (thousands of Collections on a large library). + query = session.query(Collection).options(selectinload(Collection.tag), selectinload(Collection.downloads)) if kind: query = query.filter(Collection.kind == kind) @@ -105,21 +107,20 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]: domain_stats_map = {s.collection_id: s for s in domain_stats} - # Batch query: Get channel info and video stats for all channel collections at once + # Batch query: Get channel info and video stats for all channel collections at once. + # `channel.video_count` and `channel.total_size` are maintained by triggers (see + # wrolpi/schema_ddl.py); summing FileGroup sizes here instead read a random page of the + # file_group table per video, which took minutes from a cold disk cache on a large library. channel_stats_map = {} if channel_ids: channel_stats = session.query( Channel.collection_id, Channel.id.label('channel_id'), - func.count(Video.id).label('video_count'), - func.coalesce(func.sum(FileGroup.size), 0).label('size') - ).outerjoin( - Video, Video.channel_id == Channel.id - ).outerjoin( - FileGroup, FileGroup.id == Video.file_group_id + Channel.video_count, + Channel.total_size.label('size'), ).filter( Channel.collection_id.in_(channel_ids) - ).group_by(Channel.collection_id, Channel.id).all() + ).all() channel_stats_map = {s.collection_id: s for s in channel_stats} diff --git a/wrolpi/collections/test/test_api.py b/wrolpi/collections/test/test_api.py index ea788706..e31d70c7 100644 --- a/wrolpi/collections/test/test_api.py +++ b/wrolpi/collections/test/test_api.py @@ -577,3 +577,53 @@ class TestCollectionTagging: # No conflict checking needed without a directory assert data['conflict'] is False assert data['conflict_message'] is None + + +@pytest.mark.asyncio +async def test_get_channel_collections_uses_summary_columns_and_batches_queries( + async_client, test_session, test_directory, channel_factory, video_factory, tag_factory): + """Listing channel collections reads the trigger-maintained channel summary columns and + loads downloads and tags in batches, so the query count does not grow with the channel count. + + A production DB was measured at 143 s cold for this listing: summing FileGroup sizes touched a + random page of the 2 GB file_group table per video, and each collection lazy-loaded its + downloads and tag one query at a time. + """ + from sqlalchemy import event + from wrolpi.collections import lib + from wrolpi.downloader import DownloadFrequency + + await tag_factory('one') + await tag_factory('two') + channels = [] + for i in range(6): + tag_name = ('one', 'two', None)[i % 3] + channel = channel_factory(name=f'Channel {i}', download_frequency=DownloadFrequency.weekly, tag_name=tag_name) + video_factory(channel_id=channel.id, with_video_file=True) + video_factory(channel_id=channel.id, with_video_file=True) + channels.append(channel) + test_session.commit() + test_session.expire_all() + + statements = [] + engine = test_session.get_bind() + + def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + event.listen(engine, 'before_cursor_execute', before_cursor_execute) + try: + collections = lib.get_collections(test_session, kind='channel') + finally: + event.remove(engine, 'before_cursor_execute', before_cursor_execute) + + assert len(collections) == 6 + for channel, data in zip(channels, sorted(collections, key=lambda i: i['name'])): + assert data['channel_id'] == channel.id + assert data['video_count'] == channel.video_count == 2 + assert data['total_size'] == channel.total_size > 0 + assert data['min_download_frequency'] == DownloadFrequency.weekly + assert data['tag_name'] == channel.tag_name + # BEGIN, collections, channel summaries, tags, downloads: fixed count regardless of channel count. + assert len(statements) <= 5, '\n'.join(statements) + assert not any('file_group' in s for s in statements), 'Channel sizes must come from channel.total_size'