collections: channel listing reads trigger columns and batch-loads tag/downloads

Listing channel collections summed FileGroup sizes through a video join, which reads a
random page of the file_group table per video, and lazy-loaded every collection's tag
and downloads one query at a time. On a large library with a cold disk cache the
listing took over two minutes, past the browser's 60 s timeout, so the Channels page
showed an error.

channel.video_count and channel.total_size are already maintained by triggers
(wrolpi/schema_ddl.py), so read those. Tag and downloads are selectin-loaded, so the
query count no longer grows with the number of collections.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

wrolpi/collections/lib.py

@@ -8,7 +8,7 @@ import pathlib
 from typing import List, Optional, Dict
 
 from sqlalchemy import func
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, selectinload
 
 from wrolpi import flags
 from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
@@ -72,10 +72,12 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
     """
     # Local imports to avoid circular import: collections -> archive/videos -> collections
     from modules.archive import Archive
-    from modules.videos.models import Video, Channel
+    from modules.videos.models import Channel
     from wrolpi.files.models import FileGroup
 
-    query = session.query(Collection)
+    # Serializing a Collection reads its tag and downloads; load them in two batch queries rather
+    # than one lazy query per Collection (thousands of Collections on a large library).
+    query = session.query(Collection).options(selectinload(Collection.tag), selectinload(Collection.downloads))
 
     if kind:
         query = query.filter(Collection.kind == kind)
@@ -105,21 +107,20 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
 
         domain_stats_map = {s.collection_id: s for s in domain_stats}
 
-    # Batch query: Get channel info and video stats for all channel collections at once
+    # Batch query: Get channel info and video stats for all channel collections at once.
+    # `channel.video_count` and `channel.total_size` are maintained by triggers (see
+    # wrolpi/schema_ddl.py); summing FileGroup sizes here instead read a random page of the
+    # file_group table per video, which took minutes from a cold disk cache on a large library.
     channel_stats_map = {}
     if channel_ids:
         channel_stats = session.query(
             Channel.collection_id,
             Channel.id.label('channel_id'),
-            func.count(Video.id).label('video_count'),
-            func.coalesce(func.sum(FileGroup.size), 0).label('size')
-        ).outerjoin(
-            Video, Video.channel_id == Channel.id
-        ).outerjoin(
-            FileGroup, FileGroup.id == Video.file_group_id
+            Channel.video_count,
+            Channel.total_size.label('size'),
         ).filter(
             Channel.collection_id.in_(channel_ids)
-        ).group_by(Channel.collection_id, Channel.id).all()
+        ).all()
 
         channel_stats_map = {s.collection_id: s for s in channel_stats}
 

wrolpi/collections/test/test_api.py

@@ -577,3 +577,53 @@ class TestCollectionTagging:
         # No conflict checking needed without a directory
         assert data['conflict'] is False
         assert data['conflict_message'] is None
+
+
+@pytest.mark.asyncio
+async def test_get_channel_collections_uses_summary_columns_and_batches_queries(
+        async_client, test_session, test_directory, channel_factory, video_factory, tag_factory):
+    """Listing channel collections reads the trigger-maintained channel summary columns and
+    loads downloads and tags in batches, so the query count does not grow with the channel count.
+
+    A production DB was measured at 143 s cold for this listing: summing FileGroup sizes touched a
+    random page of the 2 GB file_group table per video, and each collection lazy-loaded its
+    downloads and tag one query at a time.
+    """
+    from sqlalchemy import event
+    from wrolpi.collections import lib
+    from wrolpi.downloader import DownloadFrequency
+
+    await tag_factory('one')
+    await tag_factory('two')
+    channels = []
+    for i in range(6):
+        tag_name = ('one', 'two', None)[i % 3]
+        channel = channel_factory(name=f'Channel {i}', download_frequency=DownloadFrequency.weekly, tag_name=tag_name)
+        video_factory(channel_id=channel.id, with_video_file=True)
+        video_factory(channel_id=channel.id, with_video_file=True)
+        channels.append(channel)
+    test_session.commit()
+    test_session.expire_all()
+
+    statements = []
+    engine = test_session.get_bind()
+
+    def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
+        statements.append(statement)
+
+    event.listen(engine, 'before_cursor_execute', before_cursor_execute)
+    try:
+        collections = lib.get_collections(test_session, kind='channel')
+    finally:
+        event.remove(engine, 'before_cursor_execute', before_cursor_execute)
+
+    assert len(collections) == 6
+    for channel, data in zip(channels, sorted(collections, key=lambda i: i['name'])):
+        assert data['channel_id'] == channel.id
+        assert data['video_count'] == channel.video_count == 2
+        assert data['total_size'] == channel.total_size > 0
+        assert data['min_download_frequency'] == DownloadFrequency.weekly
+        assert data['tag_name'] == channel.tag_name
+    # BEGIN, collections, channel summaries, tags, downloads: fixed count regardless of channel count.
+    assert len(statements) <= 5, '\n'.join(statements)
+    assert not any('file_group' in s for s in statements), 'Channel sizes must come from channel.total_size'