mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 15:36:18 -05:00
collections: channel and domain listings read trigger-maintained summary columns (#739)
* collections: channel listing reads trigger columns and batch-loads tag/downloads Listing channel collections summed FileGroup sizes through a video join, which reads a random page of the file_group table per video, and lazy-loaded every collection's tag and downloads one query at a time. On a large library with a cold disk cache the listing took over two minutes, past the browser's 60 s timeout, so the Channels page showed an error. channel.video_count and channel.total_size are already maintained by triggers (wrolpi/schema_ddl.py), so read those. Tag and downloads are selectin-loaded, so the query count no longer grows with the number of collections. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * collections: domain listing reads trigger-maintained summary columns Domain Collections summed file_group.size through an archive join on every listing, one random page read per archive (about 9 s from a cold disk cache on a large library). New triggers keep collection.item_count and collection.total_size current for Domain Collections, as the channel triggers already do for Channels, and the listing reads those columns. A migration installs the triggers and backfills existing Domain Collections. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
4fea50943b
commit
20be5dfc27
5 changed files with 281 additions and 34 deletions
|
|
@ -0,0 +1,45 @@
|
|||
"""Maintain collection.item_count/total_size for Domain Collections with triggers.
|
||||
|
||||
Listing Domain Collections summed file_group.size through an archive join on every request,
|
||||
one random page read per archive. The new triggers keep the summary columns current
|
||||
(as the channel triggers already do for Channels); this migration installs them and
|
||||
backfills existing Domain Collections.
|
||||
|
||||
Revision ID: 2026_09_27_1200
|
||||
Revises: 2026_09_20_1200
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '2026_09_27_1200'
|
||||
down_revision = '2026_09_20_1200'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
TRIGGERS = (
|
||||
'archive_insert_collection_summary',
|
||||
'archive_delete_collection_summary',
|
||||
'archive_update_collection_summary',
|
||||
'file_group_size_collection_summary',
|
||||
)
|
||||
|
||||
|
||||
def upgrade():
|
||||
from wrolpi.schema_ddl import install_raw_ddl
|
||||
|
||||
# Idempotent: only the new triggers are created.
|
||||
install_raw_ddl(op.get_bind())
|
||||
|
||||
op.execute('''
|
||||
UPDATE collection SET
|
||||
item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id),
|
||||
total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a
|
||||
LEFT JOIN file_group fg ON a.file_group_id = fg.id
|
||||
WHERE a.collection_id = collection.id)
|
||||
WHERE kind = 'domain'
|
||||
''')
|
||||
|
||||
|
||||
def downgrade():
|
||||
for name in TRIGGERS:
|
||||
op.execute(f'DROP TRIGGER IF EXISTS {name}')
|
||||
|
|
@ -8,7 +8,7 @@ import pathlib
|
|||
from typing import List, Optional, Dict
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm import Session, selectinload
|
||||
|
||||
from wrolpi import flags
|
||||
from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
|
||||
|
|
@ -70,12 +70,12 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
|
|||
Returns:
|
||||
List of collection dicts with statistics for each collection type
|
||||
"""
|
||||
# Local imports to avoid circular import: collections -> archive/videos -> collections
|
||||
from modules.archive import Archive
|
||||
from modules.videos.models import Video, Channel
|
||||
from wrolpi.files.models import FileGroup
|
||||
# Local imports to avoid circular import: collections -> videos -> collections
|
||||
from modules.videos.models import Channel
|
||||
|
||||
query = session.query(Collection)
|
||||
# Serializing a Collection reads its tag and downloads; load them in two batch queries rather
|
||||
# than one lazy query per Collection (thousands of Collections on a large library).
|
||||
query = session.query(Collection).options(selectinload(Collection.tag), selectinload(Collection.downloads))
|
||||
|
||||
if kind:
|
||||
query = query.filter(Collection.kind == kind)
|
||||
|
|
@ -86,40 +86,23 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
|
|||
return []
|
||||
|
||||
# Separate collections by kind for batch processing
|
||||
domain_ids = [c.id for c in collections if c.kind == 'domain']
|
||||
channel_ids = [c.id for c in collections if c.kind == 'channel']
|
||||
generic_ids = [c.id for c in collections if c.kind not in ('domain', 'channel')]
|
||||
|
||||
# Batch query: Get archive stats for all domain collections at once
|
||||
domain_stats_map = {}
|
||||
if domain_ids:
|
||||
domain_stats = session.query(
|
||||
Archive.collection_id,
|
||||
func.count(Archive.id).label('archive_count'),
|
||||
func.coalesce(func.sum(FileGroup.size), 0).label('size')
|
||||
).outerjoin(
|
||||
FileGroup, FileGroup.id == Archive.file_group_id
|
||||
).filter(
|
||||
Archive.collection_id.in_(domain_ids)
|
||||
).group_by(Archive.collection_id).all()
|
||||
|
||||
domain_stats_map = {s.collection_id: s for s in domain_stats}
|
||||
|
||||
# Batch query: Get channel info and video stats for all channel collections at once
|
||||
# Batch query: Get channel info and video stats for all channel collections at once.
|
||||
# `channel.video_count` and `channel.total_size` are maintained by triggers (see
|
||||
# wrolpi/schema_ddl.py); summing FileGroup sizes here instead read a random page of the
|
||||
# file_group table per video, which took minutes from a cold disk cache on a large library.
|
||||
channel_stats_map = {}
|
||||
if channel_ids:
|
||||
channel_stats = session.query(
|
||||
Channel.collection_id,
|
||||
Channel.id.label('channel_id'),
|
||||
func.count(Video.id).label('video_count'),
|
||||
func.coalesce(func.sum(FileGroup.size), 0).label('size')
|
||||
).outerjoin(
|
||||
Video, Video.channel_id == Channel.id
|
||||
).outerjoin(
|
||||
FileGroup, FileGroup.id == Video.file_group_id
|
||||
Channel.video_count,
|
||||
Channel.total_size.label('size'),
|
||||
).filter(
|
||||
Channel.collection_id.in_(channel_ids)
|
||||
).group_by(Channel.collection_id, Channel.id).all()
|
||||
).all()
|
||||
|
||||
channel_stats_map = {s.collection_id: s for s in channel_stats}
|
||||
|
||||
|
|
@ -151,9 +134,10 @@ def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
|
|||
|
||||
# Add type-specific statistics from batch query results
|
||||
if collection.kind == 'domain':
|
||||
stats = domain_stats_map.get(collection.id)
|
||||
data['archive_count'] = stats.archive_count if stats else 0
|
||||
data['size'] = int(stats.size) if stats else 0
|
||||
# `collection.item_count` and `collection.total_size` are maintained by the archive
|
||||
# triggers (see wrolpi/schema_ddl.py); no join through file_group needed.
|
||||
data['archive_count'] = collection.item_count
|
||||
data['size'] = collection.total_size
|
||||
data['domain'] = data['name'] # Alias for backward compatibility
|
||||
|
||||
elif collection.kind == 'channel':
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text as sa_text
|
||||
|
||||
from wrolpi.collections.models import Collection
|
||||
|
||||
|
|
@ -577,3 +578,117 @@ class TestCollectionTagging:
|
|||
# No conflict checking needed without a directory
|
||||
assert data['conflict'] is False
|
||||
assert data['conflict_message'] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_channel_collections_uses_summary_columns_and_batches_queries(
|
||||
async_client, test_session, test_directory, channel_factory, video_factory, tag_factory):
|
||||
"""Listing channel collections reads the trigger-maintained channel summary columns and
|
||||
loads downloads and tags in batches, so the query count does not grow with the channel count.
|
||||
|
||||
A production DB was measured at 143 s cold for this listing: summing FileGroup sizes touched a
|
||||
random page of the 2 GB file_group table per video, and each collection lazy-loaded its
|
||||
downloads and tag one query at a time.
|
||||
"""
|
||||
from sqlalchemy import event
|
||||
from wrolpi.collections import lib
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
|
||||
await tag_factory('one')
|
||||
await tag_factory('two')
|
||||
channels = []
|
||||
for i in range(6):
|
||||
tag_name = ('one', 'two', None)[i % 3]
|
||||
channel = channel_factory(name=f'Channel {i}', download_frequency=DownloadFrequency.weekly, tag_name=tag_name)
|
||||
video_factory(channel_id=channel.id, with_video_file=True)
|
||||
video_factory(channel_id=channel.id, with_video_file=True)
|
||||
channels.append(channel)
|
||||
test_session.commit()
|
||||
test_session.expire_all()
|
||||
|
||||
statements = []
|
||||
engine = test_session.get_bind()
|
||||
|
||||
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
|
||||
statements.append(statement)
|
||||
|
||||
event.listen(engine, 'before_cursor_execute', before_cursor_execute)
|
||||
try:
|
||||
collections = lib.get_collections(test_session, kind='channel')
|
||||
finally:
|
||||
event.remove(engine, 'before_cursor_execute', before_cursor_execute)
|
||||
|
||||
assert len(collections) == 6
|
||||
for channel, data in zip(channels, sorted(collections, key=lambda i: i['name'])):
|
||||
assert data['channel_id'] == channel.id
|
||||
assert data['video_count'] == channel.video_count == 2
|
||||
assert data['total_size'] == channel.total_size > 0
|
||||
assert data['min_download_frequency'] == DownloadFrequency.weekly
|
||||
assert data['tag_name'] == channel.tag_name
|
||||
# BEGIN, collections, channel summaries, tags, downloads: fixed count regardless of channel count.
|
||||
assert len(statements) <= 5, '\n'.join(statements)
|
||||
assert not any('file_group' in s for s in statements), 'Channel sizes must come from channel.total_size'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_domain_collection_summary_columns_follow_archives(
|
||||
async_client, test_session, test_directory, archive_factory):
|
||||
"""`collection.item_count` and `collection.total_size` of a Domain Collection are kept current
|
||||
by triggers as Archives are added, resized, re-assigned and deleted, and the listing reads
|
||||
them without joining file_group."""
|
||||
from sqlalchemy import event
|
||||
from wrolpi.collections import lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
archive1 = archive_factory(domain='example.com', title='One', url='https://example.com/1')
|
||||
archive2 = archive_factory(domain='example.com', title='Two', url='https://example.com/2')
|
||||
archive3 = archive_factory(domain='other.org', title='Three', url='https://other.org/3')
|
||||
test_session.commit()
|
||||
|
||||
def summaries() -> dict:
|
||||
test_session.expire_all()
|
||||
return {c.name: (c.item_count, c.total_size) for c in
|
||||
test_session.query(Collection).filter_by(kind='domain')}
|
||||
|
||||
sizes = {i.file_group.id: i.file_group.size for i in (archive1, archive2, archive3)}
|
||||
assert all(sizes.values()), 'archives must have a size for this test to mean anything'
|
||||
assert summaries() == {
|
||||
'example.com': (2, sizes[archive1.file_group.id] + sizes[archive2.file_group.id]),
|
||||
'other.org': (1, sizes[archive3.file_group.id]),
|
||||
}
|
||||
|
||||
# A FileGroup's size changes (refresh found the files grew).
|
||||
test_session.query(FileGroup).filter_by(id=archive1.file_group.id).update({'size': 1_000_000})
|
||||
test_session.commit()
|
||||
assert summaries()['example.com'] == (2, 1_000_000 + sizes[archive2.file_group.id])
|
||||
|
||||
# An Archive moves to another Domain.
|
||||
other = test_session.query(Collection).filter_by(name='other.org').one()
|
||||
test_session.execute(
|
||||
sa_text('UPDATE archive SET collection_id = :cid WHERE id = :aid'),
|
||||
{'cid': other.id, 'aid': archive2.id})
|
||||
test_session.commit()
|
||||
assert summaries() == {
|
||||
'example.com': (1, 1_000_000),
|
||||
'other.org': (2, sizes[archive3.file_group.id] + sizes[archive2.file_group.id]),
|
||||
}
|
||||
|
||||
# An Archive is deleted.
|
||||
test_session.execute(sa_text('DELETE FROM archive WHERE id = :aid'), {'aid': archive1.id})
|
||||
test_session.commit()
|
||||
assert summaries()['example.com'] == (0, 0)
|
||||
|
||||
# The listing reads the columns: no file_group in any statement.
|
||||
statements = []
|
||||
engine = test_session.get_bind()
|
||||
listener = lambda conn, cursor, statement, *_: statements.append(statement) # noqa: E731
|
||||
event.listen(engine, 'before_cursor_execute', listener)
|
||||
try:
|
||||
collections = lib.get_collections(test_session, kind='domain')
|
||||
finally:
|
||||
event.remove(engine, 'before_cursor_execute', listener)
|
||||
by_name = {c['name']: c for c in collections}
|
||||
assert (by_name['example.com']['archive_count'], by_name['example.com']['size']) == (0, 0)
|
||||
assert (by_name['other.org']['archive_count'], by_name['other.org']['size']) == \
|
||||
(2, sizes[archive3.file_group.id] + sizes[archive2.file_group.id])
|
||||
assert not any('file_group' in s for s in statements), '\n'.join(statements)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ All statements are idempotent (IF NOT EXISTS).
|
|||
"""
|
||||
|
||||
# Triggers maintaining the summary columns `channel.video_count`, `channel.total_size`,
|
||||
# `channel.minimum_frequency` and `file_group.effective_datetime`.
|
||||
# `channel.minimum_frequency`, `collection.item_count`/`collection.total_size` (domains) and
|
||||
# `file_group.effective_datetime`.
|
||||
#
|
||||
# These are re-authored from the old PL/pgSQL triggers (see git history of alembic/versions/).
|
||||
# The old triggers had known bugs which are fixed here:
|
||||
|
|
@ -70,6 +71,55 @@ TRIGGER_DDL = [
|
|||
WHERE id = (SELECT channel_id FROM video WHERE file_group_id = new.id);
|
||||
END
|
||||
''',
|
||||
# collection.item_count + collection.total_size follow the Archives of a Domain Collection.
|
||||
# (Channels keep their summary on the channel table; generic Collections count collection_item.)
|
||||
'''
|
||||
CREATE TRIGGER IF NOT EXISTS archive_insert_collection_summary
|
||||
AFTER INSERT ON archive WHEN new.collection_id IS NOT NULL
|
||||
BEGIN
|
||||
UPDATE collection SET
|
||||
item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id),
|
||||
total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a
|
||||
LEFT JOIN file_group fg ON a.file_group_id = fg.id
|
||||
WHERE a.collection_id = collection.id)
|
||||
WHERE id = new.collection_id;
|
||||
END
|
||||
''',
|
||||
'''
|
||||
CREATE TRIGGER IF NOT EXISTS archive_delete_collection_summary
|
||||
AFTER DELETE ON archive WHEN old.collection_id IS NOT NULL
|
||||
BEGIN
|
||||
UPDATE collection SET
|
||||
item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id),
|
||||
total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a
|
||||
LEFT JOIN file_group fg ON a.file_group_id = fg.id
|
||||
WHERE a.collection_id = collection.id)
|
||||
WHERE id = old.collection_id;
|
||||
END
|
||||
''',
|
||||
'''
|
||||
CREATE TRIGGER IF NOT EXISTS archive_update_collection_summary
|
||||
AFTER UPDATE OF collection_id, file_group_id ON archive
|
||||
BEGIN
|
||||
UPDATE collection SET
|
||||
item_count = (SELECT COUNT(*) FROM archive WHERE collection_id = collection.id),
|
||||
total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a
|
||||
LEFT JOIN file_group fg ON a.file_group_id = fg.id
|
||||
WHERE a.collection_id = collection.id)
|
||||
WHERE id IN (old.collection_id, new.collection_id);
|
||||
END
|
||||
''',
|
||||
'''
|
||||
CREATE TRIGGER IF NOT EXISTS file_group_size_collection_summary
|
||||
AFTER UPDATE OF size ON file_group WHEN old.size IS NOT new.size
|
||||
BEGIN
|
||||
UPDATE collection SET
|
||||
total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM archive a
|
||||
LEFT JOIN file_group fg ON a.file_group_id = fg.id
|
||||
WHERE a.collection_id = collection.id)
|
||||
WHERE id = (SELECT collection_id FROM archive WHERE file_group_id = new.id);
|
||||
END
|
||||
''',
|
||||
# channel.minimum_frequency follows the Downloads of the Channel's Collection.
|
||||
'''
|
||||
CREATE TRIGGER IF NOT EXISTS download_insert_minimum_frequency
|
||||
|
|
|
|||
|
|
@ -158,6 +158,59 @@ def test_drop_file_group_idempotency_preserves_fts(test_directory):
|
|||
assert len(hits) == 1
|
||||
|
||||
|
||||
def test_domain_collection_summary_migration_backfills(test_directory):
|
||||
"""The domain-summary migration installs the archive triggers and backfills the summary
|
||||
columns of existing Domain Collections from their Archives."""
|
||||
import sqlite3
|
||||
from alembic.config import Config
|
||||
from alembic import command
|
||||
from wrolpi.db import get_db_uri, get_db_file
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
(test_directory / 'config').mkdir(parents=True, exist_ok=True)
|
||||
config = Config(str(PROJECT_DIR / 'alembic.ini'))
|
||||
config.set_main_option('sqlalchemy.url', get_db_uri())
|
||||
command.upgrade(config, '2026_09_20_1200')
|
||||
|
||||
db_file = get_db_file()
|
||||
directory = str(test_directory)
|
||||
triggers = {'archive_insert_collection_summary', 'archive_delete_collection_summary',
|
||||
'archive_update_collection_summary', 'file_group_size_collection_summary'}
|
||||
with sqlite3.connect(db_file) as conn:
|
||||
conn.execute('PRAGMA foreign_keys=ON')
|
||||
# The baseline installs the current trigger DDL, so a fresh DB already has the archive
|
||||
# triggers. A production DB created before them does not; emulate that.
|
||||
for name in triggers:
|
||||
conn.execute(f'DROP TRIGGER {name}')
|
||||
for id_, name, kind in ((1, 'example.com', 'domain'), (2, 'empty.org', 'domain'), (3, 'chan', 'channel')):
|
||||
conn.execute('INSERT INTO collection (id, name, kind, item_count, total_size) VALUES (?, ?, ?, 0, 0)',
|
||||
(id_, name, kind))
|
||||
for fg_id, stem, size in ((10, 'one', 100), (11, 'two', 250)):
|
||||
conn.execute(
|
||||
'INSERT INTO file_group (id, directory, primary_path, files, stem, size, indexed)'
|
||||
' VALUES (?,?,?,?,?,?,0)',
|
||||
(fg_id, directory, str(test_directory / f'{stem}.html'), '[]', stem, size),
|
||||
)
|
||||
conn.execute('INSERT INTO archive (collection_id, file_group_id) VALUES (1, ?)', (fg_id,))
|
||||
conn.commit()
|
||||
# Nothing maintains the columns before the migration.
|
||||
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (0, 0)
|
||||
|
||||
command.upgrade(config, 'head')
|
||||
|
||||
with sqlite3.connect(db_file) as conn:
|
||||
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (2, 350)
|
||||
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 2').fetchone() == (0, 0)
|
||||
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 3').fetchone() == (0, 0)
|
||||
installed = {r[0] for r in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='trigger' AND name LIKE '%collection_summary'")}
|
||||
assert installed == triggers
|
||||
# The triggers are live after the migration.
|
||||
conn.execute('DELETE FROM archive WHERE file_group_id = 11')
|
||||
conn.commit()
|
||||
assert conn.execute('SELECT item_count, total_size FROM collection WHERE id = 1').fetchone() == (1, 100)
|
||||
|
||||
|
||||
def test_bootstrap_lock(test_directory):
|
||||
"""Only one process can hold the bootstrap lock."""
|
||||
with db_bootstrap.bootstrap_lock() as acquired:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue