db: serializer loads per query; the guard ignores transaction control

The query guard reported the engine's own BEGIN (emitted through the same event as the first statement on a
Session with no open transaction) instead of the lazy load after it, and in production that BEGIN used up the
one warning per serializer.  Transaction-control statements are now ignored.

Mapper-level eager loads reached background code: every Video/Archive load in the modelers (one per file,
inside the write transaction) joined file_group/channel/collection/tag and selectin-loaded the Collection's
Download rows with their yt-dlp info_json.  A second refresh of 100 videos took 0.40 s instead of 0.16 s.

Serializer needs are now declared per model (`json_options()`) and added only by the queries whose results
are serialized; Download.info_json is deferred there (no serializer reads it).  Mapper defaults keep only
narrow joins nearly every use needs: TagFile.tag, Collection.tag and Channel.collection.  A second refresh is
back to master's statement count and time; search still drops from 55 to 3 statements.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-10-04 18:00:30 -06:00
parent 72a0379a7f
commit 021bdad947
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
18 changed files with 153 additions and 58 deletions

View file

@ -5,7 +5,7 @@ from typing import Iterable, List, Optional
import pytz
from sqlalchemy import Column, Integer, ForeignKey, BigInteger, Index
from sqlalchemy.orm import relationship, Session
from sqlalchemy.orm import relationship, Session, joinedload
from wrolpi import dates
from wrolpi.collections import Collection
@ -36,10 +36,15 @@ class Archive(Base, ModelHelper):
id = Column(Integer, primary_key=True)
collection_id = Column(Integer, ForeignKey('collection.id', ondelete='CASCADE'))
collection: Collection = relationship('Collection', primaryjoin='Archive.collection_id==Collection.id',
lazy='joined')
collection: Collection = relationship('Collection', primaryjoin='Archive.collection_id==Collection.id')
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
file_group: FileGroup = relationship('FileGroup', lazy='joined')
file_group: FileGroup = relationship('FileGroup')
@staticmethod
def json_options() -> tuple:
"""Loader options for a query whose Archives will be serialized: `__json__` reads the FileGroup and its Tags,
and the domain (the Collection's name)."""
return joinedload(Archive.file_group).selectinload(FileGroup.tag_files), joinedload(Archive.collection)
@serializer
def __json__(self) -> dict:

View file

@ -340,7 +340,7 @@ def _doc_response(doc) -> dict:
def _get_doc(session, file_group_id: int):
from .models import Doc
doc = session.query(Doc).filter_by(file_group_id=file_group_id).one_or_none()
doc = session.query(Doc).options(*Doc.json_options()).filter_by(file_group_id=file_group_id).one_or_none()
if not doc:
raise ValidationError(f'Doc with file_group_id {file_group_id} not found')
return doc
@ -375,7 +375,7 @@ def _search_docs(session: Session, search_str=None, author=None, subject=None, l
# the much smaller a/b/c columns.
match = fts.translate_websearch(search_str, None if deep else fts.ABC_COLUMNS) if search_str else None
query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id)
query = session.query(FileGroup).options(*FileGroup.json_options()).join(Doc, Doc.file_group_id == FileGroup.id)
fts_sq = None
if match is not None:
@ -441,7 +441,8 @@ def _search_docs(session: Session, search_str=None, author=None, subject=None, l
browse_sql = _doc_browse_sql(order_by if order_by in DOC_BROWSE_ORDERS else 'id')
if browse_sql:
fg_ids = [i for i, in session.execute(text(browse_sql), dict(limit=limit, offset=offset)).fetchall()]
by_id = {fg.id: fg for fg in session.query(FileGroup).filter(FileGroup.id.in_(fg_ids))} if fg_ids else {}
by_id = {fg.id: fg for fg in session.query(FileGroup).options(*FileGroup.json_options())
.filter(FileGroup.id.in_(fg_ids))} if fg_ids else {}
file_groups = [by_id[i] for i in fg_ids]
else:
file_groups = query.offset(offset).limit(limit).all()

View file

@ -2,7 +2,7 @@ import pathlib
from typing import Optional
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, or_
from sqlalchemy.orm import relationship, Session
from sqlalchemy.orm import relationship, Session, joinedload
from wrolpi.common import ModelHelper, Base
from wrolpi.db import get_db_session, serializer
@ -56,7 +56,7 @@ class Doc(ModelHelper, Base):
description = Column(Text)
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
file_group: FileGroup = relationship('FileGroup', lazy='joined')
file_group: FileGroup = relationship('FileGroup')
sections = relationship('DocSection', back_populates='doc',
cascade='all, delete-orphan',
passive_deletes=True)
@ -65,6 +65,11 @@ class Doc(ModelHelper, Base):
path = str(self.file_group.primary_path) if self.file_group else 'None'
return f'<Doc id={self.id} path={repr(path)} file_group_id={self.file_group_id}>'
@staticmethod
def json_options() -> tuple:
"""Loader options for a query whose Docs will be serialized: `__json__` reads the FileGroup and its Tags."""
return joinedload(Doc.file_group).selectinload(FileGroup.tag_files),
@serializer
def __json__(self) -> dict:
d = self.file_group.__json__()

View file

@ -208,6 +208,7 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5,
name_order = (func.lower(func.replace(Collection.name, ' ', '')).asc(), Collection.name.desc())
if order_by_video_count:
stmt = session.query(Channel, func.count(Video.id).label('video_count')) \
.options(*Channel.json_options()) \
.join(Collection) \
.filter(or_(
Collection.name.ilike(f'%{name}%'),
@ -220,6 +221,7 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5,
channels = [i[0] for i in stmt]
else:
stmt = session.query(Channel) \
.options(*Channel.json_options()) \
.join(Collection) \
.filter(or_(
Collection.name.ilike(f'%{name}%'),
@ -319,5 +321,6 @@ async def tag_channel(session: Session, tag_name: str | None, directory: pathlib
async def search_channels(session: Session, tag_names: List[str]) -> List[Channel]:
"""Search Tagged Channels."""
channels = session.query(Channel).join(Collection).join(Tag).filter(Tag.name.in_(tag_names)).all()
channels = session.query(Channel).options(*Channel.json_options()) \
.join(Collection).join(Tag).filter(Tag.name.in_(tag_names)).all()
return channels

View file

@ -3,7 +3,7 @@ import pathlib
from typing import Optional, Dict, List
from sqlalchemy import Column, Integer, String, Boolean, Date, ForeignKey, BigInteger, Index, text, JSON
from sqlalchemy.orm import relationship, Session, deferred, undefer
from sqlalchemy.orm import relationship, Session, deferred, undefer, joinedload, selectinload
from sqlalchemy.orm.collections import InstrumentedList
from modules.videos.errors import UnknownVideo, UnknownChannel
@ -11,6 +11,7 @@ from wrolpi.captions import read_captions, read_captions_with_timestamps
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \
background_task
from wrolpi.db import get_db_curs, get_db_session, session_curs, serializer
from wrolpi.collections.models import Collection
from wrolpi.downloader import Download
from wrolpi.files.lib import split_path_stem_and_suffix
from wrolpi.files.worker import file_worker
@ -46,10 +47,9 @@ class Video(ModelHelper, Base):
comments_failed = Column(Boolean, default=False) # see `get_missing_videos_comments`
channel_id = Column(Integer, ForeignKey('channel.id'))
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos',
lazy='joined')
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
file_group: FileGroup = relationship('FileGroup', lazy='joined')
file_group: FileGroup = relationship('FileGroup')
def __repr__(self):
v = None
@ -58,6 +58,13 @@ class Video(ModelHelper, Base):
return f'<Video id={self.id} title={repr(self.file_group.title)} path={v} channel={self.channel_id} ' \
f'source_id={repr(self.source_id)}>'
@staticmethod
def json_options() -> tuple:
"""Loader options for a query whose Videos will be serialized: `__json__` reads the codecs in the deferred
ffprobe_json, the FileGroup and its Tags, and the Channel (its Collection loads with it)."""
return undefer(Video.ffprobe_json), joinedload(Video.file_group).selectinload(FileGroup.tag_files), \
joinedload(Video.channel)
@serializer
def __json__(self) -> dict:
d = self.file_group.__json__()
@ -235,8 +242,9 @@ class Video(ModelHelper, Base):
break
# Fetch the videos by id, if they exist.
previous_video = Video.find_by_id(session, previous_id) if previous_id else None
next_video = Video.find_by_id(session, next_id) if next_id else None
# The neighbors are serialized next to this Video.
previous_video = Video.find_by_id(session, previous_id, Video.json_options()) if previous_id else None
next_video = Video.find_by_id(session, next_id, Video.json_options()) if next_id else None
return previous_video, next_video
@ -393,30 +401,27 @@ class Video(ModelHelper, Base):
return video
@staticmethod
def get_by_id(session: Session, id_: int) -> Optional['Video']:
def get_by_id(session: Session, id_: int, options: tuple = ()) -> Optional['Video']:
"""Attempt to find a Video with the provided id. Returns None if it cannot be found."""
# `__json__` reads the (deferred) ffprobe_json for the codecs.
video = session.query(Video).options(undefer(Video.ffprobe_json)).filter(Video.id == id_).one_or_none()
video = session.query(Video).options(*options).filter(Video.id == id_).one_or_none()
return video
@staticmethod
def find_by_id(session: Session, id_: int) -> 'Video':
def find_by_id(session: Session, id_: int, options: tuple = ()) -> 'Video':
"""Find a Video with the provided id, raises an exception if it cannot be found.
@raise UnknownVideo: if the Video can not be found"""
video = Video.get_by_id(session, id_)
video = Video.get_by_id(session, id_, options)
if not video:
raise UnknownVideo(f'Cannot find Video with id {id_}')
return video
@staticmethod
def find_by_file_group_id(session: Session, file_group_id: int) -> 'Video':
def find_by_file_group_id(session: Session, file_group_id: int, options: tuple = ()) -> 'Video':
"""Find a Video by its FileGroup ID, raises an exception if it cannot be found.
@raise UnknownVideo: if the Video can not be found"""
# `__json__` reads the (deferred) ffprobe_json for the codecs.
video = session.query(Video).options(undefer(Video.ffprobe_json)) \
.filter(Video.file_group_id == file_group_id).one_or_none()
video = session.query(Video).options(*options).filter(Video.file_group_id == file_group_id).one_or_none()
if not video:
raise UnknownVideo(f'Cannot find Video with file_group_id {file_group_id}')
return video
@ -625,6 +630,7 @@ class Channel(ModelHelper, Base):
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
collection_id = Column(Integer, ForeignKey('collection.id', ondelete='CASCADE'))
# Joined: a Channel's name, directory and Tag live on its Collection.
collection = relationship('Collection', foreign_keys=[collection_id], lazy='joined')
@property
@ -943,6 +949,12 @@ class Channel(ModelHelper, Base):
return None
@staticmethod
def json_options() -> tuple:
"""Loader options for a query whose Channels will be serialized: `__json__` lists the Collection's
Downloads (without their yt-dlp info_json, which is not serialized)."""
return joinedload(Channel.collection).selectinload(Collection.downloads).defer(Download.info_json),
@serializer
def __json__(self) -> dict:
d = dict(

View file

@ -127,8 +127,9 @@ async def test_video_channel_refresh(async_client, test_session, test_directory,
video2: Video = Video.get_by_path(test_session, video2_path)
assert video1.channel == channel
assert not video2.channel
# Serializers do not query: load the deferred ffprobe_json first, as the queries that serialize Videos do.
test_session.refresh(video1, ['ffprobe_json'])
# Serializers do not query: load what Video's `__json__` reads, as the queries that serialize Videos do.
video1 = test_session.query(Video).options(*Video.json_options()).populate_existing() \
.filter(Video.id == video1.id).one()
assert video1.__json__()['video']['channel']
video_channel = video1.__json__()['video']['channel']
assert video_channel['id'] == channel.id

View file

@ -30,7 +30,7 @@ def get_video_for_app(session: Session, file_group_id: int, skip_viewed: bool =
Get a Video by its FileGroup ID, with its prev/next videos. Mark the Video as viewed (a write; the
caller commits).
"""
video = Video.find_by_file_group_id(session, file_group_id)
video = Video.find_by_file_group_id(session, file_group_id, Video.json_options())
if not skip_viewed:
video.file_group.set_viewed()
previous_video, next_video = video.get_surrounding_videos()

View file

@ -323,7 +323,7 @@ def get_kiwix_catalog():
def get_kiwix_subscriptions(session: Session) -> Dict[str, ZimSubscription]:
subscriptions = session.query(ZimSubscription)
subscriptions = session.query(ZimSubscription).options(*ZimSubscription.json_options())
subscriptions_by_name = {i.name: i for i in subscriptions}
return subscriptions_by_name

View file

@ -7,7 +7,7 @@ from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional,
from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean, UniqueConstraint
from sqlalchemy.orm import relationship, Session
from sqlalchemy.orm import relationship, Session, joinedload
from sqlalchemy.orm.exc import NoResultFound # noqa
from modules.zim.errors import UnknownZimEntry, UnknownZimTagEntry, UnknownZim
@ -74,12 +74,17 @@ class Zim(Base, ModelHelper):
path: pathlib.Path = Column(MediaPathType, nullable=False)
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False)
file_group: FileGroup = relationship('FileGroup', lazy='joined')
file_group: FileGroup = relationship('FileGroup')
auto_search = Column(Boolean, default=True)
def __repr__(self):
return f'<Zim id={self.id} file_group_id={self.file_group_id} path={self.path}>'
@staticmethod
def json_options() -> tuple:
"""Loader options for a query whose Zims will be serialized: `__json__` reads the FileGroup's size."""
return joinedload(Zim.file_group),
@serializer
def __json__(self) -> dict:
d = dict(
@ -320,7 +325,7 @@ class Zims:
def get_all(session: Session) -> List[Zim]:
"""Returns all Zim records."""
zims = list()
for zim in session.query(Zim).order_by(Zim.path):
for zim in session.query(Zim).options(*Zim.json_options()).order_by(Zim.path):
try:
# It may not be possible to read the Zim file (drive not mounted), report and ignore.
zim.get_zim()
@ -383,8 +388,7 @@ class ZimSubscription(Base):
name: str = Column(Text, unique=True, nullable=False)
language: str = Column(Text, nullable=False)
download_id: int = Column(Integer, ForeignKey('download.id', ondelete='CASCADE'), nullable=False)
download: Download = relationship('Download', primaryjoin='ZimSubscription.download_id==Download.id',
lazy='joined')
download: Download = relationship('Download', primaryjoin='ZimSubscription.download_id==Download.id')
def __repr__(self):
name = self.name
@ -392,6 +396,11 @@ class ZimSubscription(Base):
download_id = self.download_id
return f'<ZimSubscription id={self.id} {name=} {language=} {download_id=}>'
@staticmethod
def json_options() -> tuple:
"""Loader options for a query whose subscriptions will be serialized: `__json__` reads the Download's URL."""
return joinedload(ZimSubscription.download).defer(Download.info_json),
@serializer
def __json__(self) -> dict:
d = dict(