mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
db: serializer loads per query; the guard ignores transaction control
The query guard reported the engine's own BEGIN (emitted through the same event as the first statement on a Session with no open transaction) instead of the lazy load after it, and in production that BEGIN used up the one warning per serializer. Transaction-control statements are now ignored. Mapper-level eager loads reached background code: every Video/Archive load in the modelers (one per file, inside the write transaction) joined file_group/channel/collection/tag and selectin-loaded the Collection's Download rows with their yt-dlp info_json. A second refresh of 100 videos took 0.40 s instead of 0.16 s. Serializer needs are now declared per model (`json_options()`) and added only by the queries whose results are serialized; Download.info_json is deferred there (no serializer reads it). Mapper defaults keep only narrow joins nearly every use needs: TagFile.tag, Collection.tag and Channel.collection. A second refresh is back to master's statement count and time; search still drops from 55 to 3 statements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
72a0379a7f
commit
021bdad947
18 changed files with 153 additions and 58 deletions
|
|
@ -5,7 +5,7 @@ from typing import Iterable, List, Optional
|
|||
|
||||
import pytz
|
||||
from sqlalchemy import Column, Integer, ForeignKey, BigInteger, Index
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
from sqlalchemy.orm import relationship, Session, joinedload
|
||||
|
||||
from wrolpi import dates
|
||||
from wrolpi.collections import Collection
|
||||
|
|
@ -36,10 +36,15 @@ class Archive(Base, ModelHelper):
|
|||
id = Column(Integer, primary_key=True)
|
||||
|
||||
collection_id = Column(Integer, ForeignKey('collection.id', ondelete='CASCADE'))
|
||||
collection: Collection = relationship('Collection', primaryjoin='Archive.collection_id==Collection.id',
|
||||
lazy='joined')
|
||||
collection: Collection = relationship('Collection', primaryjoin='Archive.collection_id==Collection.id')
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
|
||||
file_group: FileGroup = relationship('FileGroup', lazy='joined')
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose Archives will be serialized: `__json__` reads the FileGroup and its Tags,
|
||||
and the domain (the Collection's name)."""
|
||||
return joinedload(Archive.file_group).selectinload(FileGroup.tag_files), joinedload(Archive.collection)
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
|
|
|
|||
|
|
@ -340,7 +340,7 @@ def _doc_response(doc) -> dict:
|
|||
|
||||
def _get_doc(session, file_group_id: int):
|
||||
from .models import Doc
|
||||
doc = session.query(Doc).filter_by(file_group_id=file_group_id).one_or_none()
|
||||
doc = session.query(Doc).options(*Doc.json_options()).filter_by(file_group_id=file_group_id).one_or_none()
|
||||
if not doc:
|
||||
raise ValidationError(f'Doc with file_group_id {file_group_id} not found')
|
||||
return doc
|
||||
|
|
@ -375,7 +375,7 @@ def _search_docs(session: Session, search_str=None, author=None, subject=None, l
|
|||
# the much smaller a/b/c columns.
|
||||
match = fts.translate_websearch(search_str, None if deep else fts.ABC_COLUMNS) if search_str else None
|
||||
|
||||
query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id)
|
||||
query = session.query(FileGroup).options(*FileGroup.json_options()).join(Doc, Doc.file_group_id == FileGroup.id)
|
||||
|
||||
fts_sq = None
|
||||
if match is not None:
|
||||
|
|
@ -441,7 +441,8 @@ def _search_docs(session: Session, search_str=None, author=None, subject=None, l
|
|||
browse_sql = _doc_browse_sql(order_by if order_by in DOC_BROWSE_ORDERS else 'id')
|
||||
if browse_sql:
|
||||
fg_ids = [i for i, in session.execute(text(browse_sql), dict(limit=limit, offset=offset)).fetchall()]
|
||||
by_id = {fg.id: fg for fg in session.query(FileGroup).filter(FileGroup.id.in_(fg_ids))} if fg_ids else {}
|
||||
by_id = {fg.id: fg for fg in session.query(FileGroup).options(*FileGroup.json_options())
|
||||
.filter(FileGroup.id.in_(fg_ids))} if fg_ids else {}
|
||||
file_groups = [by_id[i] for i in fg_ids]
|
||||
else:
|
||||
file_groups = query.offset(offset).limit(limit).all()
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import pathlib
|
|||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, or_
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
from sqlalchemy.orm import relationship, Session, joinedload
|
||||
|
||||
from wrolpi.common import ModelHelper, Base
|
||||
from wrolpi.db import get_db_session, serializer
|
||||
|
|
@ -56,7 +56,7 @@ class Doc(ModelHelper, Base):
|
|||
description = Column(Text)
|
||||
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False, unique=True)
|
||||
file_group: FileGroup = relationship('FileGroup', lazy='joined')
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
sections = relationship('DocSection', back_populates='doc',
|
||||
cascade='all, delete-orphan',
|
||||
passive_deletes=True)
|
||||
|
|
@ -65,6 +65,11 @@ class Doc(ModelHelper, Base):
|
|||
path = str(self.file_group.primary_path) if self.file_group else 'None'
|
||||
return f'<Doc id={self.id} path={repr(path)} file_group_id={self.file_group_id}>'
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose Docs will be serialized: `__json__` reads the FileGroup and its Tags."""
|
||||
return joinedload(Doc.file_group).selectinload(FileGroup.tag_files),
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = self.file_group.__json__()
|
||||
|
|
|
|||
|
|
@ -208,6 +208,7 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5,
|
|||
name_order = (func.lower(func.replace(Collection.name, ' ', '')).asc(), Collection.name.desc())
|
||||
if order_by_video_count:
|
||||
stmt = session.query(Channel, func.count(Video.id).label('video_count')) \
|
||||
.options(*Channel.json_options()) \
|
||||
.join(Collection) \
|
||||
.filter(or_(
|
||||
Collection.name.ilike(f'%{name}%'),
|
||||
|
|
@ -220,6 +221,7 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5,
|
|||
channels = [i[0] for i in stmt]
|
||||
else:
|
||||
stmt = session.query(Channel) \
|
||||
.options(*Channel.json_options()) \
|
||||
.join(Collection) \
|
||||
.filter(or_(
|
||||
Collection.name.ilike(f'%{name}%'),
|
||||
|
|
@ -319,5 +321,6 @@ async def tag_channel(session: Session, tag_name: str | None, directory: pathlib
|
|||
|
||||
async def search_channels(session: Session, tag_names: List[str]) -> List[Channel]:
|
||||
"""Search Tagged Channels."""
|
||||
channels = session.query(Channel).join(Collection).join(Tag).filter(Tag.name.in_(tag_names)).all()
|
||||
channels = session.query(Channel).options(*Channel.json_options()) \
|
||||
.join(Collection).join(Tag).filter(Tag.name.in_(tag_names)).all()
|
||||
return channels
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import pathlib
|
|||
from typing import Optional, Dict, List
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Boolean, Date, ForeignKey, BigInteger, Index, text, JSON
|
||||
from sqlalchemy.orm import relationship, Session, deferred, undefer
|
||||
from sqlalchemy.orm import relationship, Session, deferred, undefer, joinedload, selectinload
|
||||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
|
||||
from modules.videos.errors import UnknownVideo, UnknownChannel
|
||||
|
|
@ -11,6 +11,7 @@ from wrolpi.captions import read_captions, read_captions_with_timestamps
|
|||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \
|
||||
background_task
|
||||
from wrolpi.db import get_db_curs, get_db_session, session_curs, serializer
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.downloader import Download
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
|
@ -46,10 +47,9 @@ class Video(ModelHelper, Base):
|
|||
comments_failed = Column(Boolean, default=False) # see `get_missing_videos_comments`
|
||||
|
||||
channel_id = Column(Integer, ForeignKey('channel.id'))
|
||||
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos',
|
||||
lazy='joined')
|
||||
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
|
||||
file_group: FileGroup = relationship('FileGroup', lazy='joined')
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
|
||||
def __repr__(self):
|
||||
v = None
|
||||
|
|
@ -58,6 +58,13 @@ class Video(ModelHelper, Base):
|
|||
return f'<Video id={self.id} title={repr(self.file_group.title)} path={v} channel={self.channel_id} ' \
|
||||
f'source_id={repr(self.source_id)}>'
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose Videos will be serialized: `__json__` reads the codecs in the deferred
|
||||
ffprobe_json, the FileGroup and its Tags, and the Channel (its Collection loads with it)."""
|
||||
return undefer(Video.ffprobe_json), joinedload(Video.file_group).selectinload(FileGroup.tag_files), \
|
||||
joinedload(Video.channel)
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = self.file_group.__json__()
|
||||
|
|
@ -235,8 +242,9 @@ class Video(ModelHelper, Base):
|
|||
break
|
||||
|
||||
# Fetch the videos by id, if they exist.
|
||||
previous_video = Video.find_by_id(session, previous_id) if previous_id else None
|
||||
next_video = Video.find_by_id(session, next_id) if next_id else None
|
||||
# The neighbors are serialized next to this Video.
|
||||
previous_video = Video.find_by_id(session, previous_id, Video.json_options()) if previous_id else None
|
||||
next_video = Video.find_by_id(session, next_id, Video.json_options()) if next_id else None
|
||||
|
||||
return previous_video, next_video
|
||||
|
||||
|
|
@ -393,30 +401,27 @@ class Video(ModelHelper, Base):
|
|||
return video
|
||||
|
||||
@staticmethod
|
||||
def get_by_id(session: Session, id_: int) -> Optional['Video']:
|
||||
def get_by_id(session: Session, id_: int, options: tuple = ()) -> Optional['Video']:
|
||||
"""Attempt to find a Video with the provided id. Returns None if it cannot be found."""
|
||||
# `__json__` reads the (deferred) ffprobe_json for the codecs.
|
||||
video = session.query(Video).options(undefer(Video.ffprobe_json)).filter(Video.id == id_).one_or_none()
|
||||
video = session.query(Video).options(*options).filter(Video.id == id_).one_or_none()
|
||||
return video
|
||||
|
||||
@staticmethod
|
||||
def find_by_id(session: Session, id_: int) -> 'Video':
|
||||
def find_by_id(session: Session, id_: int, options: tuple = ()) -> 'Video':
|
||||
"""Find a Video with the provided id, raises an exception if it cannot be found.
|
||||
|
||||
@raise UnknownVideo: if the Video can not be found"""
|
||||
video = Video.get_by_id(session, id_)
|
||||
video = Video.get_by_id(session, id_, options)
|
||||
if not video:
|
||||
raise UnknownVideo(f'Cannot find Video with id {id_}')
|
||||
return video
|
||||
|
||||
@staticmethod
|
||||
def find_by_file_group_id(session: Session, file_group_id: int) -> 'Video':
|
||||
def find_by_file_group_id(session: Session, file_group_id: int, options: tuple = ()) -> 'Video':
|
||||
"""Find a Video by its FileGroup ID, raises an exception if it cannot be found.
|
||||
|
||||
@raise UnknownVideo: if the Video can not be found"""
|
||||
# `__json__` reads the (deferred) ffprobe_json for the codecs.
|
||||
video = session.query(Video).options(undefer(Video.ffprobe_json)) \
|
||||
.filter(Video.file_group_id == file_group_id).one_or_none()
|
||||
video = session.query(Video).options(*options).filter(Video.file_group_id == file_group_id).one_or_none()
|
||||
if not video:
|
||||
raise UnknownVideo(f'Cannot find Video with file_group_id {file_group_id}')
|
||||
return video
|
||||
|
|
@ -625,6 +630,7 @@ class Channel(ModelHelper, Base):
|
|||
|
||||
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
|
||||
collection_id = Column(Integer, ForeignKey('collection.id', ondelete='CASCADE'))
|
||||
# Joined: a Channel's name, directory and Tag live on its Collection.
|
||||
collection = relationship('Collection', foreign_keys=[collection_id], lazy='joined')
|
||||
|
||||
@property
|
||||
|
|
@ -943,6 +949,12 @@ class Channel(ModelHelper, Base):
|
|||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose Channels will be serialized: `__json__` lists the Collection's
|
||||
Downloads (without their yt-dlp info_json, which is not serialized)."""
|
||||
return joinedload(Channel.collection).selectinload(Collection.downloads).defer(Download.info_json),
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = dict(
|
||||
|
|
|
|||
|
|
@ -127,8 +127,9 @@ async def test_video_channel_refresh(async_client, test_session, test_directory,
|
|||
video2: Video = Video.get_by_path(test_session, video2_path)
|
||||
assert video1.channel == channel
|
||||
assert not video2.channel
|
||||
# Serializers do not query: load the deferred ffprobe_json first, as the queries that serialize Videos do.
|
||||
test_session.refresh(video1, ['ffprobe_json'])
|
||||
# Serializers do not query: load what Video's `__json__` reads, as the queries that serialize Videos do.
|
||||
video1 = test_session.query(Video).options(*Video.json_options()).populate_existing() \
|
||||
.filter(Video.id == video1.id).one()
|
||||
assert video1.__json__()['video']['channel']
|
||||
video_channel = video1.__json__()['video']['channel']
|
||||
assert video_channel['id'] == channel.id
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ def get_video_for_app(session: Session, file_group_id: int, skip_viewed: bool =
|
|||
Get a Video by its FileGroup ID, with its prev/next videos. Mark the Video as viewed (a write; the
|
||||
caller commits).
|
||||
"""
|
||||
video = Video.find_by_file_group_id(session, file_group_id)
|
||||
video = Video.find_by_file_group_id(session, file_group_id, Video.json_options())
|
||||
if not skip_viewed:
|
||||
video.file_group.set_viewed()
|
||||
previous_video, next_video = video.get_surrounding_videos()
|
||||
|
|
|
|||
|
|
@ -323,7 +323,7 @@ def get_kiwix_catalog():
|
|||
|
||||
|
||||
def get_kiwix_subscriptions(session: Session) -> Dict[str, ZimSubscription]:
|
||||
subscriptions = session.query(ZimSubscription)
|
||||
subscriptions = session.query(ZimSubscription).options(*ZimSubscription.json_options())
|
||||
subscriptions_by_name = {i.name: i for i in subscriptions}
|
||||
return subscriptions_by_name
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional,
|
|||
|
||||
from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher
|
||||
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean, UniqueConstraint
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
from sqlalchemy.orm import relationship, Session, joinedload
|
||||
from sqlalchemy.orm.exc import NoResultFound # noqa
|
||||
|
||||
from modules.zim.errors import UnknownZimEntry, UnknownZimTagEntry, UnknownZim
|
||||
|
|
@ -74,12 +74,17 @@ class Zim(Base, ModelHelper):
|
|||
path: pathlib.Path = Column(MediaPathType, nullable=False)
|
||||
|
||||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False)
|
||||
file_group: FileGroup = relationship('FileGroup', lazy='joined')
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
auto_search = Column(Boolean, default=True)
|
||||
|
||||
def __repr__(self):
|
||||
return f'<Zim id={self.id} file_group_id={self.file_group_id} path={self.path}>'
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose Zims will be serialized: `__json__` reads the FileGroup's size."""
|
||||
return joinedload(Zim.file_group),
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = dict(
|
||||
|
|
@ -320,7 +325,7 @@ class Zims:
|
|||
def get_all(session: Session) -> List[Zim]:
|
||||
"""Returns all Zim records."""
|
||||
zims = list()
|
||||
for zim in session.query(Zim).order_by(Zim.path):
|
||||
for zim in session.query(Zim).options(*Zim.json_options()).order_by(Zim.path):
|
||||
try:
|
||||
# It may not be possible to read the Zim file (drive not mounted), report and ignore.
|
||||
zim.get_zim()
|
||||
|
|
@ -383,8 +388,7 @@ class ZimSubscription(Base):
|
|||
name: str = Column(Text, unique=True, nullable=False)
|
||||
language: str = Column(Text, nullable=False)
|
||||
download_id: int = Column(Integer, ForeignKey('download.id', ondelete='CASCADE'), nullable=False)
|
||||
download: Download = relationship('Download', primaryjoin='ZimSubscription.download_id==Download.id',
|
||||
lazy='joined')
|
||||
download: Download = relationship('Download', primaryjoin='ZimSubscription.download_id==Download.id')
|
||||
|
||||
def __repr__(self):
|
||||
name = self.name
|
||||
|
|
@ -392,6 +396,11 @@ class ZimSubscription(Base):
|
|||
download_id = self.download_id
|
||||
return f'<ZimSubscription id={self.id} {name=} {language=} {download_id=}>'
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose subscriptions will be serialized: `__json__` reads the Download's URL."""
|
||||
return joinedload(ZimSubscription.download).defer(Download.info_json),
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = dict(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue