mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 23:16:17 -05:00
db: serializers read only loaded data (#777)
* db: serializers read only loaded data Serializing a model must not query. A lazy load there is a hidden query per row (search serialized 24 rows with 55 statements), and would be a `MissingGreenlet` error under an async Session. - `wrolpi.db.no_db_access(reason)` and `@serializer` (on every model `__json__`, and around `json_response`): an engine `before_cursor_execute` listener raises `UnexpectedQuery` during tests, and in production logs once per serializer and lets the query run. Tests record each trip as well, and the autouse `no_unexpected_queries` fixture fails a test even when the code caught the error (Video's serializer was swallowing its per-row loads of the deferred `ffprobe_json`). - Relationships serializers read load with their parent: `selectin` for FileGroup.tag_files and Collection.downloads; `joined` for TagFile.tag, Collection.tag, Channel.collection, Video.channel, Archive.collection, ZimSubscription.download, CollectionItem.zim, and the file_group of Video, Archive, Doc, Zim and CollectionItem. - Search and the Video lookups the video page uses load the deferred `ffprobe_json` (`undefer`). - `wrolpi/test/test_serializers.py`: every serialized model, freshly loaded, serializes without SQL. Search: 55 -> 5 statements for 24 videos (31 -> 5 for archives), and a few ms faster locally. Not in this change: guarding the `dict()` serializers and `expire_on_commit=False`, which go together (about 50 serialize-after-commit sites, and 40 tests that read values changed elsewhere after a commit). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * db: serializer loads per query; the guard ignores transaction control The query guard reported the engine's own BEGIN (emitted through the same event as the first statement on a Session with no open transaction) instead of the lazy load after it, and in production that BEGIN used up the one warning per serializer. Transaction-control statements are now ignored. Mapper-level eager loads reached background code: every Video/Archive load in the modelers (one per file, inside the write transaction) joined file_group/channel/collection/tag and selectin-loaded the Collection's Download rows with their yt-dlp info_json. A second refresh of 100 videos took 0.40 s instead of 0.16 s. Serializer needs are now declared per model (`json_options()`) and added only by the queries whose results are serialized; Download.info_json is deferred there (no serializer reads it). Mapper defaults keep only narrow joins nearly every use needs: TagFile.tag, Collection.tag and Channel.collection. A second refresh is back to master's statement count and time; search still drops from 55 to 3 statements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
1f687fa825
commit
cfbed9e4d7
24 changed files with 371 additions and 72 deletions
|
|
@ -5,13 +5,14 @@ from typing import Iterable, List, Optional
|
|||
|
||||
import pytz
|
||||
from sqlalchemy import Column, Integer, ForeignKey, BigInteger, Index
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
from sqlalchemy.orm import relationship, Session, joinedload
|
||||
|
||||
from wrolpi import dates
|
||||
from wrolpi.collections import Collection
|
||||
from wrolpi.common import ModelHelper, Base, logger, get_title_from_html, get_wrolpi_config, get_media_directory, \
|
||||
strip_surrogates
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import serializer
|
||||
from wrolpi.errors import UnknownArchive
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import TagFile
|
||||
|
|
@ -39,6 +40,13 @@ class Archive(Base, ModelHelper):
|
|||
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
|
||||
file_group: FileGroup = relationship('FileGroup')
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose Archives will be serialized: `__json__` reads the FileGroup and its Tags,
|
||||
and the domain (the Collection's name)."""
|
||||
return joinedload(Archive.file_group).selectinload(FileGroup.tag_files), joinedload(Archive.collection)
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = self.file_group.__json__()
|
||||
d['archive'] = dict(
|
||||
|
|
|
|||
|
|
@ -340,7 +340,7 @@ def _doc_response(doc) -> dict:
|
|||
|
||||
def _get_doc(session, file_group_id: int):
|
||||
from .models import Doc
|
||||
doc = session.query(Doc).filter_by(file_group_id=file_group_id).one_or_none()
|
||||
doc = session.query(Doc).options(*Doc.json_options()).filter_by(file_group_id=file_group_id).one_or_none()
|
||||
if not doc:
|
||||
raise ValidationError(f'Doc with file_group_id {file_group_id} not found')
|
||||
return doc
|
||||
|
|
@ -375,7 +375,7 @@ def _search_docs(session: Session, search_str=None, author=None, subject=None, l
|
|||
# the much smaller a/b/c columns.
|
||||
match = fts.translate_websearch(search_str, None if deep else fts.ABC_COLUMNS) if search_str else None
|
||||
|
||||
query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id)
|
||||
query = session.query(FileGroup).options(*FileGroup.json_options()).join(Doc, Doc.file_group_id == FileGroup.id)
|
||||
|
||||
fts_sq = None
|
||||
if match is not None:
|
||||
|
|
@ -441,7 +441,8 @@ def _search_docs(session: Session, search_str=None, author=None, subject=None, l
|
|||
browse_sql = _doc_browse_sql(order_by if order_by in DOC_BROWSE_ORDERS else 'id')
|
||||
if browse_sql:
|
||||
fg_ids = [i for i, in session.execute(text(browse_sql), dict(limit=limit, offset=offset)).fetchall()]
|
||||
by_id = {fg.id: fg for fg in session.query(FileGroup).filter(FileGroup.id.in_(fg_ids))} if fg_ids else {}
|
||||
by_id = {fg.id: fg for fg in session.query(FileGroup).options(*FileGroup.json_options())
|
||||
.filter(FileGroup.id.in_(fg_ids))} if fg_ids else {}
|
||||
file_groups = [by_id[i] for i in fg_ids]
|
||||
else:
|
||||
file_groups = query.offset(offset).limit(limit).all()
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ import pathlib
|
|||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, or_
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
from sqlalchemy.orm import relationship, Session, joinedload
|
||||
|
||||
from wrolpi.common import ModelHelper, Base
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.db import get_db_session, serializer
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
EPUB_MIMETYPE = 'application/epub'
|
||||
|
|
@ -65,6 +65,12 @@ class Doc(ModelHelper, Base):
|
|||
path = str(self.file_group.primary_path) if self.file_group else 'None'
|
||||
return f'<Doc id={self.id} path={repr(path)} file_group_id={self.file_group_id}>'
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose Docs will be serialized: `__json__` reads the FileGroup and its Tags."""
|
||||
return joinedload(Doc.file_group).selectinload(FileGroup.tag_files),
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = self.file_group.__json__()
|
||||
if d.get('data') and d['data'].get('doc_path'):
|
||||
|
|
|
|||
|
|
@ -208,6 +208,7 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5,
|
|||
name_order = (func.lower(func.replace(Collection.name, ' ', '')).asc(), Collection.name.desc())
|
||||
if order_by_video_count:
|
||||
stmt = session.query(Channel, func.count(Video.id).label('video_count')) \
|
||||
.options(*Channel.json_options()) \
|
||||
.join(Collection) \
|
||||
.filter(or_(
|
||||
Collection.name.ilike(f'%{name}%'),
|
||||
|
|
@ -220,6 +221,7 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5,
|
|||
channels = [i[0] for i in stmt]
|
||||
else:
|
||||
stmt = session.query(Channel) \
|
||||
.options(*Channel.json_options()) \
|
||||
.join(Collection) \
|
||||
.filter(or_(
|
||||
Collection.name.ilike(f'%{name}%'),
|
||||
|
|
@ -319,5 +321,6 @@ async def tag_channel(session: Session, tag_name: str | None, directory: pathlib
|
|||
|
||||
async def search_channels(session: Session, tag_names: List[str]) -> List[Channel]:
|
||||
"""Search Tagged Channels."""
|
||||
channels = session.query(Channel).join(Collection).join(Tag).filter(Tag.name.in_(tag_names)).all()
|
||||
channels = session.query(Channel).options(*Channel.json_options()) \
|
||||
.join(Collection).join(Tag).filter(Tag.name.in_(tag_names)).all()
|
||||
return channels
|
||||
|
|
|
|||
|
|
@ -3,14 +3,15 @@ import pathlib
|
|||
from typing import Optional, Dict, List
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Boolean, Date, ForeignKey, BigInteger, Index, text, JSON
|
||||
from sqlalchemy.orm import relationship, Session, deferred
|
||||
from sqlalchemy.orm import relationship, Session, deferred, undefer, joinedload, selectinload
|
||||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
|
||||
from modules.videos.errors import UnknownVideo, UnknownChannel
|
||||
from wrolpi.captions import read_captions, read_captions_with_timestamps
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \
|
||||
background_task
|
||||
from wrolpi.db import get_db_curs, get_db_session, session_curs
|
||||
from wrolpi.db import get_db_curs, get_db_session, session_curs, serializer
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.downloader import Download
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
|
@ -57,6 +58,14 @@ class Video(ModelHelper, Base):
|
|||
return f'<Video id={self.id} title={repr(self.file_group.title)} path={v} channel={self.channel_id} ' \
|
||||
f'source_id={repr(self.source_id)}>'
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose Videos will be serialized: `__json__` reads the codecs in the deferred
|
||||
ffprobe_json, the FileGroup and its Tags, and the Channel (its Collection loads with it)."""
|
||||
return undefer(Video.ffprobe_json), joinedload(Video.file_group).selectinload(FileGroup.tag_files), \
|
||||
joinedload(Video.channel)
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = self.file_group.__json__()
|
||||
|
||||
|
|
@ -233,8 +242,9 @@ class Video(ModelHelper, Base):
|
|||
break
|
||||
|
||||
# Fetch the videos by id, if they exist.
|
||||
previous_video = Video.find_by_id(session, previous_id) if previous_id else None
|
||||
next_video = Video.find_by_id(session, next_id) if next_id else None
|
||||
# The neighbors are serialized next to this Video.
|
||||
previous_video = Video.find_by_id(session, previous_id, Video.json_options()) if previous_id else None
|
||||
next_video = Video.find_by_id(session, next_id, Video.json_options()) if next_id else None
|
||||
|
||||
return previous_video, next_video
|
||||
|
||||
|
|
@ -391,27 +401,27 @@ class Video(ModelHelper, Base):
|
|||
return video
|
||||
|
||||
@staticmethod
|
||||
def get_by_id(session: Session, id_: int) -> Optional['Video']:
|
||||
def get_by_id(session: Session, id_: int, options: tuple = ()) -> Optional['Video']:
|
||||
"""Attempt to find a Video with the provided id. Returns None if it cannot be found."""
|
||||
video = session.query(Video).filter(Video.id == id_).one_or_none()
|
||||
video = session.query(Video).options(*options).filter(Video.id == id_).one_or_none()
|
||||
return video
|
||||
|
||||
@staticmethod
|
||||
def find_by_id(session: Session, id_: int) -> 'Video':
|
||||
def find_by_id(session: Session, id_: int, options: tuple = ()) -> 'Video':
|
||||
"""Find a Video with the provided id, raises an exception if it cannot be found.
|
||||
|
||||
@raise UnknownVideo: if the Video can not be found"""
|
||||
video = Video.get_by_id(session, id_)
|
||||
video = Video.get_by_id(session, id_, options)
|
||||
if not video:
|
||||
raise UnknownVideo(f'Cannot find Video with id {id_}')
|
||||
return video
|
||||
|
||||
@staticmethod
|
||||
def find_by_file_group_id(session: Session, file_group_id: int) -> 'Video':
|
||||
def find_by_file_group_id(session: Session, file_group_id: int, options: tuple = ()) -> 'Video':
|
||||
"""Find a Video by its FileGroup ID, raises an exception if it cannot be found.
|
||||
|
||||
@raise UnknownVideo: if the Video can not be found"""
|
||||
video = session.query(Video).filter(Video.file_group_id == file_group_id).one_or_none()
|
||||
video = session.query(Video).options(*options).filter(Video.file_group_id == file_group_id).one_or_none()
|
||||
if not video:
|
||||
raise UnknownVideo(f'Cannot find Video with file_group_id {file_group_id}')
|
||||
return video
|
||||
|
|
@ -620,7 +630,8 @@ class Channel(ModelHelper, Base):
|
|||
|
||||
videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id')
|
||||
collection_id = Column(Integer, ForeignKey('collection.id', ondelete='CASCADE'))
|
||||
collection = relationship('Collection', foreign_keys=[collection_id])
|
||||
# Joined: a Channel's name, directory and Tag live on its Collection.
|
||||
collection = relationship('Collection', foreign_keys=[collection_id], lazy='joined')
|
||||
|
||||
@property
|
||||
def downloads(self) -> list:
|
||||
|
|
@ -938,6 +949,13 @@ class Channel(ModelHelper, Base):
|
|||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose Channels will be serialized: `__json__` lists the Collection's
|
||||
Downloads (without their yt-dlp info_json, which is not serialized)."""
|
||||
return joinedload(Channel.collection).selectinload(Collection.downloads).defer(Download.info_json),
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = dict(
|
||||
directory=self.directory,
|
||||
|
|
|
|||
|
|
@ -127,6 +127,9 @@ async def test_video_channel_refresh(async_client, test_session, test_directory,
|
|||
video2: Video = Video.get_by_path(test_session, video2_path)
|
||||
assert video1.channel == channel
|
||||
assert not video2.channel
|
||||
# Serializers do not query: load what Video's `__json__` reads, as the queries that serialize Videos do.
|
||||
video1 = test_session.query(Video).options(*Video.json_options()).populate_existing() \
|
||||
.filter(Video.id == video1.id).one()
|
||||
assert video1.__json__()['video']['channel']
|
||||
video_channel = video1.__json__()['video']['channel']
|
||||
assert video_channel['id'] == channel.id
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ def get_video_for_app(session: Session, file_group_id: int, skip_viewed: bool =
|
|||
Get a Video by its FileGroup ID, with its prev/next videos. Mark the Video as viewed (a write; the
|
||||
caller commits).
|
||||
"""
|
||||
video = Video.find_by_file_group_id(session, file_group_id)
|
||||
video = Video.find_by_file_group_id(session, file_group_id, Video.json_options())
|
||||
if not skip_viewed:
|
||||
video.file_group.set_viewed()
|
||||
previous_video, next_video = video.get_surrounding_videos()
|
||||
|
|
|
|||
|
|
@ -323,7 +323,7 @@ def get_kiwix_catalog():
|
|||
|
||||
|
||||
def get_kiwix_subscriptions(session: Session) -> Dict[str, ZimSubscription]:
|
||||
subscriptions = session.query(ZimSubscription)
|
||||
subscriptions = session.query(ZimSubscription).options(*ZimSubscription.json_options())
|
||||
subscriptions_by_name = {i.name: i for i in subscriptions}
|
||||
return subscriptions_by_name
|
||||
|
||||
|
|
|
|||
|
|
@ -7,14 +7,14 @@ from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional,
|
|||
|
||||
from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher
|
||||
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean, UniqueConstraint
|
||||
from sqlalchemy.orm import relationship, Session
|
||||
from sqlalchemy.orm import relationship, Session, joinedload
|
||||
from sqlalchemy.orm.exc import NoResultFound # noqa
|
||||
|
||||
from modules.zim.errors import UnknownZimEntry, UnknownZimTagEntry, UnknownZim
|
||||
from wrolpi import dates, tags
|
||||
from wrolpi.common import Base, logger, get_relative_to_media_directory, ModelHelper
|
||||
from wrolpi.dates import TZDateTime
|
||||
from wrolpi.db import session_curs
|
||||
from wrolpi.db import session_curs, serializer
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.media_path import MediaPathType
|
||||
|
|
@ -80,6 +80,12 @@ class Zim(Base, ModelHelper):
|
|||
def __repr__(self):
|
||||
return f'<Zim id={self.id} file_group_id={self.file_group_id} path={self.path}>'
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose Zims will be serialized: `__json__` reads the FileGroup's size."""
|
||||
return joinedload(Zim.file_group),
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = dict(
|
||||
id=self.id,
|
||||
|
|
@ -319,7 +325,7 @@ class Zims:
|
|||
def get_all(session: Session) -> List[Zim]:
|
||||
"""Returns all Zim records."""
|
||||
zims = list()
|
||||
for zim in session.query(Zim).order_by(Zim.path):
|
||||
for zim in session.query(Zim).options(*Zim.json_options()).order_by(Zim.path):
|
||||
try:
|
||||
# It may not be possible to read the Zim file (drive not mounted), report and ignore.
|
||||
zim.get_zim()
|
||||
|
|
@ -390,6 +396,12 @@ class ZimSubscription(Base):
|
|||
download_id = self.download_id
|
||||
return f'<ZimSubscription id={self.id} {name=} {language=} {download_id=}>'
|
||||
|
||||
@staticmethod
|
||||
def json_options() -> tuple:
|
||||
"""Loader options for a query whose subscriptions will be serialized: `__json__` reads the Download's URL."""
|
||||
return joinedload(ZimSubscription.download).defer(Download.info_json),
|
||||
|
||||
@serializer
|
||||
def __json__(self) -> dict:
|
||||
d = dict(
|
||||
id=self.id,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue