Hiding a Calibre book's cover and metadata uses the modeler's session (#936)

Doc._hide_calibre_files opened a second write session while the modeler held
the item's transaction.  SQLite has one writer, so the second session waited
out the 30 s busy timeout and failed with "database is locked"; the modeler
retried three times, so every Calibre epub cost about 98 s and stayed
unmodeled, and the cover.jpg / metadata.opf FileGroups it should have deleted
survived.  On a library of 1663 Calibre books the global refresh's modeling
phase would have taken about two days.

The nested session dates from before the SQLite cutover, when Postgres row
locks let it work.  The delete now runs in the caller's session.

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-11-02 11:04:19 -07:00 • committed by Roland
parent e6b7b9e5dd
commit c0452e29e6
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
3 changed files with 35 additions and 13 deletions

View file

@ -110,7 +110,7 @@ def _model_doc(file_group: FileGroup, session: Session) -> Doc:
doc.size = file_group.primary_path.stat().st_size if file_group.primary_path.exists() else None
# Handle cover.
_handle_doc_cover(doc, file_group, metadata)
_handle_doc_cover(session, doc, file_group, metadata)
# Store data dict.
data = file_group.data or {}
@ -154,14 +154,14 @@ def _replace_doc_sections(session: Session, doc: Doc, sections):
])
def _handle_doc_cover(doc: Doc, file_group: FileGroup, metadata):
def _handle_doc_cover(session: Session, doc: Doc, file_group: FileGroup, metadata):
"""Handle cover extraction and storage for a doc."""
# Check for Calibre cover first (epub only) — must happen before poster check
# since Calibre cover has a different stem ("cover.jpg" not matching epub stem).
if file_group.mimetype and file_group.mimetype.startswith('application/epub'):
if calibre_cover := discover_calibre_cover(file_group.primary_path):
doc._hide_calibre_files(file_group)
doc._hide_calibre_files(session, file_group)
file_group.append_files(calibre_cover)
file_group.data = file_group.data or {}
file_group.data['cover_path'] = calibre_cover.name

View file

@ -5,7 +5,7 @@ from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, In
from sqlalchemy.orm import Mapped, relationship, Session, joinedload
from wrolpi.common import loaded, ModelHelper, Base
from wrolpi.db import get_db_session, serializer
from wrolpi.db import serializer
from wrolpi.files.models import FileGroup
EPUB_MIMETYPE = 'application/epub'
@ -110,8 +110,11 @@ class Doc(ModelHelper, Base):
return cover['path']
return None
def _hide_calibre_files(self, file_group: FileGroup):
"""Delete FileGroups of the cover/metadata when this is a Calibre ebook directory."""
def _hide_calibre_files(self, session: Session, file_group: FileGroup):
"""Delete FileGroups of the cover/metadata when this is a Calibre ebook directory.
Uses the caller's session: the modeler already holds this item's write transaction, and SQLite has one
writer, so a second session would wait out the busy timeout and fail with "database is locked"."""
from modules.docs.lib import discover_calibre_cover
path = file_group.primary_path
if not discover_calibre_cover(path):
@ -120,13 +123,12 @@ class Doc(ModelHelper, Base):
metadata = path.parent / 'metadata.opf'
cover = path.parent / 'cover.jpg'
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup).filter(or_(
FileGroup.primary_path == str(metadata),
FileGroup.primary_path == str(cover),
))
for fg in file_groups:
session.delete(fg)
file_groups = session.query(FileGroup).filter(or_(
FileGroup.primary_path == str(metadata),
FileGroup.primary_path == str(cover),
))
for fg in file_groups:
session.delete(fg)
class DocSection(ModelHelper, Base):

View file

@ -295,6 +295,26 @@ async def test_doc_modeler_calibre_cover(test_session, async_client, test_direct
assert test_session.query(FileGroup).count() == 1
@pytest.mark.asyncio
async def test_doc_modeler_calibre_hides_files_in_the_modelers_session(test_session, async_client, test_directory,
example_epub, example_mobi, image_file,
refresh_files):
"""Hiding a Calibre book's cover and metadata FileGroups uses the modeler's own session. A second session
cannot take SQLite's write lock while the modeler holds the item's transaction: it waited out the busy timeout
and failed, leaving the rows and the ebook unmodeled."""
(test_directory / 'metadata.opf').touch()
shutil.move(image_file, test_directory / 'cover.jpg')
# `create=True`: the fixed module no longer imports it; the patch still catches any reintroduced use.
with patch('modules.docs.models.get_db_session', create=True,
side_effect=AssertionError('opened a second session')):
await refresh_files()
doc: Doc = test_session.query(Doc).one()
assert doc.cover_path
assert test_session.query(FileGroup).count() == 1, 'The cover and metadata FileGroups were deleted'
@pytest.mark.asyncio
async def test_doc_modeler_normalizes_author(async_client, test_session, test_directory, example_epub, refresh_files):
"""Author names are normalized before creating collections (e.g. quotes stripped)."""