mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 15:36:18 -05:00
Hiding a Calibre book's cover and metadata uses the modeler's session (#936)
Doc._hide_calibre_files opened a second write session while the modeler held the item's transaction. SQLite has one writer, so the second session waited out the 30 s busy timeout and failed with "database is locked"; the modeler retried three times, so every Calibre epub cost about 98 s and stayed unmodeled, and the cover.jpg / metadata.opf FileGroups it should have deleted survived. On a library of 1663 Calibre books the global refresh's modeling phase would have taken about two days. The nested session dates from before the SQLite cutover, when Postgres row locks let it work. The delete now runs in the caller's session. Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
e6b7b9e5dd
commit
c0452e29e6
3 changed files with 35 additions and 13 deletions
|
|
@ -110,7 +110,7 @@ def _model_doc(file_group: FileGroup, session: Session) -> Doc:
|
|||
doc.size = file_group.primary_path.stat().st_size if file_group.primary_path.exists() else None
|
||||
|
||||
# Handle cover.
|
||||
_handle_doc_cover(doc, file_group, metadata)
|
||||
_handle_doc_cover(session, doc, file_group, metadata)
|
||||
|
||||
# Store data dict.
|
||||
data = file_group.data or {}
|
||||
|
|
@ -154,14 +154,14 @@ def _replace_doc_sections(session: Session, doc: Doc, sections):
|
|||
])
|
||||
|
||||
|
||||
def _handle_doc_cover(doc: Doc, file_group: FileGroup, metadata):
|
||||
def _handle_doc_cover(session: Session, doc: Doc, file_group: FileGroup, metadata):
|
||||
"""Handle cover extraction and storage for a doc."""
|
||||
|
||||
# Check for Calibre cover first (epub only) — must happen before poster check
|
||||
# since Calibre cover has a different stem ("cover.jpg" not matching epub stem).
|
||||
if file_group.mimetype and file_group.mimetype.startswith('application/epub'):
|
||||
if calibre_cover := discover_calibre_cover(file_group.primary_path):
|
||||
doc._hide_calibre_files(file_group)
|
||||
doc._hide_calibre_files(session, file_group)
|
||||
file_group.append_files(calibre_cover)
|
||||
file_group.data = file_group.data or {}
|
||||
file_group.data['cover_path'] = calibre_cover.name
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, In
|
|||
from sqlalchemy.orm import Mapped, relationship, Session, joinedload
|
||||
|
||||
from wrolpi.common import loaded, ModelHelper, Base
|
||||
from wrolpi.db import get_db_session, serializer
|
||||
from wrolpi.db import serializer
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
EPUB_MIMETYPE = 'application/epub'
|
||||
|
|
@ -110,8 +110,11 @@ class Doc(ModelHelper, Base):
|
|||
return cover['path']
|
||||
return None
|
||||
|
||||
def _hide_calibre_files(self, file_group: FileGroup):
|
||||
"""Delete FileGroups of the cover/metadata when this is a Calibre ebook directory."""
|
||||
def _hide_calibre_files(self, session: Session, file_group: FileGroup):
|
||||
"""Delete FileGroups of the cover/metadata when this is a Calibre ebook directory.
|
||||
|
||||
Uses the caller's session: the modeler already holds this item's write transaction, and SQLite has one
|
||||
writer, so a second session would wait out the busy timeout and fail with "database is locked"."""
|
||||
from modules.docs.lib import discover_calibre_cover
|
||||
path = file_group.primary_path
|
||||
if not discover_calibre_cover(path):
|
||||
|
|
@ -120,13 +123,12 @@ class Doc(ModelHelper, Base):
|
|||
metadata = path.parent / 'metadata.opf'
|
||||
cover = path.parent / 'cover.jpg'
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
file_groups = session.query(FileGroup).filter(or_(
|
||||
FileGroup.primary_path == str(metadata),
|
||||
FileGroup.primary_path == str(cover),
|
||||
))
|
||||
for fg in file_groups:
|
||||
session.delete(fg)
|
||||
file_groups = session.query(FileGroup).filter(or_(
|
||||
FileGroup.primary_path == str(metadata),
|
||||
FileGroup.primary_path == str(cover),
|
||||
))
|
||||
for fg in file_groups:
|
||||
session.delete(fg)
|
||||
|
||||
|
||||
class DocSection(ModelHelper, Base):
|
||||
|
|
|
|||
|
|
@ -295,6 +295,26 @@ async def test_doc_modeler_calibre_cover(test_session, async_client, test_direct
|
|||
assert test_session.query(FileGroup).count() == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_calibre_hides_files_in_the_modelers_session(test_session, async_client, test_directory,
|
||||
example_epub, example_mobi, image_file,
|
||||
refresh_files):
|
||||
"""Hiding a Calibre book's cover and metadata FileGroups uses the modeler's own session. A second session
|
||||
cannot take SQLite's write lock while the modeler holds the item's transaction: it waited out the busy timeout
|
||||
and failed, leaving the rows and the ebook unmodeled."""
|
||||
(test_directory / 'metadata.opf').touch()
|
||||
shutil.move(image_file, test_directory / 'cover.jpg')
|
||||
|
||||
# `create=True`: the fixed module no longer imports it; the patch still catches any reintroduced use.
|
||||
with patch('modules.docs.models.get_db_session', create=True,
|
||||
side_effect=AssertionError('opened a second session')):
|
||||
await refresh_files()
|
||||
|
||||
doc: Doc = test_session.query(Doc).one()
|
||||
assert doc.cover_path
|
||||
assert test_session.query(FileGroup).count() == 1, 'The cover and metadata FileGroups were deleted'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_normalizes_author(async_client, test_session, test_directory, example_epub, refresh_files):
|
||||
"""Author names are normalized before creating collections (e.g. quotes stripped)."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue