From c0452e29e6509142856cc306ac24cf5b481719d2 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:04:19 -0700 Subject: [PATCH] Hiding a Calibre book's cover and metadata uses the modeler's session (#936) Doc._hide_calibre_files opened a second write session while the modeler held the item's transaction. SQLite has one writer, so the second session waited out the 30 s busy timeout and failed with "database is locked"; the modeler retried three times, so every Calibre epub cost about 98 s and stayed unmodeled, and the cover.jpg / metadata.opf FileGroups it should have deleted survived. On a library of 1663 Calibre books the global refresh's modeling phase would have taken about two days. The nested session dates from before the SQLite cutover, when Postgres row locks let it work. The delete now runs in the caller's session. Co-authored-by: Claude Fable 5.1 --- modules/docs/__init__.py | 6 +++--- modules/docs/models.py | 22 ++++++++++++---------- modules/docs/test/test_doc_modeler.py | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/modules/docs/__init__.py b/modules/docs/__init__.py index efa9d9db..66976e8c 100644 --- a/modules/docs/__init__.py +++ b/modules/docs/__init__.py @@ -110,7 +110,7 @@ def _model_doc(file_group: FileGroup, session: Session) -> Doc: doc.size = file_group.primary_path.stat().st_size if file_group.primary_path.exists() else None # Handle cover. - _handle_doc_cover(doc, file_group, metadata) + _handle_doc_cover(session, doc, file_group, metadata) # Store data dict. data = file_group.data or {} @@ -154,14 +154,14 @@ def _replace_doc_sections(session: Session, doc: Doc, sections): ]) -def _handle_doc_cover(doc: Doc, file_group: FileGroup, metadata): +def _handle_doc_cover(session: Session, doc: Doc, file_group: FileGroup, metadata): """Handle cover extraction and storage for a doc.""" # Check for Calibre cover first (epub only) — must happen before poster check # since Calibre cover has a different stem ("cover.jpg" not matching epub stem). if file_group.mimetype and file_group.mimetype.startswith('application/epub'): if calibre_cover := discover_calibre_cover(file_group.primary_path): - doc._hide_calibre_files(file_group) + doc._hide_calibre_files(session, file_group) file_group.append_files(calibre_cover) file_group.data = file_group.data or {} file_group.data['cover_path'] = calibre_cover.name diff --git a/modules/docs/models.py b/modules/docs/models.py index 16f35fcf..1112b9ca 100644 --- a/modules/docs/models.py +++ b/modules/docs/models.py @@ -5,7 +5,7 @@ from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, In from sqlalchemy.orm import Mapped, relationship, Session, joinedload from wrolpi.common import loaded, ModelHelper, Base -from wrolpi.db import get_db_session, serializer +from wrolpi.db import serializer from wrolpi.files.models import FileGroup EPUB_MIMETYPE = 'application/epub' @@ -110,8 +110,11 @@ class Doc(ModelHelper, Base): return cover['path'] return None - def _hide_calibre_files(self, file_group: FileGroup): - """Delete FileGroups of the cover/metadata when this is a Calibre ebook directory.""" + def _hide_calibre_files(self, session: Session, file_group: FileGroup): + """Delete FileGroups of the cover/metadata when this is a Calibre ebook directory. + + Uses the caller's session: the modeler already holds this item's write transaction, and SQLite has one + writer, so a second session would wait out the busy timeout and fail with "database is locked".""" from modules.docs.lib import discover_calibre_cover path = file_group.primary_path if not discover_calibre_cover(path): @@ -120,13 +123,12 @@ class Doc(ModelHelper, Base): metadata = path.parent / 'metadata.opf' cover = path.parent / 'cover.jpg' - with get_db_session(commit=True) as session: - file_groups = session.query(FileGroup).filter(or_( - FileGroup.primary_path == str(metadata), - FileGroup.primary_path == str(cover), - )) - for fg in file_groups: - session.delete(fg) + file_groups = session.query(FileGroup).filter(or_( + FileGroup.primary_path == str(metadata), + FileGroup.primary_path == str(cover), + )) + for fg in file_groups: + session.delete(fg) class DocSection(ModelHelper, Base): diff --git a/modules/docs/test/test_doc_modeler.py b/modules/docs/test/test_doc_modeler.py index 8fcda4fd..3fbc35e8 100644 --- a/modules/docs/test/test_doc_modeler.py +++ b/modules/docs/test/test_doc_modeler.py @@ -295,6 +295,26 @@ async def test_doc_modeler_calibre_cover(test_session, async_client, test_direct assert test_session.query(FileGroup).count() == 1 +@pytest.mark.asyncio +async def test_doc_modeler_calibre_hides_files_in_the_modelers_session(test_session, async_client, test_directory, + example_epub, example_mobi, image_file, + refresh_files): + """Hiding a Calibre book's cover and metadata FileGroups uses the modeler's own session. A second session + cannot take SQLite's write lock while the modeler holds the item's transaction: it waited out the busy timeout + and failed, leaving the rows and the ebook unmodeled.""" + (test_directory / 'metadata.opf').touch() + shutil.move(image_file, test_directory / 'cover.jpg') + + # `create=True`: the fixed module no longer imports it; the patch still catches any reintroduced use. + with patch('modules.docs.models.get_db_session', create=True, + side_effect=AssertionError('opened a second session')): + await refresh_files() + + doc: Doc = test_session.query(Doc).one() + assert doc.cover_path + assert test_session.query(FileGroup).count() == 1, 'The cover and metadata FileGroups were deleted' + + @pytest.mark.asyncio async def test_doc_modeler_normalizes_author(async_client, test_session, test_directory, example_epub, refresh_files): """Author names are normalized before creating collections (e.g. quotes stripped)."""