Prevent doc_modeler from looping on persistent modeling failures

Addresses review feedback: the new `Doc.id IS NULL` gate keeps matching a
doc whose _model_doc fails without persisting a Doc row. With
>=DOC_PROCESSING_LIMIT such docs, every batch re-selected the same failing
files, `processed` never dropped below the limit, and the loop hung --
starving later docs. Reproduced with a patched _model_doc that always
raises.

Track attempted FileGroup ids per run and exclude them from the query, so
the candidate pool strictly shrinks and the loop always terminates. Failed
docs are retried on the next refresh (unchanged behavior).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Roland 2026-07-20 13:05:55 -06:00
parent 56c3a16839
commit 821c3991be
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
2 changed files with 41 additions and 3 deletions

View file

@ -25,9 +25,13 @@ DOC_PROCESSING_LIMIT = 10
async def doc_modeler(progress_callback: Callable[[int], None] = None):
"""Searches for doc files (epub, mobi, pdf, docx, doc, odt, cbz, cbr) and models them."""
total_processed = 0
# Track FileGroups already attempted in this run so a doc that fails to model (and therefore
# keeps matching the `Doc.id IS NULL` gate below) cannot be re-selected forever, which would
# hang this loop and starve later docs. Failed docs are simply retried on the next refresh.
attempted_ids: set = set()
while True:
with get_db_session(commit=True) as session:
file_groups = session.query(FileGroup, Doc) \
query = session.query(FileGroup, Doc) \
.outerjoin(Doc, Doc.file_group_id == FileGroup.id) \
.filter(
or_(
@ -42,12 +46,15 @@ async def doc_modeler(progress_callback: Callable[[int], None] = None):
Doc.id.is_(None),
FileGroup.indexed == False,
),
).limit(DOC_PROCESSING_LIMIT)
file_groups: List[Tuple[FileGroup, Doc]] = list(file_groups)
)
if attempted_ids:
query = query.filter(FileGroup.id.notin_(attempted_ids))
file_groups: List[Tuple[FileGroup, Doc]] = list(query.limit(DOC_PROCESSING_LIMIT))
processed = 0
for file_group, doc in file_groups:
processed += 1
attempted_ids.add(file_group.id)
try:
if PYTEST:
session.expire(file_group)

View file

@ -1,4 +1,5 @@
"""Tests for the doc_modeler."""
import asyncio
import shutil
from unittest.mock import patch
@ -142,6 +143,36 @@ async def test_doc_modeler_models_indexed_filegroup_without_doc(async_client, te
assert doc.file_group.model == 'doc'
@pytest.mark.asyncio
async def test_doc_modeler_terminates_on_persistent_failure(async_client, test_session, test_directory):
"""doc_modeler must terminate even when modeling consistently fails and never creates a Doc row.
Regression: the `Doc.id IS NULL` gate keeps matching a doc that fails to model. Without a
per-run guard, a batch of >=DOC_PROCESSING_LIMIT permanently-failing docs is re-selected every
iteration, so the loop never breaks and later docs are starved.
"""
ebook_dir = test_directory / 'ebooks'
ebook_dir.mkdir(parents=True)
num_docs = 15 # More than one batch (DOC_PROCESSING_LIMIT == 10).
for i in range(num_docs):
path = ebook_dir / f'test_ebook_{i:03d}.epub'
shutil.copy(PROJECT_DIR / 'test/ebook example.epub', path)
for path in ebook_dir.iterdir():
fg = FileGroup.from_paths(test_session, path)
fg.indexed = True # Simulate apply_indexers having already run.
test_session.commit()
# _model_doc always raises without persisting a Doc row (patch replaces the whole function).
# PYTEST re-raises inside the loop's per-item handler; disable that so we exercise the failure
# path and the loop's termination guard, not the re-raise.
with patch('modules.docs._model_doc', side_effect=RuntimeError('boom')), \
patch('modules.docs.PYTEST', False):
await asyncio.wait_for(doc_modeler(), timeout=10)
# No Doc rows were created (every attempt failed), but the loop still terminated.
assert test_session.query(Doc).count() == 0
@pytest.mark.asyncio
async def test_doc_modeler_processes_more_than_batch_limit(async_client, test_session, test_directory):
"""Doc modeler processes more than one batch of files (limit=10)."""