mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 16:06:19 -05:00
Prevent doc_modeler from looping on persistent modeling failures
Addresses review feedback: the new `Doc.id IS NULL` gate keeps matching a doc whose _model_doc fails without persisting a Doc row. With >=DOC_PROCESSING_LIMIT such docs, every batch re-selected the same failing files, `processed` never dropped below the limit, and the loop hung -- starving later docs. Reproduced with a patched _model_doc that always raises. Track attempted FileGroup ids per run and exclude them from the query, so the candidate pool strictly shrinks and the loop always terminates. Failed docs are retried on the next refresh (unchanged behavior). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
56c3a16839
commit
821c3991be
2 changed files with 41 additions and 3 deletions
|
|
@ -25,9 +25,13 @@ DOC_PROCESSING_LIMIT = 10
|
|||
async def doc_modeler(progress_callback: Callable[[int], None] = None):
|
||||
"""Searches for doc files (epub, mobi, pdf, docx, doc, odt, cbz, cbr) and models them."""
|
||||
total_processed = 0
|
||||
# Track FileGroups already attempted in this run so a doc that fails to model (and therefore
|
||||
# keeps matching the `Doc.id IS NULL` gate below) cannot be re-selected forever, which would
|
||||
# hang this loop and starve later docs. Failed docs are simply retried on the next refresh.
|
||||
attempted_ids: set = set()
|
||||
while True:
|
||||
with get_db_session(commit=True) as session:
|
||||
file_groups = session.query(FileGroup, Doc) \
|
||||
query = session.query(FileGroup, Doc) \
|
||||
.outerjoin(Doc, Doc.file_group_id == FileGroup.id) \
|
||||
.filter(
|
||||
or_(
|
||||
|
|
@ -42,12 +46,15 @@ async def doc_modeler(progress_callback: Callable[[int], None] = None):
|
|||
Doc.id.is_(None),
|
||||
FileGroup.indexed == False,
|
||||
),
|
||||
).limit(DOC_PROCESSING_LIMIT)
|
||||
file_groups: List[Tuple[FileGroup, Doc]] = list(file_groups)
|
||||
)
|
||||
if attempted_ids:
|
||||
query = query.filter(FileGroup.id.notin_(attempted_ids))
|
||||
file_groups: List[Tuple[FileGroup, Doc]] = list(query.limit(DOC_PROCESSING_LIMIT))
|
||||
|
||||
processed = 0
|
||||
for file_group, doc in file_groups:
|
||||
processed += 1
|
||||
attempted_ids.add(file_group.id)
|
||||
try:
|
||||
if PYTEST:
|
||||
session.expire(file_group)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"""Tests for the doc_modeler."""
|
||||
import asyncio
|
||||
import shutil
|
||||
from unittest.mock import patch
|
||||
|
||||
|
|
@ -142,6 +143,36 @@ async def test_doc_modeler_models_indexed_filegroup_without_doc(async_client, te
|
|||
assert doc.file_group.model == 'doc'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_terminates_on_persistent_failure(async_client, test_session, test_directory):
|
||||
"""doc_modeler must terminate even when modeling consistently fails and never creates a Doc row.
|
||||
|
||||
Regression: the `Doc.id IS NULL` gate keeps matching a doc that fails to model. Without a
|
||||
per-run guard, a batch of >=DOC_PROCESSING_LIMIT permanently-failing docs is re-selected every
|
||||
iteration, so the loop never breaks and later docs are starved.
|
||||
"""
|
||||
ebook_dir = test_directory / 'ebooks'
|
||||
ebook_dir.mkdir(parents=True)
|
||||
num_docs = 15 # More than one batch (DOC_PROCESSING_LIMIT == 10).
|
||||
for i in range(num_docs):
|
||||
path = ebook_dir / f'test_ebook_{i:03d}.epub'
|
||||
shutil.copy(PROJECT_DIR / 'test/ebook example.epub', path)
|
||||
for path in ebook_dir.iterdir():
|
||||
fg = FileGroup.from_paths(test_session, path)
|
||||
fg.indexed = True # Simulate apply_indexers having already run.
|
||||
test_session.commit()
|
||||
|
||||
# _model_doc always raises without persisting a Doc row (patch replaces the whole function).
|
||||
# PYTEST re-raises inside the loop's per-item handler; disable that so we exercise the failure
|
||||
# path and the loop's termination guard, not the re-raise.
|
||||
with patch('modules.docs._model_doc', side_effect=RuntimeError('boom')), \
|
||||
patch('modules.docs.PYTEST', False):
|
||||
await asyncio.wait_for(doc_modeler(), timeout=10)
|
||||
|
||||
# No Doc rows were created (every attempt failed), but the loop still terminated.
|
||||
assert test_session.query(Doc).count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_modeler_processes_more_than_batch_limit(async_client, test_session, test_directory):
|
||||
"""Doc modeler processes more than one batch of files (limit=10)."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue