From 821c3991be1b40a8b78e559155c3d0dccd278fda Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 20 Jul 2026 13:05:55 -0600 Subject: [PATCH] Prevent doc_modeler from looping on persistent modeling failures Addresses review feedback: the new `Doc.id IS NULL` gate keeps matching a doc whose _model_doc fails without persisting a Doc row. With >=DOC_PROCESSING_LIMIT such docs, every batch re-selected the same failing files, `processed` never dropped below the limit, and the loop hung -- starving later docs. Reproduced with a patched _model_doc that always raises. Track attempted FileGroup ids per run and exclude them from the query, so the candidate pool strictly shrinks and the loop always terminates. Failed docs are retried on the next refresh (unchanged behavior). Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/docs/__init__.py | 13 ++++++++--- modules/docs/test/test_doc_modeler.py | 31 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/modules/docs/__init__.py b/modules/docs/__init__.py index 1f6d8afe..b72f3fa5 100644 --- a/modules/docs/__init__.py +++ b/modules/docs/__init__.py @@ -25,9 +25,13 @@ DOC_PROCESSING_LIMIT = 10 async def doc_modeler(progress_callback: Callable[[int], None] = None): """Searches for doc files (epub, mobi, pdf, docx, doc, odt, cbz, cbr) and models them.""" total_processed = 0 + # Track FileGroups already attempted in this run so a doc that fails to model (and therefore + # keeps matching the `Doc.id IS NULL` gate below) cannot be re-selected forever, which would + # hang this loop and starve later docs. Failed docs are simply retried on the next refresh. + attempted_ids: set = set() while True: with get_db_session(commit=True) as session: - file_groups = session.query(FileGroup, Doc) \ + query = session.query(FileGroup, Doc) \ .outerjoin(Doc, Doc.file_group_id == FileGroup.id) \ .filter( or_( @@ -42,12 +46,15 @@ async def doc_modeler(progress_callback: Callable[[int], None] = None): Doc.id.is_(None), FileGroup.indexed == False, ), - ).limit(DOC_PROCESSING_LIMIT) - file_groups: List[Tuple[FileGroup, Doc]] = list(file_groups) + ) + if attempted_ids: + query = query.filter(FileGroup.id.notin_(attempted_ids)) + file_groups: List[Tuple[FileGroup, Doc]] = list(query.limit(DOC_PROCESSING_LIMIT)) processed = 0 for file_group, doc in file_groups: processed += 1 + attempted_ids.add(file_group.id) try: if PYTEST: session.expire(file_group) diff --git a/modules/docs/test/test_doc_modeler.py b/modules/docs/test/test_doc_modeler.py index 8a129df1..68927d10 100644 --- a/modules/docs/test/test_doc_modeler.py +++ b/modules/docs/test/test_doc_modeler.py @@ -1,4 +1,5 @@ """Tests for the doc_modeler.""" +import asyncio import shutil from unittest.mock import patch @@ -142,6 +143,36 @@ async def test_doc_modeler_models_indexed_filegroup_without_doc(async_client, te assert doc.file_group.model == 'doc' +@pytest.mark.asyncio +async def test_doc_modeler_terminates_on_persistent_failure(async_client, test_session, test_directory): + """doc_modeler must terminate even when modeling consistently fails and never creates a Doc row. + + Regression: the `Doc.id IS NULL` gate keeps matching a doc that fails to model. Without a + per-run guard, a batch of >=DOC_PROCESSING_LIMIT permanently-failing docs is re-selected every + iteration, so the loop never breaks and later docs are starved. + """ + ebook_dir = test_directory / 'ebooks' + ebook_dir.mkdir(parents=True) + num_docs = 15 # More than one batch (DOC_PROCESSING_LIMIT == 10). + for i in range(num_docs): + path = ebook_dir / f'test_ebook_{i:03d}.epub' + shutil.copy(PROJECT_DIR / 'test/ebook example.epub', path) + for path in ebook_dir.iterdir(): + fg = FileGroup.from_paths(test_session, path) + fg.indexed = True # Simulate apply_indexers having already run. + test_session.commit() + + # _model_doc always raises without persisting a Doc row (patch replaces the whole function). + # PYTEST re-raises inside the loop's per-item handler; disable that so we exercise the failure + # path and the loop's termination guard, not the re-raise. + with patch('modules.docs._model_doc', side_effect=RuntimeError('boom')), \ + patch('modules.docs.PYTEST', False): + await asyncio.wait_for(doc_modeler(), timeout=10) + + # No Doc rows were created (every attempt failed), but the loop still terminated. + assert test_session.query(Doc).count() == 0 + + @pytest.mark.asyncio async def test_doc_modeler_processes_more_than_batch_limit(async_client, test_session, test_directory): """Doc modeler processes more than one batch of files (limit=10)."""