diff --git a/modules/docs/__init__.py b/modules/docs/__init__.py index 1f6d8afe..b72f3fa5 100644 --- a/modules/docs/__init__.py +++ b/modules/docs/__init__.py @@ -25,9 +25,13 @@ DOC_PROCESSING_LIMIT = 10 async def doc_modeler(progress_callback: Callable[[int], None] = None): """Searches for doc files (epub, mobi, pdf, docx, doc, odt, cbz, cbr) and models them.""" total_processed = 0 + # Track FileGroups already attempted in this run so a doc that fails to model (and therefore + # keeps matching the `Doc.id IS NULL` gate below) cannot be re-selected forever, which would + # hang this loop and starve later docs. Failed docs are simply retried on the next refresh. + attempted_ids: set = set() while True: with get_db_session(commit=True) as session: - file_groups = session.query(FileGroup, Doc) \ + query = session.query(FileGroup, Doc) \ .outerjoin(Doc, Doc.file_group_id == FileGroup.id) \ .filter( or_( @@ -42,12 +46,15 @@ async def doc_modeler(progress_callback: Callable[[int], None] = None): Doc.id.is_(None), FileGroup.indexed == False, ), - ).limit(DOC_PROCESSING_LIMIT) - file_groups: List[Tuple[FileGroup, Doc]] = list(file_groups) + ) + if attempted_ids: + query = query.filter(FileGroup.id.notin_(attempted_ids)) + file_groups: List[Tuple[FileGroup, Doc]] = list(query.limit(DOC_PROCESSING_LIMIT)) processed = 0 for file_group, doc in file_groups: processed += 1 + attempted_ids.add(file_group.id) try: if PYTEST: session.expire(file_group) diff --git a/modules/docs/test/test_doc_modeler.py b/modules/docs/test/test_doc_modeler.py index 8a129df1..68927d10 100644 --- a/modules/docs/test/test_doc_modeler.py +++ b/modules/docs/test/test_doc_modeler.py @@ -1,4 +1,5 @@ """Tests for the doc_modeler.""" +import asyncio import shutil from unittest.mock import patch @@ -142,6 +143,36 @@ async def test_doc_modeler_models_indexed_filegroup_without_doc(async_client, te assert doc.file_group.model == 'doc' +@pytest.mark.asyncio +async def test_doc_modeler_terminates_on_persistent_failure(async_client, test_session, test_directory): + """doc_modeler must terminate even when modeling consistently fails and never creates a Doc row. + + Regression: the `Doc.id IS NULL` gate keeps matching a doc that fails to model. Without a + per-run guard, a batch of >=DOC_PROCESSING_LIMIT permanently-failing docs is re-selected every + iteration, so the loop never breaks and later docs are starved. + """ + ebook_dir = test_directory / 'ebooks' + ebook_dir.mkdir(parents=True) + num_docs = 15 # More than one batch (DOC_PROCESSING_LIMIT == 10). + for i in range(num_docs): + path = ebook_dir / f'test_ebook_{i:03d}.epub' + shutil.copy(PROJECT_DIR / 'test/ebook example.epub', path) + for path in ebook_dir.iterdir(): + fg = FileGroup.from_paths(test_session, path) + fg.indexed = True # Simulate apply_indexers having already run. + test_session.commit() + + # _model_doc always raises without persisting a Doc row (patch replaces the whole function). + # PYTEST re-raises inside the loop's per-item handler; disable that so we exercise the failure + # path and the loop's termination guard, not the re-raise. + with patch('modules.docs._model_doc', side_effect=RuntimeError('boom')), \ + patch('modules.docs.PYTEST', False): + await asyncio.wait_for(doc_modeler(), timeout=10) + + # No Doc rows were created (every attempt failed), but the loop still terminated. + assert test_session.query(Doc).count() == 0 + + @pytest.mark.asyncio async def test_doc_modeler_processes_more_than_batch_limit(async_client, test_session, test_directory): """Doc modeler processes more than one batch of files (limit=10)."""