diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 90ff7168..85af6412 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -1,11 +1,10 @@ -import asyncio import json import os import pathlib import tempfile from abc import ABC from dataclasses import dataclass -from typing import Callable, List, Optional, Tuple, Iterable +from typing import Callable, List, Optional, Tuple from sqlalchemy.orm import Session diff --git a/modules/docs/__init__.py b/modules/docs/__init__.py index 4419f051..b7454156 100644 --- a/modules/docs/__init__.py +++ b/modules/docs/__init__.py @@ -1,9 +1,7 @@ -import asyncio import logging -from typing import Callable, List +from typing import Callable from sqlalchemy import or_ -from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session from wrolpi.common import register_modeler, register_refresh_cleanup, \ @@ -12,7 +10,7 @@ from wrolpi.db import get_db_session from wrolpi.files.indexers import Indexer from wrolpi.files.modeler import run_modeler_loop, skip_clause from wrolpi.files.models import FileGroup -from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE +from wrolpi.vars import FILE_MAX_TEXT_SIZE from .extractors import extract_metadata from .lib import get_or_create_subject_collection, get_or_create_author_collection, is_valid_author, split_authors, \ normalize_author, normalize_subject, split_subjects, is_valid_subject, discover_calibre_cover diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 1a9eaf41..95761df2 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -1,4 +1,3 @@ -import asyncio import functools import json import pathlib diff --git a/wrolpi/files/modeler.py b/wrolpi/files/modeler.py index 1cb63947..ac9b1cc9 100644 --- a/wrolpi/files/modeler.py +++ b/wrolpi/files/modeler.py @@ -17,7 +17,7 @@ from __future__ import annotations import asyncio from typing import Awaitable, Callable, Iterable -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import InvalidRequestError, OperationalError from wrolpi.common import logger from wrolpi.db import get_db_session @@ -105,6 +105,11 @@ async def _apply_batch( except SkipModeler: file_group.indexed = False skip_ids.add(file_group.id) + except InvalidRequestError: + # A previous apply already flushed and rolled the session back. + # Remaining items would each log the same error; skip the page. + skip_ids.update(ids) + break except Exception as e: skip_ids.add(file_group.id) if mark_indexed_on_failure: @@ -116,9 +121,9 @@ async def _apply_batch( # One poisoned commit must not kill the modeler. Skip this page for the rest of # the run; the next refresh retries it. skip_ids.update(ids) - logger.error(f'{name}: failed to commit batch of {len(ids)} FileGroups', exc_info=e) if PYTEST: raise + logger.error(f'{name}: failed to commit batch of {len(ids)} FileGroups', exc_info=e) async def _apply_per_item(