files: quieter batch-modeler errors; drop unused imports

Break out of a batch page on InvalidRequestError after a flush failure
so the rest of the page does not each log the same poisoned session.
Re-raise PYTEST apply failures without a misleading commit-failure
message.  Remove asyncio/List/OperationalError/PYTEST/Iterable leftovers
from docs, archive, and zim.
This commit is contained in:
Roland 2026-09-24 16:40:26 -06:00
parent 9b5f7474ff
commit 3f5e462300
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
4 changed files with 10 additions and 9 deletions

View file

@ -1,11 +1,10 @@
import asyncio
import json
import os
import pathlib
import tempfile
from abc import ABC
from dataclasses import dataclass
from typing import Callable, List, Optional, Tuple, Iterable
from typing import Callable, List, Optional, Tuple
from sqlalchemy.orm import Session

View file

@ -1,9 +1,7 @@
import asyncio
import logging
from typing import Callable, List
from typing import Callable
from sqlalchemy import or_
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session
from wrolpi.common import register_modeler, register_refresh_cleanup, \
@ -12,7 +10,7 @@ from wrolpi.db import get_db_session
from wrolpi.files.indexers import Indexer
from wrolpi.files.modeler import run_modeler_loop, skip_clause
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
from wrolpi.vars import FILE_MAX_TEXT_SIZE
from .extractors import extract_metadata
from .lib import get_or_create_subject_collection, get_or_create_author_collection, is_valid_author, split_authors, \
normalize_author, normalize_subject, split_subjects, is_valid_subject, discover_calibre_cover

View file

@ -1,4 +1,3 @@
import asyncio
import functools
import json
import pathlib

View file

@ -17,7 +17,7 @@ from __future__ import annotations
import asyncio
from typing import Awaitable, Callable, Iterable
from sqlalchemy.exc import OperationalError
from sqlalchemy.exc import InvalidRequestError, OperationalError
from wrolpi.common import logger
from wrolpi.db import get_db_session
@ -105,6 +105,11 @@ async def _apply_batch(
except SkipModeler:
file_group.indexed = False
skip_ids.add(file_group.id)
except InvalidRequestError:
# A previous apply already flushed and rolled the session back.
# Remaining items would each log the same error; skip the page.
skip_ids.update(ids)
break
except Exception as e:
skip_ids.add(file_group.id)
if mark_indexed_on_failure:
@ -116,9 +121,9 @@ async def _apply_batch(
# One poisoned commit must not kill the modeler. Skip this page for the rest of
# the run; the next refresh retries it.
skip_ids.update(ids)
logger.error(f'{name}: failed to commit batch of {len(ids)} FileGroups', exc_info=e)
if PYTEST:
raise
logger.error(f'{name}: failed to commit batch of {len(ids)} FileGroups', exc_info=e)
async def _apply_per_item(