mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 06:56:18 -05:00
files: quieter batch-modeler errors; drop unused imports
Break out of a batch page on InvalidRequestError after a flush failure so the rest of the page does not each log the same poisoned session. Re-raise PYTEST apply failures without a misleading commit-failure message. Remove asyncio/List/OperationalError/PYTEST/Iterable leftovers from docs, archive, and zim.
This commit is contained in:
parent
9b5f7474ff
commit
3f5e462300
4 changed files with 10 additions and 9 deletions
|
|
@ -1,11 +1,10 @@
|
|||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
from abc import ABC
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, List, Optional, Tuple, Iterable
|
||||
from typing import Callable, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
import asyncio
|
||||
import logging
|
||||
from typing import Callable, List
|
||||
from typing import Callable
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import register_modeler, register_refresh_cleanup, \
|
||||
|
|
@ -12,7 +10,7 @@ from wrolpi.db import get_db_session
|
|||
from wrolpi.files.indexers import Indexer
|
||||
from wrolpi.files.modeler import run_modeler_loop, skip_clause
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST, FILE_MAX_TEXT_SIZE
|
||||
from wrolpi.vars import FILE_MAX_TEXT_SIZE
|
||||
from .extractors import extract_metadata
|
||||
from .lib import get_or_create_subject_collection, get_or_create_author_collection, is_valid_author, split_authors, \
|
||||
normalize_author, normalize_subject, split_subjects, is_valid_subject, discover_calibre_cover
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import asyncio
|
||||
import functools
|
||||
import json
|
||||
import pathlib
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
from typing import Awaitable, Callable, Iterable
|
||||
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.exc import InvalidRequestError, OperationalError
|
||||
|
||||
from wrolpi.common import logger
|
||||
from wrolpi.db import get_db_session
|
||||
|
|
@ -105,6 +105,11 @@ async def _apply_batch(
|
|||
except SkipModeler:
|
||||
file_group.indexed = False
|
||||
skip_ids.add(file_group.id)
|
||||
except InvalidRequestError:
|
||||
# A previous apply already flushed and rolled the session back.
|
||||
# Remaining items would each log the same error; skip the page.
|
||||
skip_ids.update(ids)
|
||||
break
|
||||
except Exception as e:
|
||||
skip_ids.add(file_group.id)
|
||||
if mark_indexed_on_failure:
|
||||
|
|
@ -116,9 +121,9 @@ async def _apply_batch(
|
|||
# One poisoned commit must not kill the modeler. Skip this page for the rest of
|
||||
# the run; the next refresh retries it.
|
||||
skip_ids.update(ids)
|
||||
logger.error(f'{name}: failed to commit batch of {len(ids)} FileGroups', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
logger.error(f'{name}: failed to commit batch of {len(ids)} FileGroups', exc_info=e)
|
||||
|
||||
|
||||
async def _apply_per_item(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue