mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Tolerate transient "database is locked" in download dispatch
When downloading is enabled at startup, the download worker races a burst
of concurrent SQLite writes on the single DB file: retry_downloads() has
just marked every download 'new', RSS catalog refreshes call
create_downloads() (which fires its own background dispatch_downloads()),
multiple download workers claim downloads, and save_downloads_config
re-saves the config. dispatch_downloads reads-then-commits in a deferred
transaction, so when another writer commits mid-cycle it fails with
SQLITE_BUSY_SNAPSHOT ("database is locked", not absorbed by busy_timeout).
This is self-healing -- the perpetual worker retries on its next cycle and
downloads succeed -- but the raised OperationalError flooded the logs with
full tracebacks and tripped the Sanic traceback renderer (tracerite) into
its "Variable inspector failed" secondary errors.
Split the claim-and-dispatch DB work into _dispatch_new_downloads() and
have dispatch_downloads() swallow a transient lock (debug log, return) so
the next cycle retries cleanly. Non-lock database errors still propagate.
The split keeps the retry decision outside the transaction that also
awaits the download signal dispatches, so a lock is never handled by
re-dispatching (no double-dispatch).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
9310fe801f
commit
bf749bd876
2 changed files with 46 additions and 0 deletions
|
|
@ -25,6 +25,7 @@ import feedparser
|
|||
import pytz
|
||||
from feedparser import FeedParserDict
|
||||
from sqlalchemy import Column, Integer, String, Text, ForeignKey, Index, JSON
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.orm import Session, relationship
|
||||
|
||||
from wrolpi import flags
|
||||
|
|
@ -1060,6 +1061,23 @@ class DownloadManager:
|
|||
return
|
||||
|
||||
# Find download whose domain isn't already being downloaded.
|
||||
try:
|
||||
await self._dispatch_new_downloads()
|
||||
except OperationalError as e:
|
||||
if 'locked' in str(e).lower():
|
||||
# Transient SQLite write contention (e.g. the burst of RSS catalog refreshes,
|
||||
# download claims, and config saves right after downloading is enabled at startup).
|
||||
# This dispatch cycle is a no-op; the perpetual worker retries on its next cycle, so
|
||||
# don't raise -- a raised error here floods the logs with tracebacks (and trips the
|
||||
# traceback renderer's "Variable inspector failed" noise) for a self-healing condition.
|
||||
self.log_debug('dispatch_downloads skipped: database is locked (will retry next cycle)')
|
||||
return
|
||||
raise
|
||||
|
||||
async def _dispatch_new_downloads(self):
|
||||
"""Claim and dispatch the next batch of `new` downloads. Split out from `dispatch_downloads`
|
||||
so a transient "database is locked" can be handled there without retrying the signal
|
||||
dispatches inside this transaction."""
|
||||
with get_db_session(commit=True) as session:
|
||||
new_downloads = list(session.query(Download).filter(
|
||||
Download.status == 'new',
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import pathlib
|
||||
import sqlite3
|
||||
from abc import ABC
|
||||
from datetime import datetime, time, timezone, timedelta
|
||||
from http import HTTPStatus
|
||||
|
|
@ -10,6 +11,7 @@ from unittest import mock
|
|||
import pytest
|
||||
import pytz
|
||||
import yaml
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.common import get_wrolpi_config, normalize_domain
|
||||
|
|
@ -1706,6 +1708,32 @@ async def test_daily_limit_per_domain_allows_under_limit(test_session, test_down
|
|||
assert new.last_download_attempt is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_downloads_tolerates_locked_db(test_session, test_download_manager, test_downloader):
|
||||
"""A transient 'database is locked' during dispatch is swallowed (retried next cycle), not raised.
|
||||
|
||||
Regression: once downloading is enabled at startup, the download worker races the write burst
|
||||
(RSS catalog refresh, download claims, config saves) on the single SQLite file. A raised
|
||||
OperationalError flooded the logs with tracebacks -- and tripped the traceback renderer's
|
||||
'Variable inspector failed' noise -- for a condition the next perpetual cycle resolves on its own.
|
||||
"""
|
||||
locked = OperationalError('SELECT', {}, sqlite3.OperationalError('database is locked'))
|
||||
with mock.patch.object(test_download_manager, '_dispatch_new_downloads',
|
||||
new_callable=mock.AsyncMock, side_effect=locked):
|
||||
# Must return normally, not raise.
|
||||
await test_download_manager.dispatch_downloads()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_downloads_reraises_other_db_errors(test_session, test_download_manager, test_downloader):
|
||||
"""Only transient locks are tolerated; a real database error still propagates."""
|
||||
other = OperationalError('SELECT', {}, sqlite3.OperationalError('no such table: download'))
|
||||
with mock.patch.object(test_download_manager, '_dispatch_new_downloads',
|
||||
new_callable=mock.AsyncMock, side_effect=other):
|
||||
with pytest.raises(OperationalError):
|
||||
await test_download_manager.dispatch_downloads()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_limit_resets_next_day(test_session, test_download_manager, test_downloader):
|
||||
"""Downloads attempted on a previous day do not count against today's limit."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue