Tolerate transient "database is locked" in download dispatch

When downloading is enabled at startup, the download worker races a burst
of concurrent SQLite writes on the single DB file: retry_downloads() has
just marked every download 'new', RSS catalog refreshes call
create_downloads() (which fires its own background dispatch_downloads()),
multiple download workers claim downloads, and save_downloads_config
re-saves the config. dispatch_downloads reads-then-commits in a deferred
transaction, so when another writer commits mid-cycle it fails with
SQLITE_BUSY_SNAPSHOT ("database is locked", not absorbed by busy_timeout).

This is self-healing -- the perpetual worker retries on its next cycle and
downloads succeed -- but the raised OperationalError flooded the logs with
full tracebacks and tripped the Sanic traceback renderer (tracerite) into
its "Variable inspector failed" secondary errors.

Split the claim-and-dispatch DB work into _dispatch_new_downloads() and
have dispatch_downloads() swallow a transient lock (debug log, return) so
the next cycle retries cleanly. Non-lock database errors still propagate.
The split keeps the retry decision outside the transaction that also
awaits the download signal dispatches, so a lock is never handled by
re-dispatching (no double-dispatch).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Roland 2026-07-20 14:50:22 -06:00 • committed by lrnselfreliance
parent 9310fe801f
commit bf749bd876
2 changed files with 46 additions and 0 deletions

View file

@ -25,6 +25,7 @@ import feedparser
import pytz
from feedparser import FeedParserDict
from sqlalchemy import Column, Integer, String, Text, ForeignKey, Index, JSON
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session, relationship
from wrolpi import flags
@ -1060,6 +1061,23 @@ class DownloadManager:
return
# Find download whose domain isn't already being downloaded.
try:
await self._dispatch_new_downloads()
except OperationalError as e:
if 'locked' in str(e).lower():
# Transient SQLite write contention (e.g. the burst of RSS catalog refreshes,
# download claims, and config saves right after downloading is enabled at startup).
# This dispatch cycle is a no-op; the perpetual worker retries on its next cycle, so
# don't raise -- a raised error here floods the logs with tracebacks (and trips the
# traceback renderer's "Variable inspector failed" noise) for a self-healing condition.
self.log_debug('dispatch_downloads skipped: database is locked (will retry next cycle)')
return
raise
async def _dispatch_new_downloads(self):
"""Claim and dispatch the next batch of `new` downloads. Split out from `dispatch_downloads`
so a transient "database is locked" can be handled there without retrying the signal
dispatches inside this transaction."""
with get_db_session(commit=True) as session:
new_downloads = list(session.query(Download).filter(
Download.status == 'new',

View file

@ -1,6 +1,7 @@
import asyncio
import json
import pathlib
import sqlite3
from abc import ABC
from datetime import datetime, time, timezone, timedelta
from http import HTTPStatus
@ -10,6 +11,7 @@ from unittest import mock
import pytest
import pytz
import yaml
from sqlalchemy.exc import OperationalError
from wrolpi.api_utils import api_app
from wrolpi.common import get_wrolpi_config, normalize_domain
@ -1706,6 +1708,32 @@ async def test_daily_limit_per_domain_allows_under_limit(test_session, test_down
assert new.last_download_attempt is not None
@pytest.mark.asyncio
async def test_dispatch_downloads_tolerates_locked_db(test_session, test_download_manager, test_downloader):
"""A transient 'database is locked' during dispatch is swallowed (retried next cycle), not raised.
Regression: once downloading is enabled at startup, the download worker races the write burst
(RSS catalog refresh, download claims, config saves) on the single SQLite file. A raised
OperationalError flooded the logs with tracebacks -- and tripped the traceback renderer's
'Variable inspector failed' noise -- for a condition the next perpetual cycle resolves on its own.
"""
locked = OperationalError('SELECT', {}, sqlite3.OperationalError('database is locked'))
with mock.patch.object(test_download_manager, '_dispatch_new_downloads',
new_callable=mock.AsyncMock, side_effect=locked):
# Must return normally, not raise.
await test_download_manager.dispatch_downloads()
@pytest.mark.asyncio
async def test_dispatch_downloads_reraises_other_db_errors(test_session, test_download_manager, test_downloader):
"""Only transient locks are tolerated; a real database error still propagates."""
other = OperationalError('SELECT', {}, sqlite3.OperationalError('no such table: download'))
with mock.patch.object(test_download_manager, '_dispatch_new_downloads',
new_callable=mock.AsyncMock, side_effect=other):
with pytest.raises(OperationalError):
await test_download_manager.dispatch_downloads()
@pytest.mark.asyncio
async def test_daily_limit_resets_next_day(test_session, test_download_manager, test_downloader):
"""Downloads attempted on a previous day do not count against today's limit."""