From bf749bd87663b9f170c9a93926fa381b67fda330 Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 20 Jul 2026 14:50:22 -0600 Subject: [PATCH] Tolerate transient "database is locked" in download dispatch When downloading is enabled at startup, the download worker races a burst of concurrent SQLite writes on the single DB file: retry_downloads() has just marked every download 'new', RSS catalog refreshes call create_downloads() (which fires its own background dispatch_downloads()), multiple download workers claim downloads, and save_downloads_config re-saves the config. dispatch_downloads reads-then-commits in a deferred transaction, so when another writer commits mid-cycle it fails with SQLITE_BUSY_SNAPSHOT ("database is locked", not absorbed by busy_timeout). This is self-healing -- the perpetual worker retries on its next cycle and downloads succeed -- but the raised OperationalError flooded the logs with full tracebacks and tripped the Sanic traceback renderer (tracerite) into its "Variable inspector failed" secondary errors. Split the claim-and-dispatch DB work into _dispatch_new_downloads() and have dispatch_downloads() swallow a transient lock (debug log, return) so the next cycle retries cleanly. Non-lock database errors still propagate. The split keeps the retry decision outside the transaction that also awaits the download signal dispatches, so a lock is never handled by re-dispatching (no double-dispatch). Co-Authored-By: Claude Opus 4.8 (1M context) --- wrolpi/downloader.py | 18 ++++++++++++++++++ wrolpi/test/test_downloader.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 4243ea4e..e44569c4 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -25,6 +25,7 @@ import feedparser import pytz from feedparser import FeedParserDict from sqlalchemy import Column, Integer, String, Text, ForeignKey, Index, JSON +from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session, relationship from wrolpi import flags @@ -1060,6 +1061,23 @@ class DownloadManager: return # Find download whose domain isn't already being downloaded. + try: + await self._dispatch_new_downloads() + except OperationalError as e: + if 'locked' in str(e).lower(): + # Transient SQLite write contention (e.g. the burst of RSS catalog refreshes, + # download claims, and config saves right after downloading is enabled at startup). + # This dispatch cycle is a no-op; the perpetual worker retries on its next cycle, so + # don't raise -- a raised error here floods the logs with tracebacks (and trips the + # traceback renderer's "Variable inspector failed" noise) for a self-healing condition. + self.log_debug('dispatch_downloads skipped: database is locked (will retry next cycle)') + return + raise + + async def _dispatch_new_downloads(self): + """Claim and dispatch the next batch of `new` downloads. Split out from `dispatch_downloads` + so a transient "database is locked" can be handled there without retrying the signal + dispatches inside this transaction.""" with get_db_session(commit=True) as session: new_downloads = list(session.query(Download).filter( Download.status == 'new', diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index fb9539c7..f050992a 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -1,6 +1,7 @@ import asyncio import json import pathlib +import sqlite3 from abc import ABC from datetime import datetime, time, timezone, timedelta from http import HTTPStatus @@ -10,6 +11,7 @@ from unittest import mock import pytest import pytz import yaml +from sqlalchemy.exc import OperationalError from wrolpi.api_utils import api_app from wrolpi.common import get_wrolpi_config, normalize_domain @@ -1706,6 +1708,32 @@ async def test_daily_limit_per_domain_allows_under_limit(test_session, test_down assert new.last_download_attempt is not None +@pytest.mark.asyncio +async def test_dispatch_downloads_tolerates_locked_db(test_session, test_download_manager, test_downloader): + """A transient 'database is locked' during dispatch is swallowed (retried next cycle), not raised. + + Regression: once downloading is enabled at startup, the download worker races the write burst + (RSS catalog refresh, download claims, config saves) on the single SQLite file. A raised + OperationalError flooded the logs with tracebacks -- and tripped the traceback renderer's + 'Variable inspector failed' noise -- for a condition the next perpetual cycle resolves on its own. + """ + locked = OperationalError('SELECT', {}, sqlite3.OperationalError('database is locked')) + with mock.patch.object(test_download_manager, '_dispatch_new_downloads', + new_callable=mock.AsyncMock, side_effect=locked): + # Must return normally, not raise. + await test_download_manager.dispatch_downloads() + + +@pytest.mark.asyncio +async def test_dispatch_downloads_reraises_other_db_errors(test_session, test_download_manager, test_downloader): + """Only transient locks are tolerated; a real database error still propagates.""" + other = OperationalError('SELECT', {}, sqlite3.OperationalError('no such table: download')) + with mock.patch.object(test_download_manager, '_dispatch_new_downloads', + new_callable=mock.AsyncMock, side_effect=other): + with pytest.raises(OperationalError): + await test_download_manager.dispatch_downloads() + + @pytest.mark.asyncio async def test_daily_limit_resets_next_day(test_session, test_download_manager, test_downloader): """Downloads attempted on a previous day do not count against today's limit."""