mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
downloader: don't leak processing_domains on failed claim or missing row
Addresses two processing_domains leaks raised in review; each could permanently exclude a domain and eventually stall the scheduler until restart. 1. Claim-transaction failure (Greptile P1): `_add_processing_domain` mutated shared, non-transactional state before the claim transaction committed. A commit failure skipped dispatch but left the domain claimed forever. Fix: defer both the processing_domain add AND the signal dispatch to after the transaction commits; track claimed domains in a local `claimed` list so the loop's own dedup / concurrency cap is unchanged. 2. Row deleted before dispatch (Greptile P1): if a claimed download was deleted between commit and the signal running, `Download.find_by_id` raised before `download_domain` was set, so the handler's `finally` released nothing. Fix: derive `download_domain` from the URL (`urlparse(url).netloc`, the same value the dispatcher claimed) up front, so cleanup always runs. Regression tests written bug-first (both fail before this change): - test_signal_releases_processing_domain_when_download_missing - test_dispatch_releases_claimed_domains_when_transaction_fails Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
f341e386ad
commit
cd11b5b7b7
2 changed files with 82 additions and 18 deletions
|
|
@ -1113,13 +1113,19 @@ class DownloadManager:
|
|||
Uses `get_immediate_db_session` because this is a read-then-write transaction (it reads the
|
||||
`new` downloads, then claims each by setting `last_download_attempt`); a deferred lock
|
||||
upgrade here fails instantly with "database is locked" under concurrency."""
|
||||
# (download_id, url) pairs claimed this cycle. The signals are dispatched AFTER the
|
||||
# immediate (write) transaction below commits: dispatching inside it deadlocks, because the
|
||||
# awaited `signal_download_download` opens its own write session (which also emits
|
||||
# `BEGIN IMMEDIATE`, inheriting `_immediate_txn`) and blocks on the write lock this
|
||||
# transaction still holds — timing out after `busy_timeout` with "database is locked" so no
|
||||
# download ever starts.
|
||||
# Downloads claimed this cycle: dicts of (download_id, download_url, domain). Two things are
|
||||
# deferred until AFTER the immediate (write) transaction below commits:
|
||||
# * `_add_processing_domain` — it mutates shared, non-transactional state, so adding it
|
||||
# before commit would leak the domain forever if the transaction fails (dispatch is then
|
||||
# skipped and no signal handler ever runs to release it);
|
||||
# * `api_app.dispatch` — dispatching inside the transaction deadlocks, because the awaited
|
||||
# `signal_download_download` opens its own write session (also `BEGIN IMMEDIATE`,
|
||||
# inheriting `_immediate_txn`) and blocks on the write lock this transaction still holds,
|
||||
# timing out after `busy_timeout` with "database is locked" so no download ever starts.
|
||||
# `claimed` tracks the domains taken this cycle so the loop's own dedup / concurrency cap
|
||||
# still holds even though `processing_domains` is not mutated until after commit.
|
||||
to_dispatch = []
|
||||
claimed = []
|
||||
with get_immediate_db_session() as session:
|
||||
new_downloads = list(session.query(Download).filter(
|
||||
Download.status == 'new',
|
||||
|
|
@ -1166,8 +1172,8 @@ class DownloadManager:
|
|||
# Non-recurring downloads with limits enabled are grouped by their normalized domain.
|
||||
limit_domain = self._normalize_download_domain(download) \
|
||||
if (limits_enabled and not is_recurring) else None
|
||||
if domain not in self.processing_domains and len(
|
||||
self.processing_domains) < SIMULTANEOUS_DOWNLOAD_DOMAINS:
|
||||
if domain not in self.processing_domains and domain not in claimed and (
|
||||
len(self.processing_domains) + len(claimed)) < SIMULTANEOUS_DOWNLOAD_DOMAINS:
|
||||
# Daily download limits (recurring downloads bypass the gate entirely).
|
||||
if limit_domain is not None:
|
||||
if limit_global and global_count >= limit_global:
|
||||
|
|
@ -1199,14 +1205,16 @@ class DownloadManager:
|
|||
global_count += 1
|
||||
domain_counts[limit_domain] = domain_counts.get(limit_domain, 0) + 1
|
||||
|
||||
self._add_processing_domain(domain)
|
||||
# Claim now; dispatch after the transaction commits (see `to_dispatch` above).
|
||||
to_dispatch.append(dict(download_id=download.id, download_url=download.url))
|
||||
# Claim now (in the DB via last_download_attempt); the shared processing_domain is
|
||||
# added and the signal dispatched only after this transaction commits.
|
||||
claimed.append(domain)
|
||||
to_dispatch.append(dict(download_id=download.id, download_url=download.url, domain=domain))
|
||||
count += 1
|
||||
else:
|
||||
# Domain is already being processed, or all worker slots are in use. A domain
|
||||
# that stays here across many cycles points to a leaked processing_domains slot.
|
||||
reason = 'domain already processing' if domain in self.processing_domains \
|
||||
# Domain is already being processed (this cycle or a prior one), or all worker
|
||||
# slots are in use. A domain that stays here across many cycles points to a
|
||||
# leaked processing_domains slot.
|
||||
reason = 'domain already processing' if (domain in self.processing_domains or domain in claimed) \
|
||||
else 'all download slots in use'
|
||||
skip_reasons[reason] = skip_reasons.get(reason, 0) + 1
|
||||
if count:
|
||||
|
|
@ -1224,9 +1232,12 @@ class DownloadManager:
|
|||
except (AttributeError, RuntimeError):
|
||||
pass
|
||||
|
||||
# The immediate (write) transaction has now committed and released the write lock, so the
|
||||
# download signal handlers can open their own write sessions without deadlocking.
|
||||
# The immediate (write) transaction has now committed and released the write lock. Claim the
|
||||
# shared processing_domain and dispatch each signal: doing both here (not inside the txn)
|
||||
# means a failed transaction leaks nothing, and the handler's write session cannot deadlock
|
||||
# against a lock we no longer hold.
|
||||
for context in to_dispatch:
|
||||
self._add_processing_domain(context.pop('domain'))
|
||||
await api_app.dispatch('wrolpi.download.download', context=context)
|
||||
|
||||
async def do_downloads(self):
|
||||
|
|
@ -1728,7 +1739,11 @@ async def signal_download_download(download_id: int, download_url: str):
|
|||
|
||||
with timer('signal_download_download', 'trace'):
|
||||
url = download_url
|
||||
download_domain = None
|
||||
# Derive the processing-domain from the URL up front (it is `urlparse(url).netloc`, the same
|
||||
# value the dispatcher claimed): the download row may already be gone (deleted after the
|
||||
# claim committed), in which case `find_by_id` raises before we could read `download.domain`
|
||||
# -- and the `finally` below must still release the claim, or the domain leaks forever.
|
||||
download_domain = urlparse(download_url).netloc
|
||||
|
||||
name = f'download_worker'
|
||||
worker_logger = logger.getChild(name)
|
||||
|
|
@ -1740,7 +1755,6 @@ async def signal_download_download(download_id: int, download_url: str):
|
|||
# Mark the download as started in new session so the change is committed.
|
||||
download = Download.find_by_id(session, download_id)
|
||||
download.started()
|
||||
download_domain = download.domain
|
||||
|
||||
downloader: Downloader = download.get_downloader()
|
||||
if not downloader:
|
||||
|
|
|
|||
|
|
@ -1744,6 +1744,56 @@ async def test_download_dispatched_outside_immediate_transaction(test_session, t
|
|||
'download signal was dispatched inside an open BEGIN IMMEDIATE transaction (deadlocks under NullPool)'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_signal_releases_processing_domain_when_download_missing(test_session, test_download_manager,
|
||||
test_downloader):
|
||||
"""A claimed processing_domain must be released even if the Download row is gone by dispatch time.
|
||||
|
||||
Regression (Greptile P1): if a claimed download is deleted between the dispatcher committing the
|
||||
claim and `signal_download_download` running, `Download.find_by_id` raises `UnknownDownload`
|
||||
before `download_domain` is assigned, so the handler's `finally` released nothing -- leaking the
|
||||
processing domain. Enough leaked domains permanently exclude their domains and stall the
|
||||
scheduler until restart."""
|
||||
from wrolpi.downloader import signal_download_download, download_manager
|
||||
from wrolpi.conftest import production_like_sessions
|
||||
|
||||
download_manager._add_processing_domain('example.com')
|
||||
assert 'example.com' in download_manager.processing_domains
|
||||
|
||||
with production_like_sessions(test_session):
|
||||
# No Download row with this id exists (it was deleted after the claim committed).
|
||||
await signal_download_download(999_999, 'https://example.com/deleted')
|
||||
|
||||
assert 'example.com' not in download_manager.processing_domains, \
|
||||
'processing_domain leaked when the download row was missing at dispatch time'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_releases_claimed_domains_when_transaction_fails(test_session, test_download_manager,
|
||||
test_downloader):
|
||||
"""Domains claimed during a dispatch cycle must be released if that cycle's transaction fails.
|
||||
|
||||
Regression (Greptile P1): `_add_processing_domain` mutates shared, non-transactional state before
|
||||
the claim transaction commits. If the commit (or any error in the claim loop) fails, dispatch is
|
||||
skipped but the domain stays claimed forever -- future cycles exclude it, and enough leaks stall
|
||||
the scheduler until restart."""
|
||||
name = test_downloader.name
|
||||
d = Download(url='https://example.com/new', downloader=name, status='new')
|
||||
test_session.add(d)
|
||||
test_session.commit()
|
||||
assert 'example.com' not in test_download_manager.processing_domains
|
||||
|
||||
# Fail the claim transaction's commit (e.g. an I/O error while the write lock is held).
|
||||
locked = OperationalError('COMMIT', {}, sqlite3.OperationalError('database is locked'))
|
||||
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock), \
|
||||
mock.patch.object(test_session, 'commit', side_effect=locked):
|
||||
# A transient lock is swallowed by dispatch_downloads; it must not leave the claim leaked.
|
||||
await test_download_manager.dispatch_downloads()
|
||||
|
||||
assert 'example.com' not in test_download_manager.processing_domains, \
|
||||
'processing_domain leaked after the claim transaction failed'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatch_downloads_tolerates_locked_db(test_session, test_download_manager, test_downloader):
|
||||
"""A transient 'database is locked' during dispatch is swallowed (retried next cycle), not raised.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue