From cd11b5b7b7795b37b2a72ea6e1a820ce1af4f934 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 23 Jul 2026 10:23:24 -0600 Subject: [PATCH] downloader: don't leak processing_domains on failed claim or missing row Addresses two processing_domains leaks raised in review; each could permanently exclude a domain and eventually stall the scheduler until restart. 1. Claim-transaction failure (Greptile P1): `_add_processing_domain` mutated shared, non-transactional state before the claim transaction committed. A commit failure skipped dispatch but left the domain claimed forever. Fix: defer both the processing_domain add AND the signal dispatch to after the transaction commits; track claimed domains in a local `claimed` list so the loop's own dedup / concurrency cap is unchanged. 2. Row deleted before dispatch (Greptile P1): if a claimed download was deleted between commit and the signal running, `Download.find_by_id` raised before `download_domain` was set, so the handler's `finally` released nothing. Fix: derive `download_domain` from the URL (`urlparse(url).netloc`, the same value the dispatcher claimed) up front, so cleanup always runs. Regression tests written bug-first (both fail before this change): - test_signal_releases_processing_domain_when_download_missing - test_dispatch_releases_claimed_domains_when_transaction_fails Co-Authored-By: Claude Opus 4.8 (1M context) --- wrolpi/downloader.py | 50 ++++++++++++++++++++++------------ wrolpi/test/test_downloader.py | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 18 deletions(-) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 90eb2036..57ff5617 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1113,13 +1113,19 @@ class DownloadManager: Uses `get_immediate_db_session` because this is a read-then-write transaction (it reads the `new` downloads, then claims each by setting `last_download_attempt`); a deferred lock upgrade here fails instantly with "database is locked" under concurrency.""" - # (download_id, url) pairs claimed this cycle. The signals are dispatched AFTER the - # immediate (write) transaction below commits: dispatching inside it deadlocks, because the - # awaited `signal_download_download` opens its own write session (which also emits - # `BEGIN IMMEDIATE`, inheriting `_immediate_txn`) and blocks on the write lock this - # transaction still holds — timing out after `busy_timeout` with "database is locked" so no - # download ever starts. + # Downloads claimed this cycle: dicts of (download_id, download_url, domain). Two things are + # deferred until AFTER the immediate (write) transaction below commits: + # * `_add_processing_domain` — it mutates shared, non-transactional state, so adding it + # before commit would leak the domain forever if the transaction fails (dispatch is then + # skipped and no signal handler ever runs to release it); + # * `api_app.dispatch` — dispatching inside the transaction deadlocks, because the awaited + # `signal_download_download` opens its own write session (also `BEGIN IMMEDIATE`, + # inheriting `_immediate_txn`) and blocks on the write lock this transaction still holds, + # timing out after `busy_timeout` with "database is locked" so no download ever starts. + # `claimed` tracks the domains taken this cycle so the loop's own dedup / concurrency cap + # still holds even though `processing_domains` is not mutated until after commit. to_dispatch = [] + claimed = [] with get_immediate_db_session() as session: new_downloads = list(session.query(Download).filter( Download.status == 'new', @@ -1166,8 +1172,8 @@ class DownloadManager: # Non-recurring downloads with limits enabled are grouped by their normalized domain. limit_domain = self._normalize_download_domain(download) \ if (limits_enabled and not is_recurring) else None - if domain not in self.processing_domains and len( - self.processing_domains) < SIMULTANEOUS_DOWNLOAD_DOMAINS: + if domain not in self.processing_domains and domain not in claimed and ( + len(self.processing_domains) + len(claimed)) < SIMULTANEOUS_DOWNLOAD_DOMAINS: # Daily download limits (recurring downloads bypass the gate entirely). if limit_domain is not None: if limit_global and global_count >= limit_global: @@ -1199,14 +1205,16 @@ class DownloadManager: global_count += 1 domain_counts[limit_domain] = domain_counts.get(limit_domain, 0) + 1 - self._add_processing_domain(domain) - # Claim now; dispatch after the transaction commits (see `to_dispatch` above). - to_dispatch.append(dict(download_id=download.id, download_url=download.url)) + # Claim now (in the DB via last_download_attempt); the shared processing_domain is + # added and the signal dispatched only after this transaction commits. + claimed.append(domain) + to_dispatch.append(dict(download_id=download.id, download_url=download.url, domain=domain)) count += 1 else: - # Domain is already being processed, or all worker slots are in use. A domain - # that stays here across many cycles points to a leaked processing_domains slot. - reason = 'domain already processing' if domain in self.processing_domains \ + # Domain is already being processed (this cycle or a prior one), or all worker + # slots are in use. A domain that stays here across many cycles points to a + # leaked processing_domains slot. + reason = 'domain already processing' if (domain in self.processing_domains or domain in claimed) \ else 'all download slots in use' skip_reasons[reason] = skip_reasons.get(reason, 0) + 1 if count: @@ -1224,9 +1232,12 @@ class DownloadManager: except (AttributeError, RuntimeError): pass - # The immediate (write) transaction has now committed and released the write lock, so the - # download signal handlers can open their own write sessions without deadlocking. + # The immediate (write) transaction has now committed and released the write lock. Claim the + # shared processing_domain and dispatch each signal: doing both here (not inside the txn) + # means a failed transaction leaks nothing, and the handler's write session cannot deadlock + # against a lock we no longer hold. for context in to_dispatch: + self._add_processing_domain(context.pop('domain')) await api_app.dispatch('wrolpi.download.download', context=context) async def do_downloads(self): @@ -1728,7 +1739,11 @@ async def signal_download_download(download_id: int, download_url: str): with timer('signal_download_download', 'trace'): url = download_url - download_domain = None + # Derive the processing-domain from the URL up front (it is `urlparse(url).netloc`, the same + # value the dispatcher claimed): the download row may already be gone (deleted after the + # claim committed), in which case `find_by_id` raises before we could read `download.domain` + # -- and the `finally` below must still release the claim, or the domain leaks forever. + download_domain = urlparse(download_url).netloc name = f'download_worker' worker_logger = logger.getChild(name) @@ -1740,7 +1755,6 @@ async def signal_download_download(download_id: int, download_url: str): # Mark the download as started in new session so the change is committed. download = Download.find_by_id(session, download_id) download.started() - download_domain = download.domain downloader: Downloader = download.get_downloader() if not downloader: diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 67c506f2..0ed351d4 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -1744,6 +1744,56 @@ async def test_download_dispatched_outside_immediate_transaction(test_session, t 'download signal was dispatched inside an open BEGIN IMMEDIATE transaction (deadlocks under NullPool)' +@pytest.mark.asyncio +async def test_signal_releases_processing_domain_when_download_missing(test_session, test_download_manager, + test_downloader): + """A claimed processing_domain must be released even if the Download row is gone by dispatch time. + + Regression (Greptile P1): if a claimed download is deleted between the dispatcher committing the + claim and `signal_download_download` running, `Download.find_by_id` raises `UnknownDownload` + before `download_domain` is assigned, so the handler's `finally` released nothing -- leaking the + processing domain. Enough leaked domains permanently exclude their domains and stall the + scheduler until restart.""" + from wrolpi.downloader import signal_download_download, download_manager + from wrolpi.conftest import production_like_sessions + + download_manager._add_processing_domain('example.com') + assert 'example.com' in download_manager.processing_domains + + with production_like_sessions(test_session): + # No Download row with this id exists (it was deleted after the claim committed). + await signal_download_download(999_999, 'https://example.com/deleted') + + assert 'example.com' not in download_manager.processing_domains, \ + 'processing_domain leaked when the download row was missing at dispatch time' + + +@pytest.mark.asyncio +async def test_dispatch_releases_claimed_domains_when_transaction_fails(test_session, test_download_manager, + test_downloader): + """Domains claimed during a dispatch cycle must be released if that cycle's transaction fails. + + Regression (Greptile P1): `_add_processing_domain` mutates shared, non-transactional state before + the claim transaction commits. If the commit (or any error in the claim loop) fails, dispatch is + skipped but the domain stays claimed forever -- future cycles exclude it, and enough leaks stall + the scheduler until restart.""" + name = test_downloader.name + d = Download(url='https://example.com/new', downloader=name, status='new') + test_session.add(d) + test_session.commit() + assert 'example.com' not in test_download_manager.processing_domains + + # Fail the claim transaction's commit (e.g. an I/O error while the write lock is held). + locked = OperationalError('COMMIT', {}, sqlite3.OperationalError('database is locked')) + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock), \ + mock.patch.object(test_session, 'commit', side_effect=locked): + # A transient lock is swallowed by dispatch_downloads; it must not leave the claim leaked. + await test_download_manager.dispatch_downloads() + + assert 'example.com' not in test_download_manager.processing_domains, \ + 'processing_domain leaked after the claim transaction failed' + + @pytest.mark.asyncio async def test_dispatch_downloads_tolerates_locked_db(test_session, test_download_manager, test_downloader): """A transient 'database is locked' during dispatch is swallowed (retried next cycle), not raised.