perpetual: clear an in-progress flag only when its holder is dead

`with flag:` now records the holder pid in shared_ctx.flag_holders and
drops it on exit.  The reconcile step clears the flags only the perpetual
process ever enters (file_worker_counting, file_worker_discovery,
global_refresh_active) whenever they are set, and the flags API workers
also enter (file_worker_modeling / indexing / cleanup via refresh_sync,
map_search_building via the map rebuild endpoints) only when the
recorded holder is dead.  file_worker_busy is no longer touched: a
collection move in an API worker holds it as a mutex, and
process_queue drops a stale one itself.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-09-26 10:28:42 -06:00
parent 9e3ac4de5c
commit e2ab07eee9
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
4 changed files with 100 additions and 18 deletions

View file

@ -20,6 +20,8 @@ def attach_shared_contexts(app: Sanic):
from wrolpi import flags
app.shared_ctx.flags = manager.dict({i: False for i in flags.FLAG_NAMES})
# Flag name -> pid of the process inside `with flag:` (see Flag.holder_pid).
app.shared_ctx.flag_holders = manager.dict()
# ConfigFile multiprocessing_dict's.
# Shared Configs
@ -230,6 +232,8 @@ def reset_shared_contexts(app: Sanic):
del app.shared_ctx.events_history[:]
app.shared_ctx.single_tasks_started.clear()
app.shared_ctx.flags_initialized.clear()
if hasattr(app.shared_ctx, 'flag_holders'):
app.shared_ctx.flag_holders.clear()
if hasattr(app.shared_ctx, 'perpetual_restart_requested_at'):
app.shared_ctx.perpetual_restart_requested_at.value = 0.0

View file

@ -1,6 +1,7 @@
import asyncio
import contextlib
import multiprocessing
import os
from datetime import datetime
from pathlib import Path
@ -68,6 +69,27 @@ class Flag:
from wrolpi.api_utils import api_app
return api_app.shared_ctx.flags[self.name]
def holder_pid(self) -> int | None:
"""The pid of the process inside `with flag:`, or None if no process is (or the holder was not recorded).
A process killed inside the block never reaches `__exit__`; the perpetual process uses this on startup to
tell an orphaned flag from one a live process is still holding."""
if PYTEST and not TESTING_LOCK.is_set():
return None
from wrolpi.api_utils import api_app
holders = getattr(api_app.shared_ctx, 'flag_holders', None)
return holders.get(self.name) if holders is not None else None
def _record_holder(self, pid: int | None):
from wrolpi.api_utils import api_app
holders = getattr(api_app.shared_ctx, 'flag_holders', None)
if holders is None:
return
if pid is None:
holders.pop(self.name, None)
else:
holders[self.name] = pid
def __enter__(self):
if PYTEST and not TESTING_LOCK.is_set():
# Testing, but the test does not need flags.
@ -76,6 +98,7 @@ class Flag:
if self.is_set():
raise ValueError(f'{self} flag is already set!')
self.set()
self._record_holder(os.getpid())
def __exit__(self, exc_type, exc_val, exc_tb):
if PYTEST and not TESTING_LOCK.is_set():
@ -83,6 +106,7 @@ class Flag:
return
self.clear()
self._record_holder(None)
def _save(self, value: bool):
if self.store_db:

View file

@ -47,12 +47,17 @@ PERPETUAL_CANCEL_TIMEOUT = 15
SHARED_LOCK_NAMES = ('jobs_lock', 'switches_lock', 'config_save_lock', 'config_update_lock', 'events_lock',
'secure_cookies_lock', 'transcode_lock')
SHARED_LOCK_HEAL_TIMEOUT = 10
# Shared flags that mean "the perpetual process is in the middle of this". They are set and cleared by context
# managers in this process only; a SIGKILL inside one leaves it set, and the next attempt raises "flag is already
# set" forever (every file refresh, for instance).
PERPETUAL_PROGRESS_FLAGS = ('file_worker_busy', 'file_worker_counting', 'file_worker_discovery',
'file_worker_modeling', 'file_worker_indexing', 'file_worker_cleanup',
'global_refresh_active')
# In-progress flags. A SIGKILL inside `with flag:` leaves it set and the next enter raises "flag is already set"
# forever (every file refresh, or every map search-index build). Two kinds:
# * entered or set only by the perpetual process (FileWorker.process_queue, handle_refresh): at this process's
# startup any of them still set is orphaned, whoever the recorded holder is;
# * also entered by API workers (refresh_sync from the delete endpoints, the map rebuild endpoints): cleared only
# when the recorded holder pid is dead, because a live worker may be inside the block right now.
# file_worker_busy is deliberately absent: the collection-move path in an API worker holds it as a mutex, and
# process_queue already drops a stale set on its next idle tick.
PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active')
SHARED_PROGRESS_FLAGS = ('file_worker_modeling', 'file_worker_indexing', 'file_worker_cleanup',
'map_search_building')
# True only inside the process started by `run_perpetual_process`.
IN_PERPETUAL_PROCESS = False
@ -160,6 +165,37 @@ def heal_shared_locks(app, timeout: float = SHARED_LOCK_HEAL_TIMEOUT) -> list[st
return healed
def clear_orphaned_flags() -> list[str]:
"""Clear in-progress flags whose holder is gone; returns the names cleared. See PERPETUAL_ONLY_FLAGS."""
from wrolpi import flags
from wrolpi.cmd import pid_is_running
cleared = []
for name in PERPETUAL_ONLY_FLAGS + SHARED_PROGRESS_FLAGS:
try:
flag = getattr(flags, name)
if not flag.is_set():
continue
holder = flag.holder_pid()
if name in PERPETUAL_ONLY_FLAGS:
reason = f'holder pid={holder}' if holder else 'no live holder possible'
elif holder is None:
logger.warning(f'reconcile: flag {name} is set with no recorded holder; leaving it')
continue
elif pid_is_running(holder) and holder != os.getpid():
logger.info(f'reconcile: flag {name} is held by live pid={holder}; leaving it')
continue
else:
reason = f'holder pid={holder} is dead'
logger.warning(f'reconcile: clearing in-progress flag {name} ({reason})')
flag.clear()
flag._record_holder(None)
cleared.append(name)
except Exception as e:
logger.error(f'reconcile: failed to check flag {name}', exc_info=e)
return cleared
def reconcile_after_predecessor(app) -> dict:
"""Undo what a perpetual process that died mid-work left in shared state and the DB.
@ -178,15 +214,7 @@ def reconcile_after_predecessor(app) -> dict:
report['locks'] = heal_shared_locks(app)
for name in PERPETUAL_PROGRESS_FLAGS:
try:
flag = getattr(flags, name)
if flag.is_set():
logger.warning(f'reconcile: clearing in-progress flag {name} left set by a dead process')
flag.clear()
report['flags'].append(name)
except Exception as e:
logger.error(f'reconcile: failed to clear flag {name}', exc_info=e)
report['flags'] = clear_orphaned_flags()
try:
from wrolpi.jobs import fail_orphaned_jobs

View file

@ -314,8 +314,14 @@ async def test_reconcile_after_predecessor(async_client, test_session, monkeypat
created_at='', started_at='', finished_at=None, error=None)
file_worker._set_job_status('fj', 'running')
file_worker.update_status(status='refreshing')
flags.file_worker_counting.set()
flags.global_refresh_active.set()
flags.file_worker_counting.set() # perpetual-only: cleared whoever held it
flags.global_refresh_active.set() # perpetual-only, set() not entered: cleared
flags.file_worker_modeling.set() # shared, holder dead: cleared
api_app.shared_ctx.flag_holders['file_worker_modeling'] = 2 ** 31 - 1
flags.map_search_building.set() # shared, holder is a live API worker: kept
api_app.shared_ctx.flag_holders['map_search_building'] = os.getppid()
flags.file_worker_indexing.set() # shared, no holder recorded: kept (cannot tell)
flags.file_worker_busy.set() # not a reconciled flag (a live move may hold it): kept
flags.db_up.set() # not an in-progress flag; must survive
assert api_app.shared_ctx.jobs_lock.acquire(timeout=1)
monkeypatch.setattr(perpetual, 'SHARED_LOCK_HEAL_TIMEOUT', 0.1)
@ -335,11 +341,31 @@ async def test_reconcile_after_predecessor(async_client, test_session, monkeypat
{'https://example.com/a': 'new', 'https://example.com/b': 'new'}
assert report['file_worker_jobs'] == 1 and file_worker._jobs['fj']['status'] == 'failed'
assert file_worker.status['status'] == 'idle' and report['file_worker_status'] is True
assert report['flags'] == ['file_worker_counting', 'global_refresh_active']
assert report['flags'] == ['file_worker_counting', 'global_refresh_active', 'file_worker_modeling']
assert not flags.file_worker_counting.is_set() and not flags.global_refresh_active.is_set()
assert not flags.file_worker_modeling.is_set() and 'file_worker_modeling' not in api_app.shared_ctx.flag_holders
assert flags.map_search_building.is_set(), 'a flag held by a live process is not touched'
assert flags.file_worker_indexing.is_set(), 'a shared flag with no recorded holder is not touched'
assert flags.file_worker_busy.is_set(), 'file_worker_busy is not reconciled'
assert flags.db_up.is_set(), 'only in-progress flags are cleared'
with flags.file_worker_counting: # usable again; this raised "flag is already set" before
pass
for name in ('map_search_building', 'file_worker_indexing', 'file_worker_busy'):
getattr(flags, name).clear()
api_app.shared_ctx.flag_holders.clear()
# Idempotent: a clean state reports nothing.
assert not any(reconcile_after_predecessor(api_app).values())
def test_flag_context_manager_records_its_holder(async_client, flags_lock):
"""`with flag:` records this pid so a later reconcile can tell an orphaned flag from a live holder."""
from wrolpi import flags
from wrolpi.api_utils import api_app
assert flags.file_worker_modeling.holder_pid() is None
with flags.file_worker_modeling:
assert flags.file_worker_modeling.holder_pid() == os.getpid()
assert api_app.shared_ctx.flag_holders['file_worker_modeling'] == os.getpid()
assert flags.file_worker_modeling.holder_pid() is None
assert not flags.file_worker_modeling.is_set()