diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index d961cdf2..2f55cabc 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -20,6 +20,8 @@ def attach_shared_contexts(app: Sanic): from wrolpi import flags app.shared_ctx.flags = manager.dict({i: False for i in flags.FLAG_NAMES}) + # Flag name -> pid of the process inside `with flag:` (see Flag.holder_pid). + app.shared_ctx.flag_holders = manager.dict() # ConfigFile multiprocessing_dict's. # Shared Configs @@ -230,6 +232,8 @@ def reset_shared_contexts(app: Sanic): del app.shared_ctx.events_history[:] app.shared_ctx.single_tasks_started.clear() app.shared_ctx.flags_initialized.clear() + if hasattr(app.shared_ctx, 'flag_holders'): + app.shared_ctx.flag_holders.clear() if hasattr(app.shared_ctx, 'perpetual_restart_requested_at'): app.shared_ctx.perpetual_restart_requested_at.value = 0.0 diff --git a/wrolpi/flags.py b/wrolpi/flags.py index dbc59a3b..049a288f 100644 --- a/wrolpi/flags.py +++ b/wrolpi/flags.py @@ -1,6 +1,7 @@ import asyncio import contextlib import multiprocessing +import os from datetime import datetime from pathlib import Path @@ -68,6 +69,27 @@ class Flag: from wrolpi.api_utils import api_app return api_app.shared_ctx.flags[self.name] + def holder_pid(self) -> int | None: + """The pid of the process inside `with flag:`, or None if no process is (or the holder was not recorded). + + A process killed inside the block never reaches `__exit__`; the perpetual process uses this on startup to + tell an orphaned flag from one a live process is still holding.""" + if PYTEST and not TESTING_LOCK.is_set(): + return None + from wrolpi.api_utils import api_app + holders = getattr(api_app.shared_ctx, 'flag_holders', None) + return holders.get(self.name) if holders is not None else None + + def _record_holder(self, pid: int | None): + from wrolpi.api_utils import api_app + holders = getattr(api_app.shared_ctx, 'flag_holders', None) + if holders is None: + return + if pid is None: + holders.pop(self.name, None) + else: + holders[self.name] = pid + def __enter__(self): if PYTEST and not TESTING_LOCK.is_set(): # Testing, but the test does not need flags. @@ -76,6 +98,7 @@ class Flag: if self.is_set(): raise ValueError(f'{self} flag is already set!') self.set() + self._record_holder(os.getpid()) def __exit__(self, exc_type, exc_val, exc_tb): if PYTEST and not TESTING_LOCK.is_set(): @@ -83,6 +106,7 @@ class Flag: return self.clear() + self._record_holder(None) def _save(self, value: bool): if self.store_db: diff --git a/wrolpi/perpetual.py b/wrolpi/perpetual.py index 213c8fd9..afa76abb 100644 --- a/wrolpi/perpetual.py +++ b/wrolpi/perpetual.py @@ -47,12 +47,17 @@ PERPETUAL_CANCEL_TIMEOUT = 15 SHARED_LOCK_NAMES = ('jobs_lock', 'switches_lock', 'config_save_lock', 'config_update_lock', 'events_lock', 'secure_cookies_lock', 'transcode_lock') SHARED_LOCK_HEAL_TIMEOUT = 10 -# Shared flags that mean "the perpetual process is in the middle of this". They are set and cleared by context -# managers in this process only; a SIGKILL inside one leaves it set, and the next attempt raises "flag is already -# set" forever (every file refresh, for instance). -PERPETUAL_PROGRESS_FLAGS = ('file_worker_busy', 'file_worker_counting', 'file_worker_discovery', - 'file_worker_modeling', 'file_worker_indexing', 'file_worker_cleanup', - 'global_refresh_active') +# In-progress flags. A SIGKILL inside `with flag:` leaves it set and the next enter raises "flag is already set" +# forever (every file refresh, or every map search-index build). Two kinds: +# * entered or set only by the perpetual process (FileWorker.process_queue, handle_refresh): at this process's +# startup any of them still set is orphaned, whoever the recorded holder is; +# * also entered by API workers (refresh_sync from the delete endpoints, the map rebuild endpoints): cleared only +# when the recorded holder pid is dead, because a live worker may be inside the block right now. +# file_worker_busy is deliberately absent: the collection-move path in an API worker holds it as a mutex, and +# process_queue already drops a stale set on its next idle tick. +PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active') +SHARED_PROGRESS_FLAGS = ('file_worker_modeling', 'file_worker_indexing', 'file_worker_cleanup', + 'map_search_building') # True only inside the process started by `run_perpetual_process`. IN_PERPETUAL_PROCESS = False @@ -160,6 +165,37 @@ def heal_shared_locks(app, timeout: float = SHARED_LOCK_HEAL_TIMEOUT) -> list[st return healed +def clear_orphaned_flags() -> list[str]: + """Clear in-progress flags whose holder is gone; returns the names cleared. See PERPETUAL_ONLY_FLAGS.""" + from wrolpi import flags + from wrolpi.cmd import pid_is_running + + cleared = [] + for name in PERPETUAL_ONLY_FLAGS + SHARED_PROGRESS_FLAGS: + try: + flag = getattr(flags, name) + if not flag.is_set(): + continue + holder = flag.holder_pid() + if name in PERPETUAL_ONLY_FLAGS: + reason = f'holder pid={holder}' if holder else 'no live holder possible' + elif holder is None: + logger.warning(f'reconcile: flag {name} is set with no recorded holder; leaving it') + continue + elif pid_is_running(holder) and holder != os.getpid(): + logger.info(f'reconcile: flag {name} is held by live pid={holder}; leaving it') + continue + else: + reason = f'holder pid={holder} is dead' + logger.warning(f'reconcile: clearing in-progress flag {name} ({reason})') + flag.clear() + flag._record_holder(None) + cleared.append(name) + except Exception as e: + logger.error(f'reconcile: failed to check flag {name}', exc_info=e) + return cleared + + def reconcile_after_predecessor(app) -> dict: """Undo what a perpetual process that died mid-work left in shared state and the DB. @@ -178,15 +214,7 @@ def reconcile_after_predecessor(app) -> dict: report['locks'] = heal_shared_locks(app) - for name in PERPETUAL_PROGRESS_FLAGS: - try: - flag = getattr(flags, name) - if flag.is_set(): - logger.warning(f'reconcile: clearing in-progress flag {name} left set by a dead process') - flag.clear() - report['flags'].append(name) - except Exception as e: - logger.error(f'reconcile: failed to clear flag {name}', exc_info=e) + report['flags'] = clear_orphaned_flags() try: from wrolpi.jobs import fail_orphaned_jobs diff --git a/wrolpi/test/test_perpetual.py b/wrolpi/test/test_perpetual.py index 95469d03..233704c8 100644 --- a/wrolpi/test/test_perpetual.py +++ b/wrolpi/test/test_perpetual.py @@ -314,8 +314,14 @@ async def test_reconcile_after_predecessor(async_client, test_session, monkeypat created_at='', started_at='', finished_at=None, error=None) file_worker._set_job_status('fj', 'running') file_worker.update_status(status='refreshing') - flags.file_worker_counting.set() - flags.global_refresh_active.set() + flags.file_worker_counting.set() # perpetual-only: cleared whoever held it + flags.global_refresh_active.set() # perpetual-only, set() not entered: cleared + flags.file_worker_modeling.set() # shared, holder dead: cleared + api_app.shared_ctx.flag_holders['file_worker_modeling'] = 2 ** 31 - 1 + flags.map_search_building.set() # shared, holder is a live API worker: kept + api_app.shared_ctx.flag_holders['map_search_building'] = os.getppid() + flags.file_worker_indexing.set() # shared, no holder recorded: kept (cannot tell) + flags.file_worker_busy.set() # not a reconciled flag (a live move may hold it): kept flags.db_up.set() # not an in-progress flag; must survive assert api_app.shared_ctx.jobs_lock.acquire(timeout=1) monkeypatch.setattr(perpetual, 'SHARED_LOCK_HEAL_TIMEOUT', 0.1) @@ -335,11 +341,31 @@ async def test_reconcile_after_predecessor(async_client, test_session, monkeypat {'https://example.com/a': 'new', 'https://example.com/b': 'new'} assert report['file_worker_jobs'] == 1 and file_worker._jobs['fj']['status'] == 'failed' assert file_worker.status['status'] == 'idle' and report['file_worker_status'] is True - assert report['flags'] == ['file_worker_counting', 'global_refresh_active'] + assert report['flags'] == ['file_worker_counting', 'global_refresh_active', 'file_worker_modeling'] assert not flags.file_worker_counting.is_set() and not flags.global_refresh_active.is_set() + assert not flags.file_worker_modeling.is_set() and 'file_worker_modeling' not in api_app.shared_ctx.flag_holders + assert flags.map_search_building.is_set(), 'a flag held by a live process is not touched' + assert flags.file_worker_indexing.is_set(), 'a shared flag with no recorded holder is not touched' + assert flags.file_worker_busy.is_set(), 'file_worker_busy is not reconciled' assert flags.db_up.is_set(), 'only in-progress flags are cleared' with flags.file_worker_counting: # usable again; this raised "flag is already set" before pass + for name in ('map_search_building', 'file_worker_indexing', 'file_worker_busy'): + getattr(flags, name).clear() + api_app.shared_ctx.flag_holders.clear() # Idempotent: a clean state reports nothing. assert not any(reconcile_after_predecessor(api_app).values()) + + +def test_flag_context_manager_records_its_holder(async_client, flags_lock): + """`with flag:` records this pid so a later reconcile can tell an orphaned flag from a live holder.""" + from wrolpi import flags + from wrolpi.api_utils import api_app + + assert flags.file_worker_modeling.holder_pid() is None + with flags.file_worker_modeling: + assert flags.file_worker_modeling.holder_pid() == os.getpid() + assert api_app.shared_ctx.flag_holders['file_worker_modeling'] == os.getpid() + assert flags.file_worker_modeling.holder_pid() is None + assert not flags.file_worker_modeling.is_set()