mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
perpetual: clear an in-progress flag only when its holder is dead
`with flag:` now records the holder pid in shared_ctx.flag_holders and drops it on exit. The reconcile step clears the flags only the perpetual process ever enters (file_worker_counting, file_worker_discovery, global_refresh_active) whenever they are set, and the flags API workers also enter (file_worker_modeling / indexing / cleanup via refresh_sync, map_search_building via the map rebuild endpoints) only when the recorded holder is dead. file_worker_busy is no longer touched: a collection move in an API worker holds it as a mutex, and process_queue drops a stale one itself. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
9e3ac4de5c
commit
e2ab07eee9
4 changed files with 100 additions and 18 deletions
|
|
@ -20,6 +20,8 @@ def attach_shared_contexts(app: Sanic):
|
|||
from wrolpi import flags
|
||||
|
||||
app.shared_ctx.flags = manager.dict({i: False for i in flags.FLAG_NAMES})
|
||||
# Flag name -> pid of the process inside `with flag:` (see Flag.holder_pid).
|
||||
app.shared_ctx.flag_holders = manager.dict()
|
||||
|
||||
# ConfigFile multiprocessing_dict's.
|
||||
# Shared Configs
|
||||
|
|
@ -230,6 +232,8 @@ def reset_shared_contexts(app: Sanic):
|
|||
del app.shared_ctx.events_history[:]
|
||||
app.shared_ctx.single_tasks_started.clear()
|
||||
app.shared_ctx.flags_initialized.clear()
|
||||
if hasattr(app.shared_ctx, 'flag_holders'):
|
||||
app.shared_ctx.flag_holders.clear()
|
||||
if hasattr(app.shared_ctx, 'perpetual_restart_requested_at'):
|
||||
app.shared_ctx.perpetual_restart_requested_at.value = 0.0
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import multiprocessing
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -68,6 +69,27 @@ class Flag:
|
|||
from wrolpi.api_utils import api_app
|
||||
return api_app.shared_ctx.flags[self.name]
|
||||
|
||||
def holder_pid(self) -> int | None:
|
||||
"""The pid of the process inside `with flag:`, or None if no process is (or the holder was not recorded).
|
||||
|
||||
A process killed inside the block never reaches `__exit__`; the perpetual process uses this on startup to
|
||||
tell an orphaned flag from one a live process is still holding."""
|
||||
if PYTEST and not TESTING_LOCK.is_set():
|
||||
return None
|
||||
from wrolpi.api_utils import api_app
|
||||
holders = getattr(api_app.shared_ctx, 'flag_holders', None)
|
||||
return holders.get(self.name) if holders is not None else None
|
||||
|
||||
def _record_holder(self, pid: int | None):
|
||||
from wrolpi.api_utils import api_app
|
||||
holders = getattr(api_app.shared_ctx, 'flag_holders', None)
|
||||
if holders is None:
|
||||
return
|
||||
if pid is None:
|
||||
holders.pop(self.name, None)
|
||||
else:
|
||||
holders[self.name] = pid
|
||||
|
||||
def __enter__(self):
|
||||
if PYTEST and not TESTING_LOCK.is_set():
|
||||
# Testing, but the test does not need flags.
|
||||
|
|
@ -76,6 +98,7 @@ class Flag:
|
|||
if self.is_set():
|
||||
raise ValueError(f'{self} flag is already set!')
|
||||
self.set()
|
||||
self._record_holder(os.getpid())
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if PYTEST and not TESTING_LOCK.is_set():
|
||||
|
|
@ -83,6 +106,7 @@ class Flag:
|
|||
return
|
||||
|
||||
self.clear()
|
||||
self._record_holder(None)
|
||||
|
||||
def _save(self, value: bool):
|
||||
if self.store_db:
|
||||
|
|
|
|||
|
|
@ -47,12 +47,17 @@ PERPETUAL_CANCEL_TIMEOUT = 15
|
|||
SHARED_LOCK_NAMES = ('jobs_lock', 'switches_lock', 'config_save_lock', 'config_update_lock', 'events_lock',
|
||||
'secure_cookies_lock', 'transcode_lock')
|
||||
SHARED_LOCK_HEAL_TIMEOUT = 10
|
||||
# Shared flags that mean "the perpetual process is in the middle of this". They are set and cleared by context
|
||||
# managers in this process only; a SIGKILL inside one leaves it set, and the next attempt raises "flag is already
|
||||
# set" forever (every file refresh, for instance).
|
||||
PERPETUAL_PROGRESS_FLAGS = ('file_worker_busy', 'file_worker_counting', 'file_worker_discovery',
|
||||
'file_worker_modeling', 'file_worker_indexing', 'file_worker_cleanup',
|
||||
'global_refresh_active')
|
||||
# In-progress flags. A SIGKILL inside `with flag:` leaves it set and the next enter raises "flag is already set"
|
||||
# forever (every file refresh, or every map search-index build). Two kinds:
|
||||
# * entered or set only by the perpetual process (FileWorker.process_queue, handle_refresh): at this process's
|
||||
# startup any of them still set is orphaned, whoever the recorded holder is;
|
||||
# * also entered by API workers (refresh_sync from the delete endpoints, the map rebuild endpoints): cleared only
|
||||
# when the recorded holder pid is dead, because a live worker may be inside the block right now.
|
||||
# file_worker_busy is deliberately absent: the collection-move path in an API worker holds it as a mutex, and
|
||||
# process_queue already drops a stale set on its next idle tick.
|
||||
PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active')
|
||||
SHARED_PROGRESS_FLAGS = ('file_worker_modeling', 'file_worker_indexing', 'file_worker_cleanup',
|
||||
'map_search_building')
|
||||
|
||||
# True only inside the process started by `run_perpetual_process`.
|
||||
IN_PERPETUAL_PROCESS = False
|
||||
|
|
@ -160,6 +165,37 @@ def heal_shared_locks(app, timeout: float = SHARED_LOCK_HEAL_TIMEOUT) -> list[st
|
|||
return healed
|
||||
|
||||
|
||||
def clear_orphaned_flags() -> list[str]:
|
||||
"""Clear in-progress flags whose holder is gone; returns the names cleared. See PERPETUAL_ONLY_FLAGS."""
|
||||
from wrolpi import flags
|
||||
from wrolpi.cmd import pid_is_running
|
||||
|
||||
cleared = []
|
||||
for name in PERPETUAL_ONLY_FLAGS + SHARED_PROGRESS_FLAGS:
|
||||
try:
|
||||
flag = getattr(flags, name)
|
||||
if not flag.is_set():
|
||||
continue
|
||||
holder = flag.holder_pid()
|
||||
if name in PERPETUAL_ONLY_FLAGS:
|
||||
reason = f'holder pid={holder}' if holder else 'no live holder possible'
|
||||
elif holder is None:
|
||||
logger.warning(f'reconcile: flag {name} is set with no recorded holder; leaving it')
|
||||
continue
|
||||
elif pid_is_running(holder) and holder != os.getpid():
|
||||
logger.info(f'reconcile: flag {name} is held by live pid={holder}; leaving it')
|
||||
continue
|
||||
else:
|
||||
reason = f'holder pid={holder} is dead'
|
||||
logger.warning(f'reconcile: clearing in-progress flag {name} ({reason})')
|
||||
flag.clear()
|
||||
flag._record_holder(None)
|
||||
cleared.append(name)
|
||||
except Exception as e:
|
||||
logger.error(f'reconcile: failed to check flag {name}', exc_info=e)
|
||||
return cleared
|
||||
|
||||
|
||||
def reconcile_after_predecessor(app) -> dict:
|
||||
"""Undo what a perpetual process that died mid-work left in shared state and the DB.
|
||||
|
||||
|
|
@ -178,15 +214,7 @@ def reconcile_after_predecessor(app) -> dict:
|
|||
|
||||
report['locks'] = heal_shared_locks(app)
|
||||
|
||||
for name in PERPETUAL_PROGRESS_FLAGS:
|
||||
try:
|
||||
flag = getattr(flags, name)
|
||||
if flag.is_set():
|
||||
logger.warning(f'reconcile: clearing in-progress flag {name} left set by a dead process')
|
||||
flag.clear()
|
||||
report['flags'].append(name)
|
||||
except Exception as e:
|
||||
logger.error(f'reconcile: failed to clear flag {name}', exc_info=e)
|
||||
report['flags'] = clear_orphaned_flags()
|
||||
|
||||
try:
|
||||
from wrolpi.jobs import fail_orphaned_jobs
|
||||
|
|
|
|||
|
|
@ -314,8 +314,14 @@ async def test_reconcile_after_predecessor(async_client, test_session, monkeypat
|
|||
created_at='', started_at='', finished_at=None, error=None)
|
||||
file_worker._set_job_status('fj', 'running')
|
||||
file_worker.update_status(status='refreshing')
|
||||
flags.file_worker_counting.set()
|
||||
flags.global_refresh_active.set()
|
||||
flags.file_worker_counting.set() # perpetual-only: cleared whoever held it
|
||||
flags.global_refresh_active.set() # perpetual-only, set() not entered: cleared
|
||||
flags.file_worker_modeling.set() # shared, holder dead: cleared
|
||||
api_app.shared_ctx.flag_holders['file_worker_modeling'] = 2 ** 31 - 1
|
||||
flags.map_search_building.set() # shared, holder is a live API worker: kept
|
||||
api_app.shared_ctx.flag_holders['map_search_building'] = os.getppid()
|
||||
flags.file_worker_indexing.set() # shared, no holder recorded: kept (cannot tell)
|
||||
flags.file_worker_busy.set() # not a reconciled flag (a live move may hold it): kept
|
||||
flags.db_up.set() # not an in-progress flag; must survive
|
||||
assert api_app.shared_ctx.jobs_lock.acquire(timeout=1)
|
||||
monkeypatch.setattr(perpetual, 'SHARED_LOCK_HEAL_TIMEOUT', 0.1)
|
||||
|
|
@ -335,11 +341,31 @@ async def test_reconcile_after_predecessor(async_client, test_session, monkeypat
|
|||
{'https://example.com/a': 'new', 'https://example.com/b': 'new'}
|
||||
assert report['file_worker_jobs'] == 1 and file_worker._jobs['fj']['status'] == 'failed'
|
||||
assert file_worker.status['status'] == 'idle' and report['file_worker_status'] is True
|
||||
assert report['flags'] == ['file_worker_counting', 'global_refresh_active']
|
||||
assert report['flags'] == ['file_worker_counting', 'global_refresh_active', 'file_worker_modeling']
|
||||
assert not flags.file_worker_counting.is_set() and not flags.global_refresh_active.is_set()
|
||||
assert not flags.file_worker_modeling.is_set() and 'file_worker_modeling' not in api_app.shared_ctx.flag_holders
|
||||
assert flags.map_search_building.is_set(), 'a flag held by a live process is not touched'
|
||||
assert flags.file_worker_indexing.is_set(), 'a shared flag with no recorded holder is not touched'
|
||||
assert flags.file_worker_busy.is_set(), 'file_worker_busy is not reconciled'
|
||||
assert flags.db_up.is_set(), 'only in-progress flags are cleared'
|
||||
with flags.file_worker_counting: # usable again; this raised "flag is already set" before
|
||||
pass
|
||||
for name in ('map_search_building', 'file_worker_indexing', 'file_worker_busy'):
|
||||
getattr(flags, name).clear()
|
||||
api_app.shared_ctx.flag_holders.clear()
|
||||
|
||||
# Idempotent: a clean state reports nothing.
|
||||
assert not any(reconcile_after_predecessor(api_app).values())
|
||||
|
||||
|
||||
def test_flag_context_manager_records_its_holder(async_client, flags_lock):
|
||||
"""`with flag:` records this pid so a later reconcile can tell an orphaned flag from a live holder."""
|
||||
from wrolpi import flags
|
||||
from wrolpi.api_utils import api_app
|
||||
|
||||
assert flags.file_worker_modeling.holder_pid() is None
|
||||
with flags.file_worker_modeling:
|
||||
assert flags.file_worker_modeling.holder_pid() == os.getpid()
|
||||
assert api_app.shared_ctx.flag_holders['file_worker_modeling'] == os.getpid()
|
||||
assert flags.file_worker_modeling.holder_pid() is None
|
||||
assert not flags.file_worker_modeling.is_set()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue