mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
A script's result reads the journal when systemd has cleared its timestamps (#956)
The upgrade reloads systemd, which clears the run's exit timestamp, so a successful upgrade looked like a script that never ran: finished false and success null. The journal keeps systemd's unit success/failure record of the run's end, so it now decides whether a run finished, and when. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
19ff59d5e0
commit
86f168d2c1
2 changed files with 96 additions and 10 deletions
|
|
@ -5,6 +5,7 @@ Manages maintenance scripts that run as systemd services (like repair.sh).
|
|||
Uses systemd oneshot services for process survival across controller restarts.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
|
|
@ -189,7 +190,7 @@ def get_script_result(name: str) -> Optional[dict]:
|
|||
A failed upgrade restarts the services it stopped, so the services coming back up does not mean it succeeded.
|
||||
|
||||
systemd keeps Result and the exit status across the daemon-reload an upgrade does, but not the run's
|
||||
timestamps, so a run is known to have ended if it has an exit timestamp or ended badly.
|
||||
timestamps, so when the exit timestamp is missing the journal's record of the run's end is used instead.
|
||||
|
||||
Returns None for an unknown script, otherwise a dict with:
|
||||
script_name: str
|
||||
|
|
@ -249,13 +250,53 @@ def get_script_result(name: str) -> Optional[dict]:
|
|||
except ValueError:
|
||||
pass
|
||||
|
||||
# A clean Result with no exit timestamp is also what a service that never ran reports.
|
||||
if not result["finished_at"]:
|
||||
# A clean Result with no exit timestamp is also what a service that never ran reports, so ask the journal.
|
||||
journal_outcome, result["finished_at"] = _get_last_journal_outcome(service)
|
||||
failed = failed or journal_outcome is False
|
||||
if journal_outcome is None and not failed:
|
||||
return result
|
||||
|
||||
if failed or result["finished_at"]:
|
||||
result["finished"] = True
|
||||
result["success"] = not failed
|
||||
return result
|
||||
|
||||
|
||||
# systemd's journal message IDs for a unit that ended cleanly, or failed.
|
||||
UNIT_SUCCESS_MESSAGE_ID = "7ad2d189f7e94e70a38c781354912448"
|
||||
UNIT_FAILURE_MESSAGE_ID = "d9b373ed55a64feb8242e02dbe79a49c"
|
||||
|
||||
|
||||
def _get_last_journal_outcome(service: str) -> tuple[Optional[bool], Optional[str]]:
|
||||
"""
|
||||
Read how the service's last run ended from the journal, which keeps it across a daemon-reload.
|
||||
|
||||
Returns:
|
||||
Tuple of (whether it succeeded, ISO timestamp when it ended); (None, None) if the journal has no record.
|
||||
"""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["journalctl", "--no-pager", "-n", "1", "-o", "json", "--output-fields=MESSAGE_ID",
|
||||
f"UNIT={service}", f"MESSAGE_ID={UNIT_SUCCESS_MESSAGE_ID}", "+",
|
||||
f"UNIT={service}", f"MESSAGE_ID={UNIT_FAILURE_MESSAGE_ID}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, subprocess.SubprocessError, FileNotFoundError):
|
||||
return None, None
|
||||
if proc.returncode != 0 or not proc.stdout.strip():
|
||||
return None, None
|
||||
|
||||
try:
|
||||
entry = json.loads(proc.stdout.strip().splitlines()[-1])
|
||||
ended_at = datetime.fromtimestamp(int(entry["__REALTIME_TIMESTAMP"]) / 1_000_000, timezone.utc)
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None, None
|
||||
return entry.get("MESSAGE_ID") == UNIT_SUCCESS_MESSAGE_ID, ended_at.isoformat()
|
||||
|
||||
|
||||
def _get_service_timing(service: str) -> tuple[Optional[str], Optional[int]]:
|
||||
"""
|
||||
Get timing info for a running service.
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Unit tests for controller.lib.scripts module.
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from unittest import mock
|
||||
|
|
@ -375,6 +376,25 @@ def _systemctl_show(stdout: str):
|
|||
return mock.Mock(returncode=0, stdout=stdout)
|
||||
|
||||
|
||||
# systemd's message IDs for a unit that ended cleanly or failed.
|
||||
UNIT_SUCCESS = "7ad2d189f7e94e70a38c781354912448"
|
||||
UNIT_FAILURE = "d9b373ed55a64feb8242e02dbe79a49c"
|
||||
|
||||
|
||||
def _fake_run(systemctl_stdout: str, journal_stdout: str = ""):
|
||||
"""Answer `systemctl show` and `journalctl` calls with their own output."""
|
||||
|
||||
def run(args, **kwargs):
|
||||
stdout = journal_stdout if args[0] == "journalctl" else systemctl_stdout
|
||||
return mock.Mock(returncode=0, stdout=stdout)
|
||||
|
||||
return run
|
||||
|
||||
|
||||
def _journal_entry(message_id: str, realtime_us: int) -> str:
|
||||
return json.dumps({"MESSAGE_ID": message_id, "__REALTIME_TIMESTAMP": str(realtime_us)}) + "\n"
|
||||
|
||||
|
||||
class TestGetScriptResult:
|
||||
"""How the last run of a script ended, read from systemd."""
|
||||
|
||||
|
|
@ -416,34 +436,59 @@ class TestGetScriptResult:
|
|||
assert result["finished_at"] is None
|
||||
|
||||
def test_failed_run_after_daemon_reload(self):
|
||||
"""An upgrade reloads systemd, which clears the run's timestamps but keeps its Result and exit status."""
|
||||
"""An upgrade reloads systemd, which clears the run's timestamps but keeps its Result and exit status. The
|
||||
journal still has when it ended."""
|
||||
stdout = "Result=exit-code\nExecMainStatus=5\nActiveState=failed\nExecMainExitTimestamp=\n"
|
||||
journal = _journal_entry(UNIT_FAILURE, 1794151352500000)
|
||||
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
|
||||
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
|
||||
mock.patch("controller.lib.scripts.subprocess.run", side_effect=_fake_run(stdout, journal)):
|
||||
result = get_script_result("upgrade")
|
||||
assert result["running"] is False
|
||||
assert result["finished"] is True
|
||||
assert result["success"] is False
|
||||
assert result["exit_status"] == 5
|
||||
assert result["finished_at"] is None
|
||||
assert result["finished_at"] == "2026-11-08T15:22:32.500000+00:00"
|
||||
|
||||
def test_successful_run_after_daemon_reload(self):
|
||||
"""A clean run whose timestamps were cleared is not running and not failed; the upgrade page redirects."""
|
||||
"""A clean run whose timestamps were cleared looks like a service that never ran, so the journal decides."""
|
||||
stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n"
|
||||
journal = _journal_entry(UNIT_SUCCESS, 1794151352500000)
|
||||
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
|
||||
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
|
||||
mock.patch("controller.lib.scripts.subprocess.run", side_effect=_fake_run(stdout, journal)) as run:
|
||||
result = get_script_result("upgrade")
|
||||
assert result["running"] is False
|
||||
assert result["success"] is not False
|
||||
assert result["finished"] is True
|
||||
assert result["success"] is True
|
||||
assert result["finished_at"] == "2026-11-08T15:22:32.500000+00:00"
|
||||
journal_args = run.call_args_list[-1].args[0]
|
||||
assert journal_args[0] == "journalctl"
|
||||
assert "UNIT=wrolpi-upgrade.service" in journal_args
|
||||
|
||||
def test_never_run(self):
|
||||
"""A service that has never run has no result."""
|
||||
"""A service that has never run has no result, and nothing in the journal."""
|
||||
stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n"
|
||||
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
|
||||
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
|
||||
mock.patch("controller.lib.scripts.subprocess.run", side_effect=_fake_run(stdout, "")):
|
||||
result = get_script_result("upgrade")
|
||||
assert result["finished"] is False
|
||||
assert result["success"] is None
|
||||
assert result["finished_at"] is None
|
||||
|
||||
def test_journal_unreadable(self):
|
||||
"""Without the journal, a clean run with cleared timestamps is reported as not known to have finished."""
|
||||
stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n"
|
||||
|
||||
def run(args, **kwargs):
|
||||
if args[0] == "journalctl":
|
||||
raise FileNotFoundError
|
||||
return _systemctl_show(stdout)
|
||||
|
||||
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
|
||||
mock.patch("controller.lib.scripts.subprocess.run", side_effect=run):
|
||||
result = get_script_result("upgrade")
|
||||
assert result["running"] is False
|
||||
assert result["finished"] is False
|
||||
assert result["success"] is None
|
||||
|
||||
def test_systemctl_missing(self):
|
||||
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue