From 86f168d2c127556997e93a12ecf498f78343e33d Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:25:26 -0700 Subject: [PATCH] A script's result reads the journal when systemd has cleared its timestamps (#956) The upgrade reloads systemd, which clears the run's exit timestamp, so a successful upgrade looked like a script that never ran: finished false and success null. The journal keeps systemd's unit success/failure record of the run's end, so it now decides whether a run finished, and when. Co-authored-by: Claude Opus 5.5 --- controller/lib/scripts.py | 45 ++++++++++++++++++++++-- controller/test/test_scripts.py | 61 ++++++++++++++++++++++++++++----- 2 files changed, 96 insertions(+), 10 deletions(-) diff --git a/controller/lib/scripts.py b/controller/lib/scripts.py index 1072db77..360dd6cb 100644 --- a/controller/lib/scripts.py +++ b/controller/lib/scripts.py @@ -5,6 +5,7 @@ Manages maintenance scripts that run as systemd services (like repair.sh). Uses systemd oneshot services for process survival across controller restarts. """ +import json import logging import re import subprocess @@ -189,7 +190,7 @@ def get_script_result(name: str) -> Optional[dict]: A failed upgrade restarts the services it stopped, so the services coming back up does not mean it succeeded. systemd keeps Result and the exit status across the daemon-reload an upgrade does, but not the run's - timestamps, so a run is known to have ended if it has an exit timestamp or ended badly. + timestamps, so when the exit timestamp is missing the journal's record of the run's end is used instead. Returns None for an unknown script, otherwise a dict with: script_name: str @@ -249,13 +250,53 @@ def get_script_result(name: str) -> Optional[dict]: except ValueError: pass - # A clean Result with no exit timestamp is also what a service that never ran reports. + if not result["finished_at"]: + # A clean Result with no exit timestamp is also what a service that never ran reports, so ask the journal. + journal_outcome, result["finished_at"] = _get_last_journal_outcome(service) + failed = failed or journal_outcome is False + if journal_outcome is None and not failed: + return result + if failed or result["finished_at"]: result["finished"] = True result["success"] = not failed return result +# systemd's journal message IDs for a unit that ended cleanly, or failed. +UNIT_SUCCESS_MESSAGE_ID = "7ad2d189f7e94e70a38c781354912448" +UNIT_FAILURE_MESSAGE_ID = "d9b373ed55a64feb8242e02dbe79a49c" + + +def _get_last_journal_outcome(service: str) -> tuple[Optional[bool], Optional[str]]: + """ + Read how the service's last run ended from the journal, which keeps it across a daemon-reload. + + Returns: + Tuple of (whether it succeeded, ISO timestamp when it ended); (None, None) if the journal has no record. + """ + try: + proc = subprocess.run( + ["journalctl", "--no-pager", "-n", "1", "-o", "json", "--output-fields=MESSAGE_ID", + f"UNIT={service}", f"MESSAGE_ID={UNIT_SUCCESS_MESSAGE_ID}", "+", + f"UNIT={service}", f"MESSAGE_ID={UNIT_FAILURE_MESSAGE_ID}"], + capture_output=True, + text=True, + timeout=5, + ) + except (subprocess.TimeoutExpired, subprocess.SubprocessError, FileNotFoundError): + return None, None + if proc.returncode != 0 or not proc.stdout.strip(): + return None, None + + try: + entry = json.loads(proc.stdout.strip().splitlines()[-1]) + ended_at = datetime.fromtimestamp(int(entry["__REALTIME_TIMESTAMP"]) / 1_000_000, timezone.utc) + except (ValueError, KeyError, TypeError): + return None, None + return entry.get("MESSAGE_ID") == UNIT_SUCCESS_MESSAGE_ID, ended_at.isoformat() + + def _get_service_timing(service: str) -> tuple[Optional[str], Optional[int]]: """ Get timing info for a running service. diff --git a/controller/test/test_scripts.py b/controller/test/test_scripts.py index fb43c246..cdcc7787 100644 --- a/controller/test/test_scripts.py +++ b/controller/test/test_scripts.py @@ -2,6 +2,7 @@ Unit tests for controller.lib.scripts module. """ +import json import subprocess import time from unittest import mock @@ -375,6 +376,25 @@ def _systemctl_show(stdout: str): return mock.Mock(returncode=0, stdout=stdout) +# systemd's message IDs for a unit that ended cleanly or failed. +UNIT_SUCCESS = "7ad2d189f7e94e70a38c781354912448" +UNIT_FAILURE = "d9b373ed55a64feb8242e02dbe79a49c" + + +def _fake_run(systemctl_stdout: str, journal_stdout: str = ""): + """Answer `systemctl show` and `journalctl` calls with their own output.""" + + def run(args, **kwargs): + stdout = journal_stdout if args[0] == "journalctl" else systemctl_stdout + return mock.Mock(returncode=0, stdout=stdout) + + return run + + +def _journal_entry(message_id: str, realtime_us: int) -> str: + return json.dumps({"MESSAGE_ID": message_id, "__REALTIME_TIMESTAMP": str(realtime_us)}) + "\n" + + class TestGetScriptResult: """How the last run of a script ended, read from systemd.""" @@ -416,34 +436,59 @@ class TestGetScriptResult: assert result["finished_at"] is None def test_failed_run_after_daemon_reload(self): - """An upgrade reloads systemd, which clears the run's timestamps but keeps its Result and exit status.""" + """An upgrade reloads systemd, which clears the run's timestamps but keeps its Result and exit status. The + journal still has when it ended.""" stdout = "Result=exit-code\nExecMainStatus=5\nActiveState=failed\nExecMainExitTimestamp=\n" + journal = _journal_entry(UNIT_FAILURE, 1794151352500000) with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ - mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + mock.patch("controller.lib.scripts.subprocess.run", side_effect=_fake_run(stdout, journal)): result = get_script_result("upgrade") assert result["running"] is False assert result["finished"] is True assert result["success"] is False assert result["exit_status"] == 5 - assert result["finished_at"] is None + assert result["finished_at"] == "2026-11-08T15:22:32.500000+00:00" def test_successful_run_after_daemon_reload(self): - """A clean run whose timestamps were cleared is not running and not failed; the upgrade page redirects.""" + """A clean run whose timestamps were cleared looks like a service that never ran, so the journal decides.""" stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n" + journal = _journal_entry(UNIT_SUCCESS, 1794151352500000) with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ - mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + mock.patch("controller.lib.scripts.subprocess.run", side_effect=_fake_run(stdout, journal)) as run: result = get_script_result("upgrade") assert result["running"] is False - assert result["success"] is not False + assert result["finished"] is True + assert result["success"] is True + assert result["finished_at"] == "2026-11-08T15:22:32.500000+00:00" + journal_args = run.call_args_list[-1].args[0] + assert journal_args[0] == "journalctl" + assert "UNIT=wrolpi-upgrade.service" in journal_args def test_never_run(self): - """A service that has never run has no result.""" + """A service that has never run has no result, and nothing in the journal.""" stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n" with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ - mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + mock.patch("controller.lib.scripts.subprocess.run", side_effect=_fake_run(stdout, "")): result = get_script_result("upgrade") assert result["finished"] is False assert result["success"] is None + assert result["finished_at"] is None + + def test_journal_unreadable(self): + """Without the journal, a clean run with cleared timestamps is reported as not known to have finished.""" + stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n" + + def run(args, **kwargs): + if args[0] == "journalctl": + raise FileNotFoundError + return _systemctl_show(stdout) + + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", side_effect=run): + result = get_script_result("upgrade") + assert result["running"] is False + assert result["finished"] is False + assert result["success"] is None def test_systemctl_missing(self): with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \