A script's result reads the journal when systemd has cleared its timestamps (#956)

The upgrade reloads systemd, which clears the run's exit timestamp, so a
successful upgrade looked like a script that never ran: finished false and
success null. The journal keeps systemd's unit success/failure record of the
run's end, so it now decides whether a run finished, and when.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-11-08 11:25:26 -07:00 • committed by Roland
parent 19ff59d5e0
commit 86f168d2c1
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
2 changed files with 96 additions and 10 deletions

View file

@ -5,6 +5,7 @@ Manages maintenance scripts that run as systemd services (like repair.sh).
Uses systemd oneshot services for process survival across controller restarts.
"""
import json
import logging
import re
import subprocess
@ -189,7 +190,7 @@ def get_script_result(name: str) -> Optional[dict]:
A failed upgrade restarts the services it stopped, so the services coming back up does not mean it succeeded.
systemd keeps Result and the exit status across the daemon-reload an upgrade does, but not the run's
timestamps, so a run is known to have ended if it has an exit timestamp or ended badly.
timestamps, so when the exit timestamp is missing the journal's record of the run's end is used instead.
Returns None for an unknown script, otherwise a dict with:
script_name: str
@ -249,13 +250,53 @@ def get_script_result(name: str) -> Optional[dict]:
except ValueError:
pass
# A clean Result with no exit timestamp is also what a service that never ran reports.
if not result["finished_at"]:
# A clean Result with no exit timestamp is also what a service that never ran reports, so ask the journal.
journal_outcome, result["finished_at"] = _get_last_journal_outcome(service)
failed = failed or journal_outcome is False
if journal_outcome is None and not failed:
return result
if failed or result["finished_at"]:
result["finished"] = True
result["success"] = not failed
return result
# systemd's journal message IDs for a unit that ended cleanly, or failed.
UNIT_SUCCESS_MESSAGE_ID = "7ad2d189f7e94e70a38c781354912448"
UNIT_FAILURE_MESSAGE_ID = "d9b373ed55a64feb8242e02dbe79a49c"
def _get_last_journal_outcome(service: str) -> tuple[Optional[bool], Optional[str]]:
"""
Read how the service's last run ended from the journal, which keeps it across a daemon-reload.
Returns:
Tuple of (whether it succeeded, ISO timestamp when it ended); (None, None) if the journal has no record.
"""
try:
proc = subprocess.run(
["journalctl", "--no-pager", "-n", "1", "-o", "json", "--output-fields=MESSAGE_ID",
f"UNIT={service}", f"MESSAGE_ID={UNIT_SUCCESS_MESSAGE_ID}", "+",
f"UNIT={service}", f"MESSAGE_ID={UNIT_FAILURE_MESSAGE_ID}"],
capture_output=True,
text=True,
timeout=5,
)
except (subprocess.TimeoutExpired, subprocess.SubprocessError, FileNotFoundError):
return None, None
if proc.returncode != 0 or not proc.stdout.strip():
return None, None
try:
entry = json.loads(proc.stdout.strip().splitlines()[-1])
ended_at = datetime.fromtimestamp(int(entry["__REALTIME_TIMESTAMP"]) / 1_000_000, timezone.utc)
except (ValueError, KeyError, TypeError):
return None, None
return entry.get("MESSAGE_ID") == UNIT_SUCCESS_MESSAGE_ID, ended_at.isoformat()
def _get_service_timing(service: str) -> tuple[Optional[str], Optional[int]]:
"""
Get timing info for a running service.

View file

@ -2,6 +2,7 @@
Unit tests for controller.lib.scripts module.
"""
import json
import subprocess
import time
from unittest import mock
@ -375,6 +376,25 @@ def _systemctl_show(stdout: str):
return mock.Mock(returncode=0, stdout=stdout)
# systemd's message IDs for a unit that ended cleanly or failed.
UNIT_SUCCESS = "7ad2d189f7e94e70a38c781354912448"
UNIT_FAILURE = "d9b373ed55a64feb8242e02dbe79a49c"
def _fake_run(systemctl_stdout: str, journal_stdout: str = ""):
"""Answer `systemctl show` and `journalctl` calls with their own output."""
def run(args, **kwargs):
stdout = journal_stdout if args[0] == "journalctl" else systemctl_stdout
return mock.Mock(returncode=0, stdout=stdout)
return run
def _journal_entry(message_id: str, realtime_us: int) -> str:
return json.dumps({"MESSAGE_ID": message_id, "__REALTIME_TIMESTAMP": str(realtime_us)}) + "\n"
class TestGetScriptResult:
"""How the last run of a script ended, read from systemd."""
@ -416,34 +436,59 @@ class TestGetScriptResult:
assert result["finished_at"] is None
def test_failed_run_after_daemon_reload(self):
"""An upgrade reloads systemd, which clears the run's timestamps but keeps its Result and exit status."""
"""An upgrade reloads systemd, which clears the run's timestamps but keeps its Result and exit status. The
journal still has when it ended."""
stdout = "Result=exit-code\nExecMainStatus=5\nActiveState=failed\nExecMainExitTimestamp=\n"
journal = _journal_entry(UNIT_FAILURE, 1794151352500000)
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
mock.patch("controller.lib.scripts.subprocess.run", side_effect=_fake_run(stdout, journal)):
result = get_script_result("upgrade")
assert result["running"] is False
assert result["finished"] is True
assert result["success"] is False
assert result["exit_status"] == 5
assert result["finished_at"] is None
assert result["finished_at"] == "2026-11-08T15:22:32.500000+00:00"
def test_successful_run_after_daemon_reload(self):
"""A clean run whose timestamps were cleared is not running and not failed; the upgrade page redirects."""
"""A clean run whose timestamps were cleared looks like a service that never ran, so the journal decides."""
stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n"
journal = _journal_entry(UNIT_SUCCESS, 1794151352500000)
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
mock.patch("controller.lib.scripts.subprocess.run", side_effect=_fake_run(stdout, journal)) as run:
result = get_script_result("upgrade")
assert result["running"] is False
assert result["success"] is not False
assert result["finished"] is True
assert result["success"] is True
assert result["finished_at"] == "2026-11-08T15:22:32.500000+00:00"
journal_args = run.call_args_list[-1].args[0]
assert journal_args[0] == "journalctl"
assert "UNIT=wrolpi-upgrade.service" in journal_args
def test_never_run(self):
"""A service that has never run has no result."""
"""A service that has never run has no result, and nothing in the journal."""
stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n"
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
mock.patch("controller.lib.scripts.subprocess.run", side_effect=_fake_run(stdout, "")):
result = get_script_result("upgrade")
assert result["finished"] is False
assert result["success"] is None
assert result["finished_at"] is None
def test_journal_unreadable(self):
"""Without the journal, a clean run with cleared timestamps is reported as not known to have finished."""
stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n"
def run(args, **kwargs):
if args[0] == "journalctl":
raise FileNotFoundError
return _systemctl_show(stdout)
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", side_effect=run):
result = get_script_result("upgrade")
assert result["running"] is False
assert result["finished"] is False
assert result["success"] is None
def test_systemctl_missing(self):
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \