A failed upgrade is shown on the upgrade page instead of redirecting home (#955)

* A failed upgrade is shown on the upgrade page instead of redirecting home

A failed upgrade restarts the services it stopped, so the upgrade page saw
them come back and redirected home as if the upgrade had worked.  The
Controller now reports how the last run of a script ended
(GET /api/scripts/{name}/result, from systemd's Result and exit status),
and the upgrade page reads it before redirecting.  A failure turns the
banner red, gives the exit code, opens the output, and offers a link back.

Only a run the page has seen start counts, because the result of an
earlier run is still reported until the new run begins.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* An upgrade's result survives the daemon-reload it does

The upgrade reinstalls its unit and reloads systemd, which clears the run's
timestamps but keeps Result and the exit status.  A failure is now known
from those alone, and the result reports whether the script is running, so
the upgrade page waits on that instead of on a timestamp that may never come.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-11-08 11:11:41 -07:00 • committed by Roland
parent d88db90ce9
commit 19ff59d5e0
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
6 changed files with 317 additions and 11 deletions

View file

@ -449,6 +449,18 @@ class ScriptStatusResponse(BaseModel):
elapsed_seconds: Optional[int] = Field(default=None, description="Seconds since start")
class ScriptResultResponse(BaseModel):
"""Response model for how the last run of a script ended."""
script_name: str = Field(description="Name of the script")
running: bool = Field(description="Whether the script is running now")
finished: bool = Field(description="Whether a run is known to have ended (false while running, or if it never ran)")
success: Optional[bool] = Field(default=None, description="Whether the last run exited cleanly")
result: Optional[str] = Field(default=None, description="systemd's Result, e.g. 'success' or 'exit-code'")
exit_status: Optional[int] = Field(default=None, description="The script's exit status")
finished_at: Optional[str] = Field(default=None, description="ISO timestamp when the last run ended")
class ScriptStartResponse(BaseModel):
"""Response model for starting a script."""

View file

@ -12,6 +12,7 @@ from controller.api.schemas import (
BranchResponse,
ScriptInfo,
ScriptOutputResponse,
ScriptResultResponse,
ScriptStartRequest,
ScriptStartResponse,
ScriptStatusResponse,
@ -20,6 +21,7 @@ from controller.lib.config import is_docker_mode
from controller.lib.scripts import (
get_current_branch,
get_script_output,
get_script_result,
get_script_status,
list_available_scripts,
start_script,
@ -88,3 +90,18 @@ async def scripts_output(name: str, lines: int = 100) -> ScriptOutputResponse:
lines = min(max(1, lines), 5000)
result = get_script_output(name, lines)
return ScriptOutputResponse(**result)
@router.get("/api/scripts/{name}/result", response_model=ScriptResultResponse)
async def scripts_result(name: str) -> ScriptResultResponse:
"""How the last run of a script ended."""
if is_docker_mode():
raise HTTPException(
status_code=501,
detail="Scripts are not available in Docker mode"
)
result = get_script_result(name)
if result is None:
raise HTTPException(status_code=404, detail=f"Unknown script: {name}")
return ScriptResultResponse(**result)

View file

@ -182,6 +182,80 @@ def get_script_status() -> dict:
return {"running": False}
def get_script_result(name: str) -> Optional[dict]:
"""
Report how the last run of a script ended, so a page waiting on it can tell success from failure.
A failed upgrade restarts the services it stopped, so the services coming back up does not mean it succeeded.
systemd keeps Result and the exit status across the daemon-reload an upgrade does, but not the run's
timestamps, so a run is known to have ended if it has an exit timestamp or ended badly.
Returns None for an unknown script, otherwise a dict with:
script_name: str
running: bool - Whether the script is running now
finished: bool - Whether a run is known to have ended (False while running, or if it never ran)
success: Optional[bool] - Whether the last run exited cleanly; None unless finished
result: Optional[str] - systemd's Result (e.g. "success", "exit-code")
exit_status: Optional[int] - The script's exit status
finished_at: Optional[str] - ISO timestamp when the last run ended
"""
if name not in AVAILABLE_SCRIPTS:
return None
result = {
"script_name": name,
"running": False,
"finished": False,
"success": None,
"result": None,
"exit_status": None,
"finished_at": None,
}
if is_docker_mode():
return result
service = AVAILABLE_SCRIPTS[name]["service_name"]
try:
# A Unix timestamp, because systemd prints dates in the local timezone.
proc = subprocess.run(
["systemctl", "show", service, "--property=Result,ExecMainStatus,ActiveState,ExecMainExitTimestamp",
"--timestamp=unix"],
capture_output=True,
text=True,
timeout=5,
)
except (subprocess.TimeoutExpired, subprocess.SubprocessError, FileNotFoundError):
return result
if proc.returncode != 0:
return result
properties = dict(line.partition("=")[::2] for line in proc.stdout.splitlines() if "=" in line)
if properties.get("ActiveState") in ("activating", "active", "reloading"):
result["running"] = True
return result
try:
result["exit_status"] = int(properties.get("ExecMainStatus", ""))
except ValueError:
pass
result["result"] = properties.get("Result") or None
failed = result["result"] not in (None, "success") or result["exit_status"] not in (0, None)
exit_timestamp = properties.get("ExecMainExitTimestamp", "").lstrip("@")
if exit_timestamp:
try:
result["finished_at"] = datetime.fromtimestamp(float(exit_timestamp), timezone.utc).isoformat()
except ValueError:
pass
# A clean Result with no exit timestamp is also what a service that never ran reports.
if failed or result["finished_at"]:
result["finished"] = True
result["success"] = not failed
return result
def _get_service_timing(service: str) -> tuple[Optional[str], Optional[int]]:
"""
Get timing info for a running service.

View file

@ -679,6 +679,23 @@
margin-top: 1rem;
}
.script-running-banner.failed {
background: linear-gradient(135deg, #e74c3c, #c0392b);
}
.script-running-banner.failed .loading {
display: none;
}
.script-failed-message {
display: none;
margin: 0.5rem 0 0 0;
}
.script-running-banner.failed .script-failed-message {
display: block;
}
.warning-list {
background: #1a1a2e;
border: 1px solid #ff6b6b;
@ -914,8 +931,10 @@
<div id="script-running-banner" class="script-running-banner">
<h2 id="script-running-name">Script Running</h2>
<span class="loading"></span>
<p id="script-failed-message" class="script-failed-message"></p>
<div class="actions">
<button class="btn btn-info" onclick="showScriptOutput()">View Output</button>
<a id="script-failed-return" class="btn btn-success" style="display: none;" href="#">Return to WROLPi</a>
</div>
</div>
{% endif %}
@ -2643,9 +2662,81 @@
// Two-phase upgrade detection:
// Phase 1: Wait for services to go DOWN (confirms upgrade actually started).
// Phase 2: Wait for services to come back UP (upgrade complete).
// A failed upgrade also restarts the services, so the upgrade's own result decides between
// redirecting and showing the failure. Only a run this page has seen start counts: the
// result of an earlier run is still reported until the new run begins.
let upgradeServicesWentDown = sessionStorage.getItem('upgradeServicesWentDown') === 'true';
let upgradeSeenRunning = sessionStorage.getItem('upgradeSeenRunning') === 'true';
let upgradeInterval = null;
let upgradeDone = false;
const markUpgradeStarted = (key) => {
if (key === 'upgradeServicesWentDown') upgradeServicesWentDown = true;
if (key === 'upgradeSeenRunning') upgradeSeenRunning = true;
sessionStorage.setItem(key, 'true');
};
const finishUpgradeWatch = () => {
upgradeDone = true;
if (upgradeInterval) clearInterval(upgradeInterval);
sessionStorage.removeItem('upgradeServicesWentDown');
sessionStorage.removeItem('upgradeSeenRunning');
};
const showUpgradeFailed = (result) => {
finishUpgradeWatch();
const banner = document.getElementById('script-running-banner');
banner.classList.add('show', 'failed');
document.getElementById('script-running-name').textContent = 'WROLPi Upgrade Failed';
const code = result.exit_status !== null && result.exit_status !== undefined
? ` (exit code ${result.exit_status})` : '';
document.getElementById('script-failed-message').textContent =
`The upgrade did not complete${code}. See the output for the reason.`;
const returnLink = document.getElementById('script-failed-return');
returnLink.href = `${upgradeHttpsBase}/`;
returnLink.style.display = '';
showScriptOutput();
};
// The upgrade's result, or null if it cannot be read (the Controller restarts during an upgrade).
const fetchUpgradeResult = async () => {
try {
const resp = await fetch('api/scripts/upgrade/result', {
headers: {'Accept': 'application/json'},
signal: AbortSignal.timeout(5000)
});
return resp.ok ? await resp.json() : null;
} catch (e) {
return null;
}
};
const checkUpgradeComplete = async () => {
if (upgradeDone) return;
if (!upgradeSeenRunning) {
try {
const resp = await fetch('api/scripts/status', {
headers: {'Accept': 'application/json'},
signal: AbortSignal.timeout(5000)
});
const status = resp.ok ? await resp.json() : null;
if (status && status.running && status.script_name === 'upgrade') {
markUpgradeStarted('upgradeSeenRunning');
}
} catch (e) {
// The ready check below handles an unreachable Controller.
}
}
if (upgradeSeenRunning || upgradeServicesWentDown) {
const result = await fetchUpgradeResult();
if (result && result.success === false) {
showUpgradeFailed(result);
return;
}
}
try {
const resp = await fetch('api/ready', {
method: 'GET',
@ -2656,26 +2747,31 @@
const data = await resp.json();
if (!upgradeServicesWentDown) {
if (!data.api || !data.app) {
upgradeServicesWentDown = true;
sessionStorage.setItem('upgradeServicesWentDown', 'true');
markUpgradeStarted('upgradeServicesWentDown');
}
} else if (data.api && data.app) {
sessionStorage.removeItem('upgradeServicesWentDown');
setTimeout(() => {
window.location.href = `${upgradeHttpsBase}/`;
}, 2000);
// The services are back; redirect only once the upgrade itself has finished well.
const result = await fetchUpgradeResult();
if (result && result.success === false) {
showUpgradeFailed(result);
} else if (result && result.running) {
// Still running (e.g. restarting services); check again on the next poll.
} else {
finishUpgradeWatch();
setTimeout(() => {
window.location.href = `${upgradeHttpsBase}/`;
}, 2000);
}
}
} else {
upgradeServicesWentDown = true;
sessionStorage.setItem('upgradeServicesWentDown', 'true');
markUpgradeStarted('upgradeServicesWentDown');
}
} catch (e) {
upgradeServicesWentDown = true;
sessionStorage.setItem('upgradeServicesWentDown', 'true');
markUpgradeStarted('upgradeServicesWentDown');
}
};
setInterval(checkUpgradeComplete, 3000);
upgradeInterval = setInterval(checkUpgradeComplete, 3000);
checkUpgradeComplete();
}
} catch (e) {

View file

@ -13,6 +13,7 @@ from controller.lib.scripts import (
AVAILABLE_SCRIPTS,
get_current_branch,
get_script_output,
get_script_result,
get_script_status,
list_available_scripts,
start_script,
@ -368,3 +369,89 @@ class TestGetServiceTiming:
completed = subprocess.CompletedProcess(args=[], returncode=0, stdout="ExecMainStartTimestamp=\n")
with mock.patch("controller.lib.scripts.subprocess.run", return_value=completed):
assert _get_service_timing("wrolpi-upgrade.service") == (None, None)
def _systemctl_show(stdout: str):
return mock.Mock(returncode=0, stdout=stdout)
class TestGetScriptResult:
"""How the last run of a script ended, read from systemd."""
def test_failed_run(self):
"""An upgrade that exited non-zero is reported as failed, with its exit status and when it finished."""
stdout = "Result=exit-code\nExecMainStatus=5\nActiveState=failed\nExecMainExitTimestamp=@1794151352.5\n"
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)) as run:
result = get_script_result("upgrade")
assert result == {
"script_name": "upgrade",
"running": False,
"finished": True,
"success": False,
"result": "exit-code",
"exit_status": 5,
"finished_at": "2026-11-08T15:22:32.500000+00:00",
}
assert "wrolpi-upgrade.service" in run.call_args[0][0]
def test_successful_run(self):
stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=@1762615352\n"
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
result = get_script_result("upgrade")
assert result["finished"] is True
assert result["success"] is True
assert result["exit_status"] == 0
def test_running(self):
"""A run in progress has not finished, so it has neither succeeded nor failed."""
stdout = "Result=success\nExecMainStatus=0\nActiveState=activating\nExecMainExitTimestamp=\n"
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
result = get_script_result("upgrade")
assert result["running"] is True
assert result["finished"] is False
assert result["success"] is None
assert result["finished_at"] is None
def test_failed_run_after_daemon_reload(self):
"""An upgrade reloads systemd, which clears the run's timestamps but keeps its Result and exit status."""
stdout = "Result=exit-code\nExecMainStatus=5\nActiveState=failed\nExecMainExitTimestamp=\n"
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
result = get_script_result("upgrade")
assert result["running"] is False
assert result["finished"] is True
assert result["success"] is False
assert result["exit_status"] == 5
assert result["finished_at"] is None
def test_successful_run_after_daemon_reload(self):
"""A clean run whose timestamps were cleared is not running and not failed; the upgrade page redirects."""
stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n"
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
result = get_script_result("upgrade")
assert result["running"] is False
assert result["success"] is not False
def test_never_run(self):
"""A service that has never run has no result."""
stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n"
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)):
result = get_script_result("upgrade")
assert result["finished"] is False
assert result["success"] is None
def test_systemctl_missing(self):
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \
mock.patch("controller.lib.scripts.subprocess.run", side_effect=FileNotFoundError):
result = get_script_result("upgrade")
assert result["finished"] is False
assert result["success"] is None
def test_unknown_script(self):
with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False):
assert get_script_result("nonexistent") is None

View file

@ -13,6 +13,7 @@ class TestDockerModeRejectsScriptsEndpoints:
@pytest.mark.parametrize("method,endpoint", [
("post", "/api/scripts/repair/start"),
("get", "/api/scripts/repair/output"),
("get", "/api/scripts/upgrade/result"),
])
def test_action_rejected_in_docker(self, test_client_docker_mode, method, endpoint):
response = getattr(test_client_docker_mode, method)(endpoint)
@ -166,3 +167,22 @@ class TestScriptsOutputEndpoint:
# OpenAPI endpoint-presence tests are consolidated in test_api.py::TestOpenAPI.
class TestScriptResultEndpoint:
"""Tests for /api/scripts/{name}/result, which the upgrade page reads before calling an upgrade done."""
def test_reports_failed_upgrade(self, test_client):
result = {
"script_name": "upgrade", "running": False, "finished": True, "success": False, "result": "exit-code", "exit_status": 5,
"finished_at": "2026-11-08T15:22:32+00:00",
}
with mock.patch("controller.api.scripts.get_script_result", return_value=result):
response = test_client.get("/api/scripts/upgrade/result")
assert response.status_code == 200
assert response.json() == result
def test_unknown_script_is_404(self, test_client):
with mock.patch("controller.api.scripts.get_script_result", return_value=None):
response = test_client.get("/api/scripts/nonexistent/result")
assert response.status_code == 404