From 19ff59d5e00a3d0c48c2982ce976fb0ace0a57c5 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:11:41 -0700 Subject: [PATCH] A failed upgrade is shown on the upgrade page instead of redirecting home (#955) * A failed upgrade is shown on the upgrade page instead of redirecting home A failed upgrade restarts the services it stopped, so the upgrade page saw them come back and redirected home as if the upgrade had worked. The Controller now reports how the last run of a script ended (GET /api/scripts/{name}/result, from systemd's Result and exit status), and the upgrade page reads it before redirecting. A failure turns the banner red, gives the exit code, opens the output, and offers a link back. Only a run the page has seen start counts, because the result of an earlier run is still reported until the new run begins. Co-Authored-By: Claude Opus 5.5 * An upgrade's result survives the daemon-reload it does The upgrade reinstalls its unit and reloads systemd, which clears the run's timestamps but keeps Result and the exit status. A failure is now known from those alone, and the result reports whether the script is running, so the upgrade page waits on that instead of on a timestamp that may never come. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- controller/api/schemas.py | 12 +++ controller/api/scripts.py | 17 ++++ controller/lib/scripts.py | 74 +++++++++++++++++ controller/templates/index.html | 118 +++++++++++++++++++++++++--- controller/test/test_scripts.py | 87 ++++++++++++++++++++ controller/test/test_scripts_api.py | 20 +++++ 6 files changed, 317 insertions(+), 11 deletions(-) diff --git a/controller/api/schemas.py b/controller/api/schemas.py index f5befe43..d1ee16f5 100644 --- a/controller/api/schemas.py +++ b/controller/api/schemas.py @@ -449,6 +449,18 @@ class ScriptStatusResponse(BaseModel): elapsed_seconds: Optional[int] = Field(default=None, description="Seconds since start") +class ScriptResultResponse(BaseModel): + """Response model for how the last run of a script ended.""" + + script_name: str = Field(description="Name of the script") + running: bool = Field(description="Whether the script is running now") + finished: bool = Field(description="Whether a run is known to have ended (false while running, or if it never ran)") + success: Optional[bool] = Field(default=None, description="Whether the last run exited cleanly") + result: Optional[str] = Field(default=None, description="systemd's Result, e.g. 'success' or 'exit-code'") + exit_status: Optional[int] = Field(default=None, description="The script's exit status") + finished_at: Optional[str] = Field(default=None, description="ISO timestamp when the last run ended") + + class ScriptStartResponse(BaseModel): """Response model for starting a script.""" diff --git a/controller/api/scripts.py b/controller/api/scripts.py index 480b2aca..5c740601 100644 --- a/controller/api/scripts.py +++ b/controller/api/scripts.py @@ -12,6 +12,7 @@ from controller.api.schemas import ( BranchResponse, ScriptInfo, ScriptOutputResponse, + ScriptResultResponse, ScriptStartRequest, ScriptStartResponse, ScriptStatusResponse, @@ -20,6 +21,7 @@ from controller.lib.config import is_docker_mode from controller.lib.scripts import ( get_current_branch, get_script_output, + get_script_result, get_script_status, list_available_scripts, start_script, @@ -88,3 +90,18 @@ async def scripts_output(name: str, lines: int = 100) -> ScriptOutputResponse: lines = min(max(1, lines), 5000) result = get_script_output(name, lines) return ScriptOutputResponse(**result) + + +@router.get("/api/scripts/{name}/result", response_model=ScriptResultResponse) +async def scripts_result(name: str) -> ScriptResultResponse: + """How the last run of a script ended.""" + if is_docker_mode(): + raise HTTPException( + status_code=501, + detail="Scripts are not available in Docker mode" + ) + + result = get_script_result(name) + if result is None: + raise HTTPException(status_code=404, detail=f"Unknown script: {name}") + return ScriptResultResponse(**result) diff --git a/controller/lib/scripts.py b/controller/lib/scripts.py index f8c1a359..1072db77 100644 --- a/controller/lib/scripts.py +++ b/controller/lib/scripts.py @@ -182,6 +182,80 @@ def get_script_status() -> dict: return {"running": False} +def get_script_result(name: str) -> Optional[dict]: + """ + Report how the last run of a script ended, so a page waiting on it can tell success from failure. + + A failed upgrade restarts the services it stopped, so the services coming back up does not mean it succeeded. + + systemd keeps Result and the exit status across the daemon-reload an upgrade does, but not the run's + timestamps, so a run is known to have ended if it has an exit timestamp or ended badly. + + Returns None for an unknown script, otherwise a dict with: + script_name: str + running: bool - Whether the script is running now + finished: bool - Whether a run is known to have ended (False while running, or if it never ran) + success: Optional[bool] - Whether the last run exited cleanly; None unless finished + result: Optional[str] - systemd's Result (e.g. "success", "exit-code") + exit_status: Optional[int] - The script's exit status + finished_at: Optional[str] - ISO timestamp when the last run ended + """ + if name not in AVAILABLE_SCRIPTS: + return None + + result = { + "script_name": name, + "running": False, + "finished": False, + "success": None, + "result": None, + "exit_status": None, + "finished_at": None, + } + if is_docker_mode(): + return result + + service = AVAILABLE_SCRIPTS[name]["service_name"] + try: + # A Unix timestamp, because systemd prints dates in the local timezone. + proc = subprocess.run( + ["systemctl", "show", service, "--property=Result,ExecMainStatus,ActiveState,ExecMainExitTimestamp", + "--timestamp=unix"], + capture_output=True, + text=True, + timeout=5, + ) + except (subprocess.TimeoutExpired, subprocess.SubprocessError, FileNotFoundError): + return result + if proc.returncode != 0: + return result + + properties = dict(line.partition("=")[::2] for line in proc.stdout.splitlines() if "=" in line) + if properties.get("ActiveState") in ("activating", "active", "reloading"): + result["running"] = True + return result + + try: + result["exit_status"] = int(properties.get("ExecMainStatus", "")) + except ValueError: + pass + result["result"] = properties.get("Result") or None + failed = result["result"] not in (None, "success") or result["exit_status"] not in (0, None) + + exit_timestamp = properties.get("ExecMainExitTimestamp", "").lstrip("@") + if exit_timestamp: + try: + result["finished_at"] = datetime.fromtimestamp(float(exit_timestamp), timezone.utc).isoformat() + except ValueError: + pass + + # A clean Result with no exit timestamp is also what a service that never ran reports. + if failed or result["finished_at"]: + result["finished"] = True + result["success"] = not failed + return result + + def _get_service_timing(service: str) -> tuple[Optional[str], Optional[int]]: """ Get timing info for a running service. diff --git a/controller/templates/index.html b/controller/templates/index.html index 5e336587..68282d4e 100644 --- a/controller/templates/index.html +++ b/controller/templates/index.html @@ -679,6 +679,23 @@ margin-top: 1rem; } + .script-running-banner.failed { + background: linear-gradient(135deg, #e74c3c, #c0392b); + } + + .script-running-banner.failed .loading { + display: none; + } + + .script-failed-message { + display: none; + margin: 0.5rem 0 0 0; + } + + .script-running-banner.failed .script-failed-message { + display: block; + } + .warning-list { background: #1a1a2e; border: 1px solid #ff6b6b; @@ -914,8 +931,10 @@

Script Running

+

+
{% endif %} @@ -2643,9 +2662,81 @@ // Two-phase upgrade detection: // Phase 1: Wait for services to go DOWN (confirms upgrade actually started). // Phase 2: Wait for services to come back UP (upgrade complete). + // A failed upgrade also restarts the services, so the upgrade's own result decides between + // redirecting and showing the failure. Only a run this page has seen start counts: the + // result of an earlier run is still reported until the new run begins. let upgradeServicesWentDown = sessionStorage.getItem('upgradeServicesWentDown') === 'true'; + let upgradeSeenRunning = sessionStorage.getItem('upgradeSeenRunning') === 'true'; + let upgradeInterval = null; + let upgradeDone = false; + + const markUpgradeStarted = (key) => { + if (key === 'upgradeServicesWentDown') upgradeServicesWentDown = true; + if (key === 'upgradeSeenRunning') upgradeSeenRunning = true; + sessionStorage.setItem(key, 'true'); + }; + + const finishUpgradeWatch = () => { + upgradeDone = true; + if (upgradeInterval) clearInterval(upgradeInterval); + sessionStorage.removeItem('upgradeServicesWentDown'); + sessionStorage.removeItem('upgradeSeenRunning'); + }; + + const showUpgradeFailed = (result) => { + finishUpgradeWatch(); + const banner = document.getElementById('script-running-banner'); + banner.classList.add('show', 'failed'); + document.getElementById('script-running-name').textContent = 'WROLPi Upgrade Failed'; + const code = result.exit_status !== null && result.exit_status !== undefined + ? ` (exit code ${result.exit_status})` : ''; + document.getElementById('script-failed-message').textContent = + `The upgrade did not complete${code}. See the output for the reason.`; + const returnLink = document.getElementById('script-failed-return'); + returnLink.href = `${upgradeHttpsBase}/`; + returnLink.style.display = ''; + showScriptOutput(); + }; + + // The upgrade's result, or null if it cannot be read (the Controller restarts during an upgrade). + const fetchUpgradeResult = async () => { + try { + const resp = await fetch('api/scripts/upgrade/result', { + headers: {'Accept': 'application/json'}, + signal: AbortSignal.timeout(5000) + }); + return resp.ok ? await resp.json() : null; + } catch (e) { + return null; + } + }; const checkUpgradeComplete = async () => { + if (upgradeDone) return; + + if (!upgradeSeenRunning) { + try { + const resp = await fetch('api/scripts/status', { + headers: {'Accept': 'application/json'}, + signal: AbortSignal.timeout(5000) + }); + const status = resp.ok ? await resp.json() : null; + if (status && status.running && status.script_name === 'upgrade') { + markUpgradeStarted('upgradeSeenRunning'); + } + } catch (e) { + // The ready check below handles an unreachable Controller. + } + } + + if (upgradeSeenRunning || upgradeServicesWentDown) { + const result = await fetchUpgradeResult(); + if (result && result.success === false) { + showUpgradeFailed(result); + return; + } + } + try { const resp = await fetch('api/ready', { method: 'GET', @@ -2656,26 +2747,31 @@ const data = await resp.json(); if (!upgradeServicesWentDown) { if (!data.api || !data.app) { - upgradeServicesWentDown = true; - sessionStorage.setItem('upgradeServicesWentDown', 'true'); + markUpgradeStarted('upgradeServicesWentDown'); } } else if (data.api && data.app) { - sessionStorage.removeItem('upgradeServicesWentDown'); - setTimeout(() => { - window.location.href = `${upgradeHttpsBase}/`; - }, 2000); + // The services are back; redirect only once the upgrade itself has finished well. + const result = await fetchUpgradeResult(); + if (result && result.success === false) { + showUpgradeFailed(result); + } else if (result && result.running) { + // Still running (e.g. restarting services); check again on the next poll. + } else { + finishUpgradeWatch(); + setTimeout(() => { + window.location.href = `${upgradeHttpsBase}/`; + }, 2000); + } } } else { - upgradeServicesWentDown = true; - sessionStorage.setItem('upgradeServicesWentDown', 'true'); + markUpgradeStarted('upgradeServicesWentDown'); } } catch (e) { - upgradeServicesWentDown = true; - sessionStorage.setItem('upgradeServicesWentDown', 'true'); + markUpgradeStarted('upgradeServicesWentDown'); } }; - setInterval(checkUpgradeComplete, 3000); + upgradeInterval = setInterval(checkUpgradeComplete, 3000); checkUpgradeComplete(); } } catch (e) { diff --git a/controller/test/test_scripts.py b/controller/test/test_scripts.py index 3573708d..fb43c246 100644 --- a/controller/test/test_scripts.py +++ b/controller/test/test_scripts.py @@ -13,6 +13,7 @@ from controller.lib.scripts import ( AVAILABLE_SCRIPTS, get_current_branch, get_script_output, + get_script_result, get_script_status, list_available_scripts, start_script, @@ -368,3 +369,89 @@ class TestGetServiceTiming: completed = subprocess.CompletedProcess(args=[], returncode=0, stdout="ExecMainStartTimestamp=\n") with mock.patch("controller.lib.scripts.subprocess.run", return_value=completed): assert _get_service_timing("wrolpi-upgrade.service") == (None, None) + + +def _systemctl_show(stdout: str): + return mock.Mock(returncode=0, stdout=stdout) + + +class TestGetScriptResult: + """How the last run of a script ended, read from systemd.""" + + def test_failed_run(self): + """An upgrade that exited non-zero is reported as failed, with its exit status and when it finished.""" + stdout = "Result=exit-code\nExecMainStatus=5\nActiveState=failed\nExecMainExitTimestamp=@1794151352.5\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)) as run: + result = get_script_result("upgrade") + assert result == { + "script_name": "upgrade", + "running": False, + "finished": True, + "success": False, + "result": "exit-code", + "exit_status": 5, + "finished_at": "2026-11-08T15:22:32.500000+00:00", + } + assert "wrolpi-upgrade.service" in run.call_args[0][0] + + def test_successful_run(self): + stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=@1762615352\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + result = get_script_result("upgrade") + assert result["finished"] is True + assert result["success"] is True + assert result["exit_status"] == 0 + + def test_running(self): + """A run in progress has not finished, so it has neither succeeded nor failed.""" + stdout = "Result=success\nExecMainStatus=0\nActiveState=activating\nExecMainExitTimestamp=\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + result = get_script_result("upgrade") + assert result["running"] is True + assert result["finished"] is False + assert result["success"] is None + assert result["finished_at"] is None + + def test_failed_run_after_daemon_reload(self): + """An upgrade reloads systemd, which clears the run's timestamps but keeps its Result and exit status.""" + stdout = "Result=exit-code\nExecMainStatus=5\nActiveState=failed\nExecMainExitTimestamp=\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + result = get_script_result("upgrade") + assert result["running"] is False + assert result["finished"] is True + assert result["success"] is False + assert result["exit_status"] == 5 + assert result["finished_at"] is None + + def test_successful_run_after_daemon_reload(self): + """A clean run whose timestamps were cleared is not running and not failed; the upgrade page redirects.""" + stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + result = get_script_result("upgrade") + assert result["running"] is False + assert result["success"] is not False + + def test_never_run(self): + """A service that has never run has no result.""" + stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + result = get_script_result("upgrade") + assert result["finished"] is False + assert result["success"] is None + + def test_systemctl_missing(self): + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", side_effect=FileNotFoundError): + result = get_script_result("upgrade") + assert result["finished"] is False + assert result["success"] is None + + def test_unknown_script(self): + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False): + assert get_script_result("nonexistent") is None diff --git a/controller/test/test_scripts_api.py b/controller/test/test_scripts_api.py index c03b9f09..0f245231 100644 --- a/controller/test/test_scripts_api.py +++ b/controller/test/test_scripts_api.py @@ -13,6 +13,7 @@ class TestDockerModeRejectsScriptsEndpoints: @pytest.mark.parametrize("method,endpoint", [ ("post", "/api/scripts/repair/start"), ("get", "/api/scripts/repair/output"), + ("get", "/api/scripts/upgrade/result"), ]) def test_action_rejected_in_docker(self, test_client_docker_mode, method, endpoint): response = getattr(test_client_docker_mode, method)(endpoint) @@ -166,3 +167,22 @@ class TestScriptsOutputEndpoint: # OpenAPI endpoint-presence tests are consolidated in test_api.py::TestOpenAPI. + + +class TestScriptResultEndpoint: + """Tests for /api/scripts/{name}/result, which the upgrade page reads before calling an upgrade done.""" + + def test_reports_failed_upgrade(self, test_client): + result = { + "script_name": "upgrade", "running": False, "finished": True, "success": False, "result": "exit-code", "exit_status": 5, + "finished_at": "2026-11-08T15:22:32+00:00", + } + with mock.patch("controller.api.scripts.get_script_result", return_value=result): + response = test_client.get("/api/scripts/upgrade/result") + assert response.status_code == 200 + assert response.json() == result + + def test_unknown_script_is_404(self, test_client): + with mock.patch("controller.api.scripts.get_script_result", return_value=None): + response = test_client.get("/api/scripts/nonexistent/result") + assert response.status_code == 404