diff --git a/controller/api/schemas.py b/controller/api/schemas.py index f5befe43..d1ee16f5 100644 --- a/controller/api/schemas.py +++ b/controller/api/schemas.py @@ -449,6 +449,18 @@ class ScriptStatusResponse(BaseModel): elapsed_seconds: Optional[int] = Field(default=None, description="Seconds since start") +class ScriptResultResponse(BaseModel): + """Response model for how the last run of a script ended.""" + + script_name: str = Field(description="Name of the script") + running: bool = Field(description="Whether the script is running now") + finished: bool = Field(description="Whether a run is known to have ended (false while running, or if it never ran)") + success: Optional[bool] = Field(default=None, description="Whether the last run exited cleanly") + result: Optional[str] = Field(default=None, description="systemd's Result, e.g. 'success' or 'exit-code'") + exit_status: Optional[int] = Field(default=None, description="The script's exit status") + finished_at: Optional[str] = Field(default=None, description="ISO timestamp when the last run ended") + + class ScriptStartResponse(BaseModel): """Response model for starting a script.""" diff --git a/controller/api/scripts.py b/controller/api/scripts.py index 480b2aca..5c740601 100644 --- a/controller/api/scripts.py +++ b/controller/api/scripts.py @@ -12,6 +12,7 @@ from controller.api.schemas import ( BranchResponse, ScriptInfo, ScriptOutputResponse, + ScriptResultResponse, ScriptStartRequest, ScriptStartResponse, ScriptStatusResponse, @@ -20,6 +21,7 @@ from controller.lib.config import is_docker_mode from controller.lib.scripts import ( get_current_branch, get_script_output, + get_script_result, get_script_status, list_available_scripts, start_script, @@ -88,3 +90,18 @@ async def scripts_output(name: str, lines: int = 100) -> ScriptOutputResponse: lines = min(max(1, lines), 5000) result = get_script_output(name, lines) return ScriptOutputResponse(**result) + + +@router.get("/api/scripts/{name}/result", response_model=ScriptResultResponse) +async def scripts_result(name: str) -> ScriptResultResponse: + """How the last run of a script ended.""" + if is_docker_mode(): + raise HTTPException( + status_code=501, + detail="Scripts are not available in Docker mode" + ) + + result = get_script_result(name) + if result is None: + raise HTTPException(status_code=404, detail=f"Unknown script: {name}") + return ScriptResultResponse(**result) diff --git a/controller/lib/scripts.py b/controller/lib/scripts.py index f8c1a359..1072db77 100644 --- a/controller/lib/scripts.py +++ b/controller/lib/scripts.py @@ -182,6 +182,80 @@ def get_script_status() -> dict: return {"running": False} +def get_script_result(name: str) -> Optional[dict]: + """ + Report how the last run of a script ended, so a page waiting on it can tell success from failure. + + A failed upgrade restarts the services it stopped, so the services coming back up does not mean it succeeded. + + systemd keeps Result and the exit status across the daemon-reload an upgrade does, but not the run's + timestamps, so a run is known to have ended if it has an exit timestamp or ended badly. + + Returns None for an unknown script, otherwise a dict with: + script_name: str + running: bool - Whether the script is running now + finished: bool - Whether a run is known to have ended (False while running, or if it never ran) + success: Optional[bool] - Whether the last run exited cleanly; None unless finished + result: Optional[str] - systemd's Result (e.g. "success", "exit-code") + exit_status: Optional[int] - The script's exit status + finished_at: Optional[str] - ISO timestamp when the last run ended + """ + if name not in AVAILABLE_SCRIPTS: + return None + + result = { + "script_name": name, + "running": False, + "finished": False, + "success": None, + "result": None, + "exit_status": None, + "finished_at": None, + } + if is_docker_mode(): + return result + + service = AVAILABLE_SCRIPTS[name]["service_name"] + try: + # A Unix timestamp, because systemd prints dates in the local timezone. + proc = subprocess.run( + ["systemctl", "show", service, "--property=Result,ExecMainStatus,ActiveState,ExecMainExitTimestamp", + "--timestamp=unix"], + capture_output=True, + text=True, + timeout=5, + ) + except (subprocess.TimeoutExpired, subprocess.SubprocessError, FileNotFoundError): + return result + if proc.returncode != 0: + return result + + properties = dict(line.partition("=")[::2] for line in proc.stdout.splitlines() if "=" in line) + if properties.get("ActiveState") in ("activating", "active", "reloading"): + result["running"] = True + return result + + try: + result["exit_status"] = int(properties.get("ExecMainStatus", "")) + except ValueError: + pass + result["result"] = properties.get("Result") or None + failed = result["result"] not in (None, "success") or result["exit_status"] not in (0, None) + + exit_timestamp = properties.get("ExecMainExitTimestamp", "").lstrip("@") + if exit_timestamp: + try: + result["finished_at"] = datetime.fromtimestamp(float(exit_timestamp), timezone.utc).isoformat() + except ValueError: + pass + + # A clean Result with no exit timestamp is also what a service that never ran reports. + if failed or result["finished_at"]: + result["finished"] = True + result["success"] = not failed + return result + + def _get_service_timing(service: str) -> tuple[Optional[str], Optional[int]]: """ Get timing info for a running service. diff --git a/controller/templates/index.html b/controller/templates/index.html index 5e336587..68282d4e 100644 --- a/controller/templates/index.html +++ b/controller/templates/index.html @@ -679,6 +679,23 @@ margin-top: 1rem; } + .script-running-banner.failed { + background: linear-gradient(135deg, #e74c3c, #c0392b); + } + + .script-running-banner.failed .loading { + display: none; + } + + .script-failed-message { + display: none; + margin: 0.5rem 0 0 0; + } + + .script-running-banner.failed .script-failed-message { + display: block; + } + .warning-list { background: #1a1a2e; border: 1px solid #ff6b6b; @@ -914,8 +931,10 @@
{% endif %} @@ -2643,9 +2662,81 @@ // Two-phase upgrade detection: // Phase 1: Wait for services to go DOWN (confirms upgrade actually started). // Phase 2: Wait for services to come back UP (upgrade complete). + // A failed upgrade also restarts the services, so the upgrade's own result decides between + // redirecting and showing the failure. Only a run this page has seen start counts: the + // result of an earlier run is still reported until the new run begins. let upgradeServicesWentDown = sessionStorage.getItem('upgradeServicesWentDown') === 'true'; + let upgradeSeenRunning = sessionStorage.getItem('upgradeSeenRunning') === 'true'; + let upgradeInterval = null; + let upgradeDone = false; + + const markUpgradeStarted = (key) => { + if (key === 'upgradeServicesWentDown') upgradeServicesWentDown = true; + if (key === 'upgradeSeenRunning') upgradeSeenRunning = true; + sessionStorage.setItem(key, 'true'); + }; + + const finishUpgradeWatch = () => { + upgradeDone = true; + if (upgradeInterval) clearInterval(upgradeInterval); + sessionStorage.removeItem('upgradeServicesWentDown'); + sessionStorage.removeItem('upgradeSeenRunning'); + }; + + const showUpgradeFailed = (result) => { + finishUpgradeWatch(); + const banner = document.getElementById('script-running-banner'); + banner.classList.add('show', 'failed'); + document.getElementById('script-running-name').textContent = 'WROLPi Upgrade Failed'; + const code = result.exit_status !== null && result.exit_status !== undefined + ? ` (exit code ${result.exit_status})` : ''; + document.getElementById('script-failed-message').textContent = + `The upgrade did not complete${code}. See the output for the reason.`; + const returnLink = document.getElementById('script-failed-return'); + returnLink.href = `${upgradeHttpsBase}/`; + returnLink.style.display = ''; + showScriptOutput(); + }; + + // The upgrade's result, or null if it cannot be read (the Controller restarts during an upgrade). + const fetchUpgradeResult = async () => { + try { + const resp = await fetch('api/scripts/upgrade/result', { + headers: {'Accept': 'application/json'}, + signal: AbortSignal.timeout(5000) + }); + return resp.ok ? await resp.json() : null; + } catch (e) { + return null; + } + }; const checkUpgradeComplete = async () => { + if (upgradeDone) return; + + if (!upgradeSeenRunning) { + try { + const resp = await fetch('api/scripts/status', { + headers: {'Accept': 'application/json'}, + signal: AbortSignal.timeout(5000) + }); + const status = resp.ok ? await resp.json() : null; + if (status && status.running && status.script_name === 'upgrade') { + markUpgradeStarted('upgradeSeenRunning'); + } + } catch (e) { + // The ready check below handles an unreachable Controller. + } + } + + if (upgradeSeenRunning || upgradeServicesWentDown) { + const result = await fetchUpgradeResult(); + if (result && result.success === false) { + showUpgradeFailed(result); + return; + } + } + try { const resp = await fetch('api/ready', { method: 'GET', @@ -2656,26 +2747,31 @@ const data = await resp.json(); if (!upgradeServicesWentDown) { if (!data.api || !data.app) { - upgradeServicesWentDown = true; - sessionStorage.setItem('upgradeServicesWentDown', 'true'); + markUpgradeStarted('upgradeServicesWentDown'); } } else if (data.api && data.app) { - sessionStorage.removeItem('upgradeServicesWentDown'); - setTimeout(() => { - window.location.href = `${upgradeHttpsBase}/`; - }, 2000); + // The services are back; redirect only once the upgrade itself has finished well. + const result = await fetchUpgradeResult(); + if (result && result.success === false) { + showUpgradeFailed(result); + } else if (result && result.running) { + // Still running (e.g. restarting services); check again on the next poll. + } else { + finishUpgradeWatch(); + setTimeout(() => { + window.location.href = `${upgradeHttpsBase}/`; + }, 2000); + } } } else { - upgradeServicesWentDown = true; - sessionStorage.setItem('upgradeServicesWentDown', 'true'); + markUpgradeStarted('upgradeServicesWentDown'); } } catch (e) { - upgradeServicesWentDown = true; - sessionStorage.setItem('upgradeServicesWentDown', 'true'); + markUpgradeStarted('upgradeServicesWentDown'); } }; - setInterval(checkUpgradeComplete, 3000); + upgradeInterval = setInterval(checkUpgradeComplete, 3000); checkUpgradeComplete(); } } catch (e) { diff --git a/controller/test/test_scripts.py b/controller/test/test_scripts.py index 3573708d..fb43c246 100644 --- a/controller/test/test_scripts.py +++ b/controller/test/test_scripts.py @@ -13,6 +13,7 @@ from controller.lib.scripts import ( AVAILABLE_SCRIPTS, get_current_branch, get_script_output, + get_script_result, get_script_status, list_available_scripts, start_script, @@ -368,3 +369,89 @@ class TestGetServiceTiming: completed = subprocess.CompletedProcess(args=[], returncode=0, stdout="ExecMainStartTimestamp=\n") with mock.patch("controller.lib.scripts.subprocess.run", return_value=completed): assert _get_service_timing("wrolpi-upgrade.service") == (None, None) + + +def _systemctl_show(stdout: str): + return mock.Mock(returncode=0, stdout=stdout) + + +class TestGetScriptResult: + """How the last run of a script ended, read from systemd.""" + + def test_failed_run(self): + """An upgrade that exited non-zero is reported as failed, with its exit status and when it finished.""" + stdout = "Result=exit-code\nExecMainStatus=5\nActiveState=failed\nExecMainExitTimestamp=@1794151352.5\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)) as run: + result = get_script_result("upgrade") + assert result == { + "script_name": "upgrade", + "running": False, + "finished": True, + "success": False, + "result": "exit-code", + "exit_status": 5, + "finished_at": "2026-11-08T15:22:32.500000+00:00", + } + assert "wrolpi-upgrade.service" in run.call_args[0][0] + + def test_successful_run(self): + stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=@1762615352\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + result = get_script_result("upgrade") + assert result["finished"] is True + assert result["success"] is True + assert result["exit_status"] == 0 + + def test_running(self): + """A run in progress has not finished, so it has neither succeeded nor failed.""" + stdout = "Result=success\nExecMainStatus=0\nActiveState=activating\nExecMainExitTimestamp=\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + result = get_script_result("upgrade") + assert result["running"] is True + assert result["finished"] is False + assert result["success"] is None + assert result["finished_at"] is None + + def test_failed_run_after_daemon_reload(self): + """An upgrade reloads systemd, which clears the run's timestamps but keeps its Result and exit status.""" + stdout = "Result=exit-code\nExecMainStatus=5\nActiveState=failed\nExecMainExitTimestamp=\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + result = get_script_result("upgrade") + assert result["running"] is False + assert result["finished"] is True + assert result["success"] is False + assert result["exit_status"] == 5 + assert result["finished_at"] is None + + def test_successful_run_after_daemon_reload(self): + """A clean run whose timestamps were cleared is not running and not failed; the upgrade page redirects.""" + stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + result = get_script_result("upgrade") + assert result["running"] is False + assert result["success"] is not False + + def test_never_run(self): + """A service that has never run has no result.""" + stdout = "Result=success\nExecMainStatus=0\nActiveState=inactive\nExecMainExitTimestamp=\n" + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", return_value=_systemctl_show(stdout)): + result = get_script_result("upgrade") + assert result["finished"] is False + assert result["success"] is None + + def test_systemctl_missing(self): + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False), \ + mock.patch("controller.lib.scripts.subprocess.run", side_effect=FileNotFoundError): + result = get_script_result("upgrade") + assert result["finished"] is False + assert result["success"] is None + + def test_unknown_script(self): + with mock.patch("controller.lib.scripts.is_docker_mode", return_value=False): + assert get_script_result("nonexistent") is None diff --git a/controller/test/test_scripts_api.py b/controller/test/test_scripts_api.py index c03b9f09..0f245231 100644 --- a/controller/test/test_scripts_api.py +++ b/controller/test/test_scripts_api.py @@ -13,6 +13,7 @@ class TestDockerModeRejectsScriptsEndpoints: @pytest.mark.parametrize("method,endpoint", [ ("post", "/api/scripts/repair/start"), ("get", "/api/scripts/repair/output"), + ("get", "/api/scripts/upgrade/result"), ]) def test_action_rejected_in_docker(self, test_client_docker_mode, method, endpoint): response = getattr(test_client_docker_mode, method)(endpoint) @@ -166,3 +167,22 @@ class TestScriptsOutputEndpoint: # OpenAPI endpoint-presence tests are consolidated in test_api.py::TestOpenAPI. + + +class TestScriptResultEndpoint: + """Tests for /api/scripts/{name}/result, which the upgrade page reads before calling an upgrade done.""" + + def test_reports_failed_upgrade(self, test_client): + result = { + "script_name": "upgrade", "running": False, "finished": True, "success": False, "result": "exit-code", "exit_status": 5, + "finished_at": "2026-11-08T15:22:32+00:00", + } + with mock.patch("controller.api.scripts.get_script_result", return_value=result): + response = test_client.get("/api/scripts/upgrade/result") + assert response.status_code == 200 + assert response.json() == result + + def test_unknown_script_is_404(self, test_client): + with mock.patch("controller.api.scripts.get_script_result", return_value=None): + response = test_client.get("/api/scripts/nonexistent/result") + assert response.status_code == 404