upgrade: restart services when the upgrade fails before code changes (#669)

* upgrade: restart services when the upgrade fails before code changes

The upgrade script stops wrolpi-api and wrolpi-app before fetching, so a
failed fetch, signature verification, or checkout left the WROLPi down
with unchanged code on disk. The EXIT trap now restarts both services
for failures that occur before scripts/upgrade.sh begins modifying the
system. The post-checkout verification failure rolls the branch back to
the previously-running commit first, so services never restart on an
unverified commit.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* upgrade: only restart services when rollback lands on the known-good commit

Review fixes: a failed rollback no longer leaves the restart flag set, so
services cannot start on an unverified or mixed tree; the non-atomic
checkout failure path now rolls back too; trimmed narrative comments.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

upgrade.sh

@@ -89,6 +89,11 @@ trap '
     echo "WROLPi upgrade has completed"
   else
     echo "WROLPi upgrade has FAILED (exit code: $EXIT_STATUS)"
+    if [ "$RESTART_SERVICES_ON_FAILURE" = true ]; then
+      echo "Restarting WROLPi services after failed upgrade..."
+      systemctl start wrolpi-api || :
+      systemctl start wrolpi-app || :
+    fi
   fi
 ' EXIT
 
@@ -101,6 +106,18 @@ set -o pipefail
 # Units may not be installed yet if a previous install/repair failed; repair.sh installs them later.
 systemctl stop wrolpi-api || :
 systemctl stop wrolpi-app || :
+RESTART_SERVICES_ON_FAILURE=true
+
+# Restore the previously-running commit after a failed checkout or verification.
+# Services must only restart on that known-good commit, so if the rollback does
+# not land there, leave them stopped.
+rollback_or_no_restart() {
+  git -C /opt/wrolpi checkout -f -B "${BRANCH}" "${PREVIOUS_HEAD}" || :
+  if [ "$(git -C /opt/wrolpi rev-parse HEAD 2>/dev/null)" != "${PREVIOUS_HEAD}" ]; then
+    echo "ERROR: rollback to ${PREVIOUS_HEAD} failed; not restarting services."
+    RESTART_SERVICES_ON_FAILURE=false
+  fi
+}
 
 # Fetch the latest commits without modifying the working tree.
 git -C /opt/wrolpi fetch || exit 4
@@ -125,15 +142,22 @@ if ! git -C /opt/wrolpi verify-commit origin/"${BRANCH}" 2>&1; then
 fi
 
 # Signature verified, safe to apply.
+PREVIOUS_HEAD=$(git -C /opt/wrolpi rev-parse HEAD)
 # Force the checkout so local working-tree changes (e.g. app/package-lock.json
 # rewritten by npm install during a prior upgrade) cannot abort the switch.
 # Use `checkout -B <branch> origin/<branch>` so the ref can never be mistaken for a path. A tracked `release/`
 # directory otherwise collides with the `release` branch name, making a bare `git checkout release` ambiguous.
-(cd /opt/wrolpi && git checkout -f -B "${BRANCH}" "origin/${BRANCH}") || exit 4
+# checkout -B moves the ref before checking out the tree, so a failure here may
+# leave a mixed tree; roll back before the EXIT trap restarts anything.
+(cd /opt/wrolpi && git checkout -f -B "${BRANCH}" "origin/${BRANCH}") || {
+  rollback_or_no_restart
+  exit 4
+}
 
 # Verify HEAD again after checkout in case the branch was swapped between fetch and checkout.
 if ! git -C /opt/wrolpi verify-commit HEAD 2>&1; then
   echo "ERROR: HEAD commit signature verification failed after checkout. Aborting upgrade."
+  rollback_or_no_restart
   rm -rf "$GNUPGHOME"
   exit 6
 fi
@@ -141,6 +165,10 @@ fi
 rm -rf "$GNUPGHOME"
 unset GNUPGHOME
 
+# From here scripts/upgrade.sh mutates the system; a blind restart could run
+# half-upgraded code, so recovery belongs to repair.sh.
+RESTART_SERVICES_ON_FAILURE=false
+
 /opt/wrolpi/scripts/upgrade.sh 2>&1 | tee /opt/wrolpi/upgrade.log
 
 set +x