Automating build and upload of WROLPi release images.

This commit is contained in:
Roland 2026-05-29 11:02:32 -06:00 • committed by lrnselfreliance
parent f4d0468300
commit 0e3564b704
7 changed files with 353 additions and 0 deletions

3
.gitignore vendored
View file

@ -144,3 +144,6 @@ app/cypress/videos
.claude .claude
*.md *.md
/.mcp.json /.mcp.json
# Generated by release/release.sh before upload
release/latest.json

View file

@ -141,4 +141,29 @@ DEST="${SCRIPT_DIR}/WROLPi-v${VERSION}-amd64.iso"
[ -f "${DEST}" ] && (echo "Removing conflicting ISO" && rm "${DEST}") [ -f "${DEST}" ] && (echo "Removing conflicting ISO" && rm "${DEST}")
mv "${SCRIPT_DIR}"/*.iso "${DEST}" mv "${SCRIPT_DIR}"/*.iso "${DEST}"
# Sanity-check the resulting ISO before declaring success, so a truncated or
# corrupt build never ships.
MIN_ISO_BYTES=$((1024 * 1024 * 1024)) # 1 GiB floor; a real ISO is ~3 GiB.
ISO_BYTES=$(stat -c%s "${DEST}")
if [ "${ISO_BYTES}" -lt "${MIN_ISO_BYTES}" ]; then
echo "ERROR: ISO is only ${ISO_BYTES} bytes (< ${MIN_ISO_BYTES}); build likely truncated." >&2
exit 1
fi
# The ISO 9660 primary volume descriptor carries the "CD001" magic at byte
# offset 32769. It is present even in iso-hybrid images, where `file` reports
# the leading MBR instead.
if [ "$(dd if="${DEST}" bs=1 skip=32769 count=5 2>/dev/null)" != "CD001" ]; then
echo "ERROR: ${DEST} is not a valid ISO 9660 image (missing CD001 magic)." >&2
exit 1
fi
# Cross-check with blkid when available.
if command -v blkid >/dev/null 2>&1; then
ISO_FSTYPE=$(blkid -o value -s TYPE "${DEST}" 2>/dev/null || true)
if [ -n "${ISO_FSTYPE}" ] && [ "${ISO_FSTYPE}" != "iso9660" ]; then
echo "ERROR: blkid reports filesystem '${ISO_FSTYPE}', expected iso9660." >&2
exit 1
fi
fi
echo "ISO sanity checks passed ($(numfmt --to=iec "${ISO_BYTES}" 2>/dev/null || echo "${ISO_BYTES} bytes"), iso9660)."
echo "Build has completed. ISO output ${DEST}" echo "Build has completed. ISO output ${DEST}"

View file

@ -125,4 +125,28 @@ chown -R 1000:1000 "${SCRIPT_DIR}"
set +x set +x
# Sanity-check the resulting image before declaring success, so a truncated or
# corrupt build never ships.
IMG="${SCRIPT_DIR}/WROLPi-v${VERSION}-aarch64-desktop.img.xz"
MIN_IMG_BYTES=$((1024 * 1024 * 1024)) # 1 GiB floor; compressed is ~5 GiB.
MIN_UNCOMPRESSED_BYTES=$((2 * 1024 * 1024 * 1024)) # 2 GiB floor; real is ~8 GiB.
IMG_BYTES=$(stat -c%s "${IMG}")
if [ "${IMG_BYTES}" -lt "${MIN_IMG_BYTES}" ]; then
echo "ERROR: image is only ${IMG_BYTES} bytes (< ${MIN_IMG_BYTES}); build likely truncated." >&2
exit 1
fi
# Full decompression test: catches any corruption in the .xz stream.
if ! xz -t "${IMG}"; then
echo "ERROR: ${IMG} failed xz integrity check." >&2
exit 1
fi
# Confirm the uncompressed image is a sane size (field 5 of the robot totals).
UNCOMPRESSED_BYTES=$(xz --robot -l "${IMG}" 2>/dev/null | awk -F'\t' '/^totals/{print $5}')
if [ -n "${UNCOMPRESSED_BYTES}" ] && [ "${UNCOMPRESSED_BYTES}" -lt "${MIN_UNCOMPRESSED_BYTES}" ]; then
echo "ERROR: uncompressed image is only ${UNCOMPRESSED_BYTES} bytes (< ${MIN_UNCOMPRESSED_BYTES})." >&2
exit 1
fi
echo "Image sanity checks passed ($(numfmt --to=iec "${IMG_BYTES}" 2>/dev/null || echo "${IMG_BYTES} bytes") compressed, xz OK)."
echo "Build completed successfully" echo "Build completed successfully"

91
release/release-watch.sh Executable file
View file

@ -0,0 +1,91 @@
#!/bin/bash
# Poll GitHub for a newly-published WROLPi release and build it.
#
# This is the *outbound* poller that replaces an inbound webhook: GitHub never
# connects to this machine. Once an hour (via the systemd timer) we ask the
# public GitHub API for the newest release, compare its version to whatever is
# already published in latest.json on the CDN, and if they differ we run
# release.sh to build and publish it. latest.json is the source of truth for
# "already built" -- there is no local bookkeeping for success.
#
# A local state file is used only to throttle retries of a release that fails
# to build, so a permanently-broken release does not rebuild every hour
# forever.
#
# ./release-watch.sh # check once and act
#
# Honors $S3CFG (passed through to release.sh) and these optional env vars:
# STATE_DIR where to keep the lock + failure marker (default /var/lib/wrolpi-release)
# MAX_RETRIES how many times to auto-retry a failing tag (default 3)
set -euo pipefail
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
REPO="lrnselfreliance/wrolpi"
CDN_BASE="https://wrolpi.nyc3.cdn.digitaloceanspaces.com"
STATE_DIR="${STATE_DIR:-/var/lib/wrolpi-release}"
MAX_RETRIES="${MAX_RETRIES:-3}"
LOCK="${STATE_DIR}/build.lock"
FAIL_STATE="${STATE_DIR}/failed-tag"
mkdir -p "${STATE_DIR}"
log() { echo "[release-watch] $*"; }
# Normalize a release tag to a version string (drop a leading 'v').
normalize() { echo "${1#v}"; }
# Newest non-draft release tag (includes prereleases / -beta). Read-only and
# unauthenticated; the public 60 req/hr limit is irrelevant at hourly cadence.
LATEST_TAG=$(curl -fsSL -H "Accept: application/vnd.github+json" \
"https://api.github.com/repos/${REPO}/releases" \
| jq -r '[.[] | select(.draft == false)][0].tag_name // empty')
[ -n "${LATEST_TAG}" ] || { log "No releases found; nothing to do."; exit 0; }
# What is already published? A missing manifest (404) means "nothing yet".
PUBLISHED_VERSION=$(curl -fsSL "${CDN_BASE}/latest.json" 2>/dev/null \
| jq -r '.version // empty' || true)
LATEST_VERSION=$(normalize "${LATEST_TAG}")
if [ "${LATEST_VERSION}" = "${PUBLISHED_VERSION}" ]; then
log "Up to date (published v${PUBLISHED_VERSION}); nothing to do."
exit 0
fi
log "New release ${LATEST_TAG} (published: ${PUBLISHED_VERSION:-none})."
# Skip a tag that has already failed MAX_RETRIES times -- needs a human.
if [ -f "${FAIL_STATE}" ]; then
read -r failed_tag failed_count < "${FAIL_STATE}" || true
if [ "${failed_tag}" = "${LATEST_TAG}" ] && [ "${failed_count:-0}" -ge "${MAX_RETRIES}" ]; then
log "ERROR: ${LATEST_TAG} has failed ${failed_count} times; not auto-retrying."
log "Investigate, then run release.sh manually or remove ${FAIL_STATE}."
exit 1
fi
fi
# Only one build at a time. If the timer fires mid-build, just exit.
exec 9>"${LOCK}"
if ! flock -n 9; then
log "A build is already running; skipping this check."
exit 0
fi
log "Building ${LATEST_TAG}..."
if "${SCRIPT_DIR}/release.sh" -b "${LATEST_TAG}"; then
log "Published ${LATEST_TAG}."
rm -f "${FAIL_STATE}"
else
rc=$?
count=1
if [ -f "${FAIL_STATE}" ]; then
read -r ft fc < "${FAIL_STATE}" || true
[ "${ft}" = "${LATEST_TAG}" ] && count=$(( ${fc:-0} + 1 ))
fi
echo "${LATEST_TAG} ${count}" > "${FAIL_STATE}"
log "ERROR: build of ${LATEST_TAG} failed (attempt ${count}, exit ${rc})."
exit "${rc}"
fi

160
release/release.sh Executable file
View file

@ -0,0 +1,160 @@
#!/bin/bash
# Build, checksum, upload, and publish a WROLPi release.
#
# Runs the full release pipeline as a single command:
# 1. Resolve the version from the target git ref (branch or tag).
# 2. Build the amd64 ISO (debian-live-config) and aarch64 image (pi-gen).
# 3. Generate .sha256 sidecars.
# 4. Upload images + checksums to DigitalOcean Spaces (s3cmd).
# 5. Publish latest.json (the manifest wrolpi.org reads) to the bucket root.
#
# Everything here talks only to public endpoints (github.com, the Spaces CDN).
# Spaces credentials live in the s3cmd config (~/.s3cfg or $S3CFG), never here.
#
# sudo ./release.sh # build the 'release' branch
# sudo ./release.sh -b v0.23.0-beta # build a specific tag
# ./release.sh -b release -n # dry run: build nothing, show upload plan
#
# Building a tag works because the chroot hooks `git clone -b <ref>`, which
# accepts a tag (detached HEAD) just as well as a branch.
set -euo pipefail
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
REPO_DIR=$( cd -- "${SCRIPT_DIR}/.." &> /dev/null && pwd )
# Public constants. None of these are secret; the bucket and CDN host are
# already visible on wrolpi.org.
REPO="lrnselfreliance/wrolpi"
BUCKET="s3://wrolpi"
CDN_BASE="https://wrolpi.nyc3.cdn.digitaloceanspaces.com"
# Cache the manifest briefly so a new release shows up quickly; cache the
# immutable versioned images for a year.
MANIFEST_CACHE="public, max-age=300"
IMAGE_CACHE="public, max-age=31536000, immutable"
REF="release"
DRY_RUN=0
Help() {
echo "Build and publish a WROLPi release."
echo
echo "Syntax: release.sh [-h] [-b REF] [-n]"
echo "options:"
echo "h Print this help."
echo "b Build from this git branch or tag (default: 'release')."
echo "n Dry run: skip the build and only print the upload/manifest plan."
echo
}
while getopts ":hb:n" option; do
case $option in
h) Help; exit ;;
b) REF="${OPTARG}" ;;
n) DRY_RUN=1 ;;
*) echo "Error: Invalid option"; Help; exit 1 ;;
esac
done
# s3cmd reads $S3CFG if set (handy when this runs as root via systemd but the
# credentials live in a user's home), otherwise its default ~/.s3cfg.
S3CMD=(s3cmd)
[ -n "${S3CFG:-}" ] && S3CMD=(s3cmd -c "${S3CFG}")
log() { echo "[release] $*"; }
die() { echo "[release] ERROR: $*" >&2; exit 1; }
# Resolve the version straight from the target ref so the artifact names match
# the code that gets built.
VERSION=$(curl -fsSL "https://raw.githubusercontent.com/${REPO}/${REF}/wrolpi/version.txt" | tr -d '[:space:]')
[ -n "${VERSION}" ] || die "Could not fetch version from ref '${REF}'"
log "Releasing WROLPi v${VERSION} (ref: ${REF})"
ISO="${REPO_DIR}/debian-live-config/WROLPi-v${VERSION}-amd64.iso"
IMG="${REPO_DIR}/pi-gen/WROLPi-v${VERSION}-aarch64-desktop.img.xz"
if [ "${DRY_RUN}" -eq 1 ]; then
log "DRY RUN: skipping builds."
else
log "Building amd64 ISO..."
"${REPO_DIR}/debian-live-config/build.sh" -b "${REF}"
log "Building aarch64 image..."
"${REPO_DIR}/pi-gen/build.sh" -b "${REF}"
[ -s "${ISO}" ] || die "Expected ISO not found: ${ISO}"
[ -s "${IMG}" ] || die "Expected image not found: ${IMG}"
fi
# Compute checksum + size for the manifest. In a dry run the artifacts may not
# exist, so fall back to placeholders.
checksum() { [ -s "$1" ] && sha256sum "$1" | cut -d' ' -f1 || echo "DRYRUN"; }
bytesize() { [ -s "$1" ] && stat -c%s "$1" || echo 0; }
ISO_SHA=$(checksum "${ISO}")
IMG_SHA=$(checksum "${IMG}")
ISO_SIZE=$(bytesize "${ISO}")
IMG_SIZE=$(bytesize "${IMG}")
# Write .sha256 sidecars next to the artifacts (standard `sha256sum` format).
if [ "${DRY_RUN}" -eq 0 ]; then
( cd "$(dirname "${ISO}")" && sha256sum "$(basename "${ISO}")" > "${ISO}.sha256" )
( cd "$(dirname "${IMG}")" && sha256sum "$(basename "${IMG}")" > "${IMG}.sha256" )
fi
ISO_NAME=$(basename "${ISO}")
IMG_NAME=$(basename "${IMG}")
RELEASED=$(date -u +%Y-%m-%dT%H:%M:%SZ)
# Build latest.json. jq -n keeps the values properly escaped/typed.
MANIFEST=$(jq -n \
--arg version "${VERSION}" \
--arg tag "${REF}" \
--arg released "${RELEASED}" \
--arg cdn "${CDN_BASE}" \
--arg iso_name "${ISO_NAME}" --arg iso_sha "${ISO_SHA}" --argjson iso_size "${ISO_SIZE}" \
--arg img_name "${IMG_NAME}" --arg img_sha "${IMG_SHA}" --argjson img_size "${IMG_SIZE}" \
'{
version: $version,
tag: $tag,
released: $released,
downloads: [
{ key: "desktop", title: "Raspberry Pi", arch: "aarch64",
filename: $img_name, url: ($cdn + "/" + $img_name),
sha256_url: ($cdn + "/" + $img_name + ".sha256"),
sha256: $img_sha, size: $img_size },
{ key: "debian", title: "Debian", arch: "amd64",
filename: $iso_name, url: ($cdn + "/" + $iso_name),
sha256_url: ($cdn + "/" + $iso_name + ".sha256"),
sha256: $iso_sha, size: $iso_size }
]
}')
MANIFEST_FILE="${SCRIPT_DIR}/latest.json"
echo "${MANIFEST}" > "${MANIFEST_FILE}"
put() {
# put <local> <mime> <cache-control>
local src="$1" mime="$2" cache="$3"
local name; name=$(basename "${src}")
if [ "${DRY_RUN}" -eq 1 ]; then
log "DRY RUN would upload: ${name} (${mime})"
return
fi
"${S3CMD[@]}" put --acl-public --no-progress \
--mime-type="${mime}" \
--add-header="Cache-Control:${cache}" \
"${src}" "${BUCKET}/${name}"
}
log "Uploading artifacts to ${BUCKET}..."
put "${ISO}" "application/x-iso9660-image" "${IMAGE_CACHE}"
put "${ISO}.sha256" "text/plain" "${IMAGE_CACHE}"
put "${IMG}" "application/x-xz" "${IMAGE_CACHE}"
put "${IMG}.sha256" "text/plain" "${IMAGE_CACHE}"
log "Publishing manifest..."
put "${MANIFEST_FILE}" "application/json" "${MANIFEST_CACHE}"
log "Done. Published v${VERSION}:"
log " ${CDN_BASE}/${ISO_NAME}"
log " ${CDN_BASE}/${IMG_NAME}"
log " ${CDN_BASE}/latest.json"

View file

@ -0,0 +1,32 @@
# WROLPi release poller (oneshot, driven by wrolpi-release.timer).
#
# Checks GitHub for a newly-published release and builds + publishes it.
# This is OUTBOUND polling only -- GitHub never connects to this machine.
#
# Install (as root):
# cp release/wrolpi-release.service release/wrolpi-release.timer /etc/systemd/system/
# # edit paths/user below to match this host, then:
# systemctl daemon-reload
# systemctl enable --now wrolpi-release.timer
#
# The builds need root (chroot), so this runs as root. s3cmd therefore reads
# /root/.s3cfg by default -- either place the Spaces credentials there or point
# S3CFG at another location below.
[Unit]
Description=WROLPi release poller (build & publish new GitHub releases)
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
# Adjust to wherever the WROLPi repo lives on this host (production: /opt/wrolpi).
WorkingDirectory=/opt/wrolpi/release
ExecStart=/opt/wrolpi/release/release-watch.sh
# Persistent state (failure markers, build lock): /var/lib/wrolpi-release
StateDirectory=wrolpi-release
Environment=STATE_DIR=/var/lib/wrolpi-release
# Uncomment if the s3cmd config is not at /root/.s3cfg:
#Environment=S3CFG=/home/wrolpi/.s3cfg
# A full release builds two images and can run for hours.
TimeoutStartSec=0

View file

@ -0,0 +1,18 @@
# Fire the WROLPi release poller once an hour.
#
# Hourly is well within GitHub's unauthenticated API limit and means freshly
# built images land within roughly six hours of a release (the two image
# builds run sequentially). Persistent=true catches a release published while
# this host was powered off.
[Unit]
Description=Hourly WROLPi release poll
[Timer]
OnCalendar=hourly
# Spread the load off the exact top of the hour.
RandomizedDelaySec=5min
Persistent=true
[Install]
WantedBy=timers.target