From 1e112b678aa244bb70e979f505fec8258018b586 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:08:16 -0600 Subject: [PATCH] Run the CircleCI jobs on GitHub Actions (#789) * Run the CircleCI jobs on GitHub Actions Backend tests (pytest, API contract, migrations + API start) and Cypress (component, e2e) now run as GitHub Actions workflows. CircleCI stays in place until these have run green alongside it. - pip and npm caches via setup-python/setup-node, plus the Cypress binary; node_modules is no longer cached, to stay inside the 10 GB cache cap. - Superseded pull request runs are cancelled to stay under the concurrent job limit. - react-app-start is dropped: the e2e job waits for `npm start`, and jest-tests.yml already runs `npm run build`. Co-Authored-By: Claude Opus 5.5 * Skip the CI-unsupported tests on GitHub Actions too skip_circleci only read CIRCLECI, so on GitHub Actions test_archive_upload ran without the readability tooling it needs, and failed. Co-Authored-By: Claude Opus 5.5 * Run the backend tests on every vCPU -nauto counts physical cores when psutil is installed, so the four-vCPU runner got two workers. Co-Authored-By: Claude Opus 5.5 * Split the backend tests into four jobs by directory The groups are balanced by test time, so each runs about a quarter of the suite. `rest` ignores the other groups' paths rather than listing its own, so a new directory is tested before it is assigned to a group. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/backend.yml | 145 ++++++++++++++++++++++++++++++++++ .github/workflows/cypress.yml | 122 ++++++++++++++++++++++++++++ wrolpi/test/common.py | 2 +- wrolpi/vars.py | 4 +- 4 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/backend.yml create mode 100644 .github/workflows/cypress.yml diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml new file mode 100644 index 00000000..05cec023 --- /dev/null +++ b/.github/workflows/backend.yml @@ -0,0 +1,145 @@ +name: Backend Tests + +# Same triggers as jest-tests.yml: `pull_request` is unfiltered so pull requests into feature +# branches are checked too, not only those into master. +on: + push: + branches: [master, feature/*] + pull_request: + +# A new push to a pull request makes the run for the previous push worthless; cancel it so it +# does not hold runners (the free plan allows 20 concurrent jobs). Runs on master always finish. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # The suite is split by directory so it finishes in a few minutes, and so a check's name says + # which area broke. The groups are balanced by test time; `rest` runs everything the others do + # not, so a new directory is tested before anyone assigns it. A path added to any other group + # must also be added to `rest` as an --ignore, or its tests run twice. + api-tests: + name: api-tests (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: core + paths: wrolpi/test + - name: media + paths: modules/videos modules/zim modules/docs modules/map modules/inventory modules/flasher + - name: files + paths: wrolpi/files wrolpi/collections + - name: rest + paths: >- + --ignore=wrolpi/test + --ignore=modules/videos --ignore=modules/zim --ignore=modules/docs --ignore=modules/map + --ignore=modules/inventory --ignore=modules/flasher + --ignore=wrolpi/files --ignore=wrolpi/collections + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install system packages + run: sudo apt-get update && sudo apt-get install -y ffmpeg catdoc + + - name: Setup Python + uses: actions/setup-python@v5 + with: + # Trixie ships 3.13. + python-version: '3.13' + cache: 'pip' + cache-dependency-path: requirements.txt + + - name: Install dependencies + run: pip install -r requirements.txt + + # `logical`, not `auto`: with psutil installed, `auto` counts physical cores, which is two on + # the four-vCPU runner. No `-v`: the full test output inflates wall time and logs. + - name: Run pytest + env: + PATHS: ${{ matrix.paths }} + run: pytest -n logical $PATHS --junitxml=test-results.xml + + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: api-test-results-${{ matrix.name }} + path: test-results.xml + retention-days: 30 + + # Do the API and the frontend's test fixtures still agree? + # + # `api-tests` runs these too, as part of the whole suite. This job exists for the signal: + # fixture drift is silent by nature -- the frontend suite passes against a body the API stopped + # sending -- so it is worth a check whose name says what broke, failing in under a minute rather + # than somewhere inside seventeen hundred results. It needs no ffmpeg or catdoc. + api-contract-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: 'pip' + cache-dependency-path: requirements.txt + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Fixtures match the OpenAPI spec, and the spec matches the handlers + run: pytest wrolpi/test/test_api_fixtures.py -v + + # The migrations apply to an empty database, the models match them, and the API starts. + api-migrations-and-start: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: 'pip' + cache-dependency-path: requirements.txt + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Run Alembic migrations + env: + MEDIA_DIRECTORY: ${{ runner.temp }}/wrolpi-media + run: | + mkdir -p "$MEDIA_DIRECTORY/config" + python ./main.py db upgrade + + - name: Check for missing migrations + env: + MEDIA_DIRECTORY: ${{ runner.temp }}/wrolpi-media + run: python ./main.py db check + + - name: Start Sanic API + run: | + sudo mkdir -p /media/wrolpi + nohup sanic main:api_app -H 0.0.0.0 -p 8081 > sanic.log 2>&1 & + for i in $(seq 1 60); do + if curl -sf http://localhost:8081/api/echo; then + echo + echo "Sanic API is up" + exit 0 + fi + sleep 2 + done + echo "Sanic API failed to start" + cat sanic.log + exit 1 + + - name: Stop Sanic API + if: ${{ always() }} + run: pkill -f sanic || true diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml new file mode 100644 index 00000000..6fddd0e0 --- /dev/null +++ b/.github/workflows/cypress.yml @@ -0,0 +1,122 @@ +name: Cypress Tests + +on: + push: + branches: [master, feature/*] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # Component specs live under `src/`, so they need `cypress run --component`; the e2e job's + # `cypress run` will not match them. Cypress starts its own dev server, so no `npm start`. + cypress-component-tests: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: app + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20.19' + cache: 'npm' + cache-dependency-path: app/package-lock.json + + # The Cypress binary lives outside node_modules, and setup-node only caches ~/.npm. Key it + # on the lockfile so a Cypress bump fetches the matching binary. + - name: Cache Cypress binary + uses: actions/cache@v4 + with: + path: ~/.cache/Cypress + key: cypress-${{ runner.os }}-${{ hashFiles('app/package-lock.json') }} + + - name: Install dependencies + run: npm install + + # A no-op when the cached binary matches. + - name: Install Cypress binary + run: npx cypress install + + - name: Run Cypress component tests + run: npx cypress run --component + + - name: Upload screenshots + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: cypress-component-screenshots + path: app/cypress/screenshots + retention-days: 14 + + # Waiting for `npm start` to serve the app also covers the dev server starting, which + # CircleCI checked in a separate job. + cypress-e2e-tests: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: app + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20.19' + cache: 'npm' + cache-dependency-path: app/package-lock.json + + - name: Cache Cypress binary + uses: actions/cache@v4 + with: + path: ~/.cache/Cypress + key: cypress-${{ runner.os }}-${{ hashFiles('app/package-lock.json') }} + + - name: Install dependencies + run: npm install + + - name: Install Cypress binary + run: npx cypress install + + # GitHub Actions sets `CI=true`, which makes cypress.config.js use http://localhost:3000 as + # the base URL. + - name: Start React app + env: + DISABLE_ESLINT_PLUGIN: 'true' + BROWSER: none + run: | + nohup npm start > react.log 2>&1 & + for i in $(seq 1 90); do + if curl -sf http://localhost:3000/manifest.json > /dev/null; then + echo "React app is ready" + exit 0 + fi + sleep 2 + done + echo "React app failed to start" + cat react.log + exit 1 + + - name: Run Cypress e2e tests + run: npx cypress run + + - name: Upload videos and screenshots + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: cypress-e2e-media + path: | + app/cypress/videos + app/cypress/screenshots + retention-days: 14 + if-no-files-found: ignore diff --git a/wrolpi/test/common.py b/wrolpi/test/common.py index 128163ce..9dd35765 100644 --- a/wrolpi/test/common.py +++ b/wrolpi/test/common.py @@ -15,7 +15,7 @@ from wrolpi.vars import CIRCLECI, IS_MACOS TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False) -skip_circleci = pytest.mark.skipif(CIRCLECI, reason='This test is not supported in Circle CI') +skip_circleci = pytest.mark.skipif(CIRCLECI, reason='This test is not supported in CI') skip_macos = pytest.mark.skipif(IS_MACOS, reason='This test is not supported on MacOS') only_macos = pytest.mark.skipif(not IS_MACOS, reason='This test is only supported on MacOS') diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 614326fc..930f43c3 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -57,8 +57,8 @@ PYTEST = 'pytest' in sys.modules # Used to test if the Internet is up. INTERNET_SERVER = os.environ.get('INTERNET_SERVER', 'one.one.one.one') -# running on circlci -CIRCLECI = truthy_arg(os.environ.get('CIRCLECI', '')) +# Running on a hosted CI runner (CircleCI or GitHub Actions), which lacks things some tests need. +CIRCLECI = truthy_arg(os.environ.get('CIRCLECI', '')) or truthy_arg(os.environ.get('GITHUB_ACTIONS', '')) # Get the media directory from the environment. DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi')