Run the CircleCI jobs on GitHub Actions (#789)

* Run the CircleCI jobs on GitHub Actions

Backend tests (pytest, API contract, migrations + API start) and Cypress
(component, e2e) now run as GitHub Actions workflows.  CircleCI stays in
place until these have run green alongside it.

- pip and npm caches via setup-python/setup-node, plus the Cypress binary;
  node_modules is no longer cached, to stay inside the 10 GB cache cap.
- Superseded pull request runs are cancelled to stay under the concurrent
  job limit.
- react-app-start is dropped: the e2e job waits for `npm start`, and
  jest-tests.yml already runs `npm run build`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Skip the CI-unsupported tests on GitHub Actions too

skip_circleci only read CIRCLECI, so on GitHub Actions test_archive_upload
ran without the readability tooling it needs, and failed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Run the backend tests on every vCPU

-nauto counts physical cores when psutil is installed, so the four-vCPU
runner got two workers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Split the backend tests into four jobs by directory

The groups are balanced by test time, so each runs about a quarter of the
suite.  `rest` ignores the other groups' paths rather than listing its own,
so a new directory is tested before it is assigned to a group.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

.github/workflows/backend.yml

@@ -0,0 +1,145 @@
+name: Backend Tests
+
+# Same triggers as jest-tests.yml: `pull_request` is unfiltered so pull requests into feature
+# branches are checked too, not only those into master.
+on:
+  push:
+    branches: [master, feature/*]
+  pull_request:
+
+# A new push to a pull request makes the run for the previous push worthless; cancel it so it
+# does not hold runners (the free plan allows 20 concurrent jobs).  Runs on master always finish.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  # The suite is split by directory so it finishes in a few minutes, and so a check's name says
+  # which area broke.  The groups are balanced by test time; `rest` runs everything the others do
+  # not, so a new directory is tested before anyone assigns it.  A path added to any other group
+  # must also be added to `rest` as an --ignore, or its tests run twice.
+  api-tests:
+    name: api-tests (${{ matrix.name }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: core
+            paths: wrolpi/test
+          - name: media
+            paths: modules/videos modules/zim modules/docs modules/map modules/inventory modules/flasher
+          - name: files
+            paths: wrolpi/files wrolpi/collections
+          - name: rest
+            paths: >-
+              --ignore=wrolpi/test
+              --ignore=modules/videos --ignore=modules/zim --ignore=modules/docs --ignore=modules/map
+              --ignore=modules/inventory --ignore=modules/flasher
+              --ignore=wrolpi/files --ignore=wrolpi/collections
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install system packages
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg catdoc
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          # Trixie ships 3.13.
+          python-version: '3.13'
+          cache: 'pip'
+          cache-dependency-path: requirements.txt
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      # `logical`, not `auto`: with psutil installed, `auto` counts physical cores, which is two on
+      # the four-vCPU runner.  No `-v`: the full test output inflates wall time and logs.
+      - name: Run pytest
+        env:
+          PATHS: ${{ matrix.paths }}
+        run: pytest -n logical $PATHS --junitxml=test-results.xml
+
+      - name: Upload test results
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: api-test-results-${{ matrix.name }}
+          path: test-results.xml
+          retention-days: 30
+
+  # Do the API and the frontend's test fixtures still agree?
+  #
+  # `api-tests` runs these too, as part of the whole suite.  This job exists for the signal:
+  # fixture drift is silent by nature -- the frontend suite passes against a body the API stopped
+  # sending -- so it is worth a check whose name says what broke, failing in under a minute rather
+  # than somewhere inside seventeen hundred results.  It needs no ffmpeg or catdoc.
+  api-contract-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+          cache: 'pip'
+          cache-dependency-path: requirements.txt
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Fixtures match the OpenAPI spec, and the spec matches the handlers
+        run: pytest wrolpi/test/test_api_fixtures.py -v
+
+  # The migrations apply to an empty database, the models match them, and the API starts.
+  api-migrations-and-start:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+          cache: 'pip'
+          cache-dependency-path: requirements.txt
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Run Alembic migrations
+        env:
+          MEDIA_DIRECTORY: ${{ runner.temp }}/wrolpi-media
+        run: |
+          mkdir -p "$MEDIA_DIRECTORY/config"
+          python ./main.py db upgrade
+
+      - name: Check for missing migrations
+        env:
+          MEDIA_DIRECTORY: ${{ runner.temp }}/wrolpi-media
+        run: python ./main.py db check
+
+      - name: Start Sanic API
+        run: |
+          sudo mkdir -p /media/wrolpi
+          nohup sanic main:api_app -H 0.0.0.0 -p 8081 > sanic.log 2>&1 &
+          for i in $(seq 1 60); do
+            if curl -sf http://localhost:8081/api/echo; then
+              echo
+              echo "Sanic API is up"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Sanic API failed to start"
+          cat sanic.log
+          exit 1
+
+      - name: Stop Sanic API
+        if: ${{ always() }}
+        run: pkill -f sanic || true

.github/workflows/cypress.yml

@@ -0,0 +1,122 @@
+name: Cypress Tests
+
+on:
+  push:
+    branches: [master, feature/*]
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  # Component specs live under `src/`, so they need `cypress run --component`; the e2e job's
+  # `cypress run` will not match them.  Cypress starts its own dev server, so no `npm start`.
+  cypress-component-tests:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: app
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20.19'
+          cache: 'npm'
+          cache-dependency-path: app/package-lock.json
+
+      # The Cypress binary lives outside node_modules, and setup-node only caches ~/.npm.  Key it
+      # on the lockfile so a Cypress bump fetches the matching binary.
+      - name: Cache Cypress binary
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/Cypress
+          key: cypress-${{ runner.os }}-${{ hashFiles('app/package-lock.json') }}
+
+      - name: Install dependencies
+        run: npm install
+
+      # A no-op when the cached binary matches.
+      - name: Install Cypress binary
+        run: npx cypress install
+
+      - name: Run Cypress component tests
+        run: npx cypress run --component
+
+      - name: Upload screenshots
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: cypress-component-screenshots
+          path: app/cypress/screenshots
+          retention-days: 14
+
+  # Waiting for `npm start` to serve the app also covers the dev server starting, which
+  # CircleCI checked in a separate job.
+  cypress-e2e-tests:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: app
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20.19'
+          cache: 'npm'
+          cache-dependency-path: app/package-lock.json
+
+      - name: Cache Cypress binary
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/Cypress
+          key: cypress-${{ runner.os }}-${{ hashFiles('app/package-lock.json') }}
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Install Cypress binary
+        run: npx cypress install
+
+      # GitHub Actions sets `CI=true`, which makes cypress.config.js use http://localhost:3000 as
+      # the base URL.
+      - name: Start React app
+        env:
+          DISABLE_ESLINT_PLUGIN: 'true'
+          BROWSER: none
+        run: |
+          nohup npm start > react.log 2>&1 &
+          for i in $(seq 1 90); do
+            if curl -sf http://localhost:3000/manifest.json > /dev/null; then
+              echo "React app is ready"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "React app failed to start"
+          cat react.log
+          exit 1
+
+      - name: Run Cypress e2e tests
+        run: npx cypress run
+
+      - name: Upload videos and screenshots
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: cypress-e2e-media
+          path: |
+            app/cypress/videos
+            app/cypress/screenshots
+          retention-days: 14
+          if-no-files-found: ignore

wrolpi/test/common.py

@@ -15,7 +15,7 @@ from wrolpi.vars import CIRCLECI, IS_MACOS
 
 TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False)
 
-skip_circleci = pytest.mark.skipif(CIRCLECI, reason='This test is not supported in Circle CI')
+skip_circleci = pytest.mark.skipif(CIRCLECI, reason='This test is not supported in CI')
 skip_macos = pytest.mark.skipif(IS_MACOS, reason='This test is not supported on MacOS')
 only_macos = pytest.mark.skipif(not IS_MACOS, reason='This test is only supported on MacOS')
 

wrolpi/vars.py

@@ -57,8 +57,8 @@ PYTEST = 'pytest' in sys.modules
 # Used to test if the Internet is up.
 INTERNET_SERVER = os.environ.get('INTERNET_SERVER', 'one.one.one.one')
 
-# running on circlci
-CIRCLECI = truthy_arg(os.environ.get('CIRCLECI', ''))
+# Running on a hosted CI runner (CircleCI or GitHub Actions), which lacks things some tests need.
+CIRCLECI = truthy_arg(os.environ.get('CIRCLECI', '')) or truthy_arg(os.environ.get('GITHUB_ACTIONS', ''))
 
 # Get the media directory from the environment.
 DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi')