diff --git a/.github/actions/green/action.yml b/.github/actions/green/action.yml new file mode 100644 index 0000000..d1e98f7 --- /dev/null +++ b/.github/actions/green/action.yml @@ -0,0 +1,28 @@ +name: Green +description: >- + Leave the markers of a unit that passed, so that a later run with the same + key skips it (ci/plan.py). +inputs: + unit: + description: The unit as ci/plan.py gives it, as JSON + required: true +runs: + using: composite + steps: + - id: marker + shell: bash + env: + UNIT: ${{ inputs.unit }} + run: | + mkdir -p "$RUNNER_TEMP/green" + echo "$GITHUB_SHA" > "$RUNNER_TEMP/green/commit" + python3 -c 'import json, os; unit = json.loads(os.environ["UNIT"]); print("key=" + unit["key"]); print("also=" + (unit["also"][0] if unit["also"] else ""))' >> "$GITHUB_OUTPUT" + - uses: actions/cache/save@v4 + with: + path: ${{ runner.temp }}/green + key: ${{ steps.marker.outputs.key }} + - if: steps.marker.outputs.also != '' + uses: actions/cache/save@v4 + with: + path: ${{ runner.temp }}/green + key: ${{ steps.marker.outputs.also }} diff --git a/.github/actions/rust-save/action.yml b/.github/actions/rust-save/action.yml new file mode 100644 index 0000000..eb06fc1 --- /dev/null +++ b/.github/actions/rust-save/action.yml @@ -0,0 +1,25 @@ +name: Save the Rust build +description: >- + Keep the target directory for the next runs, with the tree it was built + from. Only main saves: a pull request reads main's and leaves its own. +inputs: + cache: + description: The name of the cache, as given to the rust action + required: true +runs: + using: composite + steps: + - name: Record the tree + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + shell: bash + run: python3 ci/mtimes.py save target + - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/cache/save@v4 + with: + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db + target + provision/ansible/roles/devbox/files/bin + key: cargo-${{ inputs.cache }}-${{ hashFiles('Cargo.lock', 'ci/dev.Dockerfile') }}-${{ github.sha }} diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml new file mode 100644 index 0000000..abe8801 --- /dev/null +++ b/.github/actions/rust/action.yml @@ -0,0 +1,34 @@ +name: Rust +description: >- + The toolchain of ci/dev.Dockerfile, and cargo's downloads and the target + directory from the cache of main, with the times of the files that did not + change set back so cargo builds only what did (ci/mtimes.py). +inputs: + cache: + description: The name of the cache, one for each kind of build + required: true +runs: + using: composite + steps: + - name: Install the toolchain + shell: bash + run: | + version=$(sed -n 's/^FROM rust:\([0-9.]*\)-.*/\1/p' ci/dev.Dockerfile) + rustup toolchain install "$version" --profile minimal \ + --component clippy,rustfmt --target x86_64-unknown-linux-musl --no-self-update + echo "RUSTUP_TOOLCHAIN=$version" >> "$GITHUB_ENV" + - name: Restore the build + id: restore + uses: actions/cache/restore@v4 + with: + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db + target + provision/ansible/roles/devbox/files/bin + key: cargo-${{ inputs.cache }}-${{ hashFiles('Cargo.lock', 'ci/dev.Dockerfile') }}-${{ github.sha }} + restore-keys: cargo-${{ inputs.cache }}-${{ hashFiles('Cargo.lock', 'ci/dev.Dockerfile') }}- + - name: Keep the times of what did not change + shell: bash + run: python3 ci/mtimes.py restore target diff --git a/.github/workflows/boot.yml b/.github/workflows/boot.yml index 8d188df..838e9e1 100644 --- a/.github/workflows/boot.yml +++ b/.github/workflows/boot.yml @@ -11,8 +11,11 @@ on: paths: - 'provision/**' - 'plugins/**' - - 'ci/**' - - 'tests/e2e/boot_*' + - 'ci/boot*' + - 'ci/run.sh' + - 'ci/dev.Dockerfile*' + - 'tests/e2e/boot*' + - 'tests/e2e/vm.py' - 'container/**' - '.github/workflows/boot.yml' push: @@ -20,8 +23,11 @@ on: paths: - 'provision/**' - 'plugins/**' - - 'ci/**' - - 'tests/e2e/boot_*' + - 'ci/boot*' + - 'ci/run.sh' + - 'ci/dev.Dockerfile*' + - 'tests/e2e/boot*' + - 'tests/e2e/vm.py' - 'container/**' - '.github/workflows/boot.yml' workflow_dispatch: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f13437d..49f001b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,126 +11,219 @@ permissions: concurrency: group: ci-${{ github.ref }} cancel-in-progress: true -# The checks and the end-to-end tests run ci/run.sh, as on a developer's -# machine. Image layers are kept in GitHub's cache; cargo's downloads and the -# target directory are a directory the cache action keeps. +# The checks run on the runner as they are; the end-to-end tests run +# ci/run.sh in containers, as on a developer's machine. ci/plan.py says +# which units a run needs: one that passed with the same files is skipped. +# Started by hand, everything runs. env: BERTH_CI_NO_BUILD: '1' + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: '0' jobs: - checks: + plan: runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 5 + permissions: + contents: read + # The list of the cache's keys, where the markers are. + actions: read + outputs: + rust-tests: ${{ steps.plan.outputs.rust-tests }} + clippy: ${{ steps.plan.outputs.clippy }} + python-tests: ${{ steps.plan.outputs.python-tests }} + shards: ${{ steps.plan.outputs.shards }} steps: - uses: actions/checkout@v7 - - run: echo "BERTH_CI_CACHE=$RUNNER_TEMP/berth-ci-cache" >> "$GITHUB_ENV" - - uses: docker/setup-buildx-action@v4 - - name: Build the dev image - uses: docker/build-push-action@v6 + - id: plan + env: + GH_TOKEN: ${{ github.token }} + BERTH_CI_PR: ${{ github.event.pull_request.number }} + ALL: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }} + run: python3 ci/plan.py ${ALL:+--all} + + # Quick enough to run always. + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + VALE_VERSION: 3.25.1 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 with: - context: . - file: ci/dev.Dockerfile - tags: berth-ci-dev - load: true - cache-from: type=gha,scope=berth-ci-dev - cache-to: type=gha,scope=berth-ci-dev,mode=max - - uses: actions/cache@v4 - with: - path: ${{ env.BERTH_CI_CACHE }} - key: checks-${{ hashFiles('Cargo.lock', 'ci/dev.Dockerfile') }}-${{ github.sha }} - restore-keys: checks-${{ hashFiles('Cargo.lock', 'ci/dev.Dockerfile') }}- - - name: Checks - run: ci/run.sh check - - name: Keep the cache small - if: always() - run: rm -rf "$BERTH_CI_CACHE/target/debug/incremental" "$BERTH_CI_CACHE/target/tmp" + python-version: '3.13' + cache: pip + cache-dependency-path: provision/requirements.txt + - name: Install Ansible and Vale + run: | + pip install --quiet -r provision/requirements.txt + curl -fsSL "https://github.com/errata-ai/vale/releases/download/v${VALE_VERSION}/vale_${VALE_VERSION}_Linux_64-bit.tar.gz" \ + | tar -xz -C "$RUNNER_TEMP" vale + echo "$RUNNER_TEMP" >> "$GITHUB_PATH" + - name: Install the toolchain + run: | + version=$(sed -n 's/^FROM rust:\([0-9.]*\)-.*/\1/p' ci/dev.Dockerfile) + rustup toolchain install "$version" --profile minimal \ + --component clippy,rustfmt --target x86_64-unknown-linux-musl --no-self-update + echo "RUSTUP_TOOLCHAIN=$version" >> "$GITHUB_ENV" + - run: ci/checks.sh fmt shell-syntax ansible-syntax vale - # berth is built once, here, and each shard of the end-to-end tests, the - # tests of one test host (tests/e2e/run.py), runs it in a job of its own. - # Only this job saves the cargo cache and the test host's image layers, so - # the shards do not race to save the same keys. - e2e-build: + rust-tests: + needs: plan + if: needs.plan.outputs.rust-tests != '' runs-on: ubuntu-latest - timeout-minutes: 45 - outputs: - shards: ${{ steps.shards.outputs.shards }} + timeout-minutes: 30 + # The store's tests run on PostgreSQL too. The password is made up: the + # database is reachable from this job alone. + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_USER: berth + POSTGRES_PASSWORD: berth-ci + POSTGRES_DB: berth + ports: ['5432:5432'] + options: --health-cmd "pg_isready -U berth" --health-interval 1s --health-retries 60 + env: + BERTH_TEST_POSTGRES: postgres://berth:berth-ci@localhost:5432/berth steps: - uses: actions/checkout@v7 - - run: echo "BERTH_CI_CACHE=$RUNNER_TEMP/berth-ci-cache" >> "$GITHUB_ENV" - - id: shards - run: echo "shards=$(python3 tests/e2e/run.py --shards)" >> "$GITHUB_OUTPUT" - - uses: docker/setup-buildx-action@v4 - - name: Build the dev image - uses: docker/build-push-action@v6 + - uses: ./.github/actions/rust with: - context: . - file: ci/dev.Dockerfile - tags: berth-ci-dev - load: true - cache-from: type=gha,scope=berth-ci-dev - - name: Build the test host's image - uses: docker/build-push-action@v6 + cache: test + # The SSH tests start an sshd of their own. + - name: Install sshd + run: command -v sshd >/dev/null || [ -x /usr/sbin/sshd ] || { sudo apt-get update -q && sudo apt-get install -y --no-install-recommends openssh-server; } + - run: ci/checks.sh rust-tests + - uses: ./.github/actions/rust-save with: - context: . - file: ci/host.Dockerfile - tags: berth-ci-host - cache-from: type=gha,scope=berth-ci-host - cache-to: type=gha,scope=berth-ci-host,mode=max - - uses: actions/cache@v4 + cache: test + - uses: ./.github/actions/green + with: + unit: ${{ needs.plan.outputs.rust-tests }} + + clippy: + needs: plan + if: needs.plan.outputs.clippy != '' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/rust + with: + cache: clippy + - run: ci/checks.sh clippy + - uses: ./.github/actions/rust-save + with: + cache: clippy + - uses: ./.github/actions/green + with: + unit: ${{ needs.plan.outputs.clippy }} + + python-tests: + needs: plan + if: needs.plan.outputs.python-tests != '' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 with: - path: ${{ env.BERTH_CI_CACHE }} - key: e2e-${{ hashFiles('Cargo.lock', 'ci/dev.Dockerfile') }}-${{ github.sha }} - restore-keys: e2e-${{ hashFiles('Cargo.lock', 'ci/dev.Dockerfile') }}- - - name: Build berth - run: ci/run.sh run cargo build --locked -p berth + python-version: '3.13' + cache: pip + cache-dependency-path: provision/requirements.txt + - run: pip install --quiet -r provision/requirements.txt + # The checks of the role's sshd configuration ask sshd itself. + - name: Install sshd + run: command -v sshd >/dev/null || [ -x /usr/sbin/sshd ] || { sudo apt-get update -q && sudo apt-get install -y --no-install-recommends openssh-server; } + - run: ci/checks.sh python-tests + - uses: ./.github/actions/green + with: + unit: ${{ needs.plan.outputs.python-tests }} + + # berth is built once, here, for the shards of the end-to-end tests. + build: + needs: plan + if: needs.plan.outputs.shards != '[]' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/rust + with: + cache: build + - run: cargo build --locked -p berth # berth, and the berthd and box it puts into the role's files. A tar # keeps their modes, which an artifact does not. - name: Pack what the shards test - run: | - tar -cf "$RUNNER_TEMP/e2e-berth.tar" -C "$BERTH_CI_CACHE" target/debug/berth \ - -C "$GITHUB_WORKSPACE" provision/ansible/roles/devbox/files/bin + run: tar -cf "$RUNNER_TEMP/e2e-berth.tar" target/debug/berth provision/ansible/roles/devbox/files/bin - uses: actions/upload-artifact@v4 with: name: e2e-berth path: ${{ runner.temp }}/e2e-berth.tar retention-days: 1 if-no-files-found: error - - name: Keep the cache small - if: always() - run: rm -rf "$BERTH_CI_CACHE/target/debug/incremental" "$BERTH_CI_CACHE/target/tmp" + - uses: ./.github/actions/rust-save + with: + cache: build - e2e-shard: - needs: e2e-build + # The test host's and the runner's images, built beside berth. Only this + # job saves their layers, so the shards do not race to save the same keys. + images: + needs: plan + if: needs.plan.outputs.shards != '[]' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + - uses: docker/setup-buildx-action@v4 + - uses: docker/build-push-action@v6 + with: + context: . + file: ci/host.Dockerfile + cache-from: type=gha,scope=berth-ci-host + cache-to: type=gha,scope=berth-ci-host,mode=max + outputs: type=cacheonly + - uses: docker/build-push-action@v6 + with: + context: . + file: ci/runner.Dockerfile + cache-from: type=gha,scope=berth-ci-runner + cache-to: type=gha,scope=berth-ci-runner,mode=max + outputs: type=cacheonly + + # Each shard of the end-to-end tests (SHARDS of tests/e2e/run.py) on a + # fresh test host, in a job of its own. + e2e: + needs: [plan, build, images] + if: needs.plan.outputs.shards != '[]' runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 30 strategy: fail-fast: false matrix: - shard: ${{ fromJSON(needs.e2e-build.outputs.shards) }} + shard: ${{ fromJSON(needs.plan.outputs.shards) }} + name: e2e (${{ matrix.shard.shard }}) env: BERTH_CI_PREBUILT: '1' - # The Docker engine of the tests has no way out; ci/run.sh loads this - # image into it when the shard holds the Docker plugin's tests. - BERTH_CI_DIND_IMAGES: busybox:1.37 steps: - uses: actions/checkout@v7 - run: echo "BERTH_CI_CACHE=$RUNNER_TEMP/berth-ci-cache" >> "$GITHUB_ENV" - uses: docker/setup-buildx-action@v4 - - name: Build the dev image - uses: docker/build-push-action@v6 - with: - context: . - file: ci/dev.Dockerfile - tags: berth-ci-dev - load: true - cache-from: type=gha,scope=berth-ci-dev - - name: Build the test host's image - uses: docker/build-push-action@v6 + - uses: docker/build-push-action@v6 with: context: . file: ci/host.Dockerfile tags: berth-ci-host load: true cache-from: type=gha,scope=berth-ci-host + - uses: docker/build-push-action@v6 + with: + context: . + file: ci/runner.Dockerfile + tags: berth-ci-runner + load: true + cache-from: type=gha,scope=berth-ci-runner - uses: actions/download-artifact@v4 with: name: e2e-berth @@ -140,22 +233,36 @@ jobs: mkdir -p "$BERTH_CI_CACHE" tar -xf "$RUNNER_TEMP/e2e-berth.tar" -C "$BERTH_CI_CACHE" target tar -xf "$RUNNER_TEMP/e2e-berth.tar" provision + # The tests' Docker engine has no way out; ci/run.sh loads this image + # into it for a shard with the Docker plugin's tests. - name: Pull the images the tests' Docker engine is given - run: docker pull --quiet "$BERTH_CI_DIND_IMAGES" + env: + SHARD: ${{ matrix.shard.shard }} + run: | + if python3 tests/e2e/run.py --tests-of "$SHARD" | grep -q '^e2e_provisioners'; then + docker pull --quiet busybox:1.37 + echo "BERTH_CI_DIND_IMAGES=busybox:1.37" >> "$GITHUB_ENV" + fi - name: End-to-end tests - run: ci/run.sh e2e --shard "${{ matrix.shard }}" + env: + SHARD: ${{ matrix.shard.shard }} + run: ci/run.sh e2e --shard "$SHARD" - name: Upload the test host's logs if: failure() uses: actions/upload-artifact@v4 with: - name: e2e-logs-${{ matrix.shard }} + name: e2e-logs-${{ matrix.shard.shard }} path: build/ci/logs if-no-files-found: ignore + - uses: ./.github/actions/green + with: + unit: ${{ toJSON(matrix.shard) }} - # The check that branch protection requires. A job that was skipped would - # count as passed, so this one runs always and looks at every result. - e2e: - needs: [e2e-build, e2e-shard] + # The check that branch protection can require. A unit that was skipped + # passed with the same files before; one that failed or was cancelled fails + # this. + ci: + needs: [plan, lint, rust-tests, clippy, python-tests, build, images, e2e] if: always() runs-on: ubuntu-latest timeout-minutes: 5 @@ -165,19 +272,5 @@ jobs: run: | echo "Results: $RESULTS" for result in $RESULTS; do - [[ $result == success ]] || exit 1 + [[ $result == success || $result == skipped ]] || exit 1 done - - prose: - runs-on: ubuntu-latest - timeout-minutes: 10 - env: - VALE_VERSION: 3.25.1 - steps: - - uses: actions/checkout@v7 - - name: Install Vale - run: | - curl -fsSL "https://github.com/errata-ai/vale/releases/download/v${VALE_VERSION}/vale_${VALE_VERSION}_Linux_64-bit.tar.gz" \ - | tar -xz -C "$RUNNER_TEMP" vale - echo "$RUNNER_TEMP" >> "$GITHUB_PATH" - - run: vale --minAlertLevel=warning README.md docs/content/docs plugins/*/guide.md diff --git a/AGENTS.md b/AGENTS.md index d558649..2800f35 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ With only Docker or Podman, `ci/run.sh` runs the same checks in an image that ha ```bash ci/run.sh check [CHECK…] # fmt, shell-syntax, ansible-syntax, vale, python-tests, rust-tests, clippy ci/run.sh e2e [TEST…] # the end-to-end tests, such as e2e_serve.Roles -ci/run.sh e2e --shard HOST # the tests of one test host, as a CI job runs them +ci/run.sh e2e --shard NAME # the tests of one shard on a fresh host, as a CI job runs them ci/run.sh shell # a shell in the dev container ci/run.sh boot cloud|iso # real installs in QEMU, with KVM; slow ``` @@ -83,7 +83,20 @@ That engine starts with Docker when the run is complete, names `e2e_provisioners `tests/e2e/run.py` says which test cases run on which host and in what order. Test cases that look at the same state of a host follow each other, so the state is applied once (see the shared states in `tests/e2e/harness.py`). A run ends with the time of each test and test case and the number of playbook runs. The test hosts' logs from a failed run are in `build/ci/logs/`. -CI builds berth once and runs the tests of each test host as a shard, in a job of its own on a machine of its own. Each shard is its host's list in `HOSTS` of `tests/e2e/run.py`, so no state of a host is applied twice. A test case in none of the lists stops the run before it starts. +CI runs the checks on the runner itself, each in a job of its own, and the end-to-end tests as shards. A shard is a part of a host's list that keeps the test cases of each shared state together (`SHARDS` in `tests/e2e/run.py`). Each runs in a job of its own on a fresh host, where its first test case makes the state it needs. A test case in no list, or in no shard, stops the run before it starts. + +`ci/plan.py` decides what a run does. Each unit has a key, a hash of the files it depends on and of the week. A unit that passed leaves a marker under its key in GitHub's cache, and a later run with the same files skips it. The lint job always runs. + +| Unit | Runs for a change to | +| --- | --- | +| `rust-tests`, `clippy` | anything but the docs site, the recipes and the other workflows | +| `python-tests` | anything but the Rust code and the docs site | +| The shards `studio` and `dock` | anything berth is built from: they apply only once | +| The other shards | the role, the plugins, the end-to-end tests, or the Rust code in `SEAM` of `ci/plan.py` (on main: anything berth is built from) | + +`SEAM` is the Rust code that decides what the role is given or finds on a host. The test `ansible_contract` in `crates/berth-core/src/apply.rs` compares the variables of made-up profiles with the files in `crates/berth-core/tests/fixtures/ansible/`, which are in `SEAM` too. A change elsewhere that alters them makes the test fail, and writing them again runs the shards. Code that moves into a new file that feeds the role belongs in `SEAM`. + +The Rust jobs restore the target directory from main's cache. `ci/mtimes.py` sets the files that did not change since that build to a time long past, so cargo builds only what changed. Only main saves the caches. Started by hand, CI runs everything. `ci/run.sh boot` boots berth's cloud seed and its USB installer in QEMU, lets the first boot set the host up, and then runs `tests/e2e/boot_*.py` and part of the container suite against it. Each takes 10 to 15 minutes on a machine with KVM, so run it when a change touches provisioning, the role or first boot. CI boots only the cloud image for such a change. The installer is booted once a week and when the Boot workflow is started by hand. CI runs both for changes to provisioning (the Boot workflow, on a hosted runner with KVM). Logs, timings and disk use are in `build/boot/`. diff --git a/ci/checks.sh b/ci/checks.sh index 1217a4a..1e1f3b0 100755 --- a/ci/checks.sh +++ b/ci/checks.sh @@ -1,15 +1,43 @@ #!/usr/bin/env bash -# The checks of CI, run inside the dev container by ci/run.sh. Given -# names, only those run: ci/run.sh check clippy vale +# The checks of CI, run inside the dev container by ci/run.sh, or as they +# are on a machine with the tools, as CI does. Given names, only those run: +# ci/run.sh check clippy vale set -euo pipefail -cd /src +cd "$(dirname "$0")/.." crates=(-p berth-api -p berth-pty -p berth-core -p berth-tui -p berth -p berthd -p box -p berth-page) # Each runs as the condition of an if, where set -e does not stop it: the # steps are joined with && instead. -rust-tests() { cargo test --locked; } -python-tests() { python3 -m unittest discover -s tests -v; } +# nextest runs the test programs side by side, which pays on a machine with +# many cores. On CI's four it is slower than cargo test. +rust-tests() { + if cargo nextest --version >/dev/null 2>&1; then + cargo nextest run --locked --no-fail-fast + else + cargo test --locked + fi +} +# Each file in a process of its own, side by side, its output when it is done. +python-tests() { + local file out=build/python-tests pids=() bad=0 + rm -rf "$out" && mkdir -p "$out" + for file in tests/test_*.py; do + file=$(basename "$file") + python3 -m unittest discover -s tests -p "$file" -v >"$out/$file.log" 2>&1 & + pids+=("$!:$file") + done + for entry in "${pids[@]}"; do + if wait "${entry%%:*}"; then + grep -E '^(Ran|OK)' "$out/${entry#*:}.log" | paste -sd ' ' | sed "s|^|${entry#*:}: |" + else + cat "$out/${entry#*:}.log" + echo "${entry#*:} failed" >&2 + bad=1 + fi + done + return "$bad" +} shell-syntax() { local script bad=0 for script in install.sh provision/bin/*.sh provision/bootstrap/*.sh container/*.sh ci/*.sh \ diff --git a/ci/compose.yml b/ci/compose.yml index 025f57f..6c5215f 100644 --- a/ci/compose.yml +++ b/ci/compose.yml @@ -53,8 +53,14 @@ services: POSTGRES_DB: berth tmpfs: [/var/lib/postgresql/data] + # The tests' side: berth as it was built, Ansible and SSH, without the + # toolchain of the dev image (ci/runner.Dockerfile). runner: <<: *dev + image: ${BERTH_CI_RUNNER_IMAGE:-berth-ci-runner} + build: + context: .. + dockerfile: ci/runner.Dockerfile profiles: [e2e] # Root, for the key it hands the test host in /seed. It writes nothing to # the checkout, which is mounted read-only. diff --git a/ci/dev.Dockerfile b/ci/dev.Dockerfile index 13da9fc..56b0702 100644 --- a/ci/dev.Dockerfile +++ b/ci/dev.Dockerfile @@ -3,7 +3,8 @@ # downloads (/cache/cargo) and the target directory (/src/target) are volumes. # # ci/run.sh shell -FROM rust:1-trixie +# CI installs the same version (.github/actions/rust reads it here). +FROM rust:1.99.0-trixie ARG VALE_VERSION=3.25.1 ENV DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8 RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/ci/mtimes.py b/ci/mtimes.py new file mode 100644 index 0000000..1da5cbf --- /dev/null +++ b/ci/mtimes.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Make a target directory from GitHub's cache count for a fresh checkout. + +cargo tells a changed source by its time, and a checkout gives every file +the time it was made, which is newer than any build in the cache: it would +build all of berth again. So the cache keeps the blob ids of the tree it was +built from, and after a restore every file whose blob is the same, and every +folder whose entries are the same, gets a time long past. What changed keeps +the time of the checkout and is built again. + + ci/mtimes.py save TARGET after a build, before the cache is saved + ci/mtimes.py restore TARGET after the checkout and the cache's restore +""" +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +# Older than any build: 2001-09-09. +PAST = 1_000_000_000 +NAME = '.berth-ci-tree' + + +def tree(): + listed = subprocess.run(['git', 'ls-tree', '-r', '-z', 'HEAD'], cwd=ROOT, check=True, + capture_output=True, text=True).stdout + found = {} + for entry in filter(None, listed.split('\0')): + meta, path = entry.split('\t', 1) + found[path] = meta.split()[2] + return found + + +def folders(paths): + """Each folder of the tree with the names in it.""" + found = {'.': set()} + for path in paths: + parts = path.split('/') + for depth in range(len(parts)): + parent = '/'.join(parts[:depth]) or '.' + found.setdefault(parent, set()).add(parts[depth]) + return found + + +def save(target): + (target / NAME).write_text(''.join(f'{blob} {path}\n' for path, blob in sorted(tree().items()))) + + +def restore(target): + kept = target / NAME + if not kept.is_file(): + print('No tree with the cache: everything is built', file=sys.stderr) + return + then = dict(reversed(line.split(' ', 1)) for line in kept.read_text().splitlines()) + now = tree() + same = [path for path, blob in now.items() if then.get(path) == blob] + for path in same: + file = ROOT / path + if file.is_file() and not file.is_symlink(): + os.utime(file, (PAST, PAST)) + before, after = folders(then), folders(now) + for folder, names in after.items(): + if before.get(folder) == names: + os.utime(ROOT / folder, (PAST, PAST)) + print(f'{len(same)} of {len(now)} files as the cache was built from them', file=sys.stderr) + + +if __name__ == '__main__': + if len(sys.argv) != 3 or sys.argv[1] not in ('save', 'restore'): + sys.exit(__doc__) + {'save': save, 'restore': restore}[sys.argv[1]](Path(sys.argv[2])) diff --git a/ci/plan.py b/ci/plan.py new file mode 100644 index 0000000..26f1b98 --- /dev/null +++ b/ci/plan.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""What a run of CI has to do, and what it can skip. + +Each unit of work (the Rust tests, clippy, the Python tests, each shard of +the end-to-end tests) has a key: a hash of the files in the commit it +depends on, by their git blob ids, with the week of the year so that what +floats outside the repository (base images, packages, the runner) is tried +again at least once a week. A unit that passed leaves a marker in GitHub's +cache under its key, and a later run with the same key skips it. A missing +or evicted marker only means the unit runs again. + +A unit depends on every file except those it is known not to read, so a new +file or folder counts for all of them until it is ruled out here. + + ci/plan.py the plan as JSON, looking for markers with gh + ci/plan.py --keys the keys alone, without looking + ci/plan.py --all the plan with every unit to run + +In CI it writes the plan to $GITHUB_OUTPUT. A pull request finds the markers +of its own runs and those of main. +""" +import argparse +import datetime +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tests/e2e')) +import run as e2e # noqa: E402 (the shards of the end-to-end tests) + +# Read by no unit: the documentation site, the other workflows, what agents +# read, and the files of the QEMU boots, which have a workflow of their own. +NOWHERE = ['docs/', '.agents/', '.claude/', '.github/', 'ci/boot', 'tests/e2e/boot'] +# Except these, which some unit does read. +EVERYWHERE = ['.github/workflows/ci.yml'] + +# The Rust tests compare the API documents and two pages with the code. +RUST_READS = ['docs/openapi/', 'docs/content/docs/(guide)/reference/cli.mdx', + 'docs/content/docs/(guide)/use/upgrading.mdx'] + +# Where the Rust code decides what the role is given or what it finds on the +# host: the profile and its fields, applying, the controller and the agent's +# tool the role installs, keys and logins, plugins, nodes, seeds, updates and +# the tailnet. A pull request runs the shards that apply the role only for a +# change here (or to the role and the tests); main runs them for any change. +SEAM = [ + 'crates/berth-api/', 'crates/berthd/', 'crates/box/', 'crates/berth-pty/', + 'crates/berth-core/build.rs', 'crates/berth-core/Cargo.toml', + 'crates/berth-core/tests/fixtures/ansible/', + *(f'crates/berth-core/src/{name}' for name in ( + 'apply.rs', 'rollout.rs', 'kept_profile.rs', 'node.rs', 'provision.rs', 'seed.rs', 'tailnet.rs', + 'keys/', 'plugins.rs', 'plugin_state.rs', 'paths.rs', 'profiles.rs', 'host_type.rs', 'runner.rs', + 'relay.rs', 'server_image.rs', 'import.rs')), + 'crates/berth/src/local/serve/actions.rs', + 'crates/berth-tui/src/fields.rs', 'crates/berth-tui/src/setup.rs', 'crates/berth-tui/src/flow.rs', +] + +# What each kind of unit leaves out, beyond NOWHERE, and what it keeps of that. +UNITS = { + 'rust': dict(out=['recipes/'], keep=RUST_READS), + 'python': dict(out=['crates/', 'Cargo.toml', 'Cargo.lock', 'tests/e2e/']), + # berth carries the provisioning source, so the end-to-end tests read + # most of the tree; the Python tests and the recipes are not theirs. + 'e2e': dict(out=['recipes/', 'tests/test_', 'tests/fixtures/']), + 'role': dict(out=['recipes/', 'tests/test_', 'tests/fixtures/', 'crates/'], keep=SEAM), +} + +# The shards that apply nothing but the first apply: they test berth's own +# code, so they run for any change berth is built from. +SMOKE = {'studio', 'dock'} + + +def files(): + listed = subprocess.run(['git', 'ls-tree', '-r', '-z', 'HEAD'], cwd=ROOT, check=True, + capture_output=True, text=True).stdout + for entry in filter(None, listed.split('\0')): + meta, path = entry.split('\t', 1) + yield path, meta.split()[2] + + +def week(): + year, number, _ = datetime.date.today().isocalendar() + return f'{year}-{number:02d}' + + +def keys(): + tree = list(files()) + found = {} + for unit, rule in UNITS.items(): + out, keep = NOWHERE + rule.get('out', []), EVERYWHERE + rule.get('keep', []) + digest = hashlib.sha256(f'{unit} {week()}\n'.encode()) + for path, blob in tree: + if any(path.startswith(prefix) for prefix in out) and not any(path.startswith(k) for k in keep): + continue + digest.update(f'{blob} {path}\n'.encode()) + found[unit] = digest.hexdigest()[:20] + return found + + +def markers(event, repo, number): + """The keys of the markers this run can see: its own ref's and main's.""" + refs = ['refs/heads/main'] + if event == 'pull_request' and number: + refs.append(f'refs/pull/{number}/merge') + seen = set() + for ref in refs: + page = 1 + while True: + listed = subprocess.run( + ['gh', 'api', '-X', 'GET', f'repos/{repo}/actions/caches', '-f', 'key=green-', '-f', f'ref={ref}', + '-f', 'per_page=100', '-f', f'page={page}', '--jq', '.actions_caches[].key'], + check=True, capture_output=True, text=True).stdout.split() + seen.update(listed) + if len(listed) < 100: + break + page += 1 + return seen + + +def plan(found, event, seen): + def unit(name, key, also=()): + return {'name': name, 'key': f'green-{name}-{key}', 'also': [f'green-{name}-{k}' for k in also]} + + # On main the role's shards run for any change berth is built from, and + # leave the marker a pull request looks for too. + role, role_also = (found['e2e'], [found['role']]) if event == 'push' else (found['role'], []) + jobs = [unit('rust-tests', found['rust']), unit('clippy', found['rust']), unit('python-tests', found['python'])] + shards = [] + for name in e2e.SHARDS: + shard = unit(f'e2e-{name}', found['e2e']) if name in SMOKE else unit(f'e2e-{name}', role, role_also) + shard.update(shard=name, host=e2e.host_of_shard(name)) + shards.append(shard) + todo = lambda item: item['key'] not in seen # noqa: E731 + result = {item['name']: json.dumps(item) if todo(item) else '' for item in jobs} + result['shards'] = json.dumps([item for item in shards if todo(item)]) + result['skipped'] = ' '.join(item['name'] for item in jobs + shards if not todo(item)) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--keys', action='store_true', help='print the keys and stop') + parser.add_argument('--all', action='store_true', help='run every unit, looking for no markers') + args = parser.parse_args() + found = keys() + if args.keys: + print(json.dumps(found, indent=2)) + return + event = os.environ.get('GITHUB_EVENT_NAME', '') + seen = set() + if not args.all and event in ('push', 'pull_request'): + seen = markers(event, os.environ['GITHUB_REPOSITORY'], os.environ.get('BERTH_CI_PR', '')) + result = plan(found, event, seen) + if output := os.environ.get('GITHUB_OUTPUT'): + with open(output, 'a') as out: + for name, value in result.items(): + out.write(f'{name}={value}\n') + print(json.dumps({'keys': found, **result}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/ci/run.sh b/ci/run.sh index c59d924..6122111 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -7,8 +7,9 @@ # ci/run.sh e2e [TEST…] build berth, start a test host and run the # end-to-end tests against it (names as in # unittest: e2e_serve, e2e_serve.Roles) -# ci/run.sh e2e --shard HOST the same for the tests of one test host, as a -# CI job runs them (see tests/e2e/run.py) +# ci/run.sh e2e --shard NAME the same for the tests of one shard, on a +# fresh host, as a CI job runs them (see SHARDS +# in tests/e2e/run.py) # ci/run.sh all check, then e2e # ci/run.sh boot [cloud|iso] boot real installs in QEMU and test them; needs # KVM and takes a while (see ci/boot.sh) @@ -124,7 +125,7 @@ want_docker() { esac [[ $engine == docker ]] || return 1 (($# == 0)) && return 0 - # A shard is the tests of one test host; it wants the engine when they include the Docker case. + # A shard wants the engine when its tests include the Docker case. if [[ ${1:-} == --shard ]]; then python3 tests/e2e/run.py --tests-of "${2:-}" | grep -q '^e2e_provisioners' || return 1 shift 2 @@ -148,16 +149,23 @@ e2e() { fi e2e_down trap e2e_down EXIT - compose --profile e2e up -d studio dock + # A shard needs its own host alone, and Headscale beside studio. + local hosts=(studio dock) + if [[ ${1:-} == --shard ]]; then + hosts=("$(python3 tests/e2e/run.py --host-of "${2:-}")") || return 1 + fi + local services=("${hosts[@]}") + [[ " ${hosts[*]} " == *" studio "* ]] && services+=(headscale) + compose --profile e2e up -d "${services[@]}" if want_docker "$@"; then docker_up || { echo "The Docker engine for the tests did not start" >&2; return 1; } fi local status=0 - compose --profile e2e run --rm "${tty[@]}" runner python3 tests/e2e/run.py "$@" || status=$? + compose --profile e2e run --rm --no-deps "${tty[@]}" runner python3 tests/e2e/run.py "$@" || status=$? rm -rf "$logs" mkdir -p "$logs" local host - for host in studio dock; do + for host in "${hosts[@]}"; do mkdir -p "$logs/$host" compose --profile e2e logs --no-color "$host" >"$logs/$host/init.log" 2>&1 || true compose --profile e2e cp "$host:/var/log/berth-ci/." "$logs/$host/" >/dev/null 2>&1 || true diff --git a/ci/runner.Dockerfile b/ci/runner.Dockerfile new file mode 100644 index 0000000..10a1a0e --- /dev/null +++ b/ci/runner.Dockerfile @@ -0,0 +1,17 @@ +# The runner of the end-to-end tests: what the tests and berth call, +# with berth from the target directory. It has no toolchain, so a CI +# job loads it quickly. +# +# ci/run.sh e2e +FROM debian:trixie-slim +ENV DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl docker-cli docker-compose git jq nodejs npm openssh-client procps python3 python3-venv tini xz-utils \ + && rm -rf /var/lib/apt/lists/* +COPY provision/requirements.txt /tmp/requirements.txt +RUN python3 -m venv /opt/venv && /opt/venv/bin/pip install --no-cache-dir -r /tmp/requirements.txt \ + && rm /tmp/requirements.txt +ENV PATH=/opt/venv/bin:$PATH HOME=/root +WORKDIR /src +ENTRYPOINT ["tini", "--"] +CMD ["bash"] diff --git a/ci/runner.Dockerfile.dockerignore b/ci/runner.Dockerfile.dockerignore new file mode 100644 index 0000000..20a1619 --- /dev/null +++ b/ci/runner.Dockerfile.dockerignore @@ -0,0 +1,3 @@ +# The image takes only the Python requirements; the checkout is mounted. +* +!provision/requirements.txt diff --git a/crates/berth-core/src/apply.rs b/crates/berth-core/src/apply.rs index fd48f2b..0f2a02a 100644 --- a/crates/berth-core/src/apply.rs +++ b/crates/berth-core/src/apply.rs @@ -770,4 +770,43 @@ mod tests { std::fs::write(format!("{dir}/inventory"), untemplated(&inventory)).unwrap(); } } + + /// What the role is given for the made-up profiles in + /// tests/fixtures/ansible, kept beside them. CI applies the role on a pull + /// request only for changes there or to the role, so a change to what it + /// gets has to show there. + #[test] + fn ansible_contract() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ansible"); + let mut names: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .filter_map(|entry| { + let name = entry.unwrap().file_name().into_string().unwrap(); + name.strip_suffix(".profile.json").map(str::to_owned) + }) + .collect(); + names.sort(); + assert!(!names.is_empty()); + let update = std::env::var_os("BERTH_UPDATE_DOCS").is_some(); + for name in names { + let profile = read_back(&dir.join(format!("{name}.profile.json"))); + let vars = ansible_vars( + &profile, + &json!({"docker": "/usr/lib/berth/plugins/docker", "incus": "/usr/lib/berth/plugins/incus"}), + &["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBERTH berth@studio".to_owned()], + ); + let written = serde_json::to_string_pretty(&vars).unwrap() + "\n"; + let path = dir.join(format!("{name}.vars.json")); + if update { + std::fs::write(&path, &written).unwrap(); + continue; + } + let kept = std::fs::read_to_string(&path).unwrap_or_default(); + assert!( + kept == written, + "{} is stale: write it again with BERTH_UPDATE_DOCS=1 cargo test -p berth-core ansible_contract", + path.display() + ); + } + } } diff --git a/crates/berth-core/tests/fixtures/ansible/README.md b/crates/berth-core/tests/fixtures/ansible/README.md new file mode 100644 index 0000000..67dea8b --- /dev/null +++ b/crates/berth-core/tests/fixtures/ansible/README.md @@ -0,0 +1,5 @@ +# What berth hands to Ansible + +Each `NAME.profile.json` is a made-up host's profile, and `NAME.vars.json` the variables berth gives the role for it (`apply::ansible_vars`). A test in `crates/berth-core/src/apply.rs` compares the two. Write the variables again with `BERTH_UPDATE_DOCS=1 cargo test -p berth-core ansible_contract`. + +CI runs the end-to-end tests that apply the role on a pull request only when these files, the role or the code in `SEAM` of `ci/plan.py` change. A change to what the role gets shows here, so it runs them. diff --git a/crates/berth-core/tests/fixtures/ansible/laptop.profile.json b/crates/berth-core/tests/fixtures/ansible/laptop.profile.json new file mode 100644 index 0000000..943405e --- /dev/null +++ b/crates/berth-core/tests/fixtures/ansible/laptop.profile.json @@ -0,0 +1,15 @@ +{ + "hostname": "desk", + "target": "laptop", + "operator_username": "ops", + "operator_ssh_public_keys": [], + "operator_password_hash": "$6$made-up$hash", + "agent_username": "agent", + "agent_ssh_public_keys": [], + "agent_password_hash": "", + "agent_privileges": "full", + "keyboard": "de", + "wifi_enabled": false, + "ansible_vars": {}, + "access": {"auth": "berth"} +} diff --git a/crates/berth-core/tests/fixtures/ansible/laptop.vars.json b/crates/berth-core/tests/fixtures/ansible/laptop.vars.json new file mode 100644 index 0000000..46349c5 --- /dev/null +++ b/crates/berth-core/tests/fixtures/ansible/laptop.vars.json @@ -0,0 +1,15 @@ +{ + "devbox_agent_password_hash": "", + "devbox_agent_privileges": "full", + "devbox_agent_ssh_public_key": "", + "devbox_berth_ssh_public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBERTH berth@studio", + "devbox_operator_password_hash": "$6$made-up$hash", + "devbox_operator_ssh_public_key": "", + "devbox_operator_user": "ops", + "devbox_plugin_dirs": { + "docker": "/usr/lib/berth/plugins/docker", + "incus": "/usr/lib/berth/plugins/incus" + }, + "devbox_profile": "laptop", + "devbox_user": "agent" +} diff --git a/crates/berth-core/tests/fixtures/ansible/node.profile.json b/crates/berth-core/tests/fixtures/ansible/node.profile.json new file mode 100644 index 0000000..d2dd9e2 --- /dev/null +++ b/crates/berth-core/tests/fixtures/ansible/node.profile.json @@ -0,0 +1,8 @@ +{ + "type": "node", + "target": "cloud", + "agent_username": "agent", + "operator_username": "ops", + "ansible_vars": {"devbox_plugin_features": ["incus/host"]}, + "access": {"address": "192.0.2.7", "user": "admin", "auth": "berth"} +} diff --git a/crates/berth-core/tests/fixtures/ansible/node.vars.json b/crates/berth-core/tests/fixtures/ansible/node.vars.json new file mode 100644 index 0000000..6de19cc --- /dev/null +++ b/crates/berth-core/tests/fixtures/ansible/node.vars.json @@ -0,0 +1,15 @@ +{ + "devbox_agent_ssh_public_key": "", + "devbox_berth_ssh_public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBERTH berth@studio", + "devbox_level": "node", + "devbox_node_user": "admin", + "devbox_operator_ssh_public_key": "", + "devbox_plugin_dirs": { + "docker": "/usr/lib/berth/plugins/docker", + "incus": "/usr/lib/berth/plugins/incus" + }, + "devbox_plugin_features": [ + "incus/host" + ], + "devbox_profile": "cloud" +} diff --git a/crates/berth-core/tests/fixtures/ansible/workstation.profile.json b/crates/berth-core/tests/fixtures/ansible/workstation.profile.json new file mode 100644 index 0000000..c2b32f6 --- /dev/null +++ b/crates/berth-core/tests/fixtures/ansible/workstation.profile.json @@ -0,0 +1,17 @@ +{ + "hostname": "studio", + "target": "cloud", + "operator_username": "ops", + "operator_ssh_public_keys": ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOPERATOR ops@laptop"], + "agent_username": "agent", + "agent_ssh_public_keys": ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAGENT agent@laptop"], + "agent_passwordless_sudo": true, + "agent_privileges": "limited", + "ansible_vars": { + "devbox_geometry": "1920x1080", + "devbox_plugin_features": ["docker/engine", "not a feature"], + "devbox_tailscale_key_command": ["sh", "-c", "id"], + "ansible_host": "192.0.2.9" + }, + "access": {"address": "192.0.2.5", "user": "ops", "auth": "berth"} +} diff --git a/crates/berth-core/tests/fixtures/ansible/workstation.vars.json b/crates/berth-core/tests/fixtures/ansible/workstation.vars.json new file mode 100644 index 0000000..d6191fb --- /dev/null +++ b/crates/berth-core/tests/fixtures/ansible/workstation.vars.json @@ -0,0 +1,18 @@ +{ + "devbox_agent_passwordless_sudo": true, + "devbox_agent_privileges": "limited", + "devbox_agent_ssh_public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAGENT agent@laptop", + "devbox_berth_ssh_public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBERTH berth@studio", + "devbox_geometry": "1920x1080", + "devbox_operator_ssh_public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOPERATOR ops@laptop", + "devbox_operator_user": "ops", + "devbox_plugin_dirs": { + "docker": "/usr/lib/berth/plugins/docker", + "incus": "/usr/lib/berth/plugins/incus" + }, + "devbox_plugin_features": [ + "docker/engine" + ], + "devbox_profile": "cloud", + "devbox_user": "agent" +} diff --git a/docs/content/docs/(guide)/reference/building.mdx b/docs/content/docs/(guide)/reference/building.mdx index b9aed2f..8ffb710 100644 --- a/docs/content/docs/(guide)/reference/building.mdx +++ b/docs/content/docs/(guide)/reference/building.mdx @@ -75,7 +75,7 @@ The checkout is mounted, and files are written as you. Cargo's downloads and the The end-to-end tests have two test hosts, `studio` and `dock`. Each is an Ubuntu container with SSH and an operator account, and the two run in parallel. A profile is applied to a host over SSH, with the role's tasks for the accounts, the plugins and berthd. The tests then use the host through berth's commands, berthd's API and `berth serve`. Test cases that look at the same state of a host follow each other, so the state is applied once. The desktop packages, toolchains and browsers are left out. Some modules need less than that. `e2e_access` and `e2e_controller.Controller` use the bare host `dock`, which has only SSH. `e2e_provisioners` runs without a host. `e2e_restore` starts a second `berth serve` on the host `studio` and leaves the host as it is. With Docker, `ci/run.sh` also starts an engine on the test network for the Docker plugin's cases. `BERTH_CI_DOCKER` switches it on or off, and `BERTH_CI_DIND_IMAGES` names images to load into it. The test hosts and the tests are on an internal network with no way out. Each run starts from new hosts and removes them afterwards. A run ends with the time of each test and the number of playbook runs. After a failure the hosts' logs are in `build/ci/logs/`. -To run one module or one test case, name it: `ci/run.sh e2e e2e_serve.Roles`. `ci/run.sh e2e --shard dock` runs the tests of one test host, as a CI job does. The test cases of each host are listed in order in `tests/e2e/run.py`. A new one goes there, after the ones whose state it uses. +To run one module or one test case, name it: `ci/run.sh e2e e2e_serve.Roles`. `ci/run.sh e2e --shard plain` runs one shard on a fresh host, as a CI job does. The test cases of each host are listed in order in `tests/e2e/run.py`, and cut into shards in `SHARDS` there. A new one goes after the ones whose state it uses, and into their shard. ### Boot tests diff --git a/tests/e2e/run.py b/tests/e2e/run.py index edf5089..814dd00 100644 --- a/tests/e2e/run.py +++ b/tests/e2e/run.py @@ -3,17 +3,17 @@ profile, and the others use the host it set up. ci/run.sh e2e all of them - ci/run.sh e2e --shard dock the tests of one test host, as a CI job runs them + ci/run.sh e2e --shard plain the tests of one shard, on a host of their own, as CI runs them ci/run.sh e2e e2e_serve one module ci/run.sh e2e e2e_serve.Roles one test case The tests of each test host run in a process of their own, at the same time: the output of the first host comes as it happens, that of the others -when they are done. A shard is the tests of one test host: CI runs each in -a job of its own, so the hosts run on machines apart and no state is made -twice. At the end each prints how long each test and each test -case took, slowest first, how many playbook runs there were and which tests -made them. +when they are done. A shard is a run of the tests of one host that share +its states (SHARDS): CI runs each in a job of its own, on a fresh host, +where the first test case makes the state it needs. At the end each +prints how long each test and each test case took, slowest first, how many +playbook runs there were and which tests made them. Only meant for the runner container of ci/compose.yml. """ @@ -84,6 +84,52 @@ } +# The tests of each host cut into shards, in the order HOSTS has them. Each +# shard starts from a fresh host, so it pays for the first apply and the +# states it uses; one state is never cut in two. The shards `studio` and +# `dock` are the tests that make no apply but the first: ci/plan.py runs them +# for any change to berth, and the others only for changes to what the role +# is given (its SEAM) on a pull request. +SHARDS = { + 'studio': ['e2e_apply.Apply', 'e2e_host', 'e2e_secrets', 'e2e_local', 'e2e_controller.OnAWorkstation', + 'e2e_serve.Routes', 'e2e_serve.Portal', 'e2e_serve.Roles', 'e2e_serve.SetUp', + 'e2e_serve.ClosedSetUp', 'e2e_client.Client', 'e2e_client.AccessToken', 'e2e_restore'], + 'adopt': ['e2e_adopt.Adopt'], + 'extended': ['e2e_apply.Again', 'e2e_plugins.Plugin', 'e2e_tailnet.AutomaticKey', 'e2e_update.Update', + 'e2e_update.ShippedChanged'], + 'plain': ['e2e_plugins.PluginRemoved', 'e2e_tailnet.JoinedByHand'], + 'served-extended': ['e2e_tailnet.ThroughTheServer', 'e2e_serve.ApplyJob', 'e2e_client.ClientApply', + 'e2e_update.UpdateServed'], + 'dock': ['e2e_access', 'e2e_provisioners', 'e2e_update.FromBerth052'], + 'controller': ['e2e_controller.Controller'], + 'own-key': ['e2e_own_key'], + 'node': ['e2e_node'], +} + + +def host_of_shard(shard): + return next(host for host, names in HOSTS.items() if SHARDS[shard][0] in names) + + +def unsharded(): + """What is wrong with SHARDS: a test of HOSTS in none or in two, or a shard of two hosts.""" + problems = [] + for host, names in HOSTS.items(): + for name in names: + holding = [shard for shard, listed in SHARDS.items() if name in listed] + if len(holding) != 1: + problems.append(f'{name} is in {len(holding)} shards') + for shard, listed in SHARDS.items(): + if len({host_of(name) for name in listed}) != 1 or any( + name not in HOSTS[host_of(name)] for name in listed): + problems.append(f'shard {shard} is not the tests of one host in HOSTS') + return problems + +def host_of_name_in(shard, name): + """Whether a module or test case named on the command line is of the shard.""" + return any(listed == name or listed.split('.')[0] == name.split('.')[0] for listed in SHARDS[shard]) + + def host_of(name): # A test case named on its own goes where it is named; a module where its cases are. for host, names in HOSTS.items(): @@ -220,16 +266,22 @@ def main(): sys.exit('The end-to-end tests run in their containers: ci/run.sh e2e') sys.exit(0 if run_here(sys.argv[1:], os.environ.get('BERTH_E2E_CHILD') == 'all') else 1) parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('--shard', choices=HOSTS, help='run the tests of this test host only') + parser.add_argument('--shard', choices=SHARDS, help='run the tests of this shard only') parser.add_argument('--shards', action='store_true', help='print the names of the shards as JSON') - parser.add_argument('--tests-of', choices=HOSTS, help='print the tests of this shard, one to a line') + parser.add_argument('--tests-of', choices=SHARDS, help='print the tests of this shard, one to a line') + parser.add_argument('--host-of', choices=SHARDS, help='print the test host of this shard') parser.add_argument('names', nargs='*', help='modules or test cases, as unittest names them') args = parser.parse_args() + if problems := unsharded(): + sys.exit('SHARDS of tests/e2e/run.py: ' + '; '.join(problems)) if args.shards: - print(json.dumps(list(HOSTS))) + print(json.dumps(list(SHARDS))) return if args.tests_of: - print('\n'.join(HOSTS[args.tests_of])) + print('\n'.join(SHARDS[args.tests_of])) + return + if args.host_of: + print(host_of_shard(args.host_of)) return if os.environ.get('BERTH_E2E') != '1': sys.exit('The end-to-end tests run in their containers: ci/run.sh e2e') @@ -240,7 +292,10 @@ def main(): if not names: groups = {host: list(listed) for host, listed in HOSTS.items()} if args.shard: - groups = {args.shard: groups.get(args.shard, [])} if names else {args.shard: list(HOSTS[args.shard])} + host = host_of_shard(args.shard) + listed = [name for name in groups.get(host, []) if host_of_name_in(args.shard, name)] if names \ + else list(SHARDS[args.shard]) + groups = {host: listed} started = time.monotonic() running = [] for host, listed in groups.items():