test(smoke): smoke test in rust

.github/workflows/smoke-tests.yml

@@ -1,20 +1,18 @@
 name: Compose smoke tests
 
-# Manual only: each scenario stands up a full docker-compose cluster (N nodes +
-# relay + prometheus) and observes it for 2 minutes. The resource-heavy
-# very_large scenario remains local-only. Too heavy to attach to push or
-# pull_request.
+# Manual only: each scenario runs a full docker-compose cluster for two
+# minutes, too heavy for push or pull_request. very_large stays local-only.
 on:
   workflow_dispatch:
     inputs:
       scenarios:
-        description: "Scenario filter (go test -run regex). Empty runs the CI matrix; very_large is excluded."
+        description: "Space-separated scenario names (e.g. `default_alpha pluto_dkg`). Empty runs the CI matrix; very_large is always excluded."
         type: string
         default: ""
-      go_timeout:
-        description: "go test -timeout. Must exceed the sum of the selected scenarios."
-        type: string
-        default: "50m"
+      smoke_timeout:
+        description: "Minutes allowed for the smoke run itself (image and harness builds are separate steps). Must exceed the sum of the selected scenarios' windows."
+        type: number
+        default: 50
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -24,72 +22,71 @@ permissions:
   contents: read
   actions: read
 
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  RUSTFLAGS: "-Dwarnings -C debuginfo=0"
+
 jobs:
   smoke:
     name: Compose smoke tests
     runs-on: ubuntu-24.04
-    # Covers the pluto image build (release build of pluto-cli inside docker,
-    # uncached on a fresh runner) plus the scenario matrix.
-    timeout-minutes: 90
+    timeout-minutes: 100
 
     steps:
       - name: Checkout
         uses: actions/checkout@v6
 
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: test-infra/compose/go.mod
-          cache-dependency-path: test-infra/compose/go.sum
+      - name: Cache cargo registry and target
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+
+      - name: Install `oas3-gen`
+        run: cargo install oas3-gen@0.24.0 --locked
 
       - name: Build pluto image
-        # Built here rather than letting the harness do it inside `go test`, so
-        # the release compile does not consume the -timeout budget (which should
-        # bound observation, not compilation) and a build break fails in its own
-        # step. The harness still calls `docker build` during its define step;
-        # replicating the tag and build-arg exactly makes that a cache hit.
-        #
-        # Deliberately plain `docker build`, not buildx: setup-buildx-action's
-        # docker-container driver keeps a separate cache that the harness's
-        # `docker build` would not see, so the image would be compiled twice.
+        # Built in its own step so the release compile is not charged to the smoke
+        # budget; the harness's own `docker build` then hits the cache.
         timeout-minutes: 40
         run: |
           docker build -t pluto:local \
             --build-arg "GIT_COMMIT_HASH_SHORT=$(git rev-parse --short=7 HEAD)" .
 
+      - name: Build smoke harness
+        run: cargo test --locked -p pluto-test-compose --test smoke --no-run
+
       - name: Run smoke tests
-        working-directory: test-infra/compose
-        # Inputs are passed as env vars, never interpolated into the script:
-        # `${{ inputs.* }}` inside `run:` is substituted before the shell sees
-        # it, so a crafted value would execute as shell.
+        timeout-minutes: ${{ fromJSON(inputs.smoke_timeout) }}
+        # Inputs are passed through env, never interpolated into the script.
         env:
-          # The pluto image is built from this checkout during the define step.
           PLUTO_REPO: ${{ github.workspace }}
           SCENARIOS: ${{ inputs.scenarios }}
-          GO_TIMEOUT: ${{ inputs.go_timeout }}
-          LOG_DIR: ${{ runner.temp }}/smoke-logs
+          SMOKE_LOG_DIR: ${{ runner.temp }}/smoke-logs
+          # Containers run as root; without this the runner cannot clean up.
+          SMOKE_SUDO_PERMS: "1"
         run: |
-          mkdir -p "$LOG_DIR"
-
-          args=(
-            ./smoke -v -integration
-            "-timeout=$GO_TIMEOUT"
-            "-log-dir=$LOG_DIR"
-            # Requires more CPU than a GitHub-hosted runner provides reliably.
-            "-skip=^TestSmoke/very_large$"
-            # Containers run as root, so the artefacts they leave in the compose
-            # dir are root-owned; without this the runner cannot clean them up.
-            -sudo-perms
-          )
+          mkdir -p "$SMOKE_LOG_DIR"
+
+          # very_large needs more CPU than a hosted runner has.
+          args=(--ignored --nocapture --test-threads=1 --skip scenario_very_large)
           if [ -n "$SCENARIOS" ]; then
-            args+=(-run "$SCENARIOS")
+            # Exact names: `dkg` alone would also select pluto_dkg.
+            args+=(--exact)
+            for name in $SCENARIOS; do
+              args+=("scenario_$name")
+            done
+
+            # libtest passes with 0 tests for an unknown name; every name must select one.
+            wanted=$(echo "$SCENARIOS" | wc -w | tr -d ' ')
+            found=$(cargo test --locked -p pluto-test-compose --test smoke -- "${args[@]}" --list | grep -c ': test$' || true)
+            if [ "$found" -ne "$wanted" ]; then
+              echo "::error::$found of $wanted scenario names select a test (very_large is always excluded): $SCENARIOS"
+              exit 1
+            fi
           fi
 
-          go test "${args[@]}"
+          cargo test --locked -p pluto-test-compose --test smoke -- "${args[@]}"
 
       - name: Upload scenario logs
-        # Always: a passing run's logs are the baseline for triaging the next
-        # failure, and these clusters are expensive to reproduce.
         if: always()
         uses: actions/upload-artifact@v4
         with:

.gitignore

@@ -33,6 +33,3 @@ test-infra/sszfixtures/sszfixtures
 .claude/worktrees/
 .claude/scheduled_tasks.lock
 test-cluster
-
-# Smoke-test docker-compose logs (go test -log-dir)
-test-infra/compose/**/*.log

AGENTS.md

@@ -27,6 +27,7 @@ pluto/
     p2p/                   # P2P networking (libp2p)
     peerinfo/              # Peer info utilities
     relay-server/          # Relay server implementation
+    test-compose/          # Docker-compose smoke-test harness (test infrastructure, not shipped)
     testutil/              # Test helpers/fixtures (workspace-internal)
     tracing/               # Observability/tracing utilities
   test-infra/              # Docker-compose and local infra for integration testing/observability

Cargo.toml

@@ -24,6 +24,7 @@ members = [
     "crates/frost",
     "crates/priority",
     "crates/infosync",
+    "crates/test-compose",
 ]
 # Vendored fork consumed only via [patch.crates-io]; excluded so it builds/tests
 # standalone (its upstream code isn't written to this workspace's lints) without
@@ -59,6 +60,7 @@ futures-timer = "3.0"
 backon = "1.6.0"
 hex = { version = "0.4.3" }
 hex-literal = "0.4"
+nix = { version = "0.30", features = ["user"] }
 prost = "0.14"
 prost-build = "0.14"
 prost-types = "0.14"
@@ -173,6 +175,7 @@ pluto-peerinfo = { path = "crates/peerinfo" }
 pluto-frost = { path = "crates/frost" }
 pluto-priority = { path = "crates/priority" }
 pluto-infosync = { path = "crates/infosync" }
+pluto-test-compose = { path = "crates/test-compose" }
 
 [workspace.lints.rust]
 missing_docs = "deny"

crates/cli/src/commands/create_cluster.rs

@@ -2864,7 +2864,7 @@ mod tests {
     /// `CHARON_*` env var. Charon binds env for all commands generically
     /// (viper `SetEnvPrefix`+`AutomaticEnv`), so tooling that configures a
     /// cluster purely through the environment — the compose harness in
-    /// `test-infra/compose` — works against charon and pluto alike.
+    /// `crates/test-compose` — works against charon and pluto alike.
     #[test]
     fn create_cluster_flags_use_charon_env_prefix() {
         use clap::CommandFactory as _;

crates/test-compose/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "pluto-test-compose"
+description = "Docker-compose smoke-test harness for pluto and charon clusters. Test infrastructure, not shipped."
+version.workspace = true
+edition.workspace = true
+repository.workspace = true
+license.workspace = true
+publish.workspace = true
+
+[dependencies]
+k256.workspace = true
+nix.workspace = true
+pluto-eth2util.workspace = true
+pluto-k1util.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tokio-util.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true
+test-case.workspace = true
+tokio = { workspace = true, features = ["test-util"] }
+tracing-subscriber.workspace = true
+
+[lints]
+workspace = true

crates/test-compose/README.md

@@ -0,0 +1,67 @@
+# Pluto Compose
+
+Docker-compose smoke-test harness for pluto and charon clusters, adapted from
+charon's `testutil/compose`. Test infrastructure: nothing here ships in the
+`pluto` binary.
+
+A cluster is produced in steps (`define` → `lock` → `run`), each
+rewriting `docker-compose.yml` from `config.json`. `auto` chains the steps
+against a docker daemon, brings the cluster up and watches Prometheus for
+alerts. Nodes are charon or pluto per `node_impls`; key generation follows
+`key_gen_impl`.
+
+## Smoke tests
+
+`tests/smoke.rs` holds one `#[ignore]`d test per scenario, named
+`scenario_<name>`. Each stands up a cluster for two minutes and fails on any
+firing alert. Prerequisites: docker with compose v2, and `oas3-gen` from
+`CONTRIBUTING.md` (the harness links `pluto-eth2util`, whose API types are
+generated at build time).
+
+```bash
+# one or more scenarios
+cargo test -p pluto-test-compose --test smoke -- --ignored --nocapture --exact scenario_default_alpha scenario_pluto_dkg
+# the CI matrix (very_large needs a big machine)
+cargo test -p pluto-test-compose --test smoke -- --ignored --nocapture --test-threads=1 --skip scenario_very_large
+# keep per-scenario logs
+SMOKE_LOG_DIR=. cargo test -p pluto-test-compose --test smoke -- --ignored --nocapture --exact scenario_default_alpha
+```
+
+| Variable | Effect |
+|---|---|
+| `PLUTO_REPO` | Repository root the `pluto:local` image is built from (default: this workspace). |
+| `SMOKE_SUDO_PERMS` | Set to `1` when containers run as root, so the harness can `sudo chown` its artefacts. |
+| `SMOKE_LOG_DIR` | Write `<dir>/<scenario>.log` with the `docker compose up` output. |
+| `SMOKE_EXTERNAL_RELAY` | Use this relay URL instead of the in-cluster relay. |
+
+The CI workflow (`.github/workflows/smoke-tests.yml`) is manual-only and runs
+the same command.
+
+## Alert criteria vs. charon
+
+Adapted from charon's `testutil/compose` alert rules, but the gate is corrected and the
+criteria calibrated to actually fire: charon's collector matches Prometheus alert state
+`"active"`, which is never emitted (only `inactive` / `pending` / `firing`), so upstream
+nothing is ever gated. This harness matches `"firing"`, so several rules necessarily differ:
+
+| Rule | Charon v1.7.1 | Pluto | Change & why |
+|------|---------------|-------|--------------|
+| `Pluto Down` | `up == 0` | `up == 0` | identical |
+| `Validator API Error Rate` | `increase(…{endpoint!="proxy"}[30s]) > 1` | same | identical |
+| `Proxy API Error Rate` | `increase(…{endpoint="proxy"}[30s]) > 5` | same | identical |
+| `Warn Log Rate` | `increase(app_log_warn_total[30s]) > 2` | same + `{topic!~"vmock\|tracker"}` | exclude charon mock-noise topics (vmock has no builder-registration handler; the beacon mock never includes broadcasts on-chain) |
+| `Error Log Rate` | `app_log_error_total > 0` | `increase(app_log_error_total[30s]) > 0` | windowed — an absolute counter can't recover from the inherent cold-start consensus timeout (mock-VC startup delay → no randao); a window + warmup can |
+| `Broadcast Duty Rate` | `increase(core_bcast_broadcast_total[30s]) < 0.5` | `(sum by (job) (increase(…{job=~"node[0-9]+"}[30s])) or on (job) max by (job) (0 * up)) < 0.5` | per-node sum + absent-series fallback, so a node emitting *no* broadcast series fails (charon's per-series form missed it) |
+| `Outstanding Duty Rate` | `core_bcast_broadcast_total − core_scheduler_duty_total > 50` | *removed* | dead rule — a duty is broadcast at most as often as scheduled, so it can never be positive |
+| _gate (alert state)_ | `"active"` — never emitted | `"firing"` + readiness wait + 60s warmup allowlist | charon's gate is vacuous; pluto's enforces |
+
+Scenarios that intentionally degrade the cluster tune the gate via config, not the code:
+
+| Config knob | Effect | Used by |
+|-------------|--------|---------|
+| `alert_exclude_jobs` | exempt a node from the per-node rules (never from `Pluto Down`) | `1_of_4_down`, `1_of_3_down` |
+| `alert_disable_rules` | drop an entire rule | `1_of_3_down` (disables the error-rate gates — a downed round-1 leader makes every third proposer duty unrecoverable on the mock) |
+
+## Versioning
+
+The charon image tag is `CHARON_IMAGE_TAG` in `src/smoke.rs`.