diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3bc2ef7d8c..32705b2cbd 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -43,6 +43,7 @@ env: DZ_DEVICE_HEALTH_ORACLE_IMAGE: ghcr.io/malbeclabs/dz-e2e/device-health-oracle:${{ github.event.inputs.image_tag || github.sha }} DZ_GEOPROBE_IMAGE: ghcr.io/malbeclabs/dz-e2e/geoprobe:${{ github.event.inputs.image_tag || github.sha }} DZ_SENTINEL_IMAGE: ghcr.io/malbeclabs/dz-e2e/sentinel:${{ github.event.inputs.image_tag || github.sha }} + DZ_IP_VERIFIER_IMAGE: ghcr.io/malbeclabs/dz-e2e/ip-verifier:${{ github.event.inputs.image_tag || github.sha }} DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE: ghcr.io/malbeclabs/dz-e2e/validator-metadata-service-mock:${{ github.event.inputs.image_tag || github.sha }} jobs: @@ -291,6 +292,7 @@ jobs: docker push ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }} docker push ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }} docker push ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }} + docker push ${{ env.DZ_IMAGE_REPO }}/ip-verifier:${{ env.DZ_IMAGE_TAG }} docker push ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }} - name: Discover tests and distribute across shards if: steps.gate.outputs.run-e2e == 'true' @@ -434,6 +436,7 @@ jobs: pull_with_retry ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }} pull_with_retry ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }} pull_with_retry ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }} + pull_with_retry ${{ env.DZ_IMAGE_REPO }}/ip-verifier:${{ env.DZ_IMAGE_TAG }} pull_with_retry ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }} pull_with_retry ghcr.io/malbeclabs/dz-e2e/prometheus:v2.54.1 pull_with_retry public.ecr.aws/influxdb/influxdb:1.8 diff --git a/CLAUDE.md b/CLAUDE.md index 0c0ae926fd..e8a7dd7a93 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,6 +147,7 @@ The local devnet runs in Docker containers with the naming convention `dz-local- - **Clients**: `dz-local-client-{pubkey}` - Client containers running doublezerod - **Manager**: `dz-local-manager` - Runs the doublezero CLI for admin operations - **Controller**: `dz-local-controller` - Pushes configs to devices +- **IP verifier**: `dz-local-ip-verifier` - Signs RFC-27 IP ownership proofs for `connect` (see `e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md`) ### Arista Device Interaction diff --git a/crates/doublezero-ip-verifier/src/settings.rs b/crates/doublezero-ip-verifier/src/settings.rs index 2f13ded370..8ceee73397 100644 --- a/crates/doublezero-ip-verifier/src/settings.rs +++ b/crates/doublezero-ip-verifier/src/settings.rs @@ -9,6 +9,7 @@ use clap::Parser; use doublezero_config::Environment; use ipnetwork::IpNetwork; use solana_keypair::{read_keypair_file, Keypair}; +use solana_program::pubkey::Pubkey; use std::{net::SocketAddr, path::PathBuf, sync::Arc, time::Duration}; /// Every flag also reads a `DZ_IP_VERIFIER_`-prefixed environment variable, which is how the @@ -29,6 +30,13 @@ pub struct AppArgs { #[arg(long, env = "DZ_IP_VERIFIER_LEDGER_RPC")] pub ledger_rpc: Option, + /// Serviceability program ID `GlobalState` is read from. Defaults to the program `--env` + /// names. Needed wherever the deployed program is not that one — a devnet deploys its own, + /// and reading the environment's program ID there finds no `GlobalState` account at all, so + /// the authority check has nothing to compare against and never fails. + #[arg(long, env = "DZ_IP_VERIFIER_SERVICEABILITY_PROGRAM_ID")] + pub serviceability_program_id: Option, + /// Path to the verifier keypair JSON file. Its public key must match /// `GlobalState.ip_verifier_authority_pk` or every proof this service issues is rejected /// onchain. @@ -159,17 +167,23 @@ impl AppArgs { } } + pub fn serviceability_program_id(&self) -> anyhow::Result { + match self.serviceability_program_id { + Some(program_id) => Ok(program_id), + None => Ok(self + .env + .config() + .map_err(|err| anyhow::anyhow!("{err}")) + .with_context(|| format!("no network config for environment {}", self.env))? + .serviceability_program_id), + } + } + /// The ledger client both background loops read through. pub fn ledger(&self) -> anyhow::Result { - let config = self - .env - .config() - .map_err(|err| anyhow::anyhow!("{err}")) - .with_context(|| format!("no network config for environment {}", self.env))?; - Ok(Ledger::new( self.ledger_rpc_url()?, - config.serviceability_program_id, + self.serviceability_program_id()?, )) } @@ -250,6 +264,30 @@ mod tests { assert_eq!(args.ledger_rpc_url().unwrap(), "http://localhost:8899"); } + #[test] + fn the_serviceability_program_id_defaults_to_the_environment() { + let args = parse(&["--env", "testnet"]); + assert_eq!( + args.serviceability_program_id().unwrap(), + Environment::Testnet + .config() + .unwrap() + .serviceability_program_id + ); + } + + #[test] + fn an_explicit_serviceability_program_id_wins() { + let program_id = Pubkey::new_unique(); + let args = parse(&[ + "--env", + "testnet", + "--serviceability-program-id", + &program_id.to_string(), + ]); + assert_eq!(args.serviceability_program_id().unwrap(), program_id); + } + #[test] fn a_missing_keypair_file_reports_the_path_and_nothing_else() { let args = AppArgs::try_parse_from([ diff --git a/e2e/.env.local b/e2e/.env.local index f3390d153b..f094feb134 100644 --- a/e2e/.env.local +++ b/e2e/.env.local @@ -13,3 +13,4 @@ DZ_DEVICE_HEALTH_ORACLE_IMAGE=${DZ_IMAGE_REPO}/device-health-oracle:${DZ_IMAGE_T DZ_GEOPROBE_IMAGE=${DZ_IMAGE_REPO}/geoprobe:${DZ_IMAGE_TAG} DZ_SENTINEL_IMAGE=${DZ_IMAGE_REPO}/sentinel:${DZ_IMAGE_TAG} DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE=${DZ_IMAGE_REPO}/validator-metadata-service-mock:${DZ_IMAGE_TAG} +DZ_IP_VERIFIER_IMAGE=${DZ_IMAGE_REPO}/ip-verifier:${DZ_IMAGE_TAG} diff --git a/e2e/docker/base.dockerfile b/e2e/docker/base.dockerfile index d5ce5329b9..8b37b064e6 100644 --- a/e2e/docker/base.dockerfile +++ b/e2e/docker/base.dockerfile @@ -122,6 +122,7 @@ RUN --mount=type=cache,id=cargo-${CARGO_LOCK_HASH},target=/cargo \ RUSTFLAGS="-C link-arg=-fuse-ld=mold" cargo build --workspace --release --exclude doublezero-serviceability --exclude doublezero-telemetry && \ cp /target/release/doublezero ${BIN_DIR}/ && \ cp /target/release/doublezero-sentinel ${BIN_DIR}/ && \ + cp /target/release/doublezero-ip-verifier ${BIN_DIR}/ && \ cp /target/release/fork-accounts ${BIN_DIR}/ # Force COPY in later stages to always copy the binaries, even if they appear to be the same. diff --git a/e2e/docker/ip-verifier/Dockerfile b/e2e/docker/ip-verifier/Dockerfile new file mode 100644 index 0000000000..125fa066a1 --- /dev/null +++ b/e2e/docker/ip-verifier/Dockerfile @@ -0,0 +1,13 @@ +ARG BASE_IMAGE=undefined +FROM ${BASE_IMAGE} AS base + +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y ca-certificates curl bash + +COPY --from=base /doublezero/bin/doublezero-ip-verifier /usr/local/bin/doublezero-ip-verifier + +ENTRYPOINT ["doublezero-ip-verifier"] diff --git a/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md b/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md new file mode 100644 index 0000000000..16d5544fce --- /dev/null +++ b/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md @@ -0,0 +1,92 @@ +# IP ownership verification in the local devnet + +RFC-27 ([`rfcs/rfc27-ip-verification.md`](../../rfcs/rfc27-ip-verification.md)) has `connect` +attach a proof, signed by a DoubleZero-operated verifier, that the caller can originate traffic +from the `client_ip` it is binding. Every devnet runs that verifier — `dev/dzctl` and the Go e2e +suite alike — so the local flow matches production. + +## What comes up + +The verifier is on by default: `IPVerifierSpec.Disabled` is the opt-out, so a devnet that says +nothing about it gets one. `dzctl start` brings up a `dz-local-ip-verifier` container (image +`dz-local/ip-verifier:dev`) alongside the rest of the stack; an e2e test gets the same container +named for its own deploy ID: + +- **Keypair**: generated per deploy into `dev/.deploy/dz-local/ip-verifier-keypair.json`. Devnet + only — nothing is checked in. +- **Onchain authority**: the keypair's pubkey is written to + `GlobalState.ip_verifier_authority_pk` before the container starts. The service reads the + authority from the ledger at startup and exits if it does not name its own key, so the order + matters. +- **Networks**: the default network (to reach the ledger) *and* the CYOA network, on host ID 250. +- **Client wiring**: every client container gets `DZ_IP_VERIFIER_URL` pointing at the verifier's + **CYOA** address. + +That last pair is the point. The verifier signs the source address it observes the request arrive +from, and `connect` refuses a proof for any address other than the one it is provisioning. A local +client provisions its CYOA address, so the request has to reach the verifier over the CYOA network +for the two to agree — reached over the default network instead, the observed address would be the +client's default-network address and every connect would hard-fail on the mismatch. This is the +same class of problem as the proxy handling in production, where the address the service sees is +the proxy's unless it is configured to read a forwarded one. + +The CYOA subnet is allocated from `9.128.0.0/9`, which is globally routable, so the verifier's +`not_globally_routable` refusal (which an RFC-1918 source would hit) does not fire. + +## Enforcement is off by default + +The `require-ip-ownership-proof` feature flag is **clear** in the local `GlobalState`. A proof is +obtained and attached, but the program accepts a create without one — so a stack where the +verifier is down, or a client that cannot reach it, still connects. That mirrors an environment +whose rollout has not flipped the flag yet. + +To exercise the enforcement path, turn it on: + +```bash +docker exec dz-local-manager \ + doublezero global-config feature-flags set --enable require-ip-ownership-proof +``` + +and off again: + +```bash +docker exec dz-local-manager \ + doublezero global-config feature-flags set --disable require-ip-ownership-proof +``` + +From a Go e2e test, `devnet.SetIPOwnershipProofFeatureFlag(ctx, true)` does the same thing. + +## From a Go e2e test + +Because the verifier is on by default, an ordinary `connect` in any e2e test already obtains and +attaches a real proof. Two knobs cover the cases that need something else: + +- `ClientSpec.NoIPVerifier` leaves `DZ_IP_VERIFIER_URL` unset for one client, so its `connect` + obtains no proof at all — the path an environment takes before its verifier exists. +- `IPVerifierSpec.AuthorityRefreshSecs` pins how often the service re-reads the onchain authority. + Set it long and rotate the authority with `devnet.SetIPVerifierAuthority` and the service keeps + signing with a key `GlobalState` no longer names, which is how a test produces a proof that gets + refused. + +`e2e/ip_ownership_proof_test.go` uses all three paths. + +## Poking at it + +```bash +# Health: 200 once the cached ledger epoch is fresh and the ledger names this key. +docker exec dz-local-ip-verifier curl -sS localhost:8080/health + +# What the ledger thinks the authority is. +docker exec dz-local-manager doublezero global-config authority get + +# A proof, as a client would ask for it. Run through a shell in the container: DZ_IP_VERIFIER_URL +# is set in the client's environment, and `docker exec curl` would have the host shell expand it. +docker exec dz-local-client- bash -c \ + 'curl -sS -X POST "$DZ_IP_VERIFIER_URL/v1/proof" \ + -H "content-type: application/json" \ + -d "{\"payer\":\"\",\"user_type\":0}"' +``` + +The rate limit is raised well above the production default in the devnet (burst 1000, 6000/min): +a devnet has one source address per client and a test can reconnect in a tight loop, which the +production values would turn into `rate_limited` refusals unrelated to what is being tested. diff --git a/e2e/internal/devnet/builder.go b/e2e/internal/devnet/builder.go index 1d596bf434..0ab737aed8 100644 --- a/e2e/internal/devnet/builder.go +++ b/e2e/internal/devnet/builder.go @@ -118,6 +118,12 @@ func BuildContainerImages(ctx context.Context, log *slog.Logger, workspaceDir st dockerfile: filepath.Join(dockerfilesDir, "sentinel", "Dockerfile"), args: append([]string{"--build-arg", baseImageArg}, extraArgs...), }, + { + name: "ip-verifier", + image: os.Getenv("DZ_IP_VERIFIER_IMAGE"), + dockerfile: filepath.Join(dockerfilesDir, "ip-verifier", "Dockerfile"), + args: append([]string{"--build-arg", baseImageArg}, extraArgs...), + }, { name: "validator-metadata-service-mock", image: os.Getenv("DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE"), diff --git a/e2e/internal/devnet/client.go b/e2e/internal/devnet/client.go index d1f53317fb..1e2448fdba 100644 --- a/e2e/internal/devnet/client.go +++ b/e2e/internal/devnet/client.go @@ -61,6 +61,10 @@ type ClientSpec struct { // CYOANetworkIPHostID is the offset into the host portion of the subnet (must be < 2^(32 - prefixLen)). CYOANetworkIPHostID uint32 + // NoIPVerifier leaves DZ_IP_VERIFIER_URL unset for this client, so its `connect` obtains no + // RFC-27 proof even in a devnet running a verifier. + NoIPVerifier bool + // EnableQAAgent starts the QA agent inside the client container for local QA testing. EnableQAAgent bool // QAAgentPort is the port the QA agent listens on inside the container (default: 7009). @@ -234,6 +238,25 @@ func (c *Client) Start(ctx context.Context) error { "DZ_SERVICEABILITY_PROGRAM_ID": c.dn.Manager.ServiceabilityProgramID, "DZ_CLIENT_EXTRA_ARGS": strings.Join(extraArgs, " "), } + // Point `connect` at the devnet verifier. The `--env local` config carries no verifier URL, + // so without this a client obtains no proof at all. The URL is the verifier's CYOA address, + // so the source address it observes is the same one the client binds its tunnel to — + // `connect` hard-fails on a proof for any other address. + // + // NoIPVerifier leaves it unset, which is how a test covers the no-proof path: the CLI reports + // nothing to reach and creates the user without a proof, which the program accepts while + // require-ip-ownership-proof is clear. + if c.dn.IPVerifier != nil && !c.Spec.NoIPVerifier { + // An empty URL here means Prepare never ran, which cannot happen through Devnet.Start — + // it prepares the verifier before any client is added. Refused rather than tolerated: a + // client silently dropped to the no-proof path would leave every test that connects + // still passing while no longer covering the proof, which is the property this whole + // change rests on. + if c.dn.IPVerifier.InternalURL == "" { + return fmt.Errorf("ip-verifier has no internal URL: Prepare has not run") + } + env["DZ_IP_VERIFIER_URL"] = c.dn.IPVerifier.InternalURL + } if c.Spec.EnableQAAgent { env["DZ_QAAGENT_ENABLE"] = "true" env["DZ_QAAGENT_ADDR"] = fmt.Sprintf("0.0.0.0:%d", qaAgentPort) diff --git a/e2e/internal/devnet/cmd/devnet.go b/e2e/internal/devnet/cmd/devnet.go index 169229203f..c57f6bf820 100644 --- a/e2e/internal/devnet/cmd/devnet.go +++ b/e2e/internal/devnet/cmd/devnet.go @@ -90,6 +90,10 @@ func NewLocalDevnet(log *slog.Logger, deployID string) (*LocalDevnet, error) { Verbose: true, Interval: 10 * time.Second, }, + // The RFC-27 verifier is on by default (IPVerifierSpec.Disabled), so `connect` always + // has one to reach. Enforcement is a separate switch: the require-ip-ownership-proof + // feature flag stays clear, so a proof is attached but not demanded. See + // e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md. }, log, dockerClient, subnetAllocator) if err != nil { return nil, fmt.Errorf("failed to create devnet: %w", err) diff --git a/e2e/internal/devnet/devnet.go b/e2e/internal/devnet/devnet.go index f65c75e2de..81b36766aa 100644 --- a/e2e/internal/devnet/devnet.go +++ b/e2e/internal/devnet/devnet.go @@ -41,6 +41,7 @@ const ( containerDoublezeroKeypairPath = "/root/.config/doublezero/id.json" containerSolanaKeypairPath = "/root/.config/solana/id.json" + containerIPVerifierKeypairPath = "/etc/doublezero/ip-verifier.json" // defaultNetworkBaseCIDR is the address range the devnet's default network is allocated from. // It is kept separate from the CYOA network range (9.128.0.0/9) so tests that detect interfaces @@ -94,6 +95,7 @@ type DevnetSpec struct { InfluxDB InfluxDBSpec Prometheus PrometheusSpec Sentinel SentinelSpec + IPVerifier IPVerifierSpec ValidatorMetadataServiceMock ValidatorMetadataServiceMockSpec Devices map[string]DeviceSpec Clients map[string]ClientSpec @@ -129,6 +131,7 @@ type Devnet struct { InfluxDB *InfluxDB Prometheus *Prometheus Sentinel *Sentinel + IPVerifier *IPVerifier ValidatorMetadataServiceMock *ValidatorMetadataServiceMock Devices map[string]*Device Clients map[string]*Client @@ -180,6 +183,10 @@ func (s *DevnetSpec) Validate() error { return fmt.Errorf("prometheus: %w", err) } + if err := s.IPVerifier.Validate(s.CYOANetwork); err != nil { + return fmt.Errorf("ip-verifier: %w", err) + } + if s.Devices == nil { s.Devices = make(map[string]DeviceSpec) } @@ -295,6 +302,33 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA } } + // A cloned-state devnet cannot provision its own verifier authority: its GlobalState came from + // a remote cluster and the local manager cannot write to it, so a generated key would never be + // the one the program trusts and the container would exit at startup. Default the verifier off + // there. A stack that does have a verifier the cloned state already trusts names its keypair + // explicitly, which opts back in. + if spec.SkipProgramDeploy && spec.IPVerifier.KeypairPath == "" { + spec.IPVerifier.Disabled = true + } + + // If the ip-verifier keypair path is not provided, generate a new keypair or use an existing + // one in the deploy directory if it exists. Devnet-only: it is written to the deploy + // directory rather than checked in, and its pubkey is what the local GlobalState names as the + // verifier authority. + if !spec.IPVerifier.Disabled && spec.IPVerifier.KeypairPath == "" { + ipVerifierKeypairPath := filepath.Join(spec.DeployDir, "ip-verifier-keypair.json") + generated, err := generateKeypairIfNotExists(ipVerifierKeypairPath) + if err != nil { + return nil, fmt.Errorf("failed to generate ip-verifier keypair: %w", err) + } + spec.IPVerifier.KeypairPath = ipVerifierKeypairPath + if generated { + log.Debug("--> Generated ip-verifier keypair", "path", ipVerifierKeypairPath) + } else { + log.Debug("--> Using existing ip-verifier keypair", "path", ipVerifierKeypairPath) + } + } + // Validate the spec. if err := spec.Validate(); err != nil { return nil, fmt.Errorf("failed to validate spec: %w", err) @@ -376,6 +410,12 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA dn: dn, log: log.With("component", "sentinel"), } + if !spec.IPVerifier.Disabled { + dn.IPVerifier = &IPVerifier{ + dn: dn, + log: log.With("component", "ip-verifier"), + } + } dn.ValidatorMetadataServiceMock = &ValidatorMetadataServiceMock{ dn: dn, log: log.With("component", "validator-metadata-service-mock"), @@ -535,6 +575,28 @@ func (d *Devnet) Start(ctx context.Context, buildConfig *BuildConfig) error { return fmt.Errorf("failed to create CYOA network: %w", err) } + // Start the ip-verifier if it's not already running. It comes after the CYOA network, which + // it attaches to so that a client's proof request is observed arriving from the same address + // the client binds its tunnel to. Its authority has to be onchain before it starts: the + // service reads GlobalState at startup and exits if the authority is not its own key. + if d.IPVerifier != nil { + if err := d.IPVerifier.Prepare(); err != nil { + return fmt.Errorf("failed to prepare ip-verifier: %w", err) + } + // Skipped for a stack running cloned state: its GlobalState came from a remote cluster + // and the local manager is not its authority, so the write would fail. Reaching here with + // SkipProgramDeploy means the caller named a keypair explicitly — a verifier the cloned + // state already trusts — so the authority is already correct and needs no write. + if !d.Spec.SkipProgramDeploy { + if err := d.SetIPVerifierAuthority(ctx, d.IPVerifier.Pubkey); err != nil { + return fmt.Errorf("failed to set ip-verifier authority: %w", err) + } + } + if _, err := d.IPVerifier.StartIfNotRunning(ctx); err != nil { + return fmt.Errorf("failed to start ip-verifier: %w", err) + } + } + // We don't support starting with devices yet. // The AddDevice method can be used to add devices after the devnet is started. if len(d.Spec.Devices) > 0 { diff --git a/e2e/internal/devnet/ip_verifier.go b/e2e/internal/devnet/ip_verifier.go new file mode 100644 index 0000000000..95d61a07ba --- /dev/null +++ b/e2e/internal/devnet/ip_verifier.go @@ -0,0 +1,358 @@ +package devnet + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "os" + "path/filepath" + "strconv" + "time" + + dockercontainer "github.com/docker/docker/api/types/container" + dockerfilters "github.com/docker/docker/api/types/filters" + dockernetwork "github.com/docker/docker/api/types/network" + "github.com/docker/go-connections/nat" + "github.com/malbeclabs/doublezero/e2e/internal/logging" + "github.com/malbeclabs/doublezero/e2e/internal/netutil" + "github.com/malbeclabs/doublezero/e2e/internal/poll" + "github.com/malbeclabs/doublezero/e2e/internal/solana" + "github.com/testcontainers/testcontainers-go" + tcwait "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + // ipVerifierInternalPort is the container port the proof endpoint listens on. + ipVerifierInternalPort = 8080 + + // defaultIPVerifierCYOANetworkIPHostID is the host offset the verifier takes on the CYOA + // network. Kept well clear of the ranges devices (single digits) and clients (100+) use. + defaultIPVerifierCYOANetworkIPHostID = 250 + + // ipVerifierStartupTimeout is how long the service gets to report healthy, on both the create + // and the restart path. + ipVerifierStartupTimeout = 60 * time.Second +) + +// ipVerifierHealthClient bounds a single health probe. poll.Until calls its condition +// synchronously, so without a per-request timeout a probe that hangs rather than being refused — +// an accepting docker-proxy port with nothing behind it — would block past the startup budget +// until the caller's context died, which is the opposite of the bounded failure waitForHealthy +// exists to give. +var ipVerifierHealthClient = &http.Client{Timeout: 5 * time.Second} + +// IPVerifierSpec configures the RFC-27 IP ownership verification service container. +// +// The service must sit on the CYOA network, not only on the default network. It signs the source +// address it observes the request arrive from, and `connect` refuses a proof for any address other +// than the one it is provisioning — which for a local client is its CYOA address. Reached over the +// default network the observed address would be the client's default-network address instead, and +// every connect would fail on the mismatch. +type IPVerifierSpec struct { + // Disabled leaves the verifier out of the deploy. The zero value runs it, so every devnet + // exercises the same `connect` path production does: a proof is obtained and attached, and + // the program validates it. Enforcement is separate — the require-ip-ownership-proof feature + // flag stays clear, so a create with no proof is still accepted. + Disabled bool + ContainerImage string + // KeypairPath is the host path to the verifier keypair JSON. Generated into the deploy + // directory when unset. Its pubkey is written to GlobalState.ip_verifier_authority_pk before + // the container starts: the service reads the authority from the ledger at startup and exits + // if it does not name its own key. + KeypairPath string + // CYOANetworkIPHostID is the offset into the host portion of the CYOA subnet. + CYOANetworkIPHostID uint32 + // AuthorityRefreshSecs is how often the service re-reads + // GlobalState.ip_verifier_authority_pk. Zero leaves the service default. Set it long to hold + // the service on a stale authority across a rotation, which is how a test produces a proof + // signed by a key the program no longer trusts. + AuthorityRefreshSecs int +} + +func (s *IPVerifierSpec) Validate(cyoaNetworkSpec CYOANetworkSpec) error { + if s.Disabled { + return nil + } + if s.ContainerImage == "" { + s.ContainerImage = os.Getenv("DZ_IP_VERIFIER_IMAGE") + } + if s.CYOANetworkIPHostID == 0 { + s.CYOANetworkIPHostID = defaultIPVerifierCYOANetworkIPHostID + } + // The same bounds the client and device specs enforce: the hostID may not select the network + // (0) or broadcast (max) address. 0 cannot reach here — above it is the "use the default" + // sentinel, not a caller's choice. + hostBits := 32 - cyoaNetworkSpec.CIDRPrefix + maxHostID := uint32((1 << hostBits) - 1) + if s.CYOANetworkIPHostID >= maxHostID { + return fmt.Errorf("hostID %d is out of valid range (1 to %d)", s.CYOANetworkIPHostID, maxHostID-1) + } + if s.KeypairPath != "" && !filepath.IsAbs(s.KeypairPath) { + return fmt.Errorf("keypair path must be an absolute path: %s", s.KeypairPath) + } + return nil +} + +// IPVerifier manages the ip-verifier container. +type IPVerifier struct { + dn *Devnet + log *slog.Logger + + ContainerID string + // Pubkey is the verifier's signing key, and the value written to + // GlobalState.ip_verifier_authority_pk. + Pubkey string + // CYOANetworkIP is the address clients reach the service on. + CYOANetworkIP string + // InternalURL is what a client's DZ_IP_VERIFIER_URL is set to. + InternalURL string +} + +func (v *IPVerifier) dockerContainerHostname() string { + return "ip-verifier" +} + +func (v *IPVerifier) dockerContainerName() string { + return v.dn.Spec.DeployID + "-" + v.dockerContainerHostname() +} + +func (v *IPVerifier) Exists(ctx context.Context) (bool, error) { + containers, err := v.dn.dockerClient.ContainerList(ctx, dockercontainer.ListOptions{ + All: true, + Filters: dockerfilters.NewArgs(dockerfilters.Arg("name", v.dockerContainerName())), + }) + if err != nil { + return false, fmt.Errorf("failed to list containers: %w", err) + } + for _, container := range containers { + if container.Names[0] == "/"+v.dockerContainerName() { + return true, nil + } + } + return false, nil +} + +func (v *IPVerifier) StartIfNotRunning(ctx context.Context) (bool, error) { + exists, err := v.Exists(ctx) + if err != nil { + return false, fmt.Errorf("failed to check if ip-verifier exists: %w", err) + } + if !exists { + return false, v.Start(ctx) + } + + container, err := v.dn.dockerClient.ContainerInspect(ctx, v.dockerContainerName()) + if err != nil { + return false, fmt.Errorf("failed to inspect container: %w", err) + } + started := !container.State.Running + if started { + if err := v.dn.dockerClient.ContainerStart(ctx, container.ID, dockercontainer.StartOptions{}); err != nil { + return false, fmt.Errorf("failed to start ip-verifier: %w", err) + } + } else { + v.log.Debug("--> IPVerifier already running", "container", shortContainerID(container.ID)) + } + + if err := v.setState(container.ID); err != nil { + return false, fmt.Errorf("failed to set ip-verifier state: %w", err) + } + + // A container this call started gets the same readiness gate a fresh one does. `Start` waits + // on /health through testcontainers; without the equivalent here, a verifier that exits on + // startup would leave the devnet reporting success with clients pointed at a dead service. + // That is not hypothetical for this component: the authority can rotate while the container + // is stopped, and the service refuses to start when GlobalState no longer names its key. + if started { + if err := v.waitForHealthy(ctx, container.ID); err != nil { + return false, err + } + } + return started, nil +} + +// waitForHealthy blocks until the container reports healthy, or fails as soon as it is clear it +// never will. /health is 200 only once the cached ledger epoch is fresh and the ledger names this +// container's key, so it proves the same two things the create path's wait strategy does. +func (v *IPVerifier) waitForHealthy(ctx context.Context, containerID string) error { + port, err := v.dn.waitForContainerPortExposed(ctx, containerID, ipVerifierInternalPort, 10*time.Second) + if err != nil { + // Docker drops the port bindings of a container that is no longer running, so the case + // this whole path exists for — a verifier that exits within milliseconds of starting — + // reaches the port wait as a timeout rather than as a published port. Check the state + // before reporting the port, or the diagnostic below is lost exactly where it is wanted. + if exitErr := v.exitedError(ctx, containerID); exitErr != nil { + return exitErr + } + return fmt.Errorf("failed to wait for ip-verifier port: %w", err) + } + url := fmt.Sprintf("http://%s:%d/health", v.dn.ExternalHost, port) + + err = poll.Until(ctx, func() (bool, error) { + // A container that has already exited is never going to answer, so say why now rather + // than spending the whole budget waiting on it. + if exitErr := v.exitedError(ctx, containerID); exitErr != nil { + return false, exitErr + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return false, err + } + resp, err := ipVerifierHealthClient.Do(req) + if err != nil { + return false, nil + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK, nil + }, ipVerifierStartupTimeout, time.Second) + if err != nil { + return fmt.Errorf("ip-verifier did not become healthy: %w", err) + } + return nil +} + +// exitedError reports why the container is gone, or nil if it is still running. An inspect that +// fails is itself reported: at this point the caller is already on an error path and the inspect +// result is what decides which error it gets. +func (v *IPVerifier) exitedError(ctx context.Context, containerID string) error { + container, err := v.dn.dockerClient.ContainerInspect(ctx, containerID) + if err != nil { + return fmt.Errorf("failed to inspect container: %w", err) + } + if container.State.Running { + return nil + } + // Most often the authority moved while this container was stopped: the service exits when + // GlobalState no longer names its key. Named as the likely cause, not the only one — + // `docker logs` on the container has the actual reason. + return fmt.Errorf("ip-verifier exited with code %d (check `docker logs %s`); "+ + "the usual cause is GlobalState.ip_verifier_authority_pk no longer naming its key %s", + container.State.ExitCode, v.dockerContainerName(), v.Pubkey) +} + +// Prepare reads the verifier keypair and derives the addresses the container will use, without +// starting it. The pubkey is needed onchain before the container starts, because the service +// treats an authority that is not its own key as a startup error. +func (v *IPVerifier) Prepare() error { + keypairJSON, err := os.ReadFile(v.dn.Spec.IPVerifier.KeypairPath) + if err != nil { + return fmt.Errorf("failed to read ip-verifier keypair: %w", err) + } + pubkey, err := solana.PubkeyFromKeypairJSON(keypairJSON) + if err != nil { + return fmt.Errorf("failed to parse ip-verifier pubkey: %w", err) + } + v.Pubkey = pubkey + + cyoaIP, err := netutil.DeriveIPFromCIDR(v.dn.CYOANetwork.SubnetCIDR, v.dn.Spec.IPVerifier.CYOANetworkIPHostID) + if err != nil { + return fmt.Errorf("failed to derive CYOA network IP: %w", err) + } + v.CYOANetworkIP = cyoaIP.To4().String() + v.InternalURL = fmt.Sprintf("http://%s:%d", v.CYOANetworkIP, ipVerifierInternalPort) + return nil +} + +func (v *IPVerifier) Start(ctx context.Context) error { + v.log.Debug("==> Starting ip-verifier", "image", v.dn.Spec.IPVerifier.ContainerImage) + + if err := v.Prepare(); err != nil { + return err + } + + env := map[string]string{ + "DZ_IP_VERIFIER_ENV": "local", + // Named explicitly rather than left to the environment's constant: a devnet deploys its + // own serviceability program whenever no keypair is pinned, and against the constant the + // GlobalState PDA simply does not exist — the startup authority check then finds nothing + // to compare against, only warns, and /health still answers 200. The invariant this file + // is built around (waiting on /health proves the ledger names this container's key) needs + // the program the devnet actually deployed. + "DZ_IP_VERIFIER_SERVICEABILITY_PROGRAM_ID": v.dn.Manager.ServiceabilityProgramID, + "DZ_IP_VERIFIER_LEDGER_RPC": v.dn.Ledger.InternalRPCURL, + "DZ_IP_VERIFIER_KEYPAIR": containerIPVerifierKeypairPath, + "DZ_IP_VERIFIER_LISTEN_ADDR": fmt.Sprintf("0.0.0.0:%d", ipVerifierInternalPort), + "DZ_IP_VERIFIER_LOG": "doublezero_ip_verifier=debug", + // No trusted proxies: clients reach the service directly, so the connection peer + // address is signed and forwarded headers are ignored outright. + // + // The rate limit is raised well above the production default because a devnet has one + // source address per client and a test can reconnect in a tight loop; the production + // value would turn that into `rate_limited` refusals that have nothing to do with what + // is being tested. + "DZ_IP_VERIFIER_RATE_LIMIT_BURST": "1000", + "DZ_IP_VERIFIER_RATE_LIMIT_PER_MINUTE": "6000", + } + // A long refresh holds the service on the authority it read at startup, so a rotation after + // it is up does not stop it signing. That is what lets a test hand the program a proof signed + // by a key it no longer trusts. + if v.dn.Spec.IPVerifier.AuthorityRefreshSecs > 0 { + env["DZ_IP_VERIFIER_AUTHORITY_REFRESH_SECS"] = strconv.Itoa(v.dn.Spec.IPVerifier.AuthorityRefreshSecs) + } + + req := testcontainers.ContainerRequest{ + Image: v.dn.Spec.IPVerifier.ContainerImage, + Name: v.dockerContainerName(), + ConfigModifier: func(cfg *dockercontainer.Config) { + cfg.Hostname = v.dockerContainerHostname() + }, + ExposedPorts: []string{fmt.Sprintf("%d/tcp", ipVerifierInternalPort)}, + Env: env, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: v.dn.Spec.IPVerifier.KeypairPath, + ContainerFilePath: containerIPVerifierKeypairPath, + }, + }, + Networks: []string{ + v.dn.DefaultNetwork.Name, + v.dn.CYOANetwork.Name, + }, + NetworkAliases: map[string][]string{ + v.dn.DefaultNetwork.Name: {"ip-verifier"}, + }, + EndpointSettingsModifier: func(m map[string]*dockernetwork.EndpointSettings) { + if m[v.dn.CYOANetwork.Name] == nil { + m[v.dn.CYOANetwork.Name] = &dockernetwork.EndpointSettings{} + } + m[v.dn.CYOANetwork.Name].IPAddress = v.CYOANetworkIP + m[v.dn.CYOANetwork.Name].IPAMConfig = &dockernetwork.EndpointIPAMConfig{ + IPv4Address: v.CYOANetworkIP, + } + }, + // /health is 200 only once the cached ledger epoch is fresh *and* the ledger names this + // container's key as the verifier authority, so waiting on it proves both. + WaitingFor: tcwait.ForHTTP("/health"). + WithPort(nat.Port(fmt.Sprintf("%d/tcp", ipVerifierInternalPort))). + WithStartupTimeout(ipVerifierStartupTimeout). + WithPollInterval(1 * time.Second), + Resources: dockercontainer.Resources{ + NanoCPUs: defaultContainerNanoCPUs, + Memory: defaultContainerMemory, + }, + Labels: v.dn.labels, + } + + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + Logger: logging.NewTestcontainersAdapter(v.log), + }) + if err != nil { + return fmt.Errorf("failed to start ip-verifier: %w", err) + } + + if err := v.setState(container.GetContainerID()); err != nil { + return fmt.Errorf("failed to set ip-verifier state: %w", err) + } + + v.log.Debug("--> IPVerifier started", "container", v.ContainerID, "pubkey", v.Pubkey, "url", v.InternalURL) + return nil +} + +func (v *IPVerifier) setState(containerID string) error { + v.ContainerID = shortContainerID(containerID) + return v.Prepare() +} diff --git a/e2e/internal/devnet/ip_verifier_test.go b/e2e/internal/devnet/ip_verifier_test.go new file mode 100644 index 0000000000..83950a071c --- /dev/null +++ b/e2e/internal/devnet/ip_verifier_test.go @@ -0,0 +1,66 @@ +package devnet + +import "testing" + +func TestIPVerifierSpecValidate(t *testing.T) { + cyoa := CYOANetworkSpec{CIDRPrefix: 24} + + t.Run("disabled spec is left alone", func(t *testing.T) { + // Nothing is defaulted for a disabled verifier, so a devnet that does not run one never + // derives a CYOA address for it and never points clients anywhere. + spec := IPVerifierSpec{Disabled: true} + if err := spec.Validate(cyoa); err != nil { + t.Fatalf("Validate() = %v, want nil", err) + } + if spec.CYOANetworkIPHostID != 0 { + t.Errorf("CYOANetworkIPHostID = %d, want 0", spec.CYOANetworkIPHostID) + } + }) + + t.Run("defaults the CYOA host ID", func(t *testing.T) { + // The zero value runs the verifier: every devnet gets one unless it opts out. + spec := IPVerifierSpec{} + if err := spec.Validate(cyoa); err != nil { + t.Fatalf("Validate() = %v, want nil", err) + } + if spec.CYOANetworkIPHostID != defaultIPVerifierCYOANetworkIPHostID { + t.Errorf("CYOANetworkIPHostID = %d, want %d", spec.CYOANetworkIPHostID, defaultIPVerifierCYOANetworkIPHostID) + } + }) + + t.Run("rejects a host ID outside the subnet", func(t *testing.T) { + // 256 does not fit a /24. Catching it here beats a Docker IPAM error at start time. + spec := IPVerifierSpec{CYOANetworkIPHostID: 256} + if err := spec.Validate(cyoa); err == nil { + t.Fatal("Validate() = nil, want an out-of-range error") + } + }) + + t.Run("rejects the broadcast host ID", func(t *testing.T) { + // The bound the client and device specs use. 255 is the broadcast host ID of a /24, and + // an earlier version of this check accepted it while its error message claimed otherwise. + spec := IPVerifierSpec{CYOANetworkIPHostID: 255} + if err := spec.Validate(cyoa); err == nil { + t.Fatal("Validate() = nil, want the broadcast address rejected") + } + }) + + t.Run("accepts the last usable host ID", func(t *testing.T) { + // The other side of that bound: 254 must still pass, so the fix did not go one too far. + spec := IPVerifierSpec{CYOANetworkIPHostID: 254} + if err := spec.Validate(cyoa); err != nil { + t.Fatalf("Validate() = %v, want nil", err) + } + if spec.CYOANetworkIPHostID != 254 { + t.Errorf("CYOANetworkIPHostID = %d, want it left alone", spec.CYOANetworkIPHostID) + } + }) + + t.Run("rejects a relative keypair path", func(t *testing.T) { + // The path is handed to Docker as a host mount source, which only resolves absolutely. + spec := IPVerifierSpec{KeypairPath: "ip-verifier-keypair.json"} + if err := spec.Validate(cyoa); err == nil { + t.Fatal("Validate() = nil, want an absolute-path error") + } + }) +} diff --git a/e2e/internal/devnet/smartcontract_init.go b/e2e/internal/devnet/smartcontract_init.go index 90145ea345..abc37bdb6d 100644 --- a/e2e/internal/devnet/smartcontract_init.go +++ b/e2e/internal/devnet/smartcontract_init.go @@ -161,3 +161,49 @@ func (dn *Devnet) SetOnchainAllocationFeatureFlag(ctx context.Context) error { dn.log.Debug("--> Onchain-allocation feature flag enabled") return nil } + +// SetIPVerifierAuthority writes the RFC-27 verifier pubkey to +// GlobalState.ip_verifier_authority_pk. Must run before the ip-verifier container starts: the +// service reads the authority from the ledger at startup and refuses to serve if it does not name +// its own key. +func (dn *Devnet) SetIPVerifierAuthority(ctx context.Context, pubkey string) error { + dn.log.Debug("==> Setting ip-verifier authority", "pubkey", pubkey) + + dn.onchainWriteMutex.Lock() + defer dn.onchainWriteMutex.Unlock() + + _, err := dn.Manager.Exec(ctx, []string{ + "doublezero", "global-config", "authority", "set", "--ip-verifier-authority", pubkey, + }) + if err != nil { + return fmt.Errorf("failed to set ip-verifier authority: %w", err) + } + + dn.log.Debug("--> Ip-verifier authority set", "pubkey", pubkey) + return nil +} + +// SetIPOwnershipProofFeatureFlag turns RFC-27 enforcement on or off in GlobalState. Off is the +// local default: with it clear a user is created whether or not a proof is attached, so a devnet +// with no reachable verifier still works. Turning it on makes the program reject a create without +// a valid proof, which is what a test of the enforcement path wants. +func (dn *Devnet) SetIPOwnershipProofFeatureFlag(ctx context.Context, enable bool) error { + toggle := "--disable" + if enable { + toggle = "--enable" + } + dn.log.Debug("==> Setting require-ip-ownership-proof feature flag", "enable", enable) + + dn.onchainWriteMutex.Lock() + defer dn.onchainWriteMutex.Unlock() + + _, err := dn.Manager.Exec(ctx, []string{ + "doublezero", "global-config", "feature-flags", "set", toggle, "require-ip-ownership-proof", + }) + if err != nil { + return fmt.Errorf("failed to set require-ip-ownership-proof feature flag: %w", err) + } + + dn.log.Debug("--> Require-ip-ownership-proof feature flag set", "enable", enable) + return nil +} diff --git a/e2e/ip_ownership_proof_test.go b/e2e/ip_ownership_proof_test.go new file mode 100644 index 0000000000..39b0ff724e --- /dev/null +++ b/e2e/ip_ownership_proof_test.go @@ -0,0 +1,214 @@ +//go:build e2e + +package e2e_test + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/malbeclabs/doublezero/e2e/internal/devnet" + "github.com/malbeclabs/doublezero/e2e/internal/random" + "github.com/malbeclabs/doublezero/e2e/internal/solana" + "github.com/stretchr/testify/require" +) + +// RFC-27 IP ownership proofs, end to end: `connect` asks the verification service for a proof, +// attaches it to the user creation, and the program validates it through the Ed25519 precompile. +// +// Every e2e devnet runs a verifier (IPVerifierSpec.Disabled defaults to false), so the ordinary +// connect path in every other test already carries a real proof. What is left to cover here is +// the three outcomes that path can have, which no other test distinguishes: +// +// - a proof the program accepts, +// - no proof at all, which the program also accepts while require-ip-ownership-proof is clear, +// - a proof signed by a key the program does not trust, which it must reject. + +// A valid proof is obtained and attached, and the user is created. +func TestE2E_IPOwnershipProof_ValidProof(t *testing.T) { + t.Parallel() + + dn, _, client, log := setupIPProofDevnet(t, devnet.IPVerifierSpec{}, devnet.ClientSpec{ + CYOANetworkIPHostID: 100, + }) + + out := connectIBRLForProofTest(t, log, dn, client) + + // The verifier signed the address the client is provisioning. `connect` prints this only + // after checking that the address the service observed matches the one being bound, so it is + // evidence of the whole round trip, not just of a reachable service. + require.Contains(t, out, "IP ownership verified for "+client.CYOANetworkIP, + "connect must obtain a proof for the address it is provisioning") + require.Contains(t, out, "✅ User Provisioned") + + // No proof would also have been accepted here, so assert the run did not quietly fall back. + require.NotContains(t, out, "Continuing without an IP ownership proof") + + require.NoError(t, client.WaitForTunnelUp(t.Context(), 90*time.Second), + "a user created with a proof must still come up normally") +} + +// No verifier to reach, no proof, and the create is accepted anyway: enforcement is gated on the +// require-ip-ownership-proof feature flag, which is clear in a local devnet. This is the path a +// deployed environment takes before its verifier exists, so it has to keep working. +func TestE2E_IPOwnershipProof_NoProof(t *testing.T) { + t.Parallel() + + // The devnet still runs a verifier; this client is simply not pointed at it, which is what a + // client with no configured verifier looks like. + dn, _, client, log := setupIPProofDevnet(t, devnet.IPVerifierSpec{}, devnet.ClientSpec{ + CYOANetworkIPHostID: 100, + NoIPVerifier: true, + }) + + out := connectIBRLForProofTest(t, log, dn, client) + + require.NotContains(t, out, "IP ownership verified for", + "a client with no verifier configured must not report a proof") + require.Contains(t, out, "✅ User Provisioned", + "a create without a proof must be accepted while require-ip-ownership-proof is clear") + + require.NoError(t, client.WaitForTunnelUp(t.Context(), 90*time.Second)) +} + +// A proof signed by a key the program does not trust must be rejected. +// +// The rotation is what makes the proof invalid. The verifier re-reads +// GlobalState.ip_verifier_authority_pk periodically and stops serving when it no longer names its +// own key, so the refresh is pinned long enough that it does not notice: it keeps signing with the +// key it started with while GlobalState names a different one. `connect` reads the verifier key +// from GlobalState — the proof does not carry it — and finds the signature does not verify +// against it. +// +// Where the refusal lands: the SDK checks the proof against the onchain verifier key before it +// builds the transaction, so this fails client-side rather than onchain. That is the intended +// design — the "refused before the transaction is paid for" pre-flight — but it does mean this +// test does not reach the program's own Ed25519 precompile check. Covering that would take a +// client that skips the pre-flight, which `connect` gives no way to do; the program-side check +// has unit coverage in the serviceability program. +func TestE2E_IPOwnershipProof_UntrustedSigner(t *testing.T) { + t.Parallel() + + dn, _, client, log := setupIPProofDevnet(t, devnet.IPVerifierSpec{ + // Comfortably longer than the test: the service must not observe the rotation. + AuthorityRefreshSecs: 3600, + }, devnet.ClientSpec{ + CYOANetworkIPHostID: 100, + }) + + // Rotate the trust root out from under the running verifier. + keypairJSON, err := solana.GenerateKeypairJSON() + require.NoError(t, err) + rotated, err := solana.PubkeyFromKeypairJSON(keypairJSON) + require.NoError(t, err) + require.NotEqual(t, dn.IPVerifier.Pubkey, rotated) + + log.Info("==> Rotating the verifier authority away from the running service", + "from", dn.IPVerifier.Pubkey, "to", rotated) + require.NoError(t, dn.SetIPVerifierAuthority(t.Context(), rotated)) + + setAccessPass(t, dn, client) + + log.Info("==> Connecting with a proof signed by the pre-rotation key") + out, err := client.Exec(t.Context(), []string{"bash", "-c", "doublezero connect ibrl 2>&1"}) + output := string(out) + log.Info("==> Connect output", "output", output) + + // The verifier still answers — it has not re-read the authority — so a proof is obtained and + // attached. It is the transaction that must fail. + require.Contains(t, output, "IP ownership verified for "+client.CYOANetworkIP, + "the service should still be issuing proofs; if it is not, the rotation was noticed and "+ + "this test is no longer covering an untrusted signature") + require.Error(t, err, "a proof signed by a key GlobalState does not name must not create a user") + require.Contains(t, output, "does not verify against the onchain verifier "+rotated, + "the refusal must name the rotated key it checked against") + require.NotContains(t, output, "✅ User Provisioned") + + // And nothing landed onchain: a rejected transaction must not leave a half-created user. + users, err := dn.Manager.Exec(t.Context(), []string{"bash", "-c", "doublezero user list"}) + require.NoError(t, err) + require.NotContains(t, string(users), client.CYOANetworkIP, + "no user may exist for a client whose creation was rejected") +} + +// setAccessPass grants the client a prepaid pass, which every connect needs before it gets as far +// as the proof. +func setAccessPass(t *testing.T, dn *devnet.Devnet, client *devnet.Client) { + t.Helper() + _, err := dn.Manager.Exec(t.Context(), []string{"bash", "-c", + "doublezero access-pass set --accesspass-type prepaid --epochs max --client-ip " + + client.CYOANetworkIP + " --user-payer " + client.Pubkey}) + require.NoError(t, err) +} + +func connectIBRLForProofTest(t *testing.T, log *slog.Logger, dn *devnet.Devnet, client *devnet.Client) string { + t.Helper() + + setAccessPass(t, dn, client) + + log.Info("==> Connecting IBRL") + out, err := client.Exec(t.Context(), []string{"bash", "-c", "doublezero connect ibrl 2>&1"}) + log.Info("==> Connect output", "output", string(out)) + require.NoError(t, err, "connect failed: %s", string(out)) + return string(out) +} + +// setupIPProofDevnet builds the smallest devnet these tests need: one device, one client, and the +// verifier the spec asks for. +func setupIPProofDevnet(t *testing.T, verifier devnet.IPVerifierSpec, clientSpec devnet.ClientSpec) (*devnet.Devnet, *devnet.Device, *devnet.Client, *slog.Logger) { + deployID := "dz-e2e-" + t.Name() + "-" + random.ShortID() + log := logger.With("test", t.Name(), "deployID", deployID) + + currentDir, err := os.Getwd() + require.NoError(t, err) + serviceabilityProgramKeypairPath := filepath.Join(currentDir, "data", "serviceability-program-keypair.json") + + dn, err := devnet.New(devnet.DevnetSpec{ + DeployID: deployID, + DeployDir: t.TempDir(), + + CYOANetwork: devnet.CYOANetworkSpec{ + CIDRPrefix: subnetCIDRPrefix, + }, + Manager: devnet.ManagerSpec{ + ServiceabilityProgramKeypairPath: serviceabilityProgramKeypairPath, + }, + IPVerifier: verifier, + }, log, dockerClient, subnetAllocator) + require.NoError(t, err) + + log.Info("==> Starting devnet") + require.NoError(t, dn.Start(t.Context(), nil)) + + require.NotNil(t, dn.IPVerifier, "the devnet must run a verifier for these tests") + log.Info("--> IP verifier running", "pubkey", dn.IPVerifier.Pubkey, "url", dn.IPVerifier.InternalURL) + + device, err := dn.AddDevice(t.Context(), devnet.DeviceSpec{ + Code: "ny5-dz01", + Location: "ewr", + Exchange: "xewr", + CYOANetworkIPHostID: 8, + CYOANetworkAllocatablePrefix: 29, + }) + require.NoError(t, err) + + _, err = dn.Manager.Exec(t.Context(), []string{"bash", "-c", ` + set -euo pipefail + doublezero device interface create ny5-dz01 "Ethernet2" --bandwidth 10G -w + doublezero device interface create ny5-dz01 "Loopback255" --loopback-type vpnv4 --bandwidth 10G -w + doublezero device interface create ny5-dz01 "Loopback256" --loopback-type ipv4 --bandwidth 10G -w + `}) + require.NoError(t, err) + + client, err := dn.AddClient(t.Context(), clientSpec) + require.NoError(t, err) + log.Info("--> Client added", "clientIP", client.CYOANetworkIP, "pubkey", client.Pubkey) + + // The client picks a device from its own latency measurements; connecting before they exist + // fails on endpoint selection rather than on anything this test is about. + require.NoError(t, client.WaitForLatencyResults(t.Context(), device.ID, 75*time.Second)) + + return dn, device, client, log +}