From d616cd1f937699f0503c35231324f06c0b0d2a08 Mon Sep 17 00:00:00 2001 From: Greg Mitchell Date: Tue, 25 Aug 2026 16:11:29 +0000 Subject: [PATCH 1/5] dev/dzctl: run an ip-verifier in the local devnet RFC-27 has connect obtain an IP ownership proof, so the local devnet needs a verifier to reach or the flow diverges from production. dzctl start now brings up a dz-local-ip-verifier container with a keypair generated per deploy, writes its pubkey to GlobalState.ip_verifier_authority_pk before the container starts (the service exits if the ledger names another key), and points every client at it. The container sits on the CYOA network, not only the default network, and clients are pointed at its CYOA address. The service signs the source address it observes and connect refuses a proof for any other address than the one it is provisioning, which for a local client is its CYOA address; reached over the default network the two would never agree. Same class of problem as the proxy handling in production. Enforcement stays off: the require-ip-ownership-proof feature flag is clear by default locally, so a proof is attached but not demanded. --- .github/workflows/e2e.yml | 2 + CLAUDE.md | 1 + e2e/.env.local | 1 + e2e/docker/base.dockerfile | 1 + e2e/docker/ip-verifier/Dockerfile | 13 ++ e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md | 75 +++++++ e2e/internal/devnet/builder.go | 6 + e2e/internal/devnet/client.go | 7 + e2e/internal/devnet/cmd/devnet.go | 6 + e2e/internal/devnet/devnet.go | 52 +++++ e2e/internal/devnet/ip_verifier.go | 244 ++++++++++++++++++++++ e2e/internal/devnet/ip_verifier_test.go | 45 ++++ e2e/internal/devnet/smartcontract_init.go | 46 ++++ 13 files changed, 499 insertions(+) create mode 100644 e2e/docker/ip-verifier/Dockerfile create mode 100644 e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md create mode 100644 e2e/internal/devnet/ip_verifier.go create mode 100644 e2e/internal/devnet/ip_verifier_test.go diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3bc2ef7d8c..bb2eb88ebc 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -43,6 +43,7 @@ env: DZ_DEVICE_HEALTH_ORACLE_IMAGE: ghcr.io/malbeclabs/dz-e2e/device-health-oracle:${{ github.event.inputs.image_tag || github.sha }} DZ_GEOPROBE_IMAGE: ghcr.io/malbeclabs/dz-e2e/geoprobe:${{ github.event.inputs.image_tag || github.sha }} DZ_SENTINEL_IMAGE: ghcr.io/malbeclabs/dz-e2e/sentinel:${{ github.event.inputs.image_tag || github.sha }} + DZ_IP_VERIFIER_IMAGE: ghcr.io/malbeclabs/dz-e2e/ip-verifier:${{ github.event.inputs.image_tag || github.sha }} DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE: ghcr.io/malbeclabs/dz-e2e/validator-metadata-service-mock:${{ github.event.inputs.image_tag || github.sha }} jobs: @@ -291,6 +292,7 @@ jobs: docker push ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }} docker push ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }} docker push ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }} + docker push ${{ env.DZ_IMAGE_REPO }}/ip-verifier:${{ env.DZ_IMAGE_TAG }} docker push ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }} - name: Discover tests and distribute across shards if: steps.gate.outputs.run-e2e == 'true' diff --git a/CLAUDE.md b/CLAUDE.md index 0c0ae926fd..e8a7dd7a93 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,6 +147,7 @@ The local devnet runs in Docker containers with the naming convention `dz-local- - **Clients**: `dz-local-client-{pubkey}` - Client containers running doublezerod - **Manager**: `dz-local-manager` - Runs the doublezero CLI for admin operations - **Controller**: `dz-local-controller` - Pushes configs to devices +- **IP verifier**: `dz-local-ip-verifier` - Signs RFC-27 IP ownership proofs for `connect` (see `e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md`) ### Arista Device Interaction diff --git a/e2e/.env.local b/e2e/.env.local index f3390d153b..f094feb134 100644 --- a/e2e/.env.local +++ b/e2e/.env.local @@ -13,3 +13,4 @@ DZ_DEVICE_HEALTH_ORACLE_IMAGE=${DZ_IMAGE_REPO}/device-health-oracle:${DZ_IMAGE_T DZ_GEOPROBE_IMAGE=${DZ_IMAGE_REPO}/geoprobe:${DZ_IMAGE_TAG} DZ_SENTINEL_IMAGE=${DZ_IMAGE_REPO}/sentinel:${DZ_IMAGE_TAG} DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE=${DZ_IMAGE_REPO}/validator-metadata-service-mock:${DZ_IMAGE_TAG} +DZ_IP_VERIFIER_IMAGE=${DZ_IMAGE_REPO}/ip-verifier:${DZ_IMAGE_TAG} diff --git a/e2e/docker/base.dockerfile b/e2e/docker/base.dockerfile index d5ce5329b9..8b37b064e6 100644 --- a/e2e/docker/base.dockerfile +++ b/e2e/docker/base.dockerfile @@ -122,6 +122,7 @@ RUN --mount=type=cache,id=cargo-${CARGO_LOCK_HASH},target=/cargo \ RUSTFLAGS="-C link-arg=-fuse-ld=mold" cargo build --workspace --release --exclude doublezero-serviceability --exclude doublezero-telemetry && \ cp /target/release/doublezero ${BIN_DIR}/ && \ cp /target/release/doublezero-sentinel ${BIN_DIR}/ && \ + cp /target/release/doublezero-ip-verifier ${BIN_DIR}/ && \ cp /target/release/fork-accounts ${BIN_DIR}/ # Force COPY in later stages to always copy the binaries, even if they appear to be the same. diff --git a/e2e/docker/ip-verifier/Dockerfile b/e2e/docker/ip-verifier/Dockerfile new file mode 100644 index 0000000000..125fa066a1 --- /dev/null +++ b/e2e/docker/ip-verifier/Dockerfile @@ -0,0 +1,13 @@ +ARG BASE_IMAGE=undefined +FROM ${BASE_IMAGE} AS base + +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y ca-certificates curl bash + +COPY --from=base /doublezero/bin/doublezero-ip-verifier /usr/local/bin/doublezero-ip-verifier + +ENTRYPOINT ["doublezero-ip-verifier"] diff --git a/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md b/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md new file mode 100644 index 0000000000..bde0f8714c --- /dev/null +++ b/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md @@ -0,0 +1,75 @@ +# IP ownership verification in the local devnet + +RFC-27 ([`rfcs/rfc27-ip-verification.md`](../../rfcs/rfc27-ip-verification.md)) has `connect` +attach a proof, signed by a DoubleZero-operated verifier, that the caller can originate traffic +from the `client_ip` it is binding. `dev/dzctl` runs that verifier so the local flow matches +production. + +## What comes up + +`dzctl start` brings up a `dz-local-ip-verifier` container (image `dz-local/ip-verifier:dev`) +alongside the rest of the stack: + +- **Keypair**: generated per deploy into `dev/.deploy/dz-local/ip-verifier-keypair.json`. Devnet + only — nothing is checked in. +- **Onchain authority**: the keypair's pubkey is written to + `GlobalState.ip_verifier_authority_pk` before the container starts. The service reads the + authority from the ledger at startup and exits if it does not name its own key, so the order + matters. +- **Networks**: the default network (to reach the ledger) *and* the CYOA network, on host ID 250. +- **Client wiring**: every client container gets `DZ_IP_VERIFIER_URL` pointing at the verifier's + **CYOA** address. + +That last pair is the point. The verifier signs the source address it observes the request arrive +from, and `connect` refuses a proof for any address other than the one it is provisioning. A local +client provisions its CYOA address, so the request has to reach the verifier over the CYOA network +for the two to agree — reached over the default network instead, the observed address would be the +client's default-network address and every connect would hard-fail on the mismatch. This is the +same class of problem as the proxy handling in production, where the address the service sees is +the proxy's unless it is configured to read a forwarded one. + +The CYOA subnet is allocated from `9.128.0.0/9`, which is globally routable, so the verifier's +`not_globally_routable` refusal (which an RFC-1918 source would hit) does not fire. + +## Enforcement is off by default + +The `require-ip-ownership-proof` feature flag is **clear** in the local `GlobalState`. A proof is +obtained and attached, but the program accepts a create without one — so a stack where the +verifier is down, or a client that cannot reach it, still connects. That mirrors an environment +whose rollout has not flipped the flag yet. + +To exercise the enforcement path, turn it on: + +```bash +docker exec dz-local-manager \ + doublezero global-config feature-flags set --enable require-ip-ownership-proof +``` + +and off again: + +```bash +docker exec dz-local-manager \ + doublezero global-config feature-flags set --disable require-ip-ownership-proof +``` + +From a Go e2e test, `devnet.SetIPOwnershipProofFeatureFlag(ctx, true)` does the same thing. + +## Poking at it + +```bash +# Health: 200 once the cached ledger epoch is fresh and the ledger names this key. +docker exec dz-local-ip-verifier curl -sS localhost:8080/health + +# What the ledger thinks the authority is. +docker exec dz-local-manager doublezero global-config authority get + +# A proof, as a client would ask for it. +docker exec dz-local-client- \ + curl -sS -X POST "$DZ_IP_VERIFIER_URL/v1/proof" \ + -H 'content-type: application/json' \ + -d '{"payer":"","user_type":0}' +``` + +The rate limit is raised well above the production default in the devnet (burst 1000, 6000/min): +a devnet has one source address per client and a test can reconnect in a tight loop, which the +production values would turn into `rate_limited` refusals unrelated to what is being tested. diff --git a/e2e/internal/devnet/builder.go b/e2e/internal/devnet/builder.go index 1d596bf434..0ab737aed8 100644 --- a/e2e/internal/devnet/builder.go +++ b/e2e/internal/devnet/builder.go @@ -118,6 +118,12 @@ func BuildContainerImages(ctx context.Context, log *slog.Logger, workspaceDir st dockerfile: filepath.Join(dockerfilesDir, "sentinel", "Dockerfile"), args: append([]string{"--build-arg", baseImageArg}, extraArgs...), }, + { + name: "ip-verifier", + image: os.Getenv("DZ_IP_VERIFIER_IMAGE"), + dockerfile: filepath.Join(dockerfilesDir, "ip-verifier", "Dockerfile"), + args: append([]string{"--build-arg", baseImageArg}, extraArgs...), + }, { name: "validator-metadata-service-mock", image: os.Getenv("DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE"), diff --git a/e2e/internal/devnet/client.go b/e2e/internal/devnet/client.go index d1f53317fb..e9c2e90a6d 100644 --- a/e2e/internal/devnet/client.go +++ b/e2e/internal/devnet/client.go @@ -234,6 +234,13 @@ func (c *Client) Start(ctx context.Context) error { "DZ_SERVICEABILITY_PROGRAM_ID": c.dn.Manager.ServiceabilityProgramID, "DZ_CLIENT_EXTRA_ARGS": strings.Join(extraArgs, " "), } + // Point `connect` at the devnet verifier. Without this the `--env local` default + // (http://localhost:8080) is used, which resolves to nothing inside the client container. + // The URL is the verifier's CYOA address, so the source address it observes is the same one + // the client binds its tunnel to — `connect` hard-fails on a proof for any other address. + if c.dn.IPVerifier != nil && c.dn.IPVerifier.InternalURL != "" { + env["DZ_IP_VERIFIER_URL"] = c.dn.IPVerifier.InternalURL + } if c.Spec.EnableQAAgent { env["DZ_QAAGENT_ENABLE"] = "true" env["DZ_QAAGENT_ADDR"] = fmt.Sprintf("0.0.0.0:%d", qaAgentPort) diff --git a/e2e/internal/devnet/cmd/devnet.go b/e2e/internal/devnet/cmd/devnet.go index 169229203f..514b25055a 100644 --- a/e2e/internal/devnet/cmd/devnet.go +++ b/e2e/internal/devnet/cmd/devnet.go @@ -90,6 +90,12 @@ func NewLocalDevnet(log *slog.Logger, deployID string) (*LocalDevnet, error) { Verbose: true, Interval: 10 * time.Second, }, + // The RFC-27 verifier runs in the local devnet so `connect` always has one to reach. + // Enforcement is a separate switch: the require-ip-ownership-proof feature flag stays + // clear by default, so a proof is attached but not demanded. See e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md. + IPVerifier: devnet.IPVerifierSpec{ + Enabled: true, + }, }, log, dockerClient, subnetAllocator) if err != nil { return nil, fmt.Errorf("failed to create devnet: %w", err) diff --git a/e2e/internal/devnet/devnet.go b/e2e/internal/devnet/devnet.go index f65c75e2de..2b5246dca5 100644 --- a/e2e/internal/devnet/devnet.go +++ b/e2e/internal/devnet/devnet.go @@ -41,6 +41,7 @@ const ( containerDoublezeroKeypairPath = "/root/.config/doublezero/id.json" containerSolanaKeypairPath = "/root/.config/solana/id.json" + containerIPVerifierKeypairPath = "/etc/doublezero/ip-verifier.json" // defaultNetworkBaseCIDR is the address range the devnet's default network is allocated from. // It is kept separate from the CYOA network range (9.128.0.0/9) so tests that detect interfaces @@ -94,6 +95,7 @@ type DevnetSpec struct { InfluxDB InfluxDBSpec Prometheus PrometheusSpec Sentinel SentinelSpec + IPVerifier IPVerifierSpec ValidatorMetadataServiceMock ValidatorMetadataServiceMockSpec Devices map[string]DeviceSpec Clients map[string]ClientSpec @@ -129,6 +131,7 @@ type Devnet struct { InfluxDB *InfluxDB Prometheus *Prometheus Sentinel *Sentinel + IPVerifier *IPVerifier ValidatorMetadataServiceMock *ValidatorMetadataServiceMock Devices map[string]*Device Clients map[string]*Client @@ -180,6 +183,10 @@ func (s *DevnetSpec) Validate() error { return fmt.Errorf("prometheus: %w", err) } + if err := s.IPVerifier.Validate(s.CYOANetwork); err != nil { + return fmt.Errorf("ip-verifier: %w", err) + } + if s.Devices == nil { s.Devices = make(map[string]DeviceSpec) } @@ -295,6 +302,24 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA } } + // If the ip-verifier keypair path is not provided, generate a new keypair or use an existing + // one in the deploy directory if it exists. Devnet-only: it is written to the deploy + // directory rather than checked in, and its pubkey is what the local GlobalState names as the + // verifier authority. + if spec.IPVerifier.Enabled && spec.IPVerifier.KeypairPath == "" { + ipVerifierKeypairPath := filepath.Join(spec.DeployDir, "ip-verifier-keypair.json") + generated, err := generateKeypairIfNotExists(ipVerifierKeypairPath) + if err != nil { + return nil, fmt.Errorf("failed to generate ip-verifier keypair: %w", err) + } + spec.IPVerifier.KeypairPath = ipVerifierKeypairPath + if generated { + log.Debug("--> Generated ip-verifier keypair", "path", ipVerifierKeypairPath) + } else { + log.Debug("--> Using existing ip-verifier keypair", "path", ipVerifierKeypairPath) + } + } + // Validate the spec. if err := spec.Validate(); err != nil { return nil, fmt.Errorf("failed to validate spec: %w", err) @@ -376,6 +401,12 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA dn: dn, log: log.With("component", "sentinel"), } + if spec.IPVerifier.Enabled { + dn.IPVerifier = &IPVerifier{ + dn: dn, + log: log.With("component", "ip-verifier"), + } + } dn.ValidatorMetadataServiceMock = &ValidatorMetadataServiceMock{ dn: dn, log: log.With("component", "validator-metadata-service-mock"), @@ -535,6 +566,27 @@ func (d *Devnet) Start(ctx context.Context, buildConfig *BuildConfig) error { return fmt.Errorf("failed to create CYOA network: %w", err) } + // Start the ip-verifier if it's not already running. It comes after the CYOA network, which + // it attaches to so that a client's proof request is observed arriving from the same address + // the client binds its tunnel to. Its authority has to be onchain before it starts: the + // service reads GlobalState at startup and exits if the authority is not its own key. + if d.IPVerifier != nil { + if err := d.IPVerifier.Prepare(); err != nil { + return fmt.Errorf("failed to prepare ip-verifier: %w", err) + } + // Skipped for a stack running cloned state: its GlobalState came from a remote cluster + // and the local manager is not its authority, so the write would fail. Such a stack has + // to name a verifier the cloned state already trusts. + if !d.Spec.SkipProgramDeploy { + if err := d.SetIPVerifierAuthority(ctx, d.IPVerifier.Pubkey); err != nil { + return fmt.Errorf("failed to set ip-verifier authority: %w", err) + } + } + if _, err := d.IPVerifier.StartIfNotRunning(ctx); err != nil { + return fmt.Errorf("failed to start ip-verifier: %w", err) + } + } + // We don't support starting with devices yet. // The AddDevice method can be used to add devices after the devnet is started. if len(d.Spec.Devices) > 0 { diff --git a/e2e/internal/devnet/ip_verifier.go b/e2e/internal/devnet/ip_verifier.go new file mode 100644 index 0000000000..2043906a5e --- /dev/null +++ b/e2e/internal/devnet/ip_verifier.go @@ -0,0 +1,244 @@ +package devnet + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "time" + + dockercontainer "github.com/docker/docker/api/types/container" + dockerfilters "github.com/docker/docker/api/types/filters" + dockernetwork "github.com/docker/docker/api/types/network" + "github.com/docker/go-connections/nat" + "github.com/malbeclabs/doublezero/e2e/internal/logging" + "github.com/malbeclabs/doublezero/e2e/internal/netutil" + "github.com/malbeclabs/doublezero/e2e/internal/solana" + "github.com/testcontainers/testcontainers-go" + tcwait "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + // ipVerifierInternalPort is the container port the proof endpoint listens on. + ipVerifierInternalPort = 8080 + + // defaultIPVerifierCYOANetworkIPHostID is the host offset the verifier takes on the CYOA + // network. Kept well clear of the ranges devices (single digits) and clients (100+) use. + defaultIPVerifierCYOANetworkIPHostID = 250 +) + +// IPVerifierSpec configures the RFC-27 IP ownership verification service container. +// +// The service must sit on the CYOA network, not only on the default network. It signs the source +// address it observes the request arrive from, and `connect` refuses a proof for any address other +// than the one it is provisioning — which for a local client is its CYOA address. Reached over the +// default network the observed address would be the client's default-network address instead, and +// every connect would fail on the mismatch. +type IPVerifierSpec struct { + Enabled bool + ContainerImage string + // KeypairPath is the host path to the verifier keypair JSON. Generated into the deploy + // directory when unset. Its pubkey is written to GlobalState.ip_verifier_authority_pk before + // the container starts: the service reads the authority from the ledger at startup and exits + // if it does not name its own key. + KeypairPath string + // CYOANetworkIPHostID is the offset into the host portion of the CYOA subnet. + CYOANetworkIPHostID uint32 +} + +func (s *IPVerifierSpec) Validate(cyoaNetworkSpec CYOANetworkSpec) error { + if !s.Enabled { + return nil + } + if s.ContainerImage == "" { + s.ContainerImage = os.Getenv("DZ_IP_VERIFIER_IMAGE") + } + if s.CYOANetworkIPHostID == 0 { + s.CYOANetworkIPHostID = defaultIPVerifierCYOANetworkIPHostID + } + maxHostID := uint32(1) << (32 - cyoaNetworkSpec.CIDRPrefix) + if s.CYOANetworkIPHostID >= maxHostID { + return fmt.Errorf("hostID %d is out of valid range (1 to %d)", s.CYOANetworkIPHostID, maxHostID-1) + } + if s.KeypairPath != "" && !filepath.IsAbs(s.KeypairPath) { + return fmt.Errorf("keypair path must be an absolute path: %s", s.KeypairPath) + } + return nil +} + +// IPVerifier manages the ip-verifier container. +type IPVerifier struct { + dn *Devnet + log *slog.Logger + + ContainerID string + // Pubkey is the verifier's signing key, and the value written to + // GlobalState.ip_verifier_authority_pk. + Pubkey string + // CYOANetworkIP is the address clients reach the service on. + CYOANetworkIP string + // InternalURL is what a client's DZ_IP_VERIFIER_URL is set to. + InternalURL string +} + +func (v *IPVerifier) dockerContainerHostname() string { + return "ip-verifier" +} + +func (v *IPVerifier) dockerContainerName() string { + return v.dn.Spec.DeployID + "-" + v.dockerContainerHostname() +} + +func (v *IPVerifier) Exists(ctx context.Context) (bool, error) { + containers, err := v.dn.dockerClient.ContainerList(ctx, dockercontainer.ListOptions{ + All: true, + Filters: dockerfilters.NewArgs(dockerfilters.Arg("name", v.dockerContainerName())), + }) + if err != nil { + return false, fmt.Errorf("failed to list containers: %w", err) + } + for _, container := range containers { + if container.Names[0] == "/"+v.dockerContainerName() { + return true, nil + } + } + return false, nil +} + +func (v *IPVerifier) StartIfNotRunning(ctx context.Context) (bool, error) { + exists, err := v.Exists(ctx) + if err != nil { + return false, fmt.Errorf("failed to check if ip-verifier exists: %w", err) + } + if !exists { + return false, v.Start(ctx) + } + + container, err := v.dn.dockerClient.ContainerInspect(ctx, v.dockerContainerName()) + if err != nil { + return false, fmt.Errorf("failed to inspect container: %w", err) + } + if !container.State.Running { + if err := v.dn.dockerClient.ContainerStart(ctx, container.ID, dockercontainer.StartOptions{}); err != nil { + return false, fmt.Errorf("failed to start ip-verifier: %w", err) + } + } else { + v.log.Debug("--> IPVerifier already running", "container", shortContainerID(container.ID)) + } + + if err := v.setState(container.ID); err != nil { + return false, fmt.Errorf("failed to set ip-verifier state: %w", err) + } + return !container.State.Running, nil +} + +// Prepare reads the verifier keypair and derives the addresses the container will use, without +// starting it. The pubkey is needed onchain before the container starts, because the service +// treats an authority that is not its own key as a startup error. +func (v *IPVerifier) Prepare() error { + keypairJSON, err := os.ReadFile(v.dn.Spec.IPVerifier.KeypairPath) + if err != nil { + return fmt.Errorf("failed to read ip-verifier keypair: %w", err) + } + pubkey, err := solana.PubkeyFromKeypairJSON(keypairJSON) + if err != nil { + return fmt.Errorf("failed to parse ip-verifier pubkey: %w", err) + } + v.Pubkey = pubkey + + cyoaIP, err := netutil.DeriveIPFromCIDR(v.dn.CYOANetwork.SubnetCIDR, v.dn.Spec.IPVerifier.CYOANetworkIPHostID) + if err != nil { + return fmt.Errorf("failed to derive CYOA network IP: %w", err) + } + v.CYOANetworkIP = cyoaIP.To4().String() + v.InternalURL = fmt.Sprintf("http://%s:%d", v.CYOANetworkIP, ipVerifierInternalPort) + return nil +} + +func (v *IPVerifier) Start(ctx context.Context) error { + v.log.Debug("==> Starting ip-verifier", "image", v.dn.Spec.IPVerifier.ContainerImage) + + if err := v.Prepare(); err != nil { + return err + } + + req := testcontainers.ContainerRequest{ + Image: v.dn.Spec.IPVerifier.ContainerImage, + Name: v.dockerContainerName(), + ConfigModifier: func(cfg *dockercontainer.Config) { + cfg.Hostname = v.dockerContainerHostname() + }, + ExposedPorts: []string{fmt.Sprintf("%d/tcp", ipVerifierInternalPort)}, + Env: map[string]string{ + "DZ_IP_VERIFIER_ENV": "local", + "DZ_IP_VERIFIER_LEDGER_RPC": v.dn.Ledger.InternalRPCURL, + "DZ_IP_VERIFIER_KEYPAIR": containerIPVerifierKeypairPath, + "DZ_IP_VERIFIER_LISTEN_ADDR": fmt.Sprintf("0.0.0.0:%d", ipVerifierInternalPort), + "DZ_IP_VERIFIER_LOG": "doublezero_ip_verifier=debug", + // No trusted proxies: clients reach the service directly, so the connection peer + // address is signed and forwarded headers are ignored outright. + // + // The rate limit is raised well above the production default because a devnet has one + // source address per client and a test can reconnect in a tight loop; the production + // value would turn that into `rate_limited` refusals that have nothing to do with what + // is being tested. + "DZ_IP_VERIFIER_RATE_LIMIT_BURST": "1000", + "DZ_IP_VERIFIER_RATE_LIMIT_PER_MINUTE": "6000", + }, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: v.dn.Spec.IPVerifier.KeypairPath, + ContainerFilePath: containerIPVerifierKeypairPath, + }, + }, + Networks: []string{ + v.dn.DefaultNetwork.Name, + v.dn.CYOANetwork.Name, + }, + NetworkAliases: map[string][]string{ + v.dn.DefaultNetwork.Name: {"ip-verifier"}, + }, + EndpointSettingsModifier: func(m map[string]*dockernetwork.EndpointSettings) { + if m[v.dn.CYOANetwork.Name] == nil { + m[v.dn.CYOANetwork.Name] = &dockernetwork.EndpointSettings{} + } + m[v.dn.CYOANetwork.Name].IPAddress = v.CYOANetworkIP + m[v.dn.CYOANetwork.Name].IPAMConfig = &dockernetwork.EndpointIPAMConfig{ + IPv4Address: v.CYOANetworkIP, + } + }, + // /health is 200 only once the cached ledger epoch is fresh *and* the ledger names this + // container's key as the verifier authority, so waiting on it proves both. + WaitingFor: tcwait.ForHTTP("/health"). + WithPort(nat.Port(fmt.Sprintf("%d/tcp", ipVerifierInternalPort))). + WithStartupTimeout(60 * time.Second). + WithPollInterval(1 * time.Second), + Resources: dockercontainer.Resources{ + NanoCPUs: defaultContainerNanoCPUs, + Memory: defaultContainerMemory, + }, + Labels: v.dn.labels, + } + + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + Logger: logging.NewTestcontainersAdapter(v.log), + }) + if err != nil { + return fmt.Errorf("failed to start ip-verifier: %w", err) + } + + if err := v.setState(container.GetContainerID()); err != nil { + return fmt.Errorf("failed to set ip-verifier state: %w", err) + } + + v.log.Debug("--> IPVerifier started", "container", v.ContainerID, "pubkey", v.Pubkey, "url", v.InternalURL) + return nil +} + +func (v *IPVerifier) setState(containerID string) error { + v.ContainerID = shortContainerID(containerID) + return v.Prepare() +} diff --git a/e2e/internal/devnet/ip_verifier_test.go b/e2e/internal/devnet/ip_verifier_test.go new file mode 100644 index 0000000000..5dd6d8e634 --- /dev/null +++ b/e2e/internal/devnet/ip_verifier_test.go @@ -0,0 +1,45 @@ +package devnet + +import "testing" + +func TestIPVerifierSpecValidate(t *testing.T) { + cyoa := CYOANetworkSpec{CIDRPrefix: 24} + + t.Run("disabled spec is left alone", func(t *testing.T) { + // Nothing is defaulted for a disabled verifier, so a devnet that does not run one never + // derives a CYOA address for it and never points clients anywhere. + spec := IPVerifierSpec{} + if err := spec.Validate(cyoa); err != nil { + t.Fatalf("Validate() = %v, want nil", err) + } + if spec.CYOANetworkIPHostID != 0 { + t.Errorf("CYOANetworkIPHostID = %d, want 0", spec.CYOANetworkIPHostID) + } + }) + + t.Run("defaults the CYOA host ID", func(t *testing.T) { + spec := IPVerifierSpec{Enabled: true} + if err := spec.Validate(cyoa); err != nil { + t.Fatalf("Validate() = %v, want nil", err) + } + if spec.CYOANetworkIPHostID != defaultIPVerifierCYOANetworkIPHostID { + t.Errorf("CYOANetworkIPHostID = %d, want %d", spec.CYOANetworkIPHostID, defaultIPVerifierCYOANetworkIPHostID) + } + }) + + t.Run("rejects a host ID outside the subnet", func(t *testing.T) { + // 256 does not fit a /24. Catching it here beats a Docker IPAM error at start time. + spec := IPVerifierSpec{Enabled: true, CYOANetworkIPHostID: 256} + if err := spec.Validate(cyoa); err == nil { + t.Fatal("Validate() = nil, want an out-of-range error") + } + }) + + t.Run("rejects a relative keypair path", func(t *testing.T) { + // The path is handed to Docker as a host mount source, which only resolves absolutely. + spec := IPVerifierSpec{Enabled: true, KeypairPath: "ip-verifier-keypair.json"} + if err := spec.Validate(cyoa); err == nil { + t.Fatal("Validate() = nil, want an absolute-path error") + } + }) +} diff --git a/e2e/internal/devnet/smartcontract_init.go b/e2e/internal/devnet/smartcontract_init.go index 90145ea345..abc37bdb6d 100644 --- a/e2e/internal/devnet/smartcontract_init.go +++ b/e2e/internal/devnet/smartcontract_init.go @@ -161,3 +161,49 @@ func (dn *Devnet) SetOnchainAllocationFeatureFlag(ctx context.Context) error { dn.log.Debug("--> Onchain-allocation feature flag enabled") return nil } + +// SetIPVerifierAuthority writes the RFC-27 verifier pubkey to +// GlobalState.ip_verifier_authority_pk. Must run before the ip-verifier container starts: the +// service reads the authority from the ledger at startup and refuses to serve if it does not name +// its own key. +func (dn *Devnet) SetIPVerifierAuthority(ctx context.Context, pubkey string) error { + dn.log.Debug("==> Setting ip-verifier authority", "pubkey", pubkey) + + dn.onchainWriteMutex.Lock() + defer dn.onchainWriteMutex.Unlock() + + _, err := dn.Manager.Exec(ctx, []string{ + "doublezero", "global-config", "authority", "set", "--ip-verifier-authority", pubkey, + }) + if err != nil { + return fmt.Errorf("failed to set ip-verifier authority: %w", err) + } + + dn.log.Debug("--> Ip-verifier authority set", "pubkey", pubkey) + return nil +} + +// SetIPOwnershipProofFeatureFlag turns RFC-27 enforcement on or off in GlobalState. Off is the +// local default: with it clear a user is created whether or not a proof is attached, so a devnet +// with no reachable verifier still works. Turning it on makes the program reject a create without +// a valid proof, which is what a test of the enforcement path wants. +func (dn *Devnet) SetIPOwnershipProofFeatureFlag(ctx context.Context, enable bool) error { + toggle := "--disable" + if enable { + toggle = "--enable" + } + dn.log.Debug("==> Setting require-ip-ownership-proof feature flag", "enable", enable) + + dn.onchainWriteMutex.Lock() + defer dn.onchainWriteMutex.Unlock() + + _, err := dn.Manager.Exec(ctx, []string{ + "doublezero", "global-config", "feature-flags", "set", toggle, "require-ip-ownership-proof", + }) + if err != nil { + return fmt.Errorf("failed to set require-ip-ownership-proof feature flag: %w", err) + } + + dn.log.Debug("--> Require-ip-ownership-proof feature flag set", "enable", enable) + return nil +} From 7322be6e2f4912c636761a9d04dbd2a93d384aa3 Mon Sep 17 00:00:00 2001 From: Greg Mitchell Date: Fri, 28 Aug 2026 15:37:37 +0000 Subject: [PATCH 2/5] e2e: run the ip-verifier in every devnet and cover the proof outcomes The verifier was built and pushed on every e2e run but never started: only dev/dzctl set IPVerifierSpec.Enabled, so the Go e2e suite deployed no verifier and no client was pointed at one. RFC-27's connect path had no e2e coverage at all. Flip the field to Disabled so the zero value runs it. Every existing e2e test now obtains a real proof during connect and hands it to the program, which validates it. Enforcement stays off: require-ip-ownership-proof is clear, so a create without a proof is still accepted. Add ip_ownership_proof_test.go for the three outcomes the shared path can have and that no other test distinguishes: a proof the program accepts, no proof at all, and a proof signed by a key the program does not trust. The last needs a verifier that keeps signing after the trust root moves, so IPVerifierSpec.AuthorityRefreshSecs pins how often the service re-reads GlobalState.ip_verifier_authority_pk. Set it past the test and rotate the authority and the service never notices, leaving a proof signed by a key GlobalState no longer names. That is refused by the SDK pre-flight, before the transaction is paid for, so it does not reach the program's own precompile check. ClientSpec.NoIPVerifier leaves DZ_IP_VERIFIER_URL unset for one client, which is what a client with no configured verifier looks like. --- e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md | 24 ++- e2e/internal/devnet/client.go | 18 +- e2e/internal/devnet/cmd/devnet.go | 10 +- e2e/internal/devnet/devnet.go | 4 +- e2e/internal/devnet/ip_verifier.go | 54 ++++-- e2e/internal/devnet/ip_verifier_test.go | 9 +- e2e/ip_ownership_proof_test.go | 214 ++++++++++++++++++++++++ 7 files changed, 294 insertions(+), 39 deletions(-) create mode 100644 e2e/ip_ownership_proof_test.go diff --git a/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md b/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md index bde0f8714c..51c67cafd6 100644 --- a/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md +++ b/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md @@ -2,13 +2,15 @@ RFC-27 ([`rfcs/rfc27-ip-verification.md`](../../rfcs/rfc27-ip-verification.md)) has `connect` attach a proof, signed by a DoubleZero-operated verifier, that the caller can originate traffic -from the `client_ip` it is binding. `dev/dzctl` runs that verifier so the local flow matches -production. +from the `client_ip` it is binding. Every devnet runs that verifier — `dev/dzctl` and the Go e2e +suite alike — so the local flow matches production. ## What comes up -`dzctl start` brings up a `dz-local-ip-verifier` container (image `dz-local/ip-verifier:dev`) -alongside the rest of the stack: +The verifier is on by default: `IPVerifierSpec.Disabled` is the opt-out, so a devnet that says +nothing about it gets one. `dzctl start` brings up a `dz-local-ip-verifier` container (image +`dz-local/ip-verifier:dev`) alongside the rest of the stack; an e2e test gets the same container +named for its own deploy ID: - **Keypair**: generated per deploy into `dev/.deploy/dz-local/ip-verifier-keypair.json`. Devnet only — nothing is checked in. @@ -54,6 +56,20 @@ docker exec dz-local-manager \ From a Go e2e test, `devnet.SetIPOwnershipProofFeatureFlag(ctx, true)` does the same thing. +## From a Go e2e test + +Because the verifier is on by default, an ordinary `connect` in any e2e test already obtains and +attaches a real proof. Two knobs cover the cases that need something else: + +- `ClientSpec.NoIPVerifier` leaves `DZ_IP_VERIFIER_URL` unset for one client, so its `connect` + obtains no proof at all — the path an environment takes before its verifier exists. +- `IPVerifierSpec.AuthorityRefreshSecs` pins how often the service re-reads the onchain authority. + Set it long and rotate the authority with `devnet.SetIPVerifierAuthority` and the service keeps + signing with a key `GlobalState` no longer names, which is how a test produces a proof that gets + refused. + +`e2e/ip_ownership_proof_test.go` uses all three paths. + ## Poking at it ```bash diff --git a/e2e/internal/devnet/client.go b/e2e/internal/devnet/client.go index e9c2e90a6d..9b8b864fc4 100644 --- a/e2e/internal/devnet/client.go +++ b/e2e/internal/devnet/client.go @@ -61,6 +61,10 @@ type ClientSpec struct { // CYOANetworkIPHostID is the offset into the host portion of the subnet (must be < 2^(32 - prefixLen)). CYOANetworkIPHostID uint32 + // NoIPVerifier leaves DZ_IP_VERIFIER_URL unset for this client, so its `connect` obtains no + // RFC-27 proof even in a devnet running a verifier. + NoIPVerifier bool + // EnableQAAgent starts the QA agent inside the client container for local QA testing. EnableQAAgent bool // QAAgentPort is the port the QA agent listens on inside the container (default: 7009). @@ -234,11 +238,15 @@ func (c *Client) Start(ctx context.Context) error { "DZ_SERVICEABILITY_PROGRAM_ID": c.dn.Manager.ServiceabilityProgramID, "DZ_CLIENT_EXTRA_ARGS": strings.Join(extraArgs, " "), } - // Point `connect` at the devnet verifier. Without this the `--env local` default - // (http://localhost:8080) is used, which resolves to nothing inside the client container. - // The URL is the verifier's CYOA address, so the source address it observes is the same one - // the client binds its tunnel to — `connect` hard-fails on a proof for any other address. - if c.dn.IPVerifier != nil && c.dn.IPVerifier.InternalURL != "" { + // Point `connect` at the devnet verifier. The `--env local` config carries no verifier URL, + // so without this a client obtains no proof at all. The URL is the verifier's CYOA address, + // so the source address it observes is the same one the client binds its tunnel to — + // `connect` hard-fails on a proof for any other address. + // + // NoIPVerifier leaves it unset, which is how a test covers the no-proof path: the CLI reports + // nothing to reach and creates the user without a proof, which the program accepts while + // require-ip-ownership-proof is clear. + if c.dn.IPVerifier != nil && c.dn.IPVerifier.InternalURL != "" && !c.Spec.NoIPVerifier { env["DZ_IP_VERIFIER_URL"] = c.dn.IPVerifier.InternalURL } if c.Spec.EnableQAAgent { diff --git a/e2e/internal/devnet/cmd/devnet.go b/e2e/internal/devnet/cmd/devnet.go index 514b25055a..c57f6bf820 100644 --- a/e2e/internal/devnet/cmd/devnet.go +++ b/e2e/internal/devnet/cmd/devnet.go @@ -90,12 +90,10 @@ func NewLocalDevnet(log *slog.Logger, deployID string) (*LocalDevnet, error) { Verbose: true, Interval: 10 * time.Second, }, - // The RFC-27 verifier runs in the local devnet so `connect` always has one to reach. - // Enforcement is a separate switch: the require-ip-ownership-proof feature flag stays - // clear by default, so a proof is attached but not demanded. See e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md. - IPVerifier: devnet.IPVerifierSpec{ - Enabled: true, - }, + // The RFC-27 verifier is on by default (IPVerifierSpec.Disabled), so `connect` always + // has one to reach. Enforcement is a separate switch: the require-ip-ownership-proof + // feature flag stays clear, so a proof is attached but not demanded. See + // e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md. }, log, dockerClient, subnetAllocator) if err != nil { return nil, fmt.Errorf("failed to create devnet: %w", err) diff --git a/e2e/internal/devnet/devnet.go b/e2e/internal/devnet/devnet.go index 2b5246dca5..fa6844a918 100644 --- a/e2e/internal/devnet/devnet.go +++ b/e2e/internal/devnet/devnet.go @@ -306,7 +306,7 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA // one in the deploy directory if it exists. Devnet-only: it is written to the deploy // directory rather than checked in, and its pubkey is what the local GlobalState names as the // verifier authority. - if spec.IPVerifier.Enabled && spec.IPVerifier.KeypairPath == "" { + if !spec.IPVerifier.Disabled && spec.IPVerifier.KeypairPath == "" { ipVerifierKeypairPath := filepath.Join(spec.DeployDir, "ip-verifier-keypair.json") generated, err := generateKeypairIfNotExists(ipVerifierKeypairPath) if err != nil { @@ -401,7 +401,7 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA dn: dn, log: log.With("component", "sentinel"), } - if spec.IPVerifier.Enabled { + if !spec.IPVerifier.Disabled { dn.IPVerifier = &IPVerifier{ dn: dn, log: log.With("component", "ip-verifier"), diff --git a/e2e/internal/devnet/ip_verifier.go b/e2e/internal/devnet/ip_verifier.go index 2043906a5e..711b9a5ee9 100644 --- a/e2e/internal/devnet/ip_verifier.go +++ b/e2e/internal/devnet/ip_verifier.go @@ -6,6 +6,7 @@ import ( "log/slog" "os" "path/filepath" + "strconv" "time" dockercontainer "github.com/docker/docker/api/types/container" @@ -36,7 +37,11 @@ const ( // default network the observed address would be the client's default-network address instead, and // every connect would fail on the mismatch. type IPVerifierSpec struct { - Enabled bool + // Disabled leaves the verifier out of the deploy. The zero value runs it, so every devnet + // exercises the same `connect` path production does: a proof is obtained and attached, and + // the program validates it. Enforcement is separate — the require-ip-ownership-proof feature + // flag stays clear, so a create with no proof is still accepted. + Disabled bool ContainerImage string // KeypairPath is the host path to the verifier keypair JSON. Generated into the deploy // directory when unset. Its pubkey is written to GlobalState.ip_verifier_authority_pk before @@ -45,10 +50,15 @@ type IPVerifierSpec struct { KeypairPath string // CYOANetworkIPHostID is the offset into the host portion of the CYOA subnet. CYOANetworkIPHostID uint32 + // AuthorityRefreshSecs is how often the service re-reads + // GlobalState.ip_verifier_authority_pk. Zero leaves the service default. Set it long to hold + // the service on a stale authority across a rotation, which is how a test produces a proof + // signed by a key the program no longer trusts. + AuthorityRefreshSecs int } func (s *IPVerifierSpec) Validate(cyoaNetworkSpec CYOANetworkSpec) error { - if !s.Enabled { + if s.Disabled { return nil } if s.ContainerImage == "" { @@ -163,6 +173,29 @@ func (v *IPVerifier) Start(ctx context.Context) error { return err } + env := map[string]string{ + "DZ_IP_VERIFIER_ENV": "local", + "DZ_IP_VERIFIER_LEDGER_RPC": v.dn.Ledger.InternalRPCURL, + "DZ_IP_VERIFIER_KEYPAIR": containerIPVerifierKeypairPath, + "DZ_IP_VERIFIER_LISTEN_ADDR": fmt.Sprintf("0.0.0.0:%d", ipVerifierInternalPort), + "DZ_IP_VERIFIER_LOG": "doublezero_ip_verifier=debug", + // No trusted proxies: clients reach the service directly, so the connection peer + // address is signed and forwarded headers are ignored outright. + // + // The rate limit is raised well above the production default because a devnet has one + // source address per client and a test can reconnect in a tight loop; the production + // value would turn that into `rate_limited` refusals that have nothing to do with what + // is being tested. + "DZ_IP_VERIFIER_RATE_LIMIT_BURST": "1000", + "DZ_IP_VERIFIER_RATE_LIMIT_PER_MINUTE": "6000", + } + // A long refresh holds the service on the authority it read at startup, so a rotation after + // it is up does not stop it signing. That is what lets a test hand the program a proof signed + // by a key it no longer trusts. + if v.dn.Spec.IPVerifier.AuthorityRefreshSecs > 0 { + env["DZ_IP_VERIFIER_AUTHORITY_REFRESH_SECS"] = strconv.Itoa(v.dn.Spec.IPVerifier.AuthorityRefreshSecs) + } + req := testcontainers.ContainerRequest{ Image: v.dn.Spec.IPVerifier.ContainerImage, Name: v.dockerContainerName(), @@ -170,22 +203,7 @@ func (v *IPVerifier) Start(ctx context.Context) error { cfg.Hostname = v.dockerContainerHostname() }, ExposedPorts: []string{fmt.Sprintf("%d/tcp", ipVerifierInternalPort)}, - Env: map[string]string{ - "DZ_IP_VERIFIER_ENV": "local", - "DZ_IP_VERIFIER_LEDGER_RPC": v.dn.Ledger.InternalRPCURL, - "DZ_IP_VERIFIER_KEYPAIR": containerIPVerifierKeypairPath, - "DZ_IP_VERIFIER_LISTEN_ADDR": fmt.Sprintf("0.0.0.0:%d", ipVerifierInternalPort), - "DZ_IP_VERIFIER_LOG": "doublezero_ip_verifier=debug", - // No trusted proxies: clients reach the service directly, so the connection peer - // address is signed and forwarded headers are ignored outright. - // - // The rate limit is raised well above the production default because a devnet has one - // source address per client and a test can reconnect in a tight loop; the production - // value would turn that into `rate_limited` refusals that have nothing to do with what - // is being tested. - "DZ_IP_VERIFIER_RATE_LIMIT_BURST": "1000", - "DZ_IP_VERIFIER_RATE_LIMIT_PER_MINUTE": "6000", - }, + Env: env, Files: []testcontainers.ContainerFile{ { HostFilePath: v.dn.Spec.IPVerifier.KeypairPath, diff --git a/e2e/internal/devnet/ip_verifier_test.go b/e2e/internal/devnet/ip_verifier_test.go index 5dd6d8e634..5d309a7042 100644 --- a/e2e/internal/devnet/ip_verifier_test.go +++ b/e2e/internal/devnet/ip_verifier_test.go @@ -8,7 +8,7 @@ func TestIPVerifierSpecValidate(t *testing.T) { t.Run("disabled spec is left alone", func(t *testing.T) { // Nothing is defaulted for a disabled verifier, so a devnet that does not run one never // derives a CYOA address for it and never points clients anywhere. - spec := IPVerifierSpec{} + spec := IPVerifierSpec{Disabled: true} if err := spec.Validate(cyoa); err != nil { t.Fatalf("Validate() = %v, want nil", err) } @@ -18,7 +18,8 @@ func TestIPVerifierSpecValidate(t *testing.T) { }) t.Run("defaults the CYOA host ID", func(t *testing.T) { - spec := IPVerifierSpec{Enabled: true} + // The zero value runs the verifier: every devnet gets one unless it opts out. + spec := IPVerifierSpec{} if err := spec.Validate(cyoa); err != nil { t.Fatalf("Validate() = %v, want nil", err) } @@ -29,7 +30,7 @@ func TestIPVerifierSpecValidate(t *testing.T) { t.Run("rejects a host ID outside the subnet", func(t *testing.T) { // 256 does not fit a /24. Catching it here beats a Docker IPAM error at start time. - spec := IPVerifierSpec{Enabled: true, CYOANetworkIPHostID: 256} + spec := IPVerifierSpec{CYOANetworkIPHostID: 256} if err := spec.Validate(cyoa); err == nil { t.Fatal("Validate() = nil, want an out-of-range error") } @@ -37,7 +38,7 @@ func TestIPVerifierSpecValidate(t *testing.T) { t.Run("rejects a relative keypair path", func(t *testing.T) { // The path is handed to Docker as a host mount source, which only resolves absolutely. - spec := IPVerifierSpec{Enabled: true, KeypairPath: "ip-verifier-keypair.json"} + spec := IPVerifierSpec{KeypairPath: "ip-verifier-keypair.json"} if err := spec.Validate(cyoa); err == nil { t.Fatal("Validate() = nil, want an absolute-path error") } diff --git a/e2e/ip_ownership_proof_test.go b/e2e/ip_ownership_proof_test.go new file mode 100644 index 0000000000..39b0ff724e --- /dev/null +++ b/e2e/ip_ownership_proof_test.go @@ -0,0 +1,214 @@ +//go:build e2e + +package e2e_test + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/malbeclabs/doublezero/e2e/internal/devnet" + "github.com/malbeclabs/doublezero/e2e/internal/random" + "github.com/malbeclabs/doublezero/e2e/internal/solana" + "github.com/stretchr/testify/require" +) + +// RFC-27 IP ownership proofs, end to end: `connect` asks the verification service for a proof, +// attaches it to the user creation, and the program validates it through the Ed25519 precompile. +// +// Every e2e devnet runs a verifier (IPVerifierSpec.Disabled defaults to false), so the ordinary +// connect path in every other test already carries a real proof. What is left to cover here is +// the three outcomes that path can have, which no other test distinguishes: +// +// - a proof the program accepts, +// - no proof at all, which the program also accepts while require-ip-ownership-proof is clear, +// - a proof signed by a key the program does not trust, which it must reject. + +// A valid proof is obtained and attached, and the user is created. +func TestE2E_IPOwnershipProof_ValidProof(t *testing.T) { + t.Parallel() + + dn, _, client, log := setupIPProofDevnet(t, devnet.IPVerifierSpec{}, devnet.ClientSpec{ + CYOANetworkIPHostID: 100, + }) + + out := connectIBRLForProofTest(t, log, dn, client) + + // The verifier signed the address the client is provisioning. `connect` prints this only + // after checking that the address the service observed matches the one being bound, so it is + // evidence of the whole round trip, not just of a reachable service. + require.Contains(t, out, "IP ownership verified for "+client.CYOANetworkIP, + "connect must obtain a proof for the address it is provisioning") + require.Contains(t, out, "✅ User Provisioned") + + // No proof would also have been accepted here, so assert the run did not quietly fall back. + require.NotContains(t, out, "Continuing without an IP ownership proof") + + require.NoError(t, client.WaitForTunnelUp(t.Context(), 90*time.Second), + "a user created with a proof must still come up normally") +} + +// No verifier to reach, no proof, and the create is accepted anyway: enforcement is gated on the +// require-ip-ownership-proof feature flag, which is clear in a local devnet. This is the path a +// deployed environment takes before its verifier exists, so it has to keep working. +func TestE2E_IPOwnershipProof_NoProof(t *testing.T) { + t.Parallel() + + // The devnet still runs a verifier; this client is simply not pointed at it, which is what a + // client with no configured verifier looks like. + dn, _, client, log := setupIPProofDevnet(t, devnet.IPVerifierSpec{}, devnet.ClientSpec{ + CYOANetworkIPHostID: 100, + NoIPVerifier: true, + }) + + out := connectIBRLForProofTest(t, log, dn, client) + + require.NotContains(t, out, "IP ownership verified for", + "a client with no verifier configured must not report a proof") + require.Contains(t, out, "✅ User Provisioned", + "a create without a proof must be accepted while require-ip-ownership-proof is clear") + + require.NoError(t, client.WaitForTunnelUp(t.Context(), 90*time.Second)) +} + +// A proof signed by a key the program does not trust must be rejected. +// +// The rotation is what makes the proof invalid. The verifier re-reads +// GlobalState.ip_verifier_authority_pk periodically and stops serving when it no longer names its +// own key, so the refresh is pinned long enough that it does not notice: it keeps signing with the +// key it started with while GlobalState names a different one. `connect` reads the verifier key +// from GlobalState — the proof does not carry it — and finds the signature does not verify +// against it. +// +// Where the refusal lands: the SDK checks the proof against the onchain verifier key before it +// builds the transaction, so this fails client-side rather than onchain. That is the intended +// design — the "refused before the transaction is paid for" pre-flight — but it does mean this +// test does not reach the program's own Ed25519 precompile check. Covering that would take a +// client that skips the pre-flight, which `connect` gives no way to do; the program-side check +// has unit coverage in the serviceability program. +func TestE2E_IPOwnershipProof_UntrustedSigner(t *testing.T) { + t.Parallel() + + dn, _, client, log := setupIPProofDevnet(t, devnet.IPVerifierSpec{ + // Comfortably longer than the test: the service must not observe the rotation. + AuthorityRefreshSecs: 3600, + }, devnet.ClientSpec{ + CYOANetworkIPHostID: 100, + }) + + // Rotate the trust root out from under the running verifier. + keypairJSON, err := solana.GenerateKeypairJSON() + require.NoError(t, err) + rotated, err := solana.PubkeyFromKeypairJSON(keypairJSON) + require.NoError(t, err) + require.NotEqual(t, dn.IPVerifier.Pubkey, rotated) + + log.Info("==> Rotating the verifier authority away from the running service", + "from", dn.IPVerifier.Pubkey, "to", rotated) + require.NoError(t, dn.SetIPVerifierAuthority(t.Context(), rotated)) + + setAccessPass(t, dn, client) + + log.Info("==> Connecting with a proof signed by the pre-rotation key") + out, err := client.Exec(t.Context(), []string{"bash", "-c", "doublezero connect ibrl 2>&1"}) + output := string(out) + log.Info("==> Connect output", "output", output) + + // The verifier still answers — it has not re-read the authority — so a proof is obtained and + // attached. It is the transaction that must fail. + require.Contains(t, output, "IP ownership verified for "+client.CYOANetworkIP, + "the service should still be issuing proofs; if it is not, the rotation was noticed and "+ + "this test is no longer covering an untrusted signature") + require.Error(t, err, "a proof signed by a key GlobalState does not name must not create a user") + require.Contains(t, output, "does not verify against the onchain verifier "+rotated, + "the refusal must name the rotated key it checked against") + require.NotContains(t, output, "✅ User Provisioned") + + // And nothing landed onchain: a rejected transaction must not leave a half-created user. + users, err := dn.Manager.Exec(t.Context(), []string{"bash", "-c", "doublezero user list"}) + require.NoError(t, err) + require.NotContains(t, string(users), client.CYOANetworkIP, + "no user may exist for a client whose creation was rejected") +} + +// setAccessPass grants the client a prepaid pass, which every connect needs before it gets as far +// as the proof. +func setAccessPass(t *testing.T, dn *devnet.Devnet, client *devnet.Client) { + t.Helper() + _, err := dn.Manager.Exec(t.Context(), []string{"bash", "-c", + "doublezero access-pass set --accesspass-type prepaid --epochs max --client-ip " + + client.CYOANetworkIP + " --user-payer " + client.Pubkey}) + require.NoError(t, err) +} + +func connectIBRLForProofTest(t *testing.T, log *slog.Logger, dn *devnet.Devnet, client *devnet.Client) string { + t.Helper() + + setAccessPass(t, dn, client) + + log.Info("==> Connecting IBRL") + out, err := client.Exec(t.Context(), []string{"bash", "-c", "doublezero connect ibrl 2>&1"}) + log.Info("==> Connect output", "output", string(out)) + require.NoError(t, err, "connect failed: %s", string(out)) + return string(out) +} + +// setupIPProofDevnet builds the smallest devnet these tests need: one device, one client, and the +// verifier the spec asks for. +func setupIPProofDevnet(t *testing.T, verifier devnet.IPVerifierSpec, clientSpec devnet.ClientSpec) (*devnet.Devnet, *devnet.Device, *devnet.Client, *slog.Logger) { + deployID := "dz-e2e-" + t.Name() + "-" + random.ShortID() + log := logger.With("test", t.Name(), "deployID", deployID) + + currentDir, err := os.Getwd() + require.NoError(t, err) + serviceabilityProgramKeypairPath := filepath.Join(currentDir, "data", "serviceability-program-keypair.json") + + dn, err := devnet.New(devnet.DevnetSpec{ + DeployID: deployID, + DeployDir: t.TempDir(), + + CYOANetwork: devnet.CYOANetworkSpec{ + CIDRPrefix: subnetCIDRPrefix, + }, + Manager: devnet.ManagerSpec{ + ServiceabilityProgramKeypairPath: serviceabilityProgramKeypairPath, + }, + IPVerifier: verifier, + }, log, dockerClient, subnetAllocator) + require.NoError(t, err) + + log.Info("==> Starting devnet") + require.NoError(t, dn.Start(t.Context(), nil)) + + require.NotNil(t, dn.IPVerifier, "the devnet must run a verifier for these tests") + log.Info("--> IP verifier running", "pubkey", dn.IPVerifier.Pubkey, "url", dn.IPVerifier.InternalURL) + + device, err := dn.AddDevice(t.Context(), devnet.DeviceSpec{ + Code: "ny5-dz01", + Location: "ewr", + Exchange: "xewr", + CYOANetworkIPHostID: 8, + CYOANetworkAllocatablePrefix: 29, + }) + require.NoError(t, err) + + _, err = dn.Manager.Exec(t.Context(), []string{"bash", "-c", ` + set -euo pipefail + doublezero device interface create ny5-dz01 "Ethernet2" --bandwidth 10G -w + doublezero device interface create ny5-dz01 "Loopback255" --loopback-type vpnv4 --bandwidth 10G -w + doublezero device interface create ny5-dz01 "Loopback256" --loopback-type ipv4 --bandwidth 10G -w + `}) + require.NoError(t, err) + + client, err := dn.AddClient(t.Context(), clientSpec) + require.NoError(t, err) + log.Info("--> Client added", "clientIP", client.CYOANetworkIP, "pubkey", client.Pubkey) + + // The client picks a device from its own latency measurements; connecting before they exist + // fails on endpoint selection rather than on anything this test is about. + require.NoError(t, client.WaitForLatencyResults(t.Context(), device.ID, 75*time.Second)) + + return dn, device, client, log +} From 4483270d14df8130b2ec623dc1ab6d5a23f517e3 Mon Sep 17 00:00:00 2001 From: Greg Mitchell Date: Fri, 28 Aug 2026 16:59:04 +0000 Subject: [PATCH 3/5] e2e: address review comments on the devnet ip-verifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes from review. The host ID bounds now match the ones ClientSpec and DeviceSpec enforce, which reject the network and broadcast addresses. The old check used 2^hostBits rather than 2^hostBits-1, so it accepted the broadcast host ID while its error message claimed a range that excluded it. Not reachable in practice — the CYOA Docker network is a /23 while CIDRPrefix is 24, so the bound was already conservative — but the divergence and the untrue message were both worth removing. A verifier this harness restarts now gets the same /health gate a fresh one gets from testcontainers. StartIfNotRunning called ContainerStart and returned, so a container that exits on startup left the devnet reporting success with clients pointed at a dead service. The authority can rotate while the container is stopped, and the service refuses to start when GlobalState no longer names its key, so this is the component where that gap actually bites. Ledger is the precedent: it reuses its create-path readiness check on the restart path. The other components share the gap and are left alone. A cloned-state devnet no longer runs a verifier unless it names a keypair. Its GlobalState came from a remote cluster and the local manager cannot write to it, so a generated key would never be the one the program trusts and the container would exit at startup. Latent until now: both SkipProgramDeploy callers hand-start components and never reach Devnet.Start. --- e2e/internal/devnet/devnet.go | 14 ++++- e2e/internal/devnet/ip_verifier.go | 73 +++++++++++++++++++++++-- e2e/internal/devnet/ip_verifier_test.go | 20 +++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/e2e/internal/devnet/devnet.go b/e2e/internal/devnet/devnet.go index fa6844a918..81b36766aa 100644 --- a/e2e/internal/devnet/devnet.go +++ b/e2e/internal/devnet/devnet.go @@ -302,6 +302,15 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA } } + // A cloned-state devnet cannot provision its own verifier authority: its GlobalState came from + // a remote cluster and the local manager cannot write to it, so a generated key would never be + // the one the program trusts and the container would exit at startup. Default the verifier off + // there. A stack that does have a verifier the cloned state already trusts names its keypair + // explicitly, which opts back in. + if spec.SkipProgramDeploy && spec.IPVerifier.KeypairPath == "" { + spec.IPVerifier.Disabled = true + } + // If the ip-verifier keypair path is not provided, generate a new keypair or use an existing // one in the deploy directory if it exists. Devnet-only: it is written to the deploy // directory rather than checked in, and its pubkey is what the local GlobalState names as the @@ -575,8 +584,9 @@ func (d *Devnet) Start(ctx context.Context, buildConfig *BuildConfig) error { return fmt.Errorf("failed to prepare ip-verifier: %w", err) } // Skipped for a stack running cloned state: its GlobalState came from a remote cluster - // and the local manager is not its authority, so the write would fail. Such a stack has - // to name a verifier the cloned state already trusts. + // and the local manager is not its authority, so the write would fail. Reaching here with + // SkipProgramDeploy means the caller named a keypair explicitly — a verifier the cloned + // state already trusts — so the authority is already correct and needs no write. if !d.Spec.SkipProgramDeploy { if err := d.SetIPVerifierAuthority(ctx, d.IPVerifier.Pubkey); err != nil { return fmt.Errorf("failed to set ip-verifier authority: %w", err) diff --git a/e2e/internal/devnet/ip_verifier.go b/e2e/internal/devnet/ip_verifier.go index 711b9a5ee9..adbdf538a7 100644 --- a/e2e/internal/devnet/ip_verifier.go +++ b/e2e/internal/devnet/ip_verifier.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "net/http" "os" "path/filepath" "strconv" @@ -15,6 +16,7 @@ import ( "github.com/docker/go-connections/nat" "github.com/malbeclabs/doublezero/e2e/internal/logging" "github.com/malbeclabs/doublezero/e2e/internal/netutil" + "github.com/malbeclabs/doublezero/e2e/internal/poll" "github.com/malbeclabs/doublezero/e2e/internal/solana" "github.com/testcontainers/testcontainers-go" tcwait "github.com/testcontainers/testcontainers-go/wait" @@ -27,6 +29,10 @@ const ( // defaultIPVerifierCYOANetworkIPHostID is the host offset the verifier takes on the CYOA // network. Kept well clear of the ranges devices (single digits) and clients (100+) use. defaultIPVerifierCYOANetworkIPHostID = 250 + + // ipVerifierStartupTimeout is how long the service gets to report healthy, on both the create + // and the restart path. + ipVerifierStartupTimeout = 60 * time.Second ) // IPVerifierSpec configures the RFC-27 IP ownership verification service container. @@ -67,7 +73,11 @@ func (s *IPVerifierSpec) Validate(cyoaNetworkSpec CYOANetworkSpec) error { if s.CYOANetworkIPHostID == 0 { s.CYOANetworkIPHostID = defaultIPVerifierCYOANetworkIPHostID } - maxHostID := uint32(1) << (32 - cyoaNetworkSpec.CIDRPrefix) + // The same bounds the client and device specs enforce: the hostID may not select the network + // (0) or broadcast (max) address. 0 cannot reach here — above it is the "use the default" + // sentinel, not a caller's choice. + hostBits := 32 - cyoaNetworkSpec.CIDRPrefix + maxHostID := uint32((1 << hostBits) - 1) if s.CYOANetworkIPHostID >= maxHostID { return fmt.Errorf("hostID %d is out of valid range (1 to %d)", s.CYOANetworkIPHostID, maxHostID-1) } @@ -129,7 +139,8 @@ func (v *IPVerifier) StartIfNotRunning(ctx context.Context) (bool, error) { if err != nil { return false, fmt.Errorf("failed to inspect container: %w", err) } - if !container.State.Running { + started := !container.State.Running + if started { if err := v.dn.dockerClient.ContainerStart(ctx, container.ID, dockercontainer.StartOptions{}); err != nil { return false, fmt.Errorf("failed to start ip-verifier: %w", err) } @@ -140,7 +151,61 @@ func (v *IPVerifier) StartIfNotRunning(ctx context.Context) (bool, error) { if err := v.setState(container.ID); err != nil { return false, fmt.Errorf("failed to set ip-verifier state: %w", err) } - return !container.State.Running, nil + + // A container this call started gets the same readiness gate a fresh one does. `Start` waits + // on /health through testcontainers; without the equivalent here, a verifier that exits on + // startup would leave the devnet reporting success with clients pointed at a dead service. + // That is not hypothetical for this component: the authority can rotate while the container + // is stopped, and the service refuses to start when GlobalState no longer names its key. + if started { + if err := v.waitForHealthy(ctx, container.ID); err != nil { + return false, err + } + } + return started, nil +} + +// waitForHealthy blocks until the container reports healthy, or fails as soon as it is clear it +// never will. /health is 200 only once the cached ledger epoch is fresh and the ledger names this +// container's key, so it proves the same two things the create path's wait strategy does. +func (v *IPVerifier) waitForHealthy(ctx context.Context, containerID string) error { + port, err := v.dn.waitForContainerPortExposed(ctx, containerID, ipVerifierInternalPort, 10*time.Second) + if err != nil { + return fmt.Errorf("failed to wait for ip-verifier port: %w", err) + } + url := fmt.Sprintf("http://%s:%d/health", v.dn.ExternalHost, port) + + err = poll.Until(ctx, func() (bool, error) { + // A container that has already exited is never going to answer, so say why now rather + // than spending the whole budget waiting on it. + container, err := v.dn.dockerClient.ContainerInspect(ctx, containerID) + if err != nil { + return false, fmt.Errorf("failed to inspect container: %w", err) + } + if !container.State.Running { + // Most often the authority moved while this container was stopped: the service exits + // when GlobalState no longer names its key. Named as the likely cause, not the only + // one — `docker logs` on the container has the actual reason. + return false, fmt.Errorf("ip-verifier exited with code %d (check `docker logs %s`); "+ + "the usual cause is GlobalState.ip_verifier_authority_pk no longer naming its key %s", + container.State.ExitCode, v.dockerContainerName(), v.Pubkey) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return false, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, nil + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK, nil + }, ipVerifierStartupTimeout, time.Second) + if err != nil { + return fmt.Errorf("ip-verifier did not become healthy: %w", err) + } + return nil } // Prepare reads the verifier keypair and derives the addresses the container will use, without @@ -230,7 +295,7 @@ func (v *IPVerifier) Start(ctx context.Context) error { // container's key as the verifier authority, so waiting on it proves both. WaitingFor: tcwait.ForHTTP("/health"). WithPort(nat.Port(fmt.Sprintf("%d/tcp", ipVerifierInternalPort))). - WithStartupTimeout(60 * time.Second). + WithStartupTimeout(ipVerifierStartupTimeout). WithPollInterval(1 * time.Second), Resources: dockercontainer.Resources{ NanoCPUs: defaultContainerNanoCPUs, diff --git a/e2e/internal/devnet/ip_verifier_test.go b/e2e/internal/devnet/ip_verifier_test.go index 5d309a7042..83950a071c 100644 --- a/e2e/internal/devnet/ip_verifier_test.go +++ b/e2e/internal/devnet/ip_verifier_test.go @@ -36,6 +36,26 @@ func TestIPVerifierSpecValidate(t *testing.T) { } }) + t.Run("rejects the broadcast host ID", func(t *testing.T) { + // The bound the client and device specs use. 255 is the broadcast host ID of a /24, and + // an earlier version of this check accepted it while its error message claimed otherwise. + spec := IPVerifierSpec{CYOANetworkIPHostID: 255} + if err := spec.Validate(cyoa); err == nil { + t.Fatal("Validate() = nil, want the broadcast address rejected") + } + }) + + t.Run("accepts the last usable host ID", func(t *testing.T) { + // The other side of that bound: 254 must still pass, so the fix did not go one too far. + spec := IPVerifierSpec{CYOANetworkIPHostID: 254} + if err := spec.Validate(cyoa); err != nil { + t.Fatalf("Validate() = %v, want nil", err) + } + if spec.CYOANetworkIPHostID != 254 { + t.Errorf("CYOANetworkIPHostID = %d, want it left alone", spec.CYOANetworkIPHostID) + } + }) + t.Run("rejects a relative keypair path", func(t *testing.T) { // The path is handed to Docker as a host mount source, which only resolves absolutely. spec := IPVerifierSpec{KeypairPath: "ip-verifier-keypair.json"} From 54442d0dfc941487f88c44e41f56d65f3046425e Mon Sep 17 00:00:00 2001 From: Greg Mitchell Date: Mon, 31 Aug 2026 20:15:13 +0000 Subject: [PATCH 4/5] e2e: address second-round review on the devnet ip-verifier Pull the ip-verifier image in the shard job alongside every other e2e image, rather than leaving it to testcontainers' unretried implicit pull. Give the service an explicit --serviceability-program-id (DZ_IP_VERIFIER_SERVICEABILITY_PROGRAM_ID) and pass the devnet's deployed program, so the startup authority check reads the GlobalState the devnet actually has. Under DZ_IP_VERIFIER_ENV=local it resolved the environment constant instead, and in a devnet with an unpinned program keypair that PDA does not exist: the check only warned and /health still answered 200. Check container state before the port wait in waitForHealthy. Docker drops the port bindings of a stopped container, so a verifier that exits on startup lost the exit-code diagnostic to a port-wait timeout. Run the doc's proof curl through a shell in the container, so DZ_IP_VERIFIER_URL is expanded there and not by the host shell. --- .github/workflows/e2e.yml | 1 + crates/doublezero-ip-verifier/src/settings.rs | 52 ++++++++++++++--- e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md | 11 ++-- e2e/internal/devnet/ip_verifier.go | 56 +++++++++++++------ 4 files changed, 92 insertions(+), 28 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index bb2eb88ebc..32705b2cbd 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -436,6 +436,7 @@ jobs: pull_with_retry ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }} pull_with_retry ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }} pull_with_retry ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }} + pull_with_retry ${{ env.DZ_IMAGE_REPO }}/ip-verifier:${{ env.DZ_IMAGE_TAG }} pull_with_retry ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }} pull_with_retry ghcr.io/malbeclabs/dz-e2e/prometheus:v2.54.1 pull_with_retry public.ecr.aws/influxdb/influxdb:1.8 diff --git a/crates/doublezero-ip-verifier/src/settings.rs b/crates/doublezero-ip-verifier/src/settings.rs index 2f13ded370..8ceee73397 100644 --- a/crates/doublezero-ip-verifier/src/settings.rs +++ b/crates/doublezero-ip-verifier/src/settings.rs @@ -9,6 +9,7 @@ use clap::Parser; use doublezero_config::Environment; use ipnetwork::IpNetwork; use solana_keypair::{read_keypair_file, Keypair}; +use solana_program::pubkey::Pubkey; use std::{net::SocketAddr, path::PathBuf, sync::Arc, time::Duration}; /// Every flag also reads a `DZ_IP_VERIFIER_`-prefixed environment variable, which is how the @@ -29,6 +30,13 @@ pub struct AppArgs { #[arg(long, env = "DZ_IP_VERIFIER_LEDGER_RPC")] pub ledger_rpc: Option, + /// Serviceability program ID `GlobalState` is read from. Defaults to the program `--env` + /// names. Needed wherever the deployed program is not that one — a devnet deploys its own, + /// and reading the environment's program ID there finds no `GlobalState` account at all, so + /// the authority check has nothing to compare against and never fails. + #[arg(long, env = "DZ_IP_VERIFIER_SERVICEABILITY_PROGRAM_ID")] + pub serviceability_program_id: Option, + /// Path to the verifier keypair JSON file. Its public key must match /// `GlobalState.ip_verifier_authority_pk` or every proof this service issues is rejected /// onchain. @@ -159,17 +167,23 @@ impl AppArgs { } } + pub fn serviceability_program_id(&self) -> anyhow::Result { + match self.serviceability_program_id { + Some(program_id) => Ok(program_id), + None => Ok(self + .env + .config() + .map_err(|err| anyhow::anyhow!("{err}")) + .with_context(|| format!("no network config for environment {}", self.env))? + .serviceability_program_id), + } + } + /// The ledger client both background loops read through. pub fn ledger(&self) -> anyhow::Result { - let config = self - .env - .config() - .map_err(|err| anyhow::anyhow!("{err}")) - .with_context(|| format!("no network config for environment {}", self.env))?; - Ok(Ledger::new( self.ledger_rpc_url()?, - config.serviceability_program_id, + self.serviceability_program_id()?, )) } @@ -250,6 +264,30 @@ mod tests { assert_eq!(args.ledger_rpc_url().unwrap(), "http://localhost:8899"); } + #[test] + fn the_serviceability_program_id_defaults_to_the_environment() { + let args = parse(&["--env", "testnet"]); + assert_eq!( + args.serviceability_program_id().unwrap(), + Environment::Testnet + .config() + .unwrap() + .serviceability_program_id + ); + } + + #[test] + fn an_explicit_serviceability_program_id_wins() { + let program_id = Pubkey::new_unique(); + let args = parse(&[ + "--env", + "testnet", + "--serviceability-program-id", + &program_id.to_string(), + ]); + assert_eq!(args.serviceability_program_id().unwrap(), program_id); + } + #[test] fn a_missing_keypair_file_reports_the_path_and_nothing_else() { let args = AppArgs::try_parse_from([ diff --git a/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md b/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md index 51c67cafd6..16d5544fce 100644 --- a/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md +++ b/e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md @@ -79,11 +79,12 @@ docker exec dz-local-ip-verifier curl -sS localhost:8080/health # What the ledger thinks the authority is. docker exec dz-local-manager doublezero global-config authority get -# A proof, as a client would ask for it. -docker exec dz-local-client- \ - curl -sS -X POST "$DZ_IP_VERIFIER_URL/v1/proof" \ - -H 'content-type: application/json' \ - -d '{"payer":"","user_type":0}' +# A proof, as a client would ask for it. Run through a shell in the container: DZ_IP_VERIFIER_URL +# is set in the client's environment, and `docker exec curl` would have the host shell expand it. +docker exec dz-local-client- bash -c \ + 'curl -sS -X POST "$DZ_IP_VERIFIER_URL/v1/proof" \ + -H "content-type: application/json" \ + -d "{\"payer\":\"\",\"user_type\":0}"' ``` The rate limit is raised well above the production default in the devnet (burst 1000, 6000/min): diff --git a/e2e/internal/devnet/ip_verifier.go b/e2e/internal/devnet/ip_verifier.go index adbdf538a7..069c8fae88 100644 --- a/e2e/internal/devnet/ip_verifier.go +++ b/e2e/internal/devnet/ip_verifier.go @@ -171,6 +171,13 @@ func (v *IPVerifier) StartIfNotRunning(ctx context.Context) (bool, error) { func (v *IPVerifier) waitForHealthy(ctx context.Context, containerID string) error { port, err := v.dn.waitForContainerPortExposed(ctx, containerID, ipVerifierInternalPort, 10*time.Second) if err != nil { + // Docker drops the port bindings of a container that is no longer running, so the case + // this whole path exists for — a verifier that exits within milliseconds of starting — + // reaches the port wait as a timeout rather than as a published port. Check the state + // before reporting the port, or the diagnostic below is lost exactly where it is wanted. + if exitErr := v.exitedError(ctx, containerID); exitErr != nil { + return exitErr + } return fmt.Errorf("failed to wait for ip-verifier port: %w", err) } url := fmt.Sprintf("http://%s:%d/health", v.dn.ExternalHost, port) @@ -178,17 +185,8 @@ func (v *IPVerifier) waitForHealthy(ctx context.Context, containerID string) err err = poll.Until(ctx, func() (bool, error) { // A container that has already exited is never going to answer, so say why now rather // than spending the whole budget waiting on it. - container, err := v.dn.dockerClient.ContainerInspect(ctx, containerID) - if err != nil { - return false, fmt.Errorf("failed to inspect container: %w", err) - } - if !container.State.Running { - // Most often the authority moved while this container was stopped: the service exits - // when GlobalState no longer names its key. Named as the likely cause, not the only - // one — `docker logs` on the container has the actual reason. - return false, fmt.Errorf("ip-verifier exited with code %d (check `docker logs %s`); "+ - "the usual cause is GlobalState.ip_verifier_authority_pk no longer naming its key %s", - container.State.ExitCode, v.dockerContainerName(), v.Pubkey) + if exitErr := v.exitedError(ctx, containerID); exitErr != nil { + return false, exitErr } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -208,6 +206,25 @@ func (v *IPVerifier) waitForHealthy(ctx context.Context, containerID string) err return nil } +// exitedError reports why the container is gone, or nil if it is still running. An inspect that +// fails is itself reported: at this point the caller is already on an error path and the inspect +// result is what decides which error it gets. +func (v *IPVerifier) exitedError(ctx context.Context, containerID string) error { + container, err := v.dn.dockerClient.ContainerInspect(ctx, containerID) + if err != nil { + return fmt.Errorf("failed to inspect container: %w", err) + } + if container.State.Running { + return nil + } + // Most often the authority moved while this container was stopped: the service exits when + // GlobalState no longer names its key. Named as the likely cause, not the only one — + // `docker logs` on the container has the actual reason. + return fmt.Errorf("ip-verifier exited with code %d (check `docker logs %s`); "+ + "the usual cause is GlobalState.ip_verifier_authority_pk no longer naming its key %s", + container.State.ExitCode, v.dockerContainerName(), v.Pubkey) +} + // Prepare reads the verifier keypair and derives the addresses the container will use, without // starting it. The pubkey is needed onchain before the container starts, because the service // treats an authority that is not its own key as a startup error. @@ -239,11 +256,18 @@ func (v *IPVerifier) Start(ctx context.Context) error { } env := map[string]string{ - "DZ_IP_VERIFIER_ENV": "local", - "DZ_IP_VERIFIER_LEDGER_RPC": v.dn.Ledger.InternalRPCURL, - "DZ_IP_VERIFIER_KEYPAIR": containerIPVerifierKeypairPath, - "DZ_IP_VERIFIER_LISTEN_ADDR": fmt.Sprintf("0.0.0.0:%d", ipVerifierInternalPort), - "DZ_IP_VERIFIER_LOG": "doublezero_ip_verifier=debug", + "DZ_IP_VERIFIER_ENV": "local", + // Named explicitly rather than left to the environment's constant: a devnet deploys its + // own serviceability program whenever no keypair is pinned, and against the constant the + // GlobalState PDA simply does not exist — the startup authority check then finds nothing + // to compare against, only warns, and /health still answers 200. The invariant this file + // is built around (waiting on /health proves the ledger names this container's key) needs + // the program the devnet actually deployed. + "DZ_IP_VERIFIER_SERVICEABILITY_PROGRAM_ID": v.dn.Manager.ServiceabilityProgramID, + "DZ_IP_VERIFIER_LEDGER_RPC": v.dn.Ledger.InternalRPCURL, + "DZ_IP_VERIFIER_KEYPAIR": containerIPVerifierKeypairPath, + "DZ_IP_VERIFIER_LISTEN_ADDR": fmt.Sprintf("0.0.0.0:%d", ipVerifierInternalPort), + "DZ_IP_VERIFIER_LOG": "doublezero_ip_verifier=debug", // No trusted proxies: clients reach the service directly, so the connection peer // address is signed and forwarded headers are ignored outright. // From ab7cfd764ae9f9f296e312a51f37191b3949f0a1 Mon Sep 17 00:00:00 2001 From: Greg Mitchell Date: Tue, 1 Sep 2026 00:33:11 +0000 Subject: [PATCH 5/5] e2e: bound the ip-verifier health probe and enforce the prepared URL Give the health probe its own client with a 5s timeout. poll.Until runs its condition synchronously and the request carries the caller's context, so a probe that hung rather than being refused would block past the 60s startup budget instead of failing within it. Refuse a nil-URL verifier when starting a client instead of falling through to the no-proof path. That state means Prepare never ran, which Devnet.Start makes unreachable; tolerating it would let every connecting test quietly stop covering the proof while still passing. --- e2e/internal/devnet/client.go | 10 +++++++++- e2e/internal/devnet/ip_verifier.go | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/e2e/internal/devnet/client.go b/e2e/internal/devnet/client.go index 9b8b864fc4..1e2448fdba 100644 --- a/e2e/internal/devnet/client.go +++ b/e2e/internal/devnet/client.go @@ -246,7 +246,15 @@ func (c *Client) Start(ctx context.Context) error { // NoIPVerifier leaves it unset, which is how a test covers the no-proof path: the CLI reports // nothing to reach and creates the user without a proof, which the program accepts while // require-ip-ownership-proof is clear. - if c.dn.IPVerifier != nil && c.dn.IPVerifier.InternalURL != "" && !c.Spec.NoIPVerifier { + if c.dn.IPVerifier != nil && !c.Spec.NoIPVerifier { + // An empty URL here means Prepare never ran, which cannot happen through Devnet.Start — + // it prepares the verifier before any client is added. Refused rather than tolerated: a + // client silently dropped to the no-proof path would leave every test that connects + // still passing while no longer covering the proof, which is the property this whole + // change rests on. + if c.dn.IPVerifier.InternalURL == "" { + return fmt.Errorf("ip-verifier has no internal URL: Prepare has not run") + } env["DZ_IP_VERIFIER_URL"] = c.dn.IPVerifier.InternalURL } if c.Spec.EnableQAAgent { diff --git a/e2e/internal/devnet/ip_verifier.go b/e2e/internal/devnet/ip_verifier.go index 069c8fae88..95d61a07ba 100644 --- a/e2e/internal/devnet/ip_verifier.go +++ b/e2e/internal/devnet/ip_verifier.go @@ -35,6 +35,13 @@ const ( ipVerifierStartupTimeout = 60 * time.Second ) +// ipVerifierHealthClient bounds a single health probe. poll.Until calls its condition +// synchronously, so without a per-request timeout a probe that hangs rather than being refused — +// an accepting docker-proxy port with nothing behind it — would block past the startup budget +// until the caller's context died, which is the opposite of the bounded failure waitForHealthy +// exists to give. +var ipVerifierHealthClient = &http.Client{Timeout: 5 * time.Second} + // IPVerifierSpec configures the RFC-27 IP ownership verification service container. // // The service must sit on the CYOA network, not only on the default network. It signs the source @@ -193,7 +200,7 @@ func (v *IPVerifier) waitForHealthy(ctx context.Context, containerID string) err if err != nil { return false, err } - resp, err := http.DefaultClient.Do(req) + resp, err := ipVerifierHealthClient.Do(req) if err != nil { return false, nil }