e2e: run an ip-verifier in every devnet and cover the RFC-27 proof outcomes

.github/workflows/e2e.yml

@@ -43,6 +43,7 @@ env:
   DZ_DEVICE_HEALTH_ORACLE_IMAGE: ghcr.io/malbeclabs/dz-e2e/device-health-oracle:${{ github.event.inputs.image_tag || github.sha }}
   DZ_GEOPROBE_IMAGE: ghcr.io/malbeclabs/dz-e2e/geoprobe:${{ github.event.inputs.image_tag || github.sha }}
   DZ_SENTINEL_IMAGE: ghcr.io/malbeclabs/dz-e2e/sentinel:${{ github.event.inputs.image_tag || github.sha }}
+  DZ_IP_VERIFIER_IMAGE: ghcr.io/malbeclabs/dz-e2e/ip-verifier:${{ github.event.inputs.image_tag || github.sha }}
   DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE: ghcr.io/malbeclabs/dz-e2e/validator-metadata-service-mock:${{ github.event.inputs.image_tag || github.sha }}
 
 jobs:
@@ -291,6 +292,7 @@ jobs:
           docker push ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }}
           docker push ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }}
           docker push ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }}
+          docker push ${{ env.DZ_IMAGE_REPO }}/ip-verifier:${{ env.DZ_IMAGE_TAG }}
           docker push ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }}
       - name: Discover tests and distribute across shards
         if: steps.gate.outputs.run-e2e == 'true'
@@ -434,6 +436,7 @@ jobs:
           pull_with_retry ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }}
           pull_with_retry ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }}
           pull_with_retry ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }}
+          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/ip-verifier:${{ env.DZ_IMAGE_TAG }}
           pull_with_retry ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }}
           pull_with_retry ghcr.io/malbeclabs/dz-e2e/prometheus:v2.54.1
           pull_with_retry public.ecr.aws/influxdb/influxdb:1.8

CLAUDE.md

@@ -147,6 +147,7 @@ The local devnet runs in Docker containers with the naming convention `dz-local-
 - **Clients**: `dz-local-client-{pubkey}` - Client containers running doublezerod
 - **Manager**: `dz-local-manager` - Runs the doublezero CLI for admin operations
 - **Controller**: `dz-local-controller` - Pushes configs to devices
+- **IP verifier**: `dz-local-ip-verifier` - Signs RFC-27 IP ownership proofs for `connect` (see `e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md`)
 
 ### Arista Device Interaction
 

crates/doublezero-ip-verifier/src/settings.rs

@@ -9,6 +9,7 @@ use clap::Parser;
 use doublezero_config::Environment;
 use ipnetwork::IpNetwork;
 use solana_keypair::{read_keypair_file, Keypair};
+use solana_program::pubkey::Pubkey;
 use std::{net::SocketAddr, path::PathBuf, sync::Arc, time::Duration};
 
 /// Every flag also reads a `DZ_IP_VERIFIER_`-prefixed environment variable, which is how the
@@ -29,6 +30,13 @@ pub struct AppArgs {
     #[arg(long, env = "DZ_IP_VERIFIER_LEDGER_RPC")]
     pub ledger_rpc: Option<String>,
 
+    /// Serviceability program ID `GlobalState` is read from. Defaults to the program `--env`
+    /// names. Needed wherever the deployed program is not that one — a devnet deploys its own,
+    /// and reading the environment's program ID there finds no `GlobalState` account at all, so
+    /// the authority check has nothing to compare against and never fails.
+    #[arg(long, env = "DZ_IP_VERIFIER_SERVICEABILITY_PROGRAM_ID")]
+    pub serviceability_program_id: Option<Pubkey>,
+
     /// Path to the verifier keypair JSON file. Its public key must match
     /// `GlobalState.ip_verifier_authority_pk` or every proof this service issues is rejected
     /// onchain.
@@ -159,17 +167,23 @@ impl AppArgs {
         }
     }
 
+    pub fn serviceability_program_id(&self) -> anyhow::Result<Pubkey> {
+        match self.serviceability_program_id {
+            Some(program_id) => Ok(program_id),
+            None => Ok(self
+                .env
+                .config()
+                .map_err(|err| anyhow::anyhow!("{err}"))
+                .with_context(|| format!("no network config for environment {}", self.env))?
+                .serviceability_program_id),
+        }
+    }
+
     /// The ledger client both background loops read through.
     pub fn ledger(&self) -> anyhow::Result<Ledger> {
-        let config = self
-            .env
-            .config()
-            .map_err(|err| anyhow::anyhow!("{err}"))
-            .with_context(|| format!("no network config for environment {}", self.env))?;
-
         Ok(Ledger::new(
             self.ledger_rpc_url()?,
-            config.serviceability_program_id,
+            self.serviceability_program_id()?,
         ))
     }
 
@@ -250,6 +264,30 @@ mod tests {
         assert_eq!(args.ledger_rpc_url().unwrap(), "http://localhost:8899");
     }
 
+    #[test]
+    fn the_serviceability_program_id_defaults_to_the_environment() {
+        let args = parse(&["--env", "testnet"]);
+        assert_eq!(
+            args.serviceability_program_id().unwrap(),
+            Environment::Testnet
+                .config()
+                .unwrap()
+                .serviceability_program_id
+        );
+    }
+
+    #[test]
+    fn an_explicit_serviceability_program_id_wins() {
+        let program_id = Pubkey::new_unique();
+        let args = parse(&[
+            "--env",
+            "testnet",
+            "--serviceability-program-id",
+            &program_id.to_string(),
+        ]);
+        assert_eq!(args.serviceability_program_id().unwrap(), program_id);
+    }
+
     #[test]
     fn a_missing_keypair_file_reports_the_path_and_nothing_else() {
         let args = AppArgs::try_parse_from([

e2e/.env.local

@@ -13,3 +13,4 @@ DZ_DEVICE_HEALTH_ORACLE_IMAGE=${DZ_IMAGE_REPO}/device-health-oracle:${DZ_IMAGE_T
 DZ_GEOPROBE_IMAGE=${DZ_IMAGE_REPO}/geoprobe:${DZ_IMAGE_TAG}
 DZ_SENTINEL_IMAGE=${DZ_IMAGE_REPO}/sentinel:${DZ_IMAGE_TAG}
 DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE=${DZ_IMAGE_REPO}/validator-metadata-service-mock:${DZ_IMAGE_TAG}
+DZ_IP_VERIFIER_IMAGE=${DZ_IMAGE_REPO}/ip-verifier:${DZ_IMAGE_TAG}

e2e/docker/base.dockerfile

@@ -122,6 +122,7 @@ RUN --mount=type=cache,id=cargo-${CARGO_LOCK_HASH},target=/cargo \
     RUSTFLAGS="-C link-arg=-fuse-ld=mold" cargo build --workspace --release --exclude doublezero-serviceability --exclude doublezero-telemetry && \
     cp /target/release/doublezero ${BIN_DIR}/ && \
     cp /target/release/doublezero-sentinel ${BIN_DIR}/ && \
+    cp /target/release/doublezero-ip-verifier ${BIN_DIR}/ && \
     cp /target/release/fork-accounts ${BIN_DIR}/
 
 # Force COPY in later stages to always copy the binaries, even if they appear to be the same.

e2e/docker/ip-verifier/Dockerfile

@@ -0,0 +1,13 @@
+ARG BASE_IMAGE=undefined
+FROM ${BASE_IMAGE} AS base
+
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y ca-certificates curl bash
+
+COPY --from=base /doublezero/bin/doublezero-ip-verifier /usr/local/bin/doublezero-ip-verifier
+
+ENTRYPOINT ["doublezero-ip-verifier"]

e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md

@@ -0,0 +1,92 @@
+# IP ownership verification in the local devnet
+
+RFC-27 ([`rfcs/rfc27-ip-verification.md`](../../rfcs/rfc27-ip-verification.md)) has `connect`
+attach a proof, signed by a DoubleZero-operated verifier, that the caller can originate traffic
+from the `client_ip` it is binding. Every devnet runs that verifier — `dev/dzctl` and the Go e2e
+suite alike — so the local flow matches production.
+
+## What comes up
+
+The verifier is on by default: `IPVerifierSpec.Disabled` is the opt-out, so a devnet that says
+nothing about it gets one. `dzctl start` brings up a `dz-local-ip-verifier` container (image
+`dz-local/ip-verifier:dev`) alongside the rest of the stack; an e2e test gets the same container
+named for its own deploy ID:
+
+- **Keypair**: generated per deploy into `dev/.deploy/dz-local/ip-verifier-keypair.json`. Devnet
+  only — nothing is checked in.
+- **Onchain authority**: the keypair's pubkey is written to
+  `GlobalState.ip_verifier_authority_pk` before the container starts. The service reads the
+  authority from the ledger at startup and exits if it does not name its own key, so the order
+  matters.
+- **Networks**: the default network (to reach the ledger) *and* the CYOA network, on host ID 250.
+- **Client wiring**: every client container gets `DZ_IP_VERIFIER_URL` pointing at the verifier's
+  **CYOA** address.
+
+That last pair is the point. The verifier signs the source address it observes the request arrive
+from, and `connect` refuses a proof for any address other than the one it is provisioning. A local
+client provisions its CYOA address, so the request has to reach the verifier over the CYOA network
+for the two to agree — reached over the default network instead, the observed address would be the
+client's default-network address and every connect would hard-fail on the mismatch. This is the
+same class of problem as the proxy handling in production, where the address the service sees is
+the proxy's unless it is configured to read a forwarded one.
+
+The CYOA subnet is allocated from `9.128.0.0/9`, which is globally routable, so the verifier's
+`not_globally_routable` refusal (which an RFC-1918 source would hit) does not fire.
+
+## Enforcement is off by default
+
+The `require-ip-ownership-proof` feature flag is **clear** in the local `GlobalState`. A proof is
+obtained and attached, but the program accepts a create without one — so a stack where the
+verifier is down, or a client that cannot reach it, still connects. That mirrors an environment
+whose rollout has not flipped the flag yet.
+
+To exercise the enforcement path, turn it on:
+
+```bash
+docker exec dz-local-manager \
+  doublezero global-config feature-flags set --enable require-ip-ownership-proof
+```
+
+and off again:
+
+```bash
+docker exec dz-local-manager \
+  doublezero global-config feature-flags set --disable require-ip-ownership-proof
+```
+
+From a Go e2e test, `devnet.SetIPOwnershipProofFeatureFlag(ctx, true)` does the same thing.
+
+## From a Go e2e test
+
+Because the verifier is on by default, an ordinary `connect` in any e2e test already obtains and
+attaches a real proof. Two knobs cover the cases that need something else:
+
+- `ClientSpec.NoIPVerifier` leaves `DZ_IP_VERIFIER_URL` unset for one client, so its `connect`
+  obtains no proof at all — the path an environment takes before its verifier exists.
+- `IPVerifierSpec.AuthorityRefreshSecs` pins how often the service re-reads the onchain authority.
+  Set it long and rotate the authority with `devnet.SetIPVerifierAuthority` and the service keeps
+  signing with a key `GlobalState` no longer names, which is how a test produces a proof that gets
+  refused.
+
+`e2e/ip_ownership_proof_test.go` uses all three paths.
+
+## Poking at it
+
+```bash
+# Health: 200 once the cached ledger epoch is fresh and the ledger names this key.
+docker exec dz-local-ip-verifier curl -sS localhost:8080/health
+
+# What the ledger thinks the authority is.
+docker exec dz-local-manager doublezero global-config authority get
+
+# A proof, as a client would ask for it. Run through a shell in the container: DZ_IP_VERIFIER_URL
+# is set in the client's environment, and `docker exec curl` would have the host shell expand it.
+docker exec dz-local-client-<pubkey> bash -c \
+  'curl -sS -X POST "$DZ_IP_VERIFIER_URL/v1/proof" \
+   -H "content-type: application/json" \
+   -d "{\"payer\":\"<pubkey>\",\"user_type\":0}"'
+```
+
+The rate limit is raised well above the production default in the devnet (burst 1000, 6000/min):
+a devnet has one source address per client and a test can reconnect in a tight loop, which the
+production values would turn into `rate_limited` refusals unrelated to what is being tested.

e2e/internal/devnet/builder.go

@@ -118,6 +118,12 @@ func BuildContainerImages(ctx context.Context, log *slog.Logger, workspaceDir st
 			dockerfile: filepath.Join(dockerfilesDir, "sentinel", "Dockerfile"),
 			args:       append([]string{"--build-arg", baseImageArg}, extraArgs...),
 		},
+		{
+			name:       "ip-verifier",
+			image:      os.Getenv("DZ_IP_VERIFIER_IMAGE"),
+			dockerfile: filepath.Join(dockerfilesDir, "ip-verifier", "Dockerfile"),
+			args:       append([]string{"--build-arg", baseImageArg}, extraArgs...),
+		},
 		{
 			name:       "validator-metadata-service-mock",
 			image:      os.Getenv("DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE"),

e2e/internal/devnet/client.go

@@ -61,6 +61,10 @@ type ClientSpec struct {
 	// CYOANetworkIPHostID is the offset into the host portion of the subnet (must be < 2^(32 - prefixLen)).
 	CYOANetworkIPHostID uint32
 
+	// NoIPVerifier leaves DZ_IP_VERIFIER_URL unset for this client, so its `connect` obtains no
+	// RFC-27 proof even in a devnet running a verifier.
+	NoIPVerifier bool
+
 	// EnableQAAgent starts the QA agent inside the client container for local QA testing.
 	EnableQAAgent bool
 	// QAAgentPort is the port the QA agent listens on inside the container (default: 7009).
@@ -234,6 +238,17 @@ func (c *Client) Start(ctx context.Context) error {
 		"DZ_SERVICEABILITY_PROGRAM_ID": c.dn.Manager.ServiceabilityProgramID,
 		"DZ_CLIENT_EXTRA_ARGS":         strings.Join(extraArgs, " "),
 	}
+	// Point `connect` at the devnet verifier. The `--env local` config carries no verifier URL,
+	// so without this a client obtains no proof at all. The URL is the verifier's CYOA address,
+	// so the source address it observes is the same one the client binds its tunnel to —
+	// `connect` hard-fails on a proof for any other address.
+	//
+	// NoIPVerifier leaves it unset, which is how a test covers the no-proof path: the CLI reports
+	// nothing to reach and creates the user without a proof, which the program accepts while
+	// require-ip-ownership-proof is clear.
+	if c.dn.IPVerifier != nil && c.dn.IPVerifier.InternalURL != "" && !c.Spec.NoIPVerifier {
+		env["DZ_IP_VERIFIER_URL"] = c.dn.IPVerifier.InternalURL
+	}
 	if c.Spec.EnableQAAgent {
 		env["DZ_QAAGENT_ENABLE"] = "true"
 		env["DZ_QAAGENT_ADDR"] = fmt.Sprintf("0.0.0.0:%d", qaAgentPort)

e2e/internal/devnet/cmd/devnet.go

@@ -90,6 +90,10 @@ func NewLocalDevnet(log *slog.Logger, deployID string) (*LocalDevnet, error) {
 			Verbose:  true,
 			Interval: 10 * time.Second,
 		},
+		// The RFC-27 verifier is on by default (IPVerifierSpec.Disabled), so `connect` always
+		// has one to reach. Enforcement is a separate switch: the require-ip-ownership-proof
+		// feature flag stays clear, so a proof is attached but not demanded. See
+		// e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md.
 	}, log, dockerClient, subnetAllocator)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create devnet: %w", err)

e2e/internal/devnet/devnet.go

@@ -41,6 +41,7 @@ const (
 
 	containerDoublezeroKeypairPath = "/root/.config/doublezero/id.json"
 	containerSolanaKeypairPath     = "/root/.config/solana/id.json"
+	containerIPVerifierKeypairPath = "/etc/doublezero/ip-verifier.json"
 
 	// defaultNetworkBaseCIDR is the address range the devnet's default network is allocated from.
 	// It is kept separate from the CYOA network range (9.128.0.0/9) so tests that detect interfaces
@@ -94,6 +95,7 @@ type DevnetSpec struct {
 	InfluxDB                     InfluxDBSpec
 	Prometheus                   PrometheusSpec
 	Sentinel                     SentinelSpec
+	IPVerifier                   IPVerifierSpec
 	ValidatorMetadataServiceMock ValidatorMetadataServiceMockSpec
 	Devices                      map[string]DeviceSpec
 	Clients                      map[string]ClientSpec
@@ -129,6 +131,7 @@ type Devnet struct {
 	InfluxDB                     *InfluxDB
 	Prometheus                   *Prometheus
 	Sentinel                     *Sentinel
+	IPVerifier                   *IPVerifier
 	ValidatorMetadataServiceMock *ValidatorMetadataServiceMock
 	Devices                      map[string]*Device
 	Clients                      map[string]*Client
@@ -180,6 +183,10 @@ func (s *DevnetSpec) Validate() error {
 		return fmt.Errorf("prometheus: %w", err)
 	}
 
+	if err := s.IPVerifier.Validate(s.CYOANetwork); err != nil {
+		return fmt.Errorf("ip-verifier: %w", err)
+	}
+
 	if s.Devices == nil {
 		s.Devices = make(map[string]DeviceSpec)
 	}
@@ -295,6 +302,33 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA
 		}
 	}
 
+	// A cloned-state devnet cannot provision its own verifier authority: its GlobalState came from
+	// a remote cluster and the local manager cannot write to it, so a generated key would never be
+	// the one the program trusts and the container would exit at startup. Default the verifier off
+	// there. A stack that does have a verifier the cloned state already trusts names its keypair
+	// explicitly, which opts back in.
+	if spec.SkipProgramDeploy && spec.IPVerifier.KeypairPath == "" {
+		spec.IPVerifier.Disabled = true
+	}
+
+	// If the ip-verifier keypair path is not provided, generate a new keypair or use an existing
+	// one in the deploy directory if it exists. Devnet-only: it is written to the deploy
+	// directory rather than checked in, and its pubkey is what the local GlobalState names as the
+	// verifier authority.
+	if !spec.IPVerifier.Disabled && spec.IPVerifier.KeypairPath == "" {
+		ipVerifierKeypairPath := filepath.Join(spec.DeployDir, "ip-verifier-keypair.json")
+		generated, err := generateKeypairIfNotExists(ipVerifierKeypairPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to generate ip-verifier keypair: %w", err)
+		}
+		spec.IPVerifier.KeypairPath = ipVerifierKeypairPath
+		if generated {
+			log.Debug("--> Generated ip-verifier keypair", "path", ipVerifierKeypairPath)
+		} else {
+			log.Debug("--> Using existing ip-verifier keypair", "path", ipVerifierKeypairPath)
+		}
+	}
+
 	// Validate the spec.
 	if err := spec.Validate(); err != nil {
 		return nil, fmt.Errorf("failed to validate spec: %w", err)
@@ -376,6 +410,12 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA
 		dn:  dn,
 		log: log.With("component", "sentinel"),
 	}
+	if !spec.IPVerifier.Disabled {
+		dn.IPVerifier = &IPVerifier{
+			dn:  dn,
+			log: log.With("component", "ip-verifier"),
+		}
+	}
 	dn.ValidatorMetadataServiceMock = &ValidatorMetadataServiceMock{
 		dn:  dn,
 		log: log.With("component", "validator-metadata-service-mock"),
@@ -535,6 +575,28 @@ func (d *Devnet) Start(ctx context.Context, buildConfig *BuildConfig) error {
 		return fmt.Errorf("failed to create CYOA network: %w", err)
 	}
 
+	// Start the ip-verifier if it's not already running. It comes after the CYOA network, which
+	// it attaches to so that a client's proof request is observed arriving from the same address
+	// the client binds its tunnel to. Its authority has to be onchain before it starts: the
+	// service reads GlobalState at startup and exits if the authority is not its own key.
+	if d.IPVerifier != nil {
+		if err := d.IPVerifier.Prepare(); err != nil {
+			return fmt.Errorf("failed to prepare ip-verifier: %w", err)
+		}
+		// Skipped for a stack running cloned state: its GlobalState came from a remote cluster
+		// and the local manager is not its authority, so the write would fail. Reaching here with
+		// SkipProgramDeploy means the caller named a keypair explicitly — a verifier the cloned
+		// state already trusts — so the authority is already correct and needs no write.
+		if !d.Spec.SkipProgramDeploy {
+			if err := d.SetIPVerifierAuthority(ctx, d.IPVerifier.Pubkey); err != nil {
+				return fmt.Errorf("failed to set ip-verifier authority: %w", err)
+			}
+		}
+		if _, err := d.IPVerifier.StartIfNotRunning(ctx); err != nil {
+			return fmt.Errorf("failed to start ip-verifier: %w", err)
+		}
+	}
+
 	// We don't support starting with devices yet.
 	// The AddDevice method can be used to add devices after the devnet is started.
 	if len(d.Spec.Devices) > 0 {