e2e: cover RFC-27 proof enforcement with the feature flag set

.github/workflows/e2e.yml

@@ -43,6 +43,7 @@ env:
   DZ_DEVICE_HEALTH_ORACLE_IMAGE: ghcr.io/malbeclabs/dz-e2e/device-health-oracle:${{ github.event.inputs.image_tag || github.sha }}
   DZ_GEOPROBE_IMAGE: ghcr.io/malbeclabs/dz-e2e/geoprobe:${{ github.event.inputs.image_tag || github.sha }}
   DZ_SENTINEL_IMAGE: ghcr.io/malbeclabs/dz-e2e/sentinel:${{ github.event.inputs.image_tag || github.sha }}
+  DZ_IP_VERIFIER_IMAGE: ghcr.io/malbeclabs/dz-e2e/ip-verifier:${{ github.event.inputs.image_tag || github.sha }}
   DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE: ghcr.io/malbeclabs/dz-e2e/validator-metadata-service-mock:${{ github.event.inputs.image_tag || github.sha }}
 
 jobs:
@@ -291,6 +292,7 @@ jobs:
           docker push ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }}
           docker push ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }}
           docker push ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }}
+          docker push ${{ env.DZ_IMAGE_REPO }}/ip-verifier:${{ env.DZ_IMAGE_TAG }}
           docker push ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }}
       - name: Discover tests and distribute across shards
         if: steps.gate.outputs.run-e2e == 'true'

CLAUDE.md

@@ -147,6 +147,7 @@ The local devnet runs in Docker containers with the naming convention `dz-local-
 - **Clients**: `dz-local-client-{pubkey}` - Client containers running doublezerod
 - **Manager**: `dz-local-manager` - Runs the doublezero CLI for admin operations
 - **Controller**: `dz-local-controller` - Pushes configs to devices
+- **IP verifier**: `dz-local-ip-verifier` - Signs RFC-27 IP ownership proofs for `connect` (see `e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md`)
 
 ### Arista Device Interaction
 

e2e/.env.local

@@ -13,3 +13,4 @@ DZ_DEVICE_HEALTH_ORACLE_IMAGE=${DZ_IMAGE_REPO}/device-health-oracle:${DZ_IMAGE_T
 DZ_GEOPROBE_IMAGE=${DZ_IMAGE_REPO}/geoprobe:${DZ_IMAGE_TAG}
 DZ_SENTINEL_IMAGE=${DZ_IMAGE_REPO}/sentinel:${DZ_IMAGE_TAG}
 DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE=${DZ_IMAGE_REPO}/validator-metadata-service-mock:${DZ_IMAGE_TAG}
+DZ_IP_VERIFIER_IMAGE=${DZ_IMAGE_REPO}/ip-verifier:${DZ_IMAGE_TAG}

e2e/docker/base.dockerfile

@@ -122,6 +122,7 @@ RUN --mount=type=cache,id=cargo-${CARGO_LOCK_HASH},target=/cargo \
     RUSTFLAGS="-C link-arg=-fuse-ld=mold" cargo build --workspace --release --exclude doublezero-serviceability --exclude doublezero-telemetry && \
     cp /target/release/doublezero ${BIN_DIR}/ && \
     cp /target/release/doublezero-sentinel ${BIN_DIR}/ && \
+    cp /target/release/doublezero-ip-verifier ${BIN_DIR}/ && \
     cp /target/release/fork-accounts ${BIN_DIR}/
 
 # Force COPY in later stages to always copy the binaries, even if they appear to be the same.

e2e/docker/ip-verifier/Dockerfile

@@ -0,0 +1,13 @@
+ARG BASE_IMAGE=undefined
+FROM ${BASE_IMAGE} AS base
+
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y ca-certificates curl bash
+
+COPY --from=base /doublezero/bin/doublezero-ip-verifier /usr/local/bin/doublezero-ip-verifier
+
+ENTRYPOINT ["doublezero-ip-verifier"]

e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md

@@ -0,0 +1,115 @@
+# IP ownership verification in the local devnet
+
+RFC-27 ([`rfcs/rfc27-ip-verification.md`](../../rfcs/rfc27-ip-verification.md)) has `connect`
+attach a proof, signed by a DoubleZero-operated verifier, that the caller can originate traffic
+from the `client_ip` it is binding. Every devnet runs that verifier — `dev/dzctl` and the Go e2e
+suite alike — so the local flow matches production.
+
+## What comes up
+
+The verifier is on by default: `IPVerifierSpec.Disabled` is the opt-out, so a devnet that says
+nothing about it gets one. `dzctl start` brings up a `dz-local-ip-verifier` container (image
+`dz-local/ip-verifier:dev`) alongside the rest of the stack; an e2e test gets the same container
+named for its own deploy ID:
+
+- **Keypair**: generated per deploy into `dev/.deploy/dz-local/ip-verifier-keypair.json`. Devnet
+  only — nothing is checked in.
+- **Onchain authority**: the keypair's pubkey is written to
+  `GlobalState.ip_verifier_authority_pk` before the container starts. The service reads the
+  authority from the ledger at startup and exits if it does not name its own key, so the order
+  matters.
+- **Networks**: the default network (to reach the ledger) *and* the CYOA network, on host ID 250.
+- **Client wiring**: every client container gets `DZ_IP_VERIFIER_URL` pointing at the verifier's
+  **CYOA** address.
+
+That last pair is the point. The verifier signs the source address it observes the request arrive
+from, and `connect` refuses a proof for any address other than the one it is provisioning. A local
+client provisions its CYOA address, so the request has to reach the verifier over the CYOA network
+for the two to agree — reached over the default network instead, the observed address would be the
+client's default-network address and every connect would hard-fail on the mismatch. This is the
+same class of problem as the proxy handling in production, where the address the service sees is
+the proxy's unless it is configured to read a forwarded one.
+
+The CYOA subnet is allocated from `9.128.0.0/9`, which is globally routable, so the verifier's
+`not_globally_routable` refusal (which an RFC-1918 source would hit) does not fire.
+
+## Enforcement is off by default
+
+The `require-ip-ownership-proof` feature flag is **clear** in the local `GlobalState`. A proof is
+obtained and attached, but the program accepts a create without one — so a stack where the
+verifier is down, or a client that cannot reach it, still connects. That mirrors an environment
+whose rollout has not flipped the flag yet.
+
+To exercise the enforcement path, turn it on:
+
+```bash
+docker exec dz-local-manager \
+  doublezero global-config feature-flags set --enable require-ip-ownership-proof
+```
+
+and off again:
+
+```bash
+docker exec dz-local-manager \
+  doublezero global-config feature-flags set --disable require-ip-ownership-proof
+```
+
+From a Go e2e test, `devnet.SetIPOwnershipProofFeatureFlag(ctx, true)` does the same thing.
+
+## From a Go e2e test
+
+Because the verifier is on by default, an ordinary `connect` in any e2e test already obtains and
+attaches a real proof. Two knobs cover the cases that need something else:
+
+- `ClientSpec.NoIPVerifier` leaves `DZ_IP_VERIFIER_URL` unset for one client, so its `connect`
+  obtains no proof at all — the path an environment takes before its verifier exists.
+- `IPVerifierSpec.AuthorityRefreshSecs` pins how often the service re-reads the onchain authority.
+  Set it long and rotate the authority with `devnet.SetIPVerifierAuthority` and the service keeps
+  signing with a key `GlobalState` no longer names, which is how a test produces a proof that gets
+  refused.
+
+`e2e/ip_ownership_proof_test.go` uses all three paths.
+
+### Testing enforcement
+
+`devnet.SetIPOwnershipProofFeatureFlag(ctx, true)` sets `require-ip-ownership-proof`, which changes
+exactly one thing in the program: whether a *missing* proof is an error. A supplied proof is
+validated in full either way, so most proof failures are testable with the flag clear.
+
+Two things to know before writing a "flag on rejects everything" test:
+
+- **The manager is the sentinel authority** in a local devnet (`smartcontract_init.go` runs
+  `authority set --sentinel-authority me`), and the sentinel may create a user without a proof. So
+  `doublezero user create` from the manager still succeeds under enforcement — the rejection only
+  shows up on a create paid for by someone else, which is what `doublezero connect` on a client
+  does. `is_sentinel` compares the transaction payer.
+- **Most bad-proof cases never reach the chain.** The Rust SDK pre-flights version, payer,
+  `client_ip`, `user_type` and the signature before building the transaction, and `connect` refuses
+  an address disagreement before that. Of the program's proof errors only
+  `IpOwnershipProofRequired` (105) and `IpProofEpochOutOfWindow` (110) are reachable end to end,
+  and 110 needs a ledger epoch the devnet never advances past 0. The rest have program-level
+  coverage in `user_ip_proof_test.rs`.
+
+`e2e/ip_ownership_proof_enforcement_test.go` covers the flag-set cases, including a wildcard access
+pass — a pass created with no `--client-ip`, landing at the `0.0.0.0` PDA — which is the case
+RFC-27 exists for.
+
+## Poking at it
+
+```bash
+# Health: 200 once the cached ledger epoch is fresh and the ledger names this key.
+docker exec dz-local-ip-verifier curl -sS localhost:8080/health
+
+# What the ledger thinks the authority is.
+docker exec dz-local-manager doublezero global-config authority get
+
+# A proof, as a client would ask for it.
+docker exec dz-local-client-<pubkey> \
+  curl -sS -X POST "$DZ_IP_VERIFIER_URL/v1/proof" \
+  -H 'content-type: application/json' \
+  -d '{"payer":"<pubkey>","user_type":0}'
+```
+
+The rate limit is raised well above the production default in the devnet (burst 1000, 6000/min):
+a devnet has one source address per client and a test can reconnect in a tight loop, which the
+production values would turn into `rate_limited` refusals unrelated to what is being tested.

e2e/internal/devnet/builder.go

@@ -118,6 +118,12 @@ func BuildContainerImages(ctx context.Context, log *slog.Logger, workspaceDir st
 			dockerfile: filepath.Join(dockerfilesDir, "sentinel", "Dockerfile"),
 			args:       append([]string{"--build-arg", baseImageArg}, extraArgs...),
 		},
+		{
+			name:       "ip-verifier",
+			image:      os.Getenv("DZ_IP_VERIFIER_IMAGE"),
+			dockerfile: filepath.Join(dockerfilesDir, "ip-verifier", "Dockerfile"),
+			args:       append([]string{"--build-arg", baseImageArg}, extraArgs...),
+		},
 		{
 			name:       "validator-metadata-service-mock",
 			image:      os.Getenv("DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE"),

e2e/internal/devnet/client.go

@@ -61,6 +61,16 @@ type ClientSpec struct {
 	// CYOANetworkIPHostID is the offset into the host portion of the subnet (must be < 2^(32 - prefixLen)).
 	CYOANetworkIPHostID uint32
 
+	// NoIPVerifier leaves DZ_IP_VERIFIER_URL unset for this client, so its `connect` obtains no
+	// RFC-27 proof even in a devnet running a verifier.
+	NoIPVerifier bool
+
+	// DaemonClientIP overrides the address the daemon provisions, which is otherwise this client's
+	// CYOA address. Set it to an address the container does not own and the verifier observes the
+	// real CYOA source while `connect` binds the override, which is the disagreement `connect`
+	// refuses to provision through.
+	DaemonClientIP string
+
 	// EnableQAAgent starts the QA agent inside the client container for local QA testing.
 	EnableQAAgent bool
 	// QAAgentPort is the port the QA agent listens on inside the container (default: 7009).
@@ -219,7 +229,11 @@ func (c *Client) Start(ctx context.Context) error {
 	if c.Spec.LatencyProbeTunnelEndpoints {
 		extraArgs = append(extraArgs, "-latency-probe-tunnel-endpoints")
 	}
-	extraArgs = append(extraArgs, "-client-ip", clientCYOAIP)
+	daemonClientIP := clientCYOAIP
+	if c.Spec.DaemonClientIP != "" {
+		daemonClientIP = c.Spec.DaemonClientIP
+	}
+	extraArgs = append(extraArgs, "-client-ip", daemonClientIP)
 
 	// Determine QA agent port if enabled.
 	qaAgentPort := c.Spec.QAAgentPort
@@ -234,6 +248,17 @@ func (c *Client) Start(ctx context.Context) error {
 		"DZ_SERVICEABILITY_PROGRAM_ID": c.dn.Manager.ServiceabilityProgramID,
 		"DZ_CLIENT_EXTRA_ARGS":         strings.Join(extraArgs, " "),
 	}
+	// Point `connect` at the devnet verifier. The `--env local` config carries no verifier URL,
+	// so without this a client obtains no proof at all. The URL is the verifier's CYOA address,
+	// so the source address it observes is the same one the client binds its tunnel to —
+	// `connect` hard-fails on a proof for any other address.
+	//
+	// NoIPVerifier leaves it unset, which is how a test covers the no-proof path: the CLI reports
+	// nothing to reach and creates the user without a proof, which the program accepts while
+	// require-ip-ownership-proof is clear.
+	if c.dn.IPVerifier != nil && c.dn.IPVerifier.InternalURL != "" && !c.Spec.NoIPVerifier {
+		env["DZ_IP_VERIFIER_URL"] = c.dn.IPVerifier.InternalURL
+	}
 	if c.Spec.EnableQAAgent {
 		env["DZ_QAAGENT_ENABLE"] = "true"
 		env["DZ_QAAGENT_ADDR"] = fmt.Sprintf("0.0.0.0:%d", qaAgentPort)

e2e/internal/devnet/cmd/devnet.go

@@ -90,6 +90,10 @@ func NewLocalDevnet(log *slog.Logger, deployID string) (*LocalDevnet, error) {
 			Verbose:  true,
 			Interval: 10 * time.Second,
 		},
+		// The RFC-27 verifier is on by default (IPVerifierSpec.Disabled), so `connect` always
+		// has one to reach. Enforcement is a separate switch: the require-ip-ownership-proof
+		// feature flag stays clear, so a proof is attached but not demanded. See
+		// e2e/docs/IP_VERIFIER_LOCAL_DEVNET.md.
 	}, log, dockerClient, subnetAllocator)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create devnet: %w", err)

e2e/internal/devnet/devnet.go

@@ -41,6 +41,7 @@ const (
 
 	containerDoublezeroKeypairPath = "/root/.config/doublezero/id.json"
 	containerSolanaKeypairPath     = "/root/.config/solana/id.json"
+	containerIPVerifierKeypairPath = "/etc/doublezero/ip-verifier.json"
 
 	// defaultNetworkBaseCIDR is the address range the devnet's default network is allocated from.
 	// It is kept separate from the CYOA network range (9.128.0.0/9) so tests that detect interfaces
@@ -94,6 +95,7 @@ type DevnetSpec struct {
 	InfluxDB                     InfluxDBSpec
 	Prometheus                   PrometheusSpec
 	Sentinel                     SentinelSpec
+	IPVerifier                   IPVerifierSpec
 	ValidatorMetadataServiceMock ValidatorMetadataServiceMockSpec
 	Devices                      map[string]DeviceSpec
 	Clients                      map[string]ClientSpec
@@ -129,6 +131,7 @@ type Devnet struct {
 	InfluxDB                     *InfluxDB
 	Prometheus                   *Prometheus
 	Sentinel                     *Sentinel
+	IPVerifier                   *IPVerifier
 	ValidatorMetadataServiceMock *ValidatorMetadataServiceMock
 	Devices                      map[string]*Device
 	Clients                      map[string]*Client
@@ -180,6 +183,10 @@ func (s *DevnetSpec) Validate() error {
 		return fmt.Errorf("prometheus: %w", err)
 	}
 
+	if err := s.IPVerifier.Validate(s.CYOANetwork); err != nil {
+		return fmt.Errorf("ip-verifier: %w", err)
+	}
+
 	if s.Devices == nil {
 		s.Devices = make(map[string]DeviceSpec)
 	}
@@ -295,6 +302,33 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA
 		}
 	}
 
+	// A cloned-state devnet cannot provision its own verifier authority: its GlobalState came from
+	// a remote cluster and the local manager cannot write to it, so a generated key would never be
+	// the one the program trusts and the container would exit at startup. Default the verifier off
+	// there. A stack that does have a verifier the cloned state already trusts names its keypair
+	// explicitly, which opts back in.
+	if spec.SkipProgramDeploy && spec.IPVerifier.KeypairPath == "" {
+		spec.IPVerifier.Disabled = true
+	}
+
+	// If the ip-verifier keypair path is not provided, generate a new keypair or use an existing
+	// one in the deploy directory if it exists. Devnet-only: it is written to the deploy
+	// directory rather than checked in, and its pubkey is what the local GlobalState names as the
+	// verifier authority.
+	if !spec.IPVerifier.Disabled && spec.IPVerifier.KeypairPath == "" {
+		ipVerifierKeypairPath := filepath.Join(spec.DeployDir, "ip-verifier-keypair.json")
+		generated, err := generateKeypairIfNotExists(ipVerifierKeypairPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to generate ip-verifier keypair: %w", err)
+		}
+		spec.IPVerifier.KeypairPath = ipVerifierKeypairPath
+		if generated {
+			log.Debug("--> Generated ip-verifier keypair", "path", ipVerifierKeypairPath)
+		} else {
+			log.Debug("--> Using existing ip-verifier keypair", "path", ipVerifierKeypairPath)
+		}
+	}
+
 	// Validate the spec.
 	if err := spec.Validate(); err != nil {
 		return nil, fmt.Errorf("failed to validate spec: %w", err)
@@ -376,6 +410,12 @@ func New(spec DevnetSpec, log *slog.Logger, dockerClient *client.Client, subnetA
 		dn:  dn,
 		log: log.With("component", "sentinel"),
 	}
+	if !spec.IPVerifier.Disabled {
+		dn.IPVerifier = &IPVerifier{
+			dn:  dn,
+			log: log.With("component", "ip-verifier"),
+		}
+	}
 	dn.ValidatorMetadataServiceMock = &ValidatorMetadataServiceMock{
 		dn:  dn,
 		log: log.With("component", "validator-metadata-service-mock"),
@@ -535,6 +575,28 @@ func (d *Devnet) Start(ctx context.Context, buildConfig *BuildConfig) error {
 		return fmt.Errorf("failed to create CYOA network: %w", err)
 	}
 
+	// Start the ip-verifier if it's not already running. It comes after the CYOA network, which
+	// it attaches to so that a client's proof request is observed arriving from the same address
+	// the client binds its tunnel to. Its authority has to be onchain before it starts: the
+	// service reads GlobalState at startup and exits if the authority is not its own key.
+	if d.IPVerifier != nil {
+		if err := d.IPVerifier.Prepare(); err != nil {
+			return fmt.Errorf("failed to prepare ip-verifier: %w", err)
+		}
+		// Skipped for a stack running cloned state: its GlobalState came from a remote cluster
+		// and the local manager is not its authority, so the write would fail. Reaching here with
+		// SkipProgramDeploy means the caller named a keypair explicitly — a verifier the cloned
+		// state already trusts — so the authority is already correct and needs no write.
+		if !d.Spec.SkipProgramDeploy {
+			if err := d.SetIPVerifierAuthority(ctx, d.IPVerifier.Pubkey); err != nil {
+				return fmt.Errorf("failed to set ip-verifier authority: %w", err)
+			}
+		}
+		if _, err := d.IPVerifier.StartIfNotRunning(ctx); err != nil {
+			return fmt.Errorf("failed to start ip-verifier: %w", err)
+		}
+	}
+
 	// We don't support starting with devices yet.
 	// The AddDevice method can be used to add devices after the devnet is started.
 	if len(d.Spec.Devices) > 0 {