e2e/qa: run testnet shred-subscription settlement on solana devnet

client/doublezero/src/command/connect.rs

@@ -583,7 +583,7 @@ impl ProvisioningCliCommand {
                         spinner.println("❌ Error creating user");
                         spinner.println(format!("\n{}: {:?}\n", "Error", e));
 
-                        Err(eyre::eyre!("Error creating user"))
+                        Err(eyre::eyre!("Error creating user: {e:?}"))
                     }
                 }
             }?,
@@ -829,7 +829,7 @@ impl ProvisioningCliCommand {
                     Err(e) => {
                         spinner.println("❌ Error creating user");
                         spinner.println(format!("\n{}: {:?}\n", "Error", e));
-                        return Err(eyre::eyre!("Error creating user"));
+                        return Err(eyre::eyre!("Error creating user: {e:?}"));
                     }
                 };
 

config/constants.go

@@ -19,18 +19,20 @@ const (
 	MainnetUSDCMint                     = "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v"
 
 	// Testnet constants.
-	TestnetLedgerPublicRPCURL            = "https://doublezerolocalnet.rpcpool.com/8a4fd3f4-0977-449f-88c7-63d4b0f10f16"
-	TestnetServiceabilityProgramID       = "DZtnuQ839pSaDMFG5q1ad2V95G82S5EC4RrB3Ndw2Heb"
-	TestnetTelemetryProgramID            = "3KogTMmVxc5eUHtjZnwm136H5P8tvPwVu4ufbGPvM7p1"
-	TestnetInternetLatencyCollectorPK    = "HWGQSTmXWMB85NY2vFLhM1nGpXA8f4VCARRyeGNbqDF1"
-	TestnetDeviceLocalASN                = 65342
-	TestnetTwoZOracleURL                 = "https://sol-2z-oracle-api-v1.testnet.doublezero.xyz"
-	TestnetSolanaRPC                     = "https://api.testnet.solana.com"
-	TestnetTelemetryFlowIngestURL        = "http://telemetry-flow-in.testnet.doublezero.xyz"
-	TestnetTelemetryStateIngestURL       = "https://telemetry-state-in-testnet.doublezero.xyz"
-	TestnetGeolocationProgramID          = "3AG2BCA7gAm47Q6xZzPQcUUYvnBjxAvPKnPz919cxHF4"
-	TestnetShredSubscriptionProgramID    = "dzshrr3yL57SB13sJPYHYo3TV8Bo1i1FxkyrZr3bKNE"
-	TestnetUSDCMint                      = "uSDZq2RMuxrEf7gqgDjR8wJCtCyaDAQk2e5jLAaoeeM"
+	TestnetLedgerPublicRPCURL         = "https://doublezerolocalnet.rpcpool.com/8a4fd3f4-0977-449f-88c7-63d4b0f10f16"
+	TestnetServiceabilityProgramID    = "DZtnuQ839pSaDMFG5q1ad2V95G82S5EC4RrB3Ndw2Heb"
+	TestnetTelemetryProgramID         = "3KogTMmVxc5eUHtjZnwm136H5P8tvPwVu4ufbGPvM7p1"
+	TestnetInternetLatencyCollectorPK = "HWGQSTmXWMB85NY2vFLhM1nGpXA8f4VCARRyeGNbqDF1"
+	TestnetDeviceLocalASN             = 65342
+	TestnetTwoZOracleURL              = "https://sol-2z-oracle-api-v1.testnet.doublezero.xyz"
+	TestnetSolanaRPC                  = "https://api.testnet.solana.com"
+	TestnetTelemetryFlowIngestURL     = "http://telemetry-flow-in.testnet.doublezero.xyz"
+	TestnetTelemetryStateIngestURL    = "https://telemetry-state-in-testnet.doublezero.xyz"
+	TestnetGeolocationProgramID       = "3AG2BCA7gAm47Q6xZzPQcUUYvnBjxAvPKnPz919cxHF4"
+	TestnetShredSubscriptionProgramID = "dzshrr3yL57SB13sJPYHYo3TV8Bo1i1FxkyrZr3bKNE"
+	// Testnet shred-subscription runs on Solana devnet, so settlement uses the
+	// Solana devnet USDC mint (not the DZ ledger testnet mint).
+	TestnetUSDCMint                      = "4zMMC9srt5Ri5X14GAgXhaHii3GnPAEERYPJgZJDncDU"
 	TestnetTelemetryGNMITunnelServerAddr = "gnmic-testnet.doublezero.xyz:443"
 
 	// Devnet constants.

e2e/internal/qa/client.go

@@ -26,10 +26,21 @@ import (
 )
 
 const (
-	disconnectTimeout                = 150 * time.Second
-	waitForStatusUpTimeout           = 90 * time.Second
-	waitForStatusDisconnectedTimeout = 90 * time.Second
-	waitForUserDeletionTimeout       = 90 * time.Second
+	disconnectTimeout      = 150 * time.Second
+	waitForStatusUpTimeout = 90 * time.Second
+	// Multicast (shred-subscription) tunnel UP is the oracle *subscribe* path. For
+	// a pre-existing validator-owned user the oracle adds the shred groups via its
+	// ~60s reconcile loop (not inline on the allocation ack), so subscribe latency
+	// is cadence-bound and has been observed up to ~3.5 min on devnet. This is NOT
+	// affected by the withdraw fix (doublezerofoundation/doublezero-offchain#395),
+	// so keep a generous budget until the oracle subscribes on grant.
+	waitForMulticastStatusUpTimeout = 300 * time.Second
+	// Tunnel DOWN is prompt with #395: `shreds withdraw` issues a real withdrawal,
+	// and the oracle tears the subscription down in its per-poll instant-withdrawal
+	// handler (observed ~20s), so the standard 90s budget is ample.
+	waitForMulticastStatusDisconnectedTimeout = 90 * time.Second
+	waitForStatusDisconnectedTimeout          = 90 * time.Second
+	waitForUserDeletionTimeout                = 90 * time.Second
 
 	// NOTE: This needs to be longer than 1m since BGP can sometimes throttle activity for that
 	// amount of time if too much is happening consecutively for the same peers.
@@ -125,7 +136,8 @@ type Client struct {
 
 	// Settlement config passed to doublezero-solana shreds commands.
 	// SolanaRPCURL is the Solana RPC endpoint for settlement transactions (--url).
-	// On testnet this is the DZ ledger URL; on mainnet it's the public Solana RPC.
+	// On testnet this is Solana devnet (via SOLANA_RPC_URL); on mainnet the public
+	// Solana RPC; on devnet the DZ ledger URL.
 	SolanaRPCURL               string
 	ShredSubscriptionProgramID string
 	DZLedgerURL                string
@@ -155,11 +167,14 @@ func NewClient(ctx context.Context, log *slog.Logger, hostname string, port int,
 
 	serviceabilityClient := serviceability.New(rpc.New(networkConfig.LedgerPublicRPCURL), networkConfig.ServiceabilityProgramID)
 
-	// Settlement transactions on testnet/devnet use the DZ ledger RPC endpoint
-	// (which hosts the settlement programs). Mainnet and localnet use the
+	// Settlement transactions on devnet use the DZ ledger RPC endpoint (which
+	// hosts the settlement programs there). Testnet reads/writes the
+	// shred-subscription program on Solana devnet via networkConfig.SolanaRPCURL
+	// (the SOLANA_RPC_URL override, a Solana devnet RPC endpoint in CI;
+	// defaults to the public Solana endpoint). Mainnet and localnet use the
 	// standard Solana RPC.
 	solanaRPCURL := networkConfig.SolanaRPCURL
-	if networkConfig.Moniker == config.EnvTestnet || networkConfig.Moniker == config.EnvDevnet {
+	if networkConfig.Moniker == config.EnvDevnet {
 		solanaRPCURL = networkConfig.LedgerPublicRPCURL
 	}
 
@@ -393,19 +408,19 @@ func (c *Client) WaitForStatusUp(ctx context.Context) error {
 // its session is up. Prefer this over WaitForStatusUp in multi-tunnel contexts
 // where other tunnel types may already be present.
 func (c *Client) WaitForUnicastStatusUp(ctx context.Context) error {
-	return c.waitForUserTypeStatusUp(ctx, "IBRL", FindIBRLStatus)
+	return c.waitForUserTypeStatusUp(ctx, "IBRL", FindIBRLStatus, waitForStatusUpTimeout)
 }
 
 // WaitForMulticastStatusUp polls until a Multicast status entry exists and
 // its session is up. Prefer this over WaitForStatusUp in multi-tunnel contexts
 // where other tunnel types may already be present.
 func (c *Client) WaitForMulticastStatusUp(ctx context.Context) error {
-	return c.waitForUserTypeStatusUp(ctx, "Multicast", FindMulticastStatus)
+	return c.waitForUserTypeStatusUp(ctx, "Multicast", FindMulticastStatus, waitForMulticastStatusUpTimeout)
 }
 
 // waitForUserTypeStatusUp polls until find returns a non-nil status whose
 // session is up. userType is used only for log context.
-func (c *Client) waitForUserTypeStatusUp(ctx context.Context, userType string, find func([]*pb.Status) *pb.Status) error {
+func (c *Client) waitForUserTypeStatusUp(ctx context.Context, userType string, find func([]*pb.Status) *pb.Status, timeout time.Duration) error {
 	c.log.Debug("Waiting for status to be up", "host", c.Host, "userType", userType)
 	err := poll.Until(ctx, func() (bool, error) {
 		resp, err := c.grpcClient.GetStatus(ctx, &emptypb.Empty{})
@@ -414,7 +429,7 @@ func (c *Client) waitForUserTypeStatusUp(ctx context.Context, userType string, f
 		}
 		s := find(resp.Status)
 		return s != nil && IsStatusUp(s.SessionStatus), nil
-	}, waitForStatusUpTimeout, waitInterval)
+	}, timeout, waitInterval)
 	if err != nil {
 		return fmt.Errorf("failed to wait for %s status to be up on host %s: %w", userType, c.Host, err)
 	}
@@ -503,7 +518,7 @@ func (c *Client) WaitForStatusDisconnected(ctx context.Context) error {
 // Prefer this over WaitForStatusDisconnected in multi-tunnel contexts where
 // other tunnel types (e.g. IBRL) remain up after a multicast seat is withdrawn.
 func (c *Client) WaitForMulticastStatusDisconnected(ctx context.Context) error {
-	return c.waitForUserTypeStatusDisconnected(ctx, "Multicast", FindMulticastStatus)
+	return c.waitForUserTypeStatusDisconnected(ctx, "Multicast", FindMulticastStatus, waitForMulticastStatusDisconnectedTimeout)
 }
 
 // WaitForIBRLStatusDisconnected polls until no IBRL (or IBRLWithAllocatedIP)
@@ -537,7 +552,7 @@ func (c *Client) WaitForIBRLStatusDisconnected(ctx context.Context) error {
 
 // waitForUserTypeStatusDisconnected polls until find returns nil or a status
 // whose session is disconnected. userType is used only for log context.
-func (c *Client) waitForUserTypeStatusDisconnected(ctx context.Context, userType string, find func([]*pb.Status) *pb.Status) error {
+func (c *Client) waitForUserTypeStatusDisconnected(ctx context.Context, userType string, find func([]*pb.Status) *pb.Status, timeout time.Duration) error {
 	c.log.Debug("Waiting for status to be disconnected", "host", c.Host, "userType", userType)
 	err := poll.Until(ctx, func() (bool, error) {
 		resp, err := c.grpcClient.GetStatus(ctx, &emptypb.Empty{})
@@ -546,7 +561,7 @@ func (c *Client) waitForUserTypeStatusDisconnected(ctx context.Context, userType
 		}
 		s := find(resp.Status)
 		return s == nil || s.SessionStatus == UserStatusDisconnected, nil
-	}, waitForStatusDisconnectedTimeout, waitInterval)
+	}, timeout, waitInterval)
 	if err != nil {
 		return fmt.Errorf("failed to wait for %s status to be disconnected on host %s: %w", userType, c.Host, err)
 	}

e2e/internal/qa/client_settlement.go

@@ -106,11 +106,13 @@ func (c *Client) ClosestDevice(ctx context.Context) (*Device, error) {
 	return device, nil
 }
 
-// FeedSeatPrice calls the FeedSeatPrice RPC to query device seat prices.
-// This is an idempotent read, so on RPC failure it fails over to the next
-// endpoint and retries.
-func (c *Client) FeedSeatPrice(ctx context.Context) ([]*pb.DevicePrice, error) {
-	c.log.Debug("Querying seat prices", "host", c.Host)
+// FeedSeatPrice calls the FeedSeatPrice RPC to query seat pricing for a single
+// device (by pubkey). Querying by pubkey avoids device-code resolution, which
+// the CLI refuses when it can't classify the cluster (e.g. a private Solana
+// devnet RPC URL). This is an idempotent read, so on RPC failure it fails over
+// to the next endpoint and retries.
+func (c *Client) FeedSeatPrice(ctx context.Context, devicePubkey string) ([]*pb.DevicePrice, error) {
+	c.log.Debug("Querying seat prices", "host", c.Host, "device", devicePubkey)
 	var prices []*pb.DevicePrice
 	err := c.withReadFailover(func(rpcURL string) error {
 		resp, err := c.grpcClient.FeedSeatPrice(ctx, &pb.FeedSeatPriceRequest{
@@ -119,6 +121,7 @@ func (c *Client) FeedSeatPrice(ctx context.Context) ([]*pb.DevicePrice, error) {
 			UsdcMint:                   c.USDCMint,
 			Keypair:                    c.Keypair,
 			ShredSubscriptionProgramId: c.ShredSubscriptionProgramID,
+			DevicePubkey:               devicePubkey,
 		})
 		if err != nil {
 			return err
@@ -224,7 +227,10 @@ func (c *Client) FeedSeatPay(ctx context.Context, devicePubkey string, amount st
 		c.log.Error("Seat payment failed", "host", c.Host, "device", devicePubkey, "output", resp.GetOutput())
 		return fmt.Errorf("seat payment failed on host %s: %s", c.Host, resp.GetOutput())
 	}
-	c.log.Debug("Seat payment successful", "host", c.Host, "device", devicePubkey)
+	// Surface the CLI output (which includes the on-chain tx signatures, e.g.
+	// "Fund escrow (<amount> USDC): <sig>") so failed/flaky runs are traceable
+	// on-chain from the test log.
+	c.log.Info("Seat payment successful", "host", c.Host, "device", devicePubkey, "output", resp.GetOutput())
 	return nil
 }
 
@@ -249,7 +255,9 @@ func (c *Client) FeedSeatWithdraw(ctx context.Context, devicePubkey string) erro
 	if !resp.GetSuccess() {
 		return fmt.Errorf("seat withdrawal failed on host %s: %s", c.Host, resp.GetOutput())
 	}
-	c.log.Debug("Seat withdrawal successful", "host", c.Host, "device", devicePubkey)
+	// Surface the CLI output so the branch taken (instant-withdrawal request vs.
+	// "Close payment escrow") and the tx signature are visible in the test log.
+	c.log.Info("Seat withdrawal successful", "host", c.Host, "device", devicePubkey, "output", resp.GetOutput())
 	return nil
 }
 

e2e/internal/rpc/agent.go

@@ -362,6 +362,13 @@ func (q *QAAgent) FeedSeatPrice(ctx context.Context, req *pb.FeedSeatPriceReques
 	if req.GetSolanaRpcUrl() != "" {
 		args = append(args, "--url", req.GetSolanaRpcUrl())
 	}
+	// Query a single device by pubkey rather than listing all. The list path
+	// resolves device codes via serviceability, which the CLI refuses when it
+	// can't classify the cluster (e.g. a private Solana devnet RPC URL, seen as
+	// localnet); passing --device sidesteps code resolution entirely.
+	if req.GetDevicePubkey() != "" {
+		args = append(args, "--device", req.GetDevicePubkey())
+	}
 
 	cmdCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
 	defer cancel()

e2e/proto/qa/agent.proto

@@ -238,6 +238,7 @@ message FeedSeatPriceRequest {
   string usdc_mint = 3;
   string keypair = 4;
   string shred_subscription_program_id = 5;
+  string device_pubkey = 6;
 }
 
 message DevicePrice {

e2e/qa_multicast_settlement_test.go

@@ -130,12 +130,14 @@ func TestQA_MulticastSettlement(t *testing.T) {
 	}
 
 	if !t.Run("query_seat_price", func(t *testing.T) {
-		prices, err := client.FeedSeatPrice(ctx)
+		prices, err := client.FeedSeatPrice(ctx, device.PubKey)
 		require.NoError(t, err, "failed to get seat prices")
 
+		// Match by pubkey, not code: querying by --device skips code resolution,
+		// so the returned rows may not carry a device_code.
 		var price *pb.DevicePrice
 		for _, p := range prices {
-			if p.DeviceCode == device.Code {
+			if p.DevicePubkey == device.PubKey {
 				price = p
 				break
 			}