Integration tests

.cargo/config.toml

@@ -0,0 +1,3 @@
+[alias]
+local-test = "nextest run"
+int-test = "nextest run --features integration"

.github/workflows/test.yml

@@ -50,7 +50,7 @@ jobs:
           tool: cargo-nextest
 
       - name: Cargo test
-        run: cargo nextest run --locked --all-targets --all-features --no-tests=pass --no-fail-fast
+        run: cargo nextest run --locked --all-targets --no-tests=pass --no-fail-fast
 
         # https://github.com/rust-lang/cargo/issues/6669
       - name: Run doc tests

Cargo.toml

@@ -32,5 +32,12 @@ tokio-util = "0.7.17"
 anyhow = "1.0"
 
 
+[dev-dependencies]
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+
 [features]
 tracing = []
+# Enables tests that require an external environment (e.g. Kurtosis devnet)
+integration = []

README.md

@@ -203,13 +203,46 @@ let robust: RobustProvider<Ethereum> = provider.into().await?;
 
 ## Testing
 
-Run the test suite:
+### Local Tests (Anvil)
+
+Run unit and local integration tests using Anvil instances:
+
+```bash
+cargo local-test
+```
+
+These tests verify retry logic, failover behaviour, and subscription resilience against local Anvil instances that are spawned automatically.
+
+### Integration Tests (Kurtosis)
+
+Integration tests run against real Ethereum execution clients (geth, nethermind, besu, reth) in a Kurtosis devnet.
+
+**Prerequisites:**
+- [Docker](https://docs.docker.com/get-docker/)
+- [Kurtosis CLI](https://docs.kurtosis.com/install)
+
+**Setup and run:**
+
+```bash
+# 1. Start the Kurtosis devnet (creates enclave and outputs endpoints)
+./scripts/setup-kurtosis.sh
+
+# 2. Run integration tests
+cargo int-test
+```
+
+The setup script will:
+1. Start the Kurtosis engine if not running
+2. Create a `local-eth-testnet` enclave with multiple EL clients
+3. Write endpoint URLs to `target/kurtosis-endpoints.json`
+
+**Cleanup:**
 
 ```bash
-cargo nextest run
+kurtosis enclave rm local-eth-testnet
 ```
 
-The tests use local Anvil instances to verify retry logic, failover behaviour, and subscription resilience.
+Advised that you run this before re-spinning up the testnet
 
 ---
 

scripts/setup-kurtosis.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+
+#
+# WARNING: This script was AI generated 
+#
+# Setup Kurtosis ethereum-package devnet and output EL endpoints as JSON.
+#
+# Usage:
+#   ./scripts/setup-kurtosis.sh 
+#
+# Output: target/kurtosis-endpoints.json
+#
+# JSON format:
+# [
+#   { "client": "geth", "http": "http://127.0.0.1:32003", "ws": "ws://127.0.0.1:32004" },
+#   ...
+# ]
+
+set -euo pipefail
+
+ENCLAVE="local-eth-testnet"
+OUTPUT_DIR="target"
+OUTPUT_FILE="$OUTPUT_DIR/kurtosis-endpoints.json"
+
+# Ensure output directory exists
+mkdir -p "$OUTPUT_DIR"
+
+# Check if kurtosis is installed
+if ! command -v kurtosis &> /dev/null; then
+    echo "Error: kurtosis CLI not found. Please install it first." >&2
+    echo "See: https://docs.kurtosis.com/install" >&2
+    exit 1
+fi
+
+# Check if kurtosis engine is running
+if ! kurtosis engine status &> /dev/null; then
+    echo "Starting Kurtosis engine..."
+    kurtosis engine start
+fi
+
+# Check if enclave exists
+enclave_exists() {
+    kurtosis enclave ls 2>/dev/null | grep -q "^[^ ]*[[:space:]]*$ENCLAVE[[:space:]]"
+}
+
+if ! enclave_exists; then
+    echo "Enclave '$ENCLAVE' not found. Creating new enclave with ethereum-package..."
+    kurtosis run --enclave "$ENCLAVE" github.com/ethpandaops/ethereum-package --args-file ./tests/common/network_params.yaml
+fi
+
+echo "Inspecting enclave '$ENCLAVE'..."
+
+# Get list of EL services (lines starting with "el-")
+el_services=$(kurtosis enclave inspect "$ENCLAVE" 2>/dev/null | grep -E "^[a-f0-9]+[[:space:]]+el-" | awk '{print $2}')
+
+if [ -z "$el_services" ]; then
+    echo "Error: No EL services found in enclave '$ENCLAVE'" >&2
+    exit 1
+fi
+
+# Build JSON array
+json="["
+first=true
+
+for service in $el_services; do
+    # Extract client name from service name (e.g., "el-1-geth-lighthouse" -> "geth")
+    # Format: el-{index}-{client}-{cl_client}
+    client=$(echo "$service" | sed -E 's/^el-[0-9]+-([^-]+)-.*/\1/')
+
+    # Get port mappings
+    rpc_addr=$(kurtosis port print "$ENCLAVE" "$service" rpc 2>/dev/null || echo "")
+    ws_addr=$(kurtosis port print "$ENCLAVE" "$service" ws 2>/dev/null || echo "")
+
+    if [ -z "$rpc_addr" ]; then
+        echo "Warning: Could not get RPC port for $service, skipping..." >&2
+        continue
+    fi
+
+    # Build JSON object
+    if [ "$first" = true ]; then
+        first=false
+    else
+        json+=","
+    fi
+
+    http_url="http://$rpc_addr"
+
+    if [ -n "$ws_addr" ]; then
+        ws_url="ws://$ws_addr"
+        json+=$(printf '\n  {"client": "%s", "http": "%s", "ws": "%s"}' "$client" "$http_url" "$ws_url")
+    else
+        json+=$(printf '\n  {"client": "%s", "http": "%s", "ws": null}' "$client" "$http_url")
+    fi
+done
+
+json+="\n]"
+
+# Write JSON to file
+printf "$json\n" > "$OUTPUT_FILE"
+
+echo "Wrote endpoints to $OUTPUT_FILE:"
+cat "$OUTPUT_FILE"

tests/common/mod.rs

@@ -1,14 +1,9 @@
 //! Common test utilities and helpers for integration tests.
 
 #![allow(dead_code)]
-#![allow(clippy::missing_errors_doc)]
 
 use std::time::Duration;
 
-use alloy::providers::{Provider, ProviderBuilder, RootProvider, ext::AnvilApi};
-use alloy_node_bindings::{Anvil, AnvilInstance};
-use robust_provider::{RobustProvider, RobustProviderBuilder};
-
 /// Short timeout for tests.
 pub const SHORT_TIMEOUT: Duration = Duration::from_millis(300);
 
@@ -18,31 +13,5 @@ pub const RECONNECT_INTERVAL: Duration = Duration::from_millis(500);
 /// Buffer time for async operations.
 pub const BUFFER_TIME: Duration = Duration::from_millis(100);
 
-// Setup a basic Anvil instance with a `RobustProvider`.
-pub async fn setup_anvil() -> anyhow::Result<(AnvilInstance, RobustProvider, impl Provider)> {
-    let anvil = Anvil::new().try_spawn()?;
-    let alloy_provider = ProviderBuilder::new().connect_http(anvil.endpoint_url());
-
-    let robust = RobustProviderBuilder::new(alloy_provider.clone())
-        .call_timeout(Duration::from_secs(5))
-        .build()
-        .await?;
-
-    Ok((anvil, robust, alloy_provider))
-}
-
-/// Setup an Anvil instance with pre-mined blocks.
-pub async fn setup_anvil_with_blocks(
-    num_blocks: u64,
-) -> anyhow::Result<(AnvilInstance, RobustProvider, impl Provider)> {
-    let (anvil, robust, alloy_provider) = setup_anvil().await?;
-    alloy_provider.anvil_mine(Some(num_blocks), None).await?;
-    Ok((anvil, robust, alloy_provider))
-}
-
-/// Spawn a WebSocket-enabled Anvil instance.
-pub async fn spawn_ws_anvil() -> anyhow::Result<(AnvilInstance, RootProvider)> {
-    let anvil = Anvil::new().try_spawn()?;
-    let provider = ProviderBuilder::new().connect(anvil.ws_endpoint_url().as_str()).await?;
-    Ok((anvil, provider.root().to_owned()))
-}
+pub mod setup_anvil;
+pub mod setup_kurtosis;

tests/common/network_params.yaml

@@ -0,0 +1,10 @@
+participants:
+  - el_type: geth
+  - el_type: nethermind
+  - el_type: besu
+  - el_type: reth
+    cl_type: lighthouse
+
+network_params:
+  # How long you want the network to wait before starting up
+  genesis_delay: 1

tests/common/setup_anvil.rs

@@ -0,0 +1,37 @@
+#![allow(dead_code)]
+#![allow(clippy::missing_errors_doc)]
+
+use std::time::Duration;
+
+use alloy::providers::{Provider, ProviderBuilder, RootProvider, ext::AnvilApi};
+use alloy_node_bindings::{Anvil, AnvilInstance};
+use robust_provider::{RobustProvider, RobustProviderBuilder};
+
+// Setup a basic Anvil instance with a `RobustProvider`.
+pub async fn setup_anvil() -> anyhow::Result<(AnvilInstance, RobustProvider, impl Provider)> {
+    let anvil = Anvil::new().try_spawn()?;
+    let alloy_provider = ProviderBuilder::new().connect_http(anvil.endpoint_url());
+
+    let robust = RobustProviderBuilder::new(alloy_provider.clone())
+        .call_timeout(Duration::from_secs(5))
+        .build()
+        .await?;
+
+    Ok((anvil, robust, alloy_provider))
+}
+
+/// Setup an Anvil instance with pre-mined blocks.
+pub async fn setup_anvil_with_blocks(
+    num_blocks: u64,
+) -> anyhow::Result<(AnvilInstance, RobustProvider, impl Provider)> {
+    let (anvil, robust, alloy_provider) = setup_anvil().await?;
+    alloy_provider.anvil_mine(Some(num_blocks), None).await?;
+    Ok((anvil, robust, alloy_provider))
+}
+
+/// Spawn a WebSocket-enabled Anvil instance.
+pub async fn spawn_ws_anvil() -> anyhow::Result<(AnvilInstance, RootProvider)> {
+    let anvil = Anvil::new().try_spawn()?;
+    let provider = ProviderBuilder::new().connect(anvil.ws_endpoint_url().as_str()).await?;
+    Ok((anvil, provider.root().to_owned()))
+}

tests/common/setup_kurtosis.rs

@@ -0,0 +1,44 @@
+#![allow(dead_code)]
+#![allow(clippy::missing_errors_doc)]
+
+use std::fs;
+
+use serde::Deserialize;
+
+const ENDPOINTS_FILE: &str = "target/kurtosis-endpoints.json";
+
+/// A single EL endpoint entry from the JSON file.
+#[derive(Debug, Clone, Deserialize)]
+pub struct ElEndpoint {
+    pub client: String,
+    pub http: String,
+    pub ws: Option<String>,
+}
+
+/// Load Execution Layer (EL) endpoints from the JSON file generated by `scripts/setup-kurtosis.sh`.
+///
+/// The script must be run before the tests to populate `target/kurtosis-endpoints.json`.
+///
+/// # Example
+///
+/// ```bash
+/// ./scripts/setup-kurtosis.sh
+/// cargo test --features integration
+/// ```
+pub fn load_el_endpoints() -> anyhow::Result<Vec<ElEndpoint>> {
+    let content = fs::read_to_string(ENDPOINTS_FILE).map_err(|e| {
+        anyhow::anyhow!(
+            "Failed to read '{ENDPOINTS_FILE}': {e}\n\n\
+            Run './scripts/setup-kurtosis.sh <enclave>' first to generate the endpoints file."
+        )
+    })?;
+
+    let endpoints: Vec<ElEndpoint> = serde_json::from_str(&content)
+        .map_err(|e| anyhow::anyhow!("Failed to parse '{ENDPOINTS_FILE}': {e}"))?;
+
+    if endpoints.is_empty() {
+        anyhow::bail!("No EL endpoints found in '{ENDPOINTS_FILE}'");
+    }
+
+    Ok(endpoints)
+}

tests/custom_methods.rs

@@ -3,14 +3,14 @@
 //! These tests cover methods that are unique to `RobustProvider` and not
 //! direct wrappers of standard Ethereum JSON-RPC methods.
 
-mod common;
-
-use common::setup_anvil_with_blocks;
-
 // ============================================================================
 // get_latest_confirmed
 // ============================================================================
 
+use crate::common::setup_anvil::setup_anvil_with_blocks;
+
+mod common;
+
 #[tokio::test]
 async fn test_get_latest_confirmed_succeeds() -> anyhow::Result<()> {
     let (_anvil, robust, _alloy_provider) = setup_anvil_with_blocks(100).await?;

tests/eth_namespace.rs

@@ -3,16 +3,17 @@
 //! These tests verify the behavior of standard Ethereum RPC methods wrapped
 //! by `RobustProvider` with retry and failover logic.
 
-mod common;
-
 use alloy::{
     eips::{BlockId, BlockNumberOrTag},
     primitives::BlockHash,
     providers::{Provider, ext::AnvilApi},
 };
-use common::{setup_anvil, setup_anvil_with_blocks};
 use robust_provider::Error;
 
+use crate::common::setup_anvil::{setup_anvil, setup_anvil_with_blocks};
+
+mod common;
+
 // ============================================================================
 // eth_getBlockByNumber
 // ============================================================================