Skip to content

test(e2e): bind test nodes to port 0 instead of a random fixed port #281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Nic-dorman merged 2 commits into from
Nov 5, 2026
Merged

test(e2e): bind test nodes to port 0 instead of a random fixed port #281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0da08c8
test(e2e): bind test nodes to port 0 instead of a random fixed port
Nic-dorman Nov 5, 2026
dc16dda
docs(claude): e2e test nodes use OS-assigned ports, not 20000-60000
Nic-dorman Nov 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions CLAUDE.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,9 @@ This includes:
| ant-quic | 9000-9999 | 9000 | QUIC transport layer |
| **ant-node** | **10000-10999** | **10000** | Core P2P network nodes (THIS PROJECT) |
| communitas | 11000-11999 | 11000 | Collaboration platform nodes |
| **ant-node tests** | **20000-60000** | **random** | E2E test isolation (local only) |
| **ant-node tests** | **OS-assigned** | **port 0** | E2E test isolation (local only) |

**Note:** The E2E test suite uses ports 20000-60000 with random allocation to prevent conflicts between parallel test runs and local development instances. Production deployments MUST use 10000-10999.
**Note:** E2E test nodes bind port 0, so the OS assigns each one a free port from its ephemeral range (by default Linux 32768-60999, macOS and Windows 49152-65535). That keeps parallel test runs apart and clear of local development instances. Production deployments MUST use 10000-10999.

### DO NOT DISTURB OTHER NETWORKS

Expand Down
56 changes: 15 additions & 41 deletions src/node.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1165,25 +1165,22 @@ fn jittered_interval(base: std::time::Duration) -> std::time::Duration {
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use rand::Rng;
use tempfile::TempDir;

use crate::storage::legacy_artifacts::LEGACY_ENV_DIR;

/// The e2e port range, so a test bind never lands on a production or dev instance.
const TEST_PORT_RANGE: std::ops::Range<u16> = 20000..60000;

/// How many times a bind is retried before the failure is treated as real.
const BIND_ATTEMPTS: u32 = 5;

/// A well-formed address that receives nothing; no chain is contacted in these tests.
const TEST_REWARDS_ADDRESS: &str = "0x0000000000000000000000000000000000000001";

/// A node config that builds without touching a chain or a real network.
fn local_node_config(root: &std::path::Path, port: u16) -> NodeConfig {
///
/// Port 0: the OS assigns a free port, so a build never fails on a port that
/// is in use (V2-1894). These tests used to pick a random one in 20000-60000,
/// which overlaps the OS ephemeral range.
fn local_node_config(root: &std::path::Path) -> NodeConfig {
NodeConfig {
root_dir: root.to_path_buf(),
port,
port: 0,
ipv4_only: true,
network_mode: NetworkMode::Development,
payment: crate::config::PaymentConfig {
Expand All @@ -1205,7 +1202,7 @@ mod tests {
let occupied = tokio::net::UdpSocket::bind("127.0.0.1:0")
.await
.expect("bind");
let mut config = local_node_config(&root, 0);
let mut config = local_node_config(&root);
config.webrtc_direct.enabled = true;
config.webrtc_direct.bind = occupied.local_addr().expect("address");
let mut node = NodeBuilder::new(config).build().await.expect("build");
Expand All @@ -1225,27 +1222,10 @@ mod tests {
let root = dir.path().join("node");
std::fs::create_dir_all(&root).expect("mkdir");

let mut built = None;
let mut last_err = String::new();
for _ in 0..BIND_ATTEMPTS {
let port = rand::thread_rng().gen_range(TEST_PORT_RANGE);
match NodeBuilder::new(local_node_config(&root, port))
.build()
.await
{
Ok(node) => {
built = Some(node);
break;
}
Err(e) => {
last_err = e.to_string();
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
}
}
}
let Some(node) = built else {
panic!("could not build a node after {BIND_ATTEMPTS} attempts: {last_err}");
};
let node = NodeBuilder::new(local_node_config(&root))
.build()
.await
.expect("a node builds on a clean root");

node.shutdown.cancel();
}
Expand Down Expand Up @@ -1274,8 +1254,7 @@ mod tests {
std::fs::write(env.join("data.mdb"), b"chunks that were never copied out")
.expect("seed");

let port = rand::thread_rng().gen_range(TEST_PORT_RANGE);
let mut config = local_node_config(&root, port);
let mut config = local_node_config(&root);
config.storage.enabled = storage_enabled;

let node = NodeBuilder::new(config).build().await.unwrap_or_else(|e| {
Expand Down Expand Up @@ -1313,8 +1292,7 @@ mod tests {
.expect("seed");
std::fs::write(env.join("RETIRED"), b"retired").expect("mark");

let port = rand::thread_rng().gen_range(TEST_PORT_RANGE);
let mut config = local_node_config(&root, port);
let mut config = local_node_config(&root);
config.storage.enabled = false;
let node = NodeBuilder::new(config).build().await.expect("must start");
settle();
Expand All @@ -1340,8 +1318,7 @@ mod tests {
.expect("seed");
std::fs::write(env.join("RETIRED"), b"retired").expect("mark");

let port = rand::thread_rng().gen_range(TEST_PORT_RANGE);
let node = NodeBuilder::new(local_node_config(&root, port))
let node = NodeBuilder::new(local_node_config(&root))
.build()
.await
.expect("a node with a finished leftover must start");
Expand Down Expand Up @@ -1371,10 +1348,7 @@ mod tests {
// A file where the chunk directory has to be, so the store cannot create it.
std::fs::write(root.join("chunks"), b"not a directory").expect("block the store");

let port = rand::thread_rng().gen_range(TEST_PORT_RANGE);
let built = NodeBuilder::new(local_node_config(&root, port))
.build()
.await;
let built = NodeBuilder::new(local_node_config(&root)).build().await;
assert!(
built.is_err(),
"the file store was supposed to fail to open"
Expand Down
48 changes: 11 additions & 37 deletions tests/e2e/harness.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,14 @@
//! both the ant node network and optional Anvil EVM testnet.

use super::anvil::TestAnvil;
use super::testnet::{TestNetwork, TestNetworkConfig, TestNode, TestnetError};
use super::testnet::{TestNetwork, TestNetworkConfig, TestNode};
use ant_node::client::XorName;
use ant_node::storage::DiskSpace;
use evmlib::common::TxHash;
use saorsa_core::P2PNode;
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use tracing::{info, warn};

/// How many port ranges a network is brought up on before its bind failure
/// is reported.
///
/// A node retries its own port, which clears a socket still being released.
/// It cannot clear a port the host will not hand out at all, as happens on
/// Windows runners whose reserved port ranges move between runs, so a network
/// that cannot create a node moves to a fresh random range instead.
const NETWORK_START_ATTEMPTS: u32 = 3;
use tracing::info;

/// Size of the partition the prune tests report: 1 TiB.
const TEST_DISK_BYTES: u64 = 1 << 40;
Expand All @@ -36,32 +27,15 @@ pub const ROOMY_DISK: DiskSpace = DiskSpace {
total: TEST_DISK_BYTES,
};

/// Create and start a network from `config`, moving it to a fresh port range
/// if a node cannot be created.
async fn start_network(mut config: TestNetworkConfig) -> Result<TestNetwork> {
let mut attempt = 1;
loop {
let mut network = TestNetwork::new(config.clone()).await?;
match network.start().await {
Ok(()) => return Ok(network),
Err(TestnetError::Startup(reason))
if attempt < NETWORK_START_ATTEMPTS
&& reason.starts_with("Failed to create node ") =>
{
let base_port = config.base_port;
warn!(
"Test network on base port {base_port} could not create a node \
({reason}); retrying on a fresh port range"
);
if let Err(e) = network.shutdown().await {
warn!("Cleanup after a failed bring-up failed: {e}");
}
config = config.with_fresh_ports();
attempt += 1;
}
Err(e) => return Err(e.into()),
}
}
/// Create and start a network from `config`.
///
/// Its nodes bind OS-assigned ports, so a bring-up never fails on a port that
/// is in use or reserved. This used to retry the whole network on a fresh port
/// range when a node could not be created (V2-1894).
async fn start_network(config: TestNetworkConfig) -> Result<TestNetwork> {
let mut network = TestNetwork::new(config).await?;
network.start().await?;
Ok(network)
}

/// Error type for test harness operations.
Expand Down
42 changes: 24 additions & 18 deletions tests/e2e/integration_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@

use super::testnet::{
DEFAULT_BOOTSTRAP_COUNT, DEFAULT_NODE_COUNT, MINIMAL_BOOTSTRAP_COUNT, MINIMAL_NODE_COUNT,
SMALL_NODE_COUNT, TEST_PORT_RANGE_MAX, TEST_PORT_RANGE_MIN,
SMALL_NODE_COUNT,
};
use super::{NetworkState, TestHarness, TestNetwork, TestNetworkConfig};
use saorsa_core::P2PEvent;
use serial_test::serial;
use std::net::UdpSocket;
use std::time::Duration;

/// Test that a minimal network (5 nodes) can form and stabilize.
Expand Down Expand Up @@ -172,8 +171,6 @@ fn test_config_presets() {
let default = TestNetworkConfig::default();
assert_eq!(default.node_count, DEFAULT_NODE_COUNT);
assert_eq!(default.bootstrap_count, DEFAULT_BOOTSTRAP_COUNT);
// Ports are randomly generated in a wide range to avoid collisions
assert!(default.base_port >= TEST_PORT_RANGE_MIN && default.base_port < TEST_PORT_RANGE_MAX);

let minimal = TestNetworkConfig::minimal();
assert_eq!(minimal.node_count, MINIMAL_NODE_COUNT);
Expand Down Expand Up @@ -278,23 +275,32 @@ async fn test_node_to_node_messaging() {
.expect("Failed to teardown test harness");
}

/// A network whose first node cannot bind its port is brought up on a fresh
/// port range rather than failing the test that asked for it.
/// Test nodes bind port 0, and every node reports the port the OS assigned it:
/// later nodes bootstrap from those addresses, so a node reporting port 0, or
/// two nodes sharing a port, would break the network (V2-1894).
#[tokio::test]
#[serial]
async fn a_network_whose_port_is_taken_moves_to_a_fresh_range() {
let config = TestNetworkConfig::minimal();
let taken = config.base_port;
let _holder = UdpSocket::bind(("127.0.0.1", taken)).expect("hold the first node's port");

let harness = TestHarness::setup_with_config(config)
async fn nodes_listen_on_distinct_os_assigned_ports() {
let harness = TestHarness::setup_with_config(TestNetworkConfig::minimal())
.await
.expect("the network comes up on another range");
assert_ne!(
harness.network().config().base_port,
taken,
"the network moved off the port it could not bind"
);
.expect("setup");

let mut ports = Vec::new();
for node in harness.network().nodes() {
let p2p = node.p2p_node.as_ref().expect("node is running");
let port = p2p
.listen_addrs()
.await
.iter()
.find_map(saorsa_core::MultiAddr::port)
.expect("node has a listen port");
assert_ne!(port, 0, "node {} reports port 0", node.index);
ports.push(port);
}
let mut distinct = ports.clone();
distinct.sort_unstable();
distinct.dedup();
assert_eq!(distinct.len(), ports.len(), "nodes share a port: {ports:?}");

harness.teardown().await.expect("teardown");
}
Loading
Loading