diff --git a/bdd/rust/tests/helpers/cluster.rs b/bdd/rust/tests/helpers/cluster.rs index 195bcb9a83..8a954c22af 100644 --- a/bdd/rust/tests/helpers/cluster.rs +++ b/bdd/rust/tests/helpers/cluster.rs @@ -17,6 +17,7 @@ use iggy::prelude::*; use std::env; +use std::net::{SocketAddr, ToSocketAddrs}; use std::sync::Arc; /// Resolves server address based on role and port, checking environment variables first @@ -49,6 +50,24 @@ pub async fn create_and_connect_client(addr: &str) -> IggyClient { IggyClient::create(ClientWrapper::Tcp(client), None, None) } +/// Whether two `host:port` spellings name the same endpoint. +/// +/// A client that never redirected still holds the address it was given (a +/// host name, in the BDD compose network), while a redirected one holds the +/// address the roster published (an IP). Both name the same node, so they +/// are compared once resolved, like the Go and Java suites do. +pub fn is_same_endpoint(left: &str, right: &str) -> Result { + let resolve = |address: &str| -> Result, String> { + address + .to_socket_addrs() + .map(Iterator::collect) + .map_err(|error| format!("Failed to resolve server address {address}: {error}")) + }; + let left = resolve(left)?; + let right = resolve(right)?; + Ok(left.iter().any(|candidate| right.contains(candidate))) +} + /// Verifies that a client is connected to the expected port pub async fn verify_client_connection( client: &IggyClient, diff --git a/bdd/rust/tests/steps/leader_redirection.rs b/bdd/rust/tests/steps/leader_redirection.rs index 1f6bcc5ad5..c035bd848b 100644 --- a/bdd/rust/tests/steps/leader_redirection.rs +++ b/bdd/rust/tests/steps/leader_redirection.rs @@ -288,10 +288,16 @@ async fn then_both_use_same_server(world: &mut LeaderContext) { let conn_info_a = client_a.get_connection_info().await; let conn_info_b = client_b.get_connection_info().await; - // Verify both clients are connected to the same server - assert_eq!( - conn_info_a.server_address, conn_info_b.server_address, - "Both clients should be connected to the same server" + // Verify both clients are connected to the same server. Client A holds + // the address it was configured with and client B the one the roster + // published for the leader, so the spellings differ even when the node + // is the same. + assert!( + cluster::is_same_endpoint(&conn_info_a.server_address, &conn_info_b.server_address) + .expect("Server addresses should resolve"), + "Both clients should be connected to the same server, got {} and {}", + conn_info_a.server_address, + conn_info_b.server_address ); // Verify both can communicate diff --git a/core/common/src/traits/binary_impls/personal_access_tokens.rs b/core/common/src/traits/binary_impls/personal_access_tokens.rs index 7e1299f284..8404bcc4c2 100644 --- a/core/common/src/traits/binary_impls/personal_access_tokens.rs +++ b/core/common/src/traits/binary_impls/personal_access_tokens.rs @@ -18,8 +18,9 @@ use crate::traits::binary_auth::fail_if_not_authenticated; use crate::wire_conversions::personal_access_tokens_from_wire; use crate::{ - BinaryClient, ClientState, DiagnosticEvent, IdentityInfo, IggyError, PersonalAccessTokenClient, - PersonalAccessTokenExpiry, PersonalAccessTokenInfo, RawPersonalAccessToken, + BinaryClient, ClientState, Credentials, DiagnosticEvent, IdentityInfo, IggyError, + PersonalAccessTokenClient, PersonalAccessTokenExpiry, PersonalAccessTokenInfo, + RawPersonalAccessToken, }; use iggy_binary_protocol::MAX_WIRE_NAME_LENGTH; use iggy_binary_protocol::WireName; @@ -134,6 +135,11 @@ impl PersonalAccessTokenClient for B { "authenticated against iggy server" ); self.set_state(ClientState::Authenticated).await; + self.remember_session_credentials( + Credentials::PersonalAccessToken(SecretString::from(token.to_string())), + wire_resp.user_id, + ) + .await; self.publish_event(DiagnosticEvent::SignedIn).await; Ok(IdentityInfo { user_id: wire_resp.user_id, diff --git a/core/common/src/traits/binary_impls/users.rs b/core/common/src/traits/binary_impls/users.rs index eb785109a6..38ee456f2d 100644 --- a/core/common/src/traits/binary_impls/users.rs +++ b/core/common/src/traits/binary_impls/users.rs @@ -18,8 +18,8 @@ use crate::traits::binary_auth::fail_if_not_authenticated; use crate::wire_conversions::{identifier_to_wire, permissions_to_wire, users_from_wire}; use crate::{ - BinaryClient, ClientState, DiagnosticEvent, Identifier, IdentityInfo, IggyError, Permissions, - UserClient, UserInfo, UserInfoDetails, UserStatus, UserUpdateOptions, + BinaryClient, ClientState, Credentials, DiagnosticEvent, Identifier, IdentityInfo, IggyError, + Permissions, UserClient, UserInfo, UserInfoDetails, UserStatus, UserUpdateOptions, }; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::codes::LOGIN_REGISTER_CODE; @@ -174,6 +174,7 @@ impl UserClient for B { .to_bytes(), ) .await?; + self.refresh_session_password(user_id, new_password).await; Ok(()) } @@ -218,6 +219,14 @@ impl UserClient for B { "authenticated against iggy server" ); self.set_state(ClientState::Authenticated).await; + self.remember_session_credentials( + Credentials::UsernamePassword( + username.to_owned(), + SecretString::from(password.to_string()), + ), + wire_resp.user_id, + ) + .await; self.publish_event(DiagnosticEvent::SignedIn).await; Ok(IdentityInfo { user_id: wire_resp.user_id, @@ -229,6 +238,7 @@ impl UserClient for B { fail_if_not_authenticated(self).await?; self.send_raw_with_response(LOGOUT_USER_CODE, LogoutUserRequest.to_bytes()) .await?; + self.forget_session_credentials().await; self.reset_vsr_session().await?; self.set_state(ClientState::Connected).await; self.publish_event(DiagnosticEvent::SignedOut).await; diff --git a/core/common/src/traits/binary_transport.rs b/core/common/src/traits/binary_transport.rs index 4c12db04bc..91804ddb8a 100644 --- a/core/common/src/traits/binary_transport.rs +++ b/core/common/src/traits/binary_transport.rs @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -use crate::{ClientState, DiagnosticEvent, IggyError, NonZeroIggyDuration}; +use crate::{ + ClientState, Credentials, DiagnosticEvent, Identifier, IggyError, NonZeroIggyDuration, +}; use async_trait::async_trait; use bytes::Bytes; use std::sync::Arc; @@ -51,6 +53,26 @@ mod vsr_session_sealed { pub trait VsrSessionControl: vsr_session_sealed::Sealed + BinaryTransport { async fn bind_vsr_session(&self, session: u64) -> Result<(), IggyError>; async fn reset_vsr_session(&self) -> Result<(), IggyError>; + /// Keep the credentials a sign-in succeeded with, so a transport that + /// loses its connection can re-establish the session -- on this node or, + /// after failing over, on another one. A caller that signs in by hand is + /// otherwise less reconnectable than one that configures `AutoLogin`, + /// which is a surprising difference between two ways of doing the same + /// thing. Transports that cannot reconnect leave this a no-op. + async fn remember_session_credentials(&self, _credentials: Credentials, _user_id: u32) {} + /// Drop them: after an explicit logout there is no session to restore, + /// and a reconnect must not resurrect one. + async fn forget_session_credentials(&self) {} + /// A committed password change for `user`: when it is the signed-in user, + /// the credentials the next reconnect signs in with switch to the new + /// password, or that reconnect would replay the old one and fail an + /// unrelated request with `InvalidCredentials`. Other users' changes are + /// ignored. + /// + /// This covers a configured `AutoLogin` as well as a sign-in the caller + /// ran: the configured credentials still decide *who* the client signs in + /// as, and a committed change decides what that user's password is. + async fn refresh_session_password(&self, _user: &Identifier, _new_password: &str) {} /// SDK crate version sent in the login-register version prefix. /// Implemented by the transports so the value is the SDK crate's own /// `CARGO_PKG_VERSION` (`iggy` for Rust), not `iggy_common`'s. diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs b/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs index 89d8644e8b..497db19211 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs @@ -20,10 +20,43 @@ use std::str::FromStr; #[derive(Debug, Clone)] pub struct TcpClientReconnectionConfig { + /// Whether a lost connection is redialed at all. With this off the + /// endpoints the client knows still get one pass, since they were + /// configured to be tried, but nothing is retried after it. pub enabled: bool, + /// How many passes over the known endpoints *after the first*, or `None` + /// for unlimited. `Some(0)` still makes that one pass, since the endpoints + /// were configured to be tried. + /// + /// Passes, not dials: one pass tries the endpoint the client is on, the + /// addresses it was configured with, and every node the roster named, so a + /// survivor is reached inside the first pass rather than one delay per + /// endpoint. + /// + /// The number is not portable across SDKs. Each counts the same setting in + /// its own terms, and `0` means something different in every one of them, so + /// a deployment that runs several has to set this per SDK rather than copy + /// one value across: + /// + /// | SDK | `N` | `0` | unlimited | + /// | ---- | -------------------------------------- | ----------------------------------------- | ---------------- | + /// | Rust | `N` passes after a first, unpaced one | that first pass alone | `None` | + /// | C# | as Rust | unlimited | `0` | + /// | Go | `N` passes, the first one of them | unlimited | `0` | + /// | Java | `N` passes, the first one of them | one pass, and only with several endpoints | a large `N` | + /// | Node | `N` passes, the first one of them | no pass at all | a large `N` | pub max_retries: Option, - /// Delay between connection attempts. + /// Delay between passes. The first pass runs at once when the client knows + /// more than one endpoint. pub interval: NonZeroIggyDuration, + /// Cooldown before redialing the endpoint of the last successful + /// connection, measured from when that connection was established rather + /// than from when it was lost: a session that outlived this interval is + /// redialed with no wait at all, which is the point -- the pace limit is + /// there for connections that keep dropping straight away. + /// + /// Owed to that endpoint alone: the others are dialed without waiting, and + /// the paced one goes last in the pass. pub reestablish_after: IggyDuration, } diff --git a/core/integration/tests/cluster/failover_client_continuity.rs b/core/integration/tests/cluster/failover_client_continuity.rs index ab4c93f82f..9eeed6c0dd 100644 --- a/core/integration/tests/cluster/failover_client_continuity.rs +++ b/core/integration/tests/cluster/failover_client_continuity.rs @@ -15,19 +15,19 @@ // specific language governing permissions and limitations // under the License. -//! RED SPEC, expected to FAIL: client continuity across a primary SIGKILL. +//! Client continuity across a primary SIGKILL. //! //! A producing SDK client pinned to the primary must, after the primary dies, //! complete its next operation against the surviving quorum within a small -//! budget and without an authentication error. The SDK cannot: it has no -//! multi-endpoint failover. A client is built around a single -//! `server_address`, so it knows no other endpoint to dial; the transport's -//! fail-fast gate (auto-login disabled, the shape this harness client runs -//! with) returns errors without attempting a reconnect; and the -//! leader-redirect machinery that could reroute it needs a live connection to -//! read the cluster roster. Every retry therefore redials the dead endpoint -//! and fails with a connection error until the caller gives up. Surviving a -//! primary crash needs a seed roster of endpoints, not just a redirect. +//! budget and without an authentication error. Three separate pieces of +//! client state make that possible, and the test fails if any one of them is +//! lost: the endpoints the cluster roster named while the connection was +//! healthy (the roster is unreachable exactly when it is needed), the +//! credentials the sign-in succeeded with (this harness client signs in by +//! hand rather than configuring `AutoLogin`, and a reconnect has to +//! re-establish the session on whichever node answers), and a reconnect that +//! dials those endpoints in turn instead of redialing the address the client +//! was configured with. use std::time::Duration; @@ -64,8 +64,6 @@ fn build_message(payload: &str) -> IggyMessage { /// A producing client pinned to the primary; SIGKILL the primary mid-stream; /// the same client's next send must succeed against the surviving quorum /// within `RESUME_BUDGET` and must never surface Unauthenticated. -// TODO(hubcio): fix this test -#[ignore = "SDK has no multi-endpoint failover; client redials the dead primary forever"] #[iggy_harness(cluster_nodes = 3)] async fn given_a_client_producing_when_its_primary_is_killed_should_resume_without_hang_or_unauthenticated( harness: &mut TestHarness, @@ -97,6 +95,11 @@ async fn given_a_client_producing_when_its_primary_is_killed_should_resume_witho // Pin the producing client to the primary's own endpoint, the way a // leader-aware SDK ends up connected to whichever node answers as leader. let leader = disk::leader_node_index(harness).await; + let primary_endpoint = harness + .node(leader) + .tcp_addr() + .expect("leader exposes a TCP endpoint") + .to_string(); let producer = harness .node(leader) .tcp_client() @@ -106,6 +109,12 @@ async fn given_a_client_producing_when_its_primary_is_killed_should_resume_witho .await .expect("connect the producer to the primary"); + assert_eq!( + producer.get_connection_info().await.server_address, + primary_endpoint, + "the producer must be pinned to the node this test kills, or it proves nothing" + ); + let stream = Identifier::named(STREAM_NAME).unwrap(); let topic = Identifier::named(TOPIC_NAME).unwrap(); let partitioning = Partitioning::partition_id(PARTITION_ID); @@ -160,11 +169,15 @@ async fn given_a_client_producing_when_its_primary_is_killed_should_resume_witho assert!( resumed, "a client pinned to a killed primary must complete its next operation against \ - the surviving quorum within {RESUME_BUDGET:?}, but the SDK has no \ - multi-endpoint failover: it holds only the dead node's server_address, its \ - fail-fast gate (auto-login disabled) surfaces errors without reconnecting, \ - and the leader redirect that could reroute it needs a live connection to \ - read the roster, so every retry redialed the dead endpoint \ + the surviving quorum within {RESUME_BUDGET:?}: the roster learned while the \ + connection was healthy names the survivors, and the credentials the sign-in \ + succeeded with re-establish the session on whichever one answers \ ({attempt} attempts, last error: {last_error:?})" ); + assert_ne!( + producer.get_connection_info().await.server_address, + primary_endpoint, + "the send that resumed must have landed on a survivor, so the client has to \ + have moved off the killed primary's endpoint" + ); } diff --git a/core/integration/tests/sdk/disconnect_relogin.rs b/core/integration/tests/sdk/disconnect_relogin.rs new file mode 100644 index 0000000000..956e17c5a2 --- /dev/null +++ b/core/integration/tests/sdk/disconnect_relogin.rs @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! An explicit disconnect ends the session for good: the credentials a manual +//! sign-in remembered (for reconnecting across involuntary drops and +//! failovers) must not resurrect it. Pins at the Rust layer the contract the +//! C++ e2e suite asserts through the FFI (`DisconnectThenReconnectWithoutRelogin`, +//! `GetStatsBeforeLoginThrows`), so a regression fails here first instead of +//! three suites downstream. + +use iggy::prelude::*; +use integration::iggy_harness; + +#[iggy_harness] +async fn given_a_logged_in_client_when_explicitly_disconnected_should_require_a_fresh_login( + harness: &TestHarness, +) { + let client = harness.new_client().await.unwrap(); + client + .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) + .await + .unwrap(); + client.get_me().await.expect("authenticated get_me works"); + + client.disconnect().await.unwrap(); + client.connect().await.unwrap(); + assert!( + matches!(client.get_me().await, Err(IggyError::Unauthenticated)), + "an explicit disconnect is caller intent, like a logout: the sign-in it ended \ + must not be silently replayed by the reconnect, so the client's own \ + authentication gate refuses the request before it is sent" + ); + + client.disconnect().await.unwrap(); + assert!( + matches!(client.get_stats().await, Err(IggyError::NotConnected)), + "an operation after an explicit disconnect must fail on the dead transport \ + instead of reconnecting into a resurrected session" + ); + + // The remembered sign-in exists for involuntary drops; a fresh manual + // login after the disconnect works exactly as before. + client.connect().await.unwrap(); + client + .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) + .await + .unwrap(); + client + .get_me() + .await + .expect("a fresh login restores service"); +} diff --git a/core/integration/tests/sdk/mod.rs b/core/integration/tests/sdk/mod.rs index 3b934e8c95..065b257a76 100644 --- a/core/integration/tests/sdk/mod.rs +++ b/core/integration/tests/sdk/mod.rs @@ -18,6 +18,7 @@ mod consumer_group; mod consumer_group_membership; mod consumer_offset; +mod disconnect_relogin; mod hello_world; mod http_refresh; mod options; diff --git a/core/sdk/src/clients/binary_personal_access_tokens.rs b/core/sdk/src/clients/binary_personal_access_tokens.rs index ca19ce5e48..cfbe169d5e 100644 --- a/core/sdk/src/clients/binary_personal_access_tokens.rs +++ b/core/sdk/src/clients/binary_personal_access_tokens.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::clients::redirect_login_settled; use crate::prelude::{ClientWrapper, IggyClient}; use async_trait::async_trait; use iggy_common::locking::IggyRwLockFn; @@ -77,6 +78,15 @@ impl PersonalAccessTokenClient for IggyClient { if should_redirect { info!("Redirected to leader, reconnecting and re-authenticating"); self.connect().await?; + // The reconnect signs in with the credentials this very call just + // remembered, so on a client without a configured `AutoLogin` the + // session is already this user's: signing in again would cost a + // logout and a second login (an argon2 each, on the server) for + // nothing. With `AutoLogin::Enabled` the reconnect signed in the + // configured user, who may not be this one, so the login runs. + if redirect_login_settled(&*self.client.read().await).await { + return Ok(identity); + } self.login_with_personal_access_token(token).await } else { Ok(identity) diff --git a/core/sdk/src/clients/binary_users.rs b/core/sdk/src/clients/binary_users.rs index cab16bcf61..606e11e445 100644 --- a/core/sdk/src/clients/binary_users.rs +++ b/core/sdk/src/clients/binary_users.rs @@ -16,6 +16,7 @@ // under the License. use crate::client_wrappers::client_wrapper::ClientWrapper; +use crate::clients::redirect_login_settled; use crate::prelude::IggyClient; use async_trait::async_trait; use iggy_common::UserUpdateOptions; @@ -116,6 +117,15 @@ impl UserClient for IggyClient { if should_redirect { info!("Redirected to leader, reconnecting and re-authenticating"); self.connect().await?; + // The reconnect signs in with the credentials this very call just + // remembered, so on a client without a configured `AutoLogin` the + // session is already this user's: signing in again would cost a + // logout and a second login (an argon2 each, on the server) for + // nothing. With `AutoLogin::Enabled` the reconnect signed in the + // configured user, who may not be this one, so the login runs. + if redirect_login_settled(&*self.client.read().await).await { + return Ok(identity); + } self.login_user(username, password).await } else { Ok(identity) diff --git a/core/sdk/src/clients/mod.rs b/core/sdk/src/clients/mod.rs index 3ad0df5a46..1a310450fa 100644 --- a/core/sdk/src/clients/mod.rs +++ b/core/sdk/src/clients/mod.rs @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. +use crate::client_wrappers::client_wrapper::ClientWrapper; +use iggy_common::{BinaryTransport, ClientState}; + mod binary_cluster; mod binary_consumer_group; mod binary_consumer_offset; @@ -38,5 +41,35 @@ pub mod producer_error_callback; pub mod producer_sharding; const ORDERING: std::sync::atomic::Ordering = std::sync::atomic::Ordering::SeqCst; + +/// Whether the reconnect that followed a leader redirect already left this +/// client signed in as the user the redirected sign-in was for. +/// +/// The connect flow signs in with the credentials that sign-in just +/// remembered, so on a client without a configured `AutoLogin` the session on +/// the leader is already the right one: signing in again would run a logout +/// plus a second login, an argon2 each on the server, to arrive where the +/// client already is. With `AutoLogin::Enabled(a)` the reconnect signed in the +/// configured user instead, who need not be the one signing in here, so the +/// sign-in still has to run. +pub(crate) async fn redirect_login_settled(client: &ClientWrapper) -> bool { + let (state, auto_login_configured) = match client { + ClientWrapper::Tcp(tcp_client) => ( + tcp_client.get_state().await, + tcp_client.auto_login_configured(), + ), + ClientWrapper::Quic(quic_client) => ( + quic_client.get_state().await, + quic_client.auto_login_configured(), + ), + ClientWrapper::WebSocket(ws_client) => ( + ws_client.get_state().await, + ws_client.auto_login_configured(), + ), + _ => return false, + }; + + state == ClientState::Authenticated && !auto_login_configured +} const MAX_BATCH_LENGTH: usize = 1000000; const MIB: usize = 1_048_576; diff --git a/core/sdk/src/leader_aware.rs b/core/sdk/src/leader_aware.rs index 35ffce9fca..b5c038f349 100644 --- a/core/sdk/src/leader_aware.rs +++ b/core/sdk/src/leader_aware.rs @@ -18,7 +18,7 @@ use iggy_binary_protocol::codes::GET_CLUSTER_METADATA_CODE; use iggy_common::ClusterClient; use iggy_common::{ - ClusterMetadata, ClusterNodeRole, ClusterNodeStatus, IggyError, TransportProtocol, + ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, IggyError, TransportProtocol, }; use std::net::SocketAddr; use std::str::FromStr; @@ -38,12 +38,33 @@ pub(crate) fn is_unauthenticated_metadata_probe(code: u32, error: &IggyError) -> code == GET_CLUSTER_METADATA_CODE && matches!(error, IggyError::Unauthenticated) } +/// What one leader check learned from the cluster roster. +pub struct LeaderCheck { + /// The leader's address, when it is not the node the client is on. + pub redirect: Option, + /// Every endpoint the roster named for this transport. A client keeps + /// them as failover candidates: the address it was configured with dies + /// with its node, and the roster is unreachable exactly when it is + /// needed, so it has to be remembered while the connection is healthy. + pub endpoints: Vec, +} + +impl LeaderCheck { + /// A check that learned nothing: stay where we are, remember no endpoint. + fn inconclusive() -> Self { + Self { + redirect: None, + endpoints: Vec::new(), + } + } +} + /// Check if we need to redirect to leader and return the leader address if redirection is needed pub async fn check_and_redirect_to_leader( client: &C, current_address: &str, transport: TransportProtocol, -) -> Result, IggyError> { +) -> Result { debug!("Checking cluster metadata for leader detection"); // A cluster can be transiently leaderless: a restarted node cedes the @@ -60,15 +81,31 @@ pub async fn check_and_redirect_to_leader( metadata.nodes.len(), metadata.name ); - match process_cluster_metadata(&metadata, current_address, transport) { - Outcome::Redirect(address) => return Ok(Some(address)), - Outcome::LeaderIsCurrent => return Ok(None), + let endpoints = transport_endpoints(&metadata, transport); + match process_cluster_metadata(&metadata, current_address, transport).await { + Outcome::Redirect(address) => { + return Ok(LeaderCheck { + redirect: Some(address), + endpoints, + }); + } + Outcome::LeaderIsCurrent => { + return Ok(LeaderCheck { + redirect: None, + endpoints, + }); + } Outcome::NoLeader => { if tokio::time::Instant::now() >= deadline { warn!( "No active leader found in cluster metadata within {LEADERLESS_WAIT_BUDGET:?}, connection will continue on server node {current_address}", ); - return Ok(None); + // A leaderless roster still names where the nodes + // are, and that is what failover needs. + return Ok(LeaderCheck { + redirect: None, + endpoints, + }); } tokio::time::sleep(LEADERLESS_POLL_INTERVAL).await; } @@ -82,24 +119,45 @@ pub async fn check_and_redirect_to_leader( debug!( "Cluster metadata answered Unauthenticated; the session is gone, connection will continue on server node {current_address}" ); - return Ok(None); + return Ok(LeaderCheck::inconclusive()); } Err(e) => { warn!( "Failed to get cluster metadata: {}, connection will continue on server node {}", e, current_address ); - return Ok(None); + return Ok(LeaderCheck::inconclusive()); } } } } +/// Every endpoint the roster names for `transport`, empty when the read did +/// not answer. +/// +/// No leader verdict and no waiting for an election: the caller is not moving +/// anywhere, it only wants somewhere to dial once the node it is on dies. +pub(crate) async fn read_transport_endpoints( + client: &C, + transport: TransportProtocol, +) -> Vec { + match client.get_cluster_metadata().await { + Ok(metadata) => transport_endpoints(&metadata, transport), + Err(error) => { + debug!("Failed to read the cluster roster: {error}"); + Vec::new() + } + } +} + /// How long to wait for a transiently leaderless cluster to elect before /// proceeding on the current node anyway. const LEADERLESS_WAIT_BUDGET: std::time::Duration = std::time::Duration::from_secs(5); const LEADERLESS_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(250); +/// Bound on one name lookup made to compare two addresses. +const RESOLVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); + /// One leader-check verdict from a cluster-metadata snapshot. enum Outcome { /// A healthy leader exists elsewhere; reconnect to it. @@ -110,8 +168,43 @@ enum Outcome { NoLeader, } +/// Every node's address for `transport`, in roster order. A node that does +/// not expose the transport reports port 0 and is skipped: dialing it would +/// burn a failover attempt on an endpoint that cannot answer. +fn transport_endpoints(metadata: &ClusterMetadata, transport: TransportProtocol) -> Vec { + metadata + .nodes + .iter() + .filter_map(|node| { + let port = transport_port(node, transport); + (port != 0).then(|| node_address(node, port)) + }) + .collect() +} + +/// One node's `host:port`, bracketing a literal IPv6 address. Appending a +/// port to a bare `::1` yields a spelling no dial can parse, so an IPv6 +/// cluster would hand out a roster of undialable entries that still count as +/// endpoints to fail over to. +fn node_address(node: &ClusterNode, port: u16) -> String { + if node.ip.contains(':') && !node.ip.starts_with('[') { + format!("[{}]:{port}", node.ip) + } else { + format!("{}:{port}", node.ip) + } +} + +fn transport_port(node: &ClusterNode, transport: TransportProtocol) -> u16 { + match transport { + TransportProtocol::Tcp => node.endpoints.tcp, + TransportProtocol::Quic => node.endpoints.quic, + TransportProtocol::Http => node.endpoints.http, + TransportProtocol::WebSocket => node.endpoints.websocket, + } +} + /// Process cluster metadata and determine if redirection is needed -fn process_cluster_metadata( +async fn process_cluster_metadata( metadata: &ClusterMetadata, current_address: &str, transport: TransportProtocol, @@ -132,20 +225,15 @@ fn process_cluster_metadata( match leader { Some(leader_node) => { - let leader_port = match transport { - TransportProtocol::Tcp => leader_node.endpoints.tcp, - TransportProtocol::Quic => leader_node.endpoints.quic, - TransportProtocol::Http => leader_node.endpoints.http, - TransportProtocol::WebSocket => leader_node.endpoints.websocket, - }; - let leader_address = format!("{}:{}", leader_node.ip, leader_port); + let leader_port = transport_port(leader_node, transport); + let leader_address = node_address(leader_node, leader_port); info!( "Found leader node: {} at {} (using {} transport)", leader_node.name, leader_address, transport ); - if !is_same_address(current_address, &leader_address) { + if !is_same_address(current_address, &leader_address).await { info!( "Current connection to {} is not the leader, will redirect to {}", current_address, leader_address @@ -160,15 +248,77 @@ fn process_cluster_metadata( } } -/// Check if two addresses refer to the same endpoint -/// Handles various formats like 127.0.0.1:8090 vs localhost:8090 -fn is_same_address(addr1: &str, addr2: &str) -> bool { +/// Whether two addresses are written the same way, up to canonicalization +/// (`localhost` and `[::]` spellings, and a literal address compared as an +/// address rather than as text). +/// +/// Cheap and non-blocking, which is the whole point: the resolving comparison +/// below is a `getaddrinfo`, and every caller reaches this first. +pub(crate) fn is_same_spelling(addr1: &str, addr2: &str) -> bool { match (parse_address(addr1), parse_address(addr2)) { (Some(sock1), Some(sock2)) => sock1.ip() == sock2.ip() && sock1.port() == sock2.port(), _ => normalize_address(addr1) == normalize_address(addr2), } } +/// Check if two addresses refer to the same endpoint +/// Handles various formats like 127.0.0.1:8090 vs localhost:8090 +/// +/// A host name and the address it resolves to are one endpoint too: a client +/// configured as `iggy-server:8090` whose roster advertises `10.0.0.5:8090` +/// would otherwise dial that node twice per failover sweep, and a single-node +/// deployment would be treated as a cluster. +/// +/// Resolution is the last resort, only when the spellings differ and at least +/// one side is not a literal address. It runs through the runtime's resolver +/// rather than `ToSocketAddrs`: name lookup is a blocking `getaddrinfo`, and +/// this is called from the connect and redirect paths, where stalling a +/// runtime worker on a slow resolver would stall every task sharing it. +pub(crate) async fn is_same_address(addr1: &str, addr2: &str) -> bool { + is_same_address_with(addr1, addr2, resolve_all).await +} + +/// [`is_same_address`] against a caller-provided resolver, so the fallback can +/// be exercised without depending on what the machine's resolver answers. +async fn is_same_address_with(addr1: &str, addr2: &str, resolve: R) -> bool +where + R: Fn(String) -> F, + F: Future>>, +{ + if is_same_spelling(addr1, addr2) { + return true; + } + + // Two literal addresses that did not compare equal are different + // endpoints; resolving them would only hand back what they already say. + if parse_address(addr1).is_some() && parse_address(addr2).is_some() { + return false; + } + + let (Some(first), Some(second)) = ( + resolve(addr1.to_owned()).await, + resolve(addr2.to_owned()).await, + ) else { + return false; + }; + first.iter().any(|resolved| second.contains(resolved)) +} + +/// Every socket address a host:port spelling resolves to, `None` when the +/// resolver does not know the name or does not answer in time (which then +/// compares unequal, at worst costing one extra dial). +async fn resolve_all(addr: String) -> Option> { + // A resolver that never answers must not own the request budget: this + // comparison runs on the connect and redirect paths, the redirect one + // inside the caller's request deadline, and `lookup_host` has no deadline + // of its own. + let lookup = tokio::time::timeout(RESOLVE_TIMEOUT, tokio::net::lookup_host(addr)) + .await + .ok()?; + let resolved: Vec = lookup.ok()?.collect(); + (!resolved.is_empty()).then_some(resolved) +} + /// Parse address string to SocketAddr, handling various formats fn parse_address(addr: &str) -> Option { if let Ok(socket_addr) = SocketAddr::from_str(addr) { @@ -245,12 +395,126 @@ mod tests { )); } + fn node(name: &str, ip: &str, tcp: u16, role: ClusterNodeRole) -> ClusterNode { + ClusterNode { + name: name.to_string(), + ip: ip.to_string(), + endpoints: iggy_common::TransportEndpoints::new(tcp, 0, 3000, 3001), + role, + status: ClusterNodeStatus::Healthy, + } + } + #[test] - fn test_is_same_address() { - assert!(is_same_address("127.0.0.1:8090", "127.0.0.1:8090")); - assert!(is_same_address("localhost:8090", "127.0.0.1:8090")); - assert!(!is_same_address("127.0.0.1:8090", "127.0.0.1:8091")); - assert!(!is_same_address("192.168.1.1:8090", "127.0.0.1:8090")); + fn the_roster_names_every_node_that_exposes_the_transport() { + let metadata = ClusterMetadata { + name: "iggy".to_string(), + nodes: vec![ + node("iggy-1", "10.0.0.1", 8090, ClusterNodeRole::Leader), + node("iggy-2", "10.0.0.2", 8090, ClusterNodeRole::Follower), + node("iggy-3", "10.0.0.3", 8090, ClusterNodeRole::Follower), + ], + }; + + assert_eq!( + transport_endpoints(&metadata, TransportProtocol::Tcp), + vec!["10.0.0.1:8090", "10.0.0.2:8090", "10.0.0.3:8090"] + ); + // A node that does not expose the transport reports port 0; dialing + // it would burn a failover attempt on an endpoint that cannot answer. + assert!(transport_endpoints(&metadata, TransportProtocol::Quic).is_empty()); + } + + // A port appended to a bare IPv6 address parses as neither, so the roster + // of an IPv6 cluster would name endpoints no dial can use while still + // counting as somewhere to fail over to. + #[test] + fn an_ipv6_node_is_named_as_a_bracketed_address() { + let metadata = ClusterMetadata { + name: "iggy".to_string(), + nodes: vec![ + node("iggy-1", "::1", 8090, ClusterNodeRole::Leader), + node("iggy-2", "[fd00::2]", 8090, ClusterNodeRole::Follower), + ], + }; + + let endpoints = transport_endpoints(&metadata, TransportProtocol::Tcp); + assert_eq!(endpoints, vec!["[::1]:8090", "[fd00::2]:8090"]); + for endpoint in endpoints { + assert!( + SocketAddr::from_str(&endpoint).is_ok(), + "the roster named an endpoint no dial can parse: {endpoint}" + ); + } + } + + // The address a redirect hands to the next dial comes from the same + // roster entry, so it has to be spelled the same way. + #[tokio::test] + async fn a_redirect_to_an_ipv6_leader_names_a_dialable_address() { + let metadata = ClusterMetadata { + name: "iggy".to_string(), + nodes: vec![ + node("iggy-1", "fd00::1", 8090, ClusterNodeRole::Leader), + node("iggy-2", "fd00::2", 8090, ClusterNodeRole::Follower), + ], + }; + + match process_cluster_metadata(&metadata, "[fd00::2]:8090", TransportProtocol::Tcp).await { + Outcome::Redirect(leader) => assert_eq!(leader, "[fd00::1]:8090"), + Outcome::LeaderIsCurrent => panic!("the follower was taken for the leader"), + Outcome::NoLeader => panic!("the roster named a healthy leader"), + } + } + + #[tokio::test] + async fn test_is_same_address() { + assert!(is_same_address("127.0.0.1:8090", "127.0.0.1:8090").await); + assert!(is_same_address("localhost:8090", "127.0.0.1:8090").await); + assert!(!is_same_address("127.0.0.1:8090", "127.0.0.1:8091").await); + assert!(!is_same_address("192.168.1.1:8090", "127.0.0.1:8090").await); + } + + /// A stand-in resolver: the BDD cluster's spelling of one node, which no + /// canonicalization rewrites, so only the resolving comparison can equate + /// the two. `None` for anything else, like a name the resolver does not + /// know. + async fn resolve_bdd_leader(addr: String) -> Option> { + match addr.as_str() { + "iggy-leader:8091" | "172.28.0.101:8091" => { + Some(vec![SocketAddr::from(([172, 28, 0, 101], 8091))]) + } + _ => None, + } + } + + // A host name and the address it resolves to name one endpoint. Exactly + // the case the BDD cluster hits: the client dials `iggy-leader:8091` and + // the roster advertises `172.28.0.101:8091`. + #[tokio::test] + async fn a_host_name_matches_the_address_it_resolves_to() { + assert!( + is_same_address_with("iggy-leader:8091", "172.28.0.101:8091", resolve_bdd_leader).await + ); + } + + // A name the resolver does not know compares unequal rather than + // erroring, and a resolvable name never matches another port. + #[tokio::test] + async fn an_unresolvable_name_or_another_port_is_a_different_endpoint() { + assert!( + !is_same_address_with( + "iggy-follower:8092", + "172.28.0.101:8091", + resolve_bdd_leader + ) + .await + ); + assert!( + !is_same_address_with("iggy-leader:8091", "172.28.0.101:8092", resolve_bdd_leader) + .await + ); + assert!(!is_same_address("no-such-host.invalid:8090", "127.0.0.1:8090").await); } #[test] diff --git a/core/sdk/src/quic/quic_client.rs b/core/sdk/src/quic/quic_client.rs index b253205548..b1f423a376 100644 --- a/core/sdk/src/quic/quic_client.rs +++ b/core/sdk/src/quic/quic_client.rs @@ -230,6 +230,12 @@ impl iggy_common::VsrSessionControl for QuicClient { impl BinaryClient for QuicClient {} impl QuicClient { + /// Whether an `AutoLogin` is configured on this client, which makes the + /// session after any connect the configured user's rather than whoever + /// signed in by hand. + pub(crate) fn auto_login_configured(&self) -> bool { + matches!(self.config.auto_login, AutoLogin::Enabled(_)) + } /// Creates a new QUIC client for the provided client and server addresses. pub fn new( client_address: &str, @@ -510,12 +516,15 @@ impl QuicClient { /// Returns true if redirection occurred and reconnection is needed. pub(crate) async fn handle_leader_redirection(&self) -> Result { let current_address = self.current_server_address.lock().await.clone(); + // The roster's other endpoints are dropped here: only the TCP client + // dials failover candidates so far. let leader_address = check_and_redirect_to_leader( self, ¤t_address, iggy_common::TransportProtocol::Quic, ) - .await?; + .await? + .redirect; if let Some(new_leader_address) = leader_address { let mut redirection_state = self.leader_redirection_state.lock().await; diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index 1edd3a029f..a271cef8c3 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -16,7 +16,8 @@ // under the License. use crate::leader_aware::{ - LeaderRedirectionState, check_and_redirect_to_leader, is_unauthenticated_metadata_probe, + LeaderRedirectionState, check_and_redirect_to_leader, is_same_spelling, + is_unauthenticated_metadata_probe, read_transport_endpoints, }; use crate::prelude::Client; use crate::prelude::TcpClientConfig; @@ -24,23 +25,30 @@ use crate::session::ConsensusSession; use crate::tcp::tcp_connection_stream::TcpConnectionStream; use crate::tcp::tcp_connection_stream_kind::ConnectionStreamKind; use crate::tcp::tcp_tls_connection_stream::TcpTlsConnectionStream; +use crate::vsr::operation_for_code; use async_broadcast::{Receiver, Sender, broadcast}; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; use iggy_binary_protocol::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE}; +use iggy_binary_protocol::consensus::Operation; +#[cfg(test)] +use iggy_common::TcpClientReconnectionConfig; use iggy_common::VsrSessionControl as _; use iggy_common::{ AutoLogin, ClientState, ConnectionString, ConnectionStringUtils, Credentials, DiagnosticEvent, - IggyDuration, IggyError, IggyTimestamp, NonZeroIggyDuration, TcpConnectionStringOptions, - TransportProtocol, + IdKind, Identifier, IggyDuration, IggyError, IggyTimestamp, NonZeroIggyDuration, + TcpConnectionStringOptions, TransportProtocol, }; use iggy_common::{BinaryClient, BinaryTransport, PersonalAccessTokenClient, UserClient}; use rustls::pki_types::{CertificateDer, ServerName, pem::PemObject}; -use secrecy::ExposeSecret; +use secrecy::{ExposeSecret, SecretString}; use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; use std::sync::Mutex as StdMutex; +use std::sync::atomic::{AtomicBool, Ordering}; +#[cfg(test)] +use tokio::net::TcpListener; use tokio::net::TcpStream; use tokio::sync::Mutex; use tokio::time::sleep; @@ -69,6 +77,18 @@ const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_ /// overall. const TRANSIENT_FAILOVER_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2); +/// Bound on the roster read that follows a sign-in the caller ran itself. The +/// read is a convenience for a failover that may never happen, so a cluster +/// that answers it slowly must not hold up the sign-in. +const ROSTER_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// Bound on one dial while the client has other endpoints to try. A host +/// that drops the SYN -- powered off, or partitioned away -- takes the OS +/// connect timeout to fail, which is minutes, and every other endpoint waits +/// behind it. A client that knows a single endpoint has nothing to starve, so +/// its dial stays unbounded. +const FAILOVER_DIAL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); + /// TCP client for interacting with the Iggy API. /// It requires a valid server address. #[derive(Debug)] @@ -81,6 +101,23 @@ pub struct TcpClient { pub(crate) connected_at: Mutex>, leader_redirection_state: Mutex, pub(crate) current_server_address: Mutex, + /// Every endpoint the cluster roster named, refreshed on each leader + /// check. A node dies together with its address, and the roster is + /// unreachable exactly when it is needed, so the client has to have + /// remembered it while the connection was still healthy. + roster_endpoints: Mutex>, + /// Set once a sign-in on this client has gone looking for the roster, so + /// that read happens once (see [`TcpClient::learn_roster_once`]). + roster_learned: AtomicBool, + /// Credentials a sign-in on this client succeeded with, so a reconnect -- + /// onto this node or, after a failover, another one -- can re-establish + /// the session instead of surfacing `Unauthenticated`. Cleared on logout. + session_credentials: Mutex>, + /// The password a committed change gave the user a configured `AutoLogin` + /// signs in as. The configured credentials cannot be rewritten, and the + /// password they carry is dead once the change commits, so every later + /// sign-in reads this instead. + configured_password: Mutex>, // `std::sync::Mutex` (not `tokio::sync::Mutex`): the critical section // is `encode_request_header`, which is pure CPU and never awaits. The // tokio variant would pay a waker alloc + internal semaphore on @@ -90,6 +127,31 @@ pub struct TcpClient { consumer_group_state: Arc, } +/// The sign-in a manual login on this client succeeded with, and who it signed +/// in as. The user matters because a password change has to swap the new +/// password in here, and `change_password` may well target somebody else. +#[derive(Debug)] +struct RememberedSignIn { + credentials: Credentials, + user_id: u32, +} + +/// A connection that completed every step of coming up, TLS included. +struct EstablishedConnection { + stream: ConnectionStreamKind, + client_address: SocketAddr, + remote_address: SocketAddr, +} + +/// A sign-in that did not complete, and whether the connection it ran on went +/// with it. A connection that is gone leaves the endpoints the sweep has not +/// reached yet worth dialing; one that stands means only the session is +/// missing, which no other endpoint would answer differently. +struct SignInFailure { + error: IggyError, + connection_lost: bool, +} + impl Default for TcpClient { fn default() -> Self { TcpClient::create(Arc::new(TcpClientConfig::default())).unwrap() @@ -103,7 +165,12 @@ impl Client for TcpClient { } async fn disconnect(&self) -> Result<(), IggyError> { - TcpClient::disconnect(self).await + // An explicit disconnect is caller intent, like a logout: the session + // it ends must not be resurrected by the next reconnect, so the + // remembered sign-in goes with it. Involuntary drops (a dead socket, + // a failover) go through `disconnect_transport` and keep it. + self.forget_session_credentials().await; + TcpClient::disconnect_transport(self).await } async fn shutdown(&self) -> Result<(), IggyError> { @@ -160,16 +227,30 @@ impl BinaryTransport for TcpClient { return Err(IggyError::Disconnected); } - if matches!(self.config.auto_login, AutoLogin::Disabled) && !is_login_register_code(code) { - // Without auto-login a reconnect cannot re-establish the session, - // so non-login requests fail fast. Login/register itself is the + if !is_login_register_code(code) && self.sign_in_credentials().await.is_none() { + // With no credentials -- neither configured nor remembered from a + // sign-in -- a reconnect cannot re-establish the session, so + // non-login requests fail fast. Login/register itself is the // exception: the server stays deliberately silent on transient // register failures (the server `surface_login_failure`) and // relies on the client timing out and replaying the request. return Err(error); } - self.disconnect().await?; + // Reconnecting heals the transport, but replaying the request over the + // new connection is a second attempt under a new session: + // `reset_vsr_session` drops the client id the server's dedup fence is + // keyed on, so a replicated write that committed before its reply was + // lost would apply a second time. Replay only what provably never + // reached the log -- the errors raised before the request was written, + // and the operations that never enter it. + // + // Login and register are the exception: the server stays deliberately + // silent on a transient register failure and relies on the client + // replaying, so that replay is the protocol rather than a retry. + let replay_after_reconnect = replay_is_safe(code, &error); + + self.disconnect_transport().await?; let skip_auto_login = is_login_register_code(code); if skip_auto_login { @@ -190,6 +271,16 @@ impl BinaryTransport for TcpClient { *self.skip_auto_login_once.lock().await = false; } reconnect?; + + if !replay_after_reconnect { + warn!( + "Reconnected, but command: {code} is replicated and its outcome is unknown: \ + replaying it under the new session could apply it twice, so the original \ + error is returned instead." + ); + return Err(error); + } + self.send_raw(code, payload).await } @@ -202,6 +293,38 @@ impl BinaryTransport for TcpClient { } } +/// Whether replaying `code` over a fresh connection cannot apply it twice. +/// +/// The reconnect registers a new client identity, so the server's dedup fence +/// no longer covers the original request: only requests that provably never +/// reached the log may be re-sent. +/// +/// - the errors raised before the frame was written, and the server's own +/// refusals, which precede execution. A `StaleClient` eviction is neither: +/// it arrives out of band and is consumed in place of the pending reply, so +/// the request it interrupted may already have committed; +/// - operations that never enter the log: a non-replicated read, and a logout, +/// which ends whatever session the connection carried -- the reconnect +/// brought a new one, and refusing the replay would strand +/// `logout_before_relogin`, whose failure aborts the sign-in that was about +/// to replace the session; +/// - login and register, where the replay is the protocol: the server stays +/// deliberately silent on a transient register failure and relies on the +/// client resending. +fn replay_is_safe(code: u32, error: &IggyError) -> bool { + is_login_register_code(code) + || matches!( + error, + IggyError::NotConnected + | IggyError::CannotEstablishConnection + | IggyError::Unauthenticated + ) + || matches!( + operation_for_code(code), + Operation::NonReplicated | Operation::Logout + ) +} + impl iggy_common::VsrSessionSealed for TcpClient {} #[async_trait::async_trait] @@ -231,6 +354,82 @@ impl iggy_common::VsrSessionControl for TcpClient { Ok(()) } + async fn remember_session_credentials(&self, credentials: Credentials, user_id: u32) { + self.session_credentials + .lock() + .await + .replace(RememberedSignIn { + credentials, + user_id, + }); + self.learn_roster_once().await; + } + + async fn forget_session_credentials(&self) { + self.session_credentials.lock().await.take(); + } + + async fn refresh_session_password(&self, user: &Identifier, new_password: &str) { + // Two independent copies of a password can go stale here, and a change + // may target either user: the sign-in this client remembers, and the + // one a configured `AutoLogin` carries. A caller signed in as somebody + // else -- by hand, or with a token -- can still be the one who changed + // the configured user's password. + let remembered_username = { + let mut remembered = self.session_credentials.lock().await; + remembered.as_mut().and_then(|sign_in| { + // A personal access token is not derived from the password. + let Credentials::UsernamePassword(username, password) = &mut sign_in.credentials + else { + return None; + }; + let targets_session_user = match user.kind { + IdKind::Numeric => user.get_u32_value().is_ok_and(|id| id == sign_in.user_id), + IdKind::String => user + .get_cow_str_value() + .is_ok_and(|name| name.as_ref() == username), + }; + if targets_session_user { + *password = SecretString::from(new_password.to_owned()); + } + Some((username.clone(), targets_session_user)) + }) + }; + + let AutoLogin::Enabled(Credentials::UsernamePassword(configured, _)) = + &self.config.auto_login + else { + return; + }; + + // The configured credentials cannot be rewritten -- the config is + // shared and immutable -- and the password they carry will never work + // again, so the new one is kept beside them. Kept outside the + // remembered sign-in on purpose: that record is replaced wholesale by + // every later login, so a marker on it would survive exactly one + // reconnect and the one after that would replay the dead password. + // + // A numeric identifier can only be recognised as the configured user + // through the id the signed-in user's own login reported, so a change + // made from another user's session has to name the user for the + // configured copy to be refreshed. Naming it is what an administrator + // doing this from elsewhere does anyway. + let targets_configured_user = match user.kind { + IdKind::String => user + .get_cow_str_value() + .is_ok_and(|name| name.as_ref() == configured), + IdKind::Numeric => remembered_username.is_some_and(|(username, is_session_user)| { + is_session_user && &username == configured + }), + }; + if targets_configured_user { + self.configured_password + .lock() + .await + .replace(SecretString::from(new_password.to_owned())); + } + } + fn sdk_version(&self) -> &'static str { crate::SDK_VERSION } @@ -293,6 +492,10 @@ impl TcpClient { connected_at: Mutex::new(None), leader_redirection_state: Mutex::new(LeaderRedirectionState::new()), current_server_address: Mutex::new(server_address), + roster_endpoints: Mutex::new(Vec::new()), + roster_learned: AtomicBool::new(false), + configured_password: Mutex::new(None), + session_credentials: Mutex::new(None), consensus_session: Arc::new(StdMutex::new(ConsensusSession::new())), skip_auto_login_once: Mutex::new(false), consumer_group_state: Arc::new(iggy_common::ConsumerGroupClientState::new()), @@ -301,232 +504,176 @@ impl TcpClient { async fn connect(&self) -> Result<(), IggyError> { loop { - match self.get_state().await { - ClientState::Shutdown => { - trace!("Cannot connect. Client is shutdown."); - return Err(IggyError::ClientShutdown); - } - ClientState::Connected - | ClientState::Authenticating - | ClientState::Authenticated => { - let client_address = self.get_client_address_value().await; - trace!("Client: {client_address} is already connected."); - return Ok(()); - } - ClientState::Connecting => { - trace!("Client is already connecting."); - return Ok(()); + // Read and claimed under one lock acquisition. Apart, two callers + // both find `Disconnected` and both sweep: the loser's + // `reset_vsr_session` re-mints the client id under the identity the + // winner is binding, and its `replace` below drops the live + // authenticated stream. + { + let mut state = self.state.lock().await; + match *state { + ClientState::Shutdown => { + trace!("Cannot connect. Client is shutdown."); + return Err(IggyError::ClientShutdown); + } + ClientState::Connected + | ClientState::Authenticating + | ClientState::Authenticated => { + let client_address = self.get_client_address_value().await; + trace!("Client: {client_address} is already connected."); + return Ok(()); + } + ClientState::Connecting => { + trace!("Client is already connecting."); + return Ok(()); + } + _ => *state = ClientState::Connecting, } - _ => {} } - self.set_state(ClientState::Connecting).await; - if let Some(connected_at) = self.connected_at.lock().await.as_ref() { - let now = IggyTimestamp::now(); - let elapsed = now.as_micros() - connected_at.as_micros(); - let interval = self.config.reconnection.reestablish_after.as_micros(); - trace!( - "Elapsed time since last connection: {}", - IggyDuration::from(elapsed) - ); - if elapsed < interval { - let remaining = IggyDuration::from(interval - elapsed); - info!("Trying to connect to the server in: {remaining}",); - sleep(remaining.get_duration()).await; - } + let mut candidates = self.dial_candidates().await; + // `reestablish_after` paces reconnects to the endpoint this client + // was last on, and to that one only: the other endpoints owe it no + // cooldown, and pausing before dialing them would push the failover + // past the window the caller is willing to wait. So when there is + // somewhere else to go, the paced endpoint goes last -- by which + // time its window has usually elapsed anyway -- instead of the wait + // being skipped outright. + let paced_endpoint = self.current_server_address.lock().await.clone(); + if candidates.len() > 1 && self.reestablish_wait().await.is_some() { + candidates.rotate_left(1); } - let tls_enabled = self.config.tls_enabled; - let mut retry_count = 0; - let connection_stream: ConnectionStreamKind; - let remote_address; - let client_address; - loop { - let server_address = self.current_server_address.lock().await.clone(); - info!( - "{NAME} client is connecting to server: {}...", - server_address - ); - - let connection = TcpStream::connect(&server_address).await; - if let Err(err) = &connection { - error!( - "Failed to connect to server: {}. Error: {}", - server_address, err - ); - if !self.config.reconnection.enabled { - warn!("Automatic reconnection is disabled."); - return Err(IggyError::CannotEstablishConnection); - } + let skip_auto_login = { + let mut guard = self.skip_auto_login_once.lock().await; + std::mem::take(&mut *guard) + }; - let unlimited_retries = self.config.reconnection.max_retries.is_none(); - let max_retries = self.config.reconnection.max_retries.unwrap_or_default(); - let max_retries_str = - if let Some(max_retries) = self.config.reconnection.max_retries { - max_retries.to_string() - } else { - "unlimited".to_string() - }; + let mut retry_count = 0; + let mut candidate = 0; + // A fault no retry can fix, remembered rather than returned at + // once: it belongs to the endpoint that raised it (an unreadable CA + // file, a domain that will not parse), and the endpoints behind + // that one may be perfectly usable. + let mut config_fault: Option = None; + // A sign-in that failed together with the connection it ran on. + // The sweep carries on -- a node that answers the dial and then + // goes quiet must not own the client, and it is also the endpoint + // the next connect would lead with -- and this is the reason the + // caller gets if nothing behind it works out either. + let mut sign_in_failure: Option = None; + let should_redirect = loop { + let server_address = candidates[candidate].clone(); + if server_address == paced_endpoint + && let Some(remaining) = self.reestablish_wait().await + { + info!("Trying to connect to the server: {server_address} in: {remaining}"); + sleep(remaining.get_duration()).await; + } - let interval_str = self.config.reconnection.interval.as_human_time_string(); - if unlimited_retries || retry_count < max_retries { - retry_count += 1; + info!("{NAME} client is connecting to server: {server_address}..."); + match self.establish_bounded(&server_address, &candidates).await { + Ok(connection) => { + let dialed = server_address.clone(); + // The endpoint that answered is where this client now + // lives: the leader check compares against it, and the + // next reconnect starts from it. Recorded only once the + // stream is usable, so a node that accepts TCP but + // fails the TLS handshake does not become sticky and + // shadow the endpoints behind it. + *self.current_server_address.lock().await = server_address; + let client_address = connection.client_address; + self.client_address.lock().await.replace(client_address); + let now = IggyTimestamp::now(); info!( - "Retrying to connect to server ({retry_count}/{max_retries_str}): {} in: {interval_str}", - server_address, + "{NAME} client: {client_address} has connected to server: {} at: {now}", + connection.remote_address, ); - sleep(self.config.reconnection.interval.get_duration()).await; - continue; + self.stream.lock().await.replace(connection.stream); + self.set_state(ClientState::Connected).await; + self.connected_at.lock().await.replace(now); + self.publish_event(DiagnosticEvent::Connected).await; + + match self + .establish_session(client_address, skip_auto_login) + .await + { + Ok(should_redirect) => break should_redirect, + Err(failure) if failure.connection_lost => { + warn!( + "The sign-in on the server: {dialed} did not complete: {}", + failure.error, + ); + sign_in_failure = Some(failure.error); + // The sweep owns the state again: the sign-in + // took the connection down with it, and left + // `Disconnected` another caller would start a + // second sweep alongside this one. + self.set_state(ClientState::Connecting).await; + } + // The connection stands and only the session is + // missing: rejected credentials say the same thing + // on every node, and no endpoint behind this one + // would answer differently. + Err(failure) => return Err(failure.error), + } } - - self.set_state(ClientState::Disconnected).await; - self.publish_event(DiagnosticEvent::Disconnected).await; - return Err(IggyError::CannotEstablishConnection); + Err(IggyError::CannotEstablishConnection) => {} + Err(error) => config_fault = Some(error), } - let stream = connection.map_err(|error| { - error!("Failed to establish TCP connection to the server: {error}",); - IggyError::CannotEstablishConnection - })?; - client_address = stream.local_addr().map_err(|error| { - error!("Failed to get the local address of the client: {error}",); - IggyError::CannotEstablishConnection - })?; - remote_address = stream.peer_addr().map_err(|error| { - error!("Failed to get the remote address of the server: {error}",); - IggyError::CannotEstablishConnection - })?; - self.client_address.lock().await.replace(client_address); - - if let Err(e) = stream.set_nodelay(self.config.nodelay) { - error!("Failed to set the nodelay option on the client: {e}, continuing...",); + // Every other endpoint gets its turn before the retry + // interval: the node just lost may be gone for good, and + // pausing on it helps nothing. + candidate += 1; + if candidate < candidates.len() { + continue; } - - if !tls_enabled { - connection_stream = - ConnectionStreamKind::Tcp(TcpConnectionStream::new(client_address, stream)); - break; + candidate = 0; + + // An unreadable CA file, a domain that will not parse: no + // endpoint answered and at least one said why in a way that a + // retry cannot change, so the caller gets that reason instead + // of a retry loop that buries it (`max_retries = None` would + // otherwise redial it every interval forever). + if let Some(error) = config_fault { + self.fail_connect().await; + return Err(error); } - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + // The sweep is what reconnection settings apply to, not a + // single dial: with reconnection off there are no retries, but + // the failover endpoints were configured to be tried and they + // get their one turn first. + if !self.config.reconnection.enabled { + warn!("Automatic reconnection is disabled."); + self.fail_connect().await; + return Err(sign_in_failure.unwrap_or(IggyError::CannotEstablishConnection)); + } - let config = if self.config.tls_validate_certificate { - let mut root_cert_store = rustls::RootCertStore::empty(); - if let Some(certificate_path) = &self.config.tls_ca_file { - for cert in - CertificateDer::pem_file_iter(certificate_path).map_err(|error| { - error!("Failed to read the CA file: {certificate_path}. {error}",); - IggyError::InvalidTlsCertificatePath - })? - { - let certificate = cert.map_err(|error| { - error!( - "Failed to read a certificate from the CA file: {certificate_path}. {error}", - ); - IggyError::InvalidTlsCertificate - })?; - root_cert_store.add(certificate).map_err(|error| { - error!( - "Failed to add a certificate to the root certificate store. {error}", - ); - IggyError::InvalidTlsCertificate - })?; - } + let unlimited_retries = self.config.reconnection.max_retries.is_none(); + let max_retries = self.config.reconnection.max_retries.unwrap_or_default(); + let max_retries_str = + if let Some(max_retries) = self.config.reconnection.max_retries { + max_retries.to_string() } else { - root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - } - - rustls::ClientConfig::builder() - .with_root_certificates(root_cert_store) - .with_no_client_auth() - } else { - use crate::tcp::tcp_tls_verifier::NoServerVerification; - rustls::ClientConfig::builder() - .dangerous() - .with_custom_certificate_verifier(Arc::new(NoServerVerification)) - .with_no_client_auth() - }; - let connector = TlsConnector::from(Arc::new(config)); - let tls_domain = if self.config.tls_domain.is_empty() { - // Extract hostname/IP from server_address when tls_domain is not specified - server_address - .split(':') - .next() - .unwrap_or(&server_address) - .to_string() - } else { - self.config.tls_domain.to_owned() - }; - let domain = ServerName::try_from(tls_domain).map_err(|error| { - error!("Failed to create a server name from the domain. {error}",); - IggyError::InvalidTlsDomain - })?; - let stream = connector.connect(domain, stream).await.map_err(|error| { - error!("Failed to establish a TLS connection to the server: {error}",); - IggyError::CannotEstablishConnection - })?; - connection_stream = ConnectionStreamKind::TcpTls(TcpTlsConnectionStream::new( - client_address, - TlsStream::Client(stream), - )); - break; - } - - let now = IggyTimestamp::now(); - info!( - "{NAME} client: {client_address} has connected to server: {remote_address} at: {now}", - ); - self.stream.lock().await.replace(connection_stream); - self.set_state(ClientState::Connected).await; - self.connected_at.lock().await.replace(now); - self.publish_event(DiagnosticEvent::Connected).await; - let skip_auto_login = { - let mut guard = self.skip_auto_login_once.lock().await; - std::mem::take(&mut *guard) - }; + "unlimited".to_string() + }; - // Handle auto-login - let should_redirect = match &self.config.auto_login { - AutoLogin::Disabled => { - info!("Automatic sign-in is disabled."); - // Only `IggyClient` redirects after a manual sign-in, so - // a raw transport can stay on a backup: its first - // replicated write gets `TransientNotAccepted`, the - // redirect drops the session, and the retry fails - // `Unauthenticated` until the caller signs in again. - false + if unlimited_retries || retry_count < max_retries { + retry_count += 1; + let interval_str = self.config.reconnection.interval.as_human_time_string(); + info!( + "Retrying to connect ({retry_count}/{max_retries_str}), \ + {} endpoint(s) in: {interval_str}", + candidates.len(), + ); + sleep(self.config.reconnection.interval.get_duration()).await; + continue; } - AutoLogin::Enabled(credentials) => { - if skip_auto_login { - info!("Skipping automatic sign-in for a retried login/register request."); - false - } else { - info!("{NAME} client: {client_address} is signing in..."); - self.set_state(ClientState::Authenticating).await; - match credentials { - Credentials::UsernamePassword(username, password) => { - self.login_user(username, password.expose_secret()).await?; - info!( - "{NAME} client: {client_address} has signed in with the user credentials, username: {username}", - ); - } - Credentials::PersonalAccessToken(token) => { - self.login_with_personal_access_token(token.expose_secret()) - .await?; - info!( - "{NAME} client: {client_address} has signed in with a personal access token.", - ); - } - } - // The sole leader settlement, and it runs - // authenticated. Any node completes a login now -- a - // backup forwards the register to the primary -- so - // this decides where later ops land, not whether - // sign-in works. - self.handle_leader_redirection().await? - } - } + self.fail_connect().await; + return Err(sign_in_failure.unwrap_or(IggyError::CannotEstablishConnection)); }; if should_redirect { @@ -537,18 +684,125 @@ impl TcpClient { } } + /// Re-establish the session on a connection that just came up and settle it + /// on the leader. Reports whether the leader check asks for a redirect. + async fn establish_session( + &self, + client_address: SocketAddr, + skip_auto_login: bool, + ) -> Result { + let Some(credentials) = self.sign_in_credentials().await else { + info!("No credentials to sign in with."); + // Only `IggyClient` redirects after a manual sign-in, so a raw + // transport can stay on a backup: its first replicated write gets + // `TransientNotAccepted`, the redirect drops the session, and the + // retry fails `Unauthenticated` until the caller signs in again. + return Ok(false); + }; + + if skip_auto_login { + info!("Skipping automatic sign-in for a retried login/register request."); + return Ok(false); + } + + info!("{NAME} client: {client_address} is signing in..."); + self.set_state(ClientState::Authenticating).await; + let signed_in = match &credentials { + Credentials::UsernamePassword(username, password) => self + .login_user(username, password.expose_secret()) + .await + .map(|_| format!("the user credentials, username: {username}")), + Credentials::PersonalAccessToken(token) => self + .login_with_personal_access_token(token.expose_secret()) + .await + .map(|_| "a personal access token".to_owned()), + }; + match signed_in { + Ok(how) => info!("{NAME} client: {client_address} has signed in with {how}."), + Err(error) => return Err(self.fail_sign_in(error).await), + } + + // The sole leader settlement, and it runs authenticated. Any node + // completes a login now -- a backup forwards the register to the + // primary -- so this decides where later ops land, not whether sign-in + // works. + self.handle_leader_redirection() + .await + .map_err(|error| SignInFailure { + error, + connection_lost: false, + }) + } + + /// Put the client back into a state that describes what a failed sign-in + /// left behind, and report whether the connection survived it. + async fn fail_sign_in(&self, error: IggyError) -> SignInFailure { + // A sign-in can fail because the socket died under it. Whatever is left + // of that connection cannot carry a request, so it goes rather than + // being kept behind a `Connected` that makes the next `connect()` a + // no-op and leaves every gated operation failing until someone calls + // `disconnect()` by hand. + let connection_lost = matches!( + error, + IggyError::Disconnected + | IggyError::EmptyResponse + | IggyError::NotConnected + | IggyError::CannotEstablishConnection + | IggyError::TcpError + | IggyError::StaleClient + ); + if connection_lost { + if let Err(teardown_error) = self.disconnect_transport().await { + warn!("Failed to drop the connection of a failed sign-in: {teardown_error}"); + } + } else if self.get_state().await == ClientState::Authenticating { + // With the transport up and only the session missing, the state has + // to say so: left at `Authenticating` every gated operation fails + // client-side with `Disconnected`, `connect()` returns ok without + // dialing, and nothing short of an explicit `login_user` recovers. + self.set_state(ClientState::Connected).await; + } + + // A rejected credential does not become valid on the next reconnect, + // and replaying it costs an argon2 on the server every time. Configured + // credentials stay as configured -- they are the caller's to fix -- so + // only the remembered sign-in is dropped. + if matches!( + error, + IggyError::InvalidCredentials + | IggyError::InvalidUsername + | IggyError::InvalidPassword + | IggyError::Unauthenticated + ) { + self.forget_session_credentials().await; + } + + SignInFailure { + error, + connection_lost, + } + } + /// Checks cluster metadata and handles leader redirection if needed. /// Returns true if redirection occurred and reconnection is needed. pub(crate) async fn handle_leader_redirection(&self) -> Result { let current_address = self.current_server_address.lock().await.clone(); - let leader_address = check_and_redirect_to_leader( + let leader_check = check_and_redirect_to_leader( self, ¤t_address, iggy_common::TransportProtocol::Tcp, ) .await?; - if let Some(new_leader_address) = leader_address { + // Replaced wholesale rather than merged: the roster is the cluster's + // own answer about where its nodes are, so a node it dropped should + // stop being dialed. The configured seeds are kept separately and + // outlive it. + if !leader_check.endpoints.is_empty() { + *self.roster_endpoints.lock().await = leader_check.endpoints; + } + + if let Some(new_leader_address) = leader_check.redirect { let mut redirection_state = self.leader_redirection_state.lock().await; if !redirection_state.can_redirect() { warn!("Maximum leader redirections reached, continuing with current connection"); @@ -564,7 +818,7 @@ impl TcpClient { // Clear connected_at to avoid reestablish_after delay during redirection self.connected_at.lock().await.take(); - self.disconnect().await?; + self.disconnect_transport().await?; *self.current_server_address.lock().await = new_leader_address; Ok(true) @@ -574,9 +828,295 @@ impl TcpClient { } } - async fn disconnect(&self) -> Result<(), IggyError> { - if self.get_state().await == ClientState::Disconnected { - return Ok(()); + /// Whether an `AutoLogin` is configured on this client, which makes the + /// session after any connect the configured user's rather than whoever + /// signed in by hand. + pub(crate) fn auto_login_configured(&self) -> bool { + matches!(self.config.auto_login, AutoLogin::Enabled(_)) + } + + /// Credentials to sign in with after connecting: the configured ones, or + /// else the ones a manual sign-in on this client succeeded with. A manual + /// sign-in is otherwise less reconnectable than a configured one, which + /// is a surprising difference between two ways of doing the same thing. + /// + /// A password change this client committed for the configured user is + /// applied on top: the configured password will never work again, and every + /// later reconnect would otherwise fail `InvalidCredentials`. + async fn sign_in_credentials(&self) -> Option { + // The sign-in that last succeeded, whoever ran it: a client is whoever + // it last signed in as, so a reconnect restores the session the caller + // last asked for rather than one it had moved off. A configured + // `AutoLogin` signs in through this very path, so for a client that + // never signed in by hand the remembered credentials *are* the + // configured ones. + if let Some(remembered) = self.session_credentials.lock().await.as_ref() { + return Some(remembered.credentials.clone()); + } + + match &self.config.auto_login { + // Before the first sign-in, or after a logout dropped what was + // remembered. A password change this client committed for the + // configured user is applied on top: the configured password will + // never work again, and the config cannot be rewritten. + AutoLogin::Enabled(Credentials::UsernamePassword(username, configured_password)) => { + let password = self + .configured_password + .lock() + .await + .clone() + .unwrap_or_else(|| configured_password.clone()); + Some(Credentials::UsernamePassword(username.clone(), password)) + } + AutoLogin::Enabled(credentials) => Some(credentials.clone()), + AutoLogin::Disabled => None, + } + } + + /// Read the cluster roster once, on the first sign-in that succeeds on a + /// client whose caller signs it in by hand. + /// + /// `connect()` follows the sign-in it runs itself with a leader check, and + /// that check is what refreshes the roster. A client with no configured + /// `AutoLogin` is signed in by its caller instead, and only `IggyClient` + /// follows that with a leader check, so a raw transport would know exactly + /// one endpoint -- the one it was configured with -- and redial the node + /// that died for as long as it lived. + /// + /// Once per client, which is also what keeps the read from nesting: it goes + /// through the reconnect path, whose sign-in calls straight back into here. + /// Bounded for the same reason: the read is a convenience for a failover + /// that may never happen, so it must not hold up the sign-in that triggered + /// it -- unbounded retries would do exactly that. + async fn learn_roster_once(&self) { + // Only a live session can read the roster, and only the caller's own + // sign-in leaves one behind here: a connect that signs in follows it + // with a leader check of its own. + if self.auto_login_configured() + || self.get_state().await != ClientState::Authenticated + || self.roster_learned.swap(true, Ordering::SeqCst) + { + return; + } + + let read = read_transport_endpoints(self, TransportProtocol::Tcp); + let Ok(endpoints) = tokio::time::timeout(ROSTER_READ_TIMEOUT, read).await else { + warn!("Reading the cluster roster took longer than {ROSTER_READ_TIMEOUT:?}"); + return; + }; + if endpoints.is_empty() { + return; + } + + info!( + "{NAME} client learned {} endpoint(s) to fail over to.", + endpoints.len() + ); + *self.roster_endpoints.lock().await = endpoints; + } + + /// Endpoints to dial for one connect, likeliest first: where the client + /// currently is, the address it was configured with, then the roster it + /// learned while connected. + /// + /// Configured before learned, as in the other SDKs: that is the endpoint + /// the caller vouched for, while a roster read from a cluster that has since + /// changed shape may name nodes that are gone. + async fn dial_candidates(&self) -> Vec { + let mut candidates = vec![self.current_server_address.lock().await.clone()]; + let roster = self.roster_endpoints.lock().await.clone(); + let configured = std::iter::once(&self.config.server_address); + for endpoint in configured.chain(roster.iter()) { + // Spellings only, no name resolution: one duplicate endpoint costs + // a dial that fails on its own, while a resolver that does not + // answer would stall the failover before it dialed anything. + if !candidates + .iter() + .any(|candidate| is_same_spelling(candidate, endpoint)) + { + candidates.push(endpoint.clone()); + } + } + candidates + } + + /// Bring one endpoint all the way up: TCP connect, socket options, and the + /// TLS handshake when it is configured. Nothing about the connection is + /// recorded until this succeeds, so a half-usable endpoint leaves no trace + /// for the next connect to lead with. + /// + /// `CannotEstablishConnection` means this endpoint failed and the next one + /// is worth trying; any other error is a configuration fault that no + /// endpoint can satisfy. + async fn establish(&self, server_address: &str) -> Result { + let stream = TcpStream::connect(server_address).await.map_err(|error| { + error!("Failed to connect to server: {server_address}. Error: {error}"); + IggyError::CannotEstablishConnection + })?; + let client_address = stream.local_addr().map_err(|error| { + error!("Failed to get the local address of the client: {error}"); + IggyError::CannotEstablishConnection + })?; + let remote_address = stream.peer_addr().map_err(|error| { + error!("Failed to get the remote address of the server: {error}"); + IggyError::CannotEstablishConnection + })?; + + if let Err(error) = stream.set_nodelay(self.config.nodelay) { + error!("Failed to set the nodelay option on the client: {error}, continuing..."); + } + + if !self.config.tls_enabled { + return Ok(EstablishedConnection { + stream: ConnectionStreamKind::Tcp(TcpConnectionStream::new(client_address, stream)), + client_address, + remote_address, + }); + } + + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let config = if self.config.tls_validate_certificate { + let mut root_cert_store = rustls::RootCertStore::empty(); + if let Some(certificate_path) = &self.config.tls_ca_file { + for cert in CertificateDer::pem_file_iter(certificate_path).map_err(|error| { + error!("Failed to read the CA file: {certificate_path}. {error}"); + IggyError::InvalidTlsCertificatePath + })? { + let certificate = cert.map_err(|error| { + error!( + "Failed to read a certificate from the CA file: {certificate_path}. {error}", + ); + IggyError::InvalidTlsCertificate + })?; + root_cert_store.add(certificate).map_err(|error| { + error!( + "Failed to add a certificate to the root certificate store. {error}" + ); + IggyError::InvalidTlsCertificate + })?; + } + } else { + root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + } + + rustls::ClientConfig::builder() + .with_root_certificates(root_cert_store) + .with_no_client_auth() + } else { + use crate::tcp::tcp_tls_verifier::NoServerVerification; + rustls::ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoServerVerification)) + .with_no_client_auth() + }; + + let connector = TlsConnector::from(Arc::new(config)); + let tls_domain = if self.config.tls_domain.is_empty() { + // Extract hostname/IP from server_address when tls_domain is not specified + server_address + .split(':') + .next() + .unwrap_or(server_address) + .to_string() + } else { + self.config.tls_domain.to_owned() + }; + let domain = ServerName::try_from(tls_domain).map_err(|error| { + error!("Failed to create a server name from the domain. {error}"); + IggyError::InvalidTlsDomain + })?; + let stream = connector.connect(domain, stream).await.map_err(|error| { + // The verdict describes the peer, not this client: a certificate + // that names another host, a peer that answers a ClientHello with + // something else. The endpoints behind it may be fine, and with + // one roster entry per node the SNI is a bare address that no + // certificate has to cover, so this ends the dial rather than the + // connect. + error!("Failed to establish a TLS connection to the server: {error}"); + IggyError::CannotEstablishConnection + })?; + + Ok(EstablishedConnection { + stream: ConnectionStreamKind::TcpTls(TcpTlsConnectionStream::new( + client_address, + TlsStream::Client(stream), + )), + client_address, + remote_address, + }) + } + + /// Give up on connecting. The state has to go back to `Disconnected`: + /// left at `Connecting`, the next `connect()` returns ok at the top + /// without ever dialing. + async fn fail_connect(&self) { + self.set_state(ClientState::Disconnected).await; + self.publish_event(DiagnosticEvent::Disconnected).await; + } + + /// [`Self::establish`], bounded while other endpoints are queued behind + /// this one (see `FAILOVER_DIAL_TIMEOUT`). The bound covers the handshake + /// as well as the connect: a peer that accepts TCP and then never answers + /// the ClientHello is exactly the kind of failure the survivors are there + /// for, and neither step has a deadline of its own. + async fn establish_bounded( + &self, + server_address: &str, + candidates: &[String], + ) -> Result { + if candidates.len() < 2 { + return self.establish(server_address).await; + } + + match tokio::time::timeout(FAILOVER_DIAL_TIMEOUT, self.establish(server_address)).await { + Ok(connection) => connection, + Err(_elapsed) => { + error!( + "Connecting to server: {server_address} took longer than \ + {FAILOVER_DIAL_TIMEOUT:?}" + ); + Err(IggyError::CannotEstablishConnection) + } + } + } + + /// What is left of the `reestablish_after` window since the last + /// successful connection, if any. + async fn reestablish_wait(&self) -> Option { + let connected_at = self + .connected_at + .lock() + .await + .as_ref() + .map(IggyTimestamp::as_micros)?; + let elapsed = IggyTimestamp::now().as_micros() - connected_at; + let interval = self.config.reconnection.reestablish_after.as_micros(); + trace!( + "Elapsed time since last connection: {}", + IggyDuration::from(elapsed) + ); + (elapsed < interval).then(|| IggyDuration::from(interval - elapsed)) + } + + /// Tear down the connection without touching the remembered sign-in. + /// + /// The reconnect and redirect paths use this: their disconnect is not + /// caller intent, and forgetting the credentials here would strand the + /// failover unauthenticated. The public [`Client::disconnect`] wraps this + /// and forgets them first. + async fn disconnect_transport(&self) -> Result<(), IggyError> { + match self.get_state().await { + ClientState::Disconnected => return Ok(()), + // A connect is already sweeping, and every caller here is tearing + // the connection down in order to reconnect -- which is what that + // sweep is doing. Tearing it down under the sweep would re-mint the + // client id the sign-in in flight is binding and take the stream it + // just installed. + ClientState::Connecting => { + trace!("Not disconnecting; a connect is already in flight."); + return Ok(()); + } + _ => {} } let client_address = self.get_client_address_value().await; @@ -871,6 +1411,770 @@ const fn is_login_register_code(code: u32) -> bool { #[cfg(test)] mod tests { use super::*; + use iggy_binary_protocol::codes::{GET_ME_CODE, LOGOUT_USER_CODE, SEND_MESSAGES_CODE}; + use std::sync::atomic::AtomicUsize; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + const SESSION_USER_ID: u32 = 7; + + fn client_with(server_address: &str) -> TcpClient { + TcpClient::create(Arc::new(TcpClientConfig { + server_address: server_address.to_string(), + ..TcpClientConfig::default() + })) + .expect("create the client") + } + + /// A client whose roster names `endpoints`, as a leader check leaves it. + async fn client_with_roster(server_address: &str, endpoints: Vec) -> TcpClient { + let client = client_with(server_address); + *client.roster_endpoints.lock().await = endpoints; + client + } + + /// A listener nothing ever accepts from: the kernel completes the TCP + /// handshake out of its backlog, which is all a dial needs to succeed. + async fn live_endpoint() -> (TcpListener, String) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind a listener"); + let address = listener.local_addr().expect("listener address").to_string(); + (listener, address) + } + + /// An address with nothing behind it: the dial is refused at once. + async fn dead_endpoint() -> String { + let (listener, address) = live_endpoint().await; + drop(listener); + address + } + + /// A peer that accepts TCP and hangs up without a byte: enough for the + /// dial, never enough for a TLS handshake or a sign-in. The counter says + /// how many dials reached it. + async fn counted_endpoint_that_hangs_up() -> (String, Arc) { + let (listener, address) = live_endpoint().await; + let dials = Arc::new(AtomicUsize::new(0)); + let accepted = dials.clone(); + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + accepted.fetch_add(1, Ordering::SeqCst); + drop(stream); + } + }); + (address, dials) + } + + async fn endpoint_that_hangs_up() -> String { + counted_endpoint_that_hangs_up().await.0 + } + + // With reconnection off there are no retries, but the endpoints the roster + // named are still there to be tried and each gets its one turn. + #[tokio::test] + async fn a_client_with_reconnection_disabled_still_sweeps_the_endpoints_it_knows() { + let (_listener, survivor) = live_endpoint().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: dead_endpoint().await, + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + *client.roster_endpoints.lock().await = vec![survivor.clone()]; + + TcpClient::connect(&client).await.expect("connect"); + assert_eq!(*client.current_server_address.lock().await, survivor); + } + + // A connect that gives up has to leave the state at `Disconnected`: left + // at `Connecting`, the next `connect()` returns ok at the top without ever + // dialing, and the client is wedged for good. + #[tokio::test] + async fn a_connect_that_exhausts_every_endpoint_leaves_the_client_disconnected() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: dead_endpoint().await, + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + *client.roster_endpoints.lock().await = vec![dead_endpoint().await]; + + assert!(matches!( + TcpClient::connect(&client).await, + Err(IggyError::CannotEstablishConnection) + )); + assert_eq!(client.get_state().await, ClientState::Disconnected); + assert!( + matches!( + TcpClient::connect(&client).await, + Err(IggyError::CannotEstablishConnection) + ), + "a second connect has to dial again rather than report success" + ); + } + + // An endpoint that accepts TCP but fails the TLS handshake is not where + // this client lives: recording it would make the next connect lead with + // it and shadow every endpoint behind it. + #[tokio::test] + async fn an_endpoint_that_fails_the_tls_handshake_does_not_become_the_current_one() { + let configured = dead_endpoint().await; + // Plain TCP behind a TLS client: the dial succeeds, the handshake + // cannot. + let plaintext = endpoint_that_hangs_up().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: configured.clone(), + tls_enabled: true, + tls_validate_certificate: false, + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + *client.roster_endpoints.lock().await = vec![plaintext]; + + assert!(matches!( + TcpClient::connect(&client).await, + Err(IggyError::CannotEstablishConnection) + )); + assert_eq!(*client.current_server_address.lock().await, configured); + } + + // A peer that accepts TCP and then never answers is what the other + // endpoints are there for; without a bound on the handshake the sweep + // waits on it forever. + #[tokio::test] + async fn an_endpoint_that_never_answers_the_handshake_does_not_hold_up_the_sweep() { + let (_listener, silent) = live_endpoint().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: silent, + tls_enabled: true, + tls_validate_certificate: false, + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + *client.roster_endpoints.lock().await = vec![dead_endpoint().await]; + + let sweep = tokio::time::timeout( + FAILOVER_DIAL_TIMEOUT * 3, + std::pin::pin!(TcpClient::connect(&client)), + ) + .await + .expect("the sweep has to end on its own"); + assert!(matches!(sweep, Err(IggyError::CannotEstablishConnection))); + } + + /// A peer that accepts TCP and then answers a ClientHello with something + /// else, so the handshake fails on the peer's own answer. The counter says + /// how many dials reached it. + async fn counted_endpoint_that_speaks_no_tls() -> (String, Arc) { + let (listener, address) = live_endpoint().await; + let dials = Arc::new(AtomicUsize::new(0)); + let accepted = dials.clone(); + tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + accepted.fetch_add(1, Ordering::SeqCst); + tokio::spawn(async move { + let _ = stream.write_all(b"this is not a TLS record\n").await; + // Held open, so the failure is the handshake's verdict + // rather than a closed socket. + let mut sink = [0u8; 64]; + while stream.read(&mut sink).await.is_ok_and(|read| read > 0) {} + }); + } + }); + (address, dials) + } + + // A handshake verdict describes the peer -- a certificate that names + // another host, an answer that is not TLS at all -- and not this client's + // configuration, so it ends the dial rather than the connect: the endpoints + // behind it are untried, and a redial can find a repaired node. + #[tokio::test] + async fn a_handshake_the_peer_failed_is_dialed_again() { + let (plaintext, dials) = counted_endpoint_that_speaks_no_tls().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: plaintext, + tls_enabled: true, + tls_validate_certificate: false, + reconnection: TcpClientReconnectionConfig { + // One retry, so the pass runs twice and the connect still ends + // on its own. + max_retries: Some(1), + interval: NonZeroIggyDuration::from_str("100ms").expect("duration"), + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + + let connect = tokio::time::timeout( + std::time::Duration::from_secs(10), + std::pin::pin!(TcpClient::connect(&client)), + ) + .await + .expect("the connect has to end on its own"); + assert!(matches!(connect, Err(IggyError::CannotEstablishConnection))); + assert_eq!( + dials.load(Ordering::SeqCst), + 2, + "a handshake the peer failed ended the connect instead of the dial" + ); + assert_eq!(client.get_state().await, ClientState::Disconnected); + } + + // A CA file that cannot be read is this client's own configuration, and it + // says the same thing on every attempt: reported as a lost connection it + // would be redialed every interval forever under `max_retries = None`, + // which is how a wrong CA path looks like a flaky network. + #[tokio::test] + async fn a_ca_file_that_cannot_be_read_ends_the_connect() { + let (_listener, endpoint) = live_endpoint().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: endpoint, + tls_enabled: true, + tls_validate_certificate: true, + tls_ca_file: Some("no-such-ca-file.pem".to_string()), + reconnection: TcpClientReconnectionConfig { + // Unlimited retries, so a transient classification never + // returns and this test times out instead of failing. + max_retries: None, + interval: NonZeroIggyDuration::from_str("100ms").expect("duration"), + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + + let connect = tokio::time::timeout( + std::time::Duration::from_secs(10), + std::pin::pin!(TcpClient::connect(&client)), + ) + .await + .expect("the connect has to end on its own"); + assert!(matches!(connect, Err(IggyError::InvalidTlsCertificatePath))); + assert_eq!(client.get_state().await, ClientState::Disconnected); + } + + // A node that answers the dial and then cannot carry the sign-in has to + // hand the sweep on. Ending it there leaves that node the one the client is + // recorded on, so every later connect leads with it and the endpoints + // behind it are never reached. + #[tokio::test] + async fn a_sign_in_that_failed_hands_the_sweep_on_to_the_next_endpoint() { + let (dialed_first, _) = counted_endpoint_that_hangs_up().await; + let (survivor, survivor_dials) = counted_endpoint_that_hangs_up().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: dialed_first, + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "iggy".to_string(), + "iggy".into(), + )), + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + *client.roster_endpoints.lock().await = vec![survivor]; + + assert!(TcpClient::connect(&client).await.is_err()); + assert_eq!( + survivor_dials.load(Ordering::SeqCst), + 1, + "the endpoint behind the one whose sign-in failed was never dialed" + ); + assert_eq!(client.get_state().await, ClientState::Disconnected); + } + + // A sign-in that lost its socket leaves nothing to send on, so the + // transport goes with it: kept behind a `Connected`, the next `connect()` + // would return ok without dialing and every gated operation would fail + // until someone disconnected by hand. + #[tokio::test] + async fn a_sign_in_that_lost_its_socket_leaves_the_client_disconnected() { + let (listener, address) = live_endpoint().await; + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + // Accept, then hang up: the dial succeeds and the sign-in dies + // on the socket. + drop(stream); + } + }); + + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: address, + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "iggy".to_string(), + "iggy".into(), + )), + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + + assert!(TcpClient::connect(&client).await.is_err()); + assert_eq!(client.get_state().await, ClientState::Disconnected); + assert!( + client.stream.lock().await.is_none(), + "a dead connection must not be kept for the next request to find" + ); + } + + // The reconnect registers a new client identity, so the server's dedup + // fence no longer covers the original request. + #[test] + fn only_requests_that_cannot_double_apply_are_replayed() { + // Never written, or refused before execution. + assert!(replay_is_safe(SEND_MESSAGES_CODE, &IggyError::NotConnected)); + assert!(replay_is_safe( + SEND_MESSAGES_CODE, + &IggyError::CannotEstablishConnection + )); + // Written, and its outcome unknown: a replicated write must not be + // re-sent under a session the fence cannot match it against. An + // eviction is consumed in place of the reply, so it says nothing about + // whether the write committed. + assert!(!replay_is_safe(SEND_MESSAGES_CODE, &IggyError::StaleClient)); + assert!(!replay_is_safe( + SEND_MESSAGES_CODE, + &IggyError::Disconnected + )); + assert!(!replay_is_safe( + SEND_MESSAGES_CODE, + &IggyError::EmptyResponse + )); + // A read never enters the log, and a logout ends a session the + // reconnect already replaced -- `logout_before_relogin` depends on it. + assert!(replay_is_safe(GET_ME_CODE, &IggyError::Disconnected)); + assert!(replay_is_safe(LOGOUT_USER_CODE, &IggyError::Disconnected)); + // The register replay is the protocol: the server stays silent on a + // transient failure and waits for the resend. + assert!(replay_is_safe( + LOGIN_REGISTER_CODE, + &IggyError::Disconnected + )); + } + + // `reestablish_after` paces reconnects to the endpoint that was lost. With + // somewhere else to go, that pause must not hold up the failover. + #[tokio::test] + async fn a_pending_reestablish_pause_does_not_delay_dialing_another_endpoint() { + let (_listener, survivor) = live_endpoint().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: dead_endpoint().await, + reconnection: TcpClientReconnectionConfig { + reestablish_after: IggyDuration::from_str("10s").expect("duration"), + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + *client.roster_endpoints.lock().await = vec![survivor.clone()]; + client + .connected_at + .lock() + .await + .replace(IggyTimestamp::now()); + + let started = std::time::Instant::now(); + TcpClient::connect(&client).await.expect("connect"); + assert_eq!(*client.current_server_address.lock().await, survivor); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "the failover waited out a pause it owed only the lost endpoint: {:?}", + started.elapsed() + ); + } + + // The other half of the same promise: `with_reestablish_after` is a + // cooldown on redialing the endpoint that was lost, and a known roster + // does not cancel it. + #[tokio::test] + async fn the_reestablish_pause_still_applies_to_the_endpoint_that_was_lost() { + let (_listener, current) = live_endpoint().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: current.clone(), + reconnection: TcpClientReconnectionConfig { + reestablish_after: IggyDuration::from_str("1s").expect("duration"), + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + *client.roster_endpoints.lock().await = vec![dead_endpoint().await]; + client + .connected_at + .lock() + .await + .replace(IggyTimestamp::now()); + + let started = std::time::Instant::now(); + TcpClient::connect(&client).await.expect("connect"); + assert_eq!(*client.current_server_address.lock().await, current); + assert!( + started.elapsed() >= std::time::Duration::from_millis(700), + "the cooldown on the lost endpoint was skipped: {:?}", + started.elapsed() + ); + } + + #[tokio::test] + async fn dial_candidates_lead_with_the_current_endpoint_and_name_each_other_one_once() { + let client = client_with_roster( + "127.0.0.1:8090", + vec![ + "127.0.0.1:8090".to_string(), + "localhost:8090".to_string(), + "127.0.0.1:8091".to_string(), + ], + ) + .await; + + // The current endpoint leads and the roster follows, the same order as + // the other SDKs. An endpoint the roster names again earns no second + // dial, whether it is spelled the same way or not. + assert_eq!( + client.dial_candidates().await, + vec!["127.0.0.1:8090".to_string(), "127.0.0.1:8091".to_string()] + ); + } + + #[tokio::test] + async fn a_client_that_learned_no_roster_dials_only_its_configured_endpoint() { + let client = client_with("127.0.0.1:8090"); + + assert_eq!( + client.dial_candidates().await, + vec!["127.0.0.1:8090".to_string()] + ); + } + + // The C++/Rust e2e contract: `login -> disconnect -> op` must fail until + // the caller signs in again. Only involuntary drops keep the sign-in. + #[tokio::test] + async fn an_explicit_disconnect_forgets_the_remembered_sign_in() { + let client = client_with("127.0.0.1:8090"); + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "iggy".into()), + SESSION_USER_ID, + ) + .await; + + Client::disconnect(&client).await.expect("disconnect"); + assert!( + client.sign_in_credentials().await.is_none(), + "an explicit disconnect ends the session for good, like a logout" + ); + } + + #[tokio::test] + async fn a_transport_drop_keeps_the_remembered_sign_in() { + let client = client_with("127.0.0.1:8090"); + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "iggy".into()), + SESSION_USER_ID, + ) + .await; + + client + .disconnect_transport() + .await + .expect("transport teardown"); + assert!( + client.sign_in_credentials().await.is_some(), + "an involuntary drop is what the failover exists for; the sign-in survives it" + ); + } + + #[tokio::test] + async fn a_sign_in_makes_a_client_without_auto_login_reconnectable() { + let client = client_with("127.0.0.1:8090"); + assert!(client.sign_in_credentials().await.is_none()); + + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "iggy".into()), + SESSION_USER_ID, + ) + .await; + assert!(client.sign_in_credentials().await.is_some()); + + // An explicit logout leaves no session to restore, and a reconnect + // must not resurrect one. + client.forget_session_credentials().await; + assert!(client.sign_in_credentials().await.is_none()); + } + + // A password change for the signed-in user has to reach the remembered + // sign-in, or the next reconnect replays the old password and fails an + // unrelated request with `InvalidCredentials`. + #[tokio::test] + async fn a_password_change_for_the_signed_in_user_updates_the_remembered_sign_in() { + let client = client_with("127.0.0.1:8090"); + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "old".into()), + SESSION_USER_ID, + ) + .await; + + for user in [ + Identifier::numeric(SESSION_USER_ID).expect("numeric identifier"), + Identifier::named("iggy").expect("named identifier"), + ] { + client.refresh_session_password(&user, "new").await; + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(_, password)) => { + assert_eq!(password.expose_secret(), "new", "for user: {user}"); + } + other => panic!("expected the remembered user credentials, got {other:?}"), + } + // Put it back so the second identifier form starts from the same + // place as the first. + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "old".into()), + SESSION_USER_ID, + ) + .await; + } + } + + // `change_password` can target anyone the caller may manage, and those + // changes say nothing about the credentials this client reconnects with. + #[tokio::test] + async fn a_password_change_for_another_user_leaves_the_remembered_sign_in_alone() { + let client = client_with("127.0.0.1:8090"); + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "old".into()), + SESSION_USER_ID, + ) + .await; + + for user in [ + Identifier::numeric(SESSION_USER_ID + 1).expect("numeric identifier"), + Identifier::named("someone-else").expect("named identifier"), + ] { + client.refresh_session_password(&user, "new").await; + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(_, password)) => { + assert_eq!(password.expose_secret(), "old", "for user: {user}"); + } + other => panic!("expected the remembered user credentials, got {other:?}"), + } + } + } + + // A personal access token is not derived from any password. + #[tokio::test] + async fn a_password_change_leaves_a_remembered_personal_access_token_alone() { + let client = client_with("127.0.0.1:8090"); + client + .remember_session_credentials( + Credentials::PersonalAccessToken("token".into()), + SESSION_USER_ID, + ) + .await; + + client + .refresh_session_password( + &Identifier::numeric(SESSION_USER_ID).expect("numeric identifier"), + "new", + ) + .await; + match client.sign_in_credentials().await { + Some(Credentials::PersonalAccessToken(token)) => { + assert_eq!(token.expose_secret(), "token"); + } + other => panic!("expected the remembered token, got {other:?}"), + } + } + + // The configured credentials cannot be rewritten, so a committed password + // change for the configured user has to reach the next reconnect through + // the remembered copy, or every later drop replays the password this very + // client replaced. + #[tokio::test] + async fn a_password_change_reaches_a_configured_auto_login() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "iggy".to_string(), + "old".into(), + )), + ..TcpClientConfig::default() + })) + .expect("create the client"); + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "old".into()), + SESSION_USER_ID, + ) + .await; + + client + .refresh_session_password( + &Identifier::numeric(SESSION_USER_ID).expect("numeric identifier"), + "new", + ) + .await; + + // Every later reconnect, not just the next one: each of them signs in + // and remembers that sign-in afresh, and the configured password is + // dead for good once the change commits. + for reconnect in 0..3 { + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, password)) => { + assert_eq!(username, "iggy"); + assert_eq!( + password.expose_secret(), + "new", + "the configured password came back on reconnect {reconnect}" + ); + // What the reconnect's own login does with what it signed + // in with. + client + .remember_session_credentials( + Credentials::UsernamePassword(username, password), + SESSION_USER_ID, + ) + .await; + } + other => { + panic!("expected the configured user with the new password, got {other:?}") + } + } + } + } + + // A change for somebody else says nothing about the configured user's + // password, and a sign-in as another user does not get to replace it. + #[tokio::test] + async fn a_password_change_for_another_user_leaves_a_configured_auto_login_alone() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "configured".to_string(), + "old".into(), + )), + ..TcpClientConfig::default() + })) + .expect("create the client"); + client + .remember_session_credentials( + Credentials::UsernamePassword("signed-in".to_string(), "old".into()), + SESSION_USER_ID, + ) + .await; + + client + .refresh_session_password( + &Identifier::numeric(SESSION_USER_ID).expect("numeric identifier"), + "new", + ) + .await; + + // A sign-out drops what was remembered, so what the configured + // credentials carry is what the next connect signs in with -- and this + // change was somebody else's. + client.forget_session_credentials().await; + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, password)) => { + assert_eq!(username, "configured"); + assert_eq!(password.expose_secret(), "old"); + } + other => panic!("expected the configured credentials, got {other:?}"), + } + } + + // A change made from a session signed in as somebody else still kills the + // configured password, so the next connect must not replay it. Named rather + // than numbered, since only the signed-in user's own id is known here. + #[tokio::test] + async fn a_password_change_naming_the_configured_user_reaches_it_from_another_session() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "configured".to_string(), + "old".into(), + )), + ..TcpClientConfig::default() + })) + .expect("create the client"); + client + .remember_session_credentials( + Credentials::PersonalAccessToken("token".into()), + SESSION_USER_ID, + ) + .await; + + client + .refresh_session_password( + &Identifier::named("configured").expect("named identifier"), + "new", + ) + .await; + + client.forget_session_credentials().await; + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, password)) => { + assert_eq!(username, "configured"); + assert_eq!(password.expose_secret(), "new"); + } + other => panic!("expected the configured user with the new password, got {other:?}"), + } + } + + // A client is whoever it last signed in as: the connection re-authenticates + // from the login it captured, and a redial that replayed somebody else + // would make the outcome depend on which of the two got there first. + #[tokio::test] + async fn the_last_sign_in_outranks_the_configured_credentials() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "configured".to_string(), + "iggy".into(), + )), + ..TcpClientConfig::default() + })) + .expect("create the client"); + client + .remember_session_credentials( + Credentials::UsernamePassword("signed-in".to_string(), "iggy".into()), + SESSION_USER_ID, + ) + .await; + + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, _)) => assert_eq!(username, "signed-in"), + other => panic!("expected the sign-in that last succeeded, got {other:?}"), + } + + // A sign-out leaves no session to restore, and the configured + // credentials are what every connect of this client signs in as. + client.forget_session_credentials().await; + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, _)) => assert_eq!(username, "configured"), + other => panic!("expected the configured credentials, got {other:?}"), + } + } #[test] fn should_fail_with_a_zero_heartbeat_interval() { diff --git a/core/sdk/src/vsr.rs b/core/sdk/src/vsr.rs index 4a6bda52ec..9625b020c3 100644 --- a/core/sdk/src/vsr.rs +++ b/core/sdk/src/vsr.rs @@ -159,7 +159,7 @@ pub(crate) fn encode_request_header( /// the authority: an unmapped code is forwarded as non-replicated (the code /// rides `RequestHeader.reserved`, which that path already stamps) and the /// server answers with a proper error if it does not know it. -fn operation_for_code(code: u32) -> Operation { +pub(crate) fn operation_for_code(code: u32) -> Operation { if code == LOGOUT_USER_CODE { return Operation::Logout; } diff --git a/core/sdk/src/websocket/websocket_client.rs b/core/sdk/src/websocket/websocket_client.rs index 47f7f4ac6a..56261f6d32 100644 --- a/core/sdk/src/websocket/websocket_client.rs +++ b/core/sdk/src/websocket/websocket_client.rs @@ -225,6 +225,12 @@ impl iggy_common::VsrSessionControl for WebSocketClient { impl BinaryClient for WebSocketClient {} impl WebSocketClient { + /// Whether an `AutoLogin` is configured on this client, which makes the + /// session after any connect the configured user's rather than whoever + /// signed in by hand. + pub(crate) fn auto_login_configured(&self) -> bool { + matches!(self.config.auto_login, AutoLogin::Enabled(_)) + } /// Create a new WebSocket client with the provided configuration. pub fn create(config: Arc) -> Result { let (sender, receiver) = broadcast(1000); @@ -534,12 +540,15 @@ impl WebSocketClient { /// Returns true if redirection occurred and reconnection is needed. pub(crate) async fn handle_leader_redirection(&self) -> Result { let current_address = self.current_server_address.lock().await.clone(); + // The roster's other endpoints are dropped here: only the TCP client + // dials failover candidates so far. let leader_address = check_and_redirect_to_leader( self, ¤t_address, iggy_common::TransportProtocol::WebSocket, ) - .await?; + .await? + .redirect; if let Some(new_leader_address) = leader_address { let mut redirection_state = self.leader_redirection_state.lock().await; diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs index 7a9de7fb67..882c89f881 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs @@ -104,28 +104,31 @@ public async Task EvictedClient_WithPersonalAccessTokenAutoLogin_Should_Reconnec me.ConsumerGroupsCount.ShouldBe(0); } + /// + /// An eviction is the server's heartbeat verifier reacting to silence, not caller intent, so a client + /// that signed in by hand recovers from it exactly like one whose credentials were configured: the + /// sign-in it remembered re-establishes the session. Only an explicit sign-out or Dispose ends it. + /// [Test] - public async Task EvictedClient_WithoutAutoLogin_Should_FailFast_And_NotReconnect() + public async Task EvictedClient_WithoutAutoLogin_Should_ReestablishItsSession() { using var client = await CreateClient(TimeSpan.FromHours(1), false); await client.LoginUserAsync("iggy", "iggy"); var (streamName, _) = await JoinFreshGroup(client); - var reconnected = false; - client.SubscribeConnectionEvents(args => - { - reconnected |= args.CurrentState == ConnectionState.Connecting; - return Task.CompletedTask; - }); - await Task.Delay(IdleFor); - // A reconnect could not bring the session back, so the request surfaces the loss instead of coming back - // over an unauthenticated connection. - await Should.ThrowAsync(() => client.GetMeAsync()); - reconnected.ShouldBeFalse(); - await Should.ThrowAsync(() => - client.GetStreamByIdAsync(Identifier.String(streamName))); + // A read is replay-safe, so the eviction is absorbed: the reconnect signs in again with the + // credentials the hand-run login remembered, and the request completes over the session it + // re-established. + var stream = await client.GetStreamByIdAsync(Identifier.String(streamName)); + stream.ShouldNotBeNull(); + + // The session is a new one, though: what the server evicted stays evicted, so the group membership + // that belonged to it is gone. + var me = await client.GetMeAsync(); + me.ShouldNotBeNull(); + me.ConsumerGroupsCount.ShouldBe(0); } private Task CreateClient(TimeSpan heartbeatInterval, bool autoLogin = true) diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs index 2985703fa6..e100fcc546 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs @@ -70,6 +70,13 @@ public sealed partial class TcpMessageStream : ISessionGenerationProvider /// private const int VsrMaxLeaderRedirects = 3; + /// + /// Bound on one endpoint's dial while other endpoints are queued behind it. Neither the connect nor the + /// TLS handshake has a deadline of its own, so a node whose syns are dropped would hold the sweep for + /// the whole kernel connect timeout - minutes - while a survivor goes untried. Matches the Rust SDK. + /// + private const int FailoverDialTimeout = 2_000; + /// /// Attempts a consumer-group poll gets before it gives up and reports an empty poll: one re-sync after /// the coordinator fences a stale assignment, then one retry. @@ -401,6 +408,25 @@ private async Task RedirectAsync(CancellationToken token) return true; } + /// + /// Keeps every node the roster names as a dial candidate. Replaced wholesale rather than merged: the + /// roster is the cluster's own answer about where its nodes are, so a node it dropped stops being dialed. + /// The configured address is kept separately and outlives it. A node that does not expose the tcp + /// transport reports port 0 and is skipped, since dialing it would burn an attempt on an endpoint that + /// cannot answer. + /// + private void RememberRoster(ClusterMetadata clusterMetadata) + { + var endpoints = clusterMetadata.Nodes + .Where(node => node.Endpoints.Tcp != 0) + .Select(node => ServerAddress.HostPort(node.Ip, node.Endpoints.Tcp)) + .ToArray(); + if (endpoints.Length > 0) + { + _rosterAddresses = endpoints; + } + } + private async Task GetCurrentLeaderNodeAsync(CancellationToken token) { var leaderlessDeadline = Environment.TickCount64 + VsrLeaderlessWaitMs; @@ -414,6 +440,8 @@ private async Task RedirectAsync(CancellationToken token) return null; } + RememberRoster(clusterMetadata); + if (clusterMetadata.Nodes.Count() == 1) { return null; @@ -541,6 +569,9 @@ private async Task> SendRawAsync(int code, ReadOnlyMemory Volatile.Read(ref _isConnecting) != 0; private ConnectionState State => (ConnectionState)Volatile.Read(ref _stateValue); @@ -116,6 +128,10 @@ public void Dispose() _connection?.Dispose(); _connection = null; + // Nothing can reconnect after this, and the remembered sign-in holds a plain-string password + // or token. + _rememberedLogin = null; + SetConnectionState(ConnectionState.Disconnected); _connectionEvents.Clear(); } @@ -589,7 +605,7 @@ public Task ConnectAsync(CancellationToken token = default) if (_configuration.ReconnectionSettings.Enabled && !_configuration.AutoLoginSettings.Enabled) { _logger.LogWarning( - "Reconnection is enabled without auto login: a lost session cannot be restored, requests will fail until the client logs in again"); + "Reconnection is enabled without auto login: a lost session can only be restored once the client has signed in at least once"); } return ConnectAsync(true, token); @@ -710,8 +726,16 @@ public async Task ChangePasswordAsync(Identifier userId, string currentPassword, throw new NotConnectedException(); } - return await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_CODE, + var identity = await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_CODE, LoginRegister.Serialize(userName, password), token); + _rememberedLogin = new AutoLoginSettings + { + Enabled = true, + Username = userName, + Password = password + }; + + return identity; } /// @@ -725,6 +749,11 @@ public async Task LogoutUserAsync(CancellationToken token = default) { await ResetConsensusSessionAsync(); + // An explicit sign-out leaves no session to restore, so the sign-in this client remembered + // does not outlive it. Credentials configured as AutoLoginSettings are a different promise - + // they are what every connect signs in with - and a reconnect still uses them. + _rememberedLogin = null; + if (State == ConnectionState.Authenticated) { SetConnectionState(ConnectionState.Connected); @@ -774,8 +803,11 @@ public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken /// public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) { - return await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE, + var identity = await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE, LoginRegister.SerializeWithPersonalAccessToken(token), ct); + _rememberedLogin = new AutoLoginSettings { Enabled = true, PersonalAccessToken = token }; + + return identity; } /// @@ -806,7 +838,10 @@ or ConnectionState.Authenticating return; } - if (_lastConnectionTime != DateTimeOffset.MinValue) + // The initial delay paces reconnects to the one endpoint a single-address client has. With other + // endpoints known there is somewhere else to go, and pausing first only pushes the failover past the + // window the caller is willing to wait; the dial loop's own delay still paces the retries. + if (_lastConnectionTime != DateTimeOffset.MinValue && DialCandidates().Length == 1) { await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token); } @@ -869,7 +904,7 @@ private async Task RunHeartbeatAsync(TimeSpan interval, CancellationToken token) // the ping is what brings an idle client back. var unrecoverable = State is ConnectionState.Disconnected or ConnectionState.Connecting && !(_configuration.ReconnectionSettings.Enabled - && _configuration.AutoLoginSettings.Enabled); + && SignInSettings() != null); if (IsConnecting || unrecoverable) { continue; @@ -999,22 +1034,26 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken var retryCount = 0; var redirects = 0; var delay = _configuration.ReconnectionSettings.InitialDelay; + Exception? configurationFault = null; + + if (string.IsNullOrEmpty(_currentAddress)) + { + _currentAddress = _configuration.BaseAddress; + } + + var candidates = DialCandidates(); + var candidate = 0; do { await DropConnectionAsync(); - if (string.IsNullOrEmpty(_currentAddress)) - { - _currentAddress = _configuration.BaseAddress; - } - if (!ServerAddress.TryParse(_currentAddress, out var host, out var port)) { throw new InvalidBaseAddressException(); } Socket? socket = null; - var dialed = false; + var established = false; try { socket = new Socket(ServerAddress.AddressFamilyOf(host), SocketType.Stream, ProtocolType.Tcp); @@ -1026,8 +1065,17 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken // trailing segment of a large request until the previous one is acked. socket.NoDelay = true; - await socket.ConnectAsync(host, port, token); - dialed = true; + // Neither ConnectAsync nor the TLS handshake has a deadline of its own, and nothing up the + // stack adds one: a node whose syns are dropped - or one that accepts TCP and then never + // answers the ClientHello - would hold the sweep, and the connection semaphore with it, for + // the whole kernel connect timeout while a survivor goes untried. + using var dialCancellation = candidates.Length > 1 + ? CancellationTokenSource.CreateLinkedTokenSource(token) + : null; + dialCancellation?.CancelAfter(FailoverDialTimeout); + var dialToken = dialCancellation?.Token ?? token; + + await socket.ConnectAsync(host, port, dialToken); _currentRemoteAddress = socket.RemoteEndPoint is IPEndPoint remote ? ServerAddress.HostPort(remote.Address.ToString(), (ushort)remote.Port) @@ -1038,9 +1086,14 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5); var connectionStream = _configuration.TlsSettings.Enabled - ? await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings) + ? await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings, dialToken) : new NetworkStream(socket, true); + // Established, not merely dialed: everything up to here belongs to this endpoint and the + // sweep may try the next one, while everything past it - auto login, a redirect, the leader + // lookup - fails the same way wherever the client lands. + established = true; + await _sendingSemaphore.WaitAsync(token); try { @@ -1061,9 +1114,9 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken // No pre-login roster read: the server auth-gates cluster metadata, so leadership settles after // a sign-in binds a session. A login dialed at a backup still succeeds because the server // forwards the register to the primary. - if (autoLogin && _configuration.AutoLoginSettings.Enabled) + if (autoLogin && SignInSettings() is { } signInSettings) { - await AutoLoginAsync(token); + await AutoLoginAsync(signInSettings, token); if (await RedirectAsync(token)) { @@ -1074,10 +1127,15 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken break; } - // Only a failed dial is worth another attempt. Everything past it - a rejected certificate, bad - // credentials, a leader that cannot be found - fails the same way every time, and with unlimited + // Only bringing an endpoint up is worth trying elsewhere. Everything past it - bad credentials, a + // leader that cannot be found - fails the same way wherever the client lands, and with unlimited // retries a caller would otherwise never get the error back. - catch (Exception e) when (dialed || e is OperationCanceledException || _disposed) + // + // A handshake the dial bound cut short is a failed attempt on this endpoint like any other, so it + // must not land here: only a cancellation the caller actually asked for is fatal. + catch (Exception e) when (established + || (e is OperationCanceledException && token.IsCancellationRequested) + || _disposed) { socket?.Dispose(); @@ -1095,6 +1153,37 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken _logger.LogError(e, "Failed to connect"); + if (IsTlsConfigurationFault(e)) + { + // A fault no retry can fix, kept aside rather than thrown at once: it belongs to the + // endpoint that raised it - a CA file that cannot be read - and the endpoints behind that + // one may be perfectly usable. + configurationFault = e; + } + + // Every other endpoint gets its turn before the retry delay: the node just lost may be gone for + // good, and pausing on it helps nothing. + if (++candidate < candidates.Length) + { + _currentAddress = candidates[candidate]; + continue; + } + + candidate = 0; + _currentAddress = candidates[0]; + + // No endpoint answered and at least one said why in a way no retry changes: an unreadable CA + // file. The caller gets that reason instead of a retry loop that buries it - unlimited retries + // would otherwise redial it forever. + if (configurationFault is not null) + { + SetConnectionState(ConnectionState.Disconnected); + throw configurationFault; + } + + // The sweep is what the reconnection budget applies to, not a single dial: checked per dial, + // the last round would try only the endpoint the client started on, and a client with + // reconnection turned off would never reach its other endpoints at all. if (!_configuration.ReconnectionSettings.Enabled || (_configuration.ReconnectionSettings.MaxRetries > 0 && retryCount >= _configuration.ReconnectionSettings.MaxRetries)) @@ -1137,6 +1226,10 @@ async Task BackoffOrThrowAsync() _logger.LogInformation("Following leader redirect {Redirect} to {Address}", redirects, _currentAddress); + // The redirect moved the client, so the endpoint it moved to leads the next dial. + candidates = DialCandidates(); + candidate = 0; + await Task.Delay(delay, token); } } @@ -1163,21 +1256,66 @@ private async Task DropConnectionAsync() } } - private async Task AutoLoginAsync(CancellationToken token) + private string[] DialCandidates() + { + return DialCandidates(_currentAddress, _configuration.BaseAddress, _rosterAddresses); + } + + /// + /// The endpoints one connect dials, likeliest first: where the client currently is, the address it was + /// configured with, then the roster it learned while connected. Duplicates are dropped, so an endpoint the + /// roster merely spells differently does not earn a second attempt. + /// + internal static string[] DialCandidates(string currentAddress, string baseAddress, string[] rosterAddresses) + { + var candidates = new List(); + if (!string.IsNullOrEmpty(currentAddress)) + { + candidates.Add(currentAddress); + } + + foreach (var endpoint in rosterAddresses.Prepend(baseAddress)) + { + if (!string.IsNullOrEmpty(endpoint) && + !candidates.Exists(known => ServerAddress.IsSame(known, endpoint))) + { + candidates.Add(endpoint); + } + } + + return candidates.ToArray(); + } + + private async Task AutoLoginAsync(AutoLoginSettings settings, CancellationToken token) { - var settings = _configuration.AutoLoginSettings; if (!string.IsNullOrEmpty(settings.PersonalAccessToken)) { - _logger.LogInformation("Auto login enabled. Trying to login with a personal access token"); + _logger.LogInformation("Signing in with a personal access token"); await LoginWithPersonalAccessTokenAsync(settings.PersonalAccessToken, token); return; } - _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}", settings.Username); + _logger.LogInformation("Signing in with credentials: {Username}", settings.Username); await LoginUserAsync(settings.Username, settings.Password, token); } - private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings) + /// + /// The credentials a connect signs in with: the configured ones, or else the ones a sign-in on this client + /// succeeded with. Null when nothing has ever signed in, which is when a reconnect cannot restore a + /// session at all. + /// + private AutoLoginSettings? SignInSettings() + { + if (_configuration.AutoLoginSettings.Enabled) + { + return _configuration.AutoLoginSettings; + } + + return _rememberedLogin; + } + + private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings, + CancellationToken token) { ValidateCertificatePath(tlsSettings.CertificatePath); @@ -1185,12 +1323,38 @@ private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSett _customCaStore.ImportFromPemFile(tlsSettings.CertificatePath); var stream = new NetworkStream(socket, true); var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback); - - await sslStream.AuthenticateAsClientAsync(tlsSettings.Hostname); + try + { + // The token carries the dial bound when other endpoints are queued behind this one: a peer that + // accepts TCP and never answers the ClientHello has no deadline of its own here either. + await sslStream.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions { TargetHost = tlsSettings.Hostname }, token); + } + catch + { + // A handshake that failed leaves the stream owning the socket, and the sweep moves on to the + // next endpoint: undisposed, both leak for as long as the client lives. + await sslStream.DisposeAsync(); + throw; + } return sslStream; } + /// + /// Whether bringing an endpoint up failed for a reason that says this client's own TLS configuration is + /// wrong: a CA file that cannot be read. It does not change on a retry, so the sweep reports it instead + /// of redialing forever. + /// + private static bool IsTlsConfigurationFault(Exception e) + { + // A handshake verdict describes the peer, not this client: a certificate that names another host, a + // peer that answers a ClientHello with something else. The endpoints behind it may be fine, and with + // one roster entry per node the target host is a bare address no certificate has to cover, so a failed + // handshake ends the dial rather than the connect. + return e is InvalidCertificatePathException; + } + private async Task SendAckAsync(int code, ReadOnlyMemory body, CancellationToken token) { using IMemoryOwner _ = await SendWithResponseAsync(code, body, token: token); @@ -1206,6 +1370,10 @@ private async Task> SendWithResponseAsync(int code, ReadOnlyM catch (Exception e) when (IsLostConnection(e) && !IsConnecting && !_disposed) { _logger.LogWarning("Connection lost"); + + // A stale-client eviction is not caller intent: the server's heartbeat verifier sends it after a + // gc pause or a laptop sleep, so the remembered sign-in survives it and the reconnect below + // re-establishes the session. Only an explicit sign-out or Dispose ends it. Same rule in every SDK. if (!_configuration.ReconnectionSettings.Enabled) { _logger.LogWarning("Reconnection is disabled"); @@ -1213,11 +1381,12 @@ private async Task> SendWithResponseAsync(int code, ReadOnlyM throw; } - // Without auto login a reconnect cannot re-establish the session, so the request would only come - // back unauthenticated. Login and register are the exception: they re-authenticate themselves. - if (!_configuration.AutoLoginSettings.Enabled && autoLoginOnReconnect) + // With no credentials - neither configured nor remembered from a sign-in - a reconnect cannot + // re-establish the session, so the request would only come back unauthenticated. Login and register + // are the exception: they re-authenticate themselves. + if (SignInSettings() == null && autoLoginOnReconnect) { - _logger.LogWarning("Auto login is disabled, the session cannot be re-established"); + _logger.LogWarning("No credentials to sign in with, the session cannot be re-established"); SetConnectionState(ConnectionState.Disconnected); throw; } diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/DialCandidatesTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/DialCandidatesTests.cs new file mode 100644 index 0000000000..20e0e949bc --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/DialCandidatesTests.cs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.IggyClient.Implementations; + +namespace Apache.Iggy.Tests.VsrTests; + +/// +/// Mirrors the Rust SDK's dial_candidates: a client that loses the node it is on has to dial the rest +/// of the cluster, and the two SDKs have to agree on which endpoints those are and in what order. +/// +public sealed class DialCandidatesTests +{ + [Fact] + public void LeadsWithTheCurrentEndpointThenNamesEachOtherOneOnce() + { + var candidates = TcpMessageStream.DialCandidates( + "127.0.0.1:8090", + "localhost:8090", + ["127.0.0.1:8090", "127.0.0.1:8091", "127.0.0.1:8092"]); + + // Neither the roster's copy of the current endpoint nor a configured address that only spells the same + // endpoint differently earns a second dial. + Assert.Equal(["127.0.0.1:8090", "127.0.0.1:8091", "127.0.0.1:8092"], candidates); + } + + /// + /// The configured address comes before the roster: it is the one endpoint the caller vouched for, and a + /// roster learned from a cluster that has since changed shape may name nodes that are gone. + /// + [Fact] + public void DialsTheConfiguredAddressBeforeTheLearnedRoster() + { + var candidates = TcpMessageStream.DialCandidates( + "127.0.0.1:8090", + "127.0.0.1:8099", + ["127.0.0.1:8091", "127.0.0.1:8092"]); + + Assert.Equal(["127.0.0.1:8090", "127.0.0.1:8099", "127.0.0.1:8091", "127.0.0.1:8092"], candidates); + } + + [Fact] + public void KeepsTheConfiguredAddressWhenNoRosterWasLearned() + { + var candidates = TcpMessageStream.DialCandidates("127.0.0.1:8091", "127.0.0.1:8090", []); + + Assert.Equal(["127.0.0.1:8091", "127.0.0.1:8090"], candidates); + } + + [Fact] + public void FallsBackToTheConfiguredAddressBeforeTheFirstConnect() + { + var candidates = TcpMessageStream.DialCandidates(string.Empty, "127.0.0.1:8090", []); + + Assert.Equal(["127.0.0.1:8090"], candidates); + } + + [Fact] + public void DialsOneEndpointWhenNothingElseIsKnown() + { + var candidates = TcpMessageStream.DialCandidates("127.0.0.1:8090", "127.0.0.1:8090", []); + + Assert.Equal(["127.0.0.1:8090"], candidates); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs new file mode 100644 index 0000000000..e0e0203976 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs @@ -0,0 +1,576 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using System.Net; +using System.Net.Sockets; +using System.Text; +using Apache.Iggy.Configuration; +using Apache.Iggy.Contracts.Tcp; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.IggyClient; +using Apache.Iggy.IggyClient.Implementations; +using Apache.Iggy.Vsr; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Apache.Iggy.Tests.VsrTests; + +/// +/// The node a client signed in on dies; its next request has to complete on a survivor the roster named, +/// under a session established there. Mirrors +/// core/integration/tests/cluster/failover_client_continuity.rs. +/// +public sealed class EndpointFailoverTests +{ + private const int HeaderSize = 256; + private const int SizeOffset = 48; + private const int CommandOffset = 60; + private const int RequestIdOffset = 168; + private const int RequestOperationOffset = 176; + private const int RequestReservedOffset = 196; + private const int ReplyRequestIdOffset = 200; + private const int ReplyOperationOffset = 208; + private const int ReplyStatusOffset = 216; + + private const byte CommandReply = 8; + private const byte CommandEviction = 13; + private const int EvictionReasonOffset = 255; + private const byte EvictionStaleClient = 13; + private const byte OperationRegister = 1; + private const byte OperationNonReplicated = 2; + private const int GetClusterMetadataCode = 12; + private const int PingCode = 1; + + [Fact] + public async Task ResumesOnASurvivorAfterTheSignedInNodeDies() + { + using var primary = new MockNode(); + using var survivor = new MockNode(); + + // The primary leads, so the sign-in settles there and the roster is only remembered - not acted on - + // until the node dies. + primary.Serve(request => request.Code == GetClusterMetadataCode + ? Reply(OperationNonReplicated, ClusterMetadata(primary.Port, survivor.Port, primary.Port)) + : Answer(request)); + survivor.Serve(request => request.Code == GetClusterMetadataCode + ? Reply(OperationNonReplicated, ClusterMetadata(primary.Port, survivor.Port, survivor.Port)) + : Answer(request)); + + var configuration = new IggyClientConfigurator + { + BaseAddress = $"127.0.0.1:{primary.Port}", + Protocol = Protocol.Tcp, + ReconnectionSettings = new ReconnectionSettings + { + Enabled = true, + MaxRetries = 4, + InitialDelay = TimeSpan.FromMilliseconds(20) + } + }; + using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); + + await client.ConnectAsync(TestContext.Current.CancellationToken); + // No auto login: the credentials come from the caller's own sign-in, which is the shape that could not + // reconnect at all before. + await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); + await client.PingAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, primary.Pings); + + primary.Kill(); + + // The request in flight when the node died is allowed to fail; what is not allowed is never completing + // one, which is what a client that only knows the dead endpoint does. + var (resumed, lastError) = await ResumedWithin(client, TimeSpan.FromSeconds(10)); + Assert.True(resumed, + $"the client has to resume on the survivor the roster named ({lastError}, survivor saw " + + $"{survivor.Registrations} registrations and {survivor.Pings} pings)"); + Assert.True(survivor.Registrations >= 1, "the remembered credentials signed in again on the survivor"); + Assert.True(survivor.Pings >= 1, "the request landed on the survivor"); + } + + /// + /// Mirrors the integration contract (HeartbeatTests + /// EvictedClient_WithoutAutoLogin_Should_ReestablishItsSession): an eviction comes off the server's + /// heartbeat timer rather than from the caller, so the sign-in this client remembered survives it and + /// the reconnect re-establishes the session. Same rule in every SDK. + /// + [Fact] + public async Task ServerEvictionReplaysTheRememberedSignIn() + { + using var node = new MockNode(); + var evict = false; + node.Serve(request => + { + if (request.Operation == OperationRegister) + { + return Reply(OperationRegister, RegisterBody(session: 128)); + } + + if (evict) + { + evict = false; + return EvictionFrame(EvictionStaleClient); + } + + return Reply(OperationNonReplicated, request.Code == GetClusterMetadataCode + ? ClusterMetadata(node.Port, node.Port, node.Port) + : []); + }); + + var configuration = new IggyClientConfigurator + { + BaseAddress = $"127.0.0.1:{node.Port}", + Protocol = Protocol.Tcp, + ReconnectionSettings = new ReconnectionSettings + { + Enabled = true, + MaxRetries = 2, + InitialDelay = TimeSpan.FromMilliseconds(20) + } + }; + using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); + + await client.ConnectAsync(TestContext.Current.CancellationToken); + await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); + await client.PingAsync(TestContext.Current.CancellationToken); + var registrationsBeforeEviction = node.Registrations; + + // A ping is replay-safe, so the eviction is absorbed: the reconnect signs in again with the + // remembered credentials and the request completes over the session it re-established. + evict = true; + await client.PingAsync(TestContext.Current.CancellationToken); + + Assert.True(node.Registrations > registrationsBeforeEviction, + "the reconnect signed in again with the remembered credentials"); + await client.PingAsync(TestContext.Current.CancellationToken); + } + + /// + /// The same rule when the eviction lands on a replicated write: that request is reported as + /// outcome-unknown, because its own outcome is unknown, but the session behind it is still + /// re-established for the requests that follow. + /// + [Fact] + public async Task ServerEvictionDuringAReplicatedWriteReplaysTheRememberedSignIn() + { + using var node = new MockNode(); + var evict = false; + node.Serve(request => + { + if (request.Operation == OperationRegister) + { + return Reply(OperationRegister, RegisterBody(session: 128)); + } + + if (evict) + { + evict = false; + return EvictionFrame(EvictionStaleClient); + } + + return Reply(OperationNonReplicated, request.Code == GetClusterMetadataCode + ? ClusterMetadata(node.Port, node.Port, node.Port) + : []); + }); + + var configuration = new IggyClientConfigurator + { + BaseAddress = $"127.0.0.1:{node.Port}", + Protocol = Protocol.Tcp, + ReconnectionSettings = new ReconnectionSettings + { + Enabled = true, + MaxRetries = 2, + InitialDelay = TimeSpan.FromMilliseconds(20) + } + }; + using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); + + await client.ConnectAsync(TestContext.Current.CancellationToken); + await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); + await client.PingAsync(TestContext.Current.CancellationToken); + var registrationsBeforeEviction = node.Registrations; + + evict = true; + await Assert.ThrowsAsync(() => + client.CreateStreamAsync("evicted-mid-write", token: TestContext.Current.CancellationToken)); + + await client.PingAsync(TestContext.Current.CancellationToken); + Assert.True(node.Registrations > registrationsBeforeEviction, + "the reconnect signed in again with the remembered credentials"); + } + + /// + /// A survivor that is not listening yet when its node dies still has to be found: the client keeps + /// rotating over every endpoint it knows, so one that comes up while it is retrying is dialed on a + /// later pass rather than only on the first. + /// + /// The retry budget counts rotations, not dials: a single retry buys a whole second pass over + /// both endpoints. Spent per dial, the budget would be gone before the survivor came up. + /// + /// + [Fact] + public async Task ResumesOnASurvivorThatComesUpWhileTheClientIsRetrying() + { + // A port nothing listens on: the survivor binds it only after the client has already failed on both + // endpoints, so the first pass cannot be the one that finds it. + var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + var survivorPort = (ushort)((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); + + using var primary = new MockNode(); + primary.Serve(request => request.Code == GetClusterMetadataCode + ? Reply(OperationNonReplicated, ClusterMetadata(primary.Port, survivorPort, primary.Port)) + : Answer(request)); + + var configuration = new IggyClientConfigurator + { + BaseAddress = $"127.0.0.1:{primary.Port}", + Protocol = Protocol.Tcp, + AutoLoginSettings = new AutoLoginSettings { Enabled = true, Username = "iggy", Password = "iggy" }, + ReconnectionSettings = new ReconnectionSettings + { + Enabled = true, + // One retry, so the pass that finds the survivor is the one the budget pays for. A larger + // budget would find it whether the budget counts rotations or dials. + MaxRetries = 1, + InitialDelay = TimeSpan.FromMilliseconds(200) + } + }; + using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); + + await client.ConnectAsync(TestContext.Current.CancellationToken); + await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); + await client.PingAsync(TestContext.Current.CancellationToken); + + primary.Kill(); + MockNode? survivor = null; + // Constructed inside the delay, because the listener starts in the constructor: built up front, the + // survivor would be answering from the very first dial and nothing about the later passes would be + // exercised. + var comesUp = Task.Run(async () => + { + await Task.Delay(300, TestContext.Current.CancellationToken); + survivor = new MockNode(survivorPort); + survivor.Serve(request => request.Code == GetClusterMetadataCode + ? Reply(OperationNonReplicated, ClusterMetadata(primary.Port, survivorPort, survivorPort)) + : Answer(request)); + }, TestContext.Current.CancellationToken); + + try + { + await client.PingAsync(TestContext.Current.CancellationToken); + await comesUp; + + Assert.NotNull(survivor); + Assert.True(survivor!.Registrations >= 1, "the session was re-established on the survivor"); + } + finally + { + await comesUp; + survivor?.Dispose(); + } + } + + private static byte[] EvictionFrame(byte reason) + { + var frame = new byte[HeaderSize]; + BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(SizeOffset, 4), HeaderSize); + frame[CommandOffset] = CommandEviction; + frame[EvictionReasonOffset] = reason; + return frame; + } + + [Fact] + public async Task FailsFastWhenNothingEverSignedIn() + { + using var node = new MockNode(); + node.Serve(request => request.Code == GetClusterMetadataCode + ? Reply(OperationNonReplicated, ClusterMetadata(node.Port, node.Port, node.Port)) + : Answer(request)); + + var configuration = new IggyClientConfigurator + { + BaseAddress = $"127.0.0.1:{node.Port}", + Protocol = Protocol.Tcp, + ReconnectionSettings = new ReconnectionSettings + { + Enabled = true, + MaxRetries = 2, + InitialDelay = TimeSpan.FromMilliseconds(20) + } + }; + using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); + + await client.ConnectAsync(TestContext.Current.CancellationToken); + await client.PingAsync(TestContext.Current.CancellationToken); + + // A reconnect announces itself by entering Connecting, so the absence of that transition is the + // assertion - no need to poll for a request that must never succeed. + var reconnected = false; + client.SubscribeConnectionEvents(args => + { + reconnected |= args.CurrentState == ConnectionState.Connecting; + return Task.CompletedTask; + }); + + node.Kill(); + + await Assert.ThrowsAnyAsync(() => client.PingAsync(TestContext.Current.CancellationToken)); + Assert.False(reconnected, "a client that never signed in cannot restore a session by reconnecting"); + } + + private static async Task<(bool Resumed, string LastError)> ResumedWithin(TcpMessageStream client, + TimeSpan budget) + { + var deadline = DateTimeOffset.UtcNow + budget; + var lastError = "none"; + var attempts = 0; + while (DateTimeOffset.UtcNow < deadline) + { + attempts++; + try + { + await client.PingAsync(TestContext.Current.CancellationToken); + + return (true, lastError); + } + catch (Exception error) + { + lastError = $"{attempts} attempts, last: {error.GetType().Name}: {error.Message}"; + await Task.Delay(50, TestContext.Current.CancellationToken); + } + } + + return (false, lastError); + } + + /// A reply for anything the roster read does not claim: a register, or an empty read. + private static byte[] Answer(MockRequest request) + { + return request.Operation == OperationRegister + ? Reply(OperationRegister, RegisterBody(session: 128)) + : Reply(OperationNonReplicated, []); + } + + private static byte[] Reply(byte operation, byte[] body) + { + var frame = new byte[HeaderSize + body.Length]; + BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(SizeOffset, 4), (uint)frame.Length); + frame[CommandOffset] = CommandReply; + frame[ReplyOperationOffset] = operation; + BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(ReplyStatusOffset, 4), 0); + body.CopyTo(frame.AsSpan(HeaderSize)); + + return frame; + } + + /// + /// A register reply carries a committed result section, so its four leading zero bytes announce zero + /// entries and the typed payload starts right after them. A non-replicated read carries none. + /// + private static byte[] RegisterBody(ulong session) + { + var serverVersion = Encoding.UTF8.GetBytes("0.0.0"); + var body = new byte[4 + 17 + serverVersion.Length]; + var payload = body.AsSpan(4); + BinaryPrimitives.WriteUInt32LittleEndian(payload[..4], 7); + BinaryPrimitives.WriteUInt64LittleEndian(payload[4..12], session); + BinaryPrimitives.WriteUInt32LittleEndian(payload[12..16], 11 << 10); + payload[16] = (byte)serverVersion.Length; + serverVersion.CopyTo(payload[17..]); + + return body; + } + + private static byte[] ClusterMetadata(ushort primaryPort, ushort survivorPort, ushort leaderPort) + { + var body = new List(); + WriteString(body, "test-cluster"); + body.AddRange(BitConverter.GetBytes(2u)); + WriteNode(body, "primary", primaryPort, primaryPort == leaderPort); + WriteNode(body, "survivor", survivorPort, survivorPort == leaderPort); + + return body.ToArray(); + } + + private static void WriteNode(List body, string name, ushort port, bool leader) + { + WriteString(body, name); + WriteString(body, "127.0.0.1"); + body.AddRange(BitConverter.GetBytes(port)); + body.AddRange(BitConverter.GetBytes((ushort)0)); + body.AddRange(BitConverter.GetBytes((ushort)0)); + body.AddRange(BitConverter.GetBytes((ushort)0)); + body.Add(leader ? (byte)0 : (byte)1); + body.Add(0); + } + + private static void WriteString(List body, string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + body.AddRange(BitConverter.GetBytes((uint)bytes.Length)); + body.AddRange(bytes); + } + + private readonly record struct MockRequest(byte Operation, int Code, ulong RequestId); + + /// + /// A loopback VSR node. Killing it drops the live sockets and stops accepting, so a redial is refused the + /// way a dead process refuses one. + /// + private sealed class MockNode : IDisposable + { + private readonly TcpListener _listener; + private readonly List _accepted = []; + private volatile bool _killed; + private int _connections; + private int _pings; + private int _registrations; + + /// + /// A port to bind, for a node that has to come up on an address the client already knows. Zero + /// takes whatever the OS hands out. + /// + public MockNode(ushort port = 0) + { + _listener = new TcpListener(IPAddress.Loopback, port); + _listener.Start(); + Port = (ushort)((IPEndPoint)_listener.LocalEndpoint).Port; + } + + public ushort Port { get; } + + public int Pings => Volatile.Read(ref _pings); + + public int Registrations => Volatile.Read(ref _registrations); + + public int Connections + { + get + { + lock (_accepted) + { + return _connections; + } + } + } + + public void Serve(Func handler) + { + _ = Task.Run(async () => + { + while (!_killed) + { + TcpClient connection; + try + { + connection = await _listener.AcceptTcpClientAsync(); + } + catch (Exception) + { + return; + } + + lock (_accepted) + { + _accepted.Add(connection); + _connections++; + } + + _ = Task.Run(() => Exchange(connection, handler)); + } + }); + } + + public void Kill() + { + _killed = true; + lock (_accepted) + { + foreach (var connection in _accepted) + { + connection.Close(); + } + + _accepted.Clear(); + } + + _listener.Stop(); + } + + public void Dispose() + { + Kill(); + } + + private async Task Exchange(TcpClient connection, Func handler) + { + try + { + await using var stream = connection.GetStream(); + var header = new byte[HeaderSize]; + while (!_killed) + { + await ReadExactly(stream, header); + var size = BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(SizeOffset, 4)); + var body = new byte[size - HeaderSize]; + await ReadExactly(stream, body); + + var request = new MockRequest(header[RequestOperationOffset], + BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(RequestReservedOffset, 4)), + BinaryPrimitives.ReadUInt64LittleEndian(header.AsSpan(RequestIdOffset, 8))); + if (request.Operation == OperationRegister) + { + Interlocked.Increment(ref _registrations); + } + else if (request.Code == PingCode) + { + Interlocked.Increment(ref _pings); + } + + var reply = handler(request); + BinaryPrimitives.WriteUInt64LittleEndian(reply.AsSpan(ReplyRequestIdOffset, 8), + request.RequestId); + await stream.WriteAsync(reply); + await stream.FlushAsync(); + } + } + catch (Exception) + { + // A killed node and a client that went away look the same here. + } + } + + private static async Task ReadExactly(NetworkStream stream, byte[] buffer) + { + var read = 0; + while (read < buffer.Length) + { + var chunk = await stream.ReadAsync(buffer.AsMemory(read)); + if (chunk == 0) + { + throw new EndOfStreamException("Connection closed"); + } + + read += chunk; + } + } + } +} diff --git a/foreign/csharp/README.md b/foreign/csharp/README.md index 316ef3cb04..3fed133b1b 100644 --- a/foreign/csharp/README.md +++ b/foreign/csharp/README.md @@ -108,8 +108,9 @@ var client = IggyClientFactory.CreateClient(new IggyClientConfigurator BackoffMultiplier = 2.0 }, - // Auto-login after connection. Reconnection needs it: without credentials to replay a reconnect cannot - // restore the session, so a lost connection fails the request instead + // Auto-login after connection. Optional for reconnection: a client that signs in with + // LoginUserAsync has that sign-in replayed on a reconnect too. Without either, a reconnect + // cannot restore the session and a lost connection fails the request AutoLoginSettings = AutoLoginSettings.For("your_username", "your_password"), // or AutoLoginSettings.ForPersonalAccessToken("your_token") @@ -196,14 +197,20 @@ The SDK replays a request whenever the server says it never admitted it. Two cas to `WithConnection`. Before, a builder-created client came back from a reconnect unauthenticated; now the credentials are held for the lifetime of the connection and replayed. - The TCP client now pings the server every `HeartbeatInterval` (5 seconds, always on) on its own, and - reconnection is on by default (it was off before) with unlimited retries, like the Rust client. Only a - failed dial is retried; a rejected certificate, bad credentials or a missing leader is thrown right away. - A dropped connection fails every in-flight request at once; they share a single reconnect and are replayed - on the connection it establishes. With the default `MaxRetries = 0` an unreachable server is retried - forever, so a request that passes no `CancellationToken` waits for as long as the server stays down - set - `MaxRetries` or pass a token to bound it. Reconnection only replays a request when `AutoLoginSettings` can - restore the session; a client that logged in by hand fails fast on a lost connection. Set - `ReconnectionSettings.Enabled = false` to opt out of reconnection. + reconnection is on by default (it was off before) with unlimited retries, like the Rust client. Bringing an + endpoint up is what gets another attempt, and one retry is a full pass over every endpoint the client knows - + where it is, the configured address, and every node the roster named - rather than one dial of the first. + Bad credentials or a missing leader is thrown right away, and so is a TLS fault no retry can fix (an + unreadable CA file, a certificate this client will never accept), once the pass has given the other + endpoints their turn. A dropped connection fails every in-flight request at once; they share a single + reconnect and are replayed on the connection it establishes. With the default `MaxRetries = 0` an + unreachable server is retried forever, so a request that passes no `CancellationToken` waits for as long as + the server stays down - set `MaxRetries` or pass a token to bound it. A reconnect restores the session from + `AutoLoginSettings` or from the sign-in a `LoginUserAsync` call succeeded with, so a client that logged in by + hand reconnects too; without either there is nothing to restore and the request fails. A server-side + eviction (the heartbeat verifier reacting to silence) is recovered from the same way; only `LogoutUserAsync` + or `Dispose` ends a session for good. Set `ReconnectionSettings.Enabled = false` to opt out of + reconnection. - `AutoLoginSettings` properties are now `init`-only, as is `IggyClientConfigurator.HeartbeatInterval`. Build them with an object initializer or the `AutoLoginSettings.For` / `AutoLoginSettings.ForPersonalAccessToken` factories instead of assigning after construction. diff --git a/foreign/go/client/tcp/tcp_connect_test.go b/foreign/go/client/tcp/tcp_connect_test.go index 822958b5b5..12fcd615ac 100644 --- a/foreign/go/client/tcp/tcp_connect_test.go +++ b/foreign/go/client/tcp/tcp_connect_test.go @@ -495,7 +495,18 @@ func TestExchange_DoesNotPreemptAReplayedSignIn(t *testing.T) { // The connect sign-in, the dropped explicit one, and its replay. An // automatic sign-in on the new connection would make a fourth. assert.Equal(t, 3, signIns) - assert.False(t, client.skipAutoLoginOnce, "the suppression fires exactly once") + + // The suppression rides the replay's own context, so nothing about it + // outlives that call: the next Connect signs in again. + require.NoError(t, client.disconnect()) + require.NoError(t, client.Connect(context.Background())) + signIns = 0 + for _, recorded := range server.recorded() { + if recorded.operation() == vsr.OperationRegister { + signIns++ + } + } + assert.Equal(t, 4, signIns, "the suppression leaked past the call that meant it") } func TestExchange_FailsFastWhenAutoLoginIsOff(t *testing.T) { @@ -626,7 +637,7 @@ func TestConnect_ExchangesOverTLS(t *testing.T) { func TestCreateTLSConfig_ExtractsAnIPv6ServerName(t *testing.T) { client := NewIggyTcpClient(nil, WithServerAddress("[::1]:8090"), WithTLS()) - config, err := client.createTLSConfig() + config, err := client.createTLSConfig("[::1]:8090") require.NoError(t, err) assert.Equal(t, "::1", config.ServerName) } diff --git a/foreign/go/client/tcp/tcp_core.go b/foreign/go/client/tcp/tcp_core.go index fe662af53c..057907d61b 100644 --- a/foreign/go/client/tcp/tcp_core.go +++ b/foreign/go/client/tcp/tcp_core.go @@ -52,6 +52,21 @@ func GetDefaultOptions() Options { } } +// connectAttempt is one run of Connect, shared with the callers waiting on it. +// +// The outcome is kept per attempt rather than in a field on the client: a +// waiter that read a shared field would read whatever the attempt after the one +// it waited on had written there, and a fresh attempt has no outcome yet. +type connectAttempt struct { + // done is closed once the attempt settles, whichever way it ends. + done chan struct{} + // err is the attempt's outcome, written before done is closed. + err error + // suppressesLogin records that the owner does not sign in, which is what + // makes the attempt safe to wait on from inside the sign-in transaction. + suppressesLogin bool +} + type IggyTcpClient struct { conn net.Conn // reader buffers reads off conn, so a reply costs one syscall instead of @@ -79,12 +94,20 @@ type IggyTcpClient struct { // session carries the consensus client identity and request watermark; // guarded by c.mtx. session *vsr.Session - // skipAutoLoginOnce suppresses the next automatic sign-in so a replayed - // login is not preempted by one the reconnect issues; guarded by c.mtx. - skipAutoLoginOnce bool // loggedOut records an explicit sign-out, so a reconnect's automatic // sign-in does not silently reverse it; guarded by c.mtx. loggedOut bool + // connectAttempt is the attempt a Connect is running, shared with every + // caller that arrives while it is in progress. Guarded by c.mtx. + connectAttempt *connectAttempt + // rememberedLogin holds the credentials a manual sign-in succeeded with, + // so a reconnect -- on this node or, after a failover, another one -- can + // re-establish the session instead of surfacing an unauthenticated error. + // A caller that signs in by hand is otherwise less reconnectable than one + // that configures auto-login, which is a surprising difference between + // two ways of doing the same thing. Cleared on sign-out; guarded by + // c.mtx. + rememberedLogin AutoLogin // groups caches the consumer-group assignments this client polls with. groups groupAssignmentCache // topics caches what a send needs to resolve a partition locally. @@ -294,6 +317,11 @@ const ( // A node that stopped being primary answers transient forever, so // replaying alone never recovers. failoverCheckInterval = 2 * time.Second + // failoverDialTimeout bounds one endpoint's dial and handshake while other + // endpoints are queued behind it. Neither step has a deadline of its own, + // so a node whose syns are dropped would hold the sweep for the whole + // kernel connect timeout while a survivor goes untried. + failoverDialTimeout = 2 * time.Second ) // requestBufPool reuses wire-payload buffers across RPCs. A fresh buffer @@ -453,6 +481,19 @@ func appendCommandFrame(buf []byte, cmd command.Command) ([]byte, error) { // deadlock on that lock or recurse Connect without a bound. type connectScoped struct{} +// skipAutoLogin marks the context of a Connect whose caller owns the sign-in: +// a replayed login, or a redirect inside the sign-in transaction. Carried on +// the context rather than on the client, so it cannot outlive the call that +// meant it -- a client-wide flag leaks when Connect returns early on the +// already-connected gate, and then suppresses somebody else's auto-login. +type skipAutoLogin struct{} + +// suppressAutoLogin returns ctx marked so the Connect it drives does not sign +// in by itself. +func suppressAutoLogin(ctx context.Context) context.Context { + return context.WithValue(ctx, skipAutoLogin{}, struct{}{}) +} + // localPreconditionError marks a request that failed before its frame was // written. The connection is healthy, so exchange must not tear it down and // re-dial over what is purely local state. @@ -468,6 +509,11 @@ func (c *IggyTcpClient) exchange(ctx context.Context, code uint32, frame []byte) if err == nil || !isReconnectable(err) { return response, err } + + // A stale-client eviction is not caller intent: the heartbeat verifier + // sends it after a gc pause or a laptop sleep, so the remembered sign-in + // survives it and the reconnect re-establishes the session. Only an + // explicit sign-out ends it. Same rule in every SDK. var precondition *localPreconditionError if errors.As(err, &precondition) { return nil, err @@ -480,12 +526,13 @@ func (c *IggyTcpClient) exchange(ctx context.Context, code uint32, frame []byte) return nil, err } - // Without auto-login a reconnect cannot restore the session, so anything - // but a sign-in fails here instead of replaying unauthenticated. The - // sign-in itself is the exception: the server stays silent on a transient + // With no credentials -- neither configured nor remembered from a + // sign-in -- a reconnect cannot restore the session, so anything but a + // sign-in fails here instead of replaying unauthenticated. The sign-in + // itself is the exception: the server stays silent on a transient // register failure and expects the client to replay it. login := isRegisterCode(code) - if !c.config.autoLogin.enabled && !login { + if _, ok := c.signInCredentials(); !ok && !login { return nil, err } c.mtx.Lock() @@ -505,22 +552,20 @@ func (c *IggyTcpClient) exchange(ctx context.Context, code uint32, frame []byte) if disconnectErr := c.disconnect(); disconnectErr != nil { return nil, disconnectErr } + reconnectCtx := ctx if login { - c.mtx.Lock() - c.skipAutoLoginOnce = true - c.mtx.Unlock() + // The caller replays the login itself, so the reconnect must not. + reconnectCtx = suppressAutoLogin(ctx) } + c.mtx.Lock() + serverAddress := c.currentServerAddress + c.mtx.Unlock() c.logger.Info("Reconnecting to the server...", - slog.String("server_address", c.currentServerAddress), + slog.String("server_address", serverAddress), slog.Any("error", err)) - if reconnectErr := c.Connect(ctx); reconnectErr != nil { - if login { - c.mtx.Lock() - c.skipAutoLoginOnce = false - c.mtx.Unlock() - } + if reconnectErr := c.Connect(reconnectCtx); reconnectErr != nil { return nil, reconnectErr } return c.sendFrame(ctx, code, frame) @@ -616,7 +661,15 @@ func (c *IggyTcpClient) sendFrame(ctx context.Context, code uint32, frame []byte return nil, redirectErr } if redirect { - if connectErr := c.Connect(ctx); connectErr != nil { + redirectCtx := ctx + if ctx.Value(connectScoped{}) != nil { + // Issued from inside the sign-in transaction, which holds + // registerMtx: the automatic sign-in on the reconnect path + // would wait on that lock forever. The transaction signs in + // itself on the node it lands on, so the reconnect must not. + redirectCtx = suppressAutoLogin(ctx) + } + if connectErr := c.Connect(redirectCtx); connectErr != nil { return nil, connectErr } stamped = false @@ -868,7 +921,13 @@ func (c *IggyTcpClient) GetConnectionInfo() *iggcon.ConnectionInfo { } // Connect establishes the TCP connection to the server. -func (c *IggyTcpClient) Connect(ctx context.Context) error { +// +// Single-flighted: one attempt dials, and every caller that arrives while it +// runs waits for it and shares its outcome. Reporting success to those callers +// instead would hand them a client with no connection yet, and their next +// request would fail ErrNotConnected for no reason of its own. +func (c *IggyTcpClient) Connect(ctx context.Context) (err error) { + suppressesLogin := ctx.Value(skipAutoLogin{}) != nil c.mtx.Lock() switch c.transportState { case iggcon.TransportStateShutdown: @@ -881,27 +940,73 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { c.logger.Debug("Client is already connected.", slog.String("client_address", clientAddress)) return nil case iggcon.TransportStateConnecting: + attempt := c.connectAttempt c.mtx.Unlock() - c.logger.Debug("Client is already connecting.") - return nil - default: - c.transportState = iggcon.TransportStateConnecting + if attempt == nil { + return nil + } + if suppressesLogin && !attempt.suppressesLogin { + // Only the sign-in transaction suppresses the automatic sign-in, + // and it holds registerMtx while it does. The attempt in flight + // ends in a sign-in that needs that same lock, so waiting here + // would close a cycle: the owner blocked on registerMtx, this + // goroutine blocked on the owner, and neither context cancelled. + c.logger.Debug("Another connect is signing in; not waiting for it.") + return ierror.ErrCannotEstablishConnection + } + c.logger.Debug("Client is already connecting; waiting for that attempt.") + select { + case <-attempt.done: + case <-ctx.Done(): + return ctx.Err() + case <-c.closed: + return ierror.ErrClientShutdown + } + return attempt.err + } + attempt := &connectAttempt{ + done: make(chan struct{}), + suppressesLogin: suppressesLogin, } + c.transportState = iggcon.TransportStateConnecting + c.connectAttempt = attempt connectedAt := c.connectedAt c.mtx.Unlock() - // handle reestablish interval - if !connectedAt.IsZero() { - now := time.Now() - elapsed := now.Sub(connectedAt) - reestablishAfter := c.config.reconnection.reestablishAfter - - c.logger.Debug("Elapsed time since last connection", slog.Duration("elapsed", elapsed)) - if elapsed < reestablishAfter { - remaining := reestablishAfter - elapsed - c.logger.Info("Trying to connect to the server", slog.Duration("remaining", remaining)) - time.Sleep(remaining) + // Settles the attempt for whoever is waiting on it, whichever way it ends. + defer func() { + attempt.err = err + c.mtx.Lock() + if c.connectAttempt == attempt { + c.connectAttempt = nil } + c.mtx.Unlock() + close(attempt.done) + }() + + candidates := c.connectionCandidates() + if len(candidates) == 0 { + // Nowhere to dial: a client configured with an empty server address + // and no roster. Reporting success here would leave every request + // answering ErrNotConnected while Connect keeps saying it is + // connected. + c.mtx.Lock() + c.transportState = iggcon.TransportStateDisconnected + c.mtx.Unlock() + c.logger.Error("No server address to connect to.") + return ierror.ErrCannotEstablishConnection + } + + // reestablishAfter paces reconnects to the endpoint this client was last + // on, and to that one only: the other endpoints owe it no cooldown, and + // pausing before dialing them would push the failover past the window the + // caller is willing to wait. So when there is somewhere else to go, the + // paced endpoint goes last -- by which time its window has usually + // elapsed anyway -- instead of the wait being skipped outright. + pacedEndpoint := candidates[0] + if !connectedAt.IsZero() && len(candidates) > 1 && + time.Since(connectedAt) < c.config.reconnection.reestablishAfter { + candidates = append(candidates[1:], pacedEndpoint) } attempts := uint(1) interval := time.Duration(0) @@ -909,10 +1014,7 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { attempts = uint(c.config.reconnection.maxRetries) interval = c.config.reconnection.interval } - - candidates := c.connectionCandidates() var conn net.Conn - var candidateIndex int if err := retry.New( retry.Context(ctx), retry.Attempts(attempts), @@ -923,46 +1025,42 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { }), ).Do( func() error { - address := candidates[candidateIndex%len(candidates)] - candidateIndex++ - c.logger.Info("Iggy client is connecting to server...", slog.String("server_address", address)) - connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", address) - if err != nil { - c.logger.Error("Failed to establish TCP connection to the server", slog.Any("error", err)) - return ierror.ErrCannotEstablishConnection - } - - tc := connection.(*net.TCPConn) - if err := tc.SetNoDelay(c.config.noDelay); err != nil { - c.logger.Error("Failed to set the nodelay option on the client, continuing...", slog.Any("error", err)) - } - - c.mtx.Lock() - c.clientAddress = tc.LocalAddr().String() - c.currentServerAddress = address - c.mtx.Unlock() + // Every endpoint gets its turn inside one attempt, so a full pass + // over the cluster costs one retry rather than one per endpoint: + // a pass that stopped at the first refusal would never reach the + // survivors of a client configured for a single retry. + var lastErr error + // A fault no retry can fix, kept aside rather than returned at + // once: it belongs to the endpoint that raised it, and the + // endpoints behind that one may be perfectly usable. + var configFault error + for _, address := range candidates { + if address == pacedEndpoint { + c.awaitReestablish(ctx, connectedAt) + } + connection, err := c.dialCandidate(ctx, address, len(candidates) > 1) + if err != nil { + lastErr = err + if isTLSConfigFault(err) { + configFault = err + } + continue + } - if !c.config.tlsEnabled { conn = connection return nil } - // TLS logic - tlsConfig, err := c.createTLSConfig() - if err != nil { - _ = connection.Close() - return err + // An unreadable CA file, an unparsable domain, a certificate this + // client will never accept: no endpoint answered, and at least one + // said why in a way no retry changes. Reported as unrecoverable so + // the default unlimited retries do not redial it every interval + // forever and bury it. + if configFault != nil { + return retry.Unrecoverable(configFault) } - tlsConn := tls.Client(connection, tlsConfig) - if err := tlsConn.HandshakeContext(ctx); err != nil { - c.logger.Error("Failed to establish a TLS connection to the server", slog.Any("error", err)) - _ = connection.Close() - return fmt.Errorf("TLS handshake failed: %w", err) - } - - conn = tlsConn - return nil + return lastErr }); err != nil { c.mtx.Lock() c.transportState = iggcon.TransportStateDisconnected @@ -975,6 +1073,24 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { } c.mtx.Lock() + if state := c.transportState; state != iggcon.TransportStateConnecting { + // Superseded while this attempt was dialing. The connection it just + // made is surplus either way, but what to report differs: a client + // another attempt already connected is connected, and saying + // otherwise would fail a caller whose client is up. + c.mtx.Unlock() + _ = conn.Close() + c.logger.Debug("The connect was superseded while dialing; dropping the connection.", + slog.Any("transport_state", state)) + switch state { + case iggcon.TransportStateShutdown: + return ierror.ErrClientShutdown + case iggcon.TransportStateConnected: + return nil + default: + return ierror.ErrNotConnected + } + } c.conn = conn c.reader = bufio.NewReaderSize(conn, 64*1024) c.transportState = iggcon.TransportStateConnected @@ -982,18 +1098,124 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { // The server fence does not survive the old socket, so the new connection // starts from a fresh client identity. c.session.Reset() - skipAutoLogin := c.skipAutoLoginOnce - c.skipAutoLoginOnce = false - c.logger.Info("Iggy client has connected to the Iggy server", slog.String("client_address", c.clientAddress), slog.String("server_address", c.currentServerAddress)) + clientAddress := c.clientAddress + serverAddress := c.currentServerAddress c.mtx.Unlock() + c.logger.Info("Iggy client has connected to the Iggy server", + slog.String("client_address", clientAddress), + slog.String("server_address", serverAddress)) - if err := c.establishSession(ctx, skipAutoLogin); err != nil { + if err := c.establishSession(ctx, ctx.Value(skipAutoLogin{}) != nil); err != nil { _ = c.disconnect() return err } return nil } +// isTLSConfigFault reports whether a dial failed for a reason that says the +// client's own TLS configuration is wrong -- an unreadable or unparsable CA +// file, a domain that yields no server name, or a certificate this client will +// never accept. None of those change on a retry. +// +// A peer that answered the ClientHello in plaintext is not one of them: that +// says something about the endpoint, not about this client, and the endpoints +// behind it in the roster may be speaking TLS perfectly well. +func isTLSConfigFault(err error) bool { + if errors.Is(err, ierror.ErrInvalidTlsCertificatePath) || + errors.Is(err, ierror.ErrInvalidTlsCertificate) || + errors.Is(err, ierror.ErrInvalidTlsDomain) { + return true + } + + var certificateError *tls.CertificateVerificationError + return errors.As(err, &certificateError) +} + +// awaitReestablish waits out what is left of the reestablishAfter window since +// the last successful connection, if any. +// +// The wait ends early on the caller's context or on Close: a sweep that found +// every other endpoint refused reaches the paced one in milliseconds, and +// sleeping the rest of the window regardless would hold the client in +// Connecting long past the deadline the caller gave it. +func (c *IggyTcpClient) awaitReestablish(ctx context.Context, connectedAt time.Time) { + if connectedAt.IsZero() { + return + } + + elapsed := time.Since(connectedAt) + c.logger.Debug("Elapsed time since last connection", slog.Duration("elapsed", elapsed)) + remaining := c.config.reconnection.reestablishAfter - elapsed + if remaining <= 0 { + return + } + + c.logger.Info("Trying to connect to the server", slog.Duration("remaining", remaining)) + timer := time.NewTimer(remaining) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + case <-c.closed: + } +} + +// dialCandidate brings one endpoint all the way up, wrapping it in TLS when +// configured, and records the endpoint that answered: the leader check +// compares against it and the next reconnect starts from it. +// +// bounded caps the whole attempt at failoverDialTimeout, for when other +// endpoints are queued behind this one. Neither the dial nor the handshake has +// a deadline of its own, and a node whose syns are dropped -- or one that +// accepts TCP and then never answers the ClientHello -- would hold the sweep +// for minutes while a survivor goes untried. +func (c *IggyTcpClient) dialCandidate(ctx context.Context, address string, bounded bool) (net.Conn, error) { + c.logger.Info("Iggy client is connecting to server...", slog.String("server_address", address)) + if bounded { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, failoverDialTimeout) + defer cancel() + } + + connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", address) + if err != nil { + c.logger.Error("Failed to establish TCP connection to the server", slog.Any("error", err)) + return nil, ierror.ErrCannotEstablishConnection + } + + tc := connection.(*net.TCPConn) + if err := tc.SetNoDelay(c.config.noDelay); err != nil { + c.logger.Error("Failed to set the nodelay option on the client, continuing...", slog.Any("error", err)) + } + + established := connection + if c.config.tlsEnabled { + tlsConfig, err := c.createTLSConfig(address) + if err != nil { + _ = connection.Close() + return nil, err + } + + tlsConn := tls.Client(connection, tlsConfig) + if err := tlsConn.HandshakeContext(ctx); err != nil { + c.logger.Error("Failed to establish a TLS connection to the server", slog.Any("error", err)) + _ = connection.Close() + return nil, fmt.Errorf("TLS handshake failed: %w", err) + } + established = tlsConn + } + + // Recorded only once the connection is usable: an endpoint that accepts + // TCP but fails the handshake is not where this client lives, and leading + // the next pass with it would shadow every endpoint behind it. + c.mtx.Lock() + c.clientAddress = tc.LocalAddr().String() + c.currentServerAddress = address + c.mtx.Unlock() + + return established, nil +} + func (c *IggyTcpClient) connectionCandidates() []string { c.mtx.Lock() defer c.mtx.Unlock() @@ -1025,8 +1247,9 @@ func (c *IggyTcpClient) connectionCandidates() []string { // backup: once the caller signs in, the first replicated request fails over // through the transient-deny path. func (c *IggyTcpClient) establishSession(ctx context.Context, skipAutoLogin bool) error { - if !c.config.autoLogin.enabled { - c.logger.Info("Automatic sign-in is disabled.") + credentials, ok := c.signInCredentials() + if !ok { + c.logger.Info("No credentials to sign in with.") return nil } if skipAutoLogin { @@ -1034,7 +1257,6 @@ func (c *IggyTcpClient) establishSession(ctx context.Context, skipAutoLogin bool return nil } - credentials := c.config.autoLogin.credentials if credentials.personalAccessToken != "" { _, err := c.LoginWithPersonalAccessToken(ctx, credentials.personalAccessToken) return err @@ -1043,7 +1265,42 @@ func (c *IggyTcpClient) establishSession(ctx context.Context, skipAutoLogin bool return err } -func (c *IggyTcpClient) createTLSConfig() (*tls.Config, error) { +// signInCredentials reports the credentials a reconnect signs in with: the +// configured ones, or else the ones a manual sign-in succeeded with. +func (c *IggyTcpClient) signInCredentials() (Credentials, bool) { + if c.config.autoLogin.enabled { + return c.config.autoLogin.credentials, true + } + c.mtx.Lock() + defer c.mtx.Unlock() + return c.rememberedLogin.credentials, c.rememberedLogin.enabled +} + +// rememberLogin keeps the credentials a sign-in succeeded with. Call it from +// under registerMtx (register does): remembered outside that lock, two +// concurrent sign-ins can leave A remembered while the session is B. +func (c *IggyTcpClient) rememberLogin(credentials Credentials) { + c.mtx.Lock() + defer c.mtx.Unlock() + c.rememberedLogin = NewAutoLogin(credentials) +} + +// forgetLogin drops them: after an explicit sign-out there is no session to +// restore, and a reconnect must not resurrect one. +func (c *IggyTcpClient) forgetLogin() { + c.mtx.Lock() + defer c.mtx.Unlock() + c.rememberedLogin = AutoLogin{} +} + +// createTLSConfig builds the client config for one dial. +// +// address is the candidate being dialed, which is where the SNI comes from +// when no domain is configured. Taking it from currentServerAddress instead +// would name the endpoint the client just lost: with validation on, a failover +// to a node with another name or address then fails the handshake against a +// certificate that never covered the old one. +func (c *IggyTcpClient) createTLSConfig(address string) (*tls.Config, error) { tlsConfig := &tls.Config{ InsecureSkipVerify: !c.config.tls.tlsValidateCertificate, } @@ -1051,9 +1308,9 @@ func (c *IggyTcpClient) createTLSConfig() (*tls.Config, error) { // Set server name for SNI serverName := c.config.tls.tlsDomain if serverName == "" { - host, _, err := net.SplitHostPort(c.currentServerAddress) + host, _, err := net.SplitHostPort(address) if err != nil { - host = c.currentServerAddress + host = address } serverName = host } @@ -1094,6 +1351,15 @@ func (c *IggyTcpClient) disconnect() error { if c.transportState == iggcon.TransportStateDisconnected || c.transportState == iggcon.TransportStateShutdown { return nil } + if c.transportState == iggcon.TransportStateConnecting { + // An attempt is already dialing. Every caller here is tearing the + // connection down to reconnect, which that attempt is doing anyway: + // resetting the state under it would let the next Connect start a + // second attempt, and the two would fight over which socket ends up + // installed and which error the waiters are told about. + c.logger.Debug("Not disconnecting; a connect is already in flight.") + return nil + } c.logger.Info("Iggy client is disconnecting from server...", slog.String("client_address", c.clientAddress)) c.transportState = iggcon.TransportStateDisconnected diff --git a/foreign/go/client/tcp/tcp_core_review_test.go b/foreign/go/client/tcp/tcp_core_review_test.go index f9deff4f76..4d0c2e5cdd 100644 --- a/foreign/go/client/tcp/tcp_core_review_test.go +++ b/foreign/go/client/tcp/tcp_core_review_test.go @@ -136,14 +136,19 @@ func TestConnect_SuppressedSignInSendsNothing(t *testing.T) { client := newDialingClient(t, server.address(), WithAutoLogin(NewUsernamePasswordCredentials("iggy", "iggy"))) - // The state a replayed login leaves behind before its reconnect. - client.skipAutoLoginOnce = true - require.NoError(t, client.Connect(context.Background())) + // The context a replayed login drives its reconnect with. + require.NoError(t, client.Connect(suppressAutoLogin(context.Background()))) assert.Empty(t, server.recorded(), "the replayed login owns the sign-in; Connect must not preempt it") - assert.False(t, client.skipAutoLoginOnce, "the suppression is consumed exactly once") + + // And it is that context, not the client, that carries the suppression: + // a Connect without it signs in. + require.NoError(t, client.disconnect()) + require.NoError(t, client.Connect(context.Background())) + assert.NotEmpty(t, server.recorded(), + "the suppression outlived the call that meant it") } func TestClose_InterruptsAnInFlightReplayWait(t *testing.T) { diff --git a/foreign/go/client/tcp/tcp_failover_test.go b/foreign/go/client/tcp/tcp_failover_test.go new file mode 100644 index 0000000000..e7946a462e --- /dev/null +++ b/foreign/go/client/tcp/tcp_failover_test.go @@ -0,0 +1,750 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tcp + +import ( + "context" + "crypto/tls" + "log/slog" + "net" + "sync/atomic" + "testing" + "time" + + iggcon "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" + "github.com/apache/iggy/foreign/go/internal/command" + "github.com/apache/iggy/foreign/go/internal/vsr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The node a client signed in on dies; its next request has to complete on a +// survivor the roster named, under the identity a fresh sign-in binds there. +// Mirrors `core/integration/tests/cluster/failover_client_continuity.rs`. +func TestFailover_ResumesOnASurvivorAfterTheSignedInNodeDies(t *testing.T) { + var survivor *testListener + var primary *testListener + var primaryDead atomic.Bool + + survivor = listenVSR(t, nil, func(_, _ int, read request) []byte { + switch { + case read.code() == uint32(command.GetClusterMetadataCode): + return clusterMetadataFrame(t, 1, primary.address(), survivor.address()) + case read.operation() == vsr.OperationRegister: + return registerReplyFrame(7, 512) + default: + return replyFrame(vsr.OperationNonReplicated, nil) + } + }) + + primary = listenVSR(t, nil, func(_, _ int, read request) []byte { + // A dead node answers nothing; returning nil drops the connection the + // way a killed process does. + if primaryDead.Load() { + return nil + } + switch { + case read.code() == uint32(command.GetClusterMetadataCode): + // The primary leads, so the sign-in settles here and the roster is + // only remembered -- not acted on -- until the node dies. + return clusterMetadataFrame(t, 0, primary.address(), survivor.address()) + case read.operation() == vsr.OperationRegister: + return registerReplyFrame(7, 128) + default: + return replyFrame(vsr.OperationNonReplicated, nil) + } + }) + + // No auto-login: the credentials come from the caller's own sign-in, which + // is the shape that could not reconnect at all before. + client := newDialingClient(t, primary.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "iggy", "iggy") + require.NoError(t, err) + require.NoError(t, client.Ping(ctx), "the live primary answers") + require.Equal(t, primary.address(), client.currentServerAddress) + + primaryDead.Store(true) + require.NoError(t, primary.listener.Close(), "stop accepting, so a redial is refused") + + require.NoError(t, client.Ping(ctx), + "the client has to resume on the survivor the roster named") + + assert.Equal(t, survivor.address(), client.currentServerAddress, + "the client moved off the dead endpoint") + assert.True(t, client.session.Bound(), "the session was re-established") + + var registers int + for _, read := range survivor.recorded() { + if read.operation() == vsr.OperationRegister { + registers++ + } + } + assert.Equal(t, 1, registers, + "the remembered credentials signed in again on the survivor") +} + +// Without any credentials there is nothing to sign in with, so a request on a +// dead node fails instead of reconnecting into an unauthenticated session. +func TestFailover_FailsFastWhenNothingEverSignedIn(t *testing.T) { + var server *testListener + var dead atomic.Bool + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + if dead.Load() { + return nil + } + return singleNodeHandler(t, func() string { return server.address() })(0, 0, read) + }) + + client := newDialingClient(t, server.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + require.NoError(t, client.Ping(ctx)) + + dead.Store(true) + require.NoError(t, server.listener.Close()) + + assert.Error(t, client.Ping(ctx), + "a client that never signed in cannot restore a session by reconnecting") +} + +// A stale-client eviction is not caller intent: the heartbeat verifier sends it +// after a gc pause or a laptop sleep, and a client that signed in by hand has +// to recover from it exactly like one with a configured auto-login. Same rule +// in every SDK. +func TestFailover_ServerEvictionReplaysTheRememberedSignIn(t *testing.T) { + var server *testListener + var evict atomic.Bool + var registers atomic.Int32 + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + if read.operation() == vsr.OperationRegister { + registers.Add(1) + return registerReplyFrame(7, 128) + } + if evict.CompareAndSwap(true, false) { + return evictionFrame(vsr.EvictionStaleClient, 0, 0) + } + if read.code() == uint32(command.GetClusterMetadataCode) { + return clusterMetadataFrame(t, 0, server.address()) + } + return replyFrame(vsr.OperationNonReplicated, nil) + }) + + client := newDialingClient(t, server.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "iggy", "iggy") + require.NoError(t, err) + registersBefore := registers.Load() + + evict.Store(true) + // A ping is non-replicated, so the eviction is absorbed: the reconnect it + // triggers signs in again with the credentials the sign-in remembered and + // the request completes over the session it re-established. + require.NoError(t, client.Ping(ctx), "the evicted request was not recovered") + + _, remembered := client.signInCredentials() + assert.True(t, remembered, "an eviction is not a sign-out; the credentials stay") + require.NoError(t, client.Ping(ctx), "the session came back on its own") + assert.Greater(t, registers.Load(), registersBefore, + "the reconnect re-established the session") + assert.True(t, client.session.Bound()) +} + +// An explicit sign-out is caller intent: the reconnect must not sign back in +// with the credentials the earlier sign-in used. +func TestFailover_DoesNotResurrectASignedOutSession(t *testing.T) { + var server *testListener + var dropSocket atomic.Bool + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + // A dropped connection is what makes the client reconnect at all; nil + // ends it the way a killed process does. + if dropSocket.Load() { + return nil + } + return singleNodeHandler(t, func() string { return server.address() })(0, 0, read) + }) + + client := newDialingClient(t, server.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "iggy", "iggy") + require.NoError(t, err) + require.NoError(t, client.LogoutUser(ctx)) + + credentials, ok := client.signInCredentials() + require.False(t, ok, "the sign-out forgot them") + require.Empty(t, credentials.username) + + // The socket dies under a signed-out client: the reconnect has nothing to + // restore and must not invent a session. + dropSocket.Store(true) + assert.Error(t, client.Ping(ctx), "a signed-out client cannot replay through a sign-in") + dropSocket.Store(false) + + require.NoError(t, client.Connect(ctx)) + require.NoError(t, client.Ping(ctx), "the transport recovers on its own") + + var registers int + for _, read := range server.recorded() { + if read.operation() == vsr.OperationRegister { + registers++ + } + } + assert.Equal(t, 1, registers, + "only the caller's own sign-in registered; the reconnect added none") +} + +// A re-login over a dropped transport has to complete. The logout that ends +// the old session runs while the sign-in lock is held, so a logout that enters +// the reconnect path would reconnect, sign in with the remembered credentials, +// and deadlock on that same lock. +func TestFailover_ReLoginSurvivesALogoutTheTransportSwallowed(t *testing.T) { + var server *testListener + var dropLogout atomic.Bool + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + if dropLogout.Load() && read.operation() == vsr.OperationLogout { + // The frame is swallowed and the connection ends, exactly as a + // node that dies mid-logout leaves it. + return nil + } + return singleNodeHandler(t, func() string { return server.address() })(0, 0, read) + }) + + client := newDialingClient(t, server.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "iggy", "iggy") + require.NoError(t, err) + + dropLogout.Store(true) + relogin := make(chan error, 1) + go func() { + _, err := client.LoginUser(ctx, "iggy", "iggy") + relogin <- err + }() + select { + case err := <-relogin: + require.NoError(t, err, "the sign-in has to replay on the new connection") + case <-time.After(15 * time.Second): + t.Fatal("the re-login deadlocked on the sign-in lock") + } + + assert.True(t, client.session.Bound(), "the replayed sign-in bound a session") + require.NoError(t, client.Ping(ctx)) +} + +// The other way a logout fails to land: the node answers it as not-admitted, +// which is what a node that stopped being primary does. The redirect that +// follows must not sign in on its own -- this goroutine holds the sign-in lock, +// and the reconnect's automatic sign-in would wait on it forever. +func TestFailover_ReLoginSurvivesALogoutTheOldPrimaryRefused(t *testing.T) { + var leader *testListener + var follower *testListener + var demoted atomic.Bool + + // The node the client is on: leader until the logout, then a follower that + // refuses it as not-admitted and points at the survivor. + follower = listenVSR(t, nil, func(_, _ int, read request) []byte { + switch { + case read.code() == uint32(command.GetClusterMetadataCode): + if demoted.Load() { + return clusterMetadataFrame(t, 1, follower.address(), leader.address()) + } + return clusterMetadataFrame(t, 0, follower.address(), leader.address()) + case read.operation() == vsr.OperationRegister: + return registerReplyFrame(7, 128) + case read.operation() == vsr.OperationLogout: + demoted.Store(true) + return statusReplyFrame(vsr.OperationLogout, + uint32(ierror.TransientNotAcceptedCode), nil) + default: + return replyFrame(vsr.OperationNonReplicated, nil) + } + }) + leader = listenVSR(t, nil, func(_, _ int, read request) []byte { + switch { + case read.code() == uint32(command.GetClusterMetadataCode): + return clusterMetadataFrame(t, 1, follower.address(), leader.address()) + case read.operation() == vsr.OperationRegister: + return registerReplyFrame(7, 256) + default: + return replyFrame(vsr.OperationNonReplicated, nil) + } + }) + + client := newDialingClient(t, follower.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "iggy", "iggy") + require.NoError(t, err) + + relogin := make(chan error, 1) + go func() { + _, err := client.LoginUser(ctx, "iggy", "iggy") + relogin <- err + }() + select { + case err := <-relogin: + require.NoError(t, err, "the sign-in has to settle on the node that leads") + case <-time.After(15 * time.Second): + t.Fatal("the re-login deadlocked on the sign-in lock") + } + + assert.True(t, client.session.Bound(), "the replayed sign-in bound a session") + assert.Equal(t, leader.address(), client.currentServerAddress) +} + +// A logout that never landed still ended the session it belonged to, so the +// credentials that established it must not outlive it: a sign-in that then +// fails would otherwise leave them for the next dropped request to replay, +// signing the old user back in after the caller asked for another one. +func TestFailover_ARejectedReLoginDoesNotResurrectThePreviousUser(t *testing.T) { + var server *testListener + var dropLogout atomic.Bool + var dropSocket atomic.Bool + var rejectLogin atomic.Bool + var registeredUsers atomic.Int32 + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + if dropSocket.Load() { + return nil + } + if dropLogout.Load() && read.operation() == vsr.OperationLogout { + return nil + } + if read.operation() == vsr.OperationRegister { + registeredUsers.Add(1) + if rejectLogin.Load() { + return statusReplyFrame(vsr.OperationRegister, + uint32(ierror.InvalidCredentialsCode), nil) + } + return registerReplyFrame(7, 128) + } + return singleNodeHandler(t, func() string { return server.address() })(0, 0, read) + }) + + client := newDialingClient(t, server.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "alice", "alice") + require.NoError(t, err) + + // The logout is swallowed and the sign-in that follows is rejected, so the + // client ends up with no session and no credentials it may use. + dropLogout.Store(true) + rejectLogin.Store(true) + _, err = client.LoginUser(ctx, "bob", "bob") + require.Error(t, err) + + _, remembered := client.signInCredentials() + assert.False(t, remembered, "the ended session's credentials must not survive it") + + // The socket dies with nothing remembered: the reconnect has no session to + // restore, and must not invent one out of the user who was signed in + // before. + dropLogout.Store(false) + dropSocket.Store(true) + registersBefore := registeredUsers.Load() + assert.Error(t, client.Ping(ctx), "there is no session left to restore") + assert.Equal(t, registersBefore, registeredUsers.Load(), + "the reconnect signed the previous user back in") +} + +// reestablishAfter is a cooldown on redialing the endpoint that was lost. It +// is owed to that endpoint alone, so a failover to another one must not sit +// through it. +func TestFailover_DoesNotSpendTheLostEndpointsPauseOnAnotherEndpoint(t *testing.T) { + var survivor *testListener + survivor = listenVSR(t, nil, singleNodeHandler(t, func() string { return survivor.address() })) + + client := newDialingClient(t, deadAddress(t)) + client.config.reconnection.reestablishAfter = time.Minute + client.knownServerAddresses = []string{survivor.address()} + client.connectedAt = time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + started := time.Now() + require.NoError(t, client.Connect(ctx)) + + assert.Equal(t, survivor.address(), client.currentServerAddress) + assert.Less(t, time.Since(started), 2*time.Second, + "the failover waited out a pause it owed only the lost endpoint") +} + +// The other half of the same promise: WithReestablishAfter is a cooldown on +// the endpoint that was lost, and a known roster does not cancel it. +func TestFailover_KeepsTheReestablishPauseForTheEndpointThatWasLost(t *testing.T) { + var current *testListener + current = listenVSR(t, nil, singleNodeHandler(t, func() string { return current.address() })) + + client := newDialingClient(t, current.address()) + client.config.reconnection.reestablishAfter = 500 * time.Millisecond + client.knownServerAddresses = []string{deadAddress(t)} + client.connectedAt = time.Now() + + started := time.Now() + require.NoError(t, client.Connect(context.Background())) + + assert.Equal(t, current.address(), client.currentServerAddress) + assert.GreaterOrEqual(t, time.Since(started), 350*time.Millisecond, + "the cooldown on the endpoint that was lost was skipped") +} + +// The cooldown is a pace limit, not a commitment: a caller that gave the +// connect a deadline has to get an answer inside it, and Close has to end the +// wait too. +func TestFailover_TheReestablishPauseHonoursTheCallersDeadline(t *testing.T) { + current := listenVSR(t, nil, func(_, _ int, read request) []byte { + return singleNodeHandler(t, func() string { return "127.0.0.1:8090" })(0, 0, read) + }) + + client := newDialingClient(t, current.address()) + client.config.reconnection.reestablishAfter = time.Minute + client.connectedAt = time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + started := time.Now() + _ = client.Connect(ctx) + + assert.Less(t, time.Since(started), 5*time.Second, + "the cooldown outlived the deadline the caller gave the connect") +} + +// A node whose syns are dropped must not hold the sweep: without a bound on +// the dial the survivors behind it are never reached. A black-holed address +// cannot be arranged portably, so this pins the bound itself. +func TestFailover_BoundsTheDialWhenOtherEndpointsAreQueuedBehindIt(t *testing.T) { + assert.Equal(t, 2*time.Second, failoverDialTimeout, + "the dial bound has to match the other SDKs") + + var survivor *testListener + survivor = listenVSR(t, nil, singleNodeHandler(t, func() string { return survivor.address() })) + + // A listener that accepts and never answers: the dial completes out of the + // backlog, so only the bound ends the attempt. + silent, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = silent.Close() }) + + client := newDialingClient(t, silent.Addr().String(), + WithTLS(WithTLSValidateCertificate(false))) + client.knownServerAddresses = []string{survivor.address()} + + done := make(chan error, 1) + go func() { done <- client.Connect(context.Background()) }() + select { + case <-done: + case <-time.After(3 * failoverDialTimeout): + t.Fatal("the sweep never got past an endpoint that answers nothing") + } +} + +// An endpoint that accepts TCP but fails the handshake is not where this +// client lives: recording it would make the next pass lead with it and shadow +// every endpoint behind it. +func TestFailover_DoesNotSettleOnAnEndpointThatFailedTheHandshake(t *testing.T) { + hangup, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = hangup.Close() }) + go func() { + for { + connection, err := hangup.Accept() + if err != nil { + return + } + // Plain TCP behind a TLS client: the dial succeeds, the handshake + // cannot. + _ = connection.Close() + } + }() + + configured := deadAddress(t) + client := newDialingClient(t, configured, WithTLS(WithTLSValidateCertificate(false))) + client.config.reconnection.enabled = false + client.knownServerAddresses = []string{hangup.Addr().String()} + + require.Error(t, client.Connect(context.Background())) + assert.Equal(t, configured, client.currentServerAddress, + "the endpoint that failed the handshake became the current one") +} + +// The SNI of a dial belongs to the endpoint being dialed. Taken from the +// endpoint the client just lost, a failover to a node the certificate does not +// cover fails the handshake -- which is every failover, once the addresses +// differ. +func TestFailover_UsesTheDialedEndpointAsTheServerName(t *testing.T) { + certificate, caPath := selfSignedCert(t) + survivor := listenVSR(t, + func(conn net.Conn) net.Conn { + return tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{certificate}}) + }, + singleNodeHandler(t, func() string { return "127.0.0.1:8090" })) + + // The certificate covers 127.0.0.1, and the endpoint the client starts on + // is 127.0.0.2, where nothing listens: with the server name taken from + // that endpoint, the handshake on the survivor is checked against the + // address that died. + _, port, err := net.SplitHostPort(survivor.address()) + require.NoError(t, err) + client := newDialingClient(t, "127.0.0.2:"+port, + WithTLS(WithTLSCAFile(caPath), WithTLSValidateCertificate(true))) + client.knownServerAddresses = []string{"127.0.0.1:" + port} + + require.NoError(t, client.Connect(context.Background())) + assert.Equal(t, "127.0.0.1:"+port, client.currentServerAddress) +} + +// A TLS configuration the client itself cannot satisfy says the same thing on +// every attempt, so it has to reach the caller instead of being redialed every +// interval forever -- which is what the default unlimited retries did with it. +func TestFailover_AConfigFaultEndsTheConnectInsteadOfRetryingForever(t *testing.T) { + certificate, _ := selfSignedCert(t) + server := listenVSR(t, + func(conn net.Conn) net.Conn { + return tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{certificate}}) + }, + singleNodeHandler(t, func() string { return "127.0.0.1:8090" })) + + // A CA the server's certificate was not signed by: no retry makes that + // certificate acceptable. + _, unrelatedCA := selfSignedCert(t) + client := NewIggyTcpClient(slog.New(slog.DiscardHandler), + WithServerAddress(server.address()), + WithTLS(WithTLSCAFile(unrelatedCA), WithTLSValidateCertificate(true))) + t.Cleanup(func() { _ = client.Close() }) + client.config.reconnection.maxRetries = 0 // unlimited + client.config.reconnection.interval = 10 * time.Millisecond + + done := make(chan error, 1) + go func() { done <- client.Connect(context.Background()) }() + select { + case err := <-done: + require.Error(t, err) + case <-time.After(5 * time.Second): + t.Fatal("a connect that can never succeed has to end instead of retrying forever") + } +} + +// A peer that answers the handshake in plaintext says something about that +// endpoint, not about this client's TLS configuration. Ended the whole connect, +// one misconfigured node in the roster costs the client every endpoint behind +// it, including the ones that are only down for a moment. +func TestFailover_APlaintextEndpointDoesNotEndTheSweep(t *testing.T) { + certificate, _ := selfSignedCert(t) + var accepted atomic.Int32 + var survivor *testListener + survivor = listenVSR(t, + func(conn net.Conn) net.Conn { + if accepted.Add(1) == 1 { + // Down for the first pass, up for the second: without it the + // sweep reaches this node and the pass succeeds either way. + _ = conn.Close() + return conn + } + return tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{certificate}}) + }, + singleNodeHandler(t, func() string { return survivor.address() })) + + plaintext, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = plaintext.Close() }) + go func() { + for { + conn, err := plaintext.Accept() + if err != nil { + return + } + _, _ = conn.Write([]byte("not a TLS record\n")) + _ = conn.Close() + } + }() + + client := newDialingClient(t, plaintext.Addr().String(), + WithTLS(WithTLSValidateCertificate(false))) + client.config.reconnection.maxRetries = 2 + client.knownServerAddresses = []string{survivor.address()} + + require.NoError(t, client.Connect(context.Background())) + assert.Equal(t, survivor.address(), client.currentServerAddress) +} + +// A client with nothing to dial must say so: reporting success would leave +// every request answering ErrNotConnected while Connect keeps claiming a +// connection. +func TestFailover_RejectsAConnectWithNoEndpointToDial(t *testing.T) { + client := NewIggyTcpClient(slog.New(slog.DiscardHandler), WithServerAddress("")) + t.Cleanup(func() { _ = client.Close() }) + + require.ErrorIs(t, client.Connect(context.Background()), ierror.ErrCannotEstablishConnection) + assert.Error(t, client.Ping(context.Background())) +} + +// Concurrent Connects are one attempt, and a caller that did not run it still +// gets a client it can use the moment its Connect returns. Told "connected" +// while the attempt is still signing in, its next request fails +// ErrNotConnected for no reason of its own. +func TestConnect_ConcurrentCallersShareOneAttempt(t *testing.T) { + var server *testListener + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + if read.operation() == vsr.OperationRegister { + // The sign-in is the slow part of an attempt, and it runs after the + // dial: a caller that returned early would use the client here. + time.Sleep(300 * time.Millisecond) + } + return singleNodeHandler(t, func() string { return server.address() })(0, 0, read) + }) + + client := newDialingClient(t, server.address(), + WithAutoLogin(NewUsernamePasswordCredentials("iggy", "iggy"))) + + const callers = 8 + results := make(chan error, callers) + start := make(chan struct{}) + for range callers { + go func() { + <-start + if err := client.Connect(context.Background()); err != nil { + results <- err + return + } + // Usable right away, or the Connect that returned was a lie. + results <- client.Ping(context.Background()) + }() + } + close(start) + for range callers { + require.NoError(t, <-results, "a caller was handed a client it could not use") + } + + assert.Equal(t, 1, server.connections(), "the callers dialed more than once") + var registers int + for _, read := range server.recorded() { + if read.operation() == vsr.OperationRegister { + registers++ + } + } + assert.Equal(t, 1, registers, "one attempt, one sign-in") +} + +// Two requests failing at once are one reconnect. Each tears the connection +// down before reconnecting, and a teardown that resets the state under an +// attempt already dialing lets the second caller start a second attempt: the +// two then fight over which socket is installed and which outcome the waiters +// are handed. +func TestConnect_ConcurrentReconnectsThroughExchangeShareOneAttempt(t *testing.T) { + var server *testListener + server = listenVSR(t, nil, func(connection, index int, read request) []byte { + if connection == 0 && read.code() == uint32(command.PingCode) { + // Ends the socket under both in-flight requests at once. + return nil + } + if connection > 0 && read.operation() == vsr.OperationRegister { + // The reconnect's sign-in is the slow part, so the second caller + // reliably arrives while the first attempt is still running. + time.Sleep(300 * time.Millisecond) + } + return singleNodeHandler(t, func() string { return server.address() })(connection, index, read) + }) + + client := newDialingClient(t, server.address(), + WithAutoLogin(NewUsernamePasswordCredentials("iggy", "iggy"))) + require.NoError(t, client.Connect(context.Background())) + + const callers = 2 + results := make(chan error, callers) + start := make(chan struct{}) + for range callers { + go func() { + <-start + results <- client.Ping(context.Background()) + }() + } + close(start) + for range callers { + require.NoError(t, <-results, "a request did not survive the reconnect") + } + + assert.Equal(t, 2, server.connections(), + "the two failing requests reconnected separately") +} + +// The sign-in transaction holds registerMtx across its reconnect, and an +// attempt started by a plain request ends in a sign-in that needs that same +// lock. Waiting for that attempt closes a cycle -- the owner blocked on +// registerMtx, the transaction blocked on the owner -- and callers pass a +// context with no deadline, so nothing breaks it. +func TestConnect_DoesNotWaitOnAnAttemptThatSignsIn(t *testing.T) { + certificate, _ := selfSignedCert(t) + var survivor *testListener + survivor = listenVSR(t, + func(conn net.Conn) net.Conn { + return tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{certificate}}) + }, + singleNodeHandler(t, func() string { return survivor.address() })) + + // A listener that accepts and never answers the ClientHello: the attempt + // spends the whole dial bound here, which is the window a second caller + // arrives in. + silent, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = silent.Close() }) + + client := newDialingClient(t, silent.Addr().String(), + WithTLS(WithTLSValidateCertificate(false)), + WithAutoLogin(NewUsernamePasswordCredentials("iggy", "iggy"))) + client.knownServerAddresses = []string{survivor.address()} + + // Stands in for the sign-in transaction: register holds this across the + // disconnect and the reconnect that follow it. + client.registerMtx.Lock() + owner := make(chan error, 1) + go func() { owner <- client.Connect(context.Background()) }() + require.Eventually(t, func() bool { + client.mtx.Lock() + defer client.mtx.Unlock() + return client.transportState == iggcon.TransportStateConnecting + }, time.Second, time.Millisecond, "the attempt never started dialing") + + suppressed := make(chan error, 1) + go func() { suppressed <- client.Connect(suppressAutoLogin(context.Background())) }() + select { + case err := <-suppressed: + require.Error(t, err, "the transaction was told a connection it does not have is up") + case <-time.After(2 * failoverDialTimeout): + t.Fatal("the sign-in transaction waited on an attempt that cannot finish without it") + } + + client.registerMtx.Unlock() + require.NoError(t, <-owner, "the attempt the transaction left alone did not finish") +} + +// deadAddress returns an address nothing listens on, so a dial to it is +// refused at once. +func deadAddress(t *testing.T) string { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + address := listener.Addr().String() + require.NoError(t, listener.Close()) + return address +} diff --git a/foreign/go/client/tcp/tcp_session_credentials_test.go b/foreign/go/client/tcp/tcp_session_credentials_test.go new file mode 100644 index 0000000000..c219d4f6a3 --- /dev/null +++ b/foreign/go/client/tcp/tcp_session_credentials_test.go @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tcp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newCredentialClient(autoLogin AutoLogin) *IggyTcpClient { + config := defaultTcpClientConfig() + config.autoLogin = autoLogin + return &IggyTcpClient{config: config} +} + +func TestSignInCredentials_AreAbsentUntilSomethingSignsIn(t *testing.T) { + client := newCredentialClient(AutoLogin{}) + + _, ok := client.signInCredentials() + assert.False(t, ok) +} + +func TestSignInCredentials_ComeFromAManualSignInWithoutAutoLogin(t *testing.T) { + client := newCredentialClient(AutoLogin{}) + + client.rememberLogin(NewUsernamePasswordCredentials("iggy", "secret")) + + credentials, ok := client.signInCredentials() + require.True(t, ok) + assert.Equal(t, "iggy", credentials.username) + assert.Equal(t, "secret", credentials.password) + + // An explicit sign-out leaves no session to restore, and a reconnect must + // not resurrect one. + client.forgetLogin() + _, ok = client.signInCredentials() + assert.False(t, ok) +} + +func TestSignInCredentials_PreferTheConfiguredOnes(t *testing.T) { + client := newCredentialClient(NewAutoLogin(NewUsernamePasswordCredentials("configured", "secret"))) + + client.rememberLogin(NewPersonalAccessTokenCredentials("signed-in-token")) + + credentials, ok := client.signInCredentials() + require.True(t, ok) + assert.Equal(t, "configured", credentials.username) + assert.Empty(t, credentials.personalAccessToken) +} diff --git a/foreign/go/client/tcp/tcp_session_management.go b/foreign/go/client/tcp/tcp_session_management.go index d409f7d9af..4da1fb23b5 100644 --- a/foreign/go/client/tcp/tcp_session_management.go +++ b/foreign/go/client/tcp/tcp_session_management.go @@ -33,7 +33,12 @@ func (c *IggyTcpClient) LoginUser(ctx context.Context, username string, password if err != nil { return nil, err } - return c.register(ctx, uint32(command.LoginRegisterCode), body) + return c.register( + ctx, + uint32(command.LoginRegisterCode), + body, + NewUsernamePasswordCredentials(username, password), + ) } func (c *IggyTcpClient) LoginWithPersonalAccessToken(ctx context.Context, token string) (*iggcon.IdentityInfo, error) { @@ -41,12 +46,28 @@ func (c *IggyTcpClient) LoginWithPersonalAccessToken(ctx context.Context, token if err != nil { return nil, err } - return c.register(ctx, uint32(command.LoginRegisterWithPATCode), body) + return c.register( + ctx, + uint32(command.LoginRegisterWithPATCode), + body, + NewPersonalAccessTokenCredentials(token), + ) } // register runs the sign-in handshake, binds the session the server assigned, -// and settles the connection on the cluster leader. -func (c *IggyTcpClient) register(ctx context.Context, code uint32, body []byte) (*iggcon.IdentityInfo, error) { +// settles the connection on the cluster leader, and remembers the credentials +// it succeeded with so a reconnect can re-establish the session. +// +// The credentials are remembered here rather than by the callers because this +// is what holds registerMtx: remembered outside it, two concurrent sign-ins +// could leave A remembered while the session is B, and the next reconnect +// would sign in as A. +func (c *IggyTcpClient) register( + ctx context.Context, + code uint32, + body []byte, + credentials Credentials, +) (*iggcon.IdentityInfo, error) { // One sign-in at a time. BeginRegister runs inside the exchange lock but // Bind runs after it, so two interleaved sign-ins would let the second // BeginRegister reset the identity the first is about to bind: one @@ -55,7 +76,10 @@ func (c *IggyTcpClient) register(ctx context.Context, code uint32, body []byte) c.registerMtx.Lock() defer c.registerMtx.Unlock() - c.logger.Info("Iggy client is signing in...", slog.String("client_address", c.clientAddress)) + c.mtx.Lock() + clientAddress := c.clientAddress + c.mtx.Unlock() + c.logger.Info("Iggy client is signing in...", slog.String("client_address", clientAddress)) if err := c.endBoundSession(ctx); err != nil { return nil, err @@ -70,6 +94,7 @@ func (c *IggyTcpClient) register(ctx context.Context, code uint32, body []byte) if err != nil { return nil, err } + c.rememberLogin(credentials) if settled != nil { return settled, nil } @@ -114,8 +139,11 @@ func (c *IggyTcpClient) signIn(ctx context.Context, code uint32, body []byte) (* return nil, err } + c.mtx.Lock() + signedInAddress := c.clientAddress + c.mtx.Unlock() c.logger.Info("Iggy client has signed in successfully.", - slog.String("client_address", c.clientAddress), + slog.String("client_address", signedInAddress), slog.String("server_version", registered.ServerVersion)) return &iggcon.IdentityInfo{UserId: registered.UserID}, nil } @@ -147,13 +175,7 @@ func (c *IggyTcpClient) settleOnLeader(ctx context.Context, code uint32, body [] // The replayed sign-in below owns the session; the redirected Connect // must not sign in on its own, or the replay commits a second Register. - c.mtx.Lock() - c.skipAutoLoginOnce = true - c.mtx.Unlock() - if err := c.Connect(ctx); err != nil { - c.mtx.Lock() - c.skipAutoLoginOnce = false - c.mtx.Unlock() + if err := c.Connect(suppressAutoLogin(ctx)); err != nil { return nil, err } settled, err = c.signIn(ctx, code, body) @@ -165,6 +187,14 @@ func (c *IggyTcpClient) settleOnLeader(ctx context.Context, code uint32, body [] // endBoundSession logs out a live session before a re-login, so the server // drops its client-table entry instead of leaving it to be fenced. +// +// The logout runs connect-scoped, and a failure it could recover from is +// swallowed. Both because this call holds registerMtx: a logout that entered +// the reconnect path would reconnect, sign in with the remembered credentials, +// and deadlock on that lock. There is nothing to salvage either way -- a +// session whose logout cannot be delivered died with its socket, and the +// server fences what it left behind -- and the sign-in that follows replays +// through its own reconnect. func (c *IggyTcpClient) endBoundSession(ctx context.Context) error { c.mtx.Lock() bound := c.session.Bound() @@ -172,7 +202,29 @@ func (c *IggyTcpClient) endBoundSession(ctx context.Context) error { if !bound { return nil } - return c.LogoutUser(ctx) + + err := c.LogoutUser(context.WithValue(ctx, connectScoped{}, struct{}{})) + if err == nil { + return nil + } + if !isReconnectable(err) { + return err + } + + c.logger.Debug("The bound session's logout was not delivered; its socket ended it.", + slog.Any("error", err)) + c.mtx.Lock() + c.sessionState = iggcon.SessionStateUnauthenticated + c.session.Reset() + c.groups.clear() + c.topics.clearCounts() + c.mtx.Unlock() + // The session this sign-in belonged to is over either way, so the + // credentials that established it go with it. Kept, a sign-in that then + // fails would leave them behind for the next dropped request to replay -- + // signing the old user back in after the caller asked for another one. + c.forgetLogin() + return nil } func (c *IggyTcpClient) LogoutUser(ctx context.Context) error { @@ -188,6 +240,7 @@ func (c *IggyTcpClient) LogoutUser(ctx context.Context) error { c.groups.clear() c.topics.clearCounts() c.mtx.Unlock() + c.forgetLogin() return nil } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java index e2673d965a..3f6ebd0d49 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java @@ -37,6 +37,7 @@ import org.apache.iggy.client.async.tcp.LeaderAwareness.LeaderRedirectionState; import org.apache.iggy.client.async.tcp.vsr.VsrFrameDecoder; import org.apache.iggy.config.RetryPolicy; +import org.apache.iggy.exception.IggyErrorCode; import org.apache.iggy.exception.IggyMissingCredentialsException; import org.apache.iggy.exception.IggyNotConnectedException; import org.apache.iggy.exception.IggyServerException; @@ -49,15 +50,19 @@ import java.io.File; import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Stream; /** * Async TCP client for Apache Iggy message streaming, built on Netty. @@ -111,6 +116,15 @@ */ public class AsyncIggyTcpClient { + /** + * Bound on one dial while other endpoints are queued behind it, matching + * the Rust, Go, C# and Node SDKs. Netty's own connect timeout would + * otherwise let a node whose syns are dropped hold the whole rotation. + * + * Package-private: the tests pin the bound against the other SDKs'. + */ + static final Duration FAILOVER_DIAL_TIMEOUT = Duration.ofSeconds(2); + private static final int INVALID_COMMAND_ERROR_CODE = 3; private static final Logger log = LoggerFactory.getLogger(AsyncIggyTcpClient.class); private static final RetryPolicy DEFAULT_RECONNECT_POLICY = RetryPolicy.fixedDelay(12, Duration.ofSeconds(5)); @@ -129,10 +143,38 @@ public class AsyncIggyTcpClient { private final Optional tlsCertificate; private final TcpConnectionPoolConfig poolConfig; private final ClientRoutingState routingState = new ClientRoutingState(); + private final LoginRoutingHook loginRoutingHook = new LoginRoutingHook() { + + @Override + public CompletableFuture loginOnLeader(Supplier> loginAttempt) { + return AsyncIggyTcpClient.this.loginOnLeader(loginAttempt); + } + + @Override + public void forgetLogin() { + rememberedLogin = null; + } + }; private final AtomicReference connection = new AtomicReference<>(); private final AtomicReference> loginChain = new AtomicReference<>(CompletableFuture.completedFuture(null)); private volatile ConnectionInfo connectionInfo; + /** + * Every node the roster named on the last leader check, kept as redial + * candidates. A node dies together with its address, and the roster is + * unreachable exactly when it is needed, so it has to have been + * remembered while the connection was still healthy. + */ + private volatile List rosterTargets = List.of(); + /** + * The login a successful sign-in ran, replayed after a redial so the + * session is re-established on whichever node answers. The supplier + * already carries the credentials it signed in with, so nothing new is + * stored. Cleared on an explicit sign-out, which leaves no session to + * restore. + */ + private volatile Supplier> rememberedLogin; + private volatile boolean closed; private MessagesClient messagesClient; private ConsumerGroupsClient consumerGroupsClient; @@ -235,9 +277,9 @@ public CompletableFuture connect() { consumerOffsetsClient = new ConsumerOffsetsTcpClient(currentConnection); streamsClient = new StreamsTcpClient(currentConnection); topicsClient = new TopicsTcpClient(currentConnection); - usersClient = new UsersTcpClient(currentConnection, this::loginOnLeader); + usersClient = new UsersTcpClient(currentConnection, loginRoutingHook); systemClient = new SystemTcpClient(currentConnection); - personalAccessTokensClient = new PersonalAccessTokensTcpClient(currentConnection, this::loginOnLeader); + personalAccessTokensClient = new PersonalAccessTokensTcpClient(currentConnection, loginRoutingHook); partitionsClient = new PartitionsTcpClient(currentConnection); }); } @@ -427,6 +469,10 @@ public ConsumerOffsetsClient consumerOffsets() { */ public CompletableFuture close() { closed = true; + // Closing is caller intent, like a logout: connect() clears `closed` + // again, and a session the caller ended must not come back with the + // credentials the earlier sign-in used. + rememberedLogin = null; AsyncTcpConnection currentConnection = connection.get(); if (currentConnection != null) { return currentConnection.close(); @@ -452,15 +498,55 @@ private AsyncTcpConnection openConnection(ConnectionInfo target) { enableTls, tlsCertificate, poolConfig, - connectionTimeout, + dialTimeout(), requestTimeout, heartbeatInterval, maxVsrFrameSize, this::retryTransientOnLeader, - routingState::clearAssignments, + this::onSessionReset, this::onConnectionFailure); } + /** + * How long one dial may take. + * + * With other endpoints queued behind this one, a node whose syns are + * dropped must not hold the rotation, so the wait is capped at + * {@link #FAILOVER_DIAL_TIMEOUT} - the bound the other SDKs use. A + * configured connection timeout is capped too rather than exempted: it says + * how long one endpoint may take, and a rotation that spends it on every + * endpoint reaches the survivor long after the caller gave up. A client that + * knows one endpoint keeps whatever it configured, since there is nothing + * queued behind that dial. + */ + Optional dialTimeout() { + if (redialCandidates().size() < 2) { + return connectionTimeout; + } + return Optional.of(connectionTimeout + .filter(configured -> configured.compareTo(FAILOVER_DIAL_TIMEOUT) < 0) + .orElse(FAILOVER_DIAL_TIMEOUT)); + } + + /** + * A server-side eviction reached this client. The routing state it cached + * belonged to the evicted session, so it goes. + * + * The sign-in stays. A stale-client eviction is not caller intent: the + * server's heartbeat verifier sends it after a gc pause or a laptop sleep, + * and a client that signed in by hand has to recover from it exactly like + * one whose credentials were configured. The connection re-authenticates + * the replacement channel from the login it captured, which is the same + * sign-in a redial would replay. Same rule in every SDK; only an explicit + * sign-out or close ends a session for good. + */ + private void onSessionReset(int errorCode) { + routingState.clearAssignments(); + if (errorCode == IggyErrorCode.STALE_CLIENT.getCode()) { + log.debug("The server evicted this session as stale; the next request re-establishes it"); + } + } + /** * A not-accepted request was never admitted, so it is safe to recheck the * leader, restore authentication on a new connection, and retry it within @@ -554,10 +640,12 @@ private static void releasePayload(AtomicReference payload) { * Entry point of the background redial after a pool acquire failure or an * expired reply. Requests that were in flight stay failed (their outcome * is unknown); the redial only restores the client for subsequent calls. - * Alternates the current endpoint with the seed, paced by the configured - * retry policy, and replays the builder credentials on the restored - * connection. Personal-access-token logins cannot be replayed here; those - * clients must log in again themselves. + * Each rotation sweeps every endpoint the client knows -- where it was, + * the configured seed, then the roster it learned -- and only a rotation + * that reaches none of them waits out the retry policy's delay. The + * sign-in is replayed on whichever endpoint answers, whether it was + * configured on the builder or run by the caller, personal access tokens + * included. */ private void onConnectionFailure(Throwable cause) { if (closed || !isConnectionLoss(cause)) { @@ -581,46 +669,153 @@ private CompletableFuture redialAttempt(int attempt, RetryPolicy policy) { if (closed) { return CompletableFuture.completedFuture(null); } - if (attempt > policy.getMaxRetries()) { - log.error("Redial gave up after {} attempts, next request will fail fast", policy.getMaxRetries()); + List candidates = redialCandidates(); + // The retry budget bounds the rotations, not the endpoints. A policy of + // zero retries with several endpoints known still gets one rotation: + // those endpoints - the address the client was configured with, the + // nodes the roster named - were made known in order to be tried, and + // the other SDKs sweep them once too. With one endpoint known, zero + // retries redials nothing, which is what it asked for. + boolean sweepOnce = attempt == 1 && candidates.size() > 1; + if (attempt > policy.getMaxRetries() && !sweepOnce) { + // The rotations actually run, not the configured budget: a policy of + // zero retries still sweeps once when it knows several endpoints. + log.error("Redial gave up after {} rotations, next request will fail fast", attempt - 1); return CompletableFuture.completedFuture(null); } - ConnectionInfo target = ReconnectPlan.target(connectionInfo, seedConnectionInfo, attempt); - Duration delay = ReconnectPlan.delay(policy, attempt); + // The delay paces rotations, not dials. The first rotation runs at once + // when there is somewhere else to go: pausing before dialing a survivor + // only pushes the failover past the window the caller waits in, and the + // node just lost may be gone for good. + Duration delay = attempt == 1 && candidates.size() > 1 ? Duration.ZERO : ReconnectPlan.delay(policy, attempt); Executor delayedExecutor = CompletableFuture.delayedExecutor(delay.toMillis(), TimeUnit.MILLISECONDS); - return CompletableFuture.supplyAsync(() -> null, delayedExecutor).thenCompose(ignored -> { - if (closed) { - return CompletableFuture.completedFuture(null); - } - log.info("Redial attempt {}/{} to {}", attempt, policy.getMaxRetries(), target.serverAddress()); - return retarget(target) - .thenCompose(retargeted -> replayLogin()) - .handle((ok, error) -> { - if (error == null) { - log.info("Reconnected to {}", target.serverAddress()); - return CompletableFuture.completedFuture(null); - } + return CompletableFuture.supplyAsync(() -> null, delayedExecutor) + .thenCompose(ignored -> sweepCandidates(candidates, 0, attempt, policy)); + } + + /** + * Dials one endpoint of a rotation and, if it does not come up, the next + * one. Every endpoint gets its turn inside one attempt, so a full pass over + * the cluster costs one retry rather than one per endpoint: with the + * default policy, rotating one endpoint per attempt would first dial a + * two-node survivor two delays in. + */ + private CompletableFuture sweepCandidates( + List candidates, int index, int attempt, RetryPolicy policy) { + if (closed) { + return CompletableFuture.completedFuture(null); + } + if (index >= candidates.size()) { + return redialAttempt(attempt + 1, policy); + } + ConnectionInfo target = candidates.get(index); + // A policy of zero retries that knows several endpoints still gets the + // one rotation they were made known for, so the budget shown here is + // what will actually run rather than what was configured. + int rotations = Math.max(policy.getMaxRetries(), candidates.size() > 1 ? 1 : 0); + log.info( + "Redial attempt {}/{} to {} ({}/{})", + attempt, + rotations, + target.serverAddress(), + index + 1, + candidates.size()); + return retarget(target) + .handle((retargeted, dialError) -> { + if (dialError != null) { + log.warn("Redial to {} failed: {}", target.serverAddress(), dialError.getMessage()); + return sweepCandidates(candidates, index + 1, attempt, policy); + } + return replaySignInOn(target, candidates, index, attempt, policy); + }) + .thenCompose(Function.identity()); + } + + /** + * Re-establishes the session on an endpoint that just came up. + * + * A sign-in the server rejected -- a rotated password, an expired token -- + * ends the redial: the connection is up, no other endpoint would answer + * differently, and retrying would tear the working connection down on the + * next rotation and leave the client connected but unauthenticated anyway. + * The rejected credentials are dropped so nothing replays them. + */ + private CompletableFuture replaySignInOn( + ConnectionInfo target, List candidates, int index, int attempt, RetryPolicy policy) { + return replayLogin() + .handle((ok, loginError) -> { + if (loginError == null) { + log.info("Reconnected to {}", target.serverAddress()); + return CompletableFuture.completedFuture(null); + } + if (!isSignInRejection(unwrap(loginError))) { log.warn( - "Redial attempt {} to {} failed: {}", - attempt, + "The sign-in on {} did not complete: {}", target.serverAddress(), - error.getMessage()); - return redialAttempt(attempt + 1, policy); - }) - .thenCompose(Function.identity()); - }); + loginError.getMessage()); + return sweepCandidates(candidates, index + 1, attempt, policy); + } + log.error( + "Reconnected to {} but the sign-in was rejected: {}. The connection stands" + + " unauthenticated until the caller signs in again.", + target.serverAddress(), + loginError.getMessage()); + rememberedLogin = null; + return CompletableFuture.completedFuture(null); + }) + .thenCompose(Function.identity()); } /** - * Replays the builder credentials on the freshly published connection. + * Whether the server answered the sign-in with a verdict no other endpoint + * would change: a rotated password, an expired token. + * + * Only that ends a redial. Everything else -- the channel closing before + * the reply, a timeout, a transient refusal from a node that is not the + * primary -- says nothing about the credentials, and treating it as a + * rejection would drop them and leave the client published on a node that + * is already gone, with every later call failing "not authenticated". + */ + static boolean isSignInRejection(Throwable error) { + if (!(error instanceof IggyServerException serverError)) { + return false; + } + int code = serverError.getRawErrorCode(); + return code != AsyncTcpConnection.TRANSIENT_NOT_ACCEPTED && code != AsyncTcpConnection.TRANSIENT_NOT_COMMITTED; + } + + private static Throwable unwrap(Throwable error) { + return error instanceof CompletionException && error.getCause() != null ? error.getCause() : error; + } + + /** + * Re-establishes the session on the freshly published connection with the + * sign-in that last succeeded, falling back to the credentials configured + * on the builder when no login has run yet. + * + * The last sign-in outranks the configured credentials, the same rule as in + * every other SDK: a client is whoever it last signed in as. The connection + * also re-authenticates a replacement channel from the login payload it + * captured, which is that same last sign-in, so replaying a different user + * here would make one eviction land on a different session depending on + * whether the channel or the redial got there first. A client that only ever + * used its configured credentials remembers exactly those, so nothing + * changes for it. + * * The login runs through the users client, so leader discovery retargets * again before Register when the redialed node is not the leader. */ private CompletableFuture replayLogin() { - if (username.isEmpty() || password.isEmpty() || usersClient == null) { - return CompletableFuture.completedFuture(null); + Supplier> replay = rememberedLogin; + if (replay != null) { + // Runs through loginOnLeader, so a redial that landed on a backup + // still settles on the leader before the session is used. + return loginOnLeader(replay).thenApply(identity -> null); + } + if (username.isPresent() && password.isPresent() && usersClient != null) { + return usersClient.login(username.get(), password.get()).thenApply(identity -> null); } - return usersClient.login(username.get(), password.get()).thenApply(identity -> null); + return CompletableFuture.completedFuture(null); } /** @@ -638,6 +833,12 @@ CompletableFuture loginOnLeader(Supplier callerFuture = new CompletableFuture<>(); transaction.whenComplete((identity, error) -> { gate.complete(null); + // Not after a close: a login still in flight when `close()` cleared + // this would set it again, and `connect()` clears `closed`, so the + // next loss would replay a sign-in the caller had ended. + if (error == null && !closed) { + rememberedLogin = loginAttempt; + } if (error != null) { callerFuture.completeExceptionally(error); } else { @@ -732,7 +933,57 @@ CompletableFuture> findLeaderElsewhere(ConnectionInfo c if (currentSystemClient == null) { return CompletableFuture.completedFuture(Optional.empty()); } - return LeaderAwareness.findLeaderElsewhere(currentSystemClient::getClusterMetadata, currentTarget); + return LeaderAwareness.findLeaderElsewhere(currentSystemClient::getClusterMetadata, currentTarget) + .thenApply(lookup -> { + rememberRoster(lookup); + return lookup.redirect(); + }); + } + + /** + * Keeps what a leader check learned about where the cluster's nodes are. + * + * Replaced wholesale rather than merged: the roster is the cluster's own + * answer, so a node it dropped stops being dialed. The configured seed is + * kept separately and outlives it. + * + * An inconclusive check -- an unreadable roster, a metadata read that + * failed -- names no endpoint, and that must leave the last roster + * standing: assigning it anyway would empty the redial candidates exactly + * when the cluster is unreachable, which is when they are needed. + */ + void rememberRoster(LeaderAwareness.LeaderLookup lookup) { + if (!lookup.endpoints().isEmpty()) { + rosterTargets = lookup.endpoints(); + } + } + + /** The roster this client would redial, for tests in this package. */ + List rosterTargets() { + return rosterTargets; + } + + /** Whether a sign-in is remembered for replay, for tests in this package. */ + boolean hasRememberedLogin() { + return rememberedLogin != null; + } + + /** + * Endpoints a redial rotates through, likeliest first: where the client + * currently is, the address it was configured with, then the roster it + * learned while connected. Duplicates are dropped by spelling, so an + * endpoint the roster merely writes differently does not earn a second + * attempt. Spelling only: this runs on the Netty event loop, where a + * resolver lookup per candidate pair would block it. + */ + private List redialCandidates() { + List candidates = new ArrayList<>(); + candidates.add(connectionInfo); + Stream.concat(Stream.of(seedConnectionInfo), rosterTargets.stream()) + .filter(endpoint -> + candidates.stream().noneMatch(candidate -> LeaderAwareness.isSameSpelling(candidate, endpoint))) + .forEach(candidates::add); + return List.copyOf(candidates); } CompletableFuture retarget(ConnectionInfo newTarget) { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java index 102e236fa9..48d70abb4e 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java @@ -36,6 +36,7 @@ import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslHandler; import io.netty.util.concurrent.FutureListener; import io.netty.util.concurrent.ScheduledFuture; import org.apache.iggy.client.async.tcp.vsr.ConsensusSession; @@ -68,16 +69,13 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.IntConsumer; /** * Async TCP connection using Netty for non-blocking I/O. * Manages the connection lifecycle and request/response correlation. */ public class AsyncTcpConnection { - private static final Logger log = LoggerFactory.getLogger(AsyncTcpConnection.class); - private static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofMillis(3000); - // A missing reply must not hold the single VSR-pinned channel forever. - private static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(30); // Transient VSR denials (not-committed / not-accepted) are replayed with // the same encoded frame so the server's dedup sees the same request id. // A not-committed outcome is unknown, so it replays for the whole budget. @@ -85,8 +83,15 @@ public class AsyncTcpConnection { // so after a short same-node retry it is handed to the owning client for // a leader recheck and safe replay; mirrors TRANSIENT_FAILOVER_CHECK_INTERVAL // in core/sdk/src/tcp/tcp_client.rs. - private static final int TRANSIENT_NOT_COMMITTED = 57; - private static final int TRANSIENT_NOT_ACCEPTED = 58; + // + // Package-private: the client classifies a failed sign-in by these codes, + // and a transient one is not a rejected credential. + static final int TRANSIENT_NOT_COMMITTED = 57; + static final int TRANSIENT_NOT_ACCEPTED = 58; + private static final Logger log = LoggerFactory.getLogger(AsyncTcpConnection.class); + private static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofMillis(3000); + // A missing reply must not hold the single VSR-pinned channel forever. + private static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(30); private static final long TRANSIENT_RETRY_INTERVAL_MS = 50; private static final Duration TRANSIENT_RETRY_BUDGET = Duration.ofSeconds(30); private static final Duration NOT_ACCEPTED_RETRY_BUDGET = Duration.ofSeconds(2); @@ -97,7 +102,7 @@ public class AsyncTcpConnection { private final AtomicLong authGeneration = new AtomicLong(0); private final VsrRequestEncoder vsrEncoder; private final TransientFailoverHandler transientFailoverHandler; - private final Runnable sessionResetListener; + private final IntConsumer sessionResetListener; private final Consumer connectionFailureListener; private final long requestTimeoutNanos; private final long heartbeatIntervalNanos; @@ -127,7 +132,7 @@ public AsyncTcpConnection( Duration.ofSeconds(5), VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE, null, - () -> {}, + errorCode -> {}, ignored -> {}); } @@ -143,7 +148,7 @@ public AsyncTcpConnection( Duration heartbeatInterval, int maxVsrFrameSize, TransientFailoverHandler transientFailoverHandler, - Runnable sessionResetListener, + IntConsumer sessionResetListener, Consumer connectionFailureListener) { this.transientFailoverHandler = transientFailoverHandler; this.sessionResetListener = sessionResetListener; @@ -165,12 +170,13 @@ public AsyncTcpConnection( this.vsrEncoder = new VsrRequestEncoder(consensusSession); this.eventLoopGroup = new MultiThreadIoEventLoopGroup(NioIoHandler.newFactory()); + long dialTimeoutMillis = + connectionTimeout.orElse(DEFAULT_CONNECTION_TIMEOUT).toMillis(); var bootstrap = new Bootstrap() .group(eventLoopGroup) .channel(NioSocketChannel.class) .option(ChannelOption.TCP_NODELAY, true) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) - connectionTimeout.orElse(DEFAULT_CONNECTION_TIMEOUT).toMillis()) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) dialTimeoutMillis) .option(ChannelOption.SO_KEEPALIVE, true) .remoteAddress(host, port); @@ -180,7 +186,14 @@ public AsyncTcpConnection( this.channelPool = new FixedChannelPool( bootstrap, new PoolChannelHandler( - host, port, enableTls, sslContext, consensusSession, maxVsrFrameSize, this::onSessionEvicted), + host, + port, + enableTls, + sslContext, + dialTimeoutMillis, + consensusSession, + maxVsrFrameSize, + this::onSessionEvicted), ChannelHealthChecker.ACTIVE, FixedChannelPool.AcquireTimeoutAction.FAIL, poolConfig.getAcquireTimeoutMillis(), @@ -827,10 +840,15 @@ private void handlePostResponse(Channel channel, int commandCode, boolean isLogi * channel. Bumping the generation makes the replacement channel re-run * login and Register. The fresh session invalidates cached routing state * such as consumer-group assignments. + * + * The reason travels to the listener so it can drop what belonged to the + * evicted session and log what happened. The session itself is kept + * whichever way the sign-in was made: only an explicit sign-out or close + * ends one. */ - private void onSessionEvicted() { + private void onSessionEvicted(int errorCode) { authGeneration.incrementAndGet(); - sessionResetListener.run(); + sessionResetListener.accept(errorCode); } private void captureLoginPayloadIfNeeded(int commandCode, ByteBuf payload) { @@ -892,22 +910,26 @@ private static final class PoolChannelHandler extends AbstractChannelPoolHandler private final int port; private final boolean enableTls; private final SslContext sslContext; + private final long dialTimeoutMillis; private final ConsensusSession consensusSession; private final int maxVsrFrameSize; - private final Runnable onEviction; + private final IntConsumer onEviction; + @SuppressWarnings("checkstyle:ParameterNumber") PoolChannelHandler( String host, int port, boolean enableTls, SslContext sslContext, + long dialTimeoutMillis, ConsensusSession consensusSession, int maxVsrFrameSize, - Runnable onEviction) { + IntConsumer onEviction) { this.host = host; this.port = port; this.enableTls = enableTls; this.sslContext = sslContext; + this.dialTimeoutMillis = dialTimeoutMillis; this.consensusSession = consensusSession; this.maxVsrFrameSize = maxVsrFrameSize; this.onEviction = onEviction; @@ -917,7 +939,12 @@ private static final class PoolChannelHandler extends AbstractChannelPoolHandler public void channelCreated(Channel ch) { ChannelPipeline pipeline = ch.pipeline(); if (enableTls) { - pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port)); + SslHandler ssl = sslContext.newHandler(ch.alloc(), host, port); + // A peer that accepts TCP and then never answers the + // ClientHello would otherwise hold the dial for Netty's own + // 10s default, well past the bound the rotation dials under. + ssl.setHandshakeTimeoutMillis(dialTimeoutMillis); + pipeline.addLast("ssl", ssl); } pipeline.addLast("frameDecoder", new VsrFrameDecoder(maxVsrFrameSize)); pipeline.addLast("responseHandler", new VsrResponseHandler(consensusSession, onEviction)); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java index f541e9de7c..c316733a26 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java @@ -72,12 +72,12 @@ private LeaderAwareness() {} * exceptionally, so the redirection path cannot fail the login that * triggered it. */ - static CompletableFuture> findLeaderElsewhere( + static CompletableFuture findLeaderElsewhere( Supplier> fetchMetadata, ConnectionInfo currentTarget) { return findLeaderElsewhere(fetchMetadata, currentTarget, LEADERLESS_WAIT_BUDGET, LEADERLESS_POLL_INTERVAL); } - static CompletableFuture> findLeaderElsewhere( + static CompletableFuture findLeaderElsewhere( Supplier> fetchMetadata, ConnectionInfo currentTarget, Duration leaderlessWaitBudget, @@ -87,7 +87,7 @@ static CompletableFuture> findLeaderElsewhere( fetchMetadata, currentTarget, leaderlessWaitBudget, leaderlessPollInterval, electionDeadlineNanos); } - private static CompletableFuture> pollForLeader( + private static CompletableFuture pollForLeader( Supplier> fetchMetadata, ConnectionInfo currentTarget, Duration leaderlessWaitBudget, @@ -99,16 +99,18 @@ private static CompletableFuture> pollForLeader( } catch (RuntimeException fetchError) { fetched = CompletableFuture.failedFuture(fetchError); } - return fetched.>>handleAsync((metadata, error) -> { + return fetched.>handleAsync((metadata, error) -> { if (error != null) { log.warn( "Failed to get cluster metadata: {}, connection will continue on server node {}", error.getMessage(), currentTarget.serverAddress()); - return CompletableFuture.completedFuture(Optional.empty()); + return CompletableFuture.completedFuture(LeaderLookup.inconclusive()); } LeaderCheck check; + List endpoints; try { + endpoints = nodeTargets(metadata); check = checkLeader(metadata, currentTarget); } catch (RuntimeException selectionError) { log.warn( @@ -116,10 +118,11 @@ private static CompletableFuture> pollForLeader( + " on server node {}", selectionError.getMessage(), currentTarget.serverAddress()); - return CompletableFuture.completedFuture(Optional.empty()); + return CompletableFuture.completedFuture(LeaderLookup.inconclusive()); } if (check instanceof LeaderCheck.Redirect redirect) { - return CompletableFuture.completedFuture(Optional.of(redirect.target())); + return CompletableFuture.completedFuture( + new LeaderLookup(Optional.of(redirect.target()), endpoints)); } if (check instanceof LeaderCheck.NoLeader) { if (System.nanoTime() >= electionDeadlineNanos) { @@ -128,7 +131,9 @@ private static CompletableFuture> pollForLeader( + " continue on server node {}", leaderlessWaitBudget, currentTarget.serverAddress()); - return CompletableFuture.completedFuture(Optional.empty()); + // A leaderless roster still names where the nodes + // are, and that is what a redial needs. + return CompletableFuture.completedFuture(new LeaderLookup(Optional.empty(), endpoints)); } Executor retryAfterInterval = CompletableFuture.delayedExecutor( leaderlessPollInterval.toMillis(), TimeUnit.MILLISECONDS); @@ -142,11 +147,23 @@ private static CompletableFuture> pollForLeader( retryAfterInterval) .thenCompose(Function.identity()); } - return CompletableFuture.completedFuture(Optional.empty()); + return CompletableFuture.completedFuture(new LeaderLookup(Optional.empty(), endpoints)); }) .thenCompose(Function.identity()); } + /** + * Every node's target for the tcp transport, in roster order. A node that + * does not expose the transport reports port 0 and is skipped: dialing it + * would burn a redial attempt on an endpoint that cannot answer. + */ + static List nodeTargets(ClusterMetadata metadata) { + return metadata.nodes().stream() + .filter(node -> node.endpoints().tcp() != 0) + .map(node -> new ConnectionInfo(node.ip(), node.endpoints().tcp())) + .toList(); + } + /** * One leader-check verdict from a cluster-metadata snapshot. */ @@ -200,15 +217,26 @@ static LeaderCheck checkLeader(ClusterMetadata metadata, ConnectionInfo currentT * at worst costs one redirect hop back to the same node. */ static boolean isSameAddress(ConnectionInfo target1, ConnectionInfo target2) { + if (isSameSpelling(target1, target2)) { + return true; + } if (target1.port() != target2.port()) { return false; } - var host1 = canonicalHost(target1.host()); - var host2 = canonicalHost(target2.host()); - if (host1.equals(host2)) { - return true; - } - return resolveToSameHost(host1, host2); + return resolveToSameHost(canonicalHost(target1.host()), canonicalHost(target2.host())); + } + + /** + * Whether two targets are written the same way, up to canonicalization. + * + * The cheap half of {@link #isSameAddress}, for callers that must not + * block: resolution is a synchronous DNS lookup, and the redial dedup runs + * on the Netty event loop. Two spellings of one node that only resolution + * could equate cost one wasted dial per rotation, which is not worth + * stalling an event loop for. + */ + static boolean isSameSpelling(ConnectionInfo target1, ConnectionInfo target2) { + return target1.port() == target2.port() && canonicalHost(target1.host()).equals(canonicalHost(target2.host())); } private static String canonicalHost(String host) { @@ -245,6 +273,24 @@ private static boolean reachesOnlyLocalMachine(InetAddress[] addresses) { return Arrays.stream(addresses).allMatch(address -> address.isLoopbackAddress() || address.isAnyLocalAddress()); } + /** + * What one leader check learned from the roster: where to go, and every + * node the cluster named for this transport. A client keeps the latter as + * redial candidates, because the address it was configured with dies with + * its node and the roster is unreachable exactly when it is needed. + */ + record LeaderLookup(Optional redirect, List endpoints) { + + LeaderLookup { + endpoints = List.copyOf(endpoints); + } + + /** A check that learned nothing: stay put, remember no endpoint. */ + static LeaderLookup inconclusive() { + return new LeaderLookup(Optional.empty(), List.of()); + } + } + /** * One leader-check verdict from a cluster-metadata snapshot. */ diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java index a6acb7dc09..904b492e2a 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java @@ -41,4 +41,10 @@ interface LoginRoutingHook { * @return the identity returned by the successful Register response */ CompletableFuture loginOnLeader(Supplier> loginAttempt); + + /** + * Drops any login kept for replay. Called on an explicit sign-out: there + * is no session left to restore, and a redial must not resurrect one. + */ + default void forgetLogin() {} } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java index ae596731f4..f82ee0c7d8 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java @@ -19,32 +19,17 @@ package org.apache.iggy.client.async.tcp; -import org.apache.iggy.client.ConnectionInfo; import org.apache.iggy.config.RetryPolicy; import java.time.Duration; /** - * Pure redial planning: which address to dial on a given reconnect attempt - * and how long to wait before it. + * Pure redial planning: how long to wait before a given reconnect rotation. */ final class ReconnectPlan { private ReconnectPlan() {} - /** - * Alternates reconnect dials between the current endpoint and the - * configured seed. After a leader redirect the current endpoint may die - * with the leader, and the seed is the way back to the rest of the - * cluster. Attempts are 1-based; odd attempts dial the current endpoint. - */ - static ConnectionInfo target(ConnectionInfo current, ConnectionInfo seed, int attempt) { - if (current.equals(seed)) { - return current; - } - return attempt % 2 == 1 ? current : seed; - } - /** * The delay before the given 1-based attempt: the policy's initial delay * scaled by its multiplier per prior attempt, capped at its max delay. diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java index b6e7ab1940..1bf1e7ceda 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java @@ -190,6 +190,7 @@ public CompletableFuture logout() { return connection().send(CommandCode.User.LOGOUT.getValue(), payload).thenAccept(response -> { response.release(); + routingHook.forgetLogin(); log.debug("Logged out successfully"); }); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java index 113c288e27..16a8f47474 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java @@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.IntConsumer; /** * Correlates multiplexed requests by operation and request id, decodes VSR @@ -54,10 +55,10 @@ public class VsrResponseHandler extends SimpleChannelInboundHandler { private final ConcurrentMap> pendingRequests = new ConcurrentHashMap<>(); private final ConsensusSession session; - private final Runnable onEviction; + private final IntConsumer onEviction; private final AtomicReference closeCause = new AtomicReference<>(); - public VsrResponseHandler(ConsensusSession session, Runnable onEviction) { + public VsrResponseHandler(ConsensusSession session, IntConsumer onEviction) { this.session = session; this.onEviction = onEviction; } @@ -162,7 +163,11 @@ private void handleEviction(ChannelHandlerContext ctx, ByteBuf frame) { IggyServerException error = VsrHeaders.evictionToException(frame); session.reset(); try { - onEviction.run(); + // The reason travels with the notification so the listener can drop + // what belonged to the evicted session and say which eviction it + // was. None of them ends the sign-in: the next request + // re-establishes the session. + onEviction.accept(error.getRawErrorCode()); } catch (RuntimeException listenerError) { log.warn("Eviction listener failed: {}", listenerError.getMessage()); } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java new file mode 100644 index 0000000000..e9a09de1ad --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java @@ -0,0 +1,562 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.iggy.client.ConnectionInfo; +import org.apache.iggy.config.RetryPolicy; +import org.apache.iggy.exception.IggyConnectionException; +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyServerException; +import org.junit.jupiter.api.Test; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The node a client signed in on dies; its next request has to complete on a + * survivor the roster named, under a session established there. Mirrors + * {@code core/integration/tests/cluster/failover_client_continuity.rs}. The + * mock VSR framing matches {@link AsyncIggyTcpClientTransientFailoverTest}, + * kept separate so a death mid-connection cannot disturb that suite's server. + */ +class AsyncIggyTcpClientEndpointFailoverTest { + private static final int HEADER_SIZE = 256; + private static final int SIZE_OFFSET = 48; + private static final int COMMAND_OFFSET = 60; + private static final int REQUEST_ID_OFFSET = 168; + private static final int REQUEST_OPERATION_OFFSET = 176; + private static final int REQUEST_CODE_OFFSET = 196; + private static final int REPLY_REQUEST_ID_OFFSET = 200; + private static final int REPLY_OPERATION_OFFSET = 208; + private static final int REPLY_STATUS_OFFSET = 216; + + private static final int COMMAND_REPLY = 8; + private static final int OPERATION_REGISTER = 1; + private static final int OPERATION_NON_REPLICATED = 2; + private static final int GET_CLUSTER_METADATA_CODE = 12; + private static final int PING_CODE = 1; + + @Test + void shouldResumeOnASurvivorAfterTheSignedInNodeDies() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket primarySocket = new ServerSocket(0, 4, loopback); + ServerSocket survivorSocket = new ServerSocket(0, 4, loopback)) { + int primaryPort = primarySocket.getLocalPort(); + int survivorPort = survivorSocket.getLocalPort(); + AtomicInteger survivorRegistrations = new AtomicInteger(); + AtomicInteger survivorPings = new AtomicInteger(); + List survivorLogins = new CopyOnWriteArrayList<>(); + + // The primary leads, so the sign-in settles there and the roster is + // only remembered -- not acted on -- until the node dies. + MockNode primary = MockNode.serve(primarySocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, primaryPort)); + } + if (request.operation() == OPERATION_REGISTER) { + return Response.success(OPERATION_REGISTER, registerBody(1)); + } + return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER); + }); + MockNode survivor = MockNode.serve(survivorSocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, survivorPort)); + } + if (request.operation() == OPERATION_REGISTER) { + survivorRegistrations.incrementAndGet(); + survivorLogins.add(request.bodyAsText()); + return Response.success(OPERATION_REGISTER, registerBody(2)); + } + if (request.is(PING_CODE, OPERATION_NON_REPLICATED)) { + survivorPings.incrementAndGet(); + } + return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER); + }); + + // Credentials on the builder and a hand-run login for somebody + // else. The redial replays the sign-in that last succeeded, which + // is also what the connection replays from the login it captured + // when the pool swaps a channel: replaying a different user here + // would make the same failure land on a different session depending + // on which path got there first. + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(primaryPort) + .credentials("configured", "configured") + .requestTimeout(Duration.ofSeconds(2)) + // A whole rotation dials every endpoint the client knows, so + // the survivor is reached before this delay is ever spent. + // One endpoint per attempt would need it first. + .retryPolicy(RetryPolicy.fixedDelay(8, Duration.ofSeconds(5))) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.users().login("handrun", "handrun").get(5, TimeUnit.SECONDS); + client.sendBinaryRequest(PING_CODE, new byte[0]).get(5, TimeUnit.SECONDS); + assertThat(client.getConnectionInfo().port()).isEqualTo(primaryPort); + + primary.kill(); + + // The request in flight when the node died is allowed to fail; + // what is not allowed is never completing one, which is what a + // client that only knows the dead endpoint does. + assertThat(resumeWithin(client, Duration.ofSeconds(4))) + .as("the client has to resume on the survivor inside the first rotation") + .isTrue(); + + assertThat(client.getConnectionInfo().port()) + .as("the client moved off the dead endpoint") + .isEqualTo(survivorPort); + assertThat(survivorRegistrations) + .as("the login was replayed on the survivor") + .hasValueGreaterThanOrEqualTo(1); + assertThat(survivorPings) + .as("the request landed on the survivor") + .hasValueGreaterThanOrEqualTo(1); + assertThat(survivorLogins.get(survivorLogins.size() - 1)) + .as("the redial replayed the configured user instead of the last sign-in") + .contains("handrun") + .doesNotContain("configured"); + } finally { + client.close().get(5, TimeUnit.SECONDS); + survivor.close(); + } + } + } + + /** + * An explicit sign-out is caller intent, like a close: the redial after it + * must not sign back in with the credentials that sign-in used. + */ + @Test + void shouldNotResurrectASignedOutSessionOnASurvivor() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket primarySocket = new ServerSocket(0, 4, loopback); + ServerSocket survivorSocket = new ServerSocket(0, 4, loopback)) { + int primaryPort = primarySocket.getLocalPort(); + int survivorPort = survivorSocket.getLocalPort(); + AtomicInteger survivorRegistrations = new AtomicInteger(); + + MockNode primary = MockNode.serve(primarySocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, primaryPort)); + } + if (request.operation() == OPERATION_REGISTER) { + return Response.success(OPERATION_REGISTER, registerBody(1)); + } + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + MockNode survivor = MockNode.serve(survivorSocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, survivorPort)); + } + if (request.operation() == OPERATION_REGISTER) { + survivorRegistrations.incrementAndGet(); + return Response.success(OPERATION_REGISTER, registerBody(2)); + } + // Echoed, so a logout is answered as a logout: the client + // checks the reply's operation against the request's. + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(primaryPort) + .requestTimeout(Duration.ofSeconds(2)) + .retryPolicy(RetryPolicy.fixedDelay(4, Duration.ofMillis(50))) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.users().login("iggy", "iggy").get(5, TimeUnit.SECONDS); + client.users().logout().get(5, TimeUnit.SECONDS); + + primary.kill(); + + // Every attempt is allowed to fail; none of them may register. + resumeWithin(client, Duration.ofSeconds(2)); + + assertThat(survivorRegistrations) + .as("a signed-out client has no session to restore") + .hasValue(0); + } finally { + client.close().get(5, TimeUnit.SECONDS); + survivor.close(); + } + } + } + + /** Retries until one request completes, or the budget runs out. */ + private static boolean resumeWithin(AsyncIggyTcpClient client, Duration budget) throws InterruptedException { + long deadline = System.nanoTime() + budget.toNanos(); + while (System.nanoTime() < deadline) { + try { + client.sendBinaryRequest(PING_CODE, new byte[0]).get(2, TimeUnit.SECONDS); + return true; + } catch (ExecutionException | TimeoutException stillDown) { + Thread.sleep(50); + } + } + return false; + } + + private static ByteBuf registerBody(long session) { + ByteBuf body = Unpooled.buffer(); + body.writeIntLE(0); + body.writeIntLE(1); + body.writeLongLE(session); + body.writeIntLE(11 << 10); + body.writeByte(0); + return body; + } + + private static ByteBuf clusterMetadata(int primaryPort, int survivorPort, int leaderPort) { + ByteBuf body = Unpooled.buffer(); + writeString(body, "test-cluster"); + body.writeIntLE(2); + writeNode(body, "primary", primaryPort, primaryPort == leaderPort); + writeNode(body, "survivor", survivorPort, survivorPort == leaderPort); + return body; + } + + private static void writeNode(ByteBuf body, String name, int port, boolean leader) { + writeString(body, name); + writeString(body, InetAddress.getLoopbackAddress().getHostAddress()); + body.writeShortLE(port); + body.writeShortLE(0); + body.writeShortLE(0); + body.writeShortLE(0); + body.writeByte(leader ? 0 : 1); + body.writeByte(0); + } + + private static void writeString(ByteBuf body, String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + body.writeIntLE(bytes.length); + body.writeBytes(bytes); + } + + /** + * A loopback VSR node that keeps serving every connection it accepts until + * it is killed, which drops the live sockets and stops accepting so a + * redial is refused the way a dead process refuses one. + * + *

Dedicated daemon threads, not {@code CompletableFuture.runAsync}: the + * accept loop and every connection handler block indefinitely, and parking + * them on the common pool starves it on a low-core CI runner (parallelism + * is cores minus one), which stalls the client's own async continuations + * and times the login out before the test does anything. + */ + private static final class MockNode { + private final ServerSocket server; + private final List accepted = new CopyOnWriteArrayList<>(); + private volatile boolean killed; + + private MockNode(ServerSocket server) { + this.server = server; + } + + static MockNode serve(ServerSocket server, RequestHandler handler) { + MockNode node = new MockNode(server); + Thread acceptor = new Thread( + () -> { + while (!node.killed) { + try { + Socket socket = server.accept(); + node.accepted.add(socket); + Thread exchange = new Thread(() -> node.exchange(socket, handler)); + exchange.setDaemon(true); + exchange.start(); + } catch (IOException accepted) { + return; + } + } + }, + "mock-vsr-acceptor-" + server.getLocalPort()); + acceptor.setDaemon(true); + acceptor.start(); + return node; + } + + private void exchange(Socket socket, RequestHandler handler) { + try (socket) { + InputStream input = socket.getInputStream(); + OutputStream output = socket.getOutputStream(); + Request request; + while (!killed && (request = readRequest(input)) != null) { + writeResponse(output, request, handler.handle(request)); + } + } catch (IOException closed) { + // A killed node and a client that went away look the same here. + } + } + + void kill() throws IOException { + killed = true; + for (Socket socket : accepted) { + socket.close(); + } + server.close(); + } + + void close() throws IOException { + kill(); + } + } + + private static Request readRequest(InputStream input) throws IOException { + byte[] header = input.readNBytes(HEADER_SIZE); + if (header.length == 0) { + return null; + } + if (header.length != HEADER_SIZE) { + throw new EOFException("Truncated VSR request header"); + } + ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + int size = fields.getInt(SIZE_OFFSET); + byte[] body = input.readNBytes(size - HEADER_SIZE); + if (body.length != size - HEADER_SIZE) { + throw new EOFException("Truncated VSR request body"); + } + return new Request( + Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), + fields.getInt(REQUEST_CODE_OFFSET), + fields.getLong(REQUEST_ID_OFFSET), + body); + } + + private static void writeResponse(OutputStream output, Request request, Response response) throws IOException { + byte[] body = new byte[response.body().readableBytes()]; + response.body().readBytes(body); + response.body().release(); + byte[] header = new byte[HEADER_SIZE]; + ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + fields.putInt(SIZE_OFFSET, HEADER_SIZE + body.length); + header[COMMAND_OFFSET] = (byte) COMMAND_REPLY; + fields.putLong(REPLY_REQUEST_ID_OFFSET, request.requestId()); + header[REPLY_OPERATION_OFFSET] = (byte) response.operation(); + fields.putInt(REPLY_STATUS_OFFSET, 0); + output.write(header); + output.write(body); + output.flush(); + } + + private record Request(int operation, int commandCode, long requestId, byte[] body) { + boolean is(int expectedCode, int expectedOperation) { + return commandCode == expectedCode && operation == expectedOperation; + } + + /** The request body as text, for asserting which user a login names. */ + String bodyAsText() { + return new String(body, StandardCharsets.UTF_8); + } + } + + private record Response(int operation, ByteBuf body) { + static Response success(int operation, ByteBuf body) { + return new Response(operation, body); + } + } + + @FunctionalInterface + private interface RequestHandler { + Response handle(Request request); + } + + /** + * A retry budget of zero still gets one rotation when several endpoints are + * known: those endpoints -- the address the client was configured with, the + * nodes the roster named -- were made known in order to be tried, and every + * other SDK sweeps them once too. With one endpoint known, zero retries + * redials nothing, which is what it asked for. + */ + @Test + void shouldSweepTheKnownEndpointsOnceWithNoRetries() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket primarySocket = new ServerSocket(0, 4, loopback); + ServerSocket survivorSocket = new ServerSocket(0, 4, loopback)) { + int primaryPort = primarySocket.getLocalPort(); + int survivorPort = survivorSocket.getLocalPort(); + AtomicInteger survivorRegistrations = new AtomicInteger(); + + MockNode primary = MockNode.serve(primarySocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, primaryPort)); + } + if (request.operation() == OPERATION_REGISTER) { + return Response.success(OPERATION_REGISTER, registerBody(1)); + } + return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER); + }); + MockNode survivor = MockNode.serve(survivorSocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, survivorPort)); + } + if (request.operation() == OPERATION_REGISTER) { + survivorRegistrations.incrementAndGet(); + return Response.success(OPERATION_REGISTER, registerBody(2)); + } + return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(primaryPort) + .credentials("iggy", "iggy") + .requestTimeout(Duration.ofSeconds(2)) + .retryPolicy(RetryPolicy.noRetry()) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.login().get(5, TimeUnit.SECONDS); + client.sendBinaryRequest(PING_CODE, new byte[0]).get(5, TimeUnit.SECONDS); + + primary.kill(); + + assertThat(resumeWithin(client, Duration.ofSeconds(4))) + .as("zero retries skipped the one rotation the known endpoints are for") + .isTrue(); + assertThat(client.getConnectionInfo().port()).isEqualTo(survivorPort); + assertThat(survivorRegistrations) + .as("the sign-in was replayed on the survivor") + .hasValueGreaterThanOrEqualTo(1); + } finally { + client.close().get(5, TimeUnit.SECONDS); + survivor.close(); + } + } + } + + /** + * Every dial is capped once more than one endpoint is known, a configured + * connection timeout included: it says how long one endpoint may take, and a + * rotation that spends it on each of them reaches the survivor long after + * the caller gave up. A timeout shorter than the cap is what the caller + * asked for and stands; with one endpoint known nothing is queued behind + * the dial, so the configured value stands there too. + */ + @Test + void shouldCapEveryDialWhenMoreThanOneEndpointIsKnown() { + InetAddress loopback = InetAddress.getLoopbackAddress(); + List roster = List.of(new ConnectionInfo("iggy-1", 8091)); + + AsyncIggyTcpClient patient = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(8090) + .connectionTimeout(Duration.ofSeconds(30)) + .build(); + assertThat(patient.dialTimeout()).contains(Duration.ofSeconds(30)); + patient.rememberRoster(new LeaderAwareness.LeaderLookup(Optional.empty(), roster)); + assertThat(patient.dialTimeout()) + .as("a configured timeout exempted the dial from the failover cap") + .contains(AsyncIggyTcpClient.FAILOVER_DIAL_TIMEOUT); + + AsyncIggyTcpClient impatient = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(8090) + .connectionTimeout(Duration.ofMillis(500)) + .build(); + impatient.rememberRoster(new LeaderAwareness.LeaderLookup(Optional.empty(), roster)); + assertThat(impatient.dialTimeout()) + .as("a timeout shorter than the cap is what the caller asked for") + .contains(Duration.ofMillis(500)); + + AsyncIggyTcpClient unconfigured = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(8090) + .build(); + assertThat(unconfigured.dialTimeout()).isEmpty(); + unconfigured.rememberRoster(new LeaderAwareness.LeaderLookup(Optional.empty(), roster)); + assertThat(unconfigured.dialTimeout()).contains(AsyncIggyTcpClient.FAILOVER_DIAL_TIMEOUT); + } + + /** + * A roster read that learned nothing must leave the last one standing: + * emptying the redial candidates when the cluster is unreachable takes them + * away exactly when they are needed. + */ + @Test + void shouldKeepTheLastRosterWhenALookupLearnedNothing() { + InetAddress loopback = InetAddress.getLoopbackAddress(); + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(8090) + .build(); + List roster = List.of(new ConnectionInfo("iggy-0", 8091), new ConnectionInfo("iggy-1", 8092)); + + client.rememberRoster(new LeaderAwareness.LeaderLookup(Optional.empty(), roster)); + assertThat(client.rosterTargets()).isEqualTo(roster); + + client.rememberRoster(LeaderAwareness.LeaderLookup.inconclusive()); + assertThat(client.rosterTargets()) + .as("an inconclusive check erased the endpoints the client still needs") + .isEqualTo(roster); + } + + /** + * Only the server answering "no" ends a redial. A channel that closed + * before the reply, or a node that refuses because it is not the primary, + * says nothing about the credentials -- treated as a rejection, they are + * dropped and the client is published on a node that is already gone. + */ + @Test + void shouldTreatOnlyANonTransientServerVerdictAsASignInRejection() { + assertThat(AsyncIggyTcpClient.isSignInRejection( + IggyServerException.fromTcpResponse(IggyErrorCode.INVALID_CREDENTIALS.getCode(), new byte[0]))) + .isTrue(); + assertThat(AsyncIggyTcpClient.isSignInRejection(new IggyConnectionException("channel closed"))) + .as("a channel that died mid sign-in is not a rejected credential") + .isFalse(); + assertThat(AsyncIggyTcpClient.isSignInRejection(new IOException("connection reset"))) + .isFalse(); + assertThat(AsyncIggyTcpClient.isSignInRejection( + IggyServerException.fromTcpResponse(AsyncTcpConnection.TRANSIENT_NOT_ACCEPTED, new byte[0]))) + .as("a node that is not the primary refuses transiently; another endpoint answers") + .isFalse(); + assertThat(AsyncIggyTcpClient.isSignInRejection( + IggyServerException.fromTcpResponse(AsyncTcpConnection.TRANSIENT_NOT_COMMITTED, new byte[0]))) + .isFalse(); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java index ba481bec98..897c5e476e 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java @@ -35,8 +35,11 @@ import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import static org.assertj.core.api.Assertions.assertThat; @@ -159,6 +162,190 @@ void shouldReplayTransientImplicitLoginAfterEviction() throws Exception { } } + /** + * Closing is caller intent, like a logout. A sign-in still in flight when + * it happens must not put its credentials back: `connect()` clears the + * closed flag, so the next connection loss would replay a session the + * caller had ended. + */ + @Test + void shouldNotRememberASignInThatLandedAfterClose() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { + AtomicInteger registrations = new AtomicInteger(); + CompletableFuture server = serve(serverSocket, 4, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); + } + if (request.operation() == OPERATION_REGISTER) { + int attempt = registrations.incrementAndGet(); + if (attempt > 1) { + // The second sign-in is the one racing the close. + try { + Thread.sleep(300); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + return Response.success(OPERATION_REGISTER, registerBody(attempt)); + } + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(serverSocket.getLocalPort()) + .requestTimeout(Duration.ofSeconds(5)) + .build(); + client.connect().get(5, TimeUnit.SECONDS); + client.users().login("iggy", "iggy").get(5, TimeUnit.SECONDS); + + CompletableFuture racing = client.users().login("iggy", "iggy"); + Thread.sleep(50); + client.close().get(5, TimeUnit.SECONDS); + racing.handle((ignored, error) -> null).get(5, TimeUnit.SECONDS); + + assertThat(client.hasRememberedLogin()) + .as("a sign-in that landed after the close put its credentials back") + .isFalse(); + server.completeExceptionally(new IllegalStateException("test over")); + } + } + + /** + * A stale-client eviction is not caller intent: the server's heartbeat + * verifier sends it after a gc pause or a laptop sleep. A client that + * signed in by hand recovers from it exactly like one whose credentials + * were configured (the test above), and the sign-in it recovers with is the + * one that last succeeded. Same rule in every SDK. + */ + @Test + void shouldReviveTheSignInAfterAStaleClientEviction() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { + AtomicInteger registrations = new AtomicInteger(); + List registeredLogins = new CopyOnWriteArrayList<>(); + AtomicBoolean evict = new AtomicBoolean(true); + CompletableFuture server = serve(serverSocket, 6, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); + } + if (request.operation() == OPERATION_REGISTER) { + registeredLogins.add(request.bodyAsText()); + return Response.success(OPERATION_REGISTER, registerBody(registrations.incrementAndGet())); + } + if (request.operation() == OPERATION_CREATE_STREAM) { + if (evict.compareAndSet(true, false)) { + return Response.eviction(EVICTION_STALE_CLIENT); + } + ByteBuf body = Unpooled.buffer(Integer.BYTES); + body.writeIntLE(0); + return Response.success(OPERATION_CREATE_STREAM, body); + } + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + + // No configured credentials: the only sign-in is the one run below. + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(serverSocket.getLocalPort()) + .requestTimeout(Duration.ofSeconds(2)) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.users().login("handrun", "handrun").get(5, TimeUnit.SECONDS); + int registrationsBeforeEviction = registrations.get(); + + assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .hasCauseInstanceOf(IggyServerException.class); + + assertThat(client.hasRememberedLogin()) + .as("an eviction is not a sign-out; the credentials stay") + .isTrue(); + + // The next request brings the session back, under the sign-in + // that last succeeded. + assertThat(client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .isEmpty(); + assertThat(registrations.get()) + .as("the evicted session was not re-established") + .isGreaterThan(registrationsBeforeEviction); + assertThat(registeredLogins.get(registeredLogins.size() - 1)).contains("handrun"); + } finally { + client.close().get(5, TimeUnit.SECONDS); + } + server.completeExceptionally(new IllegalStateException("test over")); + } + } + + /** + * Credentials on the builder and a hand-run sign-in for somebody else: the + * revived session is the last sign-in, the same rule as on a redial and in + * every other SDK. The connection re-authenticates a replacement channel + * from the login it captured, which is that same sign-in, so replaying the + * configured user here would make one eviction land on a different session + * depending on which path got there first. + */ + @Test + void shouldReviveTheLastSignInRatherThanTheConfiguredOne() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { + AtomicInteger registrations = new AtomicInteger(); + List registeredLogins = new CopyOnWriteArrayList<>(); + AtomicBoolean evict = new AtomicBoolean(true); + CompletableFuture server = serve(serverSocket, 6, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); + } + if (request.operation() == OPERATION_REGISTER) { + registeredLogins.add(request.bodyAsText()); + return Response.success(OPERATION_REGISTER, registerBody(registrations.incrementAndGet())); + } + if (request.operation() == OPERATION_CREATE_STREAM) { + if (evict.compareAndSet(true, false)) { + return Response.eviction(EVICTION_STALE_CLIENT); + } + ByteBuf body = Unpooled.buffer(Integer.BYTES); + body.writeIntLE(0); + return Response.success(OPERATION_CREATE_STREAM, body); + } + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(serverSocket.getLocalPort()) + .credentials("configured", "configured") + .requestTimeout(Duration.ofSeconds(2)) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.users().login("handrun", "handrun").get(5, TimeUnit.SECONDS); + int registrationsBeforeEviction = registrations.get(); + + assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .hasCauseInstanceOf(IggyServerException.class); + + assertThat(client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .isEmpty(); + assertThat(registrations.get()) + .as("the evicted session was not re-established") + .isGreaterThan(registrationsBeforeEviction); + assertThat(registeredLogins.get(registeredLogins.size() - 1)) + .as("the revived session signed in as the configured user") + .contains("handrun") + .doesNotContain("configured"); + } finally { + client.close().get(5, TimeUnit.SECONDS); + } + server.completeExceptionally(new IllegalStateException("test over")); + } + } + private static Response handleOldLeader( Request request, int oldLeaderPort, int newLeaderPort, AtomicInteger denials) { if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { @@ -235,7 +422,8 @@ private static Request readRequest(InputStream input) throws IOException { return new Request( Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), fields.getInt(REQUEST_CODE_OFFSET), - fields.getLong(REQUEST_ID_OFFSET)); + fields.getLong(REQUEST_ID_OFFSET), + body); } private static void writeResponse(OutputStream output, Request request, Response response) throws IOException { @@ -310,10 +498,15 @@ private static void writeString(ByteBuf body, String value) { body.writeBytes(bytes); } - private record Request(int operation, int commandCode, long requestId) { + private record Request(int operation, int commandCode, long requestId, byte[] body) { boolean is(int expectedCode, int expectedOperation) { return commandCode == expectedCode && operation == expectedOperation; } + + /** The request body as text, for asserting which user a login names. */ + String bodyAsText() { + return new String(body, StandardCharsets.UTF_8); + } } private record Response(int command, int operation, int status, int evictionReason, ByteBuf body) { diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java index 409e831474..354f09d95b 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java @@ -111,7 +111,7 @@ void shouldCorrelateConcurrentPartitionResponsesInReverseOrder() throws Exceptio Duration.ofHours(1), 1024 * 1024, null, - () -> {}, + errorCode -> {}, ignored -> {}); try { connection.connect().get(5, TimeUnit.SECONDS); @@ -308,7 +308,7 @@ private static AsyncTcpConnection newConnection( heartbeatInterval, 1024 * 1024, null, - () -> {}, + errorCode -> {}, ignored -> {}); } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java index 00540cd23c..a847acb1c7 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java @@ -206,6 +206,10 @@ class FindLeaderElsewhere { private final ConnectionInfo currentTarget = new ConnectionInfo("iggy-follower", 8092); private Optional findLeader(Supplier> fetch) { + return lookUpLeader(fetch).redirect(); + } + + private LeaderAwareness.LeaderLookup lookUpLeader(Supplier> fetch) { return LeaderAwareness.findLeaderElsewhere(fetch, currentTarget, BUDGET, INTERVAL) .orTimeout(30, TimeUnit.SECONDS) .join(); @@ -248,7 +252,8 @@ void shouldGiveUpOnLeaderlessClusterAfterBudget() { Duration.ofMillis(100), INTERVAL) .orTimeout(30, TimeUnit.SECONDS) - .join(); + .join() + .redirect(); assertThat(leader).isEmpty(); assertThat(fetchCount.get()).isGreaterThan(1); @@ -276,6 +281,22 @@ void shouldGiveUpWhenMetadataFetchThrowsSynchronously() { assertThat(leader).isEmpty(); } + @Test + void shouldRememberEveryNodeTheRosterNamesEvenWhileLeaderless() { + var lookup = LeaderAwareness.findLeaderElsewhere( + () -> CompletableFuture.completedFuture(leaderlessCluster()), + currentTarget, + Duration.ofMillis(100), + INTERVAL) + .orTimeout(30, TimeUnit.SECONDS) + .join(); + + // A leaderless roster still names where the nodes are, and that is + // what a redial needs. + assertThat(lookup.redirect()).isEmpty(); + assertThat(lookup.endpoints()).isNotEmpty(); + } + @Test void shouldStayWithoutPollingWhenAlreadyOnLeader() { var fetchCount = new AtomicInteger(); @@ -289,13 +310,87 @@ void shouldStayWithoutPollingWhenAlreadyOnLeader() { BUDGET, INTERVAL) .orTimeout(30, TimeUnit.SECONDS) - .join(); + .join() + .redirect(); assertThat(leader).isEmpty(); assertThat(fetchCount).hasValue(1); } } + @Nested + class NodeTargets { + + // A node with the tcp transport disabled cannot be dialed over tcp, so + // it must not join the redial candidates: dialing port 0 fails, and it + // would spend one turn of every rotation. + @Test + void shouldSkipNodesWithoutATcpEndpoint() { + var metadata = cluster( + node("tcp-node", "iggy-0", 8091, ClusterNodeRole.Leader, ClusterNodeStatus.Healthy), + node("http-only-node", "iggy-1", 0, ClusterNodeRole.Follower, ClusterNodeStatus.Healthy)); + + var targets = LeaderAwareness.nodeTargets(metadata); + + assertThat(targets).containsExactly(new ConnectionInfo("iggy-0", 8091)); + } + + // Unhealthy nodes stay: a node that is down now is where the cluster + // says it lives, and a redial candidate is a place to try, not a + // promise that it answers. + @Test + void shouldKeepUnhealthyNodesThatStillHaveATcpEndpoint() { + var metadata = cluster( + node("leader-node", "iggy-0", 8091, ClusterNodeRole.Leader, ClusterNodeStatus.Healthy), + node("down-node", "iggy-1", 8092, ClusterNodeRole.Follower, ClusterNodeStatus.Unreachable)); + + var targets = LeaderAwareness.nodeTargets(metadata); + + assertThat(targets).containsExactly(new ConnectionInfo("iggy-0", 8091), new ConnectionInfo("iggy-1", 8092)); + } + + // An inconclusive check names no endpoint, which is what lets the + // client keep the last roster it read: replacing it with an empty list + // would erase the candidates exactly when the cluster is unreachable. + @Test + void shouldNameNoEndpointWhenTheRosterCannotBeRead() { + var lookup = LeaderAwareness.LeaderLookup.inconclusive(); + + assertThat(lookup.redirect()).isEmpty(); + assertThat(lookup.endpoints()).isEmpty(); + } + } + + @Nested + class SameAddress { + + // The redial dedup runs on the Netty event loop, so it compares + // spellings only. A hostname and the address it resolves to are two + // candidates there, and one endpoint for the leader check. + @Test + void shouldCompareSpellingsWithoutResolving() { + assertThat(LeaderAwareness.isSameSpelling( + new ConnectionInfo("IGGY-0", 8090), new ConnectionInfo("iggy-0", 8090))) + .isTrue(); + assertThat(LeaderAwareness.isSameSpelling( + new ConnectionInfo("[::1]", 8090), new ConnectionInfo("::1", 8090))) + .isTrue(); + assertThat(LeaderAwareness.isSameSpelling( + new ConnectionInfo("localhost", 8090), new ConnectionInfo("127.0.0.1", 8090))) + .isFalse(); + assertThat(LeaderAwareness.isSameSpelling( + new ConnectionInfo("iggy-0", 8090), new ConnectionInfo("iggy-0", 8091))) + .isFalse(); + } + + @Test + void shouldTreatLoopbackSpellingsAsOneEndpointForTheLeaderCheck() { + assertThat(LeaderAwareness.isSameAddress( + new ConnectionInfo("localhost", 8090), new ConnectionInfo("127.0.0.1", 8090))) + .isTrue(); + } + } + @Nested class RedirectionState { diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java index ce833a58cb..0c45587b77 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java @@ -19,7 +19,6 @@ package org.apache.iggy.client.async.tcp; -import org.apache.iggy.client.ConnectionInfo; import org.apache.iggy.config.RetryPolicy; import org.junit.jupiter.api.Test; @@ -29,23 +28,6 @@ class ReconnectPlanTest { - private final ConnectionInfo seed = new ConnectionInfo("seed-node", 8090); - private final ConnectionInfo current = new ConnectionInfo("leader-node", 8090); - - @Test - void shouldAlternateBetweenCurrentAndSeed() { - assertThat(ReconnectPlan.target(current, seed, 1)).isEqualTo(current); - assertThat(ReconnectPlan.target(current, seed, 2)).isEqualTo(seed); - assertThat(ReconnectPlan.target(current, seed, 3)).isEqualTo(current); - assertThat(ReconnectPlan.target(current, seed, 4)).isEqualTo(seed); - } - - @Test - void shouldDialOnlyOneAddressWhenNeverRedirected() { - assertThat(ReconnectPlan.target(seed, seed, 1)).isEqualTo(seed); - assertThat(ReconnectPlan.target(seed, seed, 2)).isEqualTo(seed); - } - @Test void shouldKeepFixedDelayConstant() { var policy = RetryPolicy.fixedDelay(12, Duration.ofSeconds(5)); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java index e8069b9159..4a9e3bc2d6 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java @@ -39,7 +39,11 @@ class VsrResponseHandlerTest { private final ConsensusSession session = new ConsensusSession(); private final AtomicInteger evictions = new AtomicInteger(); - private final VsrResponseHandler handler = new VsrResponseHandler(session, evictions::incrementAndGet); + private final AtomicInteger lastEvictionReason = new AtomicInteger(); + private final VsrResponseHandler handler = new VsrResponseHandler(session, errorCode -> { + evictions.incrementAndGet(); + lastEvictionReason.set(errorCode); + }); private final EmbeddedChannel channel = new EmbeddedChannel(handler); @AfterEach @@ -159,6 +163,9 @@ void shouldMapEvictionReasonAndResetSession() { assertThat(rawErrorCode(future)).isEqualTo(42); assertThat(session.isBound()).isFalse(); assertThat(evictions).hasValue(1); + // The reason reaches the listener, which has to tell an eviction the + // server decided on from a transport-shaped one. + assertThat(lastEvictionReason).hasValue(42); } @Test @@ -171,6 +178,7 @@ void shouldCloseChannelOnEvictionWithoutPendingRequest() { assertThat(session.isBound()).isFalse(); assertThat(evictions).hasValue(1); + assertThat(lastEvictionReason).hasValue(VsrHeaders.ERROR_STALE_CLIENT); assertThat(frame.refCnt()).isZero(); assertThat(channel.isActive()).isFalse(); } diff --git a/foreign/node/src/client/client.connection.test.ts b/foreign/node/src/client/client.connection.test.ts index ed1f6428fa..b1269d19d0 100644 --- a/foreign/node/src/client/client.connection.test.ts +++ b/foreign/node/src/client/client.connection.test.ts @@ -393,6 +393,365 @@ describe('IggyConnection', () => { } ); + it('rotates a redial through the roster it learned while connected', + async () => { + const seed = await startServer(); + const seedPort = (seed.address() as AddressInfo).port; + const connection = new IggyConnection(connectionConfig(seed)); + connection.on('error', () => undefined); + try { + connection.rememberRoster([ + { host: '127.0.0.1', port: seedPort }, + { host: '127.0.0.1', port: seedPort + 1 }, + { host: '127.0.0.1', port: seedPort + 2 } + ]); + // The endpoint the client is on leads, the roster follows, and the + // roster's copy of that endpoint does not earn a second attempt. + assert.deepEqual( + connection._redialCandidates().map((options) => options.port), + [seedPort, seedPort + 1, seedPort + 2] + ); + } finally { + connection._destroy(); + await new Promise((resolve) => seed.close(() => resolve())); + } + } + ); + + it('dials the endpoint it is on, then the seed, then the roster', + async () => { + const seed = await startServer(); + const seedPort = (seed.address() as AddressInfo).port; + const connection = new IggyConnection(connectionConfig(seed)); + connection.on('error', () => undefined); + try { + // A redirect moves the client off its seed; the seed is still the one + // endpoint the caller vouched for, so it comes before a roster the + // cluster may have reshaped since. + connection.config.options = { + ...connection.config.options, + port: seedPort + 9 + }; + connection.rememberRoster([{ host: '127.0.0.1', port: seedPort + 5 }]); + + assert.deepEqual( + connection._redialCandidates().map((options) => options.port), + [seedPort + 9, seedPort, seedPort + 5] + ); + } finally { + connection._destroy(); + await new Promise((resolve) => seed.close(() => resolve())); + } + } + ); + + it('counts endpoints that only differ in spelling once', + async () => { + const seed = await startServer(); + const seedPort = (seed.address() as AddressInfo).port; + const connection = new IggyConnection(connectionConfig(seed)); + connection.on('error', () => undefined); + try { + // The loopback aliases and an IPv4-mapped address all name the endpoint + // the client is already on, so none of them earns a dial of its own. + connection.rememberRoster([ + { host: 'localhost', port: seedPort }, + { host: '::1', port: seedPort }, + { host: '::ffff:127.0.0.1', port: seedPort }, + { host: '127.0.0.1', port: seedPort + 1 } + ]); + + assert.deepEqual( + connection._redialCandidates().map((options) => options.port), + [seedPort, seedPort + 1] + ); + } finally { + connection._destroy(); + await new Promise((resolve) => seed.close(() => resolve())); + } + } + ); + + it('does not redial at all when reconnection is disabled', + async () => { + // `enabled: false` is what a caller says to opt out. The retry budget is + // whatever the defaults hold, so a loop that reads it without checking + // this flag would run every one of those passes - and with the backoff + // gated on the same flag, back to back. + // + // The endpoint accepts and hangs up, so the drop that would start a + // redial happens and every dial of it is counted. + const hangup = await startServer(); + const hangupPort = (hangup.address() as AddressInfo).port; + let accepted = 0; + hangup.on('connection', (socket) => { + accepted += 1; + socket.destroy(); + }); + + const connection = new IggyConnection({ + transport: 'TCP', + options: { host: '127.0.0.1', port: hangupPort }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: false, interval: 10, maxRetries: 12 }, + maxResponseFrameSize: FRAME_LIMIT + }); + connection.on('error', () => undefined); + try { + await connection.connect().catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 300)); + + assert.equal(accepted, 1, + 'a client that turned reconnection off redialed anyway' + ); + assert.equal(connection.connected, false); + } finally { + connection._destroy(); + await new Promise((resolve) => hangup.close(() => resolve())); + } + } + ); + + it('sweeps the endpoints it knows once when reconnection is disabled', + async () => { + // Opting out of retries is not opting out of the endpoints: with more + // than one known, they get exactly one pass and no backoff, as in the + // other SDKs. + const dead = await startServer(); + const deadPort = (dead.address() as AddressInfo).port; + await new Promise((resolve) => dead.close(() => resolve())); + const live = await startServer(); + const livePort = (live.address() as AddressInfo).port; + let accepted = 0; + live.on('connection', () => { accepted += 1; }); + + const connection = new IggyConnection({ + transport: 'TCP', + options: { host: '127.0.0.1', port: deadPort }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: false, interval: 10, maxRetries: 12 }, + maxResponseFrameSize: FRAME_LIMIT + }); + connection.on('error', () => undefined); + try { + connection.rememberRoster([{ host: '127.0.0.1', port: livePort }]); + await connection.connect().catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 200)); + + assert.equal(accepted, 1, + 'the known endpoints got either no pass or more than one' + ); + } finally { + connection._destroy(); + await new Promise((resolve) => live.close(() => resolve())); + } + } + ); + + it('makes one pass when every endpoint is down and reconnection is disabled', + async () => { + // One pass, not the whole retry budget: with the budget read but the + // flag ignored, a client that opted out of retries dials every endpoint + // once per pass for all of them -- and with the backoff gated on the same + // flag, back to back. + // + // Plain TCP behind a TLS client, closed at once: the dial fails, so the + // pass moves on, and every dial is counted where it lands. + const first = await startServer(); + const firstPort = (first.address() as AddressInfo).port; + let firstDials = 0; + first.on('connection', (socket) => { + firstDials += 1; + socket.destroy(); + }); + const second = await startServer(); + const secondPort = (second.address() as AddressInfo).port; + let secondDials = 0; + second.on('connection', (socket) => { + secondDials += 1; + socket.destroy(); + }); + + const connection = new IggyConnection({ + transport: 'TLS', + options: { + host: '127.0.0.1', + port: firstPort, + rejectUnauthorized: false + }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: false, interval: 10, maxRetries: 12 }, + maxResponseFrameSize: FRAME_LIMIT + }); + connection.on('error', () => undefined); + try { + connection.rememberRoster([{ host: '127.0.0.1', port: secondPort }]); + await connection.connect().catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 300)); + + assert.equal(connection.connected, false); + // The connect's own dial of the configured endpoint, then one pass over + // both: the endpoint the client starts on is dialed twice, the one + // behind it once. + assert.deepEqual([firstDials, secondDials], [2, 1], + 'a client that opted out of retries swept more than once' + ); + } finally { + connection._destroy(); + await new Promise((resolve) => first.close(() => resolve())); + await new Promise((resolve) => second.close(() => resolve())); + } + } + ); + + it('skips the first backoff when another endpoint is known', + async () => { + // The endpoint the client is on is dead and a live one sits behind it in + // the roster: waiting out the interval before the first pass would push + // the failover past what the caller waits for, and the node just lost may + // be gone for good. + const dead = await startServer(); + const deadPort = (dead.address() as AddressInfo).port; + await new Promise((resolve) => dead.close(() => resolve())); + const live = await startServer(); + const livePort = (live.address() as AddressInfo).port; + + const interval = 3000; + const connection = new IggyConnection({ + transport: 'TCP', + options: { host: '127.0.0.1', port: deadPort }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: true, interval, maxRetries: 3 }, + maxResponseFrameSize: FRAME_LIMIT + }); + connection.on('error', () => undefined); + try { + connection.rememberRoster([{ host: '127.0.0.1', port: livePort }]); + const dialed = once(live, 'connection'); + const started = Date.now(); + void connection.connect().catch(() => undefined); + await dialed; + + assert.ok(Date.now() - started < interval, + 'the failover waited out the backoff before its first pass' + ); + } finally { + connection._destroy(); + await new Promise((resolve) => live.close(() => resolve())); + } + } + ); + + it('bounds a dial that never becomes usable when others are queued behind it', + async () => { + // Plain TCP behind a TLS client: the socket connects, so only a bound on + // the handshake ends the attempt. The endpoint behind it is dead, so the + // pass has to end on its own rather than hang on the first one. + const silent = await startServer(); + const silentPort = (silent.address() as AddressInfo).port; + const held: Socket[] = []; + silent.on('connection', (socket) => { held.push(socket); }); + const dead = await startServer(); + const deadPort = (dead.address() as AddressInfo).port; + await new Promise((resolve) => dead.close(() => resolve())); + + const connection = new IggyConnection({ + transport: 'TLS', + options: { + host: '127.0.0.1', + port: silentPort, + rejectUnauthorized: false + }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: true, interval: 10, maxRetries: 3 }, + maxResponseFrameSize: FRAME_LIMIT + }); + connection.on('error', () => undefined); + try { + connection.rememberRoster([{ host: '127.0.0.1', port: deadPort }]); + void connection.connect().catch(() => undefined); + + // Unbounded, the first dial never ends and this endpoint is dialed + // exactly once, forever. + const deadline = Date.now() + 8_000; + while (held.length < 2 && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 50)); + + assert.ok(held.length >= 2, + 'a dial that never became usable held the pass' + ); + assert.equal(connection.connected, false); + } finally { + connection._destroy(); + held.forEach((socket) => socket.destroy()); + await new Promise((resolve) => silent.close(() => resolve())); + } + } + ); + + it('stops a redial pass that is destroyed part-way through', + async () => { + // The endpoint the client is on is dead, so every dial to it is refused + // - and the live roster endpoint behind it is what the pass would reach + // next, unless the destroy in between stops the pass. + const dead = await startServer(); + const deadPort = (dead.address() as AddressInfo).port; + await new Promise((resolve) => dead.close(() => resolve())); + const live = await startServer(); + const livePort = (live.address() as AddressInfo).port; + let accepted = 0; + live.on('connection', () => { accepted += 1; }); + + const connection = new IggyConnection({ + transport: 'TCP', + options: { host: '127.0.0.1', port: deadPort }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: true, interval: 10, maxRetries: 3 }, + maxResponseFrameSize: FRAME_LIMIT + }); + let destroyed = false; + let connectsAfterDestroy = 0; + connection.on('connect', () => { + if (destroyed) + connectsAfterDestroy += 1; + }); + try { + connection.rememberRoster([{ host: '127.0.0.1', port: livePort }]); + + // The first failure is the initial connect, which is what starts the + // redial pass; the next one is that pass's first candidate, so + // destroying there lands between two candidates rather than before the + // pass. + const destroyedMidPass = new Promise((resolve) => { + let failures = 0; + connection.on('error', () => { + failures += 1; + if (failures < 2 || destroyed) + return; + connection._destroy(); + destroyed = true; + resolve(); + }); + }); + + await connection.connect().catch(() => undefined); + await destroyedMidPass; + await new Promise((resolve) => setTimeout(resolve, 100)); + + assert.equal(accepted, 0, + 'a destroyed connection must not keep dialing the rest of the pass' + ); + assert.equal(connectsAfterDestroy, 0, + 'a destroyed connection must not announce a connection' + ); + assert.equal(connection.connected, false); + } finally { + connection._destroy(); + await new Promise((resolve) => live.close(() => resolve())); + } + } + ); + it('settles a dial in flight when a redirect replaces the socket', async () => { const seed = await startServer(); diff --git a/foreign/node/src/client/client.connection.ts b/foreign/node/src/client/client.connection.ts index 767da571d8..49f7752f89 100644 --- a/foreign/node/src/client/client.connection.ts +++ b/foreign/node/src/client/client.connection.ts @@ -67,9 +67,25 @@ const getTransport = (config: ClientConfig): Socket => { } }; +/** One node of the cluster, as a redial candidate. */ +export type Endpoint = { host: string, port: number }; + +/** + * Bound on one dial while other endpoints are queued behind it. Neither the + * connect nor the TLS handshake has a deadline of its own, so a node whose syns + * are dropped -- or one that accepts TCP and never answers the ClientHello -- + * would hold the whole pass. Matches the Rust SDK. + */ +const FAILOVER_DIAL_TIMEOUT_MS = 2_000; + /** * Default reconnection settings. - * Attempts reconnection every 5 seconds, up to 12 times. + * + * One retry is one full pass over every endpoint the client knows, so this is + * twelve passes rather than twelve dials, waiting 5 seconds between them. The + * first pass runs at once when more than one endpoint is known: the node just + * lost may be gone for good, and pausing before dialing a survivor only pushes + * the failover past the interval a caller is willing to wait. */ const DefaultReconnectOption: ReconnectOption = { enabled: true, @@ -108,9 +124,20 @@ export class IggyConnection extends EventEmitter { public connecting: boolean; /** Whether the connection is being intentionally closed */ public ending: boolean; + /** + * Whether the socket is being replaced by a deliberate leader redirect + * rather than lost. The drop looks the same from the outside, but nothing a + * caller submitted is in doubt: work waiting to be sent belongs on the node + * the client moves to, not in an error. + */ + public redirecting: boolean; /** Reconnection configuration */ private reconnectOption: ReconnectOption; - /** Number of reconnection attempts made */ + /** + * Number of passes made over the known endpoints. One pass dials the + * endpoint the client is on, the endpoint it was configured with, and every + * node the roster named. + */ private reconnectCount: number; /** Shared promise for concurrent callers waiting on one connection attempt */ private connectPromise?: Promise; @@ -118,6 +145,13 @@ export class IggyConnection extends EventEmitter { private reconnectPromise?: Promise; /** Endpoint the client was configured with, kept across leader redirects */ private readonly seedOptions: ClientConfig['options']; + /** + * Every node the roster named on the last read, kept as redial candidates. + * A node dies together with its address, and the roster is unreachable + * exactly when it is needed, so it has to have been remembered while the + * connection was still healthy. + */ + private rosterEndpoints: Endpoint[]; /** Incremental response frame decoder */ private responseDecoder: ResponseFrameDecoder; @@ -133,8 +167,10 @@ export class IggyConnection extends EventEmitter { this.connected = false; this.connecting = false; this.ending = false; + this.redirecting = false; this.reconnectOption = { ...DefaultReconnectOption, ...config.reconnect }; this.seedOptions = { ...config.options }; + this.rosterEndpoints = []; this.reconnectCount = 0; this.connectPromise = undefined; this.reconnectPromise = undefined; @@ -173,7 +209,10 @@ export class IggyConnection extends EventEmitter { this.emit('error', err); }); - socket.once('connect', () => { + // The readiness event, not 'connect': on TLS the socket is only usable + // once the handshake completes, and writing a request before that would + // announce a connection the peer has not agreed to yet. + socket.once(this._readyEvent(), () => { if (this.socket !== socket) return; debug('socket/connect event'); @@ -217,7 +256,13 @@ export class IggyConnection extends EventEmitter { this.connecting = true; const socket = this.socket; - const connectPromise = this._waitForConnection(socket); + // Bounded here too when the client knows somewhere else to go: this dial + // is not part of a pass, so an endpoint that never becomes usable would + // hold it with no timer of its own and the redial pass would never start. + const connectPromise = this._dialWithin( + socket, + this._redialCandidates().length > 1 + ); this.connectPromise = connectPromise; const clearConnectPromise = () => { if (this.connectPromise === connectPromise) @@ -227,10 +272,57 @@ export class IggyConnection extends EventEmitter { return connectPromise; } + /** + * Waits for one dial, bounded while other endpoints are queued behind it. + * + * A socket has no connect deadline of its own and there is no 'timeout' + * listener on it, so a node whose syns are dropped holds the pass for the + * whole OS connect timeout -- and it leads every pass, because the current + * endpoint only moves on success. The bound covers the TLS handshake too, + * which is what `_readyEvent()` waits for and has no deadline of its own + * either. It matches the Rust, Go and C# SDKs'. + */ + private async _dialWithin(socket: Socket, bounded: boolean): Promise { + if (!bounded) + return this._waitForConnection(socket); + + let expire: NodeJS.Timeout | undefined; + const bound = new Promise((_resolve, reject) => { + expire = setTimeout(() => { + // Destroying it makes the pending dial settle and releases the handle; + // left alone it would keep the event loop alive. + socket.destroy(); + reject(new Error( + `dial exceeded ${FAILOVER_DIAL_TIMEOUT_MS}ms` + )); + }, FAILOVER_DIAL_TIMEOUT_MS); + expire.unref?.(); + }); + + try { + return await Promise.race([this._waitForConnection(socket), bound]); + } finally { + clearTimeout(expire); + } + } + + /** + * The event that says a socket can carry a request. + * + * On TLS that is 'secureConnect', not 'connect': the latter fires as soon as + * the TCP handshake completes, so waiting on it would treat a peer that + * never answers the ClientHello as connected and leave the handshake with no + * deadline at all. + */ + private _readyEvent(): 'connect' | 'secureConnect' { + return this.config.transport === 'TLS' ? 'secureConnect' : 'connect'; + } + private _waitForConnection(socket: Socket): Promise { + const ready = this._readyEvent(); return new Promise((resolve, reject) => { const cleanup = () => { - socket.removeListener('connect', resolveConnect); + socket.removeListener(ready, resolveConnect); socket.removeListener('error', rejectConnect); socket.removeListener('close', rejectClosed); }; @@ -247,7 +339,7 @@ export class IggyConnection extends EventEmitter { }; socket.once('error', rejectConnect); socket.once('close', rejectClosed); - socket.once('connect', resolveConnect); + socket.once(ready, resolveConnect); }); } @@ -300,11 +392,31 @@ export class IggyConnection extends EventEmitter { ): Promise { let lastError = initialError; let expectedSocket = this.socket; - let attempt = 0; - while (enabled && this.reconnectCount < maxRetries) { + let firstPass = true; + // Reconnection settings bound the retries, not the endpoints. With them off + // and several endpoints known - the address the client was configured with, + // the nodes the roster named - those endpoints were made known in order to + // be tried, so they get one pass and no backoff, as in the other SDKs. A + // client that knows one endpoint and turned reconnection off redials + // nothing, which is what it asked for. + // Counted rather than tracked within this call: every dial the pass fails + // closes a socket, and a close starts a reconnect of its own. Bounded by + // `firstPass` alone, each of those closes would open another sweep and the + // pass would repeat for as long as the endpoints stay down. The count is + // reset when a connection is established, so a later loss sweeps again. + const sweepOnce = !enabled && this._redialCandidates().length > 1; + while ((enabled && this.reconnectCount < maxRetries) || + (sweepOnce && this.reconnectCount < 1)) { this.connecting = true; this.reconnectCount += 1; - await waitForReconnect(interval); + const candidates = this._redialCandidates(); + // The backoff paces retries against a single endpoint. With other + // endpoints known there is somewhere else to go, and pausing first only + // pushes the failover past the interval a caller is willing to wait; + // later passes still back off. + if (enabled && (!firstPass || candidates.length === 1)) + await waitForReconnect(interval); + firstPass = false; if (this.ending) throw new Error('connection is closed', { cause: lastError }); // A redirect may replace the socket at any point. Defer to the active @@ -312,24 +424,41 @@ export class IggyConnection extends EventEmitter { if (this.connected || this.socket !== expectedSocket) return this.connect(); - const options = this._reconnectTarget(attempt); - attempt += 1; - const socket = this._installSocket( - getTransport({ ...this.config, options }) - ); - this.socket = socket; - expectedSocket = socket; - try { - await this._waitForConnection(socket); - if (this.socket !== socket) + // Every endpoint gets its turn inside one attempt, so a full pass over + // the cluster costs one retry rather than one per endpoint: a pass that + // stopped at the first refusal would never reach the survivors of a + // client configured for a single retry. + for (const options of candidates) { + // Re-checked every iteration, not once above the loop: a destroy or a + // redirect mid-pass has to stop the pass. Left running, the next + // endpoint that answers would leave an open socket nobody closes, a + // 'connect' event after the destroy, and the process alive. + if (this.ending) + throw new Error('connection is closed', { cause: lastError }); + if (this.socket !== expectedSocket) return this.connect(); - this.config.options = options; - return this; - } catch (error) { - lastError = error instanceof Error - ? error - : new Error(String(error)); - debug('reconnect attempt failed', lastError); + + const socket = this._installSocket( + getTransport({ ...this.config, options }) + ); + this.socket = socket; + expectedSocket = socket; + try { + await this._dialWithin(socket, candidates.length > 1); + if (this.ending) { + socket.destroy(); + throw new Error('connection is closed', { cause: lastError }); + } + if (this.socket !== socket) + return this.connect(); + this.config.options = options; + return this; + } catch (error) { + lastError = error instanceof Error + ? error + : new Error(String(error)); + debug('reconnect attempt failed', lastError); + } } } @@ -341,16 +470,48 @@ export class IggyConnection extends EventEmitter { } /** - * Alternates reconnect dials between the current endpoint and the - * configured seed. After a leader redirect the current endpoint may die - * with the leader, and the seed is the way back to the rest of the cluster. + * Records the cluster roster as redial candidates. + * + * Replaced wholesale rather than merged: the roster is the cluster's own + * answer about where its nodes are, so a node it dropped stops being + * dialed. The configured seed is kept separately and outlives it. + */ + rememberRoster(endpoints: Endpoint[]): void { + if (endpoints.length === 0) + return; + this.rosterEndpoints = endpoints; + } + + /** + * Endpoints a redial rotates through, likeliest first: where the client + * currently is, the endpoint it was configured with, then the roster it + * learned while connected. After a leader redirect the current endpoint may + * die with the leader, and the rest of the list is the way back to the + * cluster. + * + * Duplicates are dropped by spelling: the loopback aliases and an + * IPv4-mapped IPv6 address collapse onto one endpoint. Names are not + * resolved, so a seed given as a DNS name and the roster's IP for the same + * node still count as two candidates -- one wasted dial per pass, not a + * correctness problem. */ - private _reconnectTarget(attempt: number): ClientConfig['options'] { - const current = this.config.options; - if (this.seedOptions.host === current.host && - this.seedOptions.port === current.port) - return current; - return attempt % 2 === 0 ? current : this.seedOptions; + _redialCandidates(): ClientConfig['options'][] { + const candidates = [this.config.options]; + const known = [ + this.seedOptions, + ...this.rosterEndpoints.map( + ({ host, port }) => ({ ...this.config.options, host, port }) + ) + ]; + for (const candidate of known) { + const duplicate = candidates.some( + (existing) => existing.port === candidate.port && + normalizeHost(existing.host) === normalizeHost(candidate.host) + ); + if (!duplicate) + candidates.push(candidate); + } + return candidates; } async redirect(host: string, port: number) { @@ -359,19 +520,24 @@ export class IggyConnection extends EventEmitter { ...this.config, options: redirectedOptions }; - // Destroying the old socket settles any dial still waiting on it. Its - // lifecycle listeners stay attached but go inert once the socket is - // replaced below, so surface the drop to in-flight exchanges ourselves. - this.socket.destroy(); - this.connected = false; - this.connecting = false; - this.connectPromise = undefined; - this.reconnectPromise = undefined; - this._endResponseWait(); - this.socket = this._installSocket(getTransport(redirectedConfig)); - this.emit('disconnected', false); - await this.connect(); - this.config.options = redirectedOptions; + this.redirecting = true; + try { + // Destroying the old socket settles any dial still waiting on it. Its + // lifecycle listeners stay attached but go inert once the socket is + // replaced below, so surface the drop to in-flight exchanges ourselves. + this.socket.destroy(); + this.connected = false; + this.connecting = false; + this.connectPromise = undefined; + this.reconnectPromise = undefined; + this._endResponseWait(); + this.socket = this._installSocket(getTransport(redirectedConfig)); + this.emit('disconnected', false); + await this.connect(); + this.config.options = redirectedOptions; + } finally { + this.redirecting = false; + } } abort(): void { diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts index 9605e8d86b..2b1aae15f6 100644 --- a/foreign/node/src/client/client.socket.test.ts +++ b/foreign/node/src/client/client.socket.test.ts @@ -35,7 +35,7 @@ import { import { Operation } from '../wire/vsr/operation.js'; import { VsrEvictionError } from '../wire/vsr/reply.js'; import { CommandResponseStream } from './client.socket.js'; -import type { ClientConfig } from './client.type.js'; +import type { ClientConfig, CommandResponse } from './client.type.js'; const TEST_SESSION = 42n; const TLS_CERTIFICATE = readFileSync( @@ -209,6 +209,23 @@ const vsrConfig = (port: number): ClientConfig => ({ }); /** Shrinks the leaderless poll so a test observes it without waiting on it. */ +/** The queue a command waits in, for parking one the way the client does. */ +const execQueue = (client: CommandResponseStream): { + command: number, + payload: Buffer, + handleResponse: boolean, + deadline: number, + resolve: (v: CommandResponse | PromiseLike) => void, + reject: (e: unknown) => void +}[] => (client as unknown as { _execQueue: never[] })._execQueue; + +/** The connection under a stream, for driving a redirect the way a move does. */ +const connectionOf = (client: CommandResponseStream): { + redirect: (host: string, port: number) => Promise +} => (client as unknown as { + connection: { redirect: (host: string, port: number) => Promise } +}).connection; + const compressLeaderlessPoll = ( client: CommandResponseStream, budget: number @@ -504,6 +521,111 @@ describe('VSR client socket', () => { } }); + // The node a client authenticated on dies; its next command has to complete + // on a survivor the roster named, under a session established there. + // Mirrors `core/integration/tests/cluster/failover_client_continuity.rs`. + it('resumes on a survivor after the node it authenticated on dies', + async () => { + const primarySockets = new Set(); + let primaryDead = false; + + const survivor = await startVsrServer((frame, socket) => { + const operation = frame.readUInt8(REQUEST_OFFSET.operation); + if (operation === Operation.Register) { + socket.write(replyFrame(Operation.Register, registerReplyBody())); + return; + } + const code = frame.readUInt32LE(REQUEST_OFFSET.reserved); + if (code === COMMAND_CODE.GetClusterMetadata) { + // The survivor leads once the primary is gone. + socket.write(replyFrame( + Operation.NonReplicated, + twoNodeMetadataBody(primary.port, survivor.port) + )); + return; + } + socket.write(replyFrame(operation)); + }); + + const primary = await startVsrServer((frame, socket) => { + primarySockets.add(socket); + if (primaryDead) { + socket.destroy(); + return; + } + const operation = frame.readUInt8(REQUEST_OFFSET.operation); + if (operation === Operation.Register) { + socket.write(replyFrame(Operation.Register, registerReplyBody())); + return; + } + const code = frame.readUInt32LE(REQUEST_OFFSET.reserved); + if (code === COMMAND_CODE.GetClusterMetadata) { + // The primary leads, so the login settles here and the roster is + // only remembered, not acted on, until the node dies. + socket.write(replyFrame( + Operation.NonReplicated, + twoNodeMetadataBody(survivor.port, primary.port) + )); + return; + } + socket.write(replyFrame(operation)); + }); + + const config: ClientConfig = { + ...vsrConfig(primary.port), + reconnect: { enabled: true, interval: 1, maxRetries: 3 } + }; + const client = new CommandResponseStream(config); + try { + await client.authenticate(config.credentials); + await client.sendCommand(60_021, Buffer.alloc(0)); + assert.ok( + primary.frames.some( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_021 + ), + 'the live primary answered the first command' + ); + + primaryDead = true; + for (const socket of primarySockets) + socket.destroy(); + await primary.close(); + + // The attempt in flight when the socket died is allowed to fail; the + // one after it has to land on the survivor. Two attempts, not a + // polling loop: the comment above promises at most one failed + // submission, and a loop of twenty would pass with nineteen failures. + let resumed = false; + let lastError: unknown; + for (let attempt = 0; attempt < 2 && !resumed; attempt += 1) { + try { + await client.sendCommand(60_021, Buffer.alloc(0)); + resumed = true; + } catch (error) { + lastError = error; + } + } + assert.ok(resumed, `the client never resumed: ${String(lastError)}`); + + const operations = survivor.frames.map( + (frame) => frame.readUInt8(REQUEST_OFFSET.operation) + ); + assert.ok( + operations.includes(Operation.Register), + 'the client signed in again on the survivor' + ); + assert.ok( + survivor.frames.some( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_021 + ), + 'the command landed on the survivor the roster named' + ); + } finally { + client.destroy(); + await survivor.close(); + } + }); + it('keeps a single-node login on its node', async () => { const server = await startVsrServer( (frame, socket) => singleNodeHandler(server.port)(frame, socket) @@ -864,6 +986,367 @@ describe('VSR client socket', () => { } }); + it('re-issues every request refused by a demoted node, not just the first', + async () => { + // One demotion, several refused requests: each of them re-checking on + // its own would move the client once per request, and the first + // redirect's drop would fail the others' roster reads - reporting a + // refusal they never had to. + const leader = await startVsrServer((frame, socket) => { + singleNodeHandler(leader.port)(frame, socket); + }); + // Leader at login, so the settlement leaves the client here, then + // demoted: the refusals below are what tells the client to look again. + let demotedYet = false; + const demoted = await startVsrServer((frame, socket) => { + const code = frame.readUInt32LE(REQUEST_OFFSET.reserved); + if (code === COMMAND_CODE.GetClusterMetadata) { + socket.write(replyFrame( + Operation.NonReplicated, + demotedYet + ? twoNodeMetadataBody(demoted.port, leader.port) + : twoNodeMetadataBody(leader.port, demoted.port) + )); + return; + } + if (code === 60_032) { + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + singleNodeHandler(demoted.port)(frame, socket); + }); + + const client = new CommandResponseStream(vsrConfig(demoted.port)); + try { + await client.authenticate(vsrConfig(demoted.port).credentials); + demotedYet = true; + const rosterReadsBefore = demoted.frames.filter( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === + COMMAND_CODE.GetClusterMetadata + ).length; + + // Two commands, both refused by the demoted node: one re-check between + // them, and both answered on the node it moved to. + const answers = await Promise.all([ + client.sendCommand(60_032, Buffer.alloc(0)), + client.sendCommand(60_032, Buffer.alloc(0)) + ]); + + assert.deepEqual(answers.map((answer) => answer.status), [0, 0]); + const rosterReads = demoted.frames.filter( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === + COMMAND_CODE.GetClusterMetadata + ).length - rosterReadsBefore; + assert.equal(rosterReads, 1, + 'each refusal re-read the roster on its own' + ); + const reissued = leader.frames.filter( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_032 + ).length; + assert.equal(reissued, 2, + 'a refused command was not re-issued on the node the move landed on' + ); + const connection = (client as unknown as { + connection: { isConnectedTo: (host: string, port: number) => boolean } + }).connection; + assert.equal(connection.isConnectedTo('127.0.0.1', leader.port), true); + } finally { + client.destroy(); + await leader.close(); + await demoted.close(); + } + } + ); + + it('holds a queued command instead of writing it to the node being left', + async () => { + // A refusal sends its caller to re-read the roster, and the drain that + // handed it out keeps going. Written in that window, the next queued + // command goes to the socket the move is about to replace: in flight when + // that happens, it dies with a lost-connection error nobody can act on + // instead of being re-issued on the node the move lands on. + const leader = await startVsrServer((frame, socket) => { + singleNodeHandler(leader.port)(frame, socket); + }); + let demotedYet = false; + const demoted = await startVsrServer((frame, socket) => { + const code = frame.readUInt32LE(REQUEST_OFFSET.reserved); + if (code === COMMAND_CODE.GetClusterMetadata) { + socket.write(replyFrame( + Operation.NonReplicated, + demotedYet + ? twoNodeMetadataBody(demoted.port, leader.port) + : twoNodeMetadataBody(leader.port, demoted.port) + )); + return; + } + if (code === 60_037) { + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + if (code === 60_038) { + // Accepted and never answered: a command written here is stuck until + // the move replaces the socket under it. + return; + } + singleNodeHandler(demoted.port)(frame, socket); + }); + + const client = new CommandResponseStream(vsrConfig(demoted.port)); + try { + await client.authenticate(vsrConfig(demoted.port).credentials); + demotedYet = true; + + // The second command is queued while the first is in flight, which is + // where a command caught in a move comes from. + const refused = client.sendCommand(60_037, Buffer.alloc(0)); + const behind = client.sendCommand(60_038, Buffer.alloc(0)); + refused.catch(() => undefined); + behind.catch(() => undefined); + + const settled = await Promise.race([ + Promise.all([refused, behind]).then(() => 'answered'), + new Promise((resolve) => { + setTimeout(() => resolve('stalled'), 10_000).unref(); + }) + ]); + assert.equal(settled, 'answered', + 'the command behind the refusal went out on the node being left' + ); + assert.ok( + !demoted.frames.some((frame) => + frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_038), + 'the command behind the refusal was written to the node being left' + ); + assert.ok( + leader.frames.some((frame) => + frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_038), + 'the command behind the refusal never reached the node moved to' + ); + } finally { + client.destroy(); + await leader.close(); + await demoted.close(); + } + } + ); + + it('re-issues a command queued behind a leader move instead of failing it', + async () => { + // A move replaces the socket, which looks like a drop to everything + // waiting in the queue. Nothing queued was written, though, so it belongs + // on the node the client moves to rather than in a lost-connection error + // the caller can do nothing about. + const leader = await startVsrServer((frame, socket) => { + singleNodeHandler(leader.port)(frame, socket); + }); + const demoted = await startVsrServer((frame, socket) => { + singleNodeHandler(demoted.port)(frame, socket); + }); + + const client = new CommandResponseStream(vsrConfig(demoted.port)); + try { + await client.authenticate(vsrConfig(demoted.port).credentials); + + // Parked the way a command is while something else holds the queue. + const queued = new Promise((resolve, reject) => { + execQueue(client).push({ + command: 60_034, + payload: Buffer.alloc(0), + handleResponse: true, + deadline: Date.now() + 30_000, + resolve, + reject + }); + }); + + await connectionOf(client).redirect('127.0.0.1', leader.port); + await queued; + + const landedOnLeader = leader.frames.some( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_034 + ); + assert.ok(landedOnLeader, + 'the queued command never reached the node the client moved to' + ); + } finally { + client.destroy(); + await leader.close(); + await demoted.close(); + } + } + ); + + it('surfaces the refusal rather than a timeout when the budget runs out', + async () => { + const server = await startVsrServer((frame, socket) => { + if (frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_036) { + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + singleNodeHandler(server.port)(frame, socket); + }); + const client = new CommandResponseStream(vsrConfig(server.port)); + const realNow = Date.now; + try { + await client.authenticate(vsrConfig(server.port).credentials); + // The request's budget, the first exchange, the window that hands the + // refusal out, and then a clock 10ms short of the deadline: too little + // to carry another attempt, so the caller has to see the answer the + // server gave rather than the timeout a doomed re-issue would produce. + const times = [0, 1, 2_001, 29_990]; + Date.now = () => times.shift() ?? 30_050; + + await assert.rejects( + () => client.sendCommand(60_036, Buffer.alloc(0)), + (error: unknown) => + error instanceof ResponseError && + error.commandCode === 60_036 && + error.errorCode === 58 + ); + } finally { + Date.now = realNow; + client.destroy(); + await server.close(); + } + } + ); + + it('surfaces the refusal when the move left too little of the budget', + async () => { + // The move itself costs budget: a roster read, an election it waited out, + // a redial. What is left can be positive and still too small to carry + // another exchange, and re-issued into it the request times out -- the + // caller then sees a timeout where the answer was "not admitted". + const leader = await startVsrServer((frame, socket) => { + singleNodeHandler(leader.port)(frame, socket); + }); + let demotedYet = false; + const demoted = await startVsrServer((frame, socket) => { + const code = frame.readUInt32LE(REQUEST_OFFSET.reserved); + if (code === COMMAND_CODE.GetClusterMetadata) { + socket.write(replyFrame( + Operation.NonReplicated, + demotedYet + ? twoNodeMetadataBody(demoted.port, leader.port) + : twoNodeMetadataBody(leader.port, demoted.port) + )); + return; + } + if (code === 60_039) { + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + singleNodeHandler(demoted.port)(frame, socket); + }); + + const client = new CommandResponseStream(vsrConfig(demoted.port)); + const realNow = Date.now; + let offset = 0; + try { + await client.authenticate(vsrConfig(demoted.port).credentials); + demotedYet = true; + + const deadline = realNow() + 30_000; + const connection = connectionOf(client); + const move = connection.redirect.bind(connection); + connection.redirect = async (host: string, port: number) => { + await move(host, port); + // 30ms of budget left the moment the client lands: positive, and + // below the interval one exchange needs. + offset = deadline - realNow() - 30; + }; + Date.now = () => realNow() + offset; + + await assert.rejects( + () => client.sendCommand(60_039, Buffer.alloc(0), { deadline }), + (error: unknown) => + error instanceof ResponseError && + error.commandCode === 60_039 && + error.errorCode === 58 + ); + assert.ok( + !leader.frames.some((frame) => + frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_039), + 'the request was re-issued into a budget too small to answer it' + ); + } finally { + Date.now = realNow; + client.destroy(); + await leader.close(); + await demoted.close(); + } + } + ); + + it('paces the re-issues while the roster still names this node', + async () => { + // Re-issuing is right, spinning is not: the in-connection replay window + // belongs to the request's budget and is spent after the first pass, so + // without a wait the client would hammer the node for the whole budget. + let refusals = 0; + const server = await startVsrServer((frame, socket) => { + if (frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_035) { + refusals += 1; + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + singleNodeHandler(server.port)(frame, socket); + }); + const client = new CommandResponseStream(vsrConfig(server.port)); + try { + await client.authenticate(vsrConfig(server.port).credentials); + + const pending = client.sendCommand(60_035, Buffer.alloc(0)); + pending.catch(() => undefined); + await new Promise((resolve) => { + setTimeout(resolve, 5_000).unref(); + }); + + // The first 2s window replays on the connection at its own interval; + // every window after it costs one refusal per pace. + assert.ok(refusals < 200, + `the re-issues were not paced: ${refusals} refusals in 5s` + ); + } finally { + client.destroy(); + await server.close(); + } + } + ); + + it('keeps re-issuing a not-admitted request while the roster still names this node', + async () => { + const server = await startVsrServer((frame, socket) => { + if (frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_031) { + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + singleNodeHandler(server.port)(frame, socket); + }); + const client = new CommandResponseStream(vsrConfig(server.port)); + try { + await client.authenticate(vsrConfig(server.port).credentials); + + // A refusal the roster cannot explain is a wait, not a verdict: an + // election may still be in flight, so the request keeps going for its + // whole budget instead of failing after the first re-check window. + const pending = client.sendCommand(60_031, Buffer.alloc(0)); + const outcome = await Promise.race([ + pending.then(() => 'answered', () => 'gave up'), + new Promise((resolve) => { + setTimeout(() => resolve('still trying'), 4_000).unref(); + }) + ]); + + assert.equal(outcome, 'still trying'); + } finally { + client.destroy(); + await server.close(); + } + } + ); + it('keeps a typed transient error and session at its retry deadline', async () => { const server = await startVsrServer((frame, socket) => { diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts index 2f02764c6c..3924ba14ad 100644 --- a/foreign/node/src/client/client.socket.ts +++ b/foreign/node/src/client/client.socket.ts @@ -24,7 +24,7 @@ import type { } from '../client/client.type.js'; import { ResponseError, responseError } from '../wire/error.utils.js'; import { debug } from './client.debug.js'; -import { IggyConnection } from './client.connection.js'; +import { type Endpoint, IggyConnection } from './client.connection.js'; import { LOGIN, LOGIN_WITH_TOKEN, LOGOUT, PING } from '../wire/index.js'; import { GET_CLUSTER_METADATA } from '../wire/cluster/get-cluster-metadata.command.js'; import { COMMAND_CODE } from '../wire/command.code.js'; @@ -44,6 +44,32 @@ const LEADERLESS_POLL_INTERVAL_MS = 250; const MAX_LEADER_REDIRECTS = 3; const TRANSIENT_NOT_COMMITTED = 57; const TRANSIENT_NOT_ACCEPTED = 58; +/** + * How long a `TRANSIENT_NOT_ACCEPTED` request replays on the same connection + * before the roster is re-read. A node that stopped being primary refuses + * forever, so replaying alone never recovers. Matches the Rust SDK. + */ +const VSR_FAILOVER_CHECK_MS = 2_000; + +/** + * Whether a request's budget still holds enough for another attempt. One + * exchange needs at least a replay interval to be worth starting; below that + * the attempt can only end in a timeout, which would hide the refusal that + * actually came back. + */ +const worthAnotherAttempt = (deadline: number): boolean => + deadline - Date.now() > VSR_RETRY_INTERVAL_MS; + +/** + * A request the current node keeps refusing as not-admitted. Carries the + * refusal so the caller can surface it when the roster turns out to still name + * this node as the leader. Never escapes `sendCommand`. + */ +class LeaderMovedError extends Error { + constructor(readonly refusal: ResponseError) { + super('the node refused the request as not-admitted; re-reading the roster'); + } +} /** * Command codes that can be executed without authentication. @@ -64,6 +90,12 @@ type Job = { payload: Buffer, /** Whether to parse the response */ handleResponse: boolean, + /** Whether the command is appended rather than prepended to the queue */ + last: boolean, + /** Whether a not-admitted refusal re-checks the leader and re-issues */ + followsLeaderMoves: boolean, + /** When the whole request gives up, however often it is re-issued */ + deadline: number, /** Promise resolve function */ resolve: (v: CommandResponse | PromiseLike) => void, /** Promise reject function */ @@ -99,6 +131,18 @@ export class CommandResponseStream extends EventEmitter { private authenticationPromise?: Promise; /** Whether a login is already being moved to the leader */ private settlingLeader: boolean; + /** + * The leader re-check a refused request started, shared with every other + * request refused by the same node so one demotion moves the client once. + */ + private leaderMoveInFlight?: Promise; + /** + * Refusals handed out to callers that have not decided what to do with them + * yet. The queue holds while any are outstanding: the caller of a refused + * command re-checks the leader, and a command written in the meantime goes + * out on the socket that check is about to replace. + */ + private leaderMovesUndecided: number; /** How long a leaderless roster is polled before settling in place */ private leaderlessWaitBudget: number; /** Delay between roster reads while the cluster elects */ @@ -132,6 +176,7 @@ export class CommandResponseStream extends EventEmitter { this.vsrSession = new VsrSession(); this.authenticationPromise = undefined; this.settlingLeader = false; + this.leaderMovesUndecided = 0; this.leaderlessWaitBudget = LEADERLESS_WAIT_BUDGET_MS; this.leaderlessPollInterval = LEADERLESS_POLL_INTERVAL_MS; this.pendingSubmissions = 0; @@ -153,12 +198,47 @@ export class CommandResponseStream extends EventEmitter { }); this.connection.on('disconnected', () => { this._resetSession(); + if (this.connection.redirecting) { + // The client is moving to the leader, which is its own doing: a queued + // command has not been written, so it belongs on the node being moved + // to rather than in an error. + this._reissueQueue(); + return; + } this._failQueue( new Error('connection closed before queued commands were sent') ); }); } + /** + * Re-submits queued commands through the full send path, so each one + * reconnects, re-authenticates and re-checks the leader as if it had just + * been called. + * + * Only for a drop the client caused. Nothing here was written, so there is no + * outcome in doubt: a command still in the queue when the socket is replaced + * would otherwise fail with a lost-connection error the caller can do nothing + * about. + */ + private _reissueQueue(): void { + const queued = this._execQueue; + this._execQueue = []; + for (const job of queued) { + debug('re-issuing a queued command after a leader move', job.command); + // The whole job, not just the payload: a fresh budget would let a command + // caught in a move take twice the response timeout, and a roster read + // re-issued as leader-following would answer a leader check with another + // leader check. + this.sendCommand(job.command, job.payload, { + handleResponse: job.handleResponse, + last: job.last, + followsLeaderMoves: job.followsLeaderMoves, + deadline: job.deadline + }).then(job.resolve, job.reject); + } + } + /** * Sends a command to the server. * Automatically handles connection, authentication and leader settlement. @@ -177,7 +257,8 @@ export class CommandResponseStream extends EventEmitter { try { const { handleResponse = true, - last = true + last = true, + followsLeaderMoves = true } = options; if (!this.connection.connected) @@ -186,21 +267,75 @@ export class CommandResponseStream extends EventEmitter { if (!this.isAuthenticated && !this.isUnloggedCommand(command)) await this.authenticate(this.options.credentials); - const response = await new Promise( - (resolve, reject) => { - const job = { - command, - payload, - handleResponse, - resolve, - reject - }; - if (last) - this._execQueue.push(job); - else - this._execQueue.unshift(job); - this._processQueue(); - }); + // The roster read is itself a queued command and the queue is + // single-flighted, so the leader re-check cannot happen inside + // `_processVsr`. The refusal comes back out here instead, where the + // queue is free, and the command is re-issued on the node that now + // leads. + // + // A not-admitted refusal means the request was never applied, so it is + // re-issued for the whole request budget rather than given up on after + // one window: the roster can still name this node -- an election in + // flight, a leader that has not moved yet -- and that is a wait, not a + // verdict. + // + // One budget for the whole request: the transient replays on a + // connection, the leader re-checks, and the re-issues after a move all + // spend it, so a request cannot outlive it by moving. A command re-issued + // after a move keeps the budget it was first submitted with, rather than + // opening a second one. + const deadline = options.deadline ?? Date.now() + VSR_RESPONSE_TIMEOUT_MS; + let response: CommandResponse; + for (;;) { + try { + response = await this._queueCommand(command, payload, handleResponse, + last, followsLeaderMoves, deadline); + break; + } catch (error) { + if (!(error instanceof LeaderMovedError)) + throw error; + // The roster read that a re-check runs is itself a command that can + // be refused this way, and answering a leader check with another + // leader check would recurse. Its caller reads a failure as "stay + // where you are". + // + // A budget too small to carry another attempt ends it here, with the + // refusal the server actually gave: re-issued into what is left, the + // request would time out instead and the caller would see a timeout + // where the answer was "not admitted". + let moved = false; + try { + if (!followsLeaderMoves || !worthAnotherAttempt(deadline)) + throw responseError(command, error.refusal.errorCode); + moved = await this._followLeaderMove(); + } finally { + // Released as soon as the move is decided, before the pace below + // and before any re-authentication: those go through the queue + // themselves, and a queue still held for this refusal would never + // reach them. + if (followsLeaderMoves) + this._releaseUndecidedMove(); + } + if (!moved) { + // Nowhere else to go yet: the roster still names this node, or it + // could not be read. Paced, because the in-connection replay + // window belongs to the request's budget and has already been + // spent -- re-issuing straight away would spin. + await delay(Math.min( + VSR_FAILOVER_CHECK_MS, + Math.max(0, deadline - Date.now()) + )); + } + if (!worthAnotherAttempt(deadline)) + throw responseError(command, error.refusal.errorCode); + // A move drops the session with the socket it was bound to, so the + // re-issue would otherwise go out under no session: a replicated + // command fails client-side, a non-replicated one goes out with + // session 0. + if (!this.isAuthenticated && !this.isUnloggedCommand(command)) + await this.authenticate(this.options.credentials); + } + } if (!isLoginCommand(command) || this.settlingLeader) return response; this.settlingLeader = true; @@ -216,6 +351,110 @@ export class CommandResponseStream extends EventEmitter { } } + private _queueCommand( + command: number, + payload: Buffer, + handleResponse: boolean, + last: boolean, + followsLeaderMoves: boolean, + deadline: number + ): Promise { + return new Promise((resolve, reject) => { + const job: Job = { + command, + payload, + handleResponse, + last, + followsLeaderMoves, + deadline, + resolve, + reject + }; + if (last) + this._execQueue.push(job); + else + this._execQueue.unshift(job); + this._processQueue(); + }); + } + + /** + * Re-reads the roster and moves to the leader it names. + * + * Best effort: an unreadable roster, or one that still names this node, + * leaves the client where it is and the refused request is re-issued anyway. + * + * Single-flighted, and concurrent callers share the outcome instead of + * failing: several commands are refused by the same demoted node, and each + * starting its own redirect would move the client once per command. The + * first redirect's `'disconnected'` also fails the others' roster reads, so + * a caller that raced one would report a refusal it never had to. + * + * @returns Whether the client moved + */ + private _followLeaderMove(): Promise { + const inFlight = this.leaderMoveInFlight; + if (inFlight) + return inFlight; + + const move = (async () => { + try { + const leader = await this._readLeaderEndpoint(); + if (!leader || this.connection.isConnectedTo(leader.host, leader.port)) + return false; + debug(`the leader moved to ${leader.host}:${leader.port}, following it`); + await this.connection.redirect(leader.host, leader.port); + return true; + } catch (error) { + debug('the leader could not be re-checked, staying on this node', error); + return false; + } + })(); + this.leaderMoveInFlight = move; + void move.finally(() => { + if (this.leaderMoveInFlight !== move) + return; + this.leaderMoveInFlight = undefined; + // The drain stopped while the move was being decided. A move that + // happened re-issues what was held back on the new socket; one that did + // not leaves it here, with nothing else due to pick it up. + if (!this.connection.redirecting) + void this._processQueue(); + }); + return move; + } + + /** Whether a leader move is being decided or carried out. */ + private _movePending(): boolean { + return this.leaderMovesUndecided > 0 || this.leaderMoveInFlight !== undefined; + } + + /** + * Releases the queue hold one refusal took, and drains what was held back + * once the last of them is decided. + */ + private _releaseUndecidedMove(): void { + if (this.leaderMovesUndecided > 0) + this.leaderMovesUndecided -= 1; + if (this._movePending() || this.connection.redirecting) + return; + void this._processQueue(); + } + + private _rememberRoster(response: CommandResponse): void { + try { + const metadata = GET_CLUSTER_METADATA.deserialize(response); + this.connection.rememberRoster( + metadata.nodes + .filter((node) => node.endpoints.tcp !== 0) + .map((node) => ({ host: node.ip, port: node.endpoints.tcp })) + ); + } catch (error) { + debug('an unreadable roster leaves the redial candidates as they are', + error); + } + } + /** * Processes queued commands sequentially. * Emits 'finishQueue' when all commands are processed. @@ -226,17 +465,42 @@ export class CommandResponseStream extends EventEmitter { return; this.busy = true; while (this._execQueue.length > 0 && this.connection.socket.writable) { - const next = this._execQueue.shift(); + // While a leader move is being decided, only the roster read the move + // itself runs goes out -- it is what decides where the client lands, and + // it is the one command that does not follow moves. Draining the rest + // would write them to the socket `redirect()` is about to replace, and a + // command in flight when that happens dies with a lost-connection error + // instead of being re-issued on the node the move lands on. + const index = this._movePending() + ? this._execQueue.findIndex((job) => !job.followsLeaderMoves) + : 0; + if (index < 0) break; + const [next] = this._execQueue.splice(index, 1); if (!next) break; - const { command, payload, handleResponse, resolve, reject } = next; + const { command, payload, handleResponse, deadline, resolve, reject } = next; try { - resolve(await this._processNext(command, payload, handleResponse)); + resolve(await this._processNext(command, payload, handleResponse, deadline)); } catch (err) { + if (err instanceof LeaderMovedError && next.followsLeaderMoves) + // Counted before the rejection is handed out, not after: the caller + // resumes as a microtask, so this loop would otherwise write the next + // command before the re-check it is about to start has begun. + this.leaderMovesUndecided += 1; reject(err); } } - if (this._execQueue.length > 0) - this._failQueue(new Error('connection is not writable')); + if (this._execQueue.length > 0) { + // The same distinction as on 'disconnected': the socket a leader move + // replaced stops being writable, and what is still queued belongs on the + // node being moved to. + if (this.connection.redirecting) + this._reissueQueue(); + else if (!this._movePending()) + this._failQueue(new Error('connection is not writable')); + // Otherwise the move is still being decided: these commands were never + // written, and they are drained again once it settles -- here if the + // client stays, on the new socket if it moves. + } this.busy = false; this._emitFinishQueue(); } @@ -254,38 +518,46 @@ export class CommandResponseStream extends EventEmitter { * @param command - Command code * @param payload - Command payload * @param handleResp - Whether to parse the response + * @param deadline - When the whole request gives up, shared with the leader + * re-checks and the re-issues after a move * @returns Promise resolving to the command response */ _processNext( command: number, payload: Buffer, - handleResp = true + handleResp = true, + deadline = Date.now() + VSR_RESPONSE_TIMEOUT_MS ): Promise { if (isLoginCommand(command) && this.isAuthenticated) - return this._processVsrLogin(command, payload, handleResp); - return this._processVsr(command, payload, handleResp); + return this._processVsrLogin(command, payload, handleResp, deadline); + return this._processVsr(command, payload, handleResp, deadline); } private async _processVsrLogin( command: number, payload: Buffer, - handleResp: boolean + handleResp: boolean, + deadline: number ): Promise { - await this._processVsr(LOGOUT.code, LOGOUT.serialize(), true); - return this._processVsr(command, payload, handleResp); + await this._processVsr(LOGOUT.code, LOGOUT.serialize(), true, deadline); + return this._processVsr(command, payload, handleResp, deadline); } private async _processVsr( command: number, payload: Buffer, - handleResp: boolean + handleResp: boolean, + deadline: number ): Promise { let requestWritten = false; try { const prepared = prepareVsrCommand(command, payload); // A transient retry must preserve all request identity fields. const frame = this.vsrSession.encode(prepared.command, prepared.payload); - const deadline = Date.now() + VSR_RESPONSE_TIMEOUT_MS; + // Derived from the request's own budget rather than read off the clock, + // so one request spends one budget however many times it is re-issued. + const notAcceptedDeadline = + deadline - VSR_RESPONSE_TIMEOUT_MS + VSR_FAILOVER_CHECK_MS; let lastTransientError: ResponseError | undefined; let parsed: CommandResponse; while (true) { @@ -316,6 +588,16 @@ export class CommandResponseStream extends EventEmitter { !isTransientVsrError(error.errorCode)) throw error; lastTransientError = error; + // A not-admitted refusal is a statement about who leads, not about + // load: a node that stopped being primary refuses forever, so + // replaying on this connection never recovers. Hand it back for a + // roster re-read once the window is spent. Not-committed (57) stays + // here: the request is in flight on this very node, and its outcome + // is unknown anywhere else. + if (error.errorCode === TRANSIENT_NOT_ACCEPTED && + !isLoginCommand(command) && + Date.now() >= notAcceptedDeadline) + throw new LeaderMovedError(error); const retryDelay = Math.min( VSR_RETRY_INTERVAL_MS, Math.max(0, deadline - Date.now()) @@ -335,8 +617,18 @@ export class CommandResponseStream extends EventEmitter { if (prepared.command === COMMAND_CODE.LogoutUser) { this._resetSession(); } + // Every roster read feeds the redial candidates, whoever asked for it + // and whatever it says: a node dies together with its address, the + // roster is unreachable exactly when it is needed, and reading it only + // during a login would leave the candidates stale between logins. + if (handleResp && command === GET_CLUSTER_METADATA.code) + this._rememberRoster(parsed); return parsed; } catch (error) { + // A not-admitted refusal is an answer, so the session is not in doubt + // and the request was never applied. + if (error instanceof LeaderMovedError) + throw error; // Once bytes were handed to the socket, a local transport or decode // failure leaves the request outcome ambiguous. Register a fresh session // rather than replaying that request under a different client identity. @@ -454,8 +746,7 @@ export class CommandResponseStream extends EventEmitter { * died between the login and this read), keeps the client on its current * node instead of failing a login that already succeeded. */ - private async _readLeaderEndpoint(): - Promise<{ host: string, port: number } | undefined> { + private async _readLeaderEndpoint(): Promise { // A cluster can be transiently leaderless: a restarted node cedes the // primaryship its stale view assigns it, and the roster reports no leader // until the peers' election completes. That window is roughly one heartbeat @@ -475,8 +766,11 @@ export class CommandResponseStream extends EventEmitter { const response = await this.sendCommand( GET_CLUSTER_METADATA.code, GET_CLUSTER_METADATA.serialize(), - { last: false } + { last: false, followsLeaderMoves: false } ); + // The redial candidates are fed by `_processVsr` for every roster + // read, leaderless ones included: a roster with no leader still names + // where the nodes are. const metadata = GET_CLUSTER_METADATA.deserialize(response); if (metadata.nodes.length <= 1) return undefined; diff --git a/foreign/node/src/client/client.type.ts b/foreign/node/src/client/client.type.ts index 40c29bc2cc..42ae39239d 100644 --- a/foreign/node/src/client/client.type.ts +++ b/foreign/node/src/client/client.type.ts @@ -47,7 +47,20 @@ export type SendCommandOptions = { /** Whether the response uses the standard command response decoder */ handleResponse?: boolean, /** Whether to append rather than prepend the command to the queue */ - last?: boolean + last?: boolean, + /** + * Whether a not-admitted refusal re-checks the leader and re-issues the + * command. False for the roster read a re-check itself runs: answering a + * leader check with another leader check would recurse. + */ + followsLeaderMoves?: boolean, + /** + * When the whole request gives up, as an epoch timestamp in milliseconds. + * Set when a command already carries a budget -- one re-issued after a + * leader move keeps the budget it was first submitted with, rather than + * opening a second one on top of it. Defaults to a fresh response timeout. + */ + deadline?: number }; /** @@ -100,9 +113,16 @@ export type TransportType = typeof Transports[number]; export type ReconnectOption = { /** Whether automatic reconnection is enabled */ enabled: boolean, - /** Interval between reconnection attempts in milliseconds */ + /** + * Milliseconds to wait between passes. The first pass runs at once when more + * than one endpoint is known. + */ interval: number, - /** Maximum number of reconnection attempts */ + /** + * Maximum number of passes over the known endpoints. One pass dials the + * endpoint the client is on, the endpoint it was configured with, and every + * node the roster named, so this counts passes rather than dials. + */ maxRetries: number } diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index 80137b39fa..254b23f41c 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi @@ -2020,14 +2020,21 @@ class TcpReconnectionConfig: Args: enabled: Whether to reconnect at all. Defaults to enabled. - max_retries: Attempts before giving up, or `None` for unlimited. - Defaults to unlimited, which means a call awaited while the server - is down never returns: `connect()`, `send_messages()` and + max_retries: Passes over the known endpoints after the first, or + `None` for unlimited; `0` still makes that first pass. One pass + tries the endpoint the client is on, the address it was + configured with, and every node the roster named, so this counts + passes rather than dials. Defaults + to unlimited, which means a call awaited while the server is + down never returns: `connect()`, `send_messages()` and `poll_messages()` all wait inside the retry loop. Set a finite number for request/reply style usage, so a call fails instead. - interval: Delay between attempts. Defaults to 1 second. - reestablish_after: Cooldown before reconnecting after a previously - successful connection. Defaults to 5 seconds. + interval: Delay between passes. Defaults to 1 second. The first pass + runs at once when more than one endpoint is known. + reestablish_after: Cooldown before redialing the endpoint of the last + successful connection, measured from when it was established, so + a session that outlived the interval is redialed at once. Owed to + that endpoint alone. Defaults to 5 seconds. Raises: ValueError: If a duration is negative, if `max_retries` is outside the diff --git a/foreign/python/src/config.rs b/foreign/python/src/config.rs index 79eb285b16..ed6e2a14f0 100644 --- a/foreign/python/src/config.rs +++ b/foreign/python/src/config.rs @@ -121,14 +121,21 @@ impl TcpReconnectionConfig { /// /// Args: /// enabled: Whether to reconnect at all. Defaults to enabled. - /// max_retries: Attempts before giving up, or `None` for unlimited. - /// Defaults to unlimited, which means a call awaited while the server - /// is down never returns: `connect()`, `send_messages()` and + /// max_retries: Passes over the known endpoints after the first, or + /// `None` for unlimited; `0` still makes that first pass. One pass + /// tries the endpoint the client is on, the address it was + /// configured with, and every node the roster named, so this counts + /// passes rather than dials. Defaults + /// to unlimited, which means a call awaited while the server is + /// down never returns: `connect()`, `send_messages()` and /// `poll_messages()` all wait inside the retry loop. Set a finite /// number for request/reply style usage, so a call fails instead. - /// interval: Delay between attempts. Defaults to 1 second. - /// reestablish_after: Cooldown before reconnecting after a previously - /// successful connection. Defaults to 5 seconds. + /// interval: Delay between passes. Defaults to 1 second. The first pass + /// runs at once when more than one endpoint is known. + /// reestablish_after: Cooldown before redialing the endpoint of the last + /// successful connection, measured from when it was established, so + /// a session that outlived the interval is redialed at once. Owed to + /// that endpoint alone. Defaults to 5 seconds. /// /// Raises: /// ValueError: If a duration is negative, if `max_retries` is outside the