From aa506520d3c66ed6d2563070f3e9ee74b2d142ed Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Thu, 20 Aug 2026 16:31:35 +0200 Subject: [PATCH 01/16] fix(shard): tell an unknown consensus operation from a corrupt header --- core/binary_protocol/src/consensus/error.rs | 6 ++ core/binary_protocol/src/consensus/header.rs | 14 ++++ .../src/consensus/operation.rs | 11 +++ core/server/src/partition_reconciler.rs | 59 ++++++++++++++++ core/server_common/src/consensus_message.rs | 27 ++++++-- core/shard/src/metrics.rs | 12 +++- core/shard/src/router.rs | 69 +++++++++++-------- 7 files changed, 166 insertions(+), 32 deletions(-) diff --git a/core/binary_protocol/src/consensus/error.rs b/core/binary_protocol/src/consensus/error.rs index 02fb044e64..8b6a4ba289 100644 --- a/core/binary_protocol/src/consensus/error.rs +++ b/core/binary_protocol/src/consensus/error.rs @@ -79,6 +79,12 @@ pub enum ConsensusError { #[error("invalid bit pattern in header (enum discriminant out of range)")] InvalidBitPattern, + #[error( + "operation {operation:#04x} is not known to this build; the sender runs a release that \ + added a consensus operation, so this node cannot journal or ack the frame" + )] + UnsupportedOperation { operation: u8 }, + #[error("client-bound command {0:?} cannot be dispatched on inbound path")] ClientBoundCommand(Command), } diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index 2f205df3d9..8fe0150cc9 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs @@ -103,6 +103,15 @@ pub fn frame_checksum_bytes(header: &[u8; HEADER_SIZE]) -> u128 { pub trait ConsensusHeader: Sized + CheckedBitPattern + NoUninit { const COMMAND: Command; + /// Byte offset of this header's `operation` field, `None` when it carries + /// none. + /// + /// The typed decode reads the raw byte here after a failed checked cast, + /// so an operation a newer release added is reported as version skew + /// rather than corruption. An offset rather than a getter because the cast + /// has already failed by then, so no typed view of the header exists. + const OPERATION_OFFSET: Option = None; + /// Whether a frame carrying `command` may be typed as this header. /// Defaults to an exact match; a header that serves several commands /// with one layout (e.g. `RepairDone` / `RangeEvicted`) widens it. @@ -477,6 +486,7 @@ fn validate_request_fields( } impl ConsensusHeader for RoutedRequestHeader { + const OPERATION_OFFSET: Option = Some(core::mem::offset_of!(Self, operation)); const COMMAND: Command = Command::Request; /// The client-wire [`RequestHeader`] this is promoted from is unsealed, and the /// promotion copies `checksum` verbatim, so there is nothing here to verify. @@ -511,6 +521,7 @@ impl ConsensusHeader for RoutedRequestHeader { } impl ConsensusHeader for RequestHeader { + const OPERATION_OFFSET: Option = Some(core::mem::offset_of!(Self, operation)); const COMMAND: Command = Command::Request; const FRAME_SEALED: bool = false; @@ -621,6 +632,7 @@ impl Default for ReplyHeader { } impl ConsensusHeader for ReplyHeader { + const OPERATION_OFFSET: Option = Some(core::mem::offset_of!(Self, operation)); const COMMAND: Command = Command::Reply; const FRAME_SEALED: bool = false; @@ -971,6 +983,7 @@ impl Default for PrepareHeader { } impl ConsensusHeader for PrepareHeader { + const OPERATION_OFFSET: Option = Some(core::mem::offset_of!(Self, operation)); const COMMAND: Command = Command::Prepare; const FRAME_SEALED: bool = false; @@ -1178,6 +1191,7 @@ impl Default for PrepareOkHeader { } impl ConsensusHeader for PrepareOkHeader { + const OPERATION_OFFSET: Option = Some(core::mem::offset_of!(Self, operation)); const FRAME_SEALED: bool = true; const COMMAND: Command = Command::PrepareOk; diff --git a/core/binary_protocol/src/consensus/operation.rs b/core/binary_protocol/src/consensus/operation.rs index 691bd9f1b7..95de1d4eed 100644 --- a/core/binary_protocol/src/consensus/operation.rs +++ b/core/binary_protocol/src/consensus/operation.rs @@ -98,6 +98,17 @@ pub enum Operation { } impl Operation { + /// Whether `code` is a discriminant this build defines. + /// + /// The typed decode needs this to tell an operation a newer release added + /// from a corrupted header byte: bytemuck's checked cast rejects both with + /// one undifferentiated error, and only the former is fixable by upgrading + /// this node. + #[must_use] + pub fn is_known_code(code: u8) -> bool { + bytemuck::checked::try_cast::(code).is_ok() + } + pub const INTERNAL_START: u8 = Self::CreateTopicWithAssignments as u8; pub const METADATA_START: u8 = Self::CreateStream as u8; pub const PARTITION_START: u8 = Self::SendMessages as u8; diff --git a/core/server/src/partition_reconciler.rs b/core/server/src/partition_reconciler.rs index 4df3e54020..0ab7d1be87 100644 --- a/core/server/src/partition_reconciler.rs +++ b/core/server/src/partition_reconciler.rs @@ -2350,6 +2350,65 @@ mod tests { ); } + /// The mixed-version upgrade hole from IGGY-250, at the router seam this + /// time: a wire-valid consensus frame carrying an operation only a newer + /// release defines must leave an accounted, operator-visible trace instead + /// of a bare warn log, because the frame's group stops making progress + /// until this node is upgraded. + #[compio::test] + async fn given_an_unknown_operation_when_dispatched_should_account_an_upgrade_fence_drop() { + // Far past every discriminant this build defines. + const OPERATION_FROM_A_NEWER_RELEASE: u8 = 0xEE; + + let tmp = TempDir::new().expect("tempdir for system path"); + let config = test_config(&tmp); + let shard = build_test_shard(0, &config, TestMux::default()); + + let mut owned = server_common::iobuf::Owned::<{ server_common::MESSAGE_ALIGN }>::zeroed( + iggy_binary_protocol::HEADER_SIZE, + ); + { + let frame = owned.as_mut_slice(); + let size_offset = std::mem::offset_of!(PrepareHeader, size); + let frame_size = + u32::try_from(iggy_binary_protocol::HEADER_SIZE).expect("header size fits in u32"); + frame[size_offset..size_offset + 4].copy_from_slice(&frame_size.to_le_bytes()); + frame[std::mem::offset_of!(PrepareHeader, command)] = Command::Prepare as u8; + frame[std::mem::offset_of!(PrepareHeader, operation)] = OPERATION_FROM_A_NEWER_RELEASE; + let header: &[u8; iggy_binary_protocol::HEADER_SIZE] = frame + [..iggy_binary_protocol::HEADER_SIZE] + .try_into() + .expect("frame spans a full header"); + let checksum = iggy_binary_protocol::frame_checksum_bytes(header); + frame[..size_of::()].copy_from_slice(&checksum.to_le_bytes()); + } + let message = Message::::try_from(owned) + .expect("a sealed Prepare frame is wire-valid in the generic view"); + + let before = shard.metrics().frame_drop_count( + shard::metrics::frame_drop_variant::CONSENSUS, + shard::metrics::frame_drop_reason::UNSUPPORTED_OPERATION, + ); + shard.dispatch(message); + assert_eq!( + shard.metrics().frame_drop_count( + shard::metrics::frame_drop_variant::CONSENSUS, + shard::metrics::frame_drop_reason::UNSUPPORTED_OPERATION, + ), + before + 1, + "an operation from a newer release must be accounted under its own reason, not \ + folded into the generic unparsable drop" + ); + assert_eq!( + shard.metrics().frame_drop_count( + shard::metrics::frame_drop_variant::CONSENSUS, + shard::metrics::frame_drop_reason::UNPARSABLE, + ), + 0, + "version skew must not read as header corruption" + ); + } + /// Receive half of the purge gate in `on_repair_range_reply`: while a /// committed purge has not applied locally, a repair verdict must be /// deferred wholesale -- installing the peer's floor against pre-purge diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs index cd4cd91e19..391790d0ac 100644 --- a/core/server_common/src/consensus_message.rs +++ b/core/server_common/src/consensus_message.rs @@ -242,7 +242,7 @@ where let bytes = >::header_storage(&self.backing); let typed = bytemuck::checked::try_from_bytes::(&bytes[..size_of::()]) - .map_err(|_| ConsensusError::InvalidBitPattern)?; + .map_err(|_| classify_failed_cast::(bytes))?; // Before `validate`: a header that did not survive the link intact cannot // have any of its fields believed, and `validate` reads them. typed.verify_frame()?; @@ -281,7 +281,7 @@ where let bytes = >::header_storage(&self.backing); let typed = bytemuck::checked::try_from_bytes::(&bytes[..size_of::()]) - .map_err(|_| ConsensusError::InvalidBitPattern)?; + .map_err(|_| classify_failed_cast::(bytes))?; // Before `validate`: a header that did not survive the link intact cannot // have any of its fields believed, and `validate` reads them. typed.verify_frame()?; @@ -477,7 +477,7 @@ where } let header = bytemuck::checked::try_from_bytes::(&bytes[..size_of::()]) - .map_err(|_| ConsensusError::InvalidBitPattern)?; + .map_err(|_| classify_failed_cast::(bytes))?; header.validate()?; // `size` is the whole-frame length and must at least span the header, or @@ -524,7 +524,7 @@ where } let header = bytemuck::checked::try_from_bytes::(&first.as_slice()[..size_of::()]) - .map_err(|_| ConsensusError::InvalidBitPattern)?; + .map_err(|_| classify_failed_cast::(first.as_slice()))?; header.validate()?; // See `TryFrom`: `size` must at least span the header so a @@ -774,6 +774,25 @@ impl MessageBag { } } +/// Why `H`'s header bytes failed bytemuck's checked cast. +/// +/// An operation discriminant this build does not define means the sender runs a +/// newer release; the frame is wire-valid and the node needs upgrading, which is +/// a different operator action from the corrupted-header case. bytemuck reports +/// both as one error, so the operation byte is probed here to separate them. +fn classify_failed_cast(bytes: &[u8]) -> ConsensusError +where + H: ConsensusHeader, +{ + if let Some(offset) = H::OPERATION_OFFSET + && let Some(&code) = bytes.get(offset) + && !Operation::is_known_code(code) + { + return ConsensusError::UnsupportedOperation { operation: code }; + } + ConsensusError::InvalidBitPattern +} + impl TryFrom> for MessageBag where T: ConsensusHeader, diff --git a/core/shard/src/metrics.rs b/core/shard/src/metrics.rs index 99f70a2240..6d2e725e6a 100644 --- a/core/shard/src/metrics.rs +++ b/core/shard/src/metrics.rs @@ -120,6 +120,14 @@ pub mod frame_drop_variant { /// replicated traffic has nobody to answer, so this is the only record the op /// was destroyed. pub mod frame_drop_reason { + /// Operation discriminant unknown to this build: the sender is newer. + /// + /// Distinct from `UNPARSABLE` because upgrading this node is the fix, and + /// until it is, the frame's consensus group gap-stops here. + pub const UNSUPPORTED_OPERATION: &str = "unsupported_operation"; + /// A consensus frame failed typed decode for any other reason (corrupt + /// header, bad size, client-bound command on the inbound path). + pub const UNPARSABLE: &str = "unparsable"; pub const FULL: &str = "full"; pub const DISCONNECTED: &str = "disconnected"; pub const UNROUTABLE: &str = "unroutable"; @@ -134,7 +142,7 @@ pub mod frame_drop_reason { // site actually produces it, so the unreachable corners of the 7 x 7 cross // product never appear as permanent zero-valued series. const VARIANT_COUNT: usize = 7; -const REASON_COUNT: usize = 7; +const REASON_COUNT: usize = 9; const VARIANTS: [&str; VARIANT_COUNT] = [ frame_drop_variant::CONSENSUS, @@ -147,6 +155,8 @@ const VARIANTS: [&str; VARIANT_COUNT] = [ ]; const REASONS: [&str; REASON_COUNT] = [ + frame_drop_reason::UNSUPPORTED_OPERATION, + frame_drop_reason::UNPARSABLE, frame_drop_reason::FULL, frame_drop_reason::DISCONNECTED, frame_drop_reason::UNROUTABLE, diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs index 4340761a00..d290c861ac 100644 --- a/core/shard/src/router.rs +++ b/core/shard/src/router.rs @@ -23,7 +23,7 @@ use crate::{IggyShard, LifecycleFrame, Receiver, RestorableMetadataStm, ShardFra use consensus::{MetadataHandle, PartitionsHandle}; use crossfire::TrySendError; use futures::FutureExt; -use iggy_binary_protocol::{GenericHeader, Operation, PrepareHeader}; +use iggy_binary_protocol::{ConsensusError, GenericHeader, Operation, PrepareHeader}; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; use message_bus::{ConnectionInstaller, MessageBus, ReplicaHandshakeDoneFn}; @@ -62,17 +62,33 @@ where pub fn dispatch(&self, message: Message) { let bag = match MessageBag::try_from(message) { Ok(bag) => bag, + Err(ConsensusError::UnsupportedOperation { operation }) => { + // Terminal for this consensus group, not a per-frame hiccup: + // the op is never journaled, never acked, and every later + // prepare dies on the resulting gap while quorum hides the + // outage. Repair wraps the same typed header, so it cannot + // rescue this node either -- only upgrading it can. Nothing + // fences the sending peer, so the log and the counter are the + // whole signal an operator gets. + self.metrics.record_frame_drop( + frame_drop_variant::CONSENSUS, + frame_drop_reason::UNSUPPORTED_OPERATION, + ); + tracing::error!( + shard = self.id, + operation = format_args!("{operation:#04x}"), + build_release = iggy_binary_protocol::IGGY_PROTOCOL_VERSION, + "consensus frame carries an operation this build does not know; the sender \ + runs a newer release. This node cannot journal or ack it, so its consensus \ + group stops making progress until this node is upgraded" + ); + return; + } Err(e) => { - // TODO(hubcio): this drop is the whole story for a consensus - // frame carrying an Operation this build does not know: no - // metric, no peer error, no eviction. An old node in a mixed - // cluster silently gap-stops the group here (never journals - // the op, never PrepareOks, every later prepare dies on the - // gap check) while quorum hides it, and repair wraps the same - // typed header so it cannot rescue. Rolling upgrades across - // consensus-op additions need a version fence (release_min / - // release_max bounds on the replica plane) before this arm is - // safe to hit. + self.metrics.record_frame_drop( + frame_drop_variant::CONSENSUS, + frame_drop_reason::UNPARSABLE, + ); tracing::warn!(shard = self.id, error = %e, "dropping unparsable consensus frame"); return; } @@ -804,25 +820,19 @@ mod tests { use server_common::{MESSAGE_ALIGN, Message, MessageBag}; use std::mem::offset_of; - /// RED SPEC, expected to FAIL: pins the mixed-cluster upgrade hole in - /// `dispatch`'s decode seam. - /// /// An `Operation` discriminant this build does not know, arriving on an /// otherwise wire-valid consensus frame (correct command, size, checksum: - /// exactly what a newer release sends after an op addition), decodes to - /// the same undifferentiated `ConsensusError::InvalidBitPattern` as random - /// memory corruption. `dispatch` answers both identically: a warn log and - /// a dropped frame. No metric, no peer error, no eviction, no version - /// fence. An old node in a mixed cluster therefore gap-stops its consensus - /// group silently (never journals the op, never sends a `PrepareOk`, every - /// later prepare dies on the gap check) while quorum hides the outage. + /// exactly what a newer release sends after an op addition), must decode to + /// its own error rather than the `InvalidBitPattern` that random memory + /// corruption produces. /// - /// Passes once the decode surfaces a dedicated unsupported-operation - /// signal the router can fence and account, instead of collapsing it into - /// the corruption error. + /// The two need different operator actions: version skew is fixed by + /// upgrading this node, and until it is, the frame's consensus group makes + /// no progress (the op is never journaled, never acked, and every later + /// prepare dies on the gap). `dispatch` splits its drop arms on this + /// distinction; the accounting half is pinned in the server crate by + /// `given_an_unknown_operation_when_dispatched_should_account_an_upgrade_fence_drop`. #[test] - // TODO(hubcio): fix this test - #[ignore = "unknown operation collapses into InvalidBitPattern; no upgrade fence"] fn given_an_unknown_operation_when_a_consensus_frame_decodes_should_surface_an_upgrade_fence_signal() { // Far past every defined Operation discriminant (the highest is 165). @@ -858,7 +868,12 @@ mod tests { }; assert!( - !matches!(error, ConsensusError::InvalidBitPattern), + matches!( + error, + ConsensusError::UnsupportedOperation { + operation: OPERATION_FROM_A_NEWER_RELEASE + } + ), "unknown operation {OPERATION_FROM_A_NEWER_RELEASE:#x} is silently dropped: the \ typed decode collapses a wire-valid frame from a newer release into the same \ InvalidBitPattern as corruption, and dispatch drops both with only a warn log, \ From 76460a10bc98b6db6c30a6991a4370bcf08ec7d0 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Thu, 20 Aug 2026 19:10:40 +0200 Subject: [PATCH 02/16] fix(consensus): keep a client's dedup fence across capacity eviction --- core/consensus/src/client_table.rs | 339 +++++++++++++++++- .../tests/cluster/client_table_adversarial.rs | 40 +-- 2 files changed, 348 insertions(+), 31 deletions(-) diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 1294558e44..506a61aa99 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs @@ -103,6 +103,34 @@ pub const CLIENTS_TABLE_SLOT_MAX: usize = 1 << 16; /// refcount bumps and are never persisted or transferred. pub const REPLY_RING_CAPACITY: usize = 5; +/// What capacity eviction keeps after reclaiming an entry's replies. +/// +/// At-most-once needs only the fence: the watermark says which request numbers +/// already committed, and the ring merely supplies the bytes to replay. Dropping +/// the whole entry made an evicted client's resume mint at watermark zero, so the +/// retry of a committed request re-executed. Keeping it lets the resume answer +/// from the fence instead: [`RequestStatus::Duplicate`] when the watermark's +/// reply was still ringed, [`RequestStatus::AlreadyApplied`] when it was not. +/// +/// In memory only, like the ring: a fence does not survive a checkpoint or a state +/// transfer, and losing one degrades a resume to the pre-fix behaviour rather than +/// corrupting anything. +#[derive(Debug)] +struct EvictedFence { + client_id: u128, + epoch: u64, + user_id: u32, + watermark: u64, + watermark_checksum: u128, + /// The watermark request's own reply when the ring still held it, so the + /// retry the resume contract prescribes replays its original bytes. `None` + /// when it had already aged out, or when a rebind left the register reply + /// as the newest entry: the resume then answers + /// [`RequestStatus::AlreadyApplied`], which still never re-executes. + /// One refcount bump, not a copy. + latest: Option, +} + /// Per-session entry: fence epoch + committed-request watermark + replies. /// /// The key (`client_id` today, the stable `session_id` once SDK identity @@ -407,6 +435,17 @@ pub struct ClientTable { slots: Vec>, /// `client_id` -> slot index. Rebuilt on decode. index: HashMap, + /// Fences of clients capacity eviction reclaimed, oldest at the front. + /// + /// Bounded by the slot count. A fence is the entry's header fields plus, at + /// most, the watermark request's own reply, so it costs a fraction of the + /// entry it replaces. Trimmed oldest-first. + /// + /// Replica-local best-effort, NOT replicated state: the bound is + /// `slots.len()`, which `from_snapshot` and `decode` size per node, and a + /// state transfer replaces the table wholesale. Losing a fence degrades a + /// resume to the pre-fence behaviour; it never makes one more permissive. + evicted_fences: VecDeque, } /// Whether two integrity stamps for the same request number disagree. @@ -426,6 +465,7 @@ impl ClientTable { Self { slots, index: HashMap::with_capacity(max_clients), + evicted_fences: VecDeque::new(), } } @@ -558,7 +598,12 @@ impl ClientTable { latest_commit, }); } - Ok(Self { slots, index }) + Ok(Self { + slots, + index, + // Fences are in-memory only; a restored table starts with none. + evicted_fences: VecDeque::new(), + }) } /// Check a request against the table. Epoch fence first, then the @@ -654,6 +699,11 @@ impl ClientTable { /// /// Full table evicts the oldest commit, see [`Self::evict_oldest`]. /// + /// A key this table evicted for capacity re-registers as a fresh entry that + /// RESTORES the evicted watermark (and the watermark reply when it survived) + /// from [`EvictedFence`], for the same `user_id` only. A committed `Logout` + /// forgets that fence, so a register after one starts clean. + /// /// # Panics /// If `client_id == 0` or `client_id != reply.header().client`. pub fn commit_register(&mut self, client_id: u128, user_id: u32, reply: Message) { @@ -689,6 +739,17 @@ impl ClientTable { .retain(|stored| stored.header().request != REGISTER_REQUEST_ID); entry.push_latest(cached); } else { + // A client this table evicted for capacity is resuming, not + // arriving: its committed request numbers must stay deduped, or the + // retry the resume contract prescribes re-executes. The watermark's + // own reply comes back with the fence when the ring still held it, + // so that retry replays its bytes; every other retry at or below the + // watermark answers `AlreadyApplied`, which also never re-executes. + // + // Same identity only: `client_id` is client-supplied, so a fence + // must never hand one user another user's dedup history, nor its + // cached reply bytes, merely because the key was reused. + let fence = self.take_fence(client_id, user_id); let freed = if self.index.len() >= self.slots.len() { self.evict_oldest() } else { @@ -697,16 +758,27 @@ impl ClientTable { let slot_idx = freed .or_else(|| self.first_free_slot()) .expect("eviction must free a slot"); + debug_assert!( + fence.as_ref().is_none_or(|fence| epoch > fence.epoch), + "commit_register: revived fence epoch regression" + ); let latest_commit = cached.header().commit; let mut ring = VecDeque::with_capacity(REPLY_RING_CAPACITY); + // Oldest at the front: the retained reply committed before this + // register did, and `latest()` must stay the register's own reply. + if let Some(replay) = fence.as_ref().and_then(|fence| fence.latest.as_ref()) { + ring.push_back(replay.clone()); + } ring.push_back(cached); self.slots[slot_idx] = Some(ClientEntry { epoch, user_id, client_id, latest_commit, - watermark: REGISTER_REQUEST_ID, - watermark_checksum: 0, + watermark: fence + .as_ref() + .map_or(REGISTER_REQUEST_ID, |fence| fence.watermark), + watermark_checksum: fence.as_ref().map_or(0, |fence| fence.watermark_checksum), ring, }); self.index.insert(client_id, slot_idx); @@ -797,9 +869,14 @@ impl ClientTable { .find(|stored| stored.header().request == new_request) { *stored = cached; - // The watermark's reply is the ring's back, so replacing it in - // place moves the latest commit without a push. - entry.latest_commit = new_commit; + // Re-derived, not assigned from `new_commit`: the replaced entry + // is not necessarily the ring's back. A rebind pushes the + // register reply last, and a fence-revived entry carries the + // watermark's reply at the front, so assuming otherwise lets + // `latest_commit` disagree with what `decode` rebuilds from + // `ring.back()` -- and it is `evict_oldest`'s only ranking key, + // so the two would pick different victims from one log. + entry.latest_commit = entry.latest().header().commit; } else { entry.push_latest(cached); } @@ -830,6 +907,13 @@ impl ClientTable { /// /// [`Operation::Register`]: iggy_binary_protocol::Operation pub fn remove_client(&mut self, client_id: u128) -> bool { + // A committed Logout is the explicit end of the session, so it also + // forgets any fence a capacity eviction left for this key. Otherwise a + // Logout that commits after the eviction (its prepare predates it, so + // there is no entry left to drop) would strand a fence, and a later + // register would revive a watermark the client had already ended. + self.evicted_fences + .retain(|fence| fence.client_id != client_id); let Some(slot_idx) = self.index.remove(&client_id) else { return false; }; @@ -849,9 +933,14 @@ impl ClientTable { /// session that every backup drops. /// /// A client with an uncommitted prepare is therefore evictable. Its - /// commit lands as [`CommitReply::NoEntry`] -- the reply still ships, and - /// the client learns the session is gone on its next request (`NoSession` - /// -> eviction frame -> re-register). + /// commit lands as [`CommitReply::NoEntry`] -- the reply still ships, the + /// client learns the session is gone on its next request (`NoSession` -> + /// eviction frame -> re-register), and that commit reaches no fence, so a + /// resume can re-execute exactly that request. + /// + /// The evicted session's dedup fence survives via [`Self::remember_fence`] + /// unless it had committed nothing or the fence is later trimmed, so the + /// re-registering client is normally answered rather than re-executed. /// /// **Caveat**: eviction erases the evicted session's watermark, so its /// next retry is treated as `New` (re-executes). Bounded by table @@ -876,6 +965,10 @@ impl ClientTable { let (slot_idx, _) = evictee?; let entry = self.slots[slot_idx].take().expect("evictee must exist"); self.index.remove(&entry.client_id); + // Reclaim the replies, keep the fence: the evicted client's own resume + // must not read as a first-time register, or the retry of a committed + // request re-executes. + self.remember_fence(&entry); trace!( client_id = entry.client_id, "evict_oldest: removed client from session table" @@ -883,6 +976,56 @@ impl ClientTable { Some(slot_idx) } + /// Record an evicted entry's dedup fence, trimming oldest-first. + fn remember_fence(&mut self, entry: &ClientEntry) { + // Nothing committed under this session, so there is nothing to dedup. + // Worth skipping rather than storing: `evict_oldest` ranks on the oldest + // `latest_commit`, and a session idle since its register carries its own + // register op, which makes these the PREFERRED victims -- storing them + // would crowd real fences out of a store bounded by the slot count. + if entry.watermark == REGISTER_REQUEST_ID { + return; + } + // One fence per identity: a later eviction supersedes the earlier one, + // and two fences for one key would let the older (lower) watermark be + // found first and revive a stale one. + self.evicted_fences + .retain(|fence| fence.client_id != entry.client_id || fence.user_id != entry.user_id); + self.evicted_fences.push_back(EvictedFence { + client_id: entry.client_id, + epoch: entry.epoch, + user_id: entry.user_id, + watermark: entry.watermark, + watermark_checksum: entry.watermark_checksum, + latest: entry.find_cached(entry.watermark).cloned(), + }); + while self.evicted_fences.len() > self.slots.len() { + self.evicted_fences.pop_front(); + } + } + + /// Take back the fence a previous capacity eviction left for this + /// `(client_id, user_id)` pair. The identity half is the security-relevant + /// one: `client_id` arrives off the wire. + /// + /// Linear because it runs only on a register that missed the index, which is + /// a consensus commit and already far dearer than a scan of at most + /// `slots.len()` fences. + fn take_fence(&mut self, client_id: u128, user_id: u32) -> Option { + // Both fields in the predicate, not a client_id match with the identity + // checked afterwards: `client_id` is client-supplied, so the store can + // legitimately hold one fence per user for the same key. Matching on the + // id alone would let whichever fence sits nearer the front shadow the + // caller's own, handing it a fresh watermark and re-executing a request + // it had already committed. It also leaves another user's fence in place + // rather than consuming it. + let position = self + .evicted_fences + .iter() + .position(|fence| fence.client_id == client_id && fence.user_id == user_id)?; + self.evicted_fences.remove(position) + } + fn first_free_slot(&self) -> Option { self.slots.iter().position(Option::is_none) } @@ -1245,6 +1388,184 @@ mod tests { /// assert on it (see `register_stores_user_id` for the accessor check). const TEST_USER_ID: u32 = 7; + /// Capacity eviction reclaims an entry's replies but must not reset its + /// dedup fence: the evicted client's own resume is a rebind in everything + /// but bookkeeping, and the resume contract has it retry the request it + /// never saw answered. + #[test] + fn eviction_keeps_the_fence_so_a_resumed_client_is_not_re_executed() { + const CLIENT_A: u128 = 0xA11CE; + const CHURN: [u128; 2] = [0xB0B1, 0xB0B2]; + + let mut table = ClientTable::new(2); + table.commit_register(CLIENT_A, TEST_USER_ID, make_register_reply(CLIENT_A, 1)); + // Request 1 commits for A, so its watermark is 1. + table.commit_reply(CLIENT_A, make_reply_for(CLIENT_A, 1, 2)); + + // Two fresh registers fill the table and evict A (oldest commit). + for (offset, churn) in CHURN.iter().enumerate() { + let commit = 3 + offset as u64; + table.commit_register(*churn, TEST_USER_ID, make_register_reply(*churn, commit)); + } + assert!( + table.get_epoch(CLIENT_A).is_none(), + "the churn must have evicted A for this test to mean anything" + ); + + // A resumes: fresh register under the same id, then retries request 1. + table.commit_register(CLIENT_A, TEST_USER_ID, make_register_reply(CLIENT_A, 10)); + let resumed_epoch = table.get_epoch(CLIENT_A).expect("resume registered"); + + match table.check_request(CLIENT_A, resumed_epoch, 1, 0) { + RequestStatus::Duplicate(replayed) => { + assert_eq!( + replayed.header().request, + 1, + "the retained reply must be the watermark request's own" + ); + } + other => panic!( + "a committed request retried after capacity eviction must replay its cached \ + reply, not be executed a second time; got {other:?}" + ), + } + + // A request above the restored watermark is still new. + assert!(matches!( + table.check_request(CLIENT_A, resumed_epoch, 2, 0), + RequestStatus::New + )); + } + + /// `client_id` is client-supplied, so a fence belongs to the user that + /// earned it: a register under a different identity must neither inherit the + /// dedup history (it would be handed another user's cached reply bytes) nor + /// consume the fence (anyone could then erase another client's history just + /// by presenting its key). + #[test] + fn a_fence_is_neither_inherited_nor_consumed_by_a_different_user() { + const CLIENT_A: u128 = 0xA11CE; + const OTHER_USER: u32 = TEST_USER_ID + 1; + + let mut table = ClientTable::new(2); + table.commit_register(CLIENT_A, TEST_USER_ID, make_register_reply(CLIENT_A, 1)); + table.commit_reply(CLIENT_A, make_reply_for(CLIENT_A, 1, 2)); + for (offset, churn) in [0xB0B1u128, 0xB0B2].iter().enumerate() { + let commit = 3 + offset as u64; + table.commit_register(*churn, TEST_USER_ID, make_register_reply(*churn, commit)); + } + assert!( + table.get_epoch(CLIENT_A).is_none(), + "the churn must have evicted A, leaving its fence" + ); + + table.commit_register(CLIENT_A, OTHER_USER, make_register_reply(CLIENT_A, 10)); + let squatter_epoch = table.get_epoch(CLIENT_A).expect("registered"); + assert!( + matches!( + table.check_request(CLIENT_A, squatter_epoch, 1, 0), + RequestStatus::New + ), + "a different user must start at a fresh watermark, not inherit the fence" + ); + assert!( + table + .evicted_fences + .iter() + .any(|fence| fence.client_id == CLIENT_A && fence.user_id == TEST_USER_ID), + "the owner's fence must survive a register under another identity" + ); + } + + /// Two fences can share a `client_id` with different users (the key is + /// client-supplied). Lookup must find the caller's own fence rather than + /// stopping at whichever one happens to sit closer to the front. + #[test] + fn a_fence_is_found_behind_another_users_fence_for_the_same_client_id() { + const CLIENT_A: u128 = 0xA11CE; + const FIRST_USER: u32 = TEST_USER_ID; + const SECOND_USER: u32 = TEST_USER_ID + 1; + + let mut table = ClientTable::new(2); + for (user, watermark) in [(FIRST_USER, 1u64), (SECOND_USER, 4u64)] { + table.evicted_fences.push_back(EvictedFence { + client_id: CLIENT_A, + epoch: watermark, + user_id: user, + watermark, + watermark_checksum: 0, + latest: Some(CachedReply::from_message(make_reply_for( + CLIENT_A, watermark, watermark, + ))), + }); + } + + let fence = table + .take_fence(CLIENT_A, SECOND_USER) + .expect("the second user's own fence must be reachable behind the first user's"); + assert_eq!(fence.watermark, 4); + assert!( + table + .evicted_fences + .iter() + .any(|fence| fence.user_id == FIRST_USER), + "and taking it must leave the other user's fence in place" + ); + assert!( + table.take_fence(CLIENT_A, SECOND_USER).is_none(), + "a fence is consumed on the hit, so a re-minted key cannot revive a \ + stale watermark and swallow a fresh session's requests" + ); + } + + /// A committed `Logout` ends the session explicitly, so it must not leave a + /// fence behind for a later register to revive: the client asked to be + /// forgotten, and a Logout committing after its entry was evicted finds + /// nothing to drop. + #[test] + fn logout_forgets_an_evicted_fence() { + const CLIENT_A: u128 = 0xA11CE; + + let mut table = ClientTable::new(2); + table.commit_register(CLIENT_A, TEST_USER_ID, make_register_reply(CLIENT_A, 1)); + table.commit_reply(CLIENT_A, make_reply_for(CLIENT_A, 1, 2)); + for (offset, churn) in [0xB0B1u128, 0xB0B2].iter().enumerate() { + let commit = 3 + offset as u64; + table.commit_register(*churn, TEST_USER_ID, make_register_reply(*churn, commit)); + } + assert!(table.get_epoch(CLIENT_A).is_none(), "A must be evicted"); + + table.remove_client(CLIENT_A); + table.commit_register(CLIENT_A, TEST_USER_ID, make_register_reply(CLIENT_A, 10)); + let epoch = table.get_epoch(CLIENT_A).expect("registered again"); + assert!( + matches!( + table.check_request(CLIENT_A, epoch, 1, 0), + RequestStatus::New + ), + "a register after Logout must start fresh, not revive the ended session's watermark" + ); + } + + /// The fence store is bounded by the slot count, so a churn far longer than + /// the table cannot grow it without limit. + #[test] + fn evicted_fences_stay_bounded_by_the_slot_count() { + let mut table = ClientTable::new(2); + // Each client commits an app request, so eviction has a real fence to + // keep -- a register-only session is skipped on purpose. + for client in 1..=20u128 { + let commit = (client * 2) as u64; + table.commit_register(client, TEST_USER_ID, make_register_reply(client, commit)); + table.commit_reply(client, make_reply_for(client, 1, commit + 1)); + } + assert_eq!( + table.evicted_fences.len(), + table.slots.len(), + "fences must be trimmed to the slot count" + ); + } + #[allow(clippy::cast_possible_truncation)] fn make_register_reply(client: u128, commit: u64) -> Message { let header_size = std::mem::size_of::(); diff --git a/core/integration/tests/cluster/client_table_adversarial.rs b/core/integration/tests/cluster/client_table_adversarial.rs index 10af0e749b..90c1ae245c 100644 --- a/core/integration/tests/cluster/client_table_adversarial.rs +++ b/core/integration/tests/cluster/client_table_adversarial.rs @@ -17,14 +17,14 @@ //! Adversarial specs against the VSR client table's at-most-once guarantees. //! -//! Both tests are RED SPECS, expected to FAIL: they assert the dedup contract -//! a retrying client needs, and the current table cannot honour it at its two -//! resource edges. +//! Both assert the dedup contract a retrying client needs at the table's two +//! resource edges. The capacity one now passes; the reply-ring one is still a +//! RED SPEC, expected to FAIL. //! -//! 1. Capacity: a full table evicts the entry with the oldest commit, and the -//! eviction erases that client's request watermark. A client that was -//! merely quiet (not gone) re-registers and its retry of an -//! already-committed request id re-executes. +//! 1. Capacity: a full table evicts the entry with the oldest commit. Eviction +//! keeps that client's request watermark (and the watermark's reply when the +//! ring still held it), so a client that was merely quiet re-registers and +//! its retry of an already-committed request id is answered, not re-executed. //! 2. Reply ring: each entry retains only its `REPLY_RING_CAPACITY` most //! recent committed replies. A retry of a request whose reply aged out is //! refused with the terminal `RequestAlreadyApplied` and no result payload, @@ -77,25 +77,21 @@ const REPLY_WAIT: Duration = Duration::from_secs(5); const RETRY_PAUSE: Duration = Duration::from_millis(100); -/// RED SPEC, expected to FAIL: capacity eviction must not erase a live -/// client's dedup watermark. +/// Capacity eviction must not erase a live client's dedup watermark. /// /// With the table floored at two slots, three fresh registrations evict /// `CLIENT_A` (its commit is the oldest) while its connection is still open /// and its request 1 is committed. The client then does exactly what the /// resume contract tells a disconnected client to do: reconnect, /// re-authenticate under its own identity, and retry the request it never saw -/// answered. The register finds no entry to rebind, mints a fresh one at -/// watermark zero, and the retry of the committed request re-executes. +/// answered. The register finds no entry to rebind, so it restores the fence +/// eviction left and the retry is answered from it. /// -/// The proof of re-execution is the committed duplicate-name rejection: a -/// dedup hit replays the cached success bytes, so any committed rejection -/// means the state machine ran the operation a second time. At-most-once -/// holds only for clients the table happened not to evict. -// TODO(hubcio): fix this test -#[ignore = "capacity eviction erases a live client's dedup watermark; replay re-executes"] +/// A committed duplicate-name rejection is the proof of re-execution: a dedup +/// hit replays the cached success bytes, so any committed rejection means the +/// state machine ran the operation a second time. #[iggy_harness(cluster_nodes = 1, server(metadata.clients_table_max = "2"))] -async fn given_a_low_client_table_cap_when_connects_churn_should_erase_a_live_dedup_watermark( +async fn given_a_low_client_table_cap_when_connects_churn_should_keep_a_live_dedup_watermark( harness: &mut TestHarness, ) { let addr = tcp_addr(harness); @@ -141,10 +137,10 @@ async fn given_a_low_client_table_cap_when_connects_churn_should_erase_a_live_de other => panic!( "capacity eviction erased a live client's dedup watermark: request 1 was \ committed and its reply delivered, but after the table (capacity 2) evicted \ - the entry to admit churn registrations, the resume re-registered at watermark \ - zero and the retry of request 1 was re-executed by the state machine instead \ - of being answered from the dedup cache (at-most-once broken for any client \ - the table evicts while it is merely quiet); got {other:?}" + the entry to admit churn registrations, the resume did not restore the fence, \ + so the retry of request 1 was re-executed by the state machine instead of \ + being answered from the dedup cache (at-most-once broken for any client the \ + table evicts while it is merely quiet); got {other:?}" ), } } From 99f5adf7e2f3d7f3daf18749b32118a9ceda6211 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Fri, 21 Aug 2026 11:53:42 +0200 Subject: [PATCH 03/16] feat(sdk): fail over to a surviving node when the current one dies --- .../binary_impls/personal_access_tokens.rs | 9 +- core/common/src/traits/binary_impls/users.rs | 10 +- core/common/src/traits/binary_transport.rs | 12 +- .../tcp_config/tcp_client_config.rs | 11 + .../tcp_config/tcp_client_config_builder.rs | 38 ++ .../cluster/failover_client_continuity.rs | 47 ++- core/sdk/src/client_provider.rs | 3 + core/sdk/src/leader_aware.rs | 113 +++++- core/sdk/src/quic/quic_client.rs | 5 +- core/sdk/src/tcp/tcp_client.rs | 253 +++++++++++-- core/sdk/src/websocket/websocket_client.rs | 5 +- .../Implementations/TcpMessageStream.Vsr.cs | 21 ++ .../Implementations/TcpMessageStream.cs | 129 ++++++- .../VsrTests/DialCandidatesTests.cs | 64 ++++ .../VsrTests/EndpointFailoverTests.cs | 346 ++++++++++++++++++ foreign/go/client/tcp/tcp_core.go | 149 +++++--- foreign/go/client/tcp/tcp_failover_test.go | 155 ++++++++ .../tcp/tcp_session_credentials_test.go | 66 ++++ .../go/client/tcp/tcp_session_management.go | 15 +- .../client/async/tcp/AsyncIggyTcpClient.java | 74 +++- .../client/async/tcp/LeaderAwareness.java | 53 ++- .../client/async/tcp/LoginRoutingHook.java | 6 + .../iggy/client/async/tcp/ReconnectPlan.java | 19 +- .../iggy/client/async/tcp/UsersTcpClient.java | 1 + ...syncIggyTcpClientEndpointFailoverTest.java | 307 ++++++++++++++++ .../client/async/tcp/LeaderAwarenessTest.java | 26 +- .../client/async/tcp/ReconnectPlanTest.java | 24 +- .../node/src/client/client.connection.test.ts | 25 ++ foreign/node/src/client/client.connection.ts | 94 +++-- foreign/node/src/client/client.socket.test.ts | 105 ++++++ foreign/node/src/client/client.socket.ts | 7 + 31 files changed, 2001 insertions(+), 191 deletions(-) create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/DialCandidatesTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs create mode 100644 foreign/go/client/tcp/tcp_failover_test.go create mode 100644 foreign/go/client/tcp/tcp_session_credentials_test.go create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java diff --git a/core/common/src/traits/binary_impls/personal_access_tokens.rs b/core/common/src/traits/binary_impls/personal_access_tokens.rs index 7e1299f284..d46fef8ba0 100644 --- a/core/common/src/traits/binary_impls/personal_access_tokens.rs +++ b/core/common/src/traits/binary_impls/personal_access_tokens.rs @@ -18,8 +18,9 @@ use crate::traits::binary_auth::fail_if_not_authenticated; use crate::wire_conversions::personal_access_tokens_from_wire; use crate::{ - BinaryClient, ClientState, DiagnosticEvent, IdentityInfo, IggyError, PersonalAccessTokenClient, - PersonalAccessTokenExpiry, PersonalAccessTokenInfo, RawPersonalAccessToken, + BinaryClient, ClientState, Credentials, DiagnosticEvent, IdentityInfo, IggyError, + PersonalAccessTokenClient, PersonalAccessTokenExpiry, PersonalAccessTokenInfo, + RawPersonalAccessToken, }; use iggy_binary_protocol::MAX_WIRE_NAME_LENGTH; use iggy_binary_protocol::WireName; @@ -134,6 +135,10 @@ impl PersonalAccessTokenClient for B { "authenticated against iggy server" ); self.set_state(ClientState::Authenticated).await; + self.remember_session_credentials(Credentials::PersonalAccessToken(SecretString::from( + token.to_string(), + ))) + .await; self.publish_event(DiagnosticEvent::SignedIn).await; Ok(IdentityInfo { user_id: wire_resp.user_id, diff --git a/core/common/src/traits/binary_impls/users.rs b/core/common/src/traits/binary_impls/users.rs index eb785109a6..c199f76de9 100644 --- a/core/common/src/traits/binary_impls/users.rs +++ b/core/common/src/traits/binary_impls/users.rs @@ -18,8 +18,8 @@ use crate::traits::binary_auth::fail_if_not_authenticated; use crate::wire_conversions::{identifier_to_wire, permissions_to_wire, users_from_wire}; use crate::{ - BinaryClient, ClientState, DiagnosticEvent, Identifier, IdentityInfo, IggyError, Permissions, - UserClient, UserInfo, UserInfoDetails, UserStatus, UserUpdateOptions, + BinaryClient, ClientState, Credentials, DiagnosticEvent, Identifier, IdentityInfo, IggyError, + Permissions, UserClient, UserInfo, UserInfoDetails, UserStatus, UserUpdateOptions, }; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::codes::LOGIN_REGISTER_CODE; @@ -218,6 +218,11 @@ impl UserClient for B { "authenticated against iggy server" ); self.set_state(ClientState::Authenticated).await; + self.remember_session_credentials(Credentials::UsernamePassword( + username.to_owned(), + SecretString::from(password.to_string()), + )) + .await; self.publish_event(DiagnosticEvent::SignedIn).await; Ok(IdentityInfo { user_id: wire_resp.user_id, @@ -229,6 +234,7 @@ impl UserClient for B { fail_if_not_authenticated(self).await?; self.send_raw_with_response(LOGOUT_USER_CODE, LogoutUserRequest.to_bytes()) .await?; + self.forget_session_credentials().await; self.reset_vsr_session().await?; self.set_state(ClientState::Connected).await; self.publish_event(DiagnosticEvent::SignedOut).await; diff --git a/core/common/src/traits/binary_transport.rs b/core/common/src/traits/binary_transport.rs index d3c2bd2e3a..9c6a923b62 100644 --- a/core/common/src/traits/binary_transport.rs +++ b/core/common/src/traits/binary_transport.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::{ClientState, DiagnosticEvent, IggyDuration, IggyError}; +use crate::{ClientState, Credentials, DiagnosticEvent, IggyDuration, IggyError}; use async_trait::async_trait; use bytes::Bytes; use std::sync::Arc; @@ -51,6 +51,16 @@ mod vsr_session_sealed { pub trait VsrSessionControl: vsr_session_sealed::Sealed + BinaryTransport { async fn bind_vsr_session(&self, session: u64) -> Result<(), IggyError>; async fn reset_vsr_session(&self) -> Result<(), IggyError>; + /// Keep the credentials a sign-in succeeded with, so a transport that + /// loses its connection can re-establish the session -- on this node or, + /// after failing over, on another one. A caller that signs in by hand is + /// otherwise less reconnectable than one that configures `AutoLogin`, + /// which is a surprising difference between two ways of doing the same + /// thing. Transports that cannot reconnect leave this a no-op. + async fn remember_session_credentials(&self, _credentials: Credentials) {} + /// Drop them: after an explicit logout there is no session to restore, + /// and a reconnect must not resurrect one. + async fn forget_session_credentials(&self) {} /// SDK crate version sent in the login-register version prefix. /// Implemented by the transports so the value is the SDK crate's own /// `CARGO_PKG_VERSION` (`iggy` for Rust), not `iggy_common`'s. diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_config.rs b/core/common/src/types/configuration/tcp_config/tcp_client_config.rs index 86997c6d7a..e3584130fc 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_config.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_config.rs @@ -26,6 +26,12 @@ use std::str::FromStr; pub struct TcpClientConfig { /// The address of the Iggy server. pub server_address: String, + /// Addresses of other nodes of the same cluster, dialed in order when + /// `server_address` cannot be reached. The roster the server reports is + /// remembered while the client is connected and dialed first, so these + /// seeds only have to be enough to reach the cluster once -- at the very + /// first connect, when nothing has been learned yet. + pub failover_addresses: Vec, /// Whether to use TLS when connecting to the server. pub tls_enabled: bool, /// The domain to use for TLS when connecting to the server. @@ -49,6 +55,7 @@ impl Default for TcpClientConfig { fn default() -> TcpClientConfig { TcpClientConfig { server_address: "127.0.0.1:8090".to_string(), + failover_addresses: Vec::new(), tls_enabled: false, tls_domain: "".to_string(), tls_ca_file: None, @@ -65,6 +72,10 @@ impl From> for TcpClientConfig { fn from(connection_string: ConnectionString) -> Self { TcpClientConfig { server_address: connection_string.server_address().into(), + // The connection-string grammar names a single host, so a client + // built from one starts with no seeds and learns the roster once + // it is connected. + failover_addresses: Vec::new(), auto_login: connection_string.auto_login().to_owned(), tls_enabled: connection_string.options().tls_enabled(), tls_domain: connection_string.options().tls_domain().into(), diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs b/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs index 6f665a5777..607573279d 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs @@ -20,6 +20,7 @@ use crate::{AutoLogin, IggyDuration, IggyError, TcpClientConfig, validate_server /// Builder for the TCP client configuration. /// Allows configuring the TCP client with custom settings or using defaults: /// - `server_address`: Default is "127.0.0.1:8090" +/// - `failover_addresses`: Default is empty. /// - `auto_login`: Default is AutoLogin::Disabled. /// - `reconnection`: Default is enabled unlimited retries and 1 second interval. /// - `tls_enabled`: Default is false. @@ -41,6 +42,13 @@ impl TcpClientConfigBuilder { self } + /// Sets the addresses of other nodes of the same cluster, dialed in order + /// when `server_address` cannot be reached. + pub fn with_failover_addresses(mut self, failover_addresses: Vec) -> Self { + self.config.failover_addresses = failover_addresses; + self + } + /// Sets the auto sign in during connection. pub fn with_auto_sign_in(mut self, auto_sign_in: AutoLogin) -> Self { self.config.auto_login = auto_sign_in; @@ -105,6 +113,10 @@ impl TcpClientConfigBuilder { pub fn build(mut self) -> Result { self.config.server_address = self.config.server_address.trim().to_owned(); validate_server_address(&self.config.server_address)?; + for failover_address in &mut self.config.failover_addresses { + *failover_address = failover_address.trim().to_owned(); + validate_server_address(failover_address)?; + } Ok(self.config) } @@ -182,6 +194,32 @@ mod tests { )); } + #[test] + fn valid_failover_addresses_should_succeed() { + let config = builder_with_address("127.0.0.1:8090") + .with_failover_addresses(vec![ + " 127.0.0.1:8091 ".to_string(), + "iggy-server-3:8090".to_string(), + ]) + .build() + .expect("build the configuration"); + + assert_eq!( + config.failover_addresses, + vec!["127.0.0.1:8091", "iggy-server-3:8090"] + ); + } + + #[test] + fn malformed_failover_address_should_fail() { + let builder = builder_with_address("127.0.0.1:8090") + .with_failover_addresses(vec!["127.0.0.1".to_string()]); + assert!(matches!( + builder.build(), + Err(IggyError::InvalidIpAddress(_, _)) + )); + } + #[test] fn docker_compose_service_name_should_succeed() { let builder = builder_with_address("iggy-server:8090"); diff --git a/core/integration/tests/cluster/failover_client_continuity.rs b/core/integration/tests/cluster/failover_client_continuity.rs index ab4c93f82f..9eeed6c0dd 100644 --- a/core/integration/tests/cluster/failover_client_continuity.rs +++ b/core/integration/tests/cluster/failover_client_continuity.rs @@ -15,19 +15,19 @@ // specific language governing permissions and limitations // under the License. -//! RED SPEC, expected to FAIL: client continuity across a primary SIGKILL. +//! Client continuity across a primary SIGKILL. //! //! A producing SDK client pinned to the primary must, after the primary dies, //! complete its next operation against the surviving quorum within a small -//! budget and without an authentication error. The SDK cannot: it has no -//! multi-endpoint failover. A client is built around a single -//! `server_address`, so it knows no other endpoint to dial; the transport's -//! fail-fast gate (auto-login disabled, the shape this harness client runs -//! with) returns errors without attempting a reconnect; and the -//! leader-redirect machinery that could reroute it needs a live connection to -//! read the cluster roster. Every retry therefore redials the dead endpoint -//! and fails with a connection error until the caller gives up. Surviving a -//! primary crash needs a seed roster of endpoints, not just a redirect. +//! budget and without an authentication error. Three separate pieces of +//! client state make that possible, and the test fails if any one of them is +//! lost: the endpoints the cluster roster named while the connection was +//! healthy (the roster is unreachable exactly when it is needed), the +//! credentials the sign-in succeeded with (this harness client signs in by +//! hand rather than configuring `AutoLogin`, and a reconnect has to +//! re-establish the session on whichever node answers), and a reconnect that +//! dials those endpoints in turn instead of redialing the address the client +//! was configured with. use std::time::Duration; @@ -64,8 +64,6 @@ fn build_message(payload: &str) -> IggyMessage { /// A producing client pinned to the primary; SIGKILL the primary mid-stream; /// the same client's next send must succeed against the surviving quorum /// within `RESUME_BUDGET` and must never surface Unauthenticated. -// TODO(hubcio): fix this test -#[ignore = "SDK has no multi-endpoint failover; client redials the dead primary forever"] #[iggy_harness(cluster_nodes = 3)] async fn given_a_client_producing_when_its_primary_is_killed_should_resume_without_hang_or_unauthenticated( harness: &mut TestHarness, @@ -97,6 +95,11 @@ async fn given_a_client_producing_when_its_primary_is_killed_should_resume_witho // Pin the producing client to the primary's own endpoint, the way a // leader-aware SDK ends up connected to whichever node answers as leader. let leader = disk::leader_node_index(harness).await; + let primary_endpoint = harness + .node(leader) + .tcp_addr() + .expect("leader exposes a TCP endpoint") + .to_string(); let producer = harness .node(leader) .tcp_client() @@ -106,6 +109,12 @@ async fn given_a_client_producing_when_its_primary_is_killed_should_resume_witho .await .expect("connect the producer to the primary"); + assert_eq!( + producer.get_connection_info().await.server_address, + primary_endpoint, + "the producer must be pinned to the node this test kills, or it proves nothing" + ); + let stream = Identifier::named(STREAM_NAME).unwrap(); let topic = Identifier::named(TOPIC_NAME).unwrap(); let partitioning = Partitioning::partition_id(PARTITION_ID); @@ -160,11 +169,15 @@ async fn given_a_client_producing_when_its_primary_is_killed_should_resume_witho assert!( resumed, "a client pinned to a killed primary must complete its next operation against \ - the surviving quorum within {RESUME_BUDGET:?}, but the SDK has no \ - multi-endpoint failover: it holds only the dead node's server_address, its \ - fail-fast gate (auto-login disabled) surfaces errors without reconnecting, \ - and the leader redirect that could reroute it needs a live connection to \ - read the roster, so every retry redialed the dead endpoint \ + the surviving quorum within {RESUME_BUDGET:?}: the roster learned while the \ + connection was healthy names the survivors, and the credentials the sign-in \ + succeeded with re-establish the session on whichever one answers \ ({attempt} attempts, last error: {last_error:?})" ); + assert_ne!( + producer.get_connection_info().await.server_address, + primary_endpoint, + "the send that resumed must have landed on a survivor, so the client has to \ + have moved off the killed primary's endpoint" + ); } diff --git a/core/sdk/src/client_provider.rs b/core/sdk/src/client_provider.rs index 423b7048ea..7fbe919dd3 100644 --- a/core/sdk/src/client_provider.rs +++ b/core/sdk/src/client_provider.rs @@ -132,6 +132,9 @@ impl ClientProviderConfig { TransportProtocol::Tcp => { config.tcp = Some(Arc::new(TcpClientConfig { server_address: args.tcp_server_address, + // Command-line arguments name a single server; the roster + // is learned once the client is connected. + failover_addresses: Vec::new(), tls_enabled: args.tcp_tls_enabled, tls_domain: args.tcp_tls_domain, tls_ca_file: args.tcp_tls_ca_file, diff --git a/core/sdk/src/leader_aware.rs b/core/sdk/src/leader_aware.rs index 35ffce9fca..bd99d2e32a 100644 --- a/core/sdk/src/leader_aware.rs +++ b/core/sdk/src/leader_aware.rs @@ -18,7 +18,7 @@ use iggy_binary_protocol::codes::GET_CLUSTER_METADATA_CODE; use iggy_common::ClusterClient; use iggy_common::{ - ClusterMetadata, ClusterNodeRole, ClusterNodeStatus, IggyError, TransportProtocol, + ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, IggyError, TransportProtocol, }; use std::net::SocketAddr; use std::str::FromStr; @@ -38,12 +38,33 @@ pub(crate) fn is_unauthenticated_metadata_probe(code: u32, error: &IggyError) -> code == GET_CLUSTER_METADATA_CODE && matches!(error, IggyError::Unauthenticated) } +/// What one leader check learned from the cluster roster. +pub struct LeaderCheck { + /// The leader's address, when it is not the node the client is on. + pub redirect: Option, + /// Every endpoint the roster named for this transport. A client keeps + /// them as failover candidates: the address it was configured with dies + /// with its node, and the roster is unreachable exactly when it is + /// needed, so it has to be remembered while the connection is healthy. + pub endpoints: Vec, +} + +impl LeaderCheck { + /// A check that learned nothing: stay where we are, remember no endpoint. + fn inconclusive() -> Self { + Self { + redirect: None, + endpoints: Vec::new(), + } + } +} + /// Check if we need to redirect to leader and return the leader address if redirection is needed pub async fn check_and_redirect_to_leader( client: &C, current_address: &str, transport: TransportProtocol, -) -> Result, IggyError> { +) -> Result { debug!("Checking cluster metadata for leader detection"); // A cluster can be transiently leaderless: a restarted node cedes the @@ -60,15 +81,31 @@ pub async fn check_and_redirect_to_leader( metadata.nodes.len(), metadata.name ); + let endpoints = transport_endpoints(&metadata, transport); match process_cluster_metadata(&metadata, current_address, transport) { - Outcome::Redirect(address) => return Ok(Some(address)), - Outcome::LeaderIsCurrent => return Ok(None), + Outcome::Redirect(address) => { + return Ok(LeaderCheck { + redirect: Some(address), + endpoints, + }); + } + Outcome::LeaderIsCurrent => { + return Ok(LeaderCheck { + redirect: None, + endpoints, + }); + } Outcome::NoLeader => { if tokio::time::Instant::now() >= deadline { warn!( "No active leader found in cluster metadata within {LEADERLESS_WAIT_BUDGET:?}, connection will continue on server node {current_address}", ); - return Ok(None); + // A leaderless roster still names where the nodes + // are, and that is what failover needs. + return Ok(LeaderCheck { + redirect: None, + endpoints, + }); } tokio::time::sleep(LEADERLESS_POLL_INTERVAL).await; } @@ -82,14 +119,14 @@ pub async fn check_and_redirect_to_leader( debug!( "Cluster metadata answered Unauthenticated; the session is gone, connection will continue on server node {current_address}" ); - return Ok(None); + return Ok(LeaderCheck::inconclusive()); } Err(e) => { warn!( "Failed to get cluster metadata: {}, connection will continue on server node {}", e, current_address ); - return Ok(None); + return Ok(LeaderCheck::inconclusive()); } } } @@ -110,6 +147,29 @@ enum Outcome { NoLeader, } +/// Every node's address for `transport`, in roster order. A node that does +/// not expose the transport reports port 0 and is skipped: dialing it would +/// burn a failover attempt on an endpoint that cannot answer. +fn transport_endpoints(metadata: &ClusterMetadata, transport: TransportProtocol) -> Vec { + metadata + .nodes + .iter() + .filter_map(|node| { + let port = transport_port(node, transport); + (port != 0).then(|| format!("{}:{port}", node.ip)) + }) + .collect() +} + +fn transport_port(node: &ClusterNode, transport: TransportProtocol) -> u16 { + match transport { + TransportProtocol::Tcp => node.endpoints.tcp, + TransportProtocol::Quic => node.endpoints.quic, + TransportProtocol::Http => node.endpoints.http, + TransportProtocol::WebSocket => node.endpoints.websocket, + } +} + /// Process cluster metadata and determine if redirection is needed fn process_cluster_metadata( metadata: &ClusterMetadata, @@ -132,12 +192,7 @@ fn process_cluster_metadata( match leader { Some(leader_node) => { - let leader_port = match transport { - TransportProtocol::Tcp => leader_node.endpoints.tcp, - TransportProtocol::Quic => leader_node.endpoints.quic, - TransportProtocol::Http => leader_node.endpoints.http, - TransportProtocol::WebSocket => leader_node.endpoints.websocket, - }; + let leader_port = transport_port(leader_node, transport); let leader_address = format!("{}:{}", leader_node.ip, leader_port); info!( @@ -162,7 +217,7 @@ fn process_cluster_metadata( /// Check if two addresses refer to the same endpoint /// Handles various formats like 127.0.0.1:8090 vs localhost:8090 -fn is_same_address(addr1: &str, addr2: &str) -> bool { +pub(crate) fn is_same_address(addr1: &str, addr2: &str) -> bool { match (parse_address(addr1), parse_address(addr2)) { (Some(sock1), Some(sock2)) => sock1.ip() == sock2.ip() && sock1.port() == sock2.port(), _ => normalize_address(addr1) == normalize_address(addr2), @@ -245,6 +300,36 @@ mod tests { )); } + fn node(name: &str, ip: &str, tcp: u16, role: ClusterNodeRole) -> ClusterNode { + ClusterNode { + name: name.to_string(), + ip: ip.to_string(), + endpoints: iggy_common::TransportEndpoints::new(tcp, 0, 3000, 3001), + role, + status: ClusterNodeStatus::Healthy, + } + } + + #[test] + fn the_roster_names_every_node_that_exposes_the_transport() { + let metadata = ClusterMetadata { + name: "iggy".to_string(), + nodes: vec![ + node("iggy-1", "10.0.0.1", 8090, ClusterNodeRole::Leader), + node("iggy-2", "10.0.0.2", 8090, ClusterNodeRole::Follower), + node("iggy-3", "10.0.0.3", 8090, ClusterNodeRole::Follower), + ], + }; + + assert_eq!( + transport_endpoints(&metadata, TransportProtocol::Tcp), + vec!["10.0.0.1:8090", "10.0.0.2:8090", "10.0.0.3:8090"] + ); + // A node that does not expose the transport reports port 0; dialing + // it would burn a failover attempt on an endpoint that cannot answer. + assert!(transport_endpoints(&metadata, TransportProtocol::Quic).is_empty()); + } + #[test] fn test_is_same_address() { assert!(is_same_address("127.0.0.1:8090", "127.0.0.1:8090")); diff --git a/core/sdk/src/quic/quic_client.rs b/core/sdk/src/quic/quic_client.rs index 921ef5d3dc..255e473e47 100644 --- a/core/sdk/src/quic/quic_client.rs +++ b/core/sdk/src/quic/quic_client.rs @@ -508,12 +508,15 @@ impl QuicClient { /// Returns true if redirection occurred and reconnection is needed. pub(crate) async fn handle_leader_redirection(&self) -> Result { let current_address = self.current_server_address.lock().await.clone(); + // The roster's other endpoints are dropped here: only the TCP client + // dials failover candidates so far. let leader_address = check_and_redirect_to_leader( self, ¤t_address, iggy_common::TransportProtocol::Quic, ) - .await?; + .await? + .redirect; if let Some(new_leader_address) = leader_address { let mut redirection_state = self.leader_redirection_state.lock().await; diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index 2c09e3f2ef..0b8e42b48d 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -16,7 +16,8 @@ // under the License. use crate::leader_aware::{ - LeaderRedirectionState, check_and_redirect_to_leader, is_unauthenticated_metadata_probe, + LeaderRedirectionState, check_and_redirect_to_leader, is_same_address, + is_unauthenticated_metadata_probe, }; use crate::prelude::Client; use crate::prelude::TcpClientConfig; @@ -36,6 +37,7 @@ use iggy_common::{ use iggy_common::{BinaryClient, BinaryTransport, PersonalAccessTokenClient, UserClient}; use rustls::pki_types::{CertificateDer, ServerName, pem::PemObject}; use secrecy::ExposeSecret; +use std::io; use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; @@ -68,6 +70,13 @@ const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_ /// overall. const TRANSIENT_FAILOVER_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2); +/// Bound on one dial while the client has other endpoints to try. A host +/// that drops the SYN -- powered off, or partitioned away -- takes the OS +/// connect timeout to fail, which is minutes, and every other endpoint waits +/// behind it. A client that knows a single endpoint has nothing to starve, so +/// its dial stays unbounded. +const FAILOVER_DIAL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); + /// TCP client for interacting with the Iggy API. /// It requires a valid server address. #[derive(Debug)] @@ -80,6 +89,15 @@ pub struct TcpClient { pub(crate) connected_at: Mutex>, leader_redirection_state: Mutex, pub(crate) current_server_address: Mutex, + /// Every endpoint the cluster roster named, refreshed on each leader + /// check. A node dies together with its address, and the roster is + /// unreachable exactly when it is needed, so the client has to have + /// remembered it while the connection was still healthy. + roster_endpoints: Mutex>, + /// Credentials a sign-in on this client succeeded with, so a reconnect -- + /// onto this node or, after a failover, another one -- can re-establish + /// the session instead of surfacing `Unauthenticated`. Cleared on logout. + session_credentials: Mutex>, // `std::sync::Mutex` (not `tokio::sync::Mutex`): the critical section // is `encode_request_header`, which is pure CPU and never awaits. The // tokio variant would pay a waker alloc + internal semaphore on @@ -159,9 +177,10 @@ impl BinaryTransport for TcpClient { return Err(IggyError::Disconnected); } - if matches!(self.config.auto_login, AutoLogin::Disabled) && !is_login_register_code(code) { - // Without auto-login a reconnect cannot re-establish the session, - // so non-login requests fail fast. Login/register itself is the + if !is_login_register_code(code) && self.sign_in_credentials().await.is_none() { + // With no credentials -- neither configured nor remembered from a + // sign-in -- a reconnect cannot re-establish the session, so + // non-login requests fail fast. Login/register itself is the // exception: the server stays deliberately silent on transient // register failures (the server `surface_login_failure`) and // relies on the client timing out and replaying the request. @@ -230,6 +249,14 @@ impl iggy_common::VsrSessionControl for TcpClient { Ok(()) } + async fn remember_session_credentials(&self, credentials: Credentials) { + self.session_credentials.lock().await.replace(credentials); + } + + async fn forget_session_credentials(&self) { + self.session_credentials.lock().await.take(); + } + fn sdk_version(&self) -> &'static str { crate::SDK_VERSION } @@ -292,6 +319,8 @@ impl TcpClient { connected_at: Mutex::new(None), leader_redirection_state: Mutex::new(LeaderRedirectionState::new()), current_server_address: Mutex::new(server_address), + roster_endpoints: Mutex::new(Vec::new()), + session_credentials: Mutex::new(None), consensus_session: Arc::new(StdMutex::new(ConsensusSession::new())), skip_auto_login_once: Mutex::new(false), consumer_group_state: Arc::new(iggy_common::ConsumerGroupClientState::new()), @@ -320,19 +349,20 @@ impl TcpClient { } self.set_state(ClientState::Connecting).await; - if let Some(connected_at) = self.connected_at.lock().await.as_ref() { - let now = IggyTimestamp::now(); - let elapsed = now.as_micros() - connected_at.as_micros(); - let interval = self.config.reconnection.reestablish_after.as_micros(); - trace!( - "Elapsed time since last connection: {}", - IggyDuration::from(elapsed) - ); - if elapsed < interval { - let remaining = IggyDuration::from(interval - elapsed); - info!("Trying to connect to the server in: {remaining}",); - sleep(remaining.get_duration()).await; - } + let candidates = self.dial_candidates().await; + // The reestablish delay paces reconnects to the one endpoint a + // single-address client has. With other endpoints known there is + // somewhere else to go, and pausing first only pushes the + // failover past the window the caller is willing to wait; the + // retry interval still paces the loop. + let reestablish_wait = if candidates.len() > 1 { + None + } else { + self.reestablish_wait().await + }; + if let Some(remaining) = reestablish_wait { + info!("Trying to connect to the server in: {remaining}",); + sleep(remaining.get_duration()).await; } let tls_enabled = self.config.tls_enabled; @@ -340,14 +370,15 @@ impl TcpClient { let connection_stream: ConnectionStreamKind; let remote_address; let client_address; + let mut candidate = 0; loop { - let server_address = self.current_server_address.lock().await.clone(); + let server_address = candidates[candidate].clone(); info!( "{NAME} client is connecting to server: {}...", server_address ); - let connection = TcpStream::connect(&server_address).await; + let connection = self.dial(&server_address, candidates.len() > 1).await; if let Err(err) = &connection { error!( "Failed to connect to server: {}. Error: {}", @@ -358,6 +389,15 @@ impl TcpClient { return Err(IggyError::CannotEstablishConnection); } + // Every other endpoint gets its turn before the retry + // interval: the node just lost may be gone for good, and + // pausing on it helps nothing. + candidate += 1; + if candidate < candidates.len() { + continue; + } + candidate = 0; + let unlimited_retries = self.config.reconnection.max_retries.is_none(); let max_retries = self.config.reconnection.max_retries.unwrap_or_default(); let max_retries_str = @@ -387,6 +427,10 @@ impl TcpClient { error!("Failed to establish TCP connection to the server: {error}",); IggyError::CannotEstablishConnection })?; + // The endpoint that answered is where this client now lives: + // the leader check compares against it, and the next + // reconnect starts from it. + *self.current_server_address.lock().await = server_address.clone(); client_address = stream.local_addr().map_err(|error| { error!("Failed to get the local address of the client: {error}",); IggyError::CannotEstablishConnection @@ -485,9 +529,9 @@ impl TcpClient { }; // Handle auto-login - let should_redirect = match &self.config.auto_login { - AutoLogin::Disabled => { - info!("Automatic sign-in is disabled."); + let should_redirect = match self.sign_in_credentials().await { + None => { + info!("No credentials to sign in with."); // Only `IggyClient` redirects after a manual sign-in, so // a raw transport can stay on a backup: its first // replicated write gets `TransientNotAccepted`, the @@ -495,14 +539,14 @@ impl TcpClient { // `Unauthenticated` until the caller signs in again. false } - AutoLogin::Enabled(credentials) => { + Some(credentials) => { if skip_auto_login { info!("Skipping automatic sign-in for a retried login/register request."); false } else { info!("{NAME} client: {client_address} is signing in..."); self.set_state(ClientState::Authenticating).await; - match credentials { + match &credentials { Credentials::UsernamePassword(username, password) => { self.login_user(username, password.expose_secret()).await?; info!( @@ -540,14 +584,22 @@ impl TcpClient { /// Returns true if redirection occurred and reconnection is needed. pub(crate) async fn handle_leader_redirection(&self) -> Result { let current_address = self.current_server_address.lock().await.clone(); - let leader_address = check_and_redirect_to_leader( + let leader_check = check_and_redirect_to_leader( self, ¤t_address, iggy_common::TransportProtocol::Tcp, ) .await?; - if let Some(new_leader_address) = leader_address { + // Replaced wholesale rather than merged: the roster is the cluster's + // own answer about where its nodes are, so a node it dropped should + // stop being dialed. The configured seeds are kept separately and + // outlive it. + if !leader_check.endpoints.is_empty() { + *self.roster_endpoints.lock().await = leader_check.endpoints; + } + + if let Some(new_leader_address) = leader_check.redirect { let mut redirection_state = self.leader_redirection_state.lock().await; if !redirection_state.can_redirect() { warn!("Maximum leader redirections reached, continuing with current connection"); @@ -573,6 +625,69 @@ impl TcpClient { } } + /// Credentials to sign in with after connecting: the configured ones, or + /// else the ones a manual sign-in on this client succeeded with. A manual + /// sign-in is otherwise less reconnectable than a configured one, which + /// is a surprising difference between two ways of doing the same thing. + async fn sign_in_credentials(&self) -> Option { + match &self.config.auto_login { + AutoLogin::Enabled(credentials) => Some(credentials.clone()), + AutoLogin::Disabled => self.session_credentials.lock().await.clone(), + } + } + + /// Endpoints to dial for one connect, likeliest first: where the client + /// currently is, then the roster it learned while connected, then the + /// configured seeds. + async fn dial_candidates(&self) -> Vec { + let mut candidates = vec![self.current_server_address.lock().await.clone()]; + let roster = self.roster_endpoints.lock().await.clone(); + for endpoint in roster.iter().chain(self.config.failover_addresses.iter()) { + if !candidates + .iter() + .any(|candidate| is_same_address(candidate, endpoint)) + { + candidates.push(endpoint.clone()); + } + } + candidates + } + + /// Dial one endpoint, bounding the wait while other endpoints are queued + /// behind it (see `FAILOVER_DIAL_TIMEOUT`). + async fn dial(&self, server_address: &str, bounded: bool) -> io::Result { + if !bounded { + return TcpStream::connect(server_address).await; + } + + match tokio::time::timeout(FAILOVER_DIAL_TIMEOUT, TcpStream::connect(server_address)).await + { + Ok(connection) => connection, + Err(_elapsed) => Err(io::Error::new( + io::ErrorKind::TimedOut, + format!("dialing {server_address} took longer than {FAILOVER_DIAL_TIMEOUT:?}"), + )), + } + } + + /// What is left of the `reestablish_after` window since the last + /// successful connection, if any. + async fn reestablish_wait(&self) -> Option { + let connected_at = self + .connected_at + .lock() + .await + .as_ref() + .map(IggyTimestamp::as_micros)?; + let elapsed = IggyTimestamp::now().as_micros() - connected_at; + let interval = self.config.reconnection.reestablish_after.as_micros(); + trace!( + "Elapsed time since last connection: {}", + IggyDuration::from(elapsed) + ); + (elapsed < interval).then(|| IggyDuration::from(interval - elapsed)) + } + async fn disconnect(&self) -> Result<(), IggyError> { if self.get_state().await == ClientState::Disconnected { return Ok(()); @@ -871,6 +986,92 @@ const fn is_login_register_code(code: u32) -> bool { mod tests { use super::*; + fn client_with(server_address: &str, failover_addresses: Vec) -> TcpClient { + TcpClient::create(Arc::new(TcpClientConfig { + server_address: server_address.to_string(), + failover_addresses, + ..TcpClientConfig::default() + })) + .expect("create the client") + } + + #[tokio::test] + async fn dial_candidates_lead_with_the_current_endpoint_and_name_each_other_one_once() { + let client = client_with( + "127.0.0.1:8090", + vec!["127.0.0.1:8092".to_string(), "localhost:8090".to_string()], + ); + *client.roster_endpoints.lock().await = vec![ + "127.0.0.1:8090".to_string(), + "127.0.0.1:8091".to_string(), + "127.0.0.1:8092".to_string(), + ]; + + // The current endpoint leads, the roster follows, and neither the + // roster's copy of the current endpoint nor a seed that only spells + // the same endpoint differently earns a second dial. + assert_eq!( + client.dial_candidates().await, + vec![ + "127.0.0.1:8090".to_string(), + "127.0.0.1:8091".to_string(), + "127.0.0.1:8092".to_string(), + ] + ); + } + + #[tokio::test] + async fn a_client_that_learned_no_roster_still_dials_its_configured_seeds() { + let client = client_with("127.0.0.1:8090", vec!["127.0.0.1:8091".to_string()]); + + assert_eq!( + client.dial_candidates().await, + vec!["127.0.0.1:8090".to_string(), "127.0.0.1:8091".to_string()] + ); + } + + #[tokio::test] + async fn a_sign_in_makes_a_client_without_auto_login_reconnectable() { + let client = client_with("127.0.0.1:8090", Vec::new()); + assert!(client.sign_in_credentials().await.is_none()); + + client + .remember_session_credentials(Credentials::UsernamePassword( + "iggy".to_string(), + "iggy".into(), + )) + .await; + assert!(client.sign_in_credentials().await.is_some()); + + // An explicit logout leaves no session to restore, and a reconnect + // must not resurrect one. + client.forget_session_credentials().await; + assert!(client.sign_in_credentials().await.is_none()); + } + + #[tokio::test] + async fn configured_credentials_outrank_the_ones_a_sign_in_remembered() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "configured".to_string(), + "iggy".into(), + )), + ..TcpClientConfig::default() + })) + .expect("create the client"); + client + .remember_session_credentials(Credentials::UsernamePassword( + "signed-in".to_string(), + "iggy".into(), + )) + .await; + + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, _)) => assert_eq!(username, "configured"), + other => panic!("expected the configured credentials, got {other:?}"), + } + } + #[test] fn should_fail_with_empty_connection_string() { let value = ""; diff --git a/core/sdk/src/websocket/websocket_client.rs b/core/sdk/src/websocket/websocket_client.rs index c439a38f19..362a2aa953 100644 --- a/core/sdk/src/websocket/websocket_client.rs +++ b/core/sdk/src/websocket/websocket_client.rs @@ -533,12 +533,15 @@ impl WebSocketClient { /// Returns true if redirection occurred and reconnection is needed. pub(crate) async fn handle_leader_redirection(&self) -> Result { let current_address = self.current_server_address.lock().await.clone(); + // The roster's other endpoints are dropped here: only the TCP client + // dials failover candidates so far. let leader_address = check_and_redirect_to_leader( self, ¤t_address, iggy_common::TransportProtocol::WebSocket, ) - .await?; + .await? + .redirect; if let Some(new_leader_address) = leader_address { let mut redirection_state = self.leader_redirection_state.lock().await; diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs index 1885c69faf..17d5243659 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs @@ -424,6 +424,25 @@ private async Task RedirectAsync(CancellationToken token) return true; } + /// + /// Keeps every node the roster names as a dial candidate. Replaced wholesale rather than merged: the + /// roster is the cluster's own answer about where its nodes are, so a node it dropped stops being dialed. + /// The configured address is kept separately and outlives it. A node that does not expose the tcp + /// transport reports port 0 and is skipped, since dialing it would burn an attempt on an endpoint that + /// cannot answer. + /// + private void RememberRoster(ClusterMetadata clusterMetadata) + { + var endpoints = clusterMetadata.Nodes + .Where(node => node.Endpoints.Tcp != 0) + .Select(node => ServerAddress.HostPort(node.Ip, node.Endpoints.Tcp)) + .ToArray(); + if (endpoints.Length > 0) + { + _rosterAddresses = endpoints; + } + } + private async Task GetCurrentLeaderNodeAsync(CancellationToken token) { var leaderlessDeadline = Environment.TickCount64 + VsrLeaderlessWaitMs; @@ -438,6 +457,8 @@ private async Task RedirectAsync(CancellationToken token) return null; } + RememberRoster(clusterMetadata); + if (clusterMetadata.Nodes.Count() == 1) { return null; diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index 32ea6e03ec..043a508092 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -83,6 +83,17 @@ public sealed partial class TcpMessageStream : IIggyClient private DateTimeOffset _lastConnectionTime; private int _leaderRedirectCount; + // Every node the roster named on the last read, kept as dial candidates. A node dies together with its + // address, and the roster is unreachable exactly when it is needed, so the client has to have remembered it + // while the connection was still healthy. Written by the leader probe, read by the connect loop. + private string[] _rosterAddresses = []; + + // The credentials a sign-in succeeded with, so a reconnect - on this node or, after a failover, another one - + // can re-establish the session instead of leaving every later request unauthenticated. A caller that signs in + // by hand is otherwise less reconnectable than one that configures auto login, which is a surprising + // difference between two ways of doing the same thing. Cleared on sign-out. + private AutoLoginSettings? _rememberedLogin; + // Both are written by the connect and redirect paths, which do not hold the sending semaphore the request // paths read them under, so they are accessed through Interlocked rather than as plain fields. Losing an // update to the skip flag leaves a connection reporting Connected that never authenticated; losing one to @@ -671,7 +682,7 @@ public Task ConnectAsync(CancellationToken token = default) if (_configuration.ReconnectionSettings.Enabled && !_configuration.AutoLoginSettings.Enabled) { _logger.LogWarning( - "Reconnection is enabled without auto login: a lost session cannot be restored, requests will fail until the client logs in again"); + "Reconnection is enabled without auto login: a lost session can only be restored once the client has signed in at least once"); } return ConnectAsync(true, token); @@ -814,8 +825,14 @@ public async Task ChangePasswordAsync(Identifier userId, string currentPassword, throw new NotConnectedException(); } - return await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_CODE, + var identity = await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_CODE, LoginRegister.Serialize(userName, password), token); + _rememberedLogin = new AutoLoginSettings + { + Enabled = true, Username = userName, Password = password + }; + + return identity; } /// @@ -833,6 +850,9 @@ public async Task LogoutUserAsync(CancellationToken token = default) { await ResetConsensusSessionAsync(); + // An explicit sign-out leaves no session to restore, and a reconnect must not resurrect one. + _rememberedLogin = null; + if (_state == ConnectionState.Authenticated) { SetConnectionStateAsync(ConnectionState.Connected); @@ -889,8 +909,11 @@ public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken /// public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) { - return await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE, + var identity = await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE, LoginRegister.SerializeWithPersonalAccessToken(token), ct); + _rememberedLogin = new AutoLoginSettings { Enabled = true, PersonalAccessToken = token }; + + return identity; } /// @@ -921,7 +944,10 @@ or ConnectionState.Authenticating return; } - if (_lastConnectionTime != DateTimeOffset.MinValue) + // The initial delay paces reconnects to the one endpoint a single-address client has. With other + // endpoints known there is somewhere else to go, and pausing first only pushes the failover past the + // window the caller is willing to wait; the dial loop's own delay still paces the retries. + if (_lastConnectionTime != DateTimeOffset.MinValue && DialCandidates().Length == 1) { await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token); } @@ -984,7 +1010,7 @@ private async Task RunHeartbeatAsync(TimeSpan interval, CancellationToken token) // the ping is what brings an idle client back. var unrecoverable = _state is ConnectionState.Disconnected or ConnectionState.Connecting && !(_configuration.ReconnectionSettings.Enabled - && _configuration.AutoLoginSettings.Enabled); + && SignInSettings() != null); if (IsConnecting || unrecoverable) { continue; @@ -1130,15 +1156,18 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken var retryCount = 0; var redirects = 0; var delay = _configuration.ReconnectionSettings.InitialDelay; + + if (string.IsNullOrEmpty(_currentAddress)) + { + _currentAddress = _configuration.BaseAddress; + } + + var candidates = DialCandidates(); + var candidate = 0; do { await DropStreamAsync(); - if (string.IsNullOrEmpty(_currentAddress)) - { - _currentAddress = _configuration.BaseAddress; - } - if (!ServerAddress.TryParse(_currentAddress, out var host, out var port)) { throw new InvalidBaseAddressException(); @@ -1191,9 +1220,9 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken // No pre-login roster read: the server auth-gates cluster metadata, so leadership settles after // a sign-in binds a session. A login dialed at a backup still succeeds because the server // forwards the register to the primary. - if (autoLogin && _configuration.AutoLoginSettings.Enabled && !ConsumeSkipAutoLogin()) + if (autoLogin && SignInSettings() is { } signInSettings && !ConsumeSkipAutoLogin()) { - await AutoLoginAsync(token); + await AutoLoginAsync(signInSettings, token); if (await RedirectAsync(token)) { @@ -1233,6 +1262,17 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken throw; } + // Every other endpoint gets its turn before the retry delay: the node just lost may be gone for + // good, and pausing on it helps nothing. + if (++candidate < candidates.Length) + { + _currentAddress = candidates[candidate]; + continue; + } + + candidate = 0; + _currentAddress = candidates[0]; + retryCount++; if (_configuration.ReconnectionSettings.UseExponentialBackoff) { @@ -1267,6 +1307,10 @@ async Task BackoffOrThrowAsync() _logger.LogInformation("Following leader redirect {Redirect} to {Address}", redirects, _currentAddress); + // The redirect moved the client, so the endpoint it moved to leads the next dial. + candidates = DialCandidates(); + candidate = 0; + await Task.Delay(delay, token); } } @@ -1294,20 +1338,64 @@ private async Task DropStreamAsync() } } - private async Task AutoLoginAsync(CancellationToken token) + private string[] DialCandidates() + { + return DialCandidates(_currentAddress, _configuration.BaseAddress, _rosterAddresses); + } + + /// + /// The endpoints one connect dials, likeliest first: where the client currently is, the address it was + /// configured with, then the roster it learned while connected. Duplicates are dropped, so an endpoint the + /// roster merely spells differently does not earn a second attempt. + /// + internal static string[] DialCandidates(string currentAddress, string baseAddress, string[] rosterAddresses) + { + var candidates = new List(); + if (!string.IsNullOrEmpty(currentAddress)) + { + candidates.Add(currentAddress); + } + + foreach (var endpoint in rosterAddresses.Prepend(baseAddress)) + { + if (!string.IsNullOrEmpty(endpoint) && + !candidates.Exists(known => ServerAddress.IsSame(known, endpoint))) + { + candidates.Add(endpoint); + } + } + + return candidates.ToArray(); + } + + private async Task AutoLoginAsync(AutoLoginSettings settings, CancellationToken token) { - var settings = _configuration.AutoLoginSettings; if (!string.IsNullOrEmpty(settings.PersonalAccessToken)) { - _logger.LogInformation("Auto login enabled. Trying to login with a personal access token"); + _logger.LogInformation("Signing in with a personal access token"); await LoginWithPersonalAccessTokenAsync(settings.PersonalAccessToken, token); return; } - _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}", settings.Username); + _logger.LogInformation("Signing in with credentials: {Username}", settings.Username); await LoginUserAsync(settings.Username, settings.Password, token); } + /// + /// The credentials a connect signs in with: the configured ones, or else the ones a sign-in on this client + /// succeeded with. Null when nothing has ever signed in, which is when a reconnect cannot restore a + /// session at all. + /// + private AutoLoginSettings? SignInSettings() + { + if (_configuration.AutoLoginSettings.Enabled) + { + return _configuration.AutoLoginSettings; + } + + return _rememberedLogin; + } + /// /// Whether this connect was triggered by a login or register request that will re-authenticate itself, /// so the auto-login must sit this one out. Consumes the flag. @@ -1360,11 +1448,12 @@ private async Task> SendWithResponseAsync(ReadOnlyMemory +/// Mirrors the Rust SDK's dial_candidates: a client that loses the node it is on has to dial the rest +/// of the cluster, and the two SDKs have to agree on which endpoints those are and in what order. +/// +public sealed class DialCandidatesTests +{ + [Fact] + public void LeadsWithTheCurrentEndpointThenNamesEachOtherOneOnce() + { + var candidates = TcpMessageStream.DialCandidates( + "127.0.0.1:8090", + "localhost:8090", + ["127.0.0.1:8090", "127.0.0.1:8091", "127.0.0.1:8092"]); + + // Neither the roster's copy of the current endpoint nor a configured address that only spells the same + // endpoint differently earns a second dial. + Assert.Equal(["127.0.0.1:8090", "127.0.0.1:8091", "127.0.0.1:8092"], candidates); + } + + [Fact] + public void KeepsTheConfiguredAddressWhenNoRosterWasLearned() + { + var candidates = TcpMessageStream.DialCandidates("127.0.0.1:8091", "127.0.0.1:8090", []); + + Assert.Equal(["127.0.0.1:8091", "127.0.0.1:8090"], candidates); + } + + [Fact] + public void FallsBackToTheConfiguredAddressBeforeTheFirstConnect() + { + var candidates = TcpMessageStream.DialCandidates(string.Empty, "127.0.0.1:8090", []); + + Assert.Equal(["127.0.0.1:8090"], candidates); + } + + [Fact] + public void DialsOneEndpointWhenNothingElseIsKnown() + { + var candidates = TcpMessageStream.DialCandidates("127.0.0.1:8090", "127.0.0.1:8090", []); + + Assert.Equal(["127.0.0.1:8090"], candidates); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs new file mode 100644 index 0000000000..3bc704e06e --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs @@ -0,0 +1,346 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using System.Net; +using System.Net.Sockets; +using System.Text; +using Apache.Iggy.Configuration; +using Apache.Iggy.Contracts.Tcp; +using Apache.Iggy.Enums; +using Apache.Iggy.IggyClient.Implementations; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Apache.Iggy.Tests.VsrTests; + +/// +/// The node a client signed in on dies; its next request has to complete on a survivor the roster named, +/// under a session established there. Mirrors +/// core/integration/tests/cluster/failover_client_continuity.rs. +/// +public sealed class EndpointFailoverTests +{ + private const int HeaderSize = 256; + private const int SizeOffset = 48; + private const int CommandOffset = 60; + private const int RequestIdOffset = 168; + private const int RequestOperationOffset = 176; + private const int RequestReservedOffset = 196; + private const int ReplyRequestIdOffset = 200; + private const int ReplyOperationOffset = 208; + private const int ReplyStatusOffset = 216; + + private const byte CommandReply = 8; + private const byte OperationRegister = 1; + private const byte OperationNonReplicated = 2; + private const int GetClusterMetadataCode = 12; + private const int PingCode = 1; + + [Fact] + public async Task ResumesOnASurvivorAfterTheSignedInNodeDies() + { + using var primary = new MockNode(); + using var survivor = new MockNode(); + + // The primary leads, so the sign-in settles there and the roster is only remembered - not acted on - + // until the node dies. + primary.Serve(request => request.Code == GetClusterMetadataCode + ? Reply(OperationNonReplicated, ClusterMetadata(primary.Port, survivor.Port, primary.Port)) + : Answer(request)); + survivor.Serve(request => request.Code == GetClusterMetadataCode + ? Reply(OperationNonReplicated, ClusterMetadata(primary.Port, survivor.Port, survivor.Port)) + : Answer(request)); + + var configuration = new IggyClientConfigurator + { + BaseAddress = $"127.0.0.1:{primary.Port}", + Protocol = Protocol.Tcp, + ReconnectionSettings = new ReconnectionSettings + { + Enabled = true, MaxRetries = 4, InitialDelay = TimeSpan.FromMilliseconds(20) + } + }; + using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); + + await client.ConnectAsync(TestContext.Current.CancellationToken); + // No auto login: the credentials come from the caller's own sign-in, which is the shape that could not + // reconnect at all before. + await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); + await client.PingAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, primary.Pings); + + primary.Kill(); + + // The request in flight when the node died is allowed to fail; what is not allowed is never completing + // one, which is what a client that only knows the dead endpoint does. + var (resumed, lastError) = await ResumedWithin(client, TimeSpan.FromSeconds(10)); + Assert.True(resumed, + $"the client has to resume on the survivor the roster named ({lastError}, survivor saw " + + $"{survivor.Registrations} registrations and {survivor.Pings} pings)"); + Assert.True(survivor.Registrations >= 1, "the remembered credentials signed in again on the survivor"); + Assert.True(survivor.Pings >= 1, "the request landed on the survivor"); + } + + [Fact] + public async Task FailsFastWhenNothingEverSignedIn() + { + using var node = new MockNode(); + node.Serve(request => request.Code == GetClusterMetadataCode + ? Reply(OperationNonReplicated, ClusterMetadata(node.Port, node.Port, node.Port)) + : Answer(request)); + + var configuration = new IggyClientConfigurator + { + BaseAddress = $"127.0.0.1:{node.Port}", + Protocol = Protocol.Tcp, + ReconnectionSettings = new ReconnectionSettings + { + Enabled = true, MaxRetries = 2, InitialDelay = TimeSpan.FromMilliseconds(20) + } + }; + using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); + + await client.ConnectAsync(TestContext.Current.CancellationToken); + await client.PingAsync(TestContext.Current.CancellationToken); + + node.Kill(); + + var (resumed, _) = await ResumedWithin(client, TimeSpan.FromSeconds(2)); + Assert.False(resumed, "a client that never signed in cannot restore a session by reconnecting"); + } + + private static async Task<(bool Resumed, string LastError)> ResumedWithin(TcpMessageStream client, + TimeSpan budget) + { + var deadline = DateTimeOffset.UtcNow + budget; + var lastError = "none"; + var attempts = 0; + while (DateTimeOffset.UtcNow < deadline) + { + attempts++; + try + { + await client.PingAsync(TestContext.Current.CancellationToken); + + return (true, lastError); + } + catch (Exception error) + { + lastError = $"{attempts} attempts, last: {error.GetType().Name}: {error.Message}"; + await Task.Delay(50, TestContext.Current.CancellationToken); + } + } + + return (false, lastError); + } + + /// A reply for anything the roster read does not claim: a register, or an empty read. + private static byte[] Answer(MockRequest request) + { + return request.Operation == OperationRegister + ? Reply(OperationRegister, RegisterBody(session: 128)) + : Reply(OperationNonReplicated, []); + } + + private static byte[] Reply(byte operation, byte[] body) + { + var frame = new byte[HeaderSize + body.Length]; + BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(SizeOffset, 4), (uint)frame.Length); + frame[CommandOffset] = CommandReply; + frame[ReplyOperationOffset] = operation; + BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(ReplyStatusOffset, 4), 0); + body.CopyTo(frame.AsSpan(HeaderSize)); + + return frame; + } + + /// + /// A register reply carries a committed result section, so its four leading zero bytes announce zero + /// entries and the typed payload starts right after them. A non-replicated read carries none. + /// + private static byte[] RegisterBody(ulong session) + { + var serverVersion = Encoding.UTF8.GetBytes("0.0.0"); + var body = new byte[4 + 17 + serverVersion.Length]; + var payload = body.AsSpan(4); + BinaryPrimitives.WriteUInt32LittleEndian(payload[..4], 7); + BinaryPrimitives.WriteUInt64LittleEndian(payload[4..12], session); + BinaryPrimitives.WriteUInt32LittleEndian(payload[12..16], 11 << 10); + payload[16] = (byte)serverVersion.Length; + serverVersion.CopyTo(payload[17..]); + + return body; + } + + private static byte[] ClusterMetadata(ushort primaryPort, ushort survivorPort, ushort leaderPort) + { + var body = new List(); + WriteString(body, "test-cluster"); + body.AddRange(BitConverter.GetBytes(2u)); + WriteNode(body, "primary", primaryPort, primaryPort == leaderPort); + WriteNode(body, "survivor", survivorPort, survivorPort == leaderPort); + + return body.ToArray(); + } + + private static void WriteNode(List body, string name, ushort port, bool leader) + { + WriteString(body, name); + WriteString(body, "127.0.0.1"); + body.AddRange(BitConverter.GetBytes(port)); + body.AddRange(BitConverter.GetBytes((ushort)0)); + body.AddRange(BitConverter.GetBytes((ushort)0)); + body.AddRange(BitConverter.GetBytes((ushort)0)); + body.Add(leader ? (byte)0 : (byte)1); + body.Add(0); + } + + private static void WriteString(List body, string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + body.AddRange(BitConverter.GetBytes((uint)bytes.Length)); + body.AddRange(bytes); + } + + private readonly record struct MockRequest(byte Operation, int Code, ulong RequestId); + + /// + /// A loopback VSR node. Killing it drops the live sockets and stops accepting, so a redial is refused the + /// way a dead process refuses one. + /// + private sealed class MockNode : IDisposable + { + private readonly TcpListener _listener; + private readonly List _accepted = []; + private volatile bool _killed; + private int _pings; + private int _registrations; + + public MockNode() + { + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + Port = (ushort)((IPEndPoint)_listener.LocalEndpoint).Port; + } + + public ushort Port { get; } + + public int Pings => Volatile.Read(ref _pings); + + public int Registrations => Volatile.Read(ref _registrations); + + public void Serve(Func handler) + { + _ = Task.Run(async () => + { + while (!_killed) + { + TcpClient connection; + try + { + connection = await _listener.AcceptTcpClientAsync(); + } + catch (Exception) + { + return; + } + + lock (_accepted) + { + _accepted.Add(connection); + } + + _ = Task.Run(() => Exchange(connection, handler)); + } + }); + } + + public void Kill() + { + _killed = true; + lock (_accepted) + { + foreach (var connection in _accepted) + { + connection.Close(); + } + + _accepted.Clear(); + } + + _listener.Stop(); + } + + public void Dispose() + { + Kill(); + } + + private async Task Exchange(TcpClient connection, Func handler) + { + try + { + await using var stream = connection.GetStream(); + var header = new byte[HeaderSize]; + while (!_killed) + { + await ReadExactly(stream, header); + var size = BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(SizeOffset, 4)); + var body = new byte[size - HeaderSize]; + await ReadExactly(stream, body); + + var request = new MockRequest(header[RequestOperationOffset], + BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(RequestReservedOffset, 4)), + BinaryPrimitives.ReadUInt64LittleEndian(header.AsSpan(RequestIdOffset, 8))); + if (request.Operation == OperationRegister) + { + Interlocked.Increment(ref _registrations); + } + else if (request.Code == PingCode) + { + Interlocked.Increment(ref _pings); + } + + var reply = handler(request); + BinaryPrimitives.WriteUInt64LittleEndian(reply.AsSpan(ReplyRequestIdOffset, 8), + request.RequestId); + await stream.WriteAsync(reply); + await stream.FlushAsync(); + } + } + catch (Exception) + { + // A killed node and a client that went away look the same here. + } + } + + private static async Task ReadExactly(NetworkStream stream, byte[] buffer) + { + var read = 0; + while (read < buffer.Length) + { + var chunk = await stream.ReadAsync(buffer.AsMemory(read)); + if (chunk == 0) + { + throw new EndOfStreamException("Connection closed"); + } + + read += chunk; + } + } + } +} diff --git a/foreign/go/client/tcp/tcp_core.go b/foreign/go/client/tcp/tcp_core.go index fe662af53c..5b9a46527c 100644 --- a/foreign/go/client/tcp/tcp_core.go +++ b/foreign/go/client/tcp/tcp_core.go @@ -85,6 +85,14 @@ type IggyTcpClient struct { // loggedOut records an explicit sign-out, so a reconnect's automatic // sign-in does not silently reverse it; guarded by c.mtx. loggedOut bool + // rememberedLogin holds the credentials a manual sign-in succeeded with, + // so a reconnect -- on this node or, after a failover, another one -- can + // re-establish the session instead of surfacing an unauthenticated error. + // A caller that signs in by hand is otherwise less reconnectable than one + // that configures auto-login, which is a surprising difference between + // two ways of doing the same thing. Cleared on sign-out; guarded by + // c.mtx. + rememberedLogin AutoLogin // groups caches the consumer-group assignments this client polls with. groups groupAssignmentCache // topics caches what a send needs to resolve a partition locally. @@ -480,12 +488,13 @@ func (c *IggyTcpClient) exchange(ctx context.Context, code uint32, frame []byte) return nil, err } - // Without auto-login a reconnect cannot restore the session, so anything - // but a sign-in fails here instead of replaying unauthenticated. The - // sign-in itself is the exception: the server stays silent on a transient + // With no credentials -- neither configured nor remembered from a + // sign-in -- a reconnect cannot restore the session, so anything but a + // sign-in fails here instead of replaying unauthenticated. The sign-in + // itself is the exception: the server stays silent on a transient // register failure and expects the client to replay it. login := isRegisterCode(code) - if !c.config.autoLogin.enabled && !login { + if _, ok := c.signInCredentials(); !ok && !login { return nil, err } c.mtx.Lock() @@ -890,8 +899,13 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { connectedAt := c.connectedAt c.mtx.Unlock() - // handle reestablish interval - if !connectedAt.IsZero() { + candidates := c.connectionCandidates() + + // The reestablish interval paces reconnects to the one endpoint a + // single-address client has. With other endpoints known there is somewhere + // else to go, and pausing first only pushes the failover past the window + // the caller is willing to wait; the retry interval still paces the loop. + if !connectedAt.IsZero() && len(candidates) == 1 { now := time.Now() elapsed := now.Sub(connectedAt) reestablishAfter := c.config.reconnection.reestablishAfter @@ -909,10 +923,7 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { attempts = uint(c.config.reconnection.maxRetries) interval = c.config.reconnection.interval } - - candidates := c.connectionCandidates() var conn net.Conn - var candidateIndex int if err := retry.New( retry.Context(ctx), retry.Attempts(attempts), @@ -923,46 +934,23 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { }), ).Do( func() error { - address := candidates[candidateIndex%len(candidates)] - candidateIndex++ - c.logger.Info("Iggy client is connecting to server...", slog.String("server_address", address)) - connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", address) - if err != nil { - c.logger.Error("Failed to establish TCP connection to the server", slog.Any("error", err)) - return ierror.ErrCannotEstablishConnection - } - - tc := connection.(*net.TCPConn) - if err := tc.SetNoDelay(c.config.noDelay); err != nil { - c.logger.Error("Failed to set the nodelay option on the client, continuing...", slog.Any("error", err)) - } - - c.mtx.Lock() - c.clientAddress = tc.LocalAddr().String() - c.currentServerAddress = address - c.mtx.Unlock() + // Every endpoint gets its turn inside one attempt, so a full pass + // over the cluster costs one retry rather than one per endpoint: + // a pass that stopped at the first refusal would never reach the + // survivors of a client configured for a single retry. + var lastErr error + for _, address := range candidates { + connection, err := c.dialCandidate(ctx, address) + if err != nil { + lastErr = err + continue + } - if !c.config.tlsEnabled { conn = connection return nil } - // TLS logic - tlsConfig, err := c.createTLSConfig() - if err != nil { - _ = connection.Close() - return err - } - - tlsConn := tls.Client(connection, tlsConfig) - if err := tlsConn.HandshakeContext(ctx); err != nil { - c.logger.Error("Failed to establish a TLS connection to the server", slog.Any("error", err)) - _ = connection.Close() - return fmt.Errorf("TLS handshake failed: %w", err) - } - - conn = tlsConn - return nil + return lastErr }); err != nil { c.mtx.Lock() c.transportState = iggcon.TransportStateDisconnected @@ -994,6 +982,47 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { return nil } +// dialCandidate opens one connection, wrapping it in TLS when configured, and +// records the endpoint that answered: the leader check compares against it and +// the next reconnect starts from it. +func (c *IggyTcpClient) dialCandidate(ctx context.Context, address string) (net.Conn, error) { + c.logger.Info("Iggy client is connecting to server...", slog.String("server_address", address)) + connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", address) + if err != nil { + c.logger.Error("Failed to establish TCP connection to the server", slog.Any("error", err)) + return nil, ierror.ErrCannotEstablishConnection + } + + tc := connection.(*net.TCPConn) + if err := tc.SetNoDelay(c.config.noDelay); err != nil { + c.logger.Error("Failed to set the nodelay option on the client, continuing...", slog.Any("error", err)) + } + + c.mtx.Lock() + c.clientAddress = tc.LocalAddr().String() + c.currentServerAddress = address + c.mtx.Unlock() + + if !c.config.tlsEnabled { + return connection, nil + } + + tlsConfig, err := c.createTLSConfig() + if err != nil { + _ = connection.Close() + return nil, err + } + + tlsConn := tls.Client(connection, tlsConfig) + if err := tlsConn.HandshakeContext(ctx); err != nil { + c.logger.Error("Failed to establish a TLS connection to the server", slog.Any("error", err)) + _ = connection.Close() + return nil, fmt.Errorf("TLS handshake failed: %w", err) + } + + return tlsConn, nil +} + func (c *IggyTcpClient) connectionCandidates() []string { c.mtx.Lock() defer c.mtx.Unlock() @@ -1025,8 +1054,9 @@ func (c *IggyTcpClient) connectionCandidates() []string { // backup: once the caller signs in, the first replicated request fails over // through the transient-deny path. func (c *IggyTcpClient) establishSession(ctx context.Context, skipAutoLogin bool) error { - if !c.config.autoLogin.enabled { - c.logger.Info("Automatic sign-in is disabled.") + credentials, ok := c.signInCredentials() + if !ok { + c.logger.Info("No credentials to sign in with.") return nil } if skipAutoLogin { @@ -1034,7 +1064,6 @@ func (c *IggyTcpClient) establishSession(ctx context.Context, skipAutoLogin bool return nil } - credentials := c.config.autoLogin.credentials if credentials.personalAccessToken != "" { _, err := c.LoginWithPersonalAccessToken(ctx, credentials.personalAccessToken) return err @@ -1043,6 +1072,32 @@ func (c *IggyTcpClient) establishSession(ctx context.Context, skipAutoLogin bool return err } +// signInCredentials reports the credentials a reconnect signs in with: the +// configured ones, or else the ones a manual sign-in succeeded with. +func (c *IggyTcpClient) signInCredentials() (Credentials, bool) { + if c.config.autoLogin.enabled { + return c.config.autoLogin.credentials, true + } + c.mtx.Lock() + defer c.mtx.Unlock() + return c.rememberedLogin.credentials, c.rememberedLogin.enabled +} + +// rememberLogin keeps the credentials a sign-in just succeeded with. +func (c *IggyTcpClient) rememberLogin(credentials Credentials) { + c.mtx.Lock() + defer c.mtx.Unlock() + c.rememberedLogin = NewAutoLogin(credentials) +} + +// forgetLogin drops them: after an explicit sign-out there is no session to +// restore, and a reconnect must not resurrect one. +func (c *IggyTcpClient) forgetLogin() { + c.mtx.Lock() + defer c.mtx.Unlock() + c.rememberedLogin = AutoLogin{} +} + func (c *IggyTcpClient) createTLSConfig() (*tls.Config, error) { tlsConfig := &tls.Config{ InsecureSkipVerify: !c.config.tls.tlsValidateCertificate, diff --git a/foreign/go/client/tcp/tcp_failover_test.go b/foreign/go/client/tcp/tcp_failover_test.go new file mode 100644 index 0000000000..c5735517cc --- /dev/null +++ b/foreign/go/client/tcp/tcp_failover_test.go @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tcp + +import ( + "context" + "log/slog" + "sync/atomic" + "testing" + "time" + + "github.com/apache/iggy/foreign/go/internal/command" + "github.com/apache/iggy/foreign/go/internal/vsr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The node a client signed in on dies; its next request has to complete on a +// survivor the roster named, under the identity a fresh sign-in binds there. +// Mirrors `core/integration/tests/cluster/failover_client_continuity.rs`. +func TestFailover_ResumesOnASurvivorAfterTheSignedInNodeDies(t *testing.T) { + var survivor *testListener + var primary *testListener + var primaryDead atomic.Bool + + survivor = listenVSR(t, nil, func(_, _ int, read request) []byte { + switch { + case read.code() == uint32(command.GetClusterMetadataCode): + return clusterMetadataFrame(t, 1, primary.address(), survivor.address()) + case read.operation() == vsr.OperationRegister: + return registerReplyFrame(7, 512) + default: + return replyFrame(vsr.OperationNonReplicated, nil) + } + }) + + primary = listenVSR(t, nil, func(_, _ int, read request) []byte { + // A dead node answers nothing; returning nil drops the connection the + // way a killed process does. + if primaryDead.Load() { + return nil + } + switch { + case read.code() == uint32(command.GetClusterMetadataCode): + // The primary leads, so the sign-in settles here and the roster is + // only remembered -- not acted on -- until the node dies. + return clusterMetadataFrame(t, 0, primary.address(), survivor.address()) + case read.operation() == vsr.OperationRegister: + return registerReplyFrame(7, 128) + default: + return replyFrame(vsr.OperationNonReplicated, nil) + } + }) + + // No auto-login: the credentials come from the caller's own sign-in, which + // is the shape that could not reconnect at all before. + client := newDialingClient(t, primary.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "iggy", "iggy") + require.NoError(t, err) + require.NoError(t, client.Ping(ctx), "the live primary answers") + require.Equal(t, primary.address(), client.currentServerAddress) + + primaryDead.Store(true) + require.NoError(t, primary.listener.Close(), "stop accepting, so a redial is refused") + + require.NoError(t, client.Ping(ctx), + "the client has to resume on the survivor the roster named") + + assert.Equal(t, survivor.address(), client.currentServerAddress, + "the client moved off the dead endpoint") + assert.True(t, client.session.Bound(), "the session was re-established") + + var registers int + for _, read := range survivor.recorded() { + if read.operation() == vsr.OperationRegister { + registers++ + } + } + assert.Equal(t, 1, registers, + "the remembered credentials signed in again on the survivor") +} + +// Without any credentials there is nothing to sign in with, so a request on a +// dead node fails instead of reconnecting into an unauthenticated session. +func TestFailover_FailsFastWhenNothingEverSignedIn(t *testing.T) { + var server *testListener + var dead atomic.Bool + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + if dead.Load() { + return nil + } + return singleNodeHandler(t, func() string { return server.address() })(0, 0, read) + }) + + client := newDialingClient(t, server.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + require.NoError(t, client.Ping(ctx)) + + dead.Store(true) + require.NoError(t, server.listener.Close()) + + assert.Error(t, client.Ping(ctx), + "a client that never signed in cannot restore a session by reconnecting") +} + +// An explicit sign-out is caller intent: the reconnect must not sign back in +// with the credentials the earlier sign-in used. +func TestFailover_DoesNotResurrectASignedOutSession(t *testing.T) { + var server *testListener + server = listenVSR(t, nil, singleNodeHandler(t, func() string { return server.address() })) + + client := newDialingClient(t, server.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "iggy", "iggy") + require.NoError(t, err) + require.NoError(t, client.LogoutUser(ctx)) + + credentials, ok := client.signInCredentials() + assert.False(t, ok, "the sign-out forgot them") + assert.Empty(t, credentials.username) +} + +func TestFailover_LeavesTheReestablishPauseToSingleEndpointClients(t *testing.T) { + client := NewIggyTcpClient(slog.New(slog.DiscardHandler), + WithServerAddress("127.0.0.1:8090")) + client.config.reconnection.reestablishAfter = time.Minute + client.connectedAt = time.Now() + + // One endpoint: the pause is the only thing keeping a reconnect from + // hammering the node it just lost. + require.Len(t, client.connectionCandidates(), 1) + + client.knownServerAddresses = []string{"127.0.0.1:8091"} + require.Len(t, client.connectionCandidates(), 2, + "with somewhere else to go the pause only delays the failover") +} diff --git a/foreign/go/client/tcp/tcp_session_credentials_test.go b/foreign/go/client/tcp/tcp_session_credentials_test.go new file mode 100644 index 0000000000..c219d4f6a3 --- /dev/null +++ b/foreign/go/client/tcp/tcp_session_credentials_test.go @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tcp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newCredentialClient(autoLogin AutoLogin) *IggyTcpClient { + config := defaultTcpClientConfig() + config.autoLogin = autoLogin + return &IggyTcpClient{config: config} +} + +func TestSignInCredentials_AreAbsentUntilSomethingSignsIn(t *testing.T) { + client := newCredentialClient(AutoLogin{}) + + _, ok := client.signInCredentials() + assert.False(t, ok) +} + +func TestSignInCredentials_ComeFromAManualSignInWithoutAutoLogin(t *testing.T) { + client := newCredentialClient(AutoLogin{}) + + client.rememberLogin(NewUsernamePasswordCredentials("iggy", "secret")) + + credentials, ok := client.signInCredentials() + require.True(t, ok) + assert.Equal(t, "iggy", credentials.username) + assert.Equal(t, "secret", credentials.password) + + // An explicit sign-out leaves no session to restore, and a reconnect must + // not resurrect one. + client.forgetLogin() + _, ok = client.signInCredentials() + assert.False(t, ok) +} + +func TestSignInCredentials_PreferTheConfiguredOnes(t *testing.T) { + client := newCredentialClient(NewAutoLogin(NewUsernamePasswordCredentials("configured", "secret"))) + + client.rememberLogin(NewPersonalAccessTokenCredentials("signed-in-token")) + + credentials, ok := client.signInCredentials() + require.True(t, ok) + assert.Equal(t, "configured", credentials.username) + assert.Empty(t, credentials.personalAccessToken) +} diff --git a/foreign/go/client/tcp/tcp_session_management.go b/foreign/go/client/tcp/tcp_session_management.go index d409f7d9af..b6a91e30ea 100644 --- a/foreign/go/client/tcp/tcp_session_management.go +++ b/foreign/go/client/tcp/tcp_session_management.go @@ -33,7 +33,12 @@ func (c *IggyTcpClient) LoginUser(ctx context.Context, username string, password if err != nil { return nil, err } - return c.register(ctx, uint32(command.LoginRegisterCode), body) + identity, err := c.register(ctx, uint32(command.LoginRegisterCode), body) + if err != nil { + return nil, err + } + c.rememberLogin(NewUsernamePasswordCredentials(username, password)) + return identity, nil } func (c *IggyTcpClient) LoginWithPersonalAccessToken(ctx context.Context, token string) (*iggcon.IdentityInfo, error) { @@ -41,7 +46,12 @@ func (c *IggyTcpClient) LoginWithPersonalAccessToken(ctx context.Context, token if err != nil { return nil, err } - return c.register(ctx, uint32(command.LoginRegisterWithPATCode), body) + identity, err := c.register(ctx, uint32(command.LoginRegisterWithPATCode), body) + if err != nil { + return nil, err + } + c.rememberLogin(NewPersonalAccessTokenCredentials(token)) + return identity, nil } // register runs the sign-in handshake, binds the session the server assigned, @@ -188,6 +198,7 @@ func (c *IggyTcpClient) LogoutUser(ctx context.Context) error { c.groups.clear() c.topics.clearCounts() c.mtx.Unlock() + c.forgetLogin() return nil } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java index e2673d965a..93f463e723 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java @@ -49,6 +49,8 @@ import java.io.File; import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -58,6 +60,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Stream; /** * Async TCP client for Apache Iggy message streaming, built on Netty. @@ -129,10 +132,38 @@ public class AsyncIggyTcpClient { private final Optional tlsCertificate; private final TcpConnectionPoolConfig poolConfig; private final ClientRoutingState routingState = new ClientRoutingState(); + private final LoginRoutingHook loginRoutingHook = new LoginRoutingHook() { + + @Override + public CompletableFuture loginOnLeader(Supplier> loginAttempt) { + return AsyncIggyTcpClient.this.loginOnLeader(loginAttempt); + } + + @Override + public void forgetLogin() { + rememberedLogin = null; + } + }; private final AtomicReference connection = new AtomicReference<>(); private final AtomicReference> loginChain = new AtomicReference<>(CompletableFuture.completedFuture(null)); private volatile ConnectionInfo connectionInfo; + /** + * Every node the roster named on the last leader check, kept as redial + * candidates. A node dies together with its address, and the roster is + * unreachable exactly when it is needed, so it has to have been + * remembered while the connection was still healthy. + */ + private volatile List rosterTargets = List.of(); + /** + * The login a successful sign-in ran, replayed after a redial so the + * session is re-established on whichever node answers. The supplier + * already carries the credentials it signed in with, so nothing new is + * stored. Cleared on an explicit sign-out, which leaves no session to + * restore. + */ + private volatile Supplier> rememberedLogin; + private volatile boolean closed; private MessagesClient messagesClient; private ConsumerGroupsClient consumerGroupsClient; @@ -235,9 +266,9 @@ public CompletableFuture connect() { consumerOffsetsClient = new ConsumerOffsetsTcpClient(currentConnection); streamsClient = new StreamsTcpClient(currentConnection); topicsClient = new TopicsTcpClient(currentConnection); - usersClient = new UsersTcpClient(currentConnection, this::loginOnLeader); + usersClient = new UsersTcpClient(currentConnection, loginRoutingHook); systemClient = new SystemTcpClient(currentConnection); - personalAccessTokensClient = new PersonalAccessTokensTcpClient(currentConnection, this::loginOnLeader); + personalAccessTokensClient = new PersonalAccessTokensTcpClient(currentConnection, loginRoutingHook); partitionsClient = new PartitionsTcpClient(currentConnection); }); } @@ -585,7 +616,7 @@ private CompletableFuture redialAttempt(int attempt, RetryPolicy policy) { log.error("Redial gave up after {} attempts, next request will fail fast", policy.getMaxRetries()); return CompletableFuture.completedFuture(null); } - ConnectionInfo target = ReconnectPlan.target(connectionInfo, seedConnectionInfo, attempt); + ConnectionInfo target = ReconnectPlan.target(redialCandidates(), attempt); Duration delay = ReconnectPlan.delay(policy, attempt); Executor delayedExecutor = CompletableFuture.delayedExecutor(delay.toMillis(), TimeUnit.MILLISECONDS); return CompletableFuture.supplyAsync(() -> null, delayedExecutor).thenCompose(ignored -> { @@ -617,6 +648,12 @@ private CompletableFuture redialAttempt(int attempt, RetryPolicy policy) { * again before Register when the redialed node is not the leader. */ private CompletableFuture replayLogin() { + Supplier> replay = rememberedLogin; + if (replay != null) { + // Runs through loginOnLeader, so a redial that landed on a backup + // still settles on the leader before the session is used. + return loginOnLeader(replay).thenApply(identity -> null); + } if (username.isEmpty() || password.isEmpty() || usersClient == null) { return CompletableFuture.completedFuture(null); } @@ -638,6 +675,9 @@ CompletableFuture loginOnLeader(Supplier callerFuture = new CompletableFuture<>(); transaction.whenComplete((identity, error) -> { gate.complete(null); + if (error == null) { + rememberedLogin = loginAttempt; + } if (error != null) { callerFuture.completeExceptionally(error); } else { @@ -732,7 +772,33 @@ CompletableFuture> findLeaderElsewhere(ConnectionInfo c if (currentSystemClient == null) { return CompletableFuture.completedFuture(Optional.empty()); } - return LeaderAwareness.findLeaderElsewhere(currentSystemClient::getClusterMetadata, currentTarget); + return LeaderAwareness.findLeaderElsewhere(currentSystemClient::getClusterMetadata, currentTarget) + .thenApply(lookup -> { + // Replaced wholesale rather than merged: the roster is the + // cluster's own answer about where its nodes are, so a node + // it dropped stops being dialed. The configured seed is + // kept separately and outlives it. + if (!lookup.endpoints().isEmpty()) { + rosterTargets = lookup.endpoints(); + } + return lookup.redirect(); + }); + } + + /** + * Endpoints a redial rotates through, likeliest first: where the client + * currently is, the address it was configured with, then the roster it + * learned while connected. Duplicates are dropped, so an endpoint the + * roster merely spells differently does not earn a second attempt. + */ + private List redialCandidates() { + List candidates = new ArrayList<>(); + candidates.add(connectionInfo); + Stream.concat(Stream.of(seedConnectionInfo), rosterTargets.stream()) + .filter(endpoint -> + candidates.stream().noneMatch(candidate -> LeaderAwareness.isSameAddress(candidate, endpoint))) + .forEach(candidates::add); + return List.copyOf(candidates); } CompletableFuture retarget(ConnectionInfo newTarget) { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java index f541e9de7c..3b258b7ef4 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java @@ -72,12 +72,12 @@ private LeaderAwareness() {} * exceptionally, so the redirection path cannot fail the login that * triggered it. */ - static CompletableFuture> findLeaderElsewhere( + static CompletableFuture findLeaderElsewhere( Supplier> fetchMetadata, ConnectionInfo currentTarget) { return findLeaderElsewhere(fetchMetadata, currentTarget, LEADERLESS_WAIT_BUDGET, LEADERLESS_POLL_INTERVAL); } - static CompletableFuture> findLeaderElsewhere( + static CompletableFuture findLeaderElsewhere( Supplier> fetchMetadata, ConnectionInfo currentTarget, Duration leaderlessWaitBudget, @@ -87,7 +87,7 @@ static CompletableFuture> findLeaderElsewhere( fetchMetadata, currentTarget, leaderlessWaitBudget, leaderlessPollInterval, electionDeadlineNanos); } - private static CompletableFuture> pollForLeader( + private static CompletableFuture pollForLeader( Supplier> fetchMetadata, ConnectionInfo currentTarget, Duration leaderlessWaitBudget, @@ -99,16 +99,18 @@ private static CompletableFuture> pollForLeader( } catch (RuntimeException fetchError) { fetched = CompletableFuture.failedFuture(fetchError); } - return fetched.>>handleAsync((metadata, error) -> { + return fetched.>handleAsync((metadata, error) -> { if (error != null) { log.warn( "Failed to get cluster metadata: {}, connection will continue on server node {}", error.getMessage(), currentTarget.serverAddress()); - return CompletableFuture.completedFuture(Optional.empty()); + return CompletableFuture.completedFuture(LeaderLookup.inconclusive()); } LeaderCheck check; + List endpoints; try { + endpoints = nodeTargets(metadata); check = checkLeader(metadata, currentTarget); } catch (RuntimeException selectionError) { log.warn( @@ -116,10 +118,11 @@ private static CompletableFuture> pollForLeader( + " on server node {}", selectionError.getMessage(), currentTarget.serverAddress()); - return CompletableFuture.completedFuture(Optional.empty()); + return CompletableFuture.completedFuture(LeaderLookup.inconclusive()); } if (check instanceof LeaderCheck.Redirect redirect) { - return CompletableFuture.completedFuture(Optional.of(redirect.target())); + return CompletableFuture.completedFuture( + new LeaderLookup(Optional.of(redirect.target()), endpoints)); } if (check instanceof LeaderCheck.NoLeader) { if (System.nanoTime() >= electionDeadlineNanos) { @@ -128,7 +131,9 @@ private static CompletableFuture> pollForLeader( + " continue on server node {}", leaderlessWaitBudget, currentTarget.serverAddress()); - return CompletableFuture.completedFuture(Optional.empty()); + // A leaderless roster still names where the nodes + // are, and that is what a redial needs. + return CompletableFuture.completedFuture(new LeaderLookup(Optional.empty(), endpoints)); } Executor retryAfterInterval = CompletableFuture.delayedExecutor( leaderlessPollInterval.toMillis(), TimeUnit.MILLISECONDS); @@ -142,11 +147,23 @@ private static CompletableFuture> pollForLeader( retryAfterInterval) .thenCompose(Function.identity()); } - return CompletableFuture.completedFuture(Optional.empty()); + return CompletableFuture.completedFuture(new LeaderLookup(Optional.empty(), endpoints)); }) .thenCompose(Function.identity()); } + /** + * Every node's target for the tcp transport, in roster order. A node that + * does not expose the transport reports port 0 and is skipped: dialing it + * would burn a redial attempt on an endpoint that cannot answer. + */ + static List nodeTargets(ClusterMetadata metadata) { + return metadata.nodes().stream() + .filter(node -> node.endpoints().tcp() != 0) + .map(node -> new ConnectionInfo(node.ip(), node.endpoints().tcp())) + .toList(); + } + /** * One leader-check verdict from a cluster-metadata snapshot. */ @@ -248,6 +265,24 @@ private static boolean reachesOnlyLocalMachine(InetAddress[] addresses) { /** * One leader-check verdict from a cluster-metadata snapshot. */ + /** + * What one leader check learned from the roster: where to go, and every + * node the cluster named for this transport. A client keeps the latter as + * redial candidates, because the address it was configured with dies with + * its node and the roster is unreachable exactly when it is needed. + */ + record LeaderLookup(Optional redirect, List endpoints) { + + LeaderLookup { + endpoints = List.copyOf(endpoints); + } + + /** A check that learned nothing: stay put, remember no endpoint. */ + static LeaderLookup inconclusive() { + return new LeaderLookup(Optional.empty(), List.of()); + } + } + sealed interface LeaderCheck { /** A healthy leader with an enabled tcp transport lives elsewhere; reconnect to it. */ diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java index a6acb7dc09..904b492e2a 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java @@ -41,4 +41,10 @@ interface LoginRoutingHook { * @return the identity returned by the successful Register response */ CompletableFuture loginOnLeader(Supplier> loginAttempt); + + /** + * Drops any login kept for replay. Called on an explicit sign-out: there + * is no session left to restore, and a redial must not resurrect one. + */ + default void forgetLogin() {} } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java index ae596731f4..3e9f45bf8b 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java @@ -23,6 +23,7 @@ import org.apache.iggy.config.RetryPolicy; import java.time.Duration; +import java.util.List; /** * Pure redial planning: which address to dial on a given reconnect attempt @@ -33,16 +34,18 @@ final class ReconnectPlan { private ReconnectPlan() {} /** - * Alternates reconnect dials between the current endpoint and the - * configured seed. After a leader redirect the current endpoint may die - * with the leader, and the seed is the way back to the rest of the - * cluster. Attempts are 1-based; odd attempts dial the current endpoint. + * Rotates reconnect dials through every endpoint the client knows, in the + * order the candidate list gives them. After a leader redirect the current + * endpoint may die with the leader, and the rest of the list -- the + * configured seed and the roster learned while connected -- is the way + * back to the rest of the cluster. Attempts are 1-based; the first dials + * the head of the list. */ - static ConnectionInfo target(ConnectionInfo current, ConnectionInfo seed, int attempt) { - if (current.equals(seed)) { - return current; + static ConnectionInfo target(List candidates, int attempt) { + if (candidates.isEmpty()) { + throw new IllegalArgumentException("a redial needs at least one candidate endpoint"); } - return attempt % 2 == 1 ? current : seed; + return candidates.get(Math.floorMod(attempt - 1, candidates.size())); } /** diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java index b6e7ab1940..1bf1e7ceda 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java @@ -190,6 +190,7 @@ public CompletableFuture logout() { return connection().send(CommandCode.User.LOGOUT.getValue(), payload).thenAccept(response -> { response.release(); + routingHook.forgetLogin(); log.debug("Logged out successfully"); }); } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java new file mode 100644 index 0000000000..f541701b9a --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.iggy.config.RetryPolicy; +import org.junit.jupiter.api.Test; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The node a client signed in on dies; its next request has to complete on a + * survivor the roster named, under a session established there. Mirrors + * {@code core/integration/tests/cluster/failover_client_continuity.rs}. The + * mock VSR framing matches {@link AsyncIggyTcpClientTransientFailoverTest}, + * kept separate so a death mid-connection cannot disturb that suite's server. + */ +class AsyncIggyTcpClientEndpointFailoverTest { + private static final int HEADER_SIZE = 256; + private static final int SIZE_OFFSET = 48; + private static final int COMMAND_OFFSET = 60; + private static final int REQUEST_ID_OFFSET = 168; + private static final int REQUEST_OPERATION_OFFSET = 176; + private static final int REQUEST_CODE_OFFSET = 196; + private static final int REPLY_REQUEST_ID_OFFSET = 200; + private static final int REPLY_OPERATION_OFFSET = 208; + private static final int REPLY_STATUS_OFFSET = 216; + + private static final int COMMAND_REPLY = 8; + private static final int OPERATION_REGISTER = 1; + private static final int OPERATION_NON_REPLICATED = 2; + private static final int GET_CLUSTER_METADATA_CODE = 12; + private static final int PING_CODE = 1; + + @Test + void shouldResumeOnASurvivorAfterTheSignedInNodeDies() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket primarySocket = new ServerSocket(0, 4, loopback); + ServerSocket survivorSocket = new ServerSocket(0, 4, loopback)) { + int primaryPort = primarySocket.getLocalPort(); + int survivorPort = survivorSocket.getLocalPort(); + AtomicInteger survivorRegistrations = new AtomicInteger(); + AtomicInteger survivorPings = new AtomicInteger(); + + // The primary leads, so the sign-in settles there and the roster is + // only remembered -- not acted on -- until the node dies. + MockNode primary = MockNode.serve(primarySocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, primaryPort)); + } + if (request.operation() == OPERATION_REGISTER) { + return Response.success(OPERATION_REGISTER, registerBody(1)); + } + return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER); + }); + MockNode survivor = MockNode.serve(survivorSocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, survivorPort)); + } + if (request.operation() == OPERATION_REGISTER) { + survivorRegistrations.incrementAndGet(); + return Response.success(OPERATION_REGISTER, registerBody(2)); + } + if (request.is(PING_CODE, OPERATION_NON_REPLICATED)) { + survivorPings.incrementAndGet(); + } + return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(primaryPort) + .credentials("iggy", "iggy") + .requestTimeout(Duration.ofSeconds(5)) + // A redial rotates one endpoint per attempt, so the survivor + // is the second: keep the pacing short enough to observe. + .retryPolicy(RetryPolicy.fixedDelay(8, Duration.ofMillis(50))) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.login().get(5, TimeUnit.SECONDS); + client.sendBinaryRequest(PING_CODE, new byte[0]).get(5, TimeUnit.SECONDS); + assertThat(client.getConnectionInfo().port()).isEqualTo(primaryPort); + + primary.kill(); + + // The request in flight when the node died is allowed to fail; + // what is not allowed is never completing one, which is what a + // client that only knows the dead endpoint does. + assertThat(resumeWithin(client, Duration.ofSeconds(10))) + .as("the client has to resume on the survivor the roster named") + .isTrue(); + + assertThat(client.getConnectionInfo().port()) + .as("the client moved off the dead endpoint") + .isEqualTo(survivorPort); + assertThat(survivorRegistrations) + .as("the login was replayed on the survivor") + .hasValueGreaterThanOrEqualTo(1); + assertThat(survivorPings) + .as("the request landed on the survivor") + .hasValueGreaterThanOrEqualTo(1); + } finally { + client.close().get(5, TimeUnit.SECONDS); + survivor.close(); + } + } + } + + /** Retries until one request completes, or the budget runs out. */ + private static boolean resumeWithin(AsyncIggyTcpClient client, Duration budget) throws InterruptedException { + long deadline = System.nanoTime() + budget.toNanos(); + while (System.nanoTime() < deadline) { + try { + client.sendBinaryRequest(PING_CODE, new byte[0]).get(2, TimeUnit.SECONDS); + return true; + } catch (ExecutionException | TimeoutException stillDown) { + Thread.sleep(50); + } + } + return false; + } + + private static ByteBuf registerBody(long session) { + ByteBuf body = Unpooled.buffer(); + body.writeIntLE(0); + body.writeIntLE(1); + body.writeLongLE(session); + body.writeIntLE(11 << 10); + body.writeByte(0); + return body; + } + + private static ByteBuf clusterMetadata(int primaryPort, int survivorPort, int leaderPort) { + ByteBuf body = Unpooled.buffer(); + writeString(body, "test-cluster"); + body.writeIntLE(2); + writeNode(body, "primary", primaryPort, primaryPort == leaderPort); + writeNode(body, "survivor", survivorPort, survivorPort == leaderPort); + return body; + } + + private static void writeNode(ByteBuf body, String name, int port, boolean leader) { + writeString(body, name); + writeString(body, InetAddress.getLoopbackAddress().getHostAddress()); + body.writeShortLE(port); + body.writeShortLE(0); + body.writeShortLE(0); + body.writeShortLE(0); + body.writeByte(leader ? 0 : 1); + body.writeByte(0); + } + + private static void writeString(ByteBuf body, String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + body.writeIntLE(bytes.length); + body.writeBytes(bytes); + } + + /** + * A loopback VSR node that keeps serving every connection it accepts until + * it is killed, which drops the live sockets and stops accepting so a + * redial is refused the way a dead process refuses one. + */ + private static final class MockNode { + private final ServerSocket server; + private final List accepted = new CopyOnWriteArrayList<>(); + private volatile boolean killed; + + private MockNode(ServerSocket server) { + this.server = server; + } + + static MockNode serve(ServerSocket server, RequestHandler handler) { + MockNode node = new MockNode(server); + CompletableFuture.runAsync(() -> { + while (!node.killed) { + try { + Socket socket = server.accept(); + node.accepted.add(socket); + CompletableFuture.runAsync(() -> node.exchange(socket, handler)); + } catch (IOException accepted) { + return; + } + } + }); + return node; + } + + private void exchange(Socket socket, RequestHandler handler) { + try (socket) { + InputStream input = socket.getInputStream(); + OutputStream output = socket.getOutputStream(); + Request request; + while (!killed && (request = readRequest(input)) != null) { + writeResponse(output, request, handler.handle(request)); + } + } catch (IOException closed) { + // A killed node and a client that went away look the same here. + } + } + + void kill() throws IOException { + killed = true; + for (Socket socket : accepted) { + socket.close(); + } + server.close(); + } + + void close() throws IOException { + kill(); + } + } + + private static Request readRequest(InputStream input) throws IOException { + byte[] header = input.readNBytes(HEADER_SIZE); + if (header.length == 0) { + return null; + } + if (header.length != HEADER_SIZE) { + throw new EOFException("Truncated VSR request header"); + } + ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + int size = fields.getInt(SIZE_OFFSET); + byte[] body = input.readNBytes(size - HEADER_SIZE); + if (body.length != size - HEADER_SIZE) { + throw new EOFException("Truncated VSR request body"); + } + return new Request( + Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), + fields.getInt(REQUEST_CODE_OFFSET), + fields.getLong(REQUEST_ID_OFFSET)); + } + + private static void writeResponse(OutputStream output, Request request, Response response) throws IOException { + byte[] body = new byte[response.body().readableBytes()]; + response.body().readBytes(body); + response.body().release(); + byte[] header = new byte[HEADER_SIZE]; + ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + fields.putInt(SIZE_OFFSET, HEADER_SIZE + body.length); + header[COMMAND_OFFSET] = (byte) COMMAND_REPLY; + fields.putLong(REPLY_REQUEST_ID_OFFSET, request.requestId()); + header[REPLY_OPERATION_OFFSET] = (byte) response.operation(); + fields.putInt(REPLY_STATUS_OFFSET, 0); + output.write(header); + output.write(body); + output.flush(); + } + + private record Request(int operation, int commandCode, long requestId) { + boolean is(int expectedCode, int expectedOperation) { + return commandCode == expectedCode && operation == expectedOperation; + } + } + + private record Response(int operation, ByteBuf body) { + static Response success(int operation, ByteBuf body) { + return new Response(operation, body); + } + } + + @FunctionalInterface + private interface RequestHandler { + Response handle(Request request); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java index 00540cd23c..b5b73cfb78 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java @@ -206,6 +206,10 @@ class FindLeaderElsewhere { private final ConnectionInfo currentTarget = new ConnectionInfo("iggy-follower", 8092); private Optional findLeader(Supplier> fetch) { + return lookUpLeader(fetch).redirect(); + } + + private LeaderAwareness.LeaderLookup lookUpLeader(Supplier> fetch) { return LeaderAwareness.findLeaderElsewhere(fetch, currentTarget, BUDGET, INTERVAL) .orTimeout(30, TimeUnit.SECONDS) .join(); @@ -248,7 +252,8 @@ void shouldGiveUpOnLeaderlessClusterAfterBudget() { Duration.ofMillis(100), INTERVAL) .orTimeout(30, TimeUnit.SECONDS) - .join(); + .join() + .redirect(); assertThat(leader).isEmpty(); assertThat(fetchCount.get()).isGreaterThan(1); @@ -276,6 +281,22 @@ void shouldGiveUpWhenMetadataFetchThrowsSynchronously() { assertThat(leader).isEmpty(); } + @Test + void shouldRememberEveryNodeTheRosterNamesEvenWhileLeaderless() { + var lookup = LeaderAwareness.findLeaderElsewhere( + () -> CompletableFuture.completedFuture(leaderlessCluster()), + currentTarget, + Duration.ofMillis(100), + INTERVAL) + .orTimeout(30, TimeUnit.SECONDS) + .join(); + + // A leaderless roster still names where the nodes are, and that is + // what a redial needs. + assertThat(lookup.redirect()).isEmpty(); + assertThat(lookup.endpoints()).isNotEmpty(); + } + @Test void shouldStayWithoutPollingWhenAlreadyOnLeader() { var fetchCount = new AtomicInteger(); @@ -289,7 +310,8 @@ void shouldStayWithoutPollingWhenAlreadyOnLeader() { BUDGET, INTERVAL) .orTimeout(30, TimeUnit.SECONDS) - .join(); + .join() + .redirect(); assertThat(leader).isEmpty(); assertThat(fetchCount).hasValue(1); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java index ce833a58cb..fff6114d30 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java @@ -24,26 +24,36 @@ import org.junit.jupiter.api.Test; import java.time.Duration; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class ReconnectPlanTest { private final ConnectionInfo seed = new ConnectionInfo("seed-node", 8090); private final ConnectionInfo current = new ConnectionInfo("leader-node", 8090); + private final ConnectionInfo survivor = new ConnectionInfo("survivor-node", 8090); @Test - void shouldAlternateBetweenCurrentAndSeed() { - assertThat(ReconnectPlan.target(current, seed, 1)).isEqualTo(current); - assertThat(ReconnectPlan.target(current, seed, 2)).isEqualTo(seed); - assertThat(ReconnectPlan.target(current, seed, 3)).isEqualTo(current); - assertThat(ReconnectPlan.target(current, seed, 4)).isEqualTo(seed); + void shouldRotateThroughEveryKnownEndpoint() { + var candidates = List.of(current, seed, survivor); + + assertThat(ReconnectPlan.target(candidates, 1)).isEqualTo(current); + assertThat(ReconnectPlan.target(candidates, 2)).isEqualTo(seed); + assertThat(ReconnectPlan.target(candidates, 3)).isEqualTo(survivor); + assertThat(ReconnectPlan.target(candidates, 4)).isEqualTo(current); } @Test void shouldDialOnlyOneAddressWhenNeverRedirected() { - assertThat(ReconnectPlan.target(seed, seed, 1)).isEqualTo(seed); - assertThat(ReconnectPlan.target(seed, seed, 2)).isEqualTo(seed); + assertThat(ReconnectPlan.target(List.of(seed), 1)).isEqualTo(seed); + assertThat(ReconnectPlan.target(List.of(seed), 2)).isEqualTo(seed); + } + + @Test + void shouldRefuseToPlanARedialWithoutCandidates() { + assertThatThrownBy(() -> ReconnectPlan.target(List.of(), 1)).isInstanceOf(IllegalArgumentException.class); } @Test diff --git a/foreign/node/src/client/client.connection.test.ts b/foreign/node/src/client/client.connection.test.ts index 2564cfacfd..afb034d09b 100644 --- a/foreign/node/src/client/client.connection.test.ts +++ b/foreign/node/src/client/client.connection.test.ts @@ -394,6 +394,31 @@ describe('IggyConnection', () => { } ); + it('rotates a redial through the roster it learned while connected', + async () => { + const seed = await startServer(); + const seedPort = (seed.address() as AddressInfo).port; + const connection = new IggyConnection(connectionConfig(seed)); + connection.on('error', () => undefined); + try { + connection.rememberRoster([ + { host: '127.0.0.1', port: seedPort }, + { host: '127.0.0.1', port: seedPort + 1 }, + { host: '127.0.0.1', port: seedPort + 2 } + ]); + // The endpoint the client is on leads, the roster follows, and the + // roster's copy of that endpoint does not earn a second attempt. + assert.deepEqual( + connection._redialCandidates().map((options) => options.port), + [seedPort, seedPort + 1, seedPort + 2] + ); + } finally { + connection._destroy(); + await new Promise((resolve) => seed.close(() => resolve())); + } + } + ); + it('settles a dial in flight when a redirect replaces the socket', async () => { const seed = await startServer(); diff --git a/foreign/node/src/client/client.connection.ts b/foreign/node/src/client/client.connection.ts index 10b68639d0..0ef16326f3 100644 --- a/foreign/node/src/client/client.connection.ts +++ b/foreign/node/src/client/client.connection.ts @@ -119,6 +119,13 @@ export class IggyConnection extends EventEmitter { private reconnectPromise?: Promise; /** Endpoint the client was configured with, kept across leader redirects */ private readonly seedOptions: ClientConfig['options']; + /** + * Every node the roster named on the last read, kept as redial candidates. + * A node dies together with its address, and the roster is unreachable + * exactly when it is needed, so it has to have been remembered while the + * connection was still healthy. + */ + private rosterEndpoints: { host: string, port: number }[]; /** Incremental response frame decoder */ private responseDecoder: ResponseFrameDecoder; @@ -136,6 +143,7 @@ export class IggyConnection extends EventEmitter { this.ending = false; this.reconnectOption = { ...DefaultReconnectOption, ...config.reconnect }; this.seedOptions = { ...config.options }; + this.rosterEndpoints = []; this.reconnectCount = 0; this.connectPromise = undefined; this.reconnectPromise = undefined; @@ -301,7 +309,6 @@ export class IggyConnection extends EventEmitter { ): Promise { let lastError = initialError; let expectedSocket = this.socket; - let attempt = 0; while (enabled && this.reconnectCount < maxRetries) { this.connecting = true; this.reconnectCount += 1; @@ -313,24 +320,28 @@ export class IggyConnection extends EventEmitter { if (this.connected || this.socket !== expectedSocket) return this.connect(); - const options = this._reconnectTarget(attempt); - attempt += 1; - const socket = this._installSocket( - getTransport({ ...this.config, options }) - ); - this.socket = socket; - expectedSocket = socket; - try { - await this._waitForConnection(socket); - if (this.socket !== socket) - return this.connect(); - this.config.options = options; - return this; - } catch (error) { - lastError = error instanceof Error - ? error - : new Error(String(error)); - debug('reconnect attempt failed', lastError); + // Every endpoint gets its turn inside one attempt, so a full pass over + // the cluster costs one retry rather than one per endpoint: a pass that + // stopped at the first refusal would never reach the survivors of a + // client configured for a single retry. + for (const options of this._redialCandidates()) { + const socket = this._installSocket( + getTransport({ ...this.config, options }) + ); + this.socket = socket; + expectedSocket = socket; + try { + await this._waitForConnection(socket); + if (this.socket !== socket) + return this.connect(); + this.config.options = options; + return this; + } catch (error) { + lastError = error instanceof Error + ? error + : new Error(String(error)); + debug('reconnect attempt failed', lastError); + } } } @@ -342,16 +353,43 @@ export class IggyConnection extends EventEmitter { } /** - * Alternates reconnect dials between the current endpoint and the - * configured seed. After a leader redirect the current endpoint may die - * with the leader, and the seed is the way back to the rest of the cluster. + * Records the cluster roster as redial candidates. + * + * Replaced wholesale rather than merged: the roster is the cluster's own + * answer about where its nodes are, so a node it dropped stops being + * dialed. The configured seed is kept separately and outlives it. + */ + rememberRoster(endpoints: { host: string, port: number }[]): void { + if (endpoints.length === 0) + return; + this.rosterEndpoints = endpoints; + } + + /** + * Endpoints a redial rotates through, likeliest first: where the client + * currently is, the endpoint it was configured with, then the roster it + * learned while connected. After a leader redirect the current endpoint may + * die with the leader, and the rest of the list is the way back to the + * cluster. Duplicates are dropped, so an endpoint the roster merely spells + * differently does not earn a second attempt. */ - private _reconnectTarget(attempt: number): ClientConfig['options'] { - const current = this.config.options; - if (this.seedOptions.host === current.host && - this.seedOptions.port === current.port) - return current; - return attempt % 2 === 0 ? current : this.seedOptions; + _redialCandidates(): ClientConfig['options'][] { + const candidates = [this.config.options]; + const known = [ + this.seedOptions, + ...this.rosterEndpoints.map( + ({ host, port }) => ({ ...this.config.options, host, port }) + ) + ]; + for (const candidate of known) { + const duplicate = candidates.some( + (known) => known.port === candidate.port && + normalizeHost(known.host) === normalizeHost(candidate.host) + ); + if (!duplicate) + candidates.push(candidate); + } + return candidates; } async redirect(host: string, port: number) { diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts index 0697c5f3d4..07cbcecd46 100644 --- a/foreign/node/src/client/client.socket.test.ts +++ b/foreign/node/src/client/client.socket.test.ts @@ -505,6 +505,111 @@ describe('VSR client socket', () => { } }); + // The node a client authenticated on dies; its next command has to complete + // on a survivor the roster named, under a session established there. + // Mirrors `core/integration/tests/cluster/failover_client_continuity.rs`. + it('resumes on a survivor after the node it authenticated on dies', + async () => { + const primarySockets = new Set(); + let primaryDead = false; + + const survivor = await startVsrServer((frame, socket) => { + const operation = frame.readUInt8(REQUEST_OFFSET.operation); + if (operation === Operation.Register) { + socket.write(replyFrame(Operation.Register, registerReplyBody())); + return; + } + const code = frame.readUInt32LE(REQUEST_OFFSET.reserved); + if (code === COMMAND_CODE.GetClusterMetadata) { + // The survivor leads once the primary is gone. + socket.write(replyFrame( + Operation.NonReplicated, + twoNodeMetadataBody(primary.port, survivor.port) + )); + return; + } + socket.write(replyFrame(operation)); + }); + + const primary = await startVsrServer((frame, socket) => { + primarySockets.add(socket); + if (primaryDead) { + socket.destroy(); + return; + } + const operation = frame.readUInt8(REQUEST_OFFSET.operation); + if (operation === Operation.Register) { + socket.write(replyFrame(Operation.Register, registerReplyBody())); + return; + } + const code = frame.readUInt32LE(REQUEST_OFFSET.reserved); + if (code === COMMAND_CODE.GetClusterMetadata) { + // The primary leads, so the login settles here and the roster is + // only remembered, not acted on, until the node dies. + socket.write(replyFrame( + Operation.NonReplicated, + twoNodeMetadataBody(survivor.port, primary.port) + )); + return; + } + socket.write(replyFrame(operation)); + }); + + const config: ClientConfig = { + ...vsrConfig(primary.port), + reconnect: { enabled: true, interval: 1, maxRetries: 3 } + }; + const client = new CommandResponseStream(config); + try { + await client.authenticate(config.credentials); + await client.sendCommand(60_021, Buffer.alloc(0)); + assert.ok( + primary.frames.some( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_021 + ), + 'the live primary answered the first command' + ); + + primaryDead = true; + for (const socket of primarySockets) + socket.destroy(); + await primary.close(); + + // The attempt in flight when the socket died is allowed to fail; what + // is not allowed is never completing one, which is what a client that + // only knows the dead endpoint does. + let resumed = false; + let lastError: unknown; + for (let attempt = 0; attempt < 20 && !resumed; attempt += 1) { + try { + await client.sendCommand(60_021, Buffer.alloc(0)); + resumed = true; + } catch (error) { + lastError = error; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + } + assert.ok(resumed, `the client never resumed: ${String(lastError)}`); + + const operations = survivor.frames.map( + (frame) => frame.readUInt8(REQUEST_OFFSET.operation) + ); + assert.ok( + operations.includes(Operation.Register), + 'the client signed in again on the survivor' + ); + assert.ok( + survivor.frames.some( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_021 + ), + 'the command landed on the survivor the roster named' + ); + } finally { + client.destroy(); + await survivor.close(); + } + }); + it('keeps a single-node login on its node', async () => { const server = await startVsrServer( (frame, socket) => singleNodeHandler(server.port)(frame, socket) diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts index d1e5e4fe2a..4d57387e79 100644 --- a/foreign/node/src/client/client.socket.ts +++ b/foreign/node/src/client/client.socket.ts @@ -479,6 +479,13 @@ export class CommandResponseStream extends EventEmitter { { last: false } ); const metadata = GET_CLUSTER_METADATA.deserialize(response); + // Every read feeds the redial candidates, leaderless ones included: a + // roster with no leader still names where the nodes are. + this.connection.rememberRoster( + metadata.nodes + .filter((node) => node.endpoints.tcp !== 0) + .map((node) => ({ host: node.ip, port: node.endpoints.tcp })) + ); if (metadata.nodes.length <= 1) return undefined; const leader = metadata.nodes.find( From 5fe1a460168369b1a3753a12ac5ba66e5b15d64f Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Mon, 24 Aug 2026 10:03:58 +0200 Subject: [PATCH 04/16] fix CI --- .../tests/sdk/disconnect_relogin.rs | 65 +++++++++++++++++++ core/integration/tests/sdk/mod.rs | 1 + core/sdk/src/tcp/tcp_client.rs | 58 +++++++++++++++-- .../Implementations/TcpMessageStream.cs | 4 +- .../VsrTests/EndpointFailoverTests.cs | 8 ++- ...syncIggyTcpClientEndpointFailoverTest.java | 35 ++++++---- 6 files changed, 152 insertions(+), 19 deletions(-) create mode 100644 core/integration/tests/sdk/disconnect_relogin.rs diff --git a/core/integration/tests/sdk/disconnect_relogin.rs b/core/integration/tests/sdk/disconnect_relogin.rs new file mode 100644 index 0000000000..d78acb4664 --- /dev/null +++ b/core/integration/tests/sdk/disconnect_relogin.rs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! An explicit disconnect ends the session for good: the credentials a manual +//! sign-in remembered (for reconnecting across involuntary drops and +//! failovers) must not resurrect it. Pins at the Rust layer the contract the +//! C++ e2e suite asserts through the FFI (`DisconnectThenReconnectWithoutRelogin`, +//! `GetStatsBeforeLoginThrows`), so a regression fails here first instead of +//! three suites downstream. + +use iggy::prelude::*; +use integration::iggy_harness; + +#[iggy_harness] +async fn given_a_logged_in_client_when_explicitly_disconnected_should_require_a_fresh_login( + harness: &TestHarness, +) { + let client = harness.new_client().await.unwrap(); + client + .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) + .await + .unwrap(); + client.get_me().await.expect("authenticated get_me works"); + + client.disconnect().await.unwrap(); + client.connect().await.unwrap(); + assert!( + client.get_me().await.is_err(), + "an explicit disconnect is caller intent, like a logout: the sign-in it ended \ + must not be silently replayed by the reconnect" + ); + + client.disconnect().await.unwrap(); + assert!( + client.get_stats().await.is_err(), + "an operation after an explicit disconnect must fail instead of reconnecting \ + into a resurrected session" + ); + + // The remembered sign-in exists for involuntary drops; a fresh manual + // login after the disconnect works exactly as before. + client.connect().await.unwrap(); + client + .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) + .await + .unwrap(); + client + .get_me() + .await + .expect("a fresh login restores service"); +} diff --git a/core/integration/tests/sdk/mod.rs b/core/integration/tests/sdk/mod.rs index 3b934e8c95..065b257a76 100644 --- a/core/integration/tests/sdk/mod.rs +++ b/core/integration/tests/sdk/mod.rs @@ -18,6 +18,7 @@ mod consumer_group; mod consumer_group_membership; mod consumer_offset; +mod disconnect_relogin; mod hello_world; mod http_refresh; mod options; diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index 0b8e42b48d..f7d240c7d7 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -120,7 +120,12 @@ impl Client for TcpClient { } async fn disconnect(&self) -> Result<(), IggyError> { - TcpClient::disconnect(self).await + // An explicit disconnect is caller intent, like a logout: the session + // it ends must not be resurrected by the next reconnect, so the + // remembered sign-in goes with it. Involuntary drops (a dead socket, + // a failover) go through `disconnect_transport` and keep it. + self.forget_session_credentials().await; + TcpClient::disconnect_transport(self).await } async fn shutdown(&self) -> Result<(), IggyError> { @@ -187,7 +192,7 @@ impl BinaryTransport for TcpClient { return Err(error); } - self.disconnect().await?; + self.disconnect_transport().await?; let skip_auto_login = is_login_register_code(code); if skip_auto_login { @@ -615,7 +620,7 @@ impl TcpClient { // Clear connected_at to avoid reestablish_after delay during redirection self.connected_at.lock().await.take(); - self.disconnect().await?; + self.disconnect_transport().await?; *self.current_server_address.lock().await = new_leader_address; Ok(true) @@ -688,7 +693,13 @@ impl TcpClient { (elapsed < interval).then(|| IggyDuration::from(interval - elapsed)) } - async fn disconnect(&self) -> Result<(), IggyError> { + /// Tear down the connection without touching the remembered sign-in. + /// + /// The reconnect and redirect paths use this: their disconnect is not + /// caller intent, and forgetting the credentials here would strand the + /// failover unauthenticated. The public [`Client::disconnect`] wraps this + /// and forgets them first. + async fn disconnect_transport(&self) -> Result<(), IggyError> { if self.get_state().await == ClientState::Disconnected { return Ok(()); } @@ -1030,6 +1041,45 @@ mod tests { ); } + // The C++/Rust e2e contract: `login -> disconnect -> op` must fail until + // the caller signs in again. Only involuntary drops keep the sign-in. + #[tokio::test] + async fn an_explicit_disconnect_forgets_the_remembered_sign_in() { + let client = client_with("127.0.0.1:8090", Vec::new()); + client + .remember_session_credentials(Credentials::UsernamePassword( + "iggy".to_string(), + "iggy".into(), + )) + .await; + + Client::disconnect(&client).await.expect("disconnect"); + assert!( + client.sign_in_credentials().await.is_none(), + "an explicit disconnect ends the session for good, like a logout" + ); + } + + #[tokio::test] + async fn a_transport_drop_keeps_the_remembered_sign_in() { + let client = client_with("127.0.0.1:8090", Vec::new()); + client + .remember_session_credentials(Credentials::UsernamePassword( + "iggy".to_string(), + "iggy".into(), + )) + .await; + + client + .disconnect_transport() + .await + .expect("transport teardown"); + assert!( + client.sign_in_credentials().await.is_some(), + "an involuntary drop is what the failover exists for; the sign-in survives it" + ); + } + #[tokio::test] async fn a_sign_in_makes_a_client_without_auto_login_reconnectable() { let client = client_with("127.0.0.1:8090", Vec::new()); diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index 043a508092..c97493c45f 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -829,7 +829,9 @@ public async Task ChangePasswordAsync(Identifier userId, string currentPassword, LoginRegister.Serialize(userName, password), token); _rememberedLogin = new AutoLoginSettings { - Enabled = true, Username = userName, Password = password + Enabled = true, + Username = userName, + Password = password }; return identity; diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs index 3bc704e06e..bb555474a5 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs @@ -71,7 +71,9 @@ public async Task ResumesOnASurvivorAfterTheSignedInNodeDies() Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { - Enabled = true, MaxRetries = 4, InitialDelay = TimeSpan.FromMilliseconds(20) + Enabled = true, + MaxRetries = 4, + InitialDelay = TimeSpan.FromMilliseconds(20) } }; using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); @@ -109,7 +111,9 @@ public async Task FailsFastWhenNothingEverSignedIn() Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { - Enabled = true, MaxRetries = 2, InitialDelay = TimeSpan.FromMilliseconds(20) + Enabled = true, + MaxRetries = 2, + InitialDelay = TimeSpan.FromMilliseconds(20) } }; using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java index f541701b9a..1741e342d5 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java @@ -36,7 +36,6 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.List; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -200,6 +199,12 @@ private static void writeString(ByteBuf body, String value) { * A loopback VSR node that keeps serving every connection it accepts until * it is killed, which drops the live sockets and stops accepting so a * redial is refused the way a dead process refuses one. + * + *

Dedicated daemon threads, not {@code CompletableFuture.runAsync}: the + * accept loop and every connection handler block indefinitely, and parking + * them on the common pool starves it on a low-core CI runner (parallelism + * is cores minus one), which stalls the client's own async continuations + * and times the login out before the test does anything. */ private static final class MockNode { private final ServerSocket server; @@ -212,17 +217,23 @@ private MockNode(ServerSocket server) { static MockNode serve(ServerSocket server, RequestHandler handler) { MockNode node = new MockNode(server); - CompletableFuture.runAsync(() -> { - while (!node.killed) { - try { - Socket socket = server.accept(); - node.accepted.add(socket); - CompletableFuture.runAsync(() -> node.exchange(socket, handler)); - } catch (IOException accepted) { - return; - } - } - }); + Thread acceptor = new Thread( + () -> { + while (!node.killed) { + try { + Socket socket = server.accept(); + node.accepted.add(socket); + Thread exchange = new Thread(() -> node.exchange(socket, handler)); + exchange.setDaemon(true); + exchange.start(); + } catch (IOException accepted) { + return; + } + } + }, + "mock-vsr-acceptor-" + server.getLocalPort()); + acceptor.setDaemon(true); + acceptor.start(); return node; } From 23ceb809ccd3693cb311a47b2ca7cc652a7b6924 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Mon, 24 Aug 2026 10:50:03 +0200 Subject: [PATCH 05/16] fix CI AGAIN --- core/cli/src/args/context.rs | 2 +- core/common/src/types/options/mod.rs | 12 +-- core/common/src/wire_conversions.rs | 4 +- core/configs/src/server_config/cluster.rs | 2 +- core/configs/src/server_config/server.rs | 4 +- core/configs_derive/src/lib.rs | 4 +- .../http_provider/response_extractor.rs | 2 +- core/connectors/sinks/http_sink/src/lib.rs | 2 +- core/consensus/src/client_table.rs | 6 +- core/consensus/src/dvc_merge.rs | 2 +- core/consensus/src/impls.rs | 4 +- core/consensus/src/vsr_state.rs | 2 +- .../integration/src/harness/config/resolve.rs | 2 +- core/journal/src/prepare_journal.rs | 2 +- core/message_bus/src/lib.rs | 2 +- .../src/lifecycle/connection_registry.rs | 2 +- core/message_bus/src/replica/handshake.rs | 2 +- core/message_bus/src/transports/quic.rs | 2 +- core/metadata/src/impls/metadata.rs | 4 +- core/metadata/src/stm/authz.rs | 6 +- core/metadata/src/stm/mod.rs | 8 +- core/metadata/src/stm/stream.rs | 2 +- core/partitions/src/iggy_index_reader.rs | 6 +- core/partitions/src/iggy_partition.rs | 6 +- core/partitions/src/state_transfer.rs | 4 +- core/sdk/src/tcp/tcp_client.rs | 9 +++ core/server/src/bootstrap.rs | 2 +- core/server/src/partition_reconciler.rs | 2 +- core/server/src/responses.rs | 2 +- core/server/src/users.rs | 2 +- core/shard/src/lib.rs | 10 +-- core/simulator/src/bus.rs | 2 +- core/simulator/src/deps.rs | 6 +- core/simulator/src/network.rs | 2 +- core/simulator/src/workload/mod.rs | 2 +- core/simulator/src/workload/ops/mod.rs | 2 +- core/simulator/src/workload/shadow.rs | 2 +- .../Implementations/TcpMessageStream.cs | 10 +++ .../VsrTests/EndpointFailoverTests.cs | 78 +++++++++++++++++++ foreign/go/client/tcp/tcp_core.go | 9 +++ foreign/go/client/tcp/tcp_failover_test.go | 35 +++++++++ 41 files changed, 205 insertions(+), 64 deletions(-) diff --git a/core/cli/src/args/context.rs b/core/cli/src/args/context.rs index b87404a1b0..99ff0a6ff2 100644 --- a/core/cli/src/args/context.rs +++ b/core/cli/src/args/context.rs @@ -40,7 +40,7 @@ pub(crate) enum ContextAction { /// Create a new context /// /// Creates a new named context in the contexts configuration file. - /// After creating a context, use 'iggy context use ' to activate it. + /// After creating a context, use `iggy context use ` to activate it. /// /// Examples /// iggy context create production --transport tcp --tcp-server-address 10.0.0.1:8090 diff --git a/core/common/src/types/options/mod.rs b/core/common/src/types/options/mod.rs index a7ff727f2e..f9c440b950 100644 --- a/core/common/src/types/options/mod.rs +++ b/core/common/src/types/options/mod.rs @@ -213,12 +213,12 @@ pub mod topic_option_keys { /// Must be non-zero. pub const MESSAGES_REQUIRED_TO_SAVE: &str = "messages_required_to_save"; /// Flush the journal once it holds this many bytes: `Uint64` or a - /// byte-size string. Paired with [`Self::MESSAGES_REQUIRED_TO_SAVE`]; + /// byte-size string. Paired with `Self::MESSAGES_REQUIRED_TO_SAVE`; /// whichever threshold trips first flushes. pub const SIZE_OF_MESSAGES_REQUIRED_TO_SAVE: &str = "size_of_messages_required_to_save"; /// Reserve the segment's bytes up front on a filesystem that supports it: /// `Bool`, or the strings `true` / `false`. Pairs with - /// [`Self::SEGMENT_SIZE`] -- preallocation reserves exactly that much, so + /// `Self::SEGMENT_SIZE` -- preallocation reserves exactly that much, so /// the two only make sense decided together. pub const PREALLOCATE_SEGMENTS: &str = "preallocate_segments"; } @@ -557,7 +557,7 @@ impl StreamUpdateOptions { /// /// # Errors /// - /// See [`raw_options_to_wire`]. + /// See `raw_options_to_wire`. pub fn to_wire(&self) -> Result { raw_options_to_wire(&self.raw) } @@ -579,7 +579,7 @@ impl UserUpdateOptions { /// /// # Errors /// - /// See [`raw_options_to_wire`]. + /// See `raw_options_to_wire`. pub fn to_wire(&self) -> Result { raw_options_to_wire(&self.raw) } @@ -658,7 +658,7 @@ impl TopicUpdateOptions { /// /// # Errors /// - /// See [`raw_options_map`]. + /// See `raw_options_map`. pub fn to_wire(&self) -> Result { let mut options = raw_options_map(&self.raw)?; if let Some(compression_algorithm) = self.compression_algorithm { @@ -875,7 +875,7 @@ impl TopicCreateOptions { /// /// # Errors /// - /// See [`raw_options_map`]. + /// See `raw_options_map`. pub fn to_option_map(&self) -> Result { let mut options = raw_options_map(&self.raw)?; if let Some(compression_algorithm) = self.compression_algorithm { diff --git a/core/common/src/wire_conversions.rs b/core/common/src/wire_conversions.rs index d87e392a02..e96b725252 100644 --- a/core/common/src/wire_conversions.rs +++ b/core/common/src/wire_conversions.rs @@ -732,7 +732,7 @@ fn topic_permissions_to_wire(topic_id: usize, tp: &TopicPermissions) -> WireTopi // -- User Headers conversions -- -/// Encode domain user headers into a [`WireUserHeaders`] wrapper. +/// Encode domain user headers into a `WireUserHeaders` wrapper. pub fn user_headers_to_wire( headers: &BTreeMap, ) -> iggy_binary_protocol::WireUserHeaders { @@ -762,7 +762,7 @@ pub fn user_headers_to_wire( WireUserHeaders::from_validated(buf.freeze()) } -/// Decode a [`WireUserHeaders`] wrapper into domain user headers. +/// Decode a `WireUserHeaders` wrapper into domain user headers. /// /// Wire-level validation accepts unknown kind codes for forward compatibility /// (VSR rolling upgrades). Domain-level `from_code()` rejects them - the wire diff --git a/core/configs/src/server_config/cluster.rs b/core/configs/src/server_config/cluster.rs index d71f86ae6b..033ea6f77d 100644 --- a/core/configs/src/server_config/cluster.rs +++ b/core/configs/src/server_config/cluster.rs @@ -616,7 +616,7 @@ pub struct TransportPorts { /// /// Hostnames follow RFC 1123: ASCII letters, digits and hyphens in labels of /// 1-63 characters that do not start or end with a hyphen, at most -/// [`MAX_HOSTNAME_LEN`] characters total, no port and no trailing dot. Names +/// `MAX_HOSTNAME_LEN` characters total, no port and no trailing dot. Names /// consisting solely of digits and dots are rejected as malformed IPv4 rather /// than accepted as hostnames, so `10.0.0.256` fails loudly instead of being /// handed to DNS. Hostnames normalize to lowercase and IPs to their canonical diff --git a/core/configs/src/server_config/server.rs b/core/configs/src/server_config/server.rs index bd7d168b3a..e4c6910d01 100644 --- a/core/configs/src/server_config/server.rs +++ b/core/configs/src/server_config/server.rs @@ -102,9 +102,9 @@ pub type ServerSystemConfig = SystemConfig; /// Top-level on-disk config schema for the `iggy-server` binary. /// -/// Composes the shared section types from [`crate::common`] with the +/// Composes the shared section types from `crate::common` with the /// transport, cluster, metadata and [`MessageBusConfig`] sections owned -/// by [`super`]. +/// by `super`. #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] #[config_env(prefix = "IGGY_", name = "iggy-server-config")] pub struct ServerConfig { diff --git a/core/configs_derive/src/lib.rs b/core/configs_derive/src/lib.rs index 1f5c4a48af..457b57f43e 100644 --- a/core/configs_derive/src/lib.rs +++ b/core/configs_derive/src/lib.rs @@ -33,8 +33,8 @@ //! //! - **Leaf types** (primitives, String, known value types): generate direct mappings //! - **Nested types** (structs with `ConfigEnv` derive): include their mappings recursively -//! - **Vec**: expands to indexed mappings (e.g., `FIELD_0_NAME`, `FIELD_1_NAME`, ...) -//! - **Arc**, **Box**, **Option**: transparently unwrapped +//! - **`Vec`**: expands to indexed mappings (e.g., `FIELD_0_NAME`, `FIELD_1_NAME`, ...) +//! - **`Arc`**, **`Box`**, **`Option`**: transparently unwrapped mod config_env; diff --git a/core/connectors/runtime/src/configs/connectors/http_provider/response_extractor.rs b/core/connectors/runtime/src/configs/connectors/http_provider/response_extractor.rs index a48e54a037..75731879d3 100644 --- a/core/connectors/runtime/src/configs/connectors/http_provider/response_extractor.rs +++ b/core/connectors/runtime/src/configs/connectors/http_provider/response_extractor.rs @@ -77,7 +77,7 @@ impl ResponseExtractor { /// Navigates through a JSON structure using dot-notation path /// - /// Example: "data.config" navigates to json["data"]["config"] + /// Example: "data.config" navigates to json["data"]`"config"` fn navigate_path<'a>(&self, json: &'a Value, path: &str) -> Option<&'a Value> { let parts: Vec<&str> = path.split('.').collect(); let mut current = json; diff --git a/core/connectors/sinks/http_sink/src/lib.rs b/core/connectors/sinks/http_sink/src/lib.rs index da896b0db4..0d88e427ac 100644 --- a/core/connectors/sinks/http_sink/src/lib.rs +++ b/core/connectors/sinks/http_sink/src/lib.rs @@ -150,7 +150,7 @@ struct EncodedHeader { iggy_header_encoding: &'static str, } -/// Configuration for the HTTP sink connector, deserialized from [plugin_config] in config.toml. +/// Configuration for the HTTP sink connector, deserialized from `plugin_config` in config.toml. #[derive(Debug, Serialize, Deserialize)] pub struct HttpSinkConfig { /// Target URL for HTTP requests (required). diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 506a61aa99..b23b805d0a 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs @@ -194,7 +194,7 @@ pub struct ClientEntrySnapshot { pub watermark: u64, pub watermark_checksum: u128, /// Wire bytes of the entry's latest committed reply, round-tripped through - /// [`CachedReply::from_message`]. Never empty: registration seeds the ring. + /// `CachedReply::from_message`. Never empty: registration seeds the ring. /// /// Serialized as a msgpack `bin` blob, not the integer array a plain `Vec` /// produces, which spends 2 bytes on every byte >= 0x80 and runs a checkpoint's @@ -697,11 +697,11 @@ impl ClientTable { /// previous register reply is dropped), and preserves the watermark - /// session resume keeps dedup history. /// - /// Full table evicts the oldest commit, see [`Self::evict_oldest`]. + /// Full table evicts the oldest commit, see `Self::evict_oldest`. /// /// A key this table evicted for capacity re-registers as a fresh entry that /// RESTORES the evicted watermark (and the watermark reply when it survived) - /// from [`EvictedFence`], for the same `user_id` only. A committed `Logout` + /// from `EvictedFence`, for the same `user_id` only. A committed `Logout` /// forgets that fence, so a register after one starts clean. /// /// # Panics diff --git a/core/consensus/src/dvc_merge.rs b/core/consensus/src/dvc_merge.rs index 93387dae78..516ef8ff83 100644 --- a/core/consensus/src/dvc_merge.rs +++ b/core/consensus/src/dvc_merge.rs @@ -84,7 +84,7 @@ pub struct MergedLog { /// The new primary installs these over its own log. pub headers: Vec, /// Headers non-canonical senders report committed and the canonical chain - /// corroborates. See [`committed_elsewhere`]. + /// corroborates. See `committed_elsewhere`. pub committed_elsewhere: Vec, } diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs index 3a83c0e2ce..2e4741dab2 100644 --- a/core/consensus/src/impls.rs +++ b/core/consensus/src/impls.rs @@ -1713,7 +1713,7 @@ impl> VsrConsensus { /// the head has to either stop it (nothing left to retransmit) or restart it /// (the next entry becomes the oldest, and it must be timed from now rather /// than inheriting the drained entry's elapsed ticks). Arming happens in - /// [`Self::push_prepare_entry`]; between the two the invariant is "ticking + /// `Self::push_prepare_entry`; between the two the invariant is "ticking /// iff the pipeline is non-empty". pub fn pop_committed_prepare(&self) -> Option { let popped = self.pipeline.borrow_mut().pop(); @@ -1804,7 +1804,7 @@ impl> VsrConsensus { } } - /// Undo the [`Self::push_prepare_entry`] pre-advance for a prepare whose + /// Undo the `Self::push_prepare_entry` pre-advance for a prepare whose /// journal append failed, so the op it claimed is handed back. /// /// The pre-advance runs the sequencer ahead of the WAL on purpose, so that a diff --git a/core/consensus/src/vsr_state.rs b/core/consensus/src/vsr_state.rs index b9e270168c..9040906adc 100644 --- a/core/consensus/src/vsr_state.rs +++ b/core/consensus/src/vsr_state.rs @@ -177,7 +177,7 @@ fn field(bytes: &[u8; ENCODED_LEN], start: usize) -> [u8; N] { /// Failure decoding a [`VsrState`] from bytes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VsrStateError { - /// The byte slice was not exactly [`ENCODED_LEN`] long. + /// The byte slice was not exactly `ENCODED_LEN` long. WrongLength { expected: usize, actual: usize }, /// The record violates `log_view <= view`, so it cannot be a state any replica /// reached. diff --git a/core/integration/src/harness/config/resolve.rs b/core/integration/src/harness/config/resolve.rs index 4f35b85358..fbb5360843 100644 --- a/core/integration/src/harness/config/resolve.rs +++ b/core/integration/src/harness/config/resolve.rs @@ -119,7 +119,7 @@ fn find_mapping(path: &str) -> Option<&'static EnvVarMapping> { /// /// Names outside the `IGGY_` prefix are left alone: those address the process /// environment (`RUST_LOG`, test scaffolding), not the config schema. -/// [`NON_CONFIG_ENV_VARS`] carries the `IGGY_`-prefixed names the server reads +/// `NON_CONFIG_ENV_VARS` carries the `IGGY_`-prefixed names the server reads /// outside the config struct. /// /// # Errors diff --git a/core/journal/src/prepare_journal.rs b/core/journal/src/prepare_journal.rs index a5f1fde47b..4c0f3dc54a 100644 --- a/core/journal/src/prepare_journal.rs +++ b/core/journal/src/prepare_journal.rs @@ -629,7 +629,7 @@ impl PrepareJournal { } /// How many entries the opening scan replayed unverified - /// ([`CHECKSUM_BODY_UNSEALED`]). `0` once every producer seals; the boot path + /// (`CHECKSUM_BODY_UNSEALED`). `0` once every producer seals; the boot path /// warns while it is not, so the fail-open stretch is visible to an operator. pub const fn unsealed_entry_count(&self) -> u64 { self.unsealed_entries diff --git a/core/message_bus/src/lib.rs b/core/message_bus/src/lib.rs index 414312ad0e..f41f50e30e 100644 --- a/core/message_bus/src/lib.rs +++ b/core/message_bus/src/lib.rs @@ -180,7 +180,7 @@ impl ReplicaOwnerTable { /// * `compare_exchange(OWNER_NONE -> shard_id)` wins. Common case. /// * The CAS fails because the slot already stores `shard_id`. A /// same-shard reclaim during the post-loop clear window - /// ([`IggyMessageBus::notify_connection_lost`]) is benign: the + /// (`IggyMessageBus::notify_connection_lost`) is benign: the /// slot already names us, and the stale post-loop will stand /// down once it observes a live registry entry. /// diff --git a/core/message_bus/src/lifecycle/connection_registry.rs b/core/message_bus/src/lifecycle/connection_registry.rs index cda45a8adf..0fa89a13b6 100644 --- a/core/message_bus/src/lifecycle/connection_registry.rs +++ b/core/message_bus/src/lifecycle/connection_registry.rs @@ -155,7 +155,7 @@ impl ReplyTarget { } } -/// Outcome of routing a client reply through an [`Entry`]'s reply target. +/// Outcome of routing a client reply through an `Entry`'s reply target. /// /// `Delivered` is the socket fast path: `try_send` was attempted and its /// result is carried through unchanged. `InProcess` hands the message back diff --git a/core/message_bus/src/replica/handshake.rs b/core/message_bus/src/replica/handshake.rs index 13d81ce541..c0f9fcbb4e 100644 --- a/core/message_bus/src/replica/handshake.rs +++ b/core/message_bus/src/replica/handshake.rs @@ -127,7 +127,7 @@ pub struct ReplicaTlsCtx { /// therefore binds `dialer_id = peer_id`, `acceptor_id = self_id`. /// /// On a rejection an authenticated, still-waiting dialer is answered -/// with a nonzero-status [`build_challenge_message`] (see [`reject`]) so +/// with a nonzero-status `build_challenge_message` (see `reject`) so /// it learns the cause from its own logs rather than seeing a bare /// connection close. /// diff --git a/core/message_bus/src/transports/quic.rs b/core/message_bus/src/transports/quic.rs index 26550d5471..8968a18e4d 100644 --- a/core/message_bus/src/transports/quic.rs +++ b/core/message_bus/src/transports/quic.rs @@ -49,7 +49,7 @@ //! connection share ONE request id (only metadata ops advance the dedup //! counter), so a stranded reply would be consumed by the NEXT partition //! op's bidi and shift the connection's data-plane stream off by one, -//! permanently. Instead, [`REPLY_WAIT_BACKSTOP`] (longer than the SDK's +//! permanently. Instead, `REPLY_WAIT_BACKSTOP` (longer than the SDK's //! whole-request deadline, so the client always gives up first and //! reconnects) closes the CONNECTION, which drops the mailbox and every //! pending reply with it -- nothing can strand or cross request ids. A diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 9856dd02ad..0874b18607 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -316,7 +316,7 @@ impl Snapshot for IggySnapshot { /// /// Owns the data directory path and the snapshot creation function. The /// three-phase checkpoint (persist snapshot, record the pairing durably, drain the -/// WAL) is orchestrated one layer up in [`IggyMetadata::checkpoint_if_needed`], so +/// WAL) is orchestrated one layer up in `IggyMetadata::checkpoint_if_needed`, so /// the superblock write can sit between persist and drain; this type owns only the /// snapshot I/O and the last-checkpoint bookkeeping. pub struct SnapshotCoordinator { @@ -3455,7 +3455,7 @@ where /// /// `advance_commit_min(op)` and matching `client_table` mutation /// (`commit_register` / `commit_reply`) run back-to-back, no `.await` - /// between. [`crate::metadata_helpers::is_caught_up_primary`] reads + /// between. `crate::metadata_helpers::is_caught_up_primary` reads /// `commit_min == commit_max` as proof the table is caught up; an await /// here lets another task observe transient equality with stale table, /// dispatch a fresh Register on an already-registered client, and bump diff --git a/core/metadata/src/stm/authz.rs b/core/metadata/src/stm/authz.rs index de3341451b..667b87f026 100644 --- a/core/metadata/src/stm/authz.rs +++ b/core/metadata/src/stm/authz.rs @@ -22,7 +22,7 @@ //! state, so the op commits as a deterministic no-op whose error rides the //! cached reply and replays on retry, exactly like a business rejection. //! -//! Replay-determinism invariant: [`authorize`] is a pure function of the +//! Replay-determinism invariant: `authorize` is a pure function of the //! prepare header (`operation` + the acting `user_id`, stamped into the //! replicated header at submit time) and the committed permission/stream //! state as of the op immediately before this one. That state is applied in @@ -83,7 +83,7 @@ where mux.update(prepare) } -/// Generic entry point to [`gated_apply`] for the WAL-replay path. +/// Generic entry point to `gated_apply` for the WAL-replay path. /// /// The replay path is generic over the state machine and cannot name the /// concrete accessors, so it dispatches through this trait. Implemented only @@ -92,7 +92,7 @@ where pub trait GatedApply: StateMachine, Output = ApplyReply, Error = IggyError> { - /// See [`gated_apply`]. + /// See `gated_apply`. /// /// # Errors /// Propagates the underlying [`StateMachine::update`] error. diff --git a/core/metadata/src/stm/mod.rs b/core/metadata/src/stm/mod.rs index 8dd3ae9c42..f5155aad9a 100644 --- a/core/metadata/src/stm/mod.rs +++ b/core/metadata/src/stm/mod.rs @@ -88,7 +88,7 @@ pub trait Command { /// Per-command handler for a given state type. /// /// Each command implements it for the state it mutates, returning an -/// [`ApplyReply`]: a `code` (0 = success) plus the typed reply `body` to thread +/// `ApplyReply`: a `code` (0 = success) plus the typed reply `body` to thread /// into the Reply message. /// /// Apply MUST be deterministic across replicas: both left/right buffers recompute @@ -335,9 +335,9 @@ macro_rules! define_state { } impl $state { - /// Mint a `Send + Sync` [`ReadHandleFactory`] for this state. + /// Mint a `Send + Sync` `ReadHandleFactory` for this state. /// Allows the read side to be carried across shard threads - /// without sharing the underlying `!Sync` [`ReadHandle`]. + /// without sharing the underlying `!Sync` `ReadHandle`. #[must_use] pub fn factory(&self) -> $crate::stm::LeftRightFactory<[<$state Inner>]> { self.inner.factory() @@ -345,7 +345,7 @@ macro_rules! define_state { /// Construct a reader-only state wrapper from a factory minted /// by the writer-side shard. The thread that calls this owns - /// the resulting [`ReadHandle`]; calling `apply` on the + /// the resulting `ReadHandle`; calling `apply` on the /// returned wrapper panics because `write` is `None`. #[must_use] pub fn from_factory( diff --git a/core/metadata/src/stm/stream.rs b/core/metadata/src/stm/stream.rs index eefe9e4a76..c3833b3ce2 100644 --- a/core/metadata/src/stm/stream.rs +++ b/core/metadata/src/stm/stream.rs @@ -1455,7 +1455,7 @@ impl Streams { /// `s` / topic slab `t` means creating every slab below it too; those /// fillers carry no partitions and are inert. Each level is skipped when /// already present, so seeding sibling partitions of one topic adds only - /// the missing partition. Mirrors [`Users::ensure_root_user`]: a seed + /// the missing partition. Mirrors `Users::ensure_root_user`: a seed /// helper that bypasses consensus, never a production runtime path. /// /// # Panics diff --git a/core/partitions/src/iggy_index_reader.rs b/core/partitions/src/iggy_index_reader.rs index a5dacf142c..12246cb30c 100644 --- a/core/partitions/src/iggy_index_reader.rs +++ b/core/partitions/src/iggy_index_reader.rs @@ -24,7 +24,7 @@ use tracing::trace; /// Reader for the sparse index file written by [`crate::IggyIndexWriter`]. /// -/// The on-disk stride is [`IGGY_INDEX_SIZE`] (24 bytes: `offset` u64, +/// The on-disk stride is `IGGY_INDEX_SIZE` (24 bytes: `offset` u64, /// `timestamp` u64, `position` u64, little-endian) — distinct from the legacy /// 16-byte dense per-message index that `server_common::IndexReader` parses. /// Recovery reaches for this reader so the reader matches the writer. @@ -119,7 +119,7 @@ impl IggyIndexReader { )) } - /// Load every whole entry into an [`IggyIndexCache`] for offset / timestamp + /// Load every whole entry into an `IggyIndexCache` for offset / timestamp /// lower-bound lookups in one read. Density is one sparse entry per flushed /// chunk, so an aggressive flush cadence (`messages_required_to_save = 1`) /// makes the file track every message: callers that cannot afford an @@ -158,7 +158,7 @@ impl IggyIndexReader { /// Last entry with `offset` at or below the target, binary-searching the /// file with single-entry preads instead of materializing it, for indexes /// too large to load whole. `None` when every entry sits above the target; - /// semantics match [`IggyIndexCache::offset_lower_bound`]. `entry_count` + /// semantics match `IggyIndexCache::offset_lower_bound`. `entry_count` /// comes from [`Self::entry_count`]; entries are written in ascending /// offset and timestamp order. /// diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 23a9f1b931..007d68d987 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -324,7 +324,7 @@ pub enum PurgeError { /// so the reconciler's `committed > applied` gate re-issues this purge on /// its next pass. Retry, do not fence. /// - /// Sets [`Self::purge_deferred`], which withholds `PrepareOk` for this + /// Sets `Self::purge_deferred`, which withholds `PrepareOk` for this /// group until the purge lands, so the replica goes quorum-invisible THERE /// while every other partition on the node keeps serving. Without that /// fence the counter would still name the pre-purge offset space and every @@ -347,7 +347,7 @@ pub enum PurgeError { /// reconciler re-issues the purge. Retry, do not fence: the partition is /// serviceable and re-purging an already-empty chain is cheap. /// - /// Sets [`Self::purge_deferred`] for the same reason as + /// Sets `Self::purge_deferred` for the same reason as /// [`Self::FrontierNotRecorded`]: an op acked between this failure and the /// retry would be wiped by that retry while every peer that recorded the /// generation keeps it. @@ -2095,7 +2095,7 @@ where /// against view-change-reset flipping status across `on_replicate` await. /// /// View-change safety: `reset_view_change_state` calls - /// [`crate::Pipeline::clear_request_queue`]; resumed loop breaks via + /// `crate::Pipeline::clear_request_queue`; resumed loop breaks via /// `else { break }`. /// /// # Panics diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs index 1653b1913f..4e331eb5c1 100644 --- a/core/partitions/src/state_transfer.rs +++ b/core/partitions/src/state_transfer.rs @@ -1822,7 +1822,7 @@ where /// entry are swept. /// /// A scan against a segment set this partition already scanned short- - /// circuits through [`ReuseScanMemo`]: rotating to another peer would + /// circuits through `ReuseScanMemo`: rotating to another peer would /// otherwise re-read and re-walk every staged file, up to 2 GiB each, /// sequentially, on the pump. pub async fn reuse_staged_segments( @@ -2885,7 +2885,7 @@ async fn hash_segment_range( } /// Read the first `entry.len` bytes of a served segment file and re-verify them -/// against the manifest entry, chunked through [`hash_segment_range`] with one +/// against the manifest entry, chunked through `hash_segment_range` with one /// reactor yield per chunk. /// /// The serving side runs this on the pump to answer a single chunk request, so diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index f7d240c7d7..04957b0f5e 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -178,6 +178,15 @@ impl BinaryTransport for TcpClient { return Err(error); } + // A stale-client eviction is the server ending this session + // authoritatively, like a logout: the remembered sign-in ends with it, + // so only a configured auto-login may bring the session back. + // Remembered credentials exist for transport loss, where the session + // died with the socket rather than by anyone's decision. + if matches!(error, IggyError::StaleClient) { + self.forget_session_credentials().await; + } + if !self.config.reconnection.enabled { return Err(IggyError::Disconnected); } diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs index 3d983f9e63..894a8a90bf 100644 --- a/core/server/src/bootstrap.rs +++ b/core/server/src/bootstrap.rs @@ -157,7 +157,7 @@ impl ShellBus for B {} /// The five dispatch handlers a shard is built with, plus the /// [`SessionManager`] the request-plane pair shares. /// -/// Both production ([`build_shard_for_thread`]) and the simulator's shell +/// Both production (`build_shard_for_thread`) and the simulator's shell /// mode construct these through [`wire_shell_handlers`], so the request /// plane is wired one way. The simulator's shell-off fast path uses /// [`ShellHandlers::noop`] instead. diff --git a/core/server/src/partition_reconciler.rs b/core/server/src/partition_reconciler.rs index 0ab7d1be87..c645405a49 100644 --- a/core/server/src/partition_reconciler.rs +++ b/core/server/src/partition_reconciler.rs @@ -61,7 +61,7 @@ //! frame's provenance, which is why the park stamp above is separate. //! - Nothing is left unanswered: a tombstoned namespace, an overflowing park //! buffer, and a namespace this shard has given up materialising -//! ([`reconcile_parked_frames`]) all reply with a retriable status, so a +//! (`reconcile_parked_frames`) all reply with a retriable status, so a //! lockstep transport never waits out its read timeout on silence. //! //! `shards_table` is therefore a **cache of a deterministic hash**, never a diff --git a/core/server/src/responses.rs b/core/server/src/responses.rs index 35c1b672bb..f4e6bc4e56 100644 --- a/core/server/src/responses.rs +++ b/core/server/src/responses.rs @@ -20,7 +20,7 @@ //! Assemble `get_me` / `get_clients` / `get_stream(s)` / `get_topic(s)` / //! `get_user(s)` / `get_personal_access_tokens` / stats / cluster-metadata //! responses from per-shard session state and the metadata state machine, plus the -//! [`NonReplicatedResponse`] dispatch shim and the partition-namespace +//! `NonReplicatedResponse` dispatch shim and the partition-namespace //! resolvers. use crate::bootstrap::{ShellBus, ShellShard}; diff --git a/core/server/src/users.rs b/core/server/src/users.rs index 63368cd5b4..129bbfca8f 100644 --- a/core/server/src/users.rs +++ b/core/server/src/users.rs @@ -33,7 +33,7 @@ //! replicated prepare/WAL. A mismatch is committed as a rejecting no-op //! (signalled by an empty new password) rather than denied pre-consensus, so //! the caller's request sequence stays contiguous; see -//! [`verify_and_rewrite_change_password`]. +//! `verify_and_rewrite_change_password`. use crate::bootstrap::{ShellBus, ShellShard}; use crate::wire::{request_body, rewrite_request_body}; diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 762c932f20..37de132e4c 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -1719,7 +1719,7 @@ where /// channel path -- and collects their replies. /// /// Bounded: a shard that doesn't reply within - /// [`LIST_CLIENTS_GATHER_TIMEOUT`] is skipped and the partial result is + /// `LIST_CLIENTS_GATHER_TIMEOUT` is skipped and the partial result is /// logged, so one wedged shard cannot hang the read. Callers should /// treat the result as best-effort-complete. #[allow(clippy::future_not_send)] @@ -1786,7 +1786,7 @@ where /// Routes a [`LifecycleFrame::PartitionRead`] through the shards table /// (self-sends included, so a locally-owned partition takes the same /// path). `None` = unroutable namespace, full owning-shard inbox, - /// dropped reply sender, or [`PARTITION_READ_TIMEOUT`] expiry; the + /// dropped reply sender, or `PARTITION_READ_TIMEOUT` expiry; the /// caller maps it to a client-visible error. #[allow(clippy::future_not_send)] pub async fn partition_read( @@ -2743,7 +2743,7 @@ where /// lockstep, so silence wedges the connection until the SDK read-timeout. /// /// The one retirement path a prepare still travels. It is retained - /// everywhere else (see [`ParkedFrame::passes`]); here the namespace itself + /// everywhere else (see `ParkedFrame::passes`); here the namespace itself /// is unreachable, so holding it buys nothing. pub fn discard_parked_partition_frames(&self, namespace: IggyNamespace) { // Bound the borrow to this statement: the guard in an `if let` @@ -3007,10 +3007,10 @@ where } /// Age every frame under `namespace` by one pass, answering CLIENT REQUESTS - /// past [`MAX_PARKED_PASSES`]. Returns the number answered. + /// past `MAX_PARKED_PASSES`. Returns the number answered. /// /// Prepares age but never expire. Expiry destroys a committed op with - /// nothing to recover it (see [`ParkedFrame::passes`]), and passes are + /// nothing to recover it (see `ParkedFrame::passes`), and passes are /// commit-driven: a non-empty buffer defeats the reconciler fast-skip, so a /// create burst elapses four in milliseconds, across every parked namespace /// rather than the one it concerns. Byte budgets bound them instead. Only diff --git a/core/simulator/src/bus.rs b/core/simulator/src/bus.rs index 3f542411d8..3c2ff22661 100644 --- a/core/simulator/src/bus.rs +++ b/core/simulator/src/bus.rs @@ -65,7 +65,7 @@ pub struct Envelope { /// /// Consensus code calls `send_to_replica()` / `send_to_client()` which stage /// messages here. The simulator's tick loop drains each replica's outbox and -/// feeds the messages into the [`Network`] for simulated delivery. +/// feeds the messages into the `Network` for simulated delivery. pub struct SimOutbox { /// Replica id that owns this outbox. Populated as `from_replica` on every envelope. self_id: u8, diff --git a/core/simulator/src/deps.rs b/core/simulator/src/deps.rs index 9b7f89b799..7c73f1a41a 100644 --- a/core/simulator/src/deps.rs +++ b/core/simulator/src/deps.rs @@ -347,7 +347,7 @@ impl SimJournal { /// Read the entry at `op` synchronously, for off-executor WAL replay at restart. /// Mirrors [`Journal::entry`] without the never-suspending `MemStorage` await, so - /// it needs no [`JournalAccessGuard`]: a synchronous read offers no suspension + /// it needs no `JournalAccessGuard`: a synchronous read offers no suspension /// point for another task to interleave on. /// /// # Panics @@ -401,13 +401,13 @@ impl SimSuperblock { } /// Make every subsequent write fail, a persistent write fault, so the durability - /// gate withholds view-scoped sends. See [`Self::fail_writes`]. + /// gate withholds view-scoped sends. See `Self::fail_writes`. pub fn set_fail_writes(&self) { self.fail_writes.set(true); } /// Make every subsequent write suspend once before completing, so tasks that - /// are ready at persist time interleave with it. See [`Self::yield_writes`]. + /// are ready at persist time interleave with it. See `Self::yield_writes`. pub fn set_yield_writes(&self) { self.yield_writes.set(true); } diff --git a/core/simulator/src/network.rs b/core/simulator/src/network.rs index 6a4aa795ef..cbf6e1bbb5 100644 --- a/core/simulator/src/network.rs +++ b/core/simulator/src/network.rs @@ -18,7 +18,7 @@ //! Network abstraction layer for the cluster simulator. //! //! **Note:** Currently a thin passthrough over `PacketSimulator`. Once the -//! Cluster and [`MessageBus`] layers are built, this will own +//! Cluster and `MessageBus` layers are built, this will own //! process-to-bus routing, and node enable/disable logic. use crate::packet::{ diff --git a/core/simulator/src/workload/mod.rs b/core/simulator/src/workload/mod.rs index 2e641066cb..03c7388b9f 100644 --- a/core/simulator/src/workload/mod.rs +++ b/core/simulator/src/workload/mod.rs @@ -362,7 +362,7 @@ const FAULT_SEED_SALT: u64 = 0x5A1A_F0E5_FACE_0001; /// The invariants are asserted after every tick, so a consensus or /// workload regression panics at the tick it occurs (the seed in the message /// replays it). When `crash_per_tick_ratio > 0` the driver also injects -/// crash-only faults via [`maybe_inject_crash`]. +/// crash-only faults via `maybe_inject_crash`. pub fn run( sim: &mut Simulator, workload: &mut Workload, diff --git a/core/simulator/src/workload/ops/mod.rs b/core/simulator/src/workload/ops/mod.rs index 604e110bb0..d1aa4b483f 100644 --- a/core/simulator/src/workload/ops/mod.rs +++ b/core/simulator/src/workload/ops/mod.rs @@ -26,7 +26,7 @@ //! - `classify_reply`: decode reply into a declared outcome //! - `predicted_effect`: predicted shadow mutation on commit //! -//! Dispatch via the [`op_dispatch!`] macro; missing variants are a compile +//! Dispatch via the `op_dispatch!` macro; missing variants are a compile //! error. pub mod change_password; diff --git a/core/simulator/src/workload/shadow.rs b/core/simulator/src/workload/shadow.rs index 69f64b2e47..a0df3e2a2c 100644 --- a/core/simulator/src/workload/shadow.rs +++ b/core/simulator/src/workload/shadow.rs @@ -178,7 +178,7 @@ impl Shadow { self.fresh_name(prefix) } - /// Apply a predicted effect. Returns [`SimCommand`]s for the driver + /// Apply a predicted effect. Returns `SimCommand`s for the driver /// plus an `applied` flag gating `auditor.note_committed`. /// /// `applied = false` when a precondition no longer holds (parent diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index c97493c45f..0cda6890e2 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -1443,6 +1443,16 @@ private async Task> SendWithResponseAsync(ReadOnlyMemory= 1, "the request landed on the survivor"); } + ///

+ /// Mirrors the integration contract (HeartbeatTests + /// EvictedClient_WithoutAutoLogin_Should_FailFast_And_NotReconnect): a server-side eviction ends the + /// session authoritatively, so the credentials a manual sign-in remembered must not resurrect it - the + /// evicted request surfaces the loss with no reconnect attempt. + /// + [Fact] + public async Task ServerEvictionForgetsTheRememberedSignIn() + { + using var node = new MockNode(); + var evict = false; + node.Serve(request => + { + if (request.Operation == OperationRegister) + { + return Reply(OperationRegister, RegisterBody(session: 128)); + } + + return evict + ? EvictionFrame(EvictionStaleClient) + : Reply(OperationNonReplicated, request.Code == GetClusterMetadataCode + ? ClusterMetadata(node.Port, node.Port, node.Port) + : []); + }); + + var configuration = new IggyClientConfigurator + { + BaseAddress = $"127.0.0.1:{node.Port}", + Protocol = Protocol.Tcp, + ReconnectionSettings = new ReconnectionSettings + { + Enabled = true, + MaxRetries = 2, + InitialDelay = TimeSpan.FromMilliseconds(20) + } + }; + using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); + + await client.ConnectAsync(TestContext.Current.CancellationToken); + await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); + await client.PingAsync(TestContext.Current.CancellationToken); + var connectionsBeforeEviction = node.Connections; + + evict = true; + await Assert.ThrowsAnyAsync(() => client.PingAsync(TestContext.Current.CancellationToken)); + Assert.Equal(connectionsBeforeEviction, node.Connections); + + // The dropped connection leaves the next call transport-shaped, but the eviction forgot the remembered + // sign-in, so it must fail fast instead of reconnecting into a resurrected session. + await Assert.ThrowsAnyAsync(() => client.PingAsync(TestContext.Current.CancellationToken)); + Assert.Equal(connectionsBeforeEviction, node.Connections); + } + + private static byte[] EvictionFrame(byte reason) + { + var frame = new byte[HeaderSize]; + BinaryPrimitives.WriteUInt32LittleEndian(frame.AsSpan(SizeOffset, 4), HeaderSize); + frame[CommandOffset] = CommandEviction; + frame[EvictionReasonOffset] = reason; + return frame; + } + [Fact] public async Task FailsFastWhenNothingEverSignedIn() { @@ -231,6 +296,7 @@ private sealed class MockNode : IDisposable private readonly TcpListener _listener; private readonly List _accepted = []; private volatile bool _killed; + private int _connections; private int _pings; private int _registrations; @@ -247,6 +313,17 @@ public MockNode() public int Registrations => Volatile.Read(ref _registrations); + public int Connections + { + get + { + lock (_accepted) + { + return _connections; + } + } + } + public void Serve(Func handler) { _ = Task.Run(async () => @@ -266,6 +343,7 @@ public void Serve(Func handler) lock (_accepted) { _accepted.Add(connection); + _connections++; } _ = Task.Run(() => Exchange(connection, handler)); diff --git a/foreign/go/client/tcp/tcp_core.go b/foreign/go/client/tcp/tcp_core.go index 5b9a46527c..f82520b594 100644 --- a/foreign/go/client/tcp/tcp_core.go +++ b/foreign/go/client/tcp/tcp_core.go @@ -488,6 +488,15 @@ func (c *IggyTcpClient) exchange(ctx context.Context, code uint32, frame []byte) return nil, err } + // A stale-client eviction is the server ending this session + // authoritatively, like a logout: the remembered sign-in ends with it, so + // only a configured auto-login may bring the session back. Remembered + // credentials exist for transport loss, where the session died with the + // socket rather than by anyone's decision. + if errors.Is(err, ierror.ErrStaleClient) { + c.forgetLogin() + } + // With no credentials -- neither configured nor remembered from a // sign-in -- a reconnect cannot restore the session, so anything but a // sign-in fails here instead of replaying unauthenticated. The sign-in diff --git a/foreign/go/client/tcp/tcp_failover_test.go b/foreign/go/client/tcp/tcp_failover_test.go index c5735517cc..badafeb0cf 100644 --- a/foreign/go/client/tcp/tcp_failover_test.go +++ b/foreign/go/client/tcp/tcp_failover_test.go @@ -121,6 +121,41 @@ func TestFailover_FailsFastWhenNothingEverSignedIn(t *testing.T) { "a client that never signed in cannot restore a session by reconnecting") } +// A stale-client eviction is the server ending the session authoritatively, +// like a logout: the remembered sign-in must not resurrect it, so the evicted +// request surfaces the loss instead of reconnecting into a fresh session. +func TestFailover_ServerEvictionForgetsTheRememberedSignIn(t *testing.T) { + var server *testListener + var evict atomic.Bool + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + if read.operation() == vsr.OperationRegister { + return registerReplyFrame(7, 128) + } + if evict.Load() { + return evictionFrame(vsr.EvictionStaleClient, 0, 0) + } + if read.code() == uint32(command.GetClusterMetadataCode) { + return clusterMetadataFrame(t, 0, server.address()) + } + return replyFrame(vsr.OperationNonReplicated, nil) + }) + + client := newDialingClient(t, server.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "iggy", "iggy") + require.NoError(t, err) + connectionsBefore := server.connections() + + evict.Store(true) + require.Error(t, client.Ping(ctx), "the evicted request surfaces the loss") + + _, remembered := client.signInCredentials() + assert.False(t, remembered, "the eviction forgot the remembered sign-in") + assert.Equal(t, connectionsBefore, server.connections(), + "no reconnect dial resurrected the evicted session") +} + // An explicit sign-out is caller intent: the reconnect must not sign back in // with the credentials the earlier sign-in used. func TestFailover_DoesNotResurrectASignedOutSession(t *testing.T) { From a59f2261e70929587e0484f8d71e64060dda4d52 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 25 Aug 2026 13:13:07 +0200 Subject: [PATCH 06/16] addres review comments --- .../binary_impls/personal_access_tokens.rs | 7 +- core/common/src/traits/binary_impls/users.rs | 12 +- core/common/src/traits/binary_transport.rs | 9 +- .../tests/sdk/disconnect_relogin.rs | 11 +- .../clients/binary_personal_access_tokens.rs | 10 + core/sdk/src/clients/binary_users.rs | 10 + core/sdk/src/clients/mod.rs | 33 + core/sdk/src/leader_aware.rs | 46 +- core/sdk/src/quic/quic_client.rs | 6 + core/sdk/src/tcp/tcp_client.rs | 844 +++++++++++++----- core/sdk/src/vsr.rs | 2 +- core/sdk/src/websocket/websocket_client.rs | 6 + .../Implementations/TcpMessageStream.Vsr.cs | 12 + .../Implementations/TcpMessageStream.cs | 72 +- .../VsrTests/DialCandidatesTests.cs | 15 + .../VsrTests/EndpointFailoverTests.cs | 76 +- foreign/go/client/tcp/tcp_core.go | 146 ++- foreign/go/client/tcp/tcp_failover_test.go | 198 +++- .../go/client/tcp/tcp_session_management.go | 67 +- .../client/async/tcp/AsyncIggyTcpClient.java | 178 +++- .../client/async/tcp/AsyncTcpConnection.java | 34 +- .../client/async/tcp/LeaderAwareness.java | 29 +- .../iggy/client/async/tcp/ReconnectPlan.java | 20 +- .../async/tcp/vsr/VsrResponseHandler.java | 11 +- ...syncIggyTcpClientEndpointFailoverTest.java | 84 +- ...yncIggyTcpClientTransientFailoverTest.java | 55 ++ .../AsyncTcpConnectionConcurrencyTest.java | 4 +- .../client/async/tcp/LeaderAwarenessTest.java | 73 ++ .../client/async/tcp/ReconnectPlanTest.java | 28 - .../async/tcp/vsr/VsrResponseHandlerTest.java | 10 +- .../node/src/client/client.connection.test.ts | 117 +++ foreign/node/src/client/client.connection.ts | 86 +- foreign/node/src/client/client.socket.test.ts | 10 +- foreign/node/src/client/client.socket.ts | 135 ++- 34 files changed, 1991 insertions(+), 465 deletions(-) diff --git a/core/common/src/traits/binary_impls/personal_access_tokens.rs b/core/common/src/traits/binary_impls/personal_access_tokens.rs index d46fef8ba0..8404bcc4c2 100644 --- a/core/common/src/traits/binary_impls/personal_access_tokens.rs +++ b/core/common/src/traits/binary_impls/personal_access_tokens.rs @@ -135,9 +135,10 @@ impl PersonalAccessTokenClient for B { "authenticated against iggy server" ); self.set_state(ClientState::Authenticated).await; - self.remember_session_credentials(Credentials::PersonalAccessToken(SecretString::from( - token.to_string(), - ))) + self.remember_session_credentials( + Credentials::PersonalAccessToken(SecretString::from(token.to_string())), + wire_resp.user_id, + ) .await; self.publish_event(DiagnosticEvent::SignedIn).await; Ok(IdentityInfo { diff --git a/core/common/src/traits/binary_impls/users.rs b/core/common/src/traits/binary_impls/users.rs index c199f76de9..38ee456f2d 100644 --- a/core/common/src/traits/binary_impls/users.rs +++ b/core/common/src/traits/binary_impls/users.rs @@ -174,6 +174,7 @@ impl UserClient for B { .to_bytes(), ) .await?; + self.refresh_session_password(user_id, new_password).await; Ok(()) } @@ -218,10 +219,13 @@ impl UserClient for B { "authenticated against iggy server" ); self.set_state(ClientState::Authenticated).await; - self.remember_session_credentials(Credentials::UsernamePassword( - username.to_owned(), - SecretString::from(password.to_string()), - )) + self.remember_session_credentials( + Credentials::UsernamePassword( + username.to_owned(), + SecretString::from(password.to_string()), + ), + wire_resp.user_id, + ) .await; self.publish_event(DiagnosticEvent::SignedIn).await; Ok(IdentityInfo { diff --git a/core/common/src/traits/binary_transport.rs b/core/common/src/traits/binary_transport.rs index 9c6a923b62..209ce2d580 100644 --- a/core/common/src/traits/binary_transport.rs +++ b/core/common/src/traits/binary_transport.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::{ClientState, Credentials, DiagnosticEvent, IggyDuration, IggyError}; +use crate::{ClientState, Credentials, DiagnosticEvent, Identifier, IggyDuration, IggyError}; use async_trait::async_trait; use bytes::Bytes; use std::sync::Arc; @@ -57,10 +57,15 @@ pub trait VsrSessionControl: vsr_session_sealed::Sealed + BinaryTransport { /// otherwise less reconnectable than one that configures `AutoLogin`, /// which is a surprising difference between two ways of doing the same /// thing. Transports that cannot reconnect leave this a no-op. - async fn remember_session_credentials(&self, _credentials: Credentials) {} + async fn remember_session_credentials(&self, _credentials: Credentials, _user_id: u32) {} /// Drop them: after an explicit logout there is no session to restore, /// and a reconnect must not resurrect one. async fn forget_session_credentials(&self) {} + /// A committed password change for `user`: when it is the signed-in user, + /// the remembered credentials switch to the new password, or the next + /// reconnect would sign in with the old one and fail an unrelated request + /// with `InvalidCredentials`. Other users' changes are ignored. + async fn refresh_session_password(&self, _user: &Identifier, _new_password: &str) {} /// SDK crate version sent in the login-register version prefix. /// Implemented by the transports so the value is the SDK crate's own /// `CARGO_PKG_VERSION` (`iggy` for Rust), not `iggy_common`'s. diff --git a/core/integration/tests/sdk/disconnect_relogin.rs b/core/integration/tests/sdk/disconnect_relogin.rs index d78acb4664..022bf7f94d 100644 --- a/core/integration/tests/sdk/disconnect_relogin.rs +++ b/core/integration/tests/sdk/disconnect_relogin.rs @@ -39,16 +39,17 @@ async fn given_a_logged_in_client_when_explicitly_disconnected_should_require_a_ client.disconnect().await.unwrap(); client.connect().await.unwrap(); assert!( - client.get_me().await.is_err(), + matches!(client.get_me().await, Err(IggyError::Unauthenticated)), "an explicit disconnect is caller intent, like a logout: the sign-in it ended \ - must not be silently replayed by the reconnect" + must not be silently replayed by the reconnect, so the server sees an \ + unauthenticated request" ); client.disconnect().await.unwrap(); assert!( - client.get_stats().await.is_err(), - "an operation after an explicit disconnect must fail instead of reconnecting \ - into a resurrected session" + matches!(client.get_stats().await, Err(IggyError::NotConnected)), + "an operation after an explicit disconnect must fail on the dead transport \ + instead of reconnecting into a resurrected session" ); // The remembered sign-in exists for involuntary drops; a fresh manual diff --git a/core/sdk/src/clients/binary_personal_access_tokens.rs b/core/sdk/src/clients/binary_personal_access_tokens.rs index ca19ce5e48..cfbe169d5e 100644 --- a/core/sdk/src/clients/binary_personal_access_tokens.rs +++ b/core/sdk/src/clients/binary_personal_access_tokens.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::clients::redirect_login_settled; use crate::prelude::{ClientWrapper, IggyClient}; use async_trait::async_trait; use iggy_common::locking::IggyRwLockFn; @@ -77,6 +78,15 @@ impl PersonalAccessTokenClient for IggyClient { if should_redirect { info!("Redirected to leader, reconnecting and re-authenticating"); self.connect().await?; + // The reconnect signs in with the credentials this very call just + // remembered, so on a client without a configured `AutoLogin` the + // session is already this user's: signing in again would cost a + // logout and a second login (an argon2 each, on the server) for + // nothing. With `AutoLogin::Enabled` the reconnect signed in the + // configured user, who may not be this one, so the login runs. + if redirect_login_settled(&*self.client.read().await).await { + return Ok(identity); + } self.login_with_personal_access_token(token).await } else { Ok(identity) diff --git a/core/sdk/src/clients/binary_users.rs b/core/sdk/src/clients/binary_users.rs index cab16bcf61..606e11e445 100644 --- a/core/sdk/src/clients/binary_users.rs +++ b/core/sdk/src/clients/binary_users.rs @@ -16,6 +16,7 @@ // under the License. use crate::client_wrappers::client_wrapper::ClientWrapper; +use crate::clients::redirect_login_settled; use crate::prelude::IggyClient; use async_trait::async_trait; use iggy_common::UserUpdateOptions; @@ -116,6 +117,15 @@ impl UserClient for IggyClient { if should_redirect { info!("Redirected to leader, reconnecting and re-authenticating"); self.connect().await?; + // The reconnect signs in with the credentials this very call just + // remembered, so on a client without a configured `AutoLogin` the + // session is already this user's: signing in again would cost a + // logout and a second login (an argon2 each, on the server) for + // nothing. With `AutoLogin::Enabled` the reconnect signed in the + // configured user, who may not be this one, so the login runs. + if redirect_login_settled(&*self.client.read().await).await { + return Ok(identity); + } self.login_user(username, password).await } else { Ok(identity) diff --git a/core/sdk/src/clients/mod.rs b/core/sdk/src/clients/mod.rs index 3ad0df5a46..1a310450fa 100644 --- a/core/sdk/src/clients/mod.rs +++ b/core/sdk/src/clients/mod.rs @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. +use crate::client_wrappers::client_wrapper::ClientWrapper; +use iggy_common::{BinaryTransport, ClientState}; + mod binary_cluster; mod binary_consumer_group; mod binary_consumer_offset; @@ -38,5 +41,35 @@ pub mod producer_error_callback; pub mod producer_sharding; const ORDERING: std::sync::atomic::Ordering = std::sync::atomic::Ordering::SeqCst; + +/// Whether the reconnect that followed a leader redirect already left this +/// client signed in as the user the redirected sign-in was for. +/// +/// The connect flow signs in with the credentials that sign-in just +/// remembered, so on a client without a configured `AutoLogin` the session on +/// the leader is already the right one: signing in again would run a logout +/// plus a second login, an argon2 each on the server, to arrive where the +/// client already is. With `AutoLogin::Enabled(a)` the reconnect signed in the +/// configured user instead, who need not be the one signing in here, so the +/// sign-in still has to run. +pub(crate) async fn redirect_login_settled(client: &ClientWrapper) -> bool { + let (state, auto_login_configured) = match client { + ClientWrapper::Tcp(tcp_client) => ( + tcp_client.get_state().await, + tcp_client.auto_login_configured(), + ), + ClientWrapper::Quic(quic_client) => ( + quic_client.get_state().await, + quic_client.auto_login_configured(), + ), + ClientWrapper::WebSocket(ws_client) => ( + ws_client.get_state().await, + ws_client.auto_login_configured(), + ), + _ => return false, + }; + + state == ClientState::Authenticated && !auto_login_configured +} const MAX_BATCH_LENGTH: usize = 1000000; const MIB: usize = 1_048_576; diff --git a/core/sdk/src/leader_aware.rs b/core/sdk/src/leader_aware.rs index bd99d2e32a..6e2b2a16b1 100644 --- a/core/sdk/src/leader_aware.rs +++ b/core/sdk/src/leader_aware.rs @@ -20,7 +20,7 @@ use iggy_common::ClusterClient; use iggy_common::{ ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, IggyError, TransportProtocol, }; -use std::net::SocketAddr; +use std::net::{SocketAddr, ToSocketAddrs}; use std::str::FromStr; use tracing::{debug, info, warn}; @@ -217,13 +217,41 @@ fn process_cluster_metadata( /// Check if two addresses refer to the same endpoint /// Handles various formats like 127.0.0.1:8090 vs localhost:8090 +/// +/// A host name and the address it resolves to are one endpoint too: a client +/// configured as `iggy-server:8090` whose roster advertises `10.0.0.5:8090` +/// would otherwise dial that node twice per failover sweep, and a single-node +/// deployment would be treated as a cluster. Resolution is the last resort, +/// only when the spellings differ and at least one side is not a literal +/// address, and it is a blocking lookup: this runs on the connect and redirect +/// paths, which are rare and already wait on the network. pub(crate) fn is_same_address(addr1: &str, addr2: &str) -> bool { match (parse_address(addr1), parse_address(addr2)) { (Some(sock1), Some(sock2)) => sock1.ip() == sock2.ip() && sock1.port() == sock2.port(), - _ => normalize_address(addr1) == normalize_address(addr2), + (parsed1, parsed2) => { + if normalize_address(addr1) == normalize_address(addr2) { + return true; + } + if parsed1.is_some() && parsed2.is_some() { + return false; + } + resolve_all(addr1) + .zip(resolve_all(addr2)) + .is_some_and(|(first, second)| { + first.iter().any(|resolved| second.contains(resolved)) + }) + } } } +/// Every socket address a host:port spelling resolves to, `None` when the +/// resolver does not know the name (which then compares unequal, at worst +/// costing one extra dial). +fn resolve_all(addr: &str) -> Option> { + let resolved: Vec = addr.to_socket_addrs().ok()?.collect(); + (!resolved.is_empty()).then_some(resolved) +} + /// Parse address string to SocketAddr, handling various formats fn parse_address(addr: &str) -> Option { if let Ok(socket_addr) = SocketAddr::from_str(addr) { @@ -338,6 +366,20 @@ mod tests { assert!(!is_same_address("192.168.1.1:8090", "127.0.0.1:8090")); } + // A host name and the address it resolves to name one endpoint; a name the + // resolver does not know compares unequal rather than erroring. + #[test] + fn a_host_name_matches_the_address_it_resolves_to() { + // `localhost` is rewritten before resolution, so use the loopback name + // the resolver itself answers for. + assert!(is_same_address("LOCALHOST:8090", "127.0.0.1:8090")); + assert!(!is_same_address("localhost:8090", "localhost:8091")); + assert!(!is_same_address( + "no-such-host.invalid:8090", + "127.0.0.1:8090" + )); + } + #[test] fn test_normalize_address() { assert_eq!(normalize_address("localhost:8090"), "127.0.0.1:8090"); diff --git a/core/sdk/src/quic/quic_client.rs b/core/sdk/src/quic/quic_client.rs index 255e473e47..b4d2b2ce6f 100644 --- a/core/sdk/src/quic/quic_client.rs +++ b/core/sdk/src/quic/quic_client.rs @@ -228,6 +228,12 @@ impl iggy_common::VsrSessionControl for QuicClient { impl BinaryClient for QuicClient {} impl QuicClient { + /// Whether an `AutoLogin` is configured on this client, which makes the + /// session after any connect the configured user's rather than whoever + /// signed in by hand. + pub(crate) fn auto_login_configured(&self) -> bool { + matches!(self.config.auto_login, AutoLogin::Enabled(_)) + } /// Creates a new QUIC client for the provided client and server addresses. pub fn new( client_address: &str, diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index 04957b0f5e..8bae4d2204 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -25,23 +25,29 @@ use crate::session::ConsensusSession; use crate::tcp::tcp_connection_stream::TcpConnectionStream; use crate::tcp::tcp_connection_stream_kind::ConnectionStreamKind; use crate::tcp::tcp_tls_connection_stream::TcpTlsConnectionStream; +use crate::vsr::operation_for_code; use async_broadcast::{Receiver, Sender, broadcast}; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; use iggy_binary_protocol::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE}; +use iggy_binary_protocol::consensus::Operation; +#[cfg(test)] +use iggy_common::TcpClientReconnectionConfig; use iggy_common::VsrSessionControl as _; use iggy_common::{ AutoLogin, ClientState, ConnectionString, ConnectionStringUtils, Credentials, DiagnosticEvent, - IggyDuration, IggyError, IggyTimestamp, TcpConnectionStringOptions, TransportProtocol, + IdKind, Identifier, IggyDuration, IggyError, IggyTimestamp, TcpConnectionStringOptions, + TransportProtocol, }; use iggy_common::{BinaryClient, BinaryTransport, PersonalAccessTokenClient, UserClient}; use rustls::pki_types::{CertificateDer, ServerName, pem::PemObject}; -use secrecy::ExposeSecret; -use std::io; +use secrecy::{ExposeSecret, SecretString}; use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; use std::sync::Mutex as StdMutex; +#[cfg(test)] +use tokio::net::TcpListener; use tokio::net::TcpStream; use tokio::sync::Mutex; use tokio::time::sleep; @@ -97,7 +103,7 @@ pub struct TcpClient { /// Credentials a sign-in on this client succeeded with, so a reconnect -- /// onto this node or, after a failover, another one -- can re-establish /// the session instead of surfacing `Unauthenticated`. Cleared on logout. - session_credentials: Mutex>, + session_credentials: Mutex>, // `std::sync::Mutex` (not `tokio::sync::Mutex`): the critical section // is `encode_request_header`, which is pure CPU and never awaits. The // tokio variant would pay a waker alloc + internal semaphore on @@ -107,6 +113,22 @@ pub struct TcpClient { consumer_group_state: Arc, } +/// The sign-in a manual login on this client succeeded with, and who it signed +/// in as. The user matters because a password change has to swap the new +/// password in here, and `change_password` may well target somebody else. +#[derive(Debug)] +struct RememberedSignIn { + credentials: Credentials, + user_id: u32, +} + +/// A connection that completed every step of coming up, TLS included. +struct EstablishedConnection { + stream: ConnectionStreamKind, + client_address: SocketAddr, + remote_address: SocketAddr, +} + impl Default for TcpClient { fn default() -> Self { TcpClient::create(Arc::new(TcpClientConfig::default())).unwrap() @@ -178,15 +200,6 @@ impl BinaryTransport for TcpClient { return Err(error); } - // A stale-client eviction is the server ending this session - // authoritatively, like a logout: the remembered sign-in ends with it, - // so only a configured auto-login may bring the session back. - // Remembered credentials exist for transport loss, where the session - // died with the socket rather than by anyone's decision. - if matches!(error, IggyError::StaleClient) { - self.forget_session_credentials().await; - } - if !self.config.reconnection.enabled { return Err(IggyError::Disconnected); } @@ -201,6 +214,27 @@ impl BinaryTransport for TcpClient { return Err(error); } + // Reconnecting heals the transport, but replaying the request over the + // new connection is a second attempt under a new session: + // `reset_vsr_session` drops the client id the server's dedup fence is + // keyed on, so a replicated write that committed before its reply was + // lost would apply a second time. Replay only what provably never + // reached the log -- the errors raised before the request was written, + // and the operations that never enter it. + // + // Login and register are the exception: the server stays deliberately + // silent on a transient register failure and relies on the client + // replaying, so that replay is the protocol rather than a retry. + let replay_after_reconnect = is_login_register_code(code) + || matches!( + error, + IggyError::NotConnected + | IggyError::CannotEstablishConnection + | IggyError::Unauthenticated + | IggyError::StaleClient + ) + || matches!(operation_for_code(code), Operation::NonReplicated); + self.disconnect_transport().await?; let skip_auto_login = is_login_register_code(code); @@ -222,6 +256,16 @@ impl BinaryTransport for TcpClient { *self.skip_auto_login_once.lock().await = false; } reconnect?; + + if !replay_after_reconnect { + warn!( + "Reconnected, but command: {code} is replicated and its outcome is unknown: \ + replaying it under the new session could apply it twice, so the original \ + error is returned instead." + ); + return Err(error); + } + self.send_raw(code, payload).await } @@ -263,14 +307,41 @@ impl iggy_common::VsrSessionControl for TcpClient { Ok(()) } - async fn remember_session_credentials(&self, credentials: Credentials) { - self.session_credentials.lock().await.replace(credentials); + async fn remember_session_credentials(&self, credentials: Credentials, user_id: u32) { + self.session_credentials + .lock() + .await + .replace(RememberedSignIn { + credentials, + user_id, + }); } async fn forget_session_credentials(&self) { self.session_credentials.lock().await.take(); } + async fn refresh_session_password(&self, user: &Identifier, new_password: &str) { + let mut remembered = self.session_credentials.lock().await; + let Some(sign_in) = remembered.as_mut() else { + return; + }; + // A personal access token is not derived from the password. + let Credentials::UsernamePassword(username, password) = &mut sign_in.credentials else { + return; + }; + + let targets_session_user = match user.kind { + IdKind::Numeric => user.get_u32_value().is_ok_and(|id| id == sign_in.user_id), + IdKind::String => user + .get_cow_str_value() + .is_ok_and(|name| name.as_ref() == username), + }; + if targets_session_user { + *password = SecretString::from(new_password.to_owned()); + } + } + fn sdk_version(&self) -> &'static str { crate::SDK_VERSION } @@ -363,23 +434,19 @@ impl TcpClient { } self.set_state(ClientState::Connecting).await; - let candidates = self.dial_candidates().await; - // The reestablish delay paces reconnects to the one endpoint a - // single-address client has. With other endpoints known there is - // somewhere else to go, and pausing first only pushes the - // failover past the window the caller is willing to wait; the - // retry interval still paces the loop. - let reestablish_wait = if candidates.len() > 1 { - None - } else { - self.reestablish_wait().await - }; - if let Some(remaining) = reestablish_wait { - info!("Trying to connect to the server in: {remaining}",); - sleep(remaining.get_duration()).await; + let mut candidates = self.dial_candidates().await; + // `reestablish_after` paces reconnects to the endpoint this client + // was last on, and to that one only: the other endpoints owe it no + // cooldown, and pausing before dialing them would push the failover + // past the window the caller is willing to wait. So when there is + // somewhere else to go, the paced endpoint goes last -- by which + // time its window has usually elapsed anyway -- instead of the wait + // being skipped outright. + let paced_endpoint = self.current_server_address.lock().await.clone(); + if candidates.len() > 1 && self.reestablish_wait().await.is_some() { + candidates.rotate_left(1); } - let tls_enabled = self.config.tls_enabled; let mut retry_count = 0; let connection_stream: ConnectionStreamKind; let remote_address; @@ -387,146 +454,82 @@ impl TcpClient { let mut candidate = 0; loop { let server_address = candidates[candidate].clone(); - info!( - "{NAME} client is connecting to server: {}...", - server_address - ); - - let connection = self.dial(&server_address, candidates.len() > 1).await; - if let Err(err) = &connection { - error!( - "Failed to connect to server: {}. Error: {}", - server_address, err - ); - if !self.config.reconnection.enabled { - warn!("Automatic reconnection is disabled."); - return Err(IggyError::CannotEstablishConnection); - } + if server_address == paced_endpoint + && let Some(remaining) = self.reestablish_wait().await + { + info!("Trying to connect to the server: {server_address} in: {remaining}"); + sleep(remaining.get_duration()).await; + } - // Every other endpoint gets its turn before the retry - // interval: the node just lost may be gone for good, and - // pausing on it helps nothing. - candidate += 1; - if candidate < candidates.len() { - continue; + info!("{NAME} client is connecting to server: {server_address}..."); + match self.establish_bounded(&server_address, &candidates).await { + Ok(connection) => { + // The endpoint that answered is where this client now + // lives: the leader check compares against it, and the + // next reconnect starts from it. Recorded only once the + // stream is usable, so a node that accepts TCP but + // fails the TLS handshake does not become sticky and + // shadow the endpoints behind it. + *self.current_server_address.lock().await = server_address; + client_address = connection.client_address; + remote_address = connection.remote_address; + self.client_address.lock().await.replace(client_address); + connection_stream = connection.stream; + break; } - candidate = 0; - - let unlimited_retries = self.config.reconnection.max_retries.is_none(); - let max_retries = self.config.reconnection.max_retries.unwrap_or_default(); - let max_retries_str = - if let Some(max_retries) = self.config.reconnection.max_retries { - max_retries.to_string() - } else { - "unlimited".to_string() - }; - - let interval_str = self.config.reconnection.interval.as_human_time_string(); - if unlimited_retries || retry_count < max_retries { - retry_count += 1; - info!( - "Retrying to connect to server ({retry_count}/{max_retries_str}): {} in: {interval_str}", - server_address, - ); - sleep(self.config.reconnection.interval.get_duration()).await; - continue; + Err(IggyError::CannotEstablishConnection) => {} + Err(error) => { + // An unreadable CA file or an unusable TLS domain is a + // configuration fault: no other endpoint fixes it, and + // retrying forever under `max_retries = None` would + // only bury it. + self.fail_connect().await; + return Err(error); } - - self.set_state(ClientState::Disconnected).await; - self.publish_event(DiagnosticEvent::Disconnected).await; - return Err(IggyError::CannotEstablishConnection); } - let stream = connection.map_err(|error| { - error!("Failed to establish TCP connection to the server: {error}",); - IggyError::CannotEstablishConnection - })?; - // The endpoint that answered is where this client now lives: - // the leader check compares against it, and the next - // reconnect starts from it. - *self.current_server_address.lock().await = server_address.clone(); - client_address = stream.local_addr().map_err(|error| { - error!("Failed to get the local address of the client: {error}",); - IggyError::CannotEstablishConnection - })?; - remote_address = stream.peer_addr().map_err(|error| { - error!("Failed to get the remote address of the server: {error}",); - IggyError::CannotEstablishConnection - })?; - self.client_address.lock().await.replace(client_address); - - if let Err(e) = stream.set_nodelay(self.config.nodelay) { - error!("Failed to set the nodelay option on the client: {e}, continuing...",); + // Every other endpoint gets its turn before the retry + // interval: the node just lost may be gone for good, and + // pausing on it helps nothing. + candidate += 1; + if candidate < candidates.len() { + continue; } - - if !tls_enabled { - connection_stream = - ConnectionStreamKind::Tcp(TcpConnectionStream::new(client_address, stream)); - break; + candidate = 0; + + // The sweep is what reconnection settings apply to, not a + // single dial: with reconnection off there are no retries, but + // the failover endpoints were configured to be tried and they + // get their one turn first. + if !self.config.reconnection.enabled { + warn!("Automatic reconnection is disabled."); + self.fail_connect().await; + return Err(IggyError::CannotEstablishConnection); } - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - - let config = if self.config.tls_validate_certificate { - let mut root_cert_store = rustls::RootCertStore::empty(); - if let Some(certificate_path) = &self.config.tls_ca_file { - for cert in - CertificateDer::pem_file_iter(certificate_path).map_err(|error| { - error!("Failed to read the CA file: {certificate_path}. {error}",); - IggyError::InvalidTlsCertificatePath - })? - { - let certificate = cert.map_err(|error| { - error!( - "Failed to read a certificate from the CA file: {certificate_path}. {error}", - ); - IggyError::InvalidTlsCertificate - })?; - root_cert_store.add(certificate).map_err(|error| { - error!( - "Failed to add a certificate to the root certificate store. {error}", - ); - IggyError::InvalidTlsCertificate - })?; - } + let unlimited_retries = self.config.reconnection.max_retries.is_none(); + let max_retries = self.config.reconnection.max_retries.unwrap_or_default(); + let max_retries_str = + if let Some(max_retries) = self.config.reconnection.max_retries { + max_retries.to_string() } else { - root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - } + "unlimited".to_string() + }; - rustls::ClientConfig::builder() - .with_root_certificates(root_cert_store) - .with_no_client_auth() - } else { - use crate::tcp::tcp_tls_verifier::NoServerVerification; - rustls::ClientConfig::builder() - .dangerous() - .with_custom_certificate_verifier(Arc::new(NoServerVerification)) - .with_no_client_auth() - }; - let connector = TlsConnector::from(Arc::new(config)); - let tls_domain = if self.config.tls_domain.is_empty() { - // Extract hostname/IP from server_address when tls_domain is not specified - server_address - .split(':') - .next() - .unwrap_or(&server_address) - .to_string() - } else { - self.config.tls_domain.to_owned() - }; - let domain = ServerName::try_from(tls_domain).map_err(|error| { - error!("Failed to create a server name from the domain. {error}",); - IggyError::InvalidTlsDomain - })?; - let stream = connector.connect(domain, stream).await.map_err(|error| { - error!("Failed to establish a TLS connection to the server: {error}",); - IggyError::CannotEstablishConnection - })?; - connection_stream = ConnectionStreamKind::TcpTls(TcpTlsConnectionStream::new( - client_address, - TlsStream::Client(stream), - )); - break; + if unlimited_retries || retry_count < max_retries { + retry_count += 1; + let interval_str = self.config.reconnection.interval.as_human_time_string(); + info!( + "Retrying to connect ({retry_count}/{max_retries_str}), \ + {} endpoint(s) in: {interval_str}", + candidates.len(), + ); + sleep(self.config.reconnection.interval.get_duration()).await; + continue; + } + + self.fail_connect().await; + return Err(IggyError::CannotEstablishConnection); } let now = IggyTimestamp::now(); @@ -560,19 +563,44 @@ impl TcpClient { } else { info!("{NAME} client: {client_address} is signing in..."); self.set_state(ClientState::Authenticating).await; - match &credentials { - Credentials::UsernamePassword(username, password) => { - self.login_user(username, password.expose_secret()).await?; - info!( - "{NAME} client: {client_address} has signed in with the user credentials, username: {username}", - ); + let signed_in = match &credentials { + Credentials::UsernamePassword(username, password) => self + .login_user(username, password.expose_secret()) + .await + .map(|_| format!("the user credentials, username: {username}")), + Credentials::PersonalAccessToken(token) => self + .login_with_personal_access_token(token.expose_secret()) + .await + .map(|_| "a personal access token".to_owned()), + }; + match signed_in { + Ok(how) => { + info!("{NAME} client: {client_address} has signed in with {how}.") } - Credentials::PersonalAccessToken(token) => { - self.login_with_personal_access_token(token.expose_secret()) - .await?; - info!( - "{NAME} client: {client_address} has signed in with a personal access token.", - ); + Err(error) => { + // The transport is up and only the session is + // not, so the state has to say so: left at + // `Authenticating` every gated operation fails + // client-side with `Disconnected`, `connect()` + // returns ok without dialing, and nothing short + // of an explicit `login_user` recovers. + self.set_state(ClientState::Connected).await; + // A rejected credential does not become valid on + // the next reconnect, and replaying it costs an + // argon2 on the server every time. Configured + // credentials stay as configured -- they are the + // caller's to fix -- so only the remembered + // sign-in is dropped. + if matches!( + error, + IggyError::InvalidCredentials + | IggyError::InvalidUsername + | IggyError::InvalidPassword + | IggyError::Unauthenticated + ) { + self.forget_session_credentials().await; + } + return Err(error); } } @@ -639,6 +667,13 @@ impl TcpClient { } } + /// Whether an `AutoLogin` is configured on this client, which makes the + /// session after any connect the configured user's rather than whoever + /// signed in by hand. + pub(crate) fn auto_login_configured(&self) -> bool { + matches!(self.config.auto_login, AutoLogin::Enabled(_)) + } + /// Credentials to sign in with after connecting: the configured ones, or /// else the ones a manual sign-in on this client succeeded with. A manual /// sign-in is otherwise less reconnectable than a configured one, which @@ -646,7 +681,12 @@ impl TcpClient { async fn sign_in_credentials(&self) -> Option { match &self.config.auto_login { AutoLogin::Enabled(credentials) => Some(credentials.clone()), - AutoLogin::Disabled => self.session_credentials.lock().await.clone(), + AutoLogin::Disabled => self + .session_credentials + .lock() + .await + .as_ref() + .map(|remembered| remembered.credentials.clone()), } } @@ -667,20 +707,137 @@ impl TcpClient { candidates } - /// Dial one endpoint, bounding the wait while other endpoints are queued - /// behind it (see `FAILOVER_DIAL_TIMEOUT`). - async fn dial(&self, server_address: &str, bounded: bool) -> io::Result { - if !bounded { - return TcpStream::connect(server_address).await; + /// Bring one endpoint all the way up: TCP connect, socket options, and the + /// TLS handshake when it is configured. Nothing about the connection is + /// recorded until this succeeds, so a half-usable endpoint leaves no trace + /// for the next connect to lead with. + /// + /// `CannotEstablishConnection` means this endpoint failed and the next one + /// is worth trying; any other error is a configuration fault that no + /// endpoint can satisfy. + async fn establish(&self, server_address: &str) -> Result { + let stream = TcpStream::connect(server_address).await.map_err(|error| { + error!("Failed to connect to server: {server_address}. Error: {error}"); + IggyError::CannotEstablishConnection + })?; + let client_address = stream.local_addr().map_err(|error| { + error!("Failed to get the local address of the client: {error}"); + IggyError::CannotEstablishConnection + })?; + let remote_address = stream.peer_addr().map_err(|error| { + error!("Failed to get the remote address of the server: {error}"); + IggyError::CannotEstablishConnection + })?; + + if let Err(error) = stream.set_nodelay(self.config.nodelay) { + error!("Failed to set the nodelay option on the client: {error}, continuing..."); } - match tokio::time::timeout(FAILOVER_DIAL_TIMEOUT, TcpStream::connect(server_address)).await - { - Ok(connection) => connection, - Err(_elapsed) => Err(io::Error::new( - io::ErrorKind::TimedOut, - format!("dialing {server_address} took longer than {FAILOVER_DIAL_TIMEOUT:?}"), + if !self.config.tls_enabled { + return Ok(EstablishedConnection { + stream: ConnectionStreamKind::Tcp(TcpConnectionStream::new(client_address, stream)), + client_address, + remote_address, + }); + } + + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let config = if self.config.tls_validate_certificate { + let mut root_cert_store = rustls::RootCertStore::empty(); + if let Some(certificate_path) = &self.config.tls_ca_file { + for cert in CertificateDer::pem_file_iter(certificate_path).map_err(|error| { + error!("Failed to read the CA file: {certificate_path}. {error}"); + IggyError::InvalidTlsCertificatePath + })? { + let certificate = cert.map_err(|error| { + error!( + "Failed to read a certificate from the CA file: {certificate_path}. {error}", + ); + IggyError::InvalidTlsCertificate + })?; + root_cert_store.add(certificate).map_err(|error| { + error!( + "Failed to add a certificate to the root certificate store. {error}" + ); + IggyError::InvalidTlsCertificate + })?; + } + } else { + root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + } + + rustls::ClientConfig::builder() + .with_root_certificates(root_cert_store) + .with_no_client_auth() + } else { + use crate::tcp::tcp_tls_verifier::NoServerVerification; + rustls::ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoServerVerification)) + .with_no_client_auth() + }; + + let connector = TlsConnector::from(Arc::new(config)); + let tls_domain = if self.config.tls_domain.is_empty() { + // Extract hostname/IP from server_address when tls_domain is not specified + server_address + .split(':') + .next() + .unwrap_or(server_address) + .to_string() + } else { + self.config.tls_domain.to_owned() + }; + let domain = ServerName::try_from(tls_domain).map_err(|error| { + error!("Failed to create a server name from the domain. {error}"); + IggyError::InvalidTlsDomain + })?; + let stream = connector.connect(domain, stream).await.map_err(|error| { + error!("Failed to establish a TLS connection to the server: {error}"); + IggyError::CannotEstablishConnection + })?; + + Ok(EstablishedConnection { + stream: ConnectionStreamKind::TcpTls(TcpTlsConnectionStream::new( + client_address, + TlsStream::Client(stream), )), + client_address, + remote_address, + }) + } + + /// Give up on connecting. The state has to go back to `Disconnected`: + /// left at `Connecting`, the next `connect()` returns ok at the top + /// without ever dialing. + async fn fail_connect(&self) { + self.set_state(ClientState::Disconnected).await; + self.publish_event(DiagnosticEvent::Disconnected).await; + } + + /// [`Self::establish`], bounded while other endpoints are queued behind + /// this one (see `FAILOVER_DIAL_TIMEOUT`). The bound covers the handshake + /// as well as the connect: a peer that accepts TCP and then never answers + /// the ClientHello is exactly the kind of failure the survivors are there + /// for, and neither step has a deadline of its own. + async fn establish_bounded( + &self, + server_address: &str, + candidates: &[String], + ) -> Result { + if candidates.len() < 2 { + return self.establish(server_address).await; + } + + match tokio::time::timeout(FAILOVER_DIAL_TIMEOUT, self.establish(server_address)).await { + Ok(connection) => connection, + Err(_elapsed) => { + error!( + "Connecting to server: {server_address} took longer than \ + {FAILOVER_DIAL_TIMEOUT:?}" + ); + Err(IggyError::CannotEstablishConnection) + } } } @@ -1006,6 +1163,8 @@ const fn is_login_register_code(code: u32) -> bool { mod tests { use super::*; + const SESSION_USER_ID: u32 = 7; + fn client_with(server_address: &str, failover_addresses: Vec) -> TcpClient { TcpClient::create(Arc::new(TcpClientConfig { server_address: server_address.to_string(), @@ -1015,6 +1174,205 @@ mod tests { .expect("create the client") } + /// A listener nothing ever accepts from: the kernel completes the TCP + /// handshake out of its backlog, which is all a dial needs to succeed. + async fn live_endpoint() -> (TcpListener, String) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind a listener"); + let address = listener.local_addr().expect("listener address").to_string(); + (listener, address) + } + + /// An address with nothing behind it: the dial is refused at once. + async fn dead_endpoint() -> String { + let (listener, address) = live_endpoint().await; + drop(listener); + address + } + + /// A peer that accepts TCP and hangs up without a byte: enough for the + /// dial, never enough for a TLS handshake. + async fn endpoint_that_hangs_up() -> String { + let (listener, address) = live_endpoint().await; + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + drop(stream); + } + }); + address + } + + // With reconnection off there are no retries, but the failover endpoints + // were configured to be tried and each still gets its one turn. + #[tokio::test] + async fn a_client_with_reconnection_disabled_still_sweeps_its_failover_endpoints() { + let (_listener, survivor) = live_endpoint().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: dead_endpoint().await, + failover_addresses: vec![survivor.clone()], + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + + TcpClient::connect(&client).await.expect("connect"); + assert_eq!(*client.current_server_address.lock().await, survivor); + } + + // A connect that gives up has to leave the state at `Disconnected`: left + // at `Connecting`, the next `connect()` returns ok at the top without ever + // dialing, and the client is wedged for good. + #[tokio::test] + async fn a_connect_that_exhausts_every_endpoint_leaves_the_client_disconnected() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: dead_endpoint().await, + failover_addresses: vec![dead_endpoint().await], + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + + assert!(matches!( + TcpClient::connect(&client).await, + Err(IggyError::CannotEstablishConnection) + )); + assert_eq!(client.get_state().await, ClientState::Disconnected); + assert!( + matches!( + TcpClient::connect(&client).await, + Err(IggyError::CannotEstablishConnection) + ), + "a second connect has to dial again rather than report success" + ); + } + + // An endpoint that accepts TCP but fails the TLS handshake is not where + // this client lives: recording it would make the next connect lead with + // it and shadow every endpoint behind it. + #[tokio::test] + async fn an_endpoint_that_fails_the_tls_handshake_does_not_become_the_current_one() { + let configured = dead_endpoint().await; + // Plain TCP behind a TLS client: the dial succeeds, the handshake + // cannot. + let plaintext = endpoint_that_hangs_up().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: configured.clone(), + failover_addresses: vec![plaintext], + tls_enabled: true, + tls_validate_certificate: false, + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + + assert!(matches!( + TcpClient::connect(&client).await, + Err(IggyError::CannotEstablishConnection) + )); + assert_eq!(*client.current_server_address.lock().await, configured); + } + + // A peer that accepts TCP and then never answers is what the other + // endpoints are there for; without a bound on the handshake the sweep + // waits on it forever. + #[tokio::test] + async fn an_endpoint_that_never_answers_the_handshake_does_not_hold_up_the_sweep() { + let (_listener, silent) = live_endpoint().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: silent, + failover_addresses: vec![dead_endpoint().await], + tls_enabled: true, + tls_validate_certificate: false, + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + + let sweep = tokio::time::timeout( + FAILOVER_DIAL_TIMEOUT * 3, + std::pin::pin!(TcpClient::connect(&client)), + ) + .await + .expect("the sweep has to end on its own"); + assert!(matches!(sweep, Err(IggyError::CannotEstablishConnection))); + } + + // `reestablish_after` paces reconnects to the endpoint that was lost. With + // somewhere else to go, that pause must not hold up the failover. + #[tokio::test] + async fn a_pending_reestablish_pause_does_not_delay_dialing_another_endpoint() { + let (_listener, survivor) = live_endpoint().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: dead_endpoint().await, + failover_addresses: vec![survivor.clone()], + reconnection: TcpClientReconnectionConfig { + reestablish_after: IggyDuration::from_str("10s").expect("duration"), + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + client + .connected_at + .lock() + .await + .replace(IggyTimestamp::now()); + + let started = std::time::Instant::now(); + TcpClient::connect(&client).await.expect("connect"); + assert_eq!(*client.current_server_address.lock().await, survivor); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "the failover waited out a pause it owed only the lost endpoint: {:?}", + started.elapsed() + ); + } + + // The other half of the same promise: `with_reestablish_after` is a + // cooldown on redialing the endpoint that was lost, and a known roster + // does not cancel it. + #[tokio::test] + async fn the_reestablish_pause_still_applies_to_the_endpoint_that_was_lost() { + let (_listener, current) = live_endpoint().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: current.clone(), + failover_addresses: vec![dead_endpoint().await], + reconnection: TcpClientReconnectionConfig { + reestablish_after: IggyDuration::from_str("1s").expect("duration"), + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + client + .connected_at + .lock() + .await + .replace(IggyTimestamp::now()); + + let started = std::time::Instant::now(); + TcpClient::connect(&client).await.expect("connect"); + assert_eq!(*client.current_server_address.lock().await, current); + assert!( + started.elapsed() >= std::time::Duration::from_millis(700), + "the cooldown on the lost endpoint was skipped: {:?}", + started.elapsed() + ); + } + #[tokio::test] async fn dial_candidates_lead_with_the_current_endpoint_and_name_each_other_one_once() { let client = client_with( @@ -1056,10 +1414,10 @@ mod tests { async fn an_explicit_disconnect_forgets_the_remembered_sign_in() { let client = client_with("127.0.0.1:8090", Vec::new()); client - .remember_session_credentials(Credentials::UsernamePassword( - "iggy".to_string(), - "iggy".into(), - )) + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "iggy".into()), + SESSION_USER_ID, + ) .await; Client::disconnect(&client).await.expect("disconnect"); @@ -1073,10 +1431,10 @@ mod tests { async fn a_transport_drop_keeps_the_remembered_sign_in() { let client = client_with("127.0.0.1:8090", Vec::new()); client - .remember_session_credentials(Credentials::UsernamePassword( - "iggy".to_string(), - "iggy".into(), - )) + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "iggy".into()), + SESSION_USER_ID, + ) .await; client @@ -1095,10 +1453,10 @@ mod tests { assert!(client.sign_in_credentials().await.is_none()); client - .remember_session_credentials(Credentials::UsernamePassword( - "iggy".to_string(), - "iggy".into(), - )) + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "iggy".into()), + SESSION_USER_ID, + ) .await; assert!(client.sign_in_credentials().await.is_some()); @@ -1108,6 +1466,92 @@ mod tests { assert!(client.sign_in_credentials().await.is_none()); } + // A password change for the signed-in user has to reach the remembered + // sign-in, or the next reconnect replays the old password and fails an + // unrelated request with `InvalidCredentials`. + #[tokio::test] + async fn a_password_change_for_the_signed_in_user_updates_the_remembered_sign_in() { + let client = client_with("127.0.0.1:8090", Vec::new()); + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "old".into()), + SESSION_USER_ID, + ) + .await; + + for user in [ + Identifier::numeric(SESSION_USER_ID).expect("numeric identifier"), + Identifier::named("iggy").expect("named identifier"), + ] { + client.refresh_session_password(&user, "new").await; + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(_, password)) => { + assert_eq!(password.expose_secret(), "new", "for user: {user}"); + } + other => panic!("expected the remembered user credentials, got {other:?}"), + } + // Put it back so the second identifier form starts from the same + // place as the first. + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "old".into()), + SESSION_USER_ID, + ) + .await; + } + } + + // `change_password` can target anyone the caller may manage, and those + // changes say nothing about the credentials this client reconnects with. + #[tokio::test] + async fn a_password_change_for_another_user_leaves_the_remembered_sign_in_alone() { + let client = client_with("127.0.0.1:8090", Vec::new()); + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "old".into()), + SESSION_USER_ID, + ) + .await; + + for user in [ + Identifier::numeric(SESSION_USER_ID + 1).expect("numeric identifier"), + Identifier::named("someone-else").expect("named identifier"), + ] { + client.refresh_session_password(&user, "new").await; + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(_, password)) => { + assert_eq!(password.expose_secret(), "old", "for user: {user}"); + } + other => panic!("expected the remembered user credentials, got {other:?}"), + } + } + } + + // A personal access token is not derived from any password. + #[tokio::test] + async fn a_password_change_leaves_a_remembered_personal_access_token_alone() { + let client = client_with("127.0.0.1:8090", Vec::new()); + client + .remember_session_credentials( + Credentials::PersonalAccessToken("token".into()), + SESSION_USER_ID, + ) + .await; + + client + .refresh_session_password( + &Identifier::numeric(SESSION_USER_ID).expect("numeric identifier"), + "new", + ) + .await; + match client.sign_in_credentials().await { + Some(Credentials::PersonalAccessToken(token)) => { + assert_eq!(token.expose_secret(), "token"); + } + other => panic!("expected the remembered token, got {other:?}"), + } + } + #[tokio::test] async fn configured_credentials_outrank_the_ones_a_sign_in_remembered() { let client = TcpClient::create(Arc::new(TcpClientConfig { @@ -1119,10 +1563,10 @@ mod tests { })) .expect("create the client"); client - .remember_session_credentials(Credentials::UsernamePassword( - "signed-in".to_string(), - "iggy".into(), - )) + .remember_session_credentials( + Credentials::UsernamePassword("signed-in".to_string(), "iggy".into()), + SESSION_USER_ID, + ) .await; match client.sign_in_credentials().await { diff --git a/core/sdk/src/vsr.rs b/core/sdk/src/vsr.rs index 4a6bda52ec..9625b020c3 100644 --- a/core/sdk/src/vsr.rs +++ b/core/sdk/src/vsr.rs @@ -159,7 +159,7 @@ pub(crate) fn encode_request_header( /// the authority: an unmapped code is forwarded as non-replicated (the code /// rides `RequestHeader.reserved`, which that path already stamps) and the /// server answers with a proper error if it does not know it. -fn operation_for_code(code: u32) -> Operation { +pub(crate) fn operation_for_code(code: u32) -> Operation { if code == LOGOUT_USER_CODE { return Operation::Logout; } diff --git a/core/sdk/src/websocket/websocket_client.rs b/core/sdk/src/websocket/websocket_client.rs index 362a2aa953..43b3ae4277 100644 --- a/core/sdk/src/websocket/websocket_client.rs +++ b/core/sdk/src/websocket/websocket_client.rs @@ -224,6 +224,12 @@ impl iggy_common::VsrSessionControl for WebSocketClient { impl BinaryClient for WebSocketClient {} impl WebSocketClient { + /// Whether an `AutoLogin` is configured on this client, which makes the + /// session after any connect the configured user's rather than whoever + /// signed in by hand. + pub(crate) fn auto_login_configured(&self) -> bool { + matches!(self.config.auto_login, AutoLogin::Enabled(_)) + } /// Create a new WebSocket client with the provided configuration. pub fn create(config: Arc) -> Result { let (sender, receiver) = broadcast(1000); diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs index aa01c7b383..ea41f1e0f3 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs @@ -70,6 +70,13 @@ public sealed partial class TcpMessageStream : ISessionGenerationProvider /// private const int VsrMaxLeaderRedirects = 3; + /// + /// Bound on one endpoint's dial while other endpoints are queued behind it. Neither the connect nor the + /// TLS handshake has a deadline of its own, so a node whose syns are dropped would hold the sweep for + /// the whole kernel connect timeout - minutes - while a survivor goes untried. Matches the Rust SDK. + /// + private const int FailoverDialTimeout = 2_000; + /// /// Attempts a consumer-group poll gets before it gives up and reports an empty poll: one re-sync after /// the coordinator fences a stale assignment, then one retry. @@ -562,6 +569,11 @@ private async Task> SendRawAsync(int code, ReadOnlyMemory 1 + ? CancellationTokenSource.CreateLinkedTokenSource(token) + : null; + dialCancellation?.CancelAfter(FailoverDialTimeout); + var dialToken = dialCancellation?.Token ?? token; + + await socket.ConnectAsync(host, port, dialToken); dialed = true; _currentRemoteAddress = socket.RemoteEndPoint is IPEndPoint remote @@ -1069,7 +1085,7 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5); var connectionStream = _configuration.TlsSettings.Enabled - ? await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings) + ? await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings, dialToken) : new NetworkStream(socket, true); await _sendingSemaphore.WaitAsync(token); @@ -1108,7 +1124,11 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken // Only a failed dial is worth another attempt. Everything past it - a rejected certificate, bad // credentials, a leader that cannot be found - fails the same way every time, and with unlimited // retries a caller would otherwise never get the error back. - catch (Exception e) when (dialed || e is OperationCanceledException || _disposed) + // A dial the bound above cut short is a failed dial like any other, so it must not land here: + // only a cancellation the caller actually asked for is fatal. + catch (Exception e) when (dialed + || (e is OperationCanceledException && token.IsCancellationRequested) + || _disposed) { socket?.Dispose(); @@ -1126,14 +1146,6 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken _logger.LogError(e, "Failed to connect"); - if (!_configuration.ReconnectionSettings.Enabled || - (_configuration.ReconnectionSettings.MaxRetries > 0 && - retryCount >= _configuration.ReconnectionSettings.MaxRetries)) - { - SetConnectionState(ConnectionState.Disconnected); - throw; - } - // Every other endpoint gets its turn before the retry delay: the node just lost may be gone for // good, and pausing on it helps nothing. if (++candidate < candidates.Length) @@ -1145,6 +1157,17 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken candidate = 0; _currentAddress = candidates[0]; + // The sweep is what the reconnection budget applies to, not a single dial: checked per dial, + // the last round would try only the endpoint the client started on, and a client with + // reconnection turned off would never reach its other endpoints at all. + if (!_configuration.ReconnectionSettings.Enabled || + (_configuration.ReconnectionSettings.MaxRetries > 0 && + retryCount >= _configuration.ReconnectionSettings.MaxRetries)) + { + SetConnectionState(ConnectionState.Disconnected); + throw; + } + retryCount++; if (_configuration.ReconnectionSettings.UseExponentialBackoff) { @@ -1267,7 +1290,8 @@ private async Task AutoLoginAsync(AutoLoginSettings settings, CancellationToken return _rememberedLogin; } - private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings) + private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings, + CancellationToken token) { ValidateCertificatePath(tlsSettings.CertificatePath); @@ -1276,7 +1300,10 @@ private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSett var stream = new NetworkStream(socket, true); var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback); - await sslStream.AuthenticateAsClientAsync(tlsSettings.Hostname); + // The token carries the dial bound when other endpoints are queued behind this one: a peer that + // accepts TCP and never answers the ClientHello has no deadline of its own here either. + await sslStream.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions { TargetHost = tlsSettings.Hostname }, token); return sslStream; } @@ -1301,10 +1328,7 @@ private async Task> SendWithResponseAsync(int code, ReadOnlyM // remembered sign-in ends with it, so only a configured auto login may bring the session back. // Remembered credentials exist for transport loss, where the session died with the socket rather // than by anyone's decision. - if (e is IggyInvalidStatusCodeException { StatusCode: VsrError.STALE_CLIENT, FromServer: true }) - { - _rememberedLogin = null; - } + ForgetSessionAfterEviction(e); if (!_configuration.ReconnectionSettings.Enabled) { @@ -1335,6 +1359,18 @@ private static bool IsLostConnection(Exception e) || e is IggyInvalidStatusCodeException { StatusCode: VsrError.STALE_CLIENT, FromServer: true }; } + // An eviction ends this session authoritatively, like a logout: the sign-in this client remembered goes + // with it, so only a configured auto login may bring the session back. Called from the consensus layer, + // which sees the eviction whatever it interrupted - an eviction that landed on a replicated write is + // reported as VsrRequestOutcomeUnknownException and never reaches the lost-connection path at all. + private void ForgetSessionAfterEviction(Exception verdict) + { + if (verdict is IggyInvalidStatusCodeException { StatusCode: VsrError.STALE_CLIENT, FromServer: true }) + { + _rememberedLogin = null; + } + } + private async Task> HandleReconnectionAsync(int code, ReadOnlyMemory body, bool autoLogin, CancellationToken token) { diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/DialCandidatesTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/DialCandidatesTests.cs index 7fdaf99076..20e0e949bc 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/DialCandidatesTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/DialCandidatesTests.cs @@ -38,6 +38,21 @@ public void LeadsWithTheCurrentEndpointThenNamesEachOtherOneOnce() Assert.Equal(["127.0.0.1:8090", "127.0.0.1:8091", "127.0.0.1:8092"], candidates); } + /// + /// The configured address comes before the roster: it is the one endpoint the caller vouched for, and a + /// roster learned from a cluster that has since changed shape may name nodes that are gone. + /// + [Fact] + public void DialsTheConfiguredAddressBeforeTheLearnedRoster() + { + var candidates = TcpMessageStream.DialCandidates( + "127.0.0.1:8090", + "127.0.0.1:8099", + ["127.0.0.1:8091", "127.0.0.1:8092"]); + + Assert.Equal(["127.0.0.1:8090", "127.0.0.1:8099", "127.0.0.1:8091", "127.0.0.1:8092"], candidates); + } + [Fact] public void KeepsTheConfiguredAddressWhenNoRosterWasLearned() { diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs index c1554133d2..b0f8d96e21 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs @@ -22,7 +22,10 @@ using Apache.Iggy.Configuration; using Apache.Iggy.Contracts.Tcp; using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.IggyClient; using Apache.Iggy.IggyClient.Implementations; +using Apache.Iggy.Vsr; using Microsoft.Extensions.Logging.Abstractions; namespace Apache.Iggy.Tests.VsrTests; @@ -144,12 +147,68 @@ public async Task ServerEvictionForgetsTheRememberedSignIn() var connectionsBeforeEviction = node.Connections; evict = true; - await Assert.ThrowsAnyAsync(() => client.PingAsync(TestContext.Current.CancellationToken)); + var evicted = await Assert.ThrowsAsync(() => + client.PingAsync(TestContext.Current.CancellationToken)); + Assert.Equal(VsrError.STALE_CLIENT, evicted.StatusCode); + Assert.True(evicted.FromServer); Assert.Equal(connectionsBeforeEviction, node.Connections); // The dropped connection leaves the next call transport-shaped, but the eviction forgot the remembered // sign-in, so it must fail fast instead of reconnecting into a resurrected session. - await Assert.ThrowsAnyAsync(() => client.PingAsync(TestContext.Current.CancellationToken)); + await Assert.ThrowsAsync(() => + client.PingAsync(TestContext.Current.CancellationToken)); + Assert.Equal(connectionsBeforeEviction, node.Connections); + } + + /// + /// The same contract when the eviction lands on a replicated write: that request is reported as + /// outcome-unknown rather than as a lost connection, so it never passes through the lost-connection + /// path, and the session it evicted still has to be forgotten. + /// + [Fact] + public async Task ServerEvictionDuringAReplicatedWriteForgetsTheRememberedSignIn() + { + using var node = new MockNode(); + var evict = false; + node.Serve(request => + { + if (request.Operation == OperationRegister) + { + return Reply(OperationRegister, RegisterBody(session: 128)); + } + + return evict + ? EvictionFrame(EvictionStaleClient) + : Reply(OperationNonReplicated, request.Code == GetClusterMetadataCode + ? ClusterMetadata(node.Port, node.Port, node.Port) + : []); + }); + + var configuration = new IggyClientConfigurator + { + BaseAddress = $"127.0.0.1:{node.Port}", + Protocol = Protocol.Tcp, + ReconnectionSettings = new ReconnectionSettings + { + Enabled = true, + MaxRetries = 2, + InitialDelay = TimeSpan.FromMilliseconds(20) + } + }; + using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); + + await client.ConnectAsync(TestContext.Current.CancellationToken); + await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); + await client.PingAsync(TestContext.Current.CancellationToken); + var connectionsBeforeEviction = node.Connections; + + evict = true; + await Assert.ThrowsAsync(() => + client.CreateStreamAsync("evicted-mid-write", token: TestContext.Current.CancellationToken)); + Assert.Equal(connectionsBeforeEviction, node.Connections); + + await Assert.ThrowsAsync(() => + client.PingAsync(TestContext.Current.CancellationToken)); Assert.Equal(connectionsBeforeEviction, node.Connections); } @@ -186,10 +245,19 @@ public async Task FailsFastWhenNothingEverSignedIn() await client.ConnectAsync(TestContext.Current.CancellationToken); await client.PingAsync(TestContext.Current.CancellationToken); + // A reconnect announces itself by entering Connecting, so the absence of that transition is the + // assertion - no need to poll for a request that must never succeed. + var reconnected = false; + client.SubscribeConnectionEvents(args => + { + reconnected |= args.CurrentState == ConnectionState.Connecting; + return Task.CompletedTask; + }); + node.Kill(); - var (resumed, _) = await ResumedWithin(client, TimeSpan.FromSeconds(2)); - Assert.False(resumed, "a client that never signed in cannot restore a session by reconnecting"); + await Assert.ThrowsAnyAsync(() => client.PingAsync(TestContext.Current.CancellationToken)); + Assert.False(reconnected, "a client that never signed in cannot restore a session by reconnecting"); } private static async Task<(bool Resumed, string LastError)> ResumedWithin(TcpMessageStream client, diff --git a/foreign/go/client/tcp/tcp_core.go b/foreign/go/client/tcp/tcp_core.go index f82520b594..b721abf124 100644 --- a/foreign/go/client/tcp/tcp_core.go +++ b/foreign/go/client/tcp/tcp_core.go @@ -302,6 +302,11 @@ const ( // A node that stopped being primary answers transient forever, so // replaying alone never recovers. failoverCheckInterval = 2 * time.Second + // failoverDialTimeout bounds one endpoint's dial and handshake while other + // endpoints are queued behind it. Neither step has a deadline of its own, + // so a node whose syns are dropped would hold the sweep for the whole + // kernel connect timeout while a survivor goes untried. + failoverDialTimeout = 2 * time.Second ) // requestBufPool reuses wire-payload buffers across RPCs. A fresh buffer @@ -476,6 +481,19 @@ func (c *IggyTcpClient) exchange(ctx context.Context, code uint32, frame []byte) if err == nil || !isReconnectable(err) { return response, err } + + // A stale-client eviction is the server ending this session + // authoritatively, like a logout: the remembered sign-in ends with it, so + // only a configured auto-login may bring the session back. Remembered + // credentials exist for transport loss, where the session died with the + // socket rather than by anyone's decision. This runs before the gates + // below because they all return: with reconnection disabled the eviction + // would otherwise never be forgotten, and the next manual Connect would + // sign in with the evicted session's credentials. + if errors.Is(err, ierror.ErrStaleClient) { + c.forgetLogin() + } + var precondition *localPreconditionError if errors.As(err, &precondition) { return nil, err @@ -488,15 +506,6 @@ func (c *IggyTcpClient) exchange(ctx context.Context, code uint32, frame []byte) return nil, err } - // A stale-client eviction is the server ending this session - // authoritatively, like a logout: the remembered sign-in ends with it, so - // only a configured auto-login may bring the session back. Remembered - // credentials exist for transport loss, where the session died with the - // socket rather than by anyone's decision. - if errors.Is(err, ierror.ErrStaleClient) { - c.forgetLogin() - } - // With no credentials -- neither configured nor remembered from a // sign-in -- a reconnect cannot restore the session, so anything but a // sign-in fails here instead of replaying unauthenticated. The sign-in @@ -909,22 +918,28 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { c.mtx.Unlock() candidates := c.connectionCandidates() + if len(candidates) == 0 { + // Nowhere to dial: a client configured with an empty server address + // and no roster. Reporting success here would leave every request + // answering ErrNotConnected while Connect keeps saying it is + // connected. + c.mtx.Lock() + c.transportState = iggcon.TransportStateDisconnected + c.mtx.Unlock() + c.logger.Error("No server address to connect to.") + return ierror.ErrCannotEstablishConnection + } - // The reestablish interval paces reconnects to the one endpoint a - // single-address client has. With other endpoints known there is somewhere - // else to go, and pausing first only pushes the failover past the window - // the caller is willing to wait; the retry interval still paces the loop. - if !connectedAt.IsZero() && len(candidates) == 1 { - now := time.Now() - elapsed := now.Sub(connectedAt) - reestablishAfter := c.config.reconnection.reestablishAfter - - c.logger.Debug("Elapsed time since last connection", slog.Duration("elapsed", elapsed)) - if elapsed < reestablishAfter { - remaining := reestablishAfter - elapsed - c.logger.Info("Trying to connect to the server", slog.Duration("remaining", remaining)) - time.Sleep(remaining) - } + // reestablishAfter paces reconnects to the endpoint this client was last + // on, and to that one only: the other endpoints owe it no cooldown, and + // pausing before dialing them would push the failover past the window the + // caller is willing to wait. So when there is somewhere else to go, the + // paced endpoint goes last -- by which time its window has usually + // elapsed anyway -- instead of the wait being skipped outright. + pacedEndpoint := candidates[0] + if !connectedAt.IsZero() && len(candidates) > 1 && + time.Since(connectedAt) < c.config.reconnection.reestablishAfter { + candidates = append(candidates[1:], pacedEndpoint) } attempts := uint(1) interval := time.Duration(0) @@ -949,7 +964,10 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { // survivors of a client configured for a single retry. var lastErr error for _, address := range candidates { - connection, err := c.dialCandidate(ctx, address) + if address == pacedEndpoint { + c.awaitReestablish(connectedAt) + } + connection, err := c.dialCandidate(ctx, address, len(candidates) > 1) if err != nil { lastErr = err continue @@ -991,11 +1009,38 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { return nil } -// dialCandidate opens one connection, wrapping it in TLS when configured, and -// records the endpoint that answered: the leader check compares against it and -// the next reconnect starts from it. -func (c *IggyTcpClient) dialCandidate(ctx context.Context, address string) (net.Conn, error) { +// awaitReestablish waits out what is left of the reestablishAfter window since +// the last successful connection, if any. +func (c *IggyTcpClient) awaitReestablish(connectedAt time.Time) { + if connectedAt.IsZero() { + return + } + + elapsed := time.Since(connectedAt) + c.logger.Debug("Elapsed time since last connection", slog.Duration("elapsed", elapsed)) + if remaining := c.config.reconnection.reestablishAfter - elapsed; remaining > 0 { + c.logger.Info("Trying to connect to the server", slog.Duration("remaining", remaining)) + time.Sleep(remaining) + } +} + +// dialCandidate brings one endpoint all the way up, wrapping it in TLS when +// configured, and records the endpoint that answered: the leader check +// compares against it and the next reconnect starts from it. +// +// bounded caps the whole attempt at failoverDialTimeout, for when other +// endpoints are queued behind this one. Neither the dial nor the handshake has +// a deadline of its own, and a node whose syns are dropped -- or one that +// accepts TCP and then never answers the ClientHello -- would hold the sweep +// for minutes while a survivor goes untried. +func (c *IggyTcpClient) dialCandidate(ctx context.Context, address string, bounded bool) (net.Conn, error) { c.logger.Info("Iggy client is connecting to server...", slog.String("server_address", address)) + if bounded { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, failoverDialTimeout) + defer cancel() + } + connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", address) if err != nil { c.logger.Error("Failed to establish TCP connection to the server", slog.Any("error", err)) @@ -1007,29 +1052,32 @@ func (c *IggyTcpClient) dialCandidate(ctx context.Context, address string) (net. c.logger.Error("Failed to set the nodelay option on the client, continuing...", slog.Any("error", err)) } + established := connection + if c.config.tlsEnabled { + tlsConfig, err := c.createTLSConfig() + if err != nil { + _ = connection.Close() + return nil, err + } + + tlsConn := tls.Client(connection, tlsConfig) + if err := tlsConn.HandshakeContext(ctx); err != nil { + c.logger.Error("Failed to establish a TLS connection to the server", slog.Any("error", err)) + _ = connection.Close() + return nil, fmt.Errorf("TLS handshake failed: %w", err) + } + established = tlsConn + } + + // Recorded only once the connection is usable: an endpoint that accepts + // TCP but fails the handshake is not where this client lives, and leading + // the next pass with it would shadow every endpoint behind it. c.mtx.Lock() c.clientAddress = tc.LocalAddr().String() c.currentServerAddress = address c.mtx.Unlock() - if !c.config.tlsEnabled { - return connection, nil - } - - tlsConfig, err := c.createTLSConfig() - if err != nil { - _ = connection.Close() - return nil, err - } - - tlsConn := tls.Client(connection, tlsConfig) - if err := tlsConn.HandshakeContext(ctx); err != nil { - c.logger.Error("Failed to establish a TLS connection to the server", slog.Any("error", err)) - _ = connection.Close() - return nil, fmt.Errorf("TLS handshake failed: %w", err) - } - - return tlsConn, nil + return established, nil } func (c *IggyTcpClient) connectionCandidates() []string { @@ -1092,7 +1140,9 @@ func (c *IggyTcpClient) signInCredentials() (Credentials, bool) { return c.rememberedLogin.credentials, c.rememberedLogin.enabled } -// rememberLogin keeps the credentials a sign-in just succeeded with. +// rememberLogin keeps the credentials a sign-in succeeded with. Call it from +// under registerMtx (register does): remembered outside that lock, two +// concurrent sign-ins can leave A remembered while the session is B. func (c *IggyTcpClient) rememberLogin(credentials Credentials) { c.mtx.Lock() defer c.mtx.Unlock() diff --git a/foreign/go/client/tcp/tcp_failover_test.go b/foreign/go/client/tcp/tcp_failover_test.go index badafeb0cf..2531ef8d96 100644 --- a/foreign/go/client/tcp/tcp_failover_test.go +++ b/foreign/go/client/tcp/tcp_failover_test.go @@ -20,10 +20,12 @@ package tcp import ( "context" "log/slog" + "net" "sync/atomic" "testing" "time" + ierror "github.com/apache/iggy/foreign/go/errors" "github.com/apache/iggy/foreign/go/internal/command" "github.com/apache/iggy/foreign/go/internal/vsr" "github.com/stretchr/testify/assert" @@ -160,7 +162,15 @@ func TestFailover_ServerEvictionForgetsTheRememberedSignIn(t *testing.T) { // with the credentials the earlier sign-in used. func TestFailover_DoesNotResurrectASignedOutSession(t *testing.T) { var server *testListener - server = listenVSR(t, nil, singleNodeHandler(t, func() string { return server.address() })) + var dropSocket atomic.Bool + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + // A dropped connection is what makes the client reconnect at all; nil + // ends it the way a killed process does. + if dropSocket.Load() { + return nil + } + return singleNodeHandler(t, func() string { return server.address() })(0, 0, read) + }) client := newDialingClient(t, server.address()) ctx := context.Background() @@ -170,21 +180,185 @@ func TestFailover_DoesNotResurrectASignedOutSession(t *testing.T) { require.NoError(t, client.LogoutUser(ctx)) credentials, ok := client.signInCredentials() - assert.False(t, ok, "the sign-out forgot them") - assert.Empty(t, credentials.username) + require.False(t, ok, "the sign-out forgot them") + require.Empty(t, credentials.username) + + // The socket dies under a signed-out client: the reconnect has nothing to + // restore and must not invent a session. + dropSocket.Store(true) + assert.Error(t, client.Ping(ctx), "a signed-out client cannot replay through a sign-in") + dropSocket.Store(false) + + require.NoError(t, client.Connect(ctx)) + require.NoError(t, client.Ping(ctx), "the transport recovers on its own") + + var registers int + for _, read := range server.recorded() { + if read.operation() == vsr.OperationRegister { + registers++ + } + } + assert.Equal(t, 1, registers, + "only the caller's own sign-in registered; the reconnect added none") +} + +// A re-login over a dropped transport has to complete. The logout that ends +// the old session runs while the sign-in lock is held, so a logout that enters +// the reconnect path would reconnect, sign in with the remembered credentials, +// and deadlock on that same lock. +func TestFailover_ReLoginSurvivesALogoutTheTransportSwallowed(t *testing.T) { + var server *testListener + var dropLogout atomic.Bool + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + if dropLogout.Load() && read.operation() == vsr.OperationLogout { + // The frame is swallowed and the connection ends, exactly as a + // node that dies mid-logout leaves it. + return nil + } + return singleNodeHandler(t, func() string { return server.address() })(0, 0, read) + }) + + client := newDialingClient(t, server.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "iggy", "iggy") + require.NoError(t, err) + + dropLogout.Store(true) + relogin := make(chan error, 1) + go func() { + _, err := client.LoginUser(ctx, "iggy", "iggy") + relogin <- err + }() + select { + case err := <-relogin: + require.NoError(t, err, "the sign-in has to replay on the new connection") + case <-time.After(15 * time.Second): + t.Fatal("the re-login deadlocked on the sign-in lock") + } + + assert.True(t, client.session.Bound(), "the replayed sign-in bound a session") + require.NoError(t, client.Ping(ctx)) } -func TestFailover_LeavesTheReestablishPauseToSingleEndpointClients(t *testing.T) { - client := NewIggyTcpClient(slog.New(slog.DiscardHandler), - WithServerAddress("127.0.0.1:8090")) +// reestablishAfter is a cooldown on redialing the endpoint that was lost. It +// is owed to that endpoint alone, so a failover to another one must not sit +// through it. +func TestFailover_DoesNotSpendTheLostEndpointsPauseOnAnotherEndpoint(t *testing.T) { + var survivor *testListener + survivor = listenVSR(t, nil, singleNodeHandler(t, func() string { return survivor.address() })) + + client := newDialingClient(t, deadAddress(t)) client.config.reconnection.reestablishAfter = time.Minute + client.knownServerAddresses = []string{survivor.address()} + client.connectedAt = time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + started := time.Now() + require.NoError(t, client.Connect(ctx)) + + assert.Equal(t, survivor.address(), client.currentServerAddress) + assert.Less(t, time.Since(started), 2*time.Second, + "the failover waited out a pause it owed only the lost endpoint") +} + +// The other half of the same promise: WithReestablishAfter is a cooldown on +// the endpoint that was lost, and a known roster does not cancel it. +func TestFailover_KeepsTheReestablishPauseForTheEndpointThatWasLost(t *testing.T) { + var current *testListener + current = listenVSR(t, nil, singleNodeHandler(t, func() string { return current.address() })) + + client := newDialingClient(t, current.address()) + client.config.reconnection.reestablishAfter = 500 * time.Millisecond + client.knownServerAddresses = []string{deadAddress(t)} client.connectedAt = time.Now() - // One endpoint: the pause is the only thing keeping a reconnect from - // hammering the node it just lost. - require.Len(t, client.connectionCandidates(), 1) + started := time.Now() + require.NoError(t, client.Connect(context.Background())) + + assert.Equal(t, current.address(), client.currentServerAddress) + assert.GreaterOrEqual(t, time.Since(started), 350*time.Millisecond, + "the cooldown on the endpoint that was lost was skipped") +} + +// A node whose syns are dropped must not hold the sweep: without a bound on +// the dial the survivors behind it are never reached. A black-holed address +// cannot be arranged portably, so this pins the bound itself. +func TestFailover_BoundsTheDialWhenOtherEndpointsAreQueuedBehindIt(t *testing.T) { + assert.Equal(t, 2*time.Second, failoverDialTimeout, + "the dial bound has to match the other SDKs") + + var survivor *testListener + survivor = listenVSR(t, nil, singleNodeHandler(t, func() string { return survivor.address() })) + + // A listener that accepts and never answers: the dial completes out of the + // backlog, so only the bound ends the attempt. + silent, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = silent.Close() }) + + client := newDialingClient(t, silent.Addr().String(), + WithTLS(WithTLSValidateCertificate(false))) + client.knownServerAddresses = []string{survivor.address()} + + done := make(chan error, 1) + go func() { done <- client.Connect(context.Background()) }() + select { + case <-done: + case <-time.After(3 * failoverDialTimeout): + t.Fatal("the sweep never got past an endpoint that answers nothing") + } +} + +// An endpoint that accepts TCP but fails the handshake is not where this +// client lives: recording it would make the next pass lead with it and shadow +// every endpoint behind it. +func TestFailover_DoesNotSettleOnAnEndpointThatFailedTheHandshake(t *testing.T) { + hangup, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = hangup.Close() }) + go func() { + for { + connection, err := hangup.Accept() + if err != nil { + return + } + // Plain TCP behind a TLS client: the dial succeeds, the handshake + // cannot. + _ = connection.Close() + } + }() - client.knownServerAddresses = []string{"127.0.0.1:8091"} - require.Len(t, client.connectionCandidates(), 2, - "with somewhere else to go the pause only delays the failover") + configured := deadAddress(t) + client := newDialingClient(t, configured, WithTLS(WithTLSValidateCertificate(false))) + client.config.reconnection.enabled = false + client.knownServerAddresses = []string{hangup.Addr().String()} + + require.Error(t, client.Connect(context.Background())) + assert.Equal(t, configured, client.currentServerAddress, + "the endpoint that failed the handshake became the current one") +} + +// A client with nothing to dial must say so: reporting success would leave +// every request answering ErrNotConnected while Connect keeps claiming a +// connection. +func TestFailover_RejectsAConnectWithNoEndpointToDial(t *testing.T) { + client := NewIggyTcpClient(slog.New(slog.DiscardHandler), WithServerAddress("")) + t.Cleanup(func() { _ = client.Close() }) + + require.ErrorIs(t, client.Connect(context.Background()), ierror.ErrCannotEstablishConnection) + assert.Error(t, client.Ping(context.Background())) +} + +// deadAddress returns an address nothing listens on, so a dial to it is +// refused at once. +func deadAddress(t *testing.T) string { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + address := listener.Addr().String() + require.NoError(t, listener.Close()) + return address } diff --git a/foreign/go/client/tcp/tcp_session_management.go b/foreign/go/client/tcp/tcp_session_management.go index b6a91e30ea..530420e898 100644 --- a/foreign/go/client/tcp/tcp_session_management.go +++ b/foreign/go/client/tcp/tcp_session_management.go @@ -33,12 +33,12 @@ func (c *IggyTcpClient) LoginUser(ctx context.Context, username string, password if err != nil { return nil, err } - identity, err := c.register(ctx, uint32(command.LoginRegisterCode), body) - if err != nil { - return nil, err - } - c.rememberLogin(NewUsernamePasswordCredentials(username, password)) - return identity, nil + return c.register( + ctx, + uint32(command.LoginRegisterCode), + body, + NewUsernamePasswordCredentials(username, password), + ) } func (c *IggyTcpClient) LoginWithPersonalAccessToken(ctx context.Context, token string) (*iggcon.IdentityInfo, error) { @@ -46,17 +46,28 @@ func (c *IggyTcpClient) LoginWithPersonalAccessToken(ctx context.Context, token if err != nil { return nil, err } - identity, err := c.register(ctx, uint32(command.LoginRegisterWithPATCode), body) - if err != nil { - return nil, err - } - c.rememberLogin(NewPersonalAccessTokenCredentials(token)) - return identity, nil + return c.register( + ctx, + uint32(command.LoginRegisterWithPATCode), + body, + NewPersonalAccessTokenCredentials(token), + ) } // register runs the sign-in handshake, binds the session the server assigned, -// and settles the connection on the cluster leader. -func (c *IggyTcpClient) register(ctx context.Context, code uint32, body []byte) (*iggcon.IdentityInfo, error) { +// settles the connection on the cluster leader, and remembers the credentials +// it succeeded with so a reconnect can re-establish the session. +// +// The credentials are remembered here rather than by the callers because this +// is what holds registerMtx: remembered outside it, two concurrent sign-ins +// could leave A remembered while the session is B, and the next reconnect +// would sign in as A. +func (c *IggyTcpClient) register( + ctx context.Context, + code uint32, + body []byte, + credentials Credentials, +) (*iggcon.IdentityInfo, error) { // One sign-in at a time. BeginRegister runs inside the exchange lock but // Bind runs after it, so two interleaved sign-ins would let the second // BeginRegister reset the identity the first is about to bind: one @@ -80,6 +91,7 @@ func (c *IggyTcpClient) register(ctx context.Context, code uint32, body []byte) if err != nil { return nil, err } + c.rememberLogin(credentials) if settled != nil { return settled, nil } @@ -175,6 +187,14 @@ func (c *IggyTcpClient) settleOnLeader(ctx context.Context, code uint32, body [] // endBoundSession logs out a live session before a re-login, so the server // drops its client-table entry instead of leaving it to be fenced. +// +// The logout runs connect-scoped, and a failure it could recover from is +// swallowed. Both because this call holds registerMtx: a logout that entered +// the reconnect path would reconnect, sign in with the remembered credentials, +// and deadlock on that lock. There is nothing to salvage either way -- a +// session whose logout cannot be delivered died with its socket, and the +// server fences what it left behind -- and the sign-in that follows replays +// through its own reconnect. func (c *IggyTcpClient) endBoundSession(ctx context.Context) error { c.mtx.Lock() bound := c.session.Bound() @@ -182,7 +202,24 @@ func (c *IggyTcpClient) endBoundSession(ctx context.Context) error { if !bound { return nil } - return c.LogoutUser(ctx) + + err := c.LogoutUser(context.WithValue(ctx, connectScoped{}, struct{}{})) + if err == nil { + return nil + } + if !isReconnectable(err) { + return err + } + + c.logger.Debug("The bound session's logout was not delivered; its socket ended it.", + slog.Any("error", err)) + c.mtx.Lock() + c.sessionState = iggcon.SessionStateUnauthenticated + c.session.Reset() + c.groups.clear() + c.topics.clearCounts() + c.mtx.Unlock() + return nil } func (c *IggyTcpClient) LogoutUser(ctx context.Context) error { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java index 93f463e723..34fe409b4d 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java @@ -37,6 +37,7 @@ import org.apache.iggy.client.async.tcp.LeaderAwareness.LeaderRedirectionState; import org.apache.iggy.client.async.tcp.vsr.VsrFrameDecoder; import org.apache.iggy.config.RetryPolicy; +import org.apache.iggy.exception.IggyErrorCode; import org.apache.iggy.exception.IggyMissingCredentialsException; import org.apache.iggy.exception.IggyNotConnectedException; import org.apache.iggy.exception.IggyServerException; @@ -54,6 +55,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -458,6 +460,10 @@ public ConsumerOffsetsClient consumerOffsets() { */ public CompletableFuture close() { closed = true; + // Closing is caller intent, like a logout: connect() clears `closed` + // again, and a session the caller ended must not come back with the + // credentials the earlier sign-in used. + rememberedLogin = null; AsyncTcpConnection currentConnection = connection.get(); if (currentConnection != null) { return currentConnection.close(); @@ -488,10 +494,40 @@ private AsyncTcpConnection openConnection(ConnectionInfo target) { heartbeatInterval, maxVsrFrameSize, this::retryTransientOnLeader, - routingState::clearAssignments, + this::onSessionReset, this::onConnectionFailure); } + /** + * A server-side eviction reached this client. The routing state it cached + * belonged to the evicted session, and a stale-client eviction is the + * server ending that session authoritatively, like a logout: the + * remembered sign-in ends with it, so only credentials configured on the + * builder may bring the session back. + */ + private void onSessionReset(int errorCode) { + routingState.clearAssignments(); + if (errorCode != IggyErrorCode.STALE_CLIENT.getCode()) { + return; + } + // Credentials configured on the builder are what every connect of this + // client signs in as, so an eviction does not revoke them and the + // client recovers on its own. A sign-in a caller ran is different: the + // server ended that session deliberately, and reviving it behind the + // caller's back is what an explicit logout must not be able to do + // either. Dropped in both places, because the connection replays its + // own captured login to bring up a replacement channel. + if (username.isPresent() && password.isPresent()) { + return; + } + log.warn("The server evicted this session as stale; the sign-in it ran will not be replayed"); + rememberedLogin = null; + AsyncTcpConnection currentConnection = connection.get(); + if (currentConnection != null) { + currentConnection.forgetCapturedLogin(); + } + } + /** * A not-accepted request was never admitted, so it is safe to recheck the * leader, restore authentication on a new connection, and retry it within @@ -585,10 +621,12 @@ private static void releasePayload(AtomicReference payload) { * Entry point of the background redial after a pool acquire failure or an * expired reply. Requests that were in flight stay failed (their outcome * is unknown); the redial only restores the client for subsequent calls. - * Alternates the current endpoint with the seed, paced by the configured - * retry policy, and replays the builder credentials on the restored - * connection. Personal-access-token logins cannot be replayed here; those - * clients must log in again themselves. + * Each rotation sweeps every endpoint the client knows -- where it was, + * the configured seed, then the roster it learned -- and only a rotation + * that reaches none of them waits out the retry policy's delay. The + * sign-in is replayed on whichever endpoint answers, whether it was + * configured on the builder or run by the caller, personal access tokens + * included. */ private void onConnectionFailure(Throwable cause) { if (closed || !isConnectionLoss(cause)) { @@ -616,48 +654,112 @@ private CompletableFuture redialAttempt(int attempt, RetryPolicy policy) { log.error("Redial gave up after {} attempts, next request will fail fast", policy.getMaxRetries()); return CompletableFuture.completedFuture(null); } - ConnectionInfo target = ReconnectPlan.target(redialCandidates(), attempt); - Duration delay = ReconnectPlan.delay(policy, attempt); + List candidates = redialCandidates(); + // The delay paces rotations, not dials. The first rotation runs at once + // when there is somewhere else to go: pausing before dialing a survivor + // only pushes the failover past the window the caller waits in, and the + // node just lost may be gone for good. + Duration delay = attempt == 1 && candidates.size() > 1 ? Duration.ZERO : ReconnectPlan.delay(policy, attempt); Executor delayedExecutor = CompletableFuture.delayedExecutor(delay.toMillis(), TimeUnit.MILLISECONDS); - return CompletableFuture.supplyAsync(() -> null, delayedExecutor).thenCompose(ignored -> { - if (closed) { - return CompletableFuture.completedFuture(null); - } - log.info("Redial attempt {}/{} to {}", attempt, policy.getMaxRetries(), target.serverAddress()); - return retarget(target) - .thenCompose(retargeted -> replayLogin()) - .handle((ok, error) -> { - if (error == null) { - log.info("Reconnected to {}", target.serverAddress()); - return CompletableFuture.completedFuture(null); - } + return CompletableFuture.supplyAsync(() -> null, delayedExecutor) + .thenCompose(ignored -> sweepCandidates(candidates, 0, attempt, policy)); + } + + /** + * Dials one endpoint of a rotation and, if it does not come up, the next + * one. Every endpoint gets its turn inside one attempt, so a full pass over + * the cluster costs one retry rather than one per endpoint: with the + * default policy, rotating one endpoint per attempt would first dial a + * two-node survivor two delays in. + */ + private CompletableFuture sweepCandidates( + List candidates, int index, int attempt, RetryPolicy policy) { + if (closed) { + return CompletableFuture.completedFuture(null); + } + if (index >= candidates.size()) { + return redialAttempt(attempt + 1, policy); + } + ConnectionInfo target = candidates.get(index); + log.info( + "Redial attempt {}/{} to {} ({}/{})", + attempt, + policy.getMaxRetries(), + target.serverAddress(), + index + 1, + candidates.size()); + return retarget(target) + .handle((retargeted, dialError) -> { + if (dialError != null) { + log.warn("Redial to {} failed: {}", target.serverAddress(), dialError.getMessage()); + return sweepCandidates(candidates, index + 1, attempt, policy); + } + return replaySignInOn(target, candidates, index, attempt, policy); + }) + .thenCompose(Function.identity()); + } + + /** + * Re-establishes the session on an endpoint that just came up. + * + * A sign-in the server rejected -- a rotated password, an expired token -- + * ends the redial: the connection is up, no other endpoint would answer + * differently, and retrying would tear the working connection down on the + * next rotation and leave the client connected but unauthenticated anyway. + * The rejected credentials are dropped so nothing replays them. + */ + private CompletableFuture replaySignInOn( + ConnectionInfo target, List candidates, int index, int attempt, RetryPolicy policy) { + return replayLogin() + .handle((ok, loginError) -> { + if (loginError == null) { + log.info("Reconnected to {}", target.serverAddress()); + return CompletableFuture.completedFuture(null); + } + if (isConnectionLoss(unwrap(loginError))) { log.warn( - "Redial attempt {} to {} failed: {}", - attempt, + "The sign-in on {} was lost with the connection: {}", target.serverAddress(), - error.getMessage()); - return redialAttempt(attempt + 1, policy); - }) - .thenCompose(Function.identity()); - }); + loginError.getMessage()); + return sweepCandidates(candidates, index + 1, attempt, policy); + } + log.error( + "Reconnected to {} but the sign-in was rejected: {}. The connection stands" + + " unauthenticated until the caller signs in again.", + target.serverAddress(), + loginError.getMessage()); + rememberedLogin = null; + return CompletableFuture.completedFuture(null); + }) + .thenCompose(Function.identity()); + } + + private static Throwable unwrap(Throwable error) { + return error instanceof CompletionException && error.getCause() != null ? error.getCause() : error; } /** - * Replays the builder credentials on the freshly published connection. + * Re-establishes the session on the freshly published connection: with the + * credentials configured on the builder, or else the sign-in a caller ran + * by hand. Configured credentials win, as in the Rust and Go SDKs -- they + * are what every connect of this client is meant to sign in as, and a + * remembered sign-in exists to make a hand-run login as reconnectable as + * a configured one, not to override it. + * * The login runs through the users client, so leader discovery retargets * again before Register when the redialed node is not the leader. */ private CompletableFuture replayLogin() { - Supplier> replay = rememberedLogin; - if (replay != null) { - // Runs through loginOnLeader, so a redial that landed on a backup - // still settles on the leader before the session is used. - return loginOnLeader(replay).thenApply(identity -> null); + if (username.isPresent() && password.isPresent() && usersClient != null) { + return usersClient.login(username.get(), password.get()).thenApply(identity -> null); } - if (username.isEmpty() || password.isEmpty() || usersClient == null) { + Supplier> replay = rememberedLogin; + if (replay == null) { return CompletableFuture.completedFuture(null); } - return usersClient.login(username.get(), password.get()).thenApply(identity -> null); + // Runs through loginOnLeader, so a redial that landed on a backup + // still settles on the leader before the session is used. + return loginOnLeader(replay).thenApply(identity -> null); } /** @@ -788,15 +890,17 @@ CompletableFuture> findLeaderElsewhere(ConnectionInfo c /** * Endpoints a redial rotates through, likeliest first: where the client * currently is, the address it was configured with, then the roster it - * learned while connected. Duplicates are dropped, so an endpoint the - * roster merely spells differently does not earn a second attempt. + * learned while connected. Duplicates are dropped by spelling, so an + * endpoint the roster merely writes differently does not earn a second + * attempt. Spelling only: this runs on the Netty event loop, where a + * resolver lookup per candidate pair would block it. */ private List redialCandidates() { List candidates = new ArrayList<>(); candidates.add(connectionInfo); Stream.concat(Stream.of(seedConnectionInfo), rosterTargets.stream()) .filter(endpoint -> - candidates.stream().noneMatch(candidate -> LeaderAwareness.isSameAddress(candidate, endpoint))) + candidates.stream().noneMatch(candidate -> LeaderAwareness.isSameSpelling(candidate, endpoint))) .forEach(candidates::add); return List.copyOf(candidates); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java index 102e236fa9..5d815e07e2 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java @@ -68,6 +68,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.IntConsumer; /** * Async TCP connection using Netty for non-blocking I/O. @@ -97,7 +98,7 @@ public class AsyncTcpConnection { private final AtomicLong authGeneration = new AtomicLong(0); private final VsrRequestEncoder vsrEncoder; private final TransientFailoverHandler transientFailoverHandler; - private final Runnable sessionResetListener; + private final IntConsumer sessionResetListener; private final Consumer connectionFailureListener; private final long requestTimeoutNanos; private final long heartbeatIntervalNanos; @@ -127,7 +128,7 @@ public AsyncTcpConnection( Duration.ofSeconds(5), VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE, null, - () -> {}, + errorCode -> {}, ignored -> {}); } @@ -143,7 +144,7 @@ public AsyncTcpConnection( Duration heartbeatInterval, int maxVsrFrameSize, TransientFailoverHandler transientFailoverHandler, - Runnable sessionResetListener, + IntConsumer sessionResetListener, Consumer connectionFailureListener) { this.transientFailoverHandler = transientFailoverHandler; this.sessionResetListener = sessionResetListener; @@ -827,10 +828,29 @@ private void handlePostResponse(Channel channel, int commandCode, boolean isLogi * channel. Bumping the generation makes the replacement channel re-run * login and Register. The fresh session invalidates cached routing state * such as consumer-group assignments. + * + * The reason travels to the listener, which owns the question of whether + * the session may be re-established at all: only it knows whether the + * sign-in was configured on the client or run by a caller. */ - private void onSessionEvicted() { + private void onSessionEvicted(int errorCode) { authGeneration.incrementAndGet(); - sessionResetListener.run(); + sessionResetListener.accept(errorCode); + } + + /** + * Drops the captured sign-in, so the next channel comes up + * unauthenticated instead of replaying it. + * + * The channel replays the payload it captured to re-authenticate a + * replacement channel, which is right for a lost connection and wrong + * after an eviction the server decided on: that would resurrect the very + * session the server ended. + */ + void forgetCapturedLogin() { + authenticated = false; + authGeneration.incrementAndGet(); + releaseLoginPayload(); } private void captureLoginPayloadIfNeeded(int commandCode, ByteBuf payload) { @@ -894,7 +914,7 @@ private static final class PoolChannelHandler extends AbstractChannelPoolHandler private final SslContext sslContext; private final ConsensusSession consensusSession; private final int maxVsrFrameSize; - private final Runnable onEviction; + private final IntConsumer onEviction; PoolChannelHandler( String host, @@ -903,7 +923,7 @@ private static final class PoolChannelHandler extends AbstractChannelPoolHandler SslContext sslContext, ConsensusSession consensusSession, int maxVsrFrameSize, - Runnable onEviction) { + IntConsumer onEviction) { this.host = host; this.port = port; this.enableTls = enableTls; diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java index 3b258b7ef4..c316733a26 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LeaderAwareness.java @@ -217,15 +217,26 @@ static LeaderCheck checkLeader(ClusterMetadata metadata, ConnectionInfo currentT * at worst costs one redirect hop back to the same node. */ static boolean isSameAddress(ConnectionInfo target1, ConnectionInfo target2) { + if (isSameSpelling(target1, target2)) { + return true; + } if (target1.port() != target2.port()) { return false; } - var host1 = canonicalHost(target1.host()); - var host2 = canonicalHost(target2.host()); - if (host1.equals(host2)) { - return true; - } - return resolveToSameHost(host1, host2); + return resolveToSameHost(canonicalHost(target1.host()), canonicalHost(target2.host())); + } + + /** + * Whether two targets are written the same way, up to canonicalization. + * + * The cheap half of {@link #isSameAddress}, for callers that must not + * block: resolution is a synchronous DNS lookup, and the redial dedup runs + * on the Netty event loop. Two spellings of one node that only resolution + * could equate cost one wasted dial per rotation, which is not worth + * stalling an event loop for. + */ + static boolean isSameSpelling(ConnectionInfo target1, ConnectionInfo target2) { + return target1.port() == target2.port() && canonicalHost(target1.host()).equals(canonicalHost(target2.host())); } private static String canonicalHost(String host) { @@ -262,9 +273,6 @@ private static boolean reachesOnlyLocalMachine(InetAddress[] addresses) { return Arrays.stream(addresses).allMatch(address -> address.isLoopbackAddress() || address.isAnyLocalAddress()); } - /** - * One leader-check verdict from a cluster-metadata snapshot. - */ /** * What one leader check learned from the roster: where to go, and every * node the cluster named for this transport. A client keeps the latter as @@ -283,6 +291,9 @@ static LeaderLookup inconclusive() { } } + /** + * One leader-check verdict from a cluster-metadata snapshot. + */ sealed interface LeaderCheck { /** A healthy leader with an enabled tcp transport lives elsewhere; reconnect to it. */ diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java index 3e9f45bf8b..f82ee0c7d8 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java @@ -19,35 +19,17 @@ package org.apache.iggy.client.async.tcp; -import org.apache.iggy.client.ConnectionInfo; import org.apache.iggy.config.RetryPolicy; import java.time.Duration; -import java.util.List; /** - * Pure redial planning: which address to dial on a given reconnect attempt - * and how long to wait before it. + * Pure redial planning: how long to wait before a given reconnect rotation. */ final class ReconnectPlan { private ReconnectPlan() {} - /** - * Rotates reconnect dials through every endpoint the client knows, in the - * order the candidate list gives them. After a leader redirect the current - * endpoint may die with the leader, and the rest of the list -- the - * configured seed and the roster learned while connected -- is the way - * back to the rest of the cluster. Attempts are 1-based; the first dials - * the head of the list. - */ - static ConnectionInfo target(List candidates, int attempt) { - if (candidates.isEmpty()) { - throw new IllegalArgumentException("a redial needs at least one candidate endpoint"); - } - return candidates.get(Math.floorMod(attempt - 1, candidates.size())); - } - /** * The delay before the given 1-based attempt: the policy's initial delay * scaled by its multiplier per prior attempt, capped at its max delay. diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java index 113c288e27..ac182227f9 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java @@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.IntConsumer; /** * Correlates multiplexed requests by operation and request id, decodes VSR @@ -54,10 +55,10 @@ public class VsrResponseHandler extends SimpleChannelInboundHandler { private final ConcurrentMap> pendingRequests = new ConcurrentHashMap<>(); private final ConsensusSession session; - private final Runnable onEviction; + private final IntConsumer onEviction; private final AtomicReference closeCause = new AtomicReference<>(); - public VsrResponseHandler(ConsensusSession session, Runnable onEviction) { + public VsrResponseHandler(ConsensusSession session, IntConsumer onEviction) { this.session = session; this.onEviction = onEviction; } @@ -162,7 +163,11 @@ private void handleEviction(ChannelHandlerContext ctx, ByteBuf frame) { IggyServerException error = VsrHeaders.evictionToException(frame); session.reset(); try { - onEviction.run(); + // The reason travels with the notification: an eviction the server + // decided on (a stale client) ends the session authoritatively, + // while the rest are transport-shaped, and the listener has to + // tell them apart. + onEviction.accept(error.getRawErrorCode()); } catch (RuntimeException listenerError) { log.warn("Eviction listener failed: {}", listenerError.getMessage()); } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java index 1741e342d5..5e68d50a80 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java @@ -105,18 +105,23 @@ void shouldResumeOnASurvivorAfterTheSignedInNodeDies() throws Exception { return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER); }); + // No builder credentials: the replay can only come from the + // sign-in the caller ran, which is the shape that could not + // reconnect at all before. With credentials configured, the replay + // falls back to them and the test would pass without a remembered + // sign-in at all. AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() .host(loopback.getHostAddress()) .port(primaryPort) - .credentials("iggy", "iggy") - .requestTimeout(Duration.ofSeconds(5)) - // A redial rotates one endpoint per attempt, so the survivor - // is the second: keep the pacing short enough to observe. - .retryPolicy(RetryPolicy.fixedDelay(8, Duration.ofMillis(50))) + .requestTimeout(Duration.ofSeconds(2)) + // A whole rotation dials every endpoint the client knows, so + // the survivor is reached before this delay is ever spent. + // One endpoint per attempt would need it first. + .retryPolicy(RetryPolicy.fixedDelay(8, Duration.ofSeconds(5))) .build(); try { client.connect().get(5, TimeUnit.SECONDS); - client.login().get(5, TimeUnit.SECONDS); + client.users().login("iggy", "iggy").get(5, TimeUnit.SECONDS); client.sendBinaryRequest(PING_CODE, new byte[0]).get(5, TimeUnit.SECONDS); assertThat(client.getConnectionInfo().port()).isEqualTo(primaryPort); @@ -125,8 +130,8 @@ void shouldResumeOnASurvivorAfterTheSignedInNodeDies() throws Exception { // The request in flight when the node died is allowed to fail; // what is not allowed is never completing one, which is what a // client that only knows the dead endpoint does. - assertThat(resumeWithin(client, Duration.ofSeconds(10))) - .as("the client has to resume on the survivor the roster named") + assertThat(resumeWithin(client, Duration.ofSeconds(4))) + .as("the client has to resume on the survivor inside the first rotation") .isTrue(); assertThat(client.getConnectionInfo().port()) @@ -145,6 +150,69 @@ void shouldResumeOnASurvivorAfterTheSignedInNodeDies() throws Exception { } } + /** + * An explicit sign-out is caller intent, like a close: the redial after it + * must not sign back in with the credentials that sign-in used. + */ + @Test + void shouldNotResurrectASignedOutSessionOnASurvivor() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket primarySocket = new ServerSocket(0, 4, loopback); + ServerSocket survivorSocket = new ServerSocket(0, 4, loopback)) { + int primaryPort = primarySocket.getLocalPort(); + int survivorPort = survivorSocket.getLocalPort(); + AtomicInteger survivorRegistrations = new AtomicInteger(); + + MockNode primary = MockNode.serve(primarySocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, primaryPort)); + } + if (request.operation() == OPERATION_REGISTER) { + return Response.success(OPERATION_REGISTER, registerBody(1)); + } + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + MockNode survivor = MockNode.serve(survivorSocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, survivorPort)); + } + if (request.operation() == OPERATION_REGISTER) { + survivorRegistrations.incrementAndGet(); + return Response.success(OPERATION_REGISTER, registerBody(2)); + } + // Echoed, so a logout is answered as a logout: the client + // checks the reply's operation against the request's. + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(primaryPort) + .requestTimeout(Duration.ofSeconds(2)) + .retryPolicy(RetryPolicy.fixedDelay(4, Duration.ofMillis(50))) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.users().login("iggy", "iggy").get(5, TimeUnit.SECONDS); + client.users().logout().get(5, TimeUnit.SECONDS); + + primary.kill(); + + // Every attempt is allowed to fail; none of them may register. + resumeWithin(client, Duration.ofSeconds(2)); + + assertThat(survivorRegistrations) + .as("a signed-out client has no session to restore") + .hasValue(0); + } finally { + client.close().get(5, TimeUnit.SECONDS); + survivor.close(); + } + } + } + /** Retries until one request completes, or the budget runs out. */ private static boolean resumeWithin(AsyncIggyTcpClient client, Duration budget) throws InterruptedException { long deadline = System.nanoTime() + budget.toNanos(); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java index ba481bec98..fe32b9cece 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java @@ -159,6 +159,61 @@ void shouldReplayTransientImplicitLoginAfterEviction() throws Exception { } } + /** + * A stale-client eviction is the server ending the session + * authoritatively, like a logout. A client whose credentials were + * configured signs in again on every connect and recovers (the test + * above); one whose session came from a caller's own sign-in must not have + * that session revived behind the caller's back. + */ + @Test + void shouldNotReviveAHandRunSignInAfterAStaleClientEviction() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { + AtomicInteger registrations = new AtomicInteger(); + CompletableFuture server = serve(serverSocket, 4, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); + } + if (request.operation() == OPERATION_REGISTER) { + return Response.success(OPERATION_REGISTER, registerBody(registrations.incrementAndGet())); + } + if (request.operation() == OPERATION_CREATE_STREAM) { + return Response.eviction(EVICTION_STALE_CLIENT); + } + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + + // No configured credentials: the only sign-in is the one run below. + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(serverSocket.getLocalPort()) + .requestTimeout(Duration.ofSeconds(2)) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.users().login("iggy", "iggy").get(5, TimeUnit.SECONDS); + int registrationsBeforeEviction = registrations.get(); + + assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .hasCauseInstanceOf(IggyServerException.class); + + // Whatever the caller does next, nothing may sign this session + // back in on its own. + assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .isNotNull(); + assertThat(registrations) + .as("the evicted session was signed back in") + .hasValue(registrationsBeforeEviction); + } finally { + client.close().get(5, TimeUnit.SECONDS); + } + server.completeExceptionally(new IllegalStateException("test over")); + } + } + private static Response handleOldLeader( Request request, int oldLeaderPort, int newLeaderPort, AtomicInteger denials) { if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java index 409e831474..354f09d95b 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java @@ -111,7 +111,7 @@ void shouldCorrelateConcurrentPartitionResponsesInReverseOrder() throws Exceptio Duration.ofHours(1), 1024 * 1024, null, - () -> {}, + errorCode -> {}, ignored -> {}); try { connection.connect().get(5, TimeUnit.SECONDS); @@ -308,7 +308,7 @@ private static AsyncTcpConnection newConnection( heartbeatInterval, 1024 * 1024, null, - () -> {}, + errorCode -> {}, ignored -> {}); } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java index b5b73cfb78..a847acb1c7 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LeaderAwarenessTest.java @@ -318,6 +318,79 @@ void shouldStayWithoutPollingWhenAlreadyOnLeader() { } } + @Nested + class NodeTargets { + + // A node with the tcp transport disabled cannot be dialed over tcp, so + // it must not join the redial candidates: dialing port 0 fails, and it + // would spend one turn of every rotation. + @Test + void shouldSkipNodesWithoutATcpEndpoint() { + var metadata = cluster( + node("tcp-node", "iggy-0", 8091, ClusterNodeRole.Leader, ClusterNodeStatus.Healthy), + node("http-only-node", "iggy-1", 0, ClusterNodeRole.Follower, ClusterNodeStatus.Healthy)); + + var targets = LeaderAwareness.nodeTargets(metadata); + + assertThat(targets).containsExactly(new ConnectionInfo("iggy-0", 8091)); + } + + // Unhealthy nodes stay: a node that is down now is where the cluster + // says it lives, and a redial candidate is a place to try, not a + // promise that it answers. + @Test + void shouldKeepUnhealthyNodesThatStillHaveATcpEndpoint() { + var metadata = cluster( + node("leader-node", "iggy-0", 8091, ClusterNodeRole.Leader, ClusterNodeStatus.Healthy), + node("down-node", "iggy-1", 8092, ClusterNodeRole.Follower, ClusterNodeStatus.Unreachable)); + + var targets = LeaderAwareness.nodeTargets(metadata); + + assertThat(targets).containsExactly(new ConnectionInfo("iggy-0", 8091), new ConnectionInfo("iggy-1", 8092)); + } + + // An inconclusive check names no endpoint, which is what lets the + // client keep the last roster it read: replacing it with an empty list + // would erase the candidates exactly when the cluster is unreachable. + @Test + void shouldNameNoEndpointWhenTheRosterCannotBeRead() { + var lookup = LeaderAwareness.LeaderLookup.inconclusive(); + + assertThat(lookup.redirect()).isEmpty(); + assertThat(lookup.endpoints()).isEmpty(); + } + } + + @Nested + class SameAddress { + + // The redial dedup runs on the Netty event loop, so it compares + // spellings only. A hostname and the address it resolves to are two + // candidates there, and one endpoint for the leader check. + @Test + void shouldCompareSpellingsWithoutResolving() { + assertThat(LeaderAwareness.isSameSpelling( + new ConnectionInfo("IGGY-0", 8090), new ConnectionInfo("iggy-0", 8090))) + .isTrue(); + assertThat(LeaderAwareness.isSameSpelling( + new ConnectionInfo("[::1]", 8090), new ConnectionInfo("::1", 8090))) + .isTrue(); + assertThat(LeaderAwareness.isSameSpelling( + new ConnectionInfo("localhost", 8090), new ConnectionInfo("127.0.0.1", 8090))) + .isFalse(); + assertThat(LeaderAwareness.isSameSpelling( + new ConnectionInfo("iggy-0", 8090), new ConnectionInfo("iggy-0", 8091))) + .isFalse(); + } + + @Test + void shouldTreatLoopbackSpellingsAsOneEndpointForTheLeaderCheck() { + assertThat(LeaderAwareness.isSameAddress( + new ConnectionInfo("localhost", 8090), new ConnectionInfo("127.0.0.1", 8090))) + .isTrue(); + } + } + @Nested class RedirectionState { diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java index fff6114d30..0c45587b77 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java @@ -19,43 +19,15 @@ package org.apache.iggy.client.async.tcp; -import org.apache.iggy.client.ConnectionInfo; import org.apache.iggy.config.RetryPolicy; import org.junit.jupiter.api.Test; import java.time.Duration; -import java.util.List; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; class ReconnectPlanTest { - private final ConnectionInfo seed = new ConnectionInfo("seed-node", 8090); - private final ConnectionInfo current = new ConnectionInfo("leader-node", 8090); - private final ConnectionInfo survivor = new ConnectionInfo("survivor-node", 8090); - - @Test - void shouldRotateThroughEveryKnownEndpoint() { - var candidates = List.of(current, seed, survivor); - - assertThat(ReconnectPlan.target(candidates, 1)).isEqualTo(current); - assertThat(ReconnectPlan.target(candidates, 2)).isEqualTo(seed); - assertThat(ReconnectPlan.target(candidates, 3)).isEqualTo(survivor); - assertThat(ReconnectPlan.target(candidates, 4)).isEqualTo(current); - } - - @Test - void shouldDialOnlyOneAddressWhenNeverRedirected() { - assertThat(ReconnectPlan.target(List.of(seed), 1)).isEqualTo(seed); - assertThat(ReconnectPlan.target(List.of(seed), 2)).isEqualTo(seed); - } - - @Test - void shouldRefuseToPlanARedialWithoutCandidates() { - assertThatThrownBy(() -> ReconnectPlan.target(List.of(), 1)).isInstanceOf(IllegalArgumentException.class); - } - @Test void shouldKeepFixedDelayConstant() { var policy = RetryPolicy.fixedDelay(12, Duration.ofSeconds(5)); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java index e8069b9159..4a9e3bc2d6 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java @@ -39,7 +39,11 @@ class VsrResponseHandlerTest { private final ConsensusSession session = new ConsensusSession(); private final AtomicInteger evictions = new AtomicInteger(); - private final VsrResponseHandler handler = new VsrResponseHandler(session, evictions::incrementAndGet); + private final AtomicInteger lastEvictionReason = new AtomicInteger(); + private final VsrResponseHandler handler = new VsrResponseHandler(session, errorCode -> { + evictions.incrementAndGet(); + lastEvictionReason.set(errorCode); + }); private final EmbeddedChannel channel = new EmbeddedChannel(handler); @AfterEach @@ -159,6 +163,9 @@ void shouldMapEvictionReasonAndResetSession() { assertThat(rawErrorCode(future)).isEqualTo(42); assertThat(session.isBound()).isFalse(); assertThat(evictions).hasValue(1); + // The reason reaches the listener, which has to tell an eviction the + // server decided on from a transport-shaped one. + assertThat(lastEvictionReason).hasValue(42); } @Test @@ -171,6 +178,7 @@ void shouldCloseChannelOnEvictionWithoutPendingRequest() { assertThat(session.isBound()).isFalse(); assertThat(evictions).hasValue(1); + assertThat(lastEvictionReason).hasValue(VsrHeaders.ERROR_STALE_CLIENT); assertThat(frame.refCnt()).isZero(); assertThat(channel.isActive()).isFalse(); } diff --git a/foreign/node/src/client/client.connection.test.ts b/foreign/node/src/client/client.connection.test.ts index ed6787df44..eb1b353e50 100644 --- a/foreign/node/src/client/client.connection.test.ts +++ b/foreign/node/src/client/client.connection.test.ts @@ -418,6 +418,123 @@ describe('IggyConnection', () => { } ); + it('dials the endpoint it is on, then the seed, then the roster', + async () => { + const seed = await startServer(); + const seedPort = (seed.address() as AddressInfo).port; + const connection = new IggyConnection(connectionConfig(seed)); + connection.on('error', () => undefined); + try { + // A redirect moves the client off its seed; the seed is still the one + // endpoint the caller vouched for, so it comes before a roster the + // cluster may have reshaped since. + connection.config.options = { + ...connection.config.options, + port: seedPort + 9 + }; + connection.rememberRoster([{ host: '127.0.0.1', port: seedPort + 5 }]); + + assert.deepEqual( + connection._redialCandidates().map((options) => options.port), + [seedPort + 9, seedPort, seedPort + 5] + ); + } finally { + connection._destroy(); + await new Promise((resolve) => seed.close(() => resolve())); + } + } + ); + + it('counts endpoints that only differ in spelling once', + async () => { + const seed = await startServer(); + const seedPort = (seed.address() as AddressInfo).port; + const connection = new IggyConnection(connectionConfig(seed)); + connection.on('error', () => undefined); + try { + // The loopback aliases and an IPv4-mapped address all name the endpoint + // the client is already on, so none of them earns a dial of its own. + connection.rememberRoster([ + { host: 'localhost', port: seedPort }, + { host: '::1', port: seedPort }, + { host: '::ffff:127.0.0.1', port: seedPort }, + { host: '127.0.0.1', port: seedPort + 1 } + ]); + + assert.deepEqual( + connection._redialCandidates().map((options) => options.port), + [seedPort, seedPort + 1] + ); + } finally { + connection._destroy(); + await new Promise((resolve) => seed.close(() => resolve())); + } + } + ); + + it('stops a redial pass that is destroyed part-way through', + async () => { + // The endpoint the client is on is dead, so every dial to it is refused + // - and the live roster endpoint behind it is what the pass would reach + // next, unless the destroy in between stops the pass. + const dead = await startServer(); + const deadPort = (dead.address() as AddressInfo).port; + await new Promise((resolve) => dead.close(() => resolve())); + const live = await startServer(); + const livePort = (live.address() as AddressInfo).port; + let accepted = 0; + live.on('connection', () => { accepted += 1; }); + + const connection = new IggyConnection({ + transport: 'TCP', + options: { host: '127.0.0.1', port: deadPort }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: true, interval: 10, maxRetries: 3 }, + maxResponseFrameSize: FRAME_LIMIT + }); + let destroyed = false; + let connectsAfterDestroy = 0; + connection.on('connect', () => { + if (destroyed) + connectsAfterDestroy += 1; + }); + try { + connection.rememberRoster([{ host: '127.0.0.1', port: livePort }]); + + // The first failure is the initial connect, which is what starts the + // redial pass; the next one is that pass's first candidate, so + // destroying there lands between two candidates rather than before the + // pass. + const destroyedMidPass = new Promise((resolve) => { + let failures = 0; + connection.on('error', () => { + failures += 1; + if (failures < 2 || destroyed) + return; + connection._destroy(); + destroyed = true; + resolve(); + }); + }); + + await connection.connect().catch(() => undefined); + await destroyedMidPass; + await new Promise((resolve) => setTimeout(resolve, 100)); + + assert.equal(accepted, 0, + 'a destroyed connection must not keep dialing the rest of the pass' + ); + assert.equal(connectsAfterDestroy, 0, + 'a destroyed connection must not announce a connection' + ); + assert.equal(connection.connected, false); + } finally { + connection._destroy(); + await new Promise((resolve) => live.close(() => resolve())); + } + } + ); + it('settles a dial in flight when a redirect replaces the socket', async () => { const seed = await startServer(); diff --git a/foreign/node/src/client/client.connection.ts b/foreign/node/src/client/client.connection.ts index bd37ac31a0..16269a1538 100644 --- a/foreign/node/src/client/client.connection.ts +++ b/foreign/node/src/client/client.connection.ts @@ -67,6 +67,16 @@ const getTransport = (config: ClientConfig): Socket => { } }; +/** One node of the cluster, as a redial candidate. */ +export type Endpoint = { host: string, port: number }; + +/** + * Bound on one dial while other endpoints are queued behind it. A socket has + * no connect deadline of its own, so a node whose syns are dropped would hold + * the whole pass. Matches the Rust SDK. + */ +const FAILOVER_DIAL_TIMEOUT_MS = 2_000; + /** * Default reconnection settings. * Attempts reconnection every 5 seconds, up to 12 times. @@ -124,7 +134,7 @@ export class IggyConnection extends EventEmitter { * exactly when it is needed, so it has to have been remembered while the * connection was still healthy. */ - private rosterEndpoints: { host: string, port: number }[]; + private rosterEndpoints: Endpoint[]; /** Incremental response frame decoder */ private responseDecoder: ResponseFrameDecoder; @@ -235,6 +245,38 @@ export class IggyConnection extends EventEmitter { return connectPromise; } + /** + * Waits for one dial, bounded while other endpoints are queued behind it. + * + * A socket has no connect deadline of its own and there is no 'timeout' + * listener on it, so a node whose syns are dropped holds the pass for the + * whole OS connect timeout -- and it leads every pass, because the current + * endpoint only moves on success. The bound matches the Rust SDK's. + */ + private async _dialWithin(socket: Socket, bounded: boolean): Promise { + if (!bounded) + return this._waitForConnection(socket); + + let expire: NodeJS.Timeout | undefined; + const bound = new Promise((_resolve, reject) => { + expire = setTimeout(() => { + // Destroying it makes the pending dial settle and releases the handle; + // left alone it would keep the event loop alive. + socket.destroy(); + reject(new Error( + `dial exceeded ${FAILOVER_DIAL_TIMEOUT_MS}ms` + )); + }, FAILOVER_DIAL_TIMEOUT_MS); + expire.unref?.(); + }); + + try { + return await Promise.race([this._waitForConnection(socket), bound]); + } finally { + clearTimeout(expire); + } + } + private _waitForConnection(socket: Socket): Promise { return new Promise((resolve, reject) => { const cleanup = () => { @@ -308,10 +350,18 @@ export class IggyConnection extends EventEmitter { ): Promise { let lastError = initialError; let expectedSocket = this.socket; + let firstPass = true; while (enabled && this.reconnectCount < maxRetries) { this.connecting = true; this.reconnectCount += 1; - await waitForReconnect(interval); + const candidates = this._redialCandidates(); + // The backoff paces retries against a single endpoint. With other + // endpoints known there is somewhere else to go, and pausing first only + // pushes the failover past the interval a caller is willing to wait; + // later passes still back off. + if (!firstPass || candidates.length === 1) + await waitForReconnect(interval); + firstPass = false; if (this.ending) throw new Error('connection is closed', { cause: lastError }); // A redirect may replace the socket at any point. Defer to the active @@ -323,14 +373,27 @@ export class IggyConnection extends EventEmitter { // the cluster costs one retry rather than one per endpoint: a pass that // stopped at the first refusal would never reach the survivors of a // client configured for a single retry. - for (const options of this._redialCandidates()) { + for (const options of candidates) { + // Re-checked every iteration, not once above the loop: a destroy or a + // redirect mid-pass has to stop the pass. Left running, the next + // endpoint that answers would leave an open socket nobody closes, a + // 'connect' event after the destroy, and the process alive. + if (this.ending) + throw new Error('connection is closed', { cause: lastError }); + if (this.socket !== expectedSocket) + return this.connect(); + const socket = this._installSocket( getTransport({ ...this.config, options }) ); this.socket = socket; expectedSocket = socket; try { - await this._waitForConnection(socket); + await this._dialWithin(socket, candidates.length > 1); + if (this.ending) { + socket.destroy(); + throw new Error('connection is closed', { cause: lastError }); + } if (this.socket !== socket) return this.connect(); this.config.options = options; @@ -358,7 +421,7 @@ export class IggyConnection extends EventEmitter { * answer about where its nodes are, so a node it dropped stops being * dialed. The configured seed is kept separately and outlives it. */ - rememberRoster(endpoints: { host: string, port: number }[]): void { + rememberRoster(endpoints: Endpoint[]): void { if (endpoints.length === 0) return; this.rosterEndpoints = endpoints; @@ -369,8 +432,13 @@ export class IggyConnection extends EventEmitter { * currently is, the endpoint it was configured with, then the roster it * learned while connected. After a leader redirect the current endpoint may * die with the leader, and the rest of the list is the way back to the - * cluster. Duplicates are dropped, so an endpoint the roster merely spells - * differently does not earn a second attempt. + * cluster. + * + * Duplicates are dropped by spelling: the loopback aliases and an + * IPv4-mapped IPv6 address collapse onto one endpoint. Names are not + * resolved, so a seed given as a DNS name and the roster's IP for the same + * node still count as two candidates -- one wasted dial per pass, not a + * correctness problem. */ _redialCandidates(): ClientConfig['options'][] { const candidates = [this.config.options]; @@ -382,8 +450,8 @@ export class IggyConnection extends EventEmitter { ]; for (const candidate of known) { const duplicate = candidates.some( - (known) => known.port === candidate.port && - normalizeHost(known.host) === normalizeHost(candidate.host) + (existing) => existing.port === candidate.port && + normalizeHost(existing.host) === normalizeHost(candidate.host) ); if (!duplicate) candidates.push(candidate); diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts index 1f93a1555e..3a6719a740 100644 --- a/foreign/node/src/client/client.socket.test.ts +++ b/foreign/node/src/client/client.socket.test.ts @@ -574,18 +574,18 @@ describe('VSR client socket', () => { socket.destroy(); await primary.close(); - // The attempt in flight when the socket died is allowed to fail; what - // is not allowed is never completing one, which is what a client that - // only knows the dead endpoint does. + // The attempt in flight when the socket died is allowed to fail; the + // one after it has to land on the survivor. Two attempts, not a + // polling loop: the comment above promises at most one failed + // submission, and a loop of twenty would pass with nineteen failures. let resumed = false; let lastError: unknown; - for (let attempt = 0; attempt < 20 && !resumed; attempt += 1) { + for (let attempt = 0; attempt < 2 && !resumed; attempt += 1) { try { await client.sendCommand(60_021, Buffer.alloc(0)); resumed = true; } catch (error) { lastError = error; - await new Promise((resolve) => setTimeout(resolve, 10)); } } assert.ok(resumed, `the client never resumed: ${String(lastError)}`); diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts index 2aa8d1dfc6..746f43e2bd 100644 --- a/foreign/node/src/client/client.socket.ts +++ b/foreign/node/src/client/client.socket.ts @@ -44,6 +44,23 @@ const LEADERLESS_POLL_INTERVAL_MS = 250; const MAX_LEADER_REDIRECTS = 3; const TRANSIENT_NOT_COMMITTED = 57; const TRANSIENT_NOT_ACCEPTED = 58; +/** + * How long a `TRANSIENT_NOT_ACCEPTED` request replays on the same connection + * before the roster is re-read. A node that stopped being primary refuses + * forever, so replaying alone never recovers. Matches the Rust SDK. + */ +const VSR_FAILOVER_CHECK_MS = 2_000; + +/** + * A request the current node keeps refusing as not-admitted. Carries the + * refusal so the caller can surface it when the roster turns out to still name + * this node as the leader. Never escapes `sendCommand`. + */ +class LeaderMovedError extends Error { + constructor(readonly refusal: ResponseError) { + super('the node refused the request as not-admitted; re-reading the roster'); + } +} /** * Command codes that can be executed without authentication. @@ -186,21 +203,24 @@ export class CommandResponseStream extends EventEmitter { if (!this.isAuthenticated && !this.isUnloggedCommand(command)) await this.authenticate(this.options.credentials); - const response = await new Promise( - (resolve, reject) => { - const job = { - command, - payload, - handleResponse, - resolve, - reject - }; - if (last) - this._execQueue.push(job); - else - this._execQueue.unshift(job); - this._processQueue(); - }); + // The roster read is itself a queued command and the queue is + // single-flighted, so the leader re-check cannot happen inside + // `_processVsr`. The refusal comes back out here instead, where the + // queue is free, and the command is re-issued on the node that now + // leads. + let response: CommandResponse; + for (let move = 0; ; move += 1) { + try { + response = await this._queueCommand(command, payload, handleResponse, + last); + break; + } catch (error) { + if (!(error instanceof LeaderMovedError)) + throw error; + if (move >= MAX_LEADER_REDIRECTS || !await this._followLeaderMove()) + throw responseError(command, error.refusal.errorCode); + } + } if (!isLoginCommand(command) || this.settlingLeader) return response; this.settlingLeader = true; @@ -216,6 +236,57 @@ export class CommandResponseStream extends EventEmitter { } } + private _queueCommand( + command: number, + payload: Buffer, + handleResponse: boolean, + last: boolean + ): Promise { + return new Promise((resolve, reject) => { + const job = { + command, + payload, + handleResponse, + resolve, + reject + }; + if (last) + this._execQueue.push(job); + else + this._execQueue.unshift(job); + this._processQueue(); + }); + } + + /** + * Re-reads the roster and moves to the leader it names. + * + * @returns Whether the client moved, so the refused request is worth + * re-issuing + */ + private async _followLeaderMove(): Promise { + const leader = await this._readLeaderEndpoint(); + if (!leader || this.connection.isConnectedTo(leader.host, leader.port)) + return false; + debug(`the leader moved to ${leader.host}:${leader.port}, following it`); + await this.connection.redirect(leader.host, leader.port); + return true; + } + + private _rememberRoster(response: CommandResponse): void { + try { + const metadata = GET_CLUSTER_METADATA.deserialize(response); + this.connection.rememberRoster( + metadata.nodes + .filter((node) => node.endpoints.tcp !== 0) + .map((node) => ({ host: node.ip, port: node.endpoints.tcp })) + ); + } catch (error) { + debug('an unreadable roster leaves the redial candidates as they are', + error); + } + } + /** * Processes queued commands sequentially. * Emits 'finishQueue' when all commands are processed. @@ -285,7 +356,9 @@ export class CommandResponseStream extends EventEmitter { const prepared = prepareVsrCommand(command, payload); // A transient retry must preserve all request identity fields. const frame = this.vsrSession.encode(prepared.command, prepared.payload); - const deadline = Date.now() + VSR_RESPONSE_TIMEOUT_MS; + const startedAt = Date.now(); + const deadline = startedAt + VSR_RESPONSE_TIMEOUT_MS; + const notAcceptedDeadline = startedAt + VSR_FAILOVER_CHECK_MS; let lastTransientError: ResponseError | undefined; let parsed: CommandResponse; while (true) { @@ -316,6 +389,16 @@ export class CommandResponseStream extends EventEmitter { !isTransientVsrError(error.errorCode)) throw error; lastTransientError = error; + // A not-admitted refusal is a statement about who leads, not about + // load: a node that stopped being primary refuses forever, so + // replaying on this connection never recovers. Hand it back for a + // roster re-read once the window is spent. Not-committed (57) stays + // here: the request is in flight on this very node, and its outcome + // is unknown anywhere else. + if (error.errorCode === TRANSIENT_NOT_ACCEPTED && + !isLoginCommand(command) && + Date.now() >= notAcceptedDeadline) + throw new LeaderMovedError(error); const retryDelay = Math.min( VSR_RETRY_INTERVAL_MS, Math.max(0, deadline - Date.now()) @@ -335,8 +418,18 @@ export class CommandResponseStream extends EventEmitter { if (prepared.command === COMMAND_CODE.LogoutUser) { this._resetSession(); } + // Every roster read feeds the redial candidates, whoever asked for it + // and whatever it says: a node dies together with its address, the + // roster is unreachable exactly when it is needed, and reading it only + // during a login would leave the candidates stale between logins. + if (handleResp && command === GET_CLUSTER_METADATA.code) + this._rememberRoster(parsed); return parsed; } catch (error) { + // A not-admitted refusal is an answer, so the session is not in doubt + // and the request was never applied. + if (error instanceof LeaderMovedError) + throw error; // Once bytes were handed to the socket, a local transport or decode // failure leaves the request outcome ambiguous. Register a fresh session // rather than replaying that request under a different client identity. @@ -477,14 +570,10 @@ export class CommandResponseStream extends EventEmitter { GET_CLUSTER_METADATA.serialize(), { last: false } ); + // The redial candidates are fed by `_processVsr` for every roster + // read, leaderless ones included: a roster with no leader still names + // where the nodes are. const metadata = GET_CLUSTER_METADATA.deserialize(response); - // Every read feeds the redial candidates, leaderless ones included: a - // roster with no leader still names where the nodes are. - this.connection.rememberRoster( - metadata.nodes - .filter((node) => node.endpoints.tcp !== 0) - .map((node) => ({ host: node.ip, port: node.endpoints.tcp })) - ); if (metadata.nodes.length <= 1) return undefined; const leader = metadata.nodes.find( From e62f2dde15ed959784cd3a00d32e9c4d64f7e29e Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 25 Aug 2026 13:36:22 +0200 Subject: [PATCH 07/16] fix CI --- bdd/rust/tests/helpers/cluster.rs | 19 +++++++++++++++++++ bdd/rust/tests/steps/leader_redirection.rs | 14 ++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/bdd/rust/tests/helpers/cluster.rs b/bdd/rust/tests/helpers/cluster.rs index 195bcb9a83..8a954c22af 100644 --- a/bdd/rust/tests/helpers/cluster.rs +++ b/bdd/rust/tests/helpers/cluster.rs @@ -17,6 +17,7 @@ use iggy::prelude::*; use std::env; +use std::net::{SocketAddr, ToSocketAddrs}; use std::sync::Arc; /// Resolves server address based on role and port, checking environment variables first @@ -49,6 +50,24 @@ pub async fn create_and_connect_client(addr: &str) -> IggyClient { IggyClient::create(ClientWrapper::Tcp(client), None, None) } +/// Whether two `host:port` spellings name the same endpoint. +/// +/// A client that never redirected still holds the address it was given (a +/// host name, in the BDD compose network), while a redirected one holds the +/// address the roster published (an IP). Both name the same node, so they +/// are compared once resolved, like the Go and Java suites do. +pub fn is_same_endpoint(left: &str, right: &str) -> Result { + let resolve = |address: &str| -> Result, String> { + address + .to_socket_addrs() + .map(Iterator::collect) + .map_err(|error| format!("Failed to resolve server address {address}: {error}")) + }; + let left = resolve(left)?; + let right = resolve(right)?; + Ok(left.iter().any(|candidate| right.contains(candidate))) +} + /// Verifies that a client is connected to the expected port pub async fn verify_client_connection( client: &IggyClient, diff --git a/bdd/rust/tests/steps/leader_redirection.rs b/bdd/rust/tests/steps/leader_redirection.rs index 1f6bcc5ad5..c035bd848b 100644 --- a/bdd/rust/tests/steps/leader_redirection.rs +++ b/bdd/rust/tests/steps/leader_redirection.rs @@ -288,10 +288,16 @@ async fn then_both_use_same_server(world: &mut LeaderContext) { let conn_info_a = client_a.get_connection_info().await; let conn_info_b = client_b.get_connection_info().await; - // Verify both clients are connected to the same server - assert_eq!( - conn_info_a.server_address, conn_info_b.server_address, - "Both clients should be connected to the same server" + // Verify both clients are connected to the same server. Client A holds + // the address it was configured with and client B the one the roster + // published for the leader, so the spellings differ even when the node + // is the same. + assert!( + cluster::is_same_endpoint(&conn_info_a.server_address, &conn_info_b.server_address) + .expect("Server addresses should resolve"), + "Both clients should be connected to the same server, got {} and {}", + conn_info_a.server_address, + conn_info_b.server_address ); // Verify both can communicate From 3a0d9d5654ac1eefb1a03e08e5dd9cc6bf8ef614 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 25 Aug 2026 17:47:59 +0200 Subject: [PATCH 08/16] address review comments --- core/common/src/traits/binary_transport.rs | 11 +- .../tests/sdk/disconnect_relogin.rs | 4 +- core/sdk/src/leader_aware.rs | 143 ++++--- core/sdk/src/tcp/tcp_client.rs | 356 ++++++++++++++++-- .../Implementations/TcpMessageStream.cs | 67 +++- .../VsrTests/EndpointFailoverTests.cs | 64 +++- foreign/go/client/tcp/tcp_connect_test.go | 2 +- foreign/go/client/tcp/tcp_core.go | 87 ++++- foreign/go/client/tcp/tcp_failover_test.go | 195 ++++++++++ .../go/client/tcp/tcp_session_management.go | 5 + .../client/async/tcp/AsyncIggyTcpClient.java | 108 ++++-- .../client/async/tcp/AsyncTcpConnection.java | 15 +- ...syncIggyTcpClientEndpointFailoverTest.java | 53 +++ ...yncIggyTcpClientTransientFailoverTest.java | 132 ++++++- .../node/src/client/client.connection.test.ts | 85 +++++ foreign/node/src/client/client.connection.ts | 48 ++- foreign/node/src/client/client.socket.test.ts | 32 ++ foreign/node/src/client/client.socket.ts | 101 +++-- foreign/node/src/client/client.type.ts | 11 +- 19 files changed, 1346 insertions(+), 173 deletions(-) diff --git a/core/common/src/traits/binary_transport.rs b/core/common/src/traits/binary_transport.rs index 209ce2d580..f9e1965645 100644 --- a/core/common/src/traits/binary_transport.rs +++ b/core/common/src/traits/binary_transport.rs @@ -62,9 +62,14 @@ pub trait VsrSessionControl: vsr_session_sealed::Sealed + BinaryTransport { /// and a reconnect must not resurrect one. async fn forget_session_credentials(&self) {} /// A committed password change for `user`: when it is the signed-in user, - /// the remembered credentials switch to the new password, or the next - /// reconnect would sign in with the old one and fail an unrelated request - /// with `InvalidCredentials`. Other users' changes are ignored. + /// the credentials the next reconnect signs in with switch to the new + /// password, or that reconnect would replay the old one and fail an + /// unrelated request with `InvalidCredentials`. Other users' changes are + /// ignored. + /// + /// This covers a configured `AutoLogin` as well as a sign-in the caller + /// ran: the configured credentials still decide *who* the client signs in + /// as, and a committed change decides what that user's password is. async fn refresh_session_password(&self, _user: &Identifier, _new_password: &str) {} /// SDK crate version sent in the login-register version prefix. /// Implemented by the transports so the value is the SDK crate's own diff --git a/core/integration/tests/sdk/disconnect_relogin.rs b/core/integration/tests/sdk/disconnect_relogin.rs index 022bf7f94d..956e17c5a2 100644 --- a/core/integration/tests/sdk/disconnect_relogin.rs +++ b/core/integration/tests/sdk/disconnect_relogin.rs @@ -41,8 +41,8 @@ async fn given_a_logged_in_client_when_explicitly_disconnected_should_require_a_ assert!( matches!(client.get_me().await, Err(IggyError::Unauthenticated)), "an explicit disconnect is caller intent, like a logout: the sign-in it ended \ - must not be silently replayed by the reconnect, so the server sees an \ - unauthenticated request" + must not be silently replayed by the reconnect, so the client's own \ + authentication gate refuses the request before it is sent" ); client.disconnect().await.unwrap(); diff --git a/core/sdk/src/leader_aware.rs b/core/sdk/src/leader_aware.rs index 6e2b2a16b1..66cdad35f3 100644 --- a/core/sdk/src/leader_aware.rs +++ b/core/sdk/src/leader_aware.rs @@ -20,7 +20,7 @@ use iggy_common::ClusterClient; use iggy_common::{ ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, IggyError, TransportProtocol, }; -use std::net::{SocketAddr, ToSocketAddrs}; +use std::net::SocketAddr; use std::str::FromStr; use tracing::{debug, info, warn}; @@ -82,7 +82,7 @@ pub async fn check_and_redirect_to_leader( metadata.name ); let endpoints = transport_endpoints(&metadata, transport); - match process_cluster_metadata(&metadata, current_address, transport) { + match process_cluster_metadata(&metadata, current_address, transport).await { Outcome::Redirect(address) => { return Ok(LeaderCheck { redirect: Some(address), @@ -171,7 +171,7 @@ fn transport_port(node: &ClusterNode, transport: TransportProtocol) -> u16 { } /// Process cluster metadata and determine if redirection is needed -fn process_cluster_metadata( +async fn process_cluster_metadata( metadata: &ClusterMetadata, current_address: &str, transport: TransportProtocol, @@ -200,7 +200,7 @@ fn process_cluster_metadata( leader_node.name, leader_address, transport ); - if !is_same_address(current_address, &leader_address) { + if !is_same_address(current_address, &leader_address).await { info!( "Current connection to {} is not the leader, will redirect to {}", current_address, leader_address @@ -215,40 +215,67 @@ fn process_cluster_metadata( } } +/// Whether two addresses are written the same way, up to canonicalization +/// (`localhost` and `[::]` spellings, and a literal address compared as an +/// address rather than as text). +/// +/// Cheap and non-blocking, which is the whole point: the resolving comparison +/// below is a `getaddrinfo`, and every caller reaches this first. +pub(crate) fn is_same_spelling(addr1: &str, addr2: &str) -> bool { + match (parse_address(addr1), parse_address(addr2)) { + (Some(sock1), Some(sock2)) => sock1.ip() == sock2.ip() && sock1.port() == sock2.port(), + _ => normalize_address(addr1) == normalize_address(addr2), + } +} + /// Check if two addresses refer to the same endpoint /// Handles various formats like 127.0.0.1:8090 vs localhost:8090 /// /// A host name and the address it resolves to are one endpoint too: a client /// configured as `iggy-server:8090` whose roster advertises `10.0.0.5:8090` /// would otherwise dial that node twice per failover sweep, and a single-node -/// deployment would be treated as a cluster. Resolution is the last resort, -/// only when the spellings differ and at least one side is not a literal -/// address, and it is a blocking lookup: this runs on the connect and redirect -/// paths, which are rare and already wait on the network. -pub(crate) fn is_same_address(addr1: &str, addr2: &str) -> bool { - match (parse_address(addr1), parse_address(addr2)) { - (Some(sock1), Some(sock2)) => sock1.ip() == sock2.ip() && sock1.port() == sock2.port(), - (parsed1, parsed2) => { - if normalize_address(addr1) == normalize_address(addr2) { - return true; - } - if parsed1.is_some() && parsed2.is_some() { - return false; - } - resolve_all(addr1) - .zip(resolve_all(addr2)) - .is_some_and(|(first, second)| { - first.iter().any(|resolved| second.contains(resolved)) - }) - } +/// deployment would be treated as a cluster. +/// +/// Resolution is the last resort, only when the spellings differ and at least +/// one side is not a literal address. It runs through the runtime's resolver +/// rather than `ToSocketAddrs`: name lookup is a blocking `getaddrinfo`, and +/// this is called from the connect and redirect paths, where stalling a +/// runtime worker on a slow resolver would stall every task sharing it. +pub(crate) async fn is_same_address(addr1: &str, addr2: &str) -> bool { + is_same_address_with(addr1, addr2, resolve_all).await +} + +/// [`is_same_address`] against a caller-provided resolver, so the fallback can +/// be exercised without depending on what the machine's resolver answers. +async fn is_same_address_with(addr1: &str, addr2: &str, resolve: R) -> bool +where + R: Fn(String) -> F, + F: Future>>, +{ + if is_same_spelling(addr1, addr2) { + return true; } + + // Two literal addresses that did not compare equal are different + // endpoints; resolving them would only hand back what they already say. + if parse_address(addr1).is_some() && parse_address(addr2).is_some() { + return false; + } + + let (Some(first), Some(second)) = ( + resolve(addr1.to_owned()).await, + resolve(addr2.to_owned()).await, + ) else { + return false; + }; + first.iter().any(|resolved| second.contains(resolved)) } /// Every socket address a host:port spelling resolves to, `None` when the /// resolver does not know the name (which then compares unequal, at worst /// costing one extra dial). -fn resolve_all(addr: &str) -> Option> { - let resolved: Vec = addr.to_socket_addrs().ok()?.collect(); +async fn resolve_all(addr: String) -> Option> { + let resolved: Vec = tokio::net::lookup_host(addr).await.ok()?.collect(); (!resolved.is_empty()).then_some(resolved) } @@ -358,26 +385,54 @@ mod tests { assert!(transport_endpoints(&metadata, TransportProtocol::Quic).is_empty()); } - #[test] - fn test_is_same_address() { - assert!(is_same_address("127.0.0.1:8090", "127.0.0.1:8090")); - assert!(is_same_address("localhost:8090", "127.0.0.1:8090")); - assert!(!is_same_address("127.0.0.1:8090", "127.0.0.1:8091")); - assert!(!is_same_address("192.168.1.1:8090", "127.0.0.1:8090")); + #[tokio::test] + async fn test_is_same_address() { + assert!(is_same_address("127.0.0.1:8090", "127.0.0.1:8090").await); + assert!(is_same_address("localhost:8090", "127.0.0.1:8090").await); + assert!(!is_same_address("127.0.0.1:8090", "127.0.0.1:8091").await); + assert!(!is_same_address("192.168.1.1:8090", "127.0.0.1:8090").await); } - // A host name and the address it resolves to name one endpoint; a name the - // resolver does not know compares unequal rather than erroring. - #[test] - fn a_host_name_matches_the_address_it_resolves_to() { - // `localhost` is rewritten before resolution, so use the loopback name - // the resolver itself answers for. - assert!(is_same_address("LOCALHOST:8090", "127.0.0.1:8090")); - assert!(!is_same_address("localhost:8090", "localhost:8091")); - assert!(!is_same_address( - "no-such-host.invalid:8090", - "127.0.0.1:8090" - )); + /// A stand-in resolver: the BDD cluster's spelling of one node, which no + /// canonicalization rewrites, so only the resolving comparison can equate + /// the two. `None` for anything else, like a name the resolver does not + /// know. + async fn resolve_bdd_leader(addr: String) -> Option> { + match addr.as_str() { + "iggy-leader:8091" | "172.28.0.101:8091" => { + Some(vec![SocketAddr::from(([172, 28, 0, 101], 8091))]) + } + _ => None, + } + } + + // A host name and the address it resolves to name one endpoint. Exactly + // the case the BDD cluster hits: the client dials `iggy-leader:8091` and + // the roster advertises `172.28.0.101:8091`. + #[tokio::test] + async fn a_host_name_matches_the_address_it_resolves_to() { + assert!( + is_same_address_with("iggy-leader:8091", "172.28.0.101:8091", resolve_bdd_leader).await + ); + } + + // A name the resolver does not know compares unequal rather than + // erroring, and a resolvable name never matches another port. + #[tokio::test] + async fn an_unresolvable_name_or_another_port_is_a_different_endpoint() { + assert!( + !is_same_address_with( + "iggy-follower:8092", + "172.28.0.101:8091", + resolve_bdd_leader + ) + .await + ); + assert!( + !is_same_address_with("iggy-leader:8091", "172.28.0.101:8092", resolve_bdd_leader) + .await + ); + assert!(!is_same_address("no-such-host.invalid:8090", "127.0.0.1:8090").await); } #[test] diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index 8bae4d2204..7022e773fb 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -120,6 +120,11 @@ pub struct TcpClient { struct RememberedSignIn { credentials: Credentials, user_id: u32, + /// Set by a committed password change for `user_id`. The configured + /// `AutoLogin` credentials cannot be rewritten -- the config is shared and + /// immutable -- so this marks the remembered copy as the newer one for + /// that same user, and [`TcpClient::sign_in_credentials`] prefers it. + password_refreshed: bool, } /// A connection that completed every step of coming up, TLS included. @@ -225,15 +230,7 @@ impl BinaryTransport for TcpClient { // Login and register are the exception: the server stays deliberately // silent on a transient register failure and relies on the client // replaying, so that replay is the protocol rather than a retry. - let replay_after_reconnect = is_login_register_code(code) - || matches!( - error, - IggyError::NotConnected - | IggyError::CannotEstablishConnection - | IggyError::Unauthenticated - | IggyError::StaleClient - ) - || matches!(operation_for_code(code), Operation::NonReplicated); + let replay_after_reconnect = replay_is_safe(code, &error); self.disconnect_transport().await?; @@ -278,6 +275,59 @@ impl BinaryTransport for TcpClient { } } +/// Whether replaying `code` over a fresh connection cannot apply it twice. +/// +/// The reconnect registers a new client identity, so the server's dedup fence +/// no longer covers the original request: only requests that provably never +/// reached the log may be re-sent. +/// +/// - the errors raised before the frame was written, and the server's own +/// refusals, which precede execution; +/// - operations that never enter the log: a non-replicated read, and a logout, +/// which ends whatever session the connection carried -- the reconnect +/// brought a new one, and refusing the replay would strand +/// `logout_before_relogin`, whose failure aborts the sign-in that was about +/// to replace the session; +/// - login and register, where the replay is the protocol: the server stays +/// deliberately silent on a transient register failure and relies on the +/// client resending. +fn replay_is_safe(code: u32, error: &IggyError) -> bool { + is_login_register_code(code) + || matches!( + error, + IggyError::NotConnected + | IggyError::CannotEstablishConnection + | IggyError::Unauthenticated + | IggyError::StaleClient + ) + || matches!( + operation_for_code(code), + Operation::NonReplicated | Operation::Logout + ) +} + +/// Why a TLS handshake failed, as far as retrying is concerned. +/// +/// A certificate this client will never accept -- the wrong CA, a name it does +/// not cover, a peer that answers a ClientHello with something else -- says the +/// same thing on every attempt. Reported as a configuration fault it ends the +/// connect after one sweep; reported as a lost connection it would be redialed +/// every interval forever under `max_retries = None`, which is how a wrong CA +/// looks like a flaky network. +fn classify_handshake_failure(error: &std::io::Error) -> IggyError { + match error + .get_ref() + .and_then(|inner| inner.downcast_ref::()) + { + Some( + rustls::Error::InvalidCertificate(_) + | rustls::Error::NoCertificatesPresented + | rustls::Error::InvalidMessage(_), + ) => IggyError::InvalidTlsCertificate, + _ => IggyError::CannotEstablishConnection, + } +} + impl iggy_common::VsrSessionSealed for TcpClient {} #[async_trait::async_trait] @@ -314,6 +364,7 @@ impl iggy_common::VsrSessionControl for TcpClient { .replace(RememberedSignIn { credentials, user_id, + password_refreshed: false, }); } @@ -339,6 +390,7 @@ impl iggy_common::VsrSessionControl for TcpClient { }; if targets_session_user { *password = SecretString::from(new_password.to_owned()); + sign_in.password_refreshed = true; } } @@ -452,6 +504,11 @@ impl TcpClient { let remote_address; let client_address; let mut candidate = 0; + // A fault no retry can fix, remembered rather than returned at + // once: it belongs to the endpoint that raised it (a certificate + // that names another host, a domain that will not parse), and the + // endpoints behind that one may be perfectly usable. + let mut config_fault: Option = None; loop { let server_address = candidates[candidate].clone(); if server_address == paced_endpoint @@ -478,14 +535,7 @@ impl TcpClient { break; } Err(IggyError::CannotEstablishConnection) => {} - Err(error) => { - // An unreadable CA file or an unusable TLS domain is a - // configuration fault: no other endpoint fixes it, and - // retrying forever under `max_retries = None` would - // only bury it. - self.fail_connect().await; - return Err(error); - } + Err(error) => config_fault = Some(error), } // Every other endpoint gets its turn before the retry @@ -497,6 +547,17 @@ impl TcpClient { } candidate = 0; + // An unreadable CA file, a certificate that names another + // host, a domain that will not parse: no endpoint answered and + // at least one said why in a way that a retry cannot change, + // so the caller gets that reason instead of a retry loop that + // buries it (`max_retries = None` would otherwise redial it + // every interval forever). + if let Some(error) = config_fault { + self.fail_connect().await; + return Err(error); + } + // The sweep is what reconnection settings apply to, not a // single dial: with reconnection off there are no retries, but // the failover endpoints were configured to be tried and they @@ -578,13 +639,33 @@ impl TcpClient { info!("{NAME} client: {client_address} has signed in with {how}.") } Err(error) => { - // The transport is up and only the session is - // not, so the state has to say so: left at + // With the transport up and only the session + // missing, the state has to say so: left at // `Authenticating` every gated operation fails // client-side with `Disconnected`, `connect()` // returns ok without dialing, and nothing short // of an explicit `login_user` recovers. - self.set_state(ClientState::Connected).await; + // + // A sign-in can also fail because the socket + // died under it. Whatever is left of that + // connection cannot carry a request, so it goes + // rather than being kept behind a `Connected` + // that makes the next `connect()` a no-op and + // leaves every gated operation failing until + // someone calls `disconnect()` by hand. + if matches!( + error, + IggyError::Disconnected + | IggyError::EmptyResponse + | IggyError::NotConnected + | IggyError::CannotEstablishConnection + | IggyError::TcpError + | IggyError::StaleClient + ) { + self.disconnect_transport().await?; + } else if self.get_state().await == ClientState::Authenticating { + self.set_state(ClientState::Connected).await; + } // A rejected credential does not become valid on // the next reconnect, and replaying it costs an // argon2 on the server every time. Configured @@ -679,14 +760,33 @@ impl TcpClient { /// sign-in is otherwise less reconnectable than a configured one, which /// is a surprising difference between two ways of doing the same thing. async fn sign_in_credentials(&self) -> Option { - match &self.config.auto_login { - AutoLogin::Enabled(credentials) => Some(credentials.clone()), - AutoLogin::Disabled => self - .session_credentials + let remembered = + self.session_credentials .lock() .await .as_ref() - .map(|remembered| remembered.credentials.clone()), + .map(|remembered: &RememberedSignIn| { + ( + remembered.credentials.clone(), + remembered.password_refreshed, + ) + }); + + match (&self.config.auto_login, remembered) { + // A committed password change for the configured user outranks the + // configured password: the config cannot be rewritten, and signing + // in with the password this client itself replaced would fail + // `InvalidCredentials` on every later reconnect. Same user either + // way -- `refresh_session_password` only marks a change that + // targeted the signed-in one. + ( + AutoLogin::Enabled(Credentials::UsernamePassword(configured_username, _)), + Some((Credentials::UsernamePassword(username, password), true)), + ) if configured_username == &username => { + Some(Credentials::UsernamePassword(username, password)) + } + (AutoLogin::Enabled(configured), _) => Some(configured.clone()), + (AutoLogin::Disabled, remembered) => remembered.map(|(credentials, _)| credentials), } } @@ -697,10 +797,14 @@ impl TcpClient { let mut candidates = vec![self.current_server_address.lock().await.clone()]; let roster = self.roster_endpoints.lock().await.clone(); for endpoint in roster.iter().chain(self.config.failover_addresses.iter()) { - if !candidates - .iter() - .any(|candidate| is_same_address(candidate, endpoint)) - { + let mut known = false; + for candidate in &candidates { + if is_same_address(candidate, endpoint).await { + known = true; + break; + } + } + if !known { candidates.push(endpoint.clone()); } } @@ -794,7 +898,7 @@ impl TcpClient { })?; let stream = connector.connect(domain, stream).await.map_err(|error| { error!("Failed to establish a TLS connection to the server: {error}"); - IggyError::CannotEstablishConnection + classify_handshake_failure(&error) })?; Ok(EstablishedConnection { @@ -1162,6 +1266,8 @@ const fn is_login_register_code(code: u32) -> bool { #[cfg(test)] mod tests { use super::*; + use iggy_binary_protocol::codes::{GET_ME_CODE, LOGOUT_USER_CODE, SEND_MESSAGES_CODE}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; const SESSION_USER_ID: u32 = 7; @@ -1310,6 +1416,124 @@ mod tests { assert!(matches!(sweep, Err(IggyError::CannotEstablishConnection))); } + /// A peer that accepts TCP and then answers a ClientHello with something + /// else: the handshake fails for a reason no retry changes. + async fn endpoint_that_speaks_no_tls() -> String { + let (listener, address) = live_endpoint().await; + tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + tokio::spawn(async move { + let _ = stream.write_all(b"this is not a TLS record\n").await; + // Held open, so the failure is the handshake's verdict + // rather than a closed socket. + let mut sink = [0u8; 64]; + while stream.read(&mut sink).await.is_ok_and(|read| read > 0) {} + }); + } + }); + address + } + + // A certificate this client will never accept says the same thing on every + // attempt, so it has to reach the caller instead of being redialed every + // interval forever -- which is what `max_retries = None` did with it. + #[tokio::test] + async fn a_handshake_no_retry_can_fix_ends_the_connect() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: endpoint_that_speaks_no_tls().await, + tls_enabled: true, + tls_validate_certificate: false, + reconnection: TcpClientReconnectionConfig { + // Unlimited retries, so a transient classification never + // returns and this test times out instead of failing. + max_retries: None, + interval: IggyDuration::from_str("100ms").expect("duration"), + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + + let connect = tokio::time::timeout( + std::time::Duration::from_secs(10), + std::pin::pin!(TcpClient::connect(&client)), + ) + .await + .expect("the connect has to end on its own"); + assert!(matches!(connect, Err(IggyError::InvalidTlsCertificate))); + assert_eq!(client.get_state().await, ClientState::Disconnected); + } + + // A sign-in that lost its socket leaves nothing to send on, so the + // transport goes with it: kept behind a `Connected`, the next `connect()` + // would return ok without dialing and every gated operation would fail + // until someone disconnected by hand. + #[tokio::test] + async fn a_sign_in_that_lost_its_socket_leaves_the_client_disconnected() { + let (listener, address) = live_endpoint().await; + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + // Accept, then hang up: the dial succeeds and the sign-in dies + // on the socket. + drop(stream); + } + }); + + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: address, + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "iggy".to_string(), + "iggy".into(), + )), + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + + assert!(TcpClient::connect(&client).await.is_err()); + assert_eq!(client.get_state().await, ClientState::Disconnected); + assert!( + client.stream.lock().await.is_none(), + "a dead connection must not be kept for the next request to find" + ); + } + + // The reconnect registers a new client identity, so the server's dedup + // fence no longer covers the original request. + #[test] + fn only_requests_that_cannot_double_apply_are_replayed() { + // Never written, or refused before execution. + assert!(replay_is_safe(SEND_MESSAGES_CODE, &IggyError::NotConnected)); + assert!(replay_is_safe( + SEND_MESSAGES_CODE, + &IggyError::CannotEstablishConnection + )); + assert!(replay_is_safe(SEND_MESSAGES_CODE, &IggyError::StaleClient)); + // Written, and its outcome unknown: a replicated write must not be + // re-sent under a session the fence cannot match it against. + assert!(!replay_is_safe( + SEND_MESSAGES_CODE, + &IggyError::Disconnected + )); + assert!(!replay_is_safe( + SEND_MESSAGES_CODE, + &IggyError::EmptyResponse + )); + // A read never enters the log, and a logout ends a session the + // reconnect already replaced -- `logout_before_relogin` depends on it. + assert!(replay_is_safe(GET_ME_CODE, &IggyError::Disconnected)); + assert!(replay_is_safe(LOGOUT_USER_CODE, &IggyError::Disconnected)); + // The register replay is the protocol: the server stays silent on a + // transient failure and waits for the resend. + assert!(replay_is_safe( + LOGIN_REGISTER_CODE, + &IggyError::Disconnected + )); + } + // `reestablish_after` paces reconnects to the endpoint that was lost. With // somewhere else to go, that pause must not hold up the failover. #[tokio::test] @@ -1552,6 +1776,78 @@ mod tests { } } + // The configured credentials cannot be rewritten, so a committed password + // change for the configured user has to reach the next reconnect through + // the remembered copy, or every later drop replays the password this very + // client replaced. + #[tokio::test] + async fn a_password_change_reaches_a_configured_auto_login() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "iggy".to_string(), + "old".into(), + )), + ..TcpClientConfig::default() + })) + .expect("create the client"); + client + .remember_session_credentials( + Credentials::UsernamePassword("iggy".to_string(), "old".into()), + SESSION_USER_ID, + ) + .await; + + client + .refresh_session_password( + &Identifier::numeric(SESSION_USER_ID).expect("numeric identifier"), + "new", + ) + .await; + + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, password)) => { + assert_eq!(username, "iggy"); + assert_eq!(password.expose_secret(), "new"); + } + other => panic!("expected the configured user with the new password, got {other:?}"), + } + } + + // A change for somebody else says nothing about the configured user's + // password, and a sign-in as another user does not get to replace it. + #[tokio::test] + async fn a_password_change_for_another_user_leaves_a_configured_auto_login_alone() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "configured".to_string(), + "old".into(), + )), + ..TcpClientConfig::default() + })) + .expect("create the client"); + client + .remember_session_credentials( + Credentials::UsernamePassword("signed-in".to_string(), "old".into()), + SESSION_USER_ID, + ) + .await; + + client + .refresh_session_password( + &Identifier::numeric(SESSION_USER_ID).expect("numeric identifier"), + "new", + ) + .await; + + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, password)) => { + assert_eq!(username, "configured"); + assert_eq!(password.expose_secret(), "old"); + } + other => panic!("expected the configured credentials, got {other:?}"), + } + } + #[tokio::test] async fn configured_credentials_outrank_the_ones_a_sign_in_remembered() { let client = TcpClient::create(Arc::new(TcpClientConfig { diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index 16f2731bbf..a1e267c3c1 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -21,6 +21,7 @@ using System.Net.Security; using System.Net.Sockets; using System.Runtime.InteropServices; +using System.Security.Authentication; using System.Security.Cryptography.X509Certificates; using Apache.Iggy.Configuration; using Apache.Iggy.Contracts; @@ -1033,6 +1034,7 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken var retryCount = 0; var redirects = 0; var delay = _configuration.ReconnectionSettings.InitialDelay; + Exception? configurationFault = null; if (string.IsNullOrEmpty(_currentAddress)) { @@ -1051,7 +1053,7 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken } Socket? socket = null; - var dialed = false; + var established = false; try { socket = new Socket(ServerAddress.AddressFamilyOf(host), SocketType.Stream, ProtocolType.Tcp); @@ -1074,7 +1076,6 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken var dialToken = dialCancellation?.Token ?? token; await socket.ConnectAsync(host, port, dialToken); - dialed = true; _currentRemoteAddress = socket.RemoteEndPoint is IPEndPoint remote ? ServerAddress.HostPort(remote.Address.ToString(), (ushort)remote.Port) @@ -1088,6 +1089,11 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken ? await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings, dialToken) : new NetworkStream(socket, true); + // Established, not merely dialed: everything up to here belongs to this endpoint and the + // sweep may try the next one, while everything past it - auto login, a redirect, the leader + // lookup - fails the same way wherever the client lands. + established = true; + await _sendingSemaphore.WaitAsync(token); try { @@ -1121,12 +1127,13 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken break; } - // Only a failed dial is worth another attempt. Everything past it - a rejected certificate, bad - // credentials, a leader that cannot be found - fails the same way every time, and with unlimited + // Only bringing an endpoint up is worth trying elsewhere. Everything past it - bad credentials, a + // leader that cannot be found - fails the same way wherever the client lands, and with unlimited // retries a caller would otherwise never get the error back. - // A dial the bound above cut short is a failed dial like any other, so it must not land here: - // only a cancellation the caller actually asked for is fatal. - catch (Exception e) when (dialed + // + // A handshake the dial bound cut short is a failed attempt on this endpoint like any other, so it + // must not land here: only a cancellation the caller actually asked for is fatal. + catch (Exception e) when (established || (e is OperationCanceledException && token.IsCancellationRequested) || _disposed) { @@ -1146,6 +1153,14 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken _logger.LogError(e, "Failed to connect"); + if (IsTlsConfigurationFault(e)) + { + // A fault no retry can fix, kept aside rather than thrown at once: it belongs to the + // endpoint that raised it - a certificate that names another host - and the endpoints + // behind that one may be perfectly usable. + configurationFault = e; + } + // Every other endpoint gets its turn before the retry delay: the node just lost may be gone for // good, and pausing on it helps nothing. if (++candidate < candidates.Length) @@ -1157,6 +1172,15 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken candidate = 0; _currentAddress = candidates[0]; + // No endpoint answered and at least one said why in a way no retry changes: an unreadable CA + // file, a certificate this client will never accept. The caller gets that reason instead of a + // retry loop that buries it - unlimited retries would otherwise redial it forever. + if (configurationFault is not null) + { + SetConnectionState(ConnectionState.Disconnected); + throw configurationFault; + } + // The sweep is what the reconnection budget applies to, not a single dial: checked per dial, // the last round would try only the endpoint the client started on, and a client with // reconnection turned off would never reach its other endpoints at all. @@ -1299,15 +1323,34 @@ private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSett _customCaStore.ImportFromPemFile(tlsSettings.CertificatePath); var stream = new NetworkStream(socket, true); var sslStream = new SslStream(stream, false, RemoteCertificateValidationCallback); - - // The token carries the dial bound when other endpoints are queued behind this one: a peer that - // accepts TCP and never answers the ClientHello has no deadline of its own here either. - await sslStream.AuthenticateAsClientAsync( - new SslClientAuthenticationOptions { TargetHost = tlsSettings.Hostname }, token); + try + { + // The token carries the dial bound when other endpoints are queued behind this one: a peer that + // accepts TCP and never answers the ClientHello has no deadline of its own here either. + await sslStream.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions { TargetHost = tlsSettings.Hostname }, token); + } + catch + { + // A handshake that failed leaves the stream owning the socket, and the sweep moves on to the + // next endpoint: undisposed, both leak for as long as the client lives. + await sslStream.DisposeAsync(); + throw; + } return sslStream; } + /// + /// Whether bringing an endpoint up failed for a reason that says this client's own TLS configuration is + /// wrong: a CA file that cannot be read, or a certificate it will never accept. Neither changes on a + /// retry, so the sweep reports it instead of redialing forever. + /// + private static bool IsTlsConfigurationFault(Exception e) + { + return e is AuthenticationException or InvalidCertificatePathException; + } + private async Task SendAckAsync(int code, ReadOnlyMemory body, CancellationToken token) { using IMemoryOwner _ = await SendWithResponseAsync(code, body, token: token); diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs index b0f8d96e21..2d88801845 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs @@ -212,6 +212,62 @@ await Assert.ThrowsAsync(() => Assert.Equal(connectionsBeforeEviction, node.Connections); } + /// + /// A survivor that only comes up after the first rotation still has to be found. The reconnection + /// budget counts rotations, not dials, so one retry is one full pass over every endpoint the client + /// knows rather than one dial of the endpoint it started from. + /// + [Fact] + public async Task ResumesOnASurvivorThatComesUpAfterTheFirstRotation() + { + // A port nothing listens on yet: the survivor comes up on it only after the first rotation has + // already failed on both endpoints. + var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + var survivorPort = (ushort)((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); + + using var primary = new MockNode(); + primary.Serve(request => request.Code == GetClusterMetadataCode + ? Reply(OperationNonReplicated, ClusterMetadata(primary.Port, survivorPort, primary.Port)) + : Answer(request)); + + var configuration = new IggyClientConfigurator + { + BaseAddress = $"127.0.0.1:{primary.Port}", + Protocol = Protocol.Tcp, + AutoLoginSettings = new AutoLoginSettings { Enabled = true, Username = "iggy", Password = "iggy" }, + ReconnectionSettings = new ReconnectionSettings + { + Enabled = true, + // One retry: the rotation after the first failure is the last one, which is where the + // budget check used to cut the sweep short. + MaxRetries = 1, + InitialDelay = TimeSpan.FromMilliseconds(600) + } + }; + using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); + + await client.ConnectAsync(TestContext.Current.CancellationToken); + await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); + await client.PingAsync(TestContext.Current.CancellationToken); + + primary.Kill(); + using var survivor = new MockNode(survivorPort); + var comesUp = Task.Run(async () => + { + await Task.Delay(250, TestContext.Current.CancellationToken); + survivor.Serve(request => request.Code == GetClusterMetadataCode + ? Reply(OperationNonReplicated, ClusterMetadata(primary.Port, survivorPort, survivorPort)) + : Answer(request)); + }, TestContext.Current.CancellationToken); + + await client.PingAsync(TestContext.Current.CancellationToken); + await comesUp; + + Assert.True(survivor.Registrations >= 1, "the session was re-established on the survivor"); + } + private static byte[] EvictionFrame(byte reason) { var frame = new byte[HeaderSize]; @@ -368,9 +424,13 @@ private sealed class MockNode : IDisposable private int _pings; private int _registrations; - public MockNode() + /// + /// A port to bind, for a node that has to come up on an address the client already knows. Zero + /// takes whatever the OS hands out. + /// + public MockNode(ushort port = 0) { - _listener = new TcpListener(IPAddress.Loopback, 0); + _listener = new TcpListener(IPAddress.Loopback, port); _listener.Start(); Port = (ushort)((IPEndPoint)_listener.LocalEndpoint).Port; } diff --git a/foreign/go/client/tcp/tcp_connect_test.go b/foreign/go/client/tcp/tcp_connect_test.go index 822958b5b5..aa3ce3a12e 100644 --- a/foreign/go/client/tcp/tcp_connect_test.go +++ b/foreign/go/client/tcp/tcp_connect_test.go @@ -626,7 +626,7 @@ func TestConnect_ExchangesOverTLS(t *testing.T) { func TestCreateTLSConfig_ExtractsAnIPv6ServerName(t *testing.T) { client := NewIggyTcpClient(nil, WithServerAddress("[::1]:8090"), WithTLS()) - config, err := client.createTLSConfig() + config, err := client.createTLSConfig("[::1]:8090") require.NoError(t, err) assert.Equal(t, "::1", config.ServerName) } diff --git a/foreign/go/client/tcp/tcp_core.go b/foreign/go/client/tcp/tcp_core.go index b721abf124..0ebb91c312 100644 --- a/foreign/go/client/tcp/tcp_core.go +++ b/foreign/go/client/tcp/tcp_core.go @@ -643,7 +643,23 @@ func (c *IggyTcpClient) sendFrame(ctx context.Context, code uint32, frame []byte return nil, redirectErr } if redirect { + // A connect-scoped request is issued from inside the sign-in + // transaction, which holds registerMtx: the automatic sign-in + // on the reconnect path would wait on that lock forever. The + // transaction signs in itself on the node it lands on, so the + // reconnect must not. + connectScopedRequest := ctx.Value(connectScoped{}) != nil + if connectScopedRequest { + c.mtx.Lock() + c.skipAutoLoginOnce = true + c.mtx.Unlock() + } if connectErr := c.Connect(ctx); connectErr != nil { + if connectScopedRequest { + c.mtx.Lock() + c.skipAutoLoginOnce = false + c.mtx.Unlock() + } return nil, connectErr } stamped = false @@ -963,13 +979,20 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { // a pass that stopped at the first refusal would never reach the // survivors of a client configured for a single retry. var lastErr error + // A fault no retry can fix, kept aside rather than returned at + // once: it belongs to the endpoint that raised it, and the + // endpoints behind that one may be perfectly usable. + var configFault error for _, address := range candidates { if address == pacedEndpoint { - c.awaitReestablish(connectedAt) + c.awaitReestablish(ctx, connectedAt) } connection, err := c.dialCandidate(ctx, address, len(candidates) > 1) if err != nil { lastErr = err + if isTLSConfigFault(err) { + configFault = err + } continue } @@ -977,6 +1000,15 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { return nil } + // An unreadable CA file, an unparsable domain, a certificate this + // client will never accept: no endpoint answered, and at least one + // said why in a way no retry changes. Reported as unrecoverable so + // the default unlimited retries do not redial it every interval + // forever and bury it. + if configFault != nil { + return retry.Unrecoverable(configFault) + } + return lastErr }); err != nil { c.mtx.Lock() @@ -1009,18 +1041,48 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { return nil } +// isTLSConfigFault reports whether a dial failed for a reason that says the +// client's own TLS configuration is wrong -- an unreadable or unparsable CA +// file, a domain that yields no server name, or a certificate this client will +// never accept. None of those change on a retry. +func isTLSConfigFault(err error) bool { + if errors.Is(err, ierror.ErrInvalidTlsCertificatePath) || + errors.Is(err, ierror.ErrInvalidTlsCertificate) || + errors.Is(err, ierror.ErrInvalidTlsDomain) { + return true + } + + var certificateError *tls.CertificateVerificationError + var recordError tls.RecordHeaderError + return errors.As(err, &certificateError) || errors.As(err, &recordError) +} + // awaitReestablish waits out what is left of the reestablishAfter window since // the last successful connection, if any. -func (c *IggyTcpClient) awaitReestablish(connectedAt time.Time) { +// +// The wait ends early on the caller's context or on Close: a sweep that found +// every other endpoint refused reaches the paced one in milliseconds, and +// sleeping the rest of the window regardless would hold the client in +// Connecting long past the deadline the caller gave it. +func (c *IggyTcpClient) awaitReestablish(ctx context.Context, connectedAt time.Time) { if connectedAt.IsZero() { return } elapsed := time.Since(connectedAt) c.logger.Debug("Elapsed time since last connection", slog.Duration("elapsed", elapsed)) - if remaining := c.config.reconnection.reestablishAfter - elapsed; remaining > 0 { - c.logger.Info("Trying to connect to the server", slog.Duration("remaining", remaining)) - time.Sleep(remaining) + remaining := c.config.reconnection.reestablishAfter - elapsed + if remaining <= 0 { + return + } + + c.logger.Info("Trying to connect to the server", slog.Duration("remaining", remaining)) + timer := time.NewTimer(remaining) + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + case <-c.closed: } } @@ -1054,7 +1116,7 @@ func (c *IggyTcpClient) dialCandidate(ctx context.Context, address string, bound established := connection if c.config.tlsEnabled { - tlsConfig, err := c.createTLSConfig() + tlsConfig, err := c.createTLSConfig(address) if err != nil { _ = connection.Close() return nil, err @@ -1157,7 +1219,14 @@ func (c *IggyTcpClient) forgetLogin() { c.rememberedLogin = AutoLogin{} } -func (c *IggyTcpClient) createTLSConfig() (*tls.Config, error) { +// createTLSConfig builds the client config for one dial. +// +// address is the candidate being dialed, which is where the SNI comes from +// when no domain is configured. Taking it from currentServerAddress instead +// would name the endpoint the client just lost: with validation on, a failover +// to a node with another name or address then fails the handshake against a +// certificate that never covered the old one. +func (c *IggyTcpClient) createTLSConfig(address string) (*tls.Config, error) { tlsConfig := &tls.Config{ InsecureSkipVerify: !c.config.tls.tlsValidateCertificate, } @@ -1165,9 +1234,9 @@ func (c *IggyTcpClient) createTLSConfig() (*tls.Config, error) { // Set server name for SNI serverName := c.config.tls.tlsDomain if serverName == "" { - host, _, err := net.SplitHostPort(c.currentServerAddress) + host, _, err := net.SplitHostPort(address) if err != nil { - host = c.currentServerAddress + host = address } serverName = host } diff --git a/foreign/go/client/tcp/tcp_failover_test.go b/foreign/go/client/tcp/tcp_failover_test.go index 2531ef8d96..30badd8a34 100644 --- a/foreign/go/client/tcp/tcp_failover_test.go +++ b/foreign/go/client/tcp/tcp_failover_test.go @@ -19,6 +19,7 @@ package tcp import ( "context" + "crypto/tls" "log/slog" "net" "sync/atomic" @@ -241,6 +242,122 @@ func TestFailover_ReLoginSurvivesALogoutTheTransportSwallowed(t *testing.T) { require.NoError(t, client.Ping(ctx)) } +// The other way a logout fails to land: the node answers it as not-admitted, +// which is what a node that stopped being primary does. The redirect that +// follows must not sign in on its own -- this goroutine holds the sign-in lock, +// and the reconnect's automatic sign-in would wait on it forever. +func TestFailover_ReLoginSurvivesALogoutTheOldPrimaryRefused(t *testing.T) { + var leader *testListener + var follower *testListener + var demoted atomic.Bool + + // The node the client is on: leader until the logout, then a follower that + // refuses it as not-admitted and points at the survivor. + follower = listenVSR(t, nil, func(_, _ int, read request) []byte { + switch { + case read.code() == uint32(command.GetClusterMetadataCode): + if demoted.Load() { + return clusterMetadataFrame(t, 1, follower.address(), leader.address()) + } + return clusterMetadataFrame(t, 0, follower.address(), leader.address()) + case read.operation() == vsr.OperationRegister: + return registerReplyFrame(7, 128) + case read.operation() == vsr.OperationLogout: + demoted.Store(true) + return statusReplyFrame(vsr.OperationLogout, + uint32(ierror.TransientNotAcceptedCode), nil) + default: + return replyFrame(vsr.OperationNonReplicated, nil) + } + }) + leader = listenVSR(t, nil, func(_, _ int, read request) []byte { + switch { + case read.code() == uint32(command.GetClusterMetadataCode): + return clusterMetadataFrame(t, 1, follower.address(), leader.address()) + case read.operation() == vsr.OperationRegister: + return registerReplyFrame(7, 256) + default: + return replyFrame(vsr.OperationNonReplicated, nil) + } + }) + + client := newDialingClient(t, follower.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "iggy", "iggy") + require.NoError(t, err) + + relogin := make(chan error, 1) + go func() { + _, err := client.LoginUser(ctx, "iggy", "iggy") + relogin <- err + }() + select { + case err := <-relogin: + require.NoError(t, err, "the sign-in has to settle on the node that leads") + case <-time.After(15 * time.Second): + t.Fatal("the re-login deadlocked on the sign-in lock") + } + + assert.True(t, client.session.Bound(), "the replayed sign-in bound a session") + assert.Equal(t, leader.address(), client.currentServerAddress) +} + +// A logout that never landed still ended the session it belonged to, so the +// credentials that established it must not outlive it: a sign-in that then +// fails would otherwise leave them for the next dropped request to replay, +// signing the old user back in after the caller asked for another one. +func TestFailover_ARejectedReLoginDoesNotResurrectThePreviousUser(t *testing.T) { + var server *testListener + var dropLogout atomic.Bool + var dropSocket atomic.Bool + var rejectLogin atomic.Bool + var registeredUsers atomic.Int32 + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + if dropSocket.Load() { + return nil + } + if dropLogout.Load() && read.operation() == vsr.OperationLogout { + return nil + } + if read.operation() == vsr.OperationRegister { + registeredUsers.Add(1) + if rejectLogin.Load() { + return statusReplyFrame(vsr.OperationRegister, + uint32(ierror.InvalidCredentialsCode), nil) + } + return registerReplyFrame(7, 128) + } + return singleNodeHandler(t, func() string { return server.address() })(0, 0, read) + }) + + client := newDialingClient(t, server.address()) + ctx := context.Background() + require.NoError(t, client.Connect(ctx)) + _, err := client.LoginUser(ctx, "alice", "alice") + require.NoError(t, err) + + // The logout is swallowed and the sign-in that follows is rejected, so the + // client ends up with no session and no credentials it may use. + dropLogout.Store(true) + rejectLogin.Store(true) + _, err = client.LoginUser(ctx, "bob", "bob") + require.Error(t, err) + + _, remembered := client.signInCredentials() + assert.False(t, remembered, "the ended session's credentials must not survive it") + + // The socket dies with nothing remembered: the reconnect has no session to + // restore, and must not invent one out of the user who was signed in + // before. + dropLogout.Store(false) + dropSocket.Store(true) + registersBefore := registeredUsers.Load() + assert.Error(t, client.Ping(ctx), "there is no session left to restore") + assert.Equal(t, registersBefore, registeredUsers.Load(), + "the reconnect signed the previous user back in") +} + // reestablishAfter is a cooldown on redialing the endpoint that was lost. It // is owed to that endpoint alone, so a failover to another one must not sit // through it. @@ -282,6 +399,27 @@ func TestFailover_KeepsTheReestablishPauseForTheEndpointThatWasLost(t *testing.T "the cooldown on the endpoint that was lost was skipped") } +// The cooldown is a pace limit, not a commitment: a caller that gave the +// connect a deadline has to get an answer inside it, and Close has to end the +// wait too. +func TestFailover_TheReestablishPauseHonoursTheCallersDeadline(t *testing.T) { + current := listenVSR(t, nil, func(_, _ int, read request) []byte { + return singleNodeHandler(t, func() string { return "127.0.0.1:8090" })(0, 0, read) + }) + + client := newDialingClient(t, current.address()) + client.config.reconnection.reestablishAfter = time.Minute + client.connectedAt = time.Now() + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + started := time.Now() + _ = client.Connect(ctx) + + assert.Less(t, time.Since(started), 5*time.Second, + "the cooldown outlived the deadline the caller gave the connect") +} + // A node whose syns are dropped must not hold the sweep: without a bound on // the dial the survivors behind it are never reached. A black-holed address // cannot be arranged portably, so this pins the bound itself. @@ -340,6 +478,63 @@ func TestFailover_DoesNotSettleOnAnEndpointThatFailedTheHandshake(t *testing.T) "the endpoint that failed the handshake became the current one") } +// The SNI of a dial belongs to the endpoint being dialed. Taken from the +// endpoint the client just lost, a failover to a node the certificate does not +// cover fails the handshake -- which is every failover, once the addresses +// differ. +func TestFailover_UsesTheDialedEndpointAsTheServerName(t *testing.T) { + certificate, caPath := selfSignedCert(t) + survivor := listenVSR(t, + func(conn net.Conn) net.Conn { + return tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{certificate}}) + }, + singleNodeHandler(t, func() string { return "127.0.0.1:8090" })) + + // The certificate covers 127.0.0.1, and the endpoint the client starts on + // is 127.0.0.2, where nothing listens: with the server name taken from + // that endpoint, the handshake on the survivor is checked against the + // address that died. + _, port, err := net.SplitHostPort(survivor.address()) + require.NoError(t, err) + client := newDialingClient(t, "127.0.0.2:"+port, + WithTLS(WithTLSCAFile(caPath), WithTLSValidateCertificate(true))) + client.knownServerAddresses = []string{"127.0.0.1:" + port} + + require.NoError(t, client.Connect(context.Background())) + assert.Equal(t, "127.0.0.1:"+port, client.currentServerAddress) +} + +// A TLS configuration the client itself cannot satisfy says the same thing on +// every attempt, so it has to reach the caller instead of being redialed every +// interval forever -- which is what the default unlimited retries did with it. +func TestFailover_AConfigFaultEndsTheConnectInsteadOfRetryingForever(t *testing.T) { + certificate, _ := selfSignedCert(t) + server := listenVSR(t, + func(conn net.Conn) net.Conn { + return tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{certificate}}) + }, + singleNodeHandler(t, func() string { return "127.0.0.1:8090" })) + + // A CA the server's certificate was not signed by: no retry makes that + // certificate acceptable. + _, unrelatedCA := selfSignedCert(t) + client := NewIggyTcpClient(slog.New(slog.DiscardHandler), + WithServerAddress(server.address()), + WithTLS(WithTLSCAFile(unrelatedCA), WithTLSValidateCertificate(true))) + t.Cleanup(func() { _ = client.Close() }) + client.config.reconnection.maxRetries = 0 // unlimited + client.config.reconnection.interval = 10 * time.Millisecond + + done := make(chan error, 1) + go func() { done <- client.Connect(context.Background()) }() + select { + case err := <-done: + require.Error(t, err) + case <-time.After(5 * time.Second): + t.Fatal("a connect that can never succeed has to end instead of retrying forever") + } +} + // A client with nothing to dial must say so: reporting success would leave // every request answering ErrNotConnected while Connect keeps claiming a // connection. diff --git a/foreign/go/client/tcp/tcp_session_management.go b/foreign/go/client/tcp/tcp_session_management.go index 530420e898..d2e5341161 100644 --- a/foreign/go/client/tcp/tcp_session_management.go +++ b/foreign/go/client/tcp/tcp_session_management.go @@ -219,6 +219,11 @@ func (c *IggyTcpClient) endBoundSession(ctx context.Context) error { c.groups.clear() c.topics.clearCounts() c.mtx.Unlock() + // The session this sign-in belonged to is over either way, so the + // credentials that established it go with it. Kept, a sign-in that then + // fails would leave them behind for the next dropped request to replay -- + // signing the old user back in after the caller asked for another one. + c.forgetLogin() return nil } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java index 34fe409b4d..ef2baa5ed7 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java @@ -501,31 +501,41 @@ private AsyncTcpConnection openConnection(ConnectionInfo target) { /** * A server-side eviction reached this client. The routing state it cached * belonged to the evicted session, and a stale-client eviction is the - * server ending that session authoritatively, like a logout: the - * remembered sign-in ends with it, so only credentials configured on the - * builder may bring the session back. + * server ending that session authoritatively, like a logout: nothing may + * revive it behind the caller's back. + * + * The captured login always goes, whatever is configured. The connection + * replays it to bring up a replacement channel, so keeping it would revive + * exactly the session the server ended -- and on a client whose caller + * signed in as somebody other than the configured user, it would revive a + * different user than a redial replays, making the outcome depend on which + * failure ran. + * + * What may re-establish the session is what every connect of this client + * signs in as: credentials configured on the builder. A sign-in the caller + * ran is theirs to repeat. */ private void onSessionReset(int errorCode) { routingState.clearAssignments(); if (errorCode != IggyErrorCode.STALE_CLIENT.getCode()) { return; } - // Credentials configured on the builder are what every connect of this - // client signs in as, so an eviction does not revoke them and the - // client recovers on its own. A sign-in a caller ran is different: the - // server ended that session deliberately, and reviving it behind the - // caller's back is what an explicit logout must not be able to do - // either. Dropped in both places, because the connection replays its - // own captured login to bring up a replacement channel. - if (username.isPresent() && password.isPresent()) { - return; - } - log.warn("The server evicted this session as stale; the sign-in it ran will not be replayed"); - rememberedLogin = null; AsyncTcpConnection currentConnection = connection.get(); if (currentConnection != null) { currentConnection.forgetCapturedLogin(); } + if (username.isEmpty() || password.isEmpty()) { + log.warn("The server evicted this session as stale; the sign-in it ran will not be replayed"); + rememberedLogin = null; + return; + } + + log.info("The server evicted this session as stale; signing in again with the configured credentials"); + replayLogin().whenComplete((ignored, error) -> { + if (error != null) { + log.warn("Signing in again after the eviction failed: {}", error.getMessage()); + } + }); } /** @@ -716,9 +726,9 @@ private CompletableFuture replaySignInOn( log.info("Reconnected to {}", target.serverAddress()); return CompletableFuture.completedFuture(null); } - if (isConnectionLoss(unwrap(loginError))) { + if (!isSignInRejection(unwrap(loginError))) { log.warn( - "The sign-in on {} was lost with the connection: {}", + "The sign-in on {} did not complete: {}", target.serverAddress(), loginError.getMessage()); return sweepCandidates(candidates, index + 1, attempt, policy); @@ -734,6 +744,24 @@ private CompletableFuture replaySignInOn( .thenCompose(Function.identity()); } + /** + * Whether the server answered the sign-in with a verdict no other endpoint + * would change: a rotated password, an expired token. + * + * Only that ends a redial. Everything else -- the channel closing before + * the reply, a timeout, a transient refusal from a node that is not the + * primary -- says nothing about the credentials, and treating it as a + * rejection would drop them and leave the client published on a node that + * is already gone, with every later call failing "not authenticated". + */ + static boolean isSignInRejection(Throwable error) { + if (!(error instanceof IggyServerException serverError)) { + return false; + } + int code = serverError.getRawErrorCode(); + return code != AsyncTcpConnection.TRANSIENT_NOT_ACCEPTED && code != AsyncTcpConnection.TRANSIENT_NOT_COMMITTED; + } + private static Throwable unwrap(Throwable error) { return error instanceof CompletionException && error.getCause() != null ? error.getCause() : error; } @@ -777,7 +805,10 @@ CompletableFuture loginOnLeader(Supplier callerFuture = new CompletableFuture<>(); transaction.whenComplete((identity, error) -> { gate.complete(null); - if (error == null) { + // Not after a close: a login still in flight when `close()` cleared + // this would set it again, and `connect()` clears `closed`, so the + // next loss would replay a sign-in the caller had ended. + if (error == null && !closed) { rememberedLogin = loginAttempt; } if (error != null) { @@ -876,17 +907,44 @@ CompletableFuture> findLeaderElsewhere(ConnectionInfo c } return LeaderAwareness.findLeaderElsewhere(currentSystemClient::getClusterMetadata, currentTarget) .thenApply(lookup -> { - // Replaced wholesale rather than merged: the roster is the - // cluster's own answer about where its nodes are, so a node - // it dropped stops being dialed. The configured seed is - // kept separately and outlives it. - if (!lookup.endpoints().isEmpty()) { - rosterTargets = lookup.endpoints(); - } + rememberRoster(lookup); return lookup.redirect(); }); } + /** + * Keeps what a leader check learned about where the cluster's nodes are. + * + * Replaced wholesale rather than merged: the roster is the cluster's own + * answer, so a node it dropped stops being dialed. The configured seed is + * kept separately and outlives it. + * + * An inconclusive check -- an unreadable roster, a metadata read that + * failed -- names no endpoint, and that must leave the last roster + * standing: assigning it anyway would empty the redial candidates exactly + * when the cluster is unreachable, which is when they are needed. + */ + void rememberRoster(LeaderAwareness.LeaderLookup lookup) { + if (!lookup.endpoints().isEmpty()) { + rosterTargets = lookup.endpoints(); + } + } + + /** The roster this client would redial, for tests in this package. */ + List rosterTargets() { + return rosterTargets; + } + + /** Whether a sign-in is remembered for replay, for tests in this package. */ + boolean hasRememberedLogin() { + return rememberedLogin != null; + } + + /** The live connection, for tests in this package. */ + AsyncTcpConnection currentConnection() { + return connection.get(); + } + /** * Endpoints a redial rotates through, likeliest first: where the client * currently is, the address it was configured with, then the roster it diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java index 5d815e07e2..94515314cf 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java @@ -75,10 +75,6 @@ * Manages the connection lifecycle and request/response correlation. */ public class AsyncTcpConnection { - private static final Logger log = LoggerFactory.getLogger(AsyncTcpConnection.class); - private static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofMillis(3000); - // A missing reply must not hold the single VSR-pinned channel forever. - private static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(30); // Transient VSR denials (not-committed / not-accepted) are replayed with // the same encoded frame so the server's dedup sees the same request id. // A not-committed outcome is unknown, so it replays for the whole budget. @@ -86,8 +82,15 @@ public class AsyncTcpConnection { // so after a short same-node retry it is handed to the owning client for // a leader recheck and safe replay; mirrors TRANSIENT_FAILOVER_CHECK_INTERVAL // in core/sdk/src/tcp/tcp_client.rs. - private static final int TRANSIENT_NOT_COMMITTED = 57; - private static final int TRANSIENT_NOT_ACCEPTED = 58; + // + // Package-private: the client classifies a failed sign-in by these codes, + // and a transient one is not a rejected credential. + static final int TRANSIENT_NOT_COMMITTED = 57; + static final int TRANSIENT_NOT_ACCEPTED = 58; + private static final Logger log = LoggerFactory.getLogger(AsyncTcpConnection.class); + private static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofMillis(3000); + // A missing reply must not hold the single VSR-pinned channel forever. + private static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(30); private static final long TRANSIENT_RETRY_INTERVAL_MS = 50; private static final Duration TRANSIENT_RETRY_BUDGET = Duration.ofSeconds(30); private static final Duration NOT_ACCEPTED_RETRY_BUDGET = Duration.ofSeconds(2); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java index 5e68d50a80..fd8080755a 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java @@ -21,7 +21,11 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import org.apache.iggy.client.ConnectionInfo; import org.apache.iggy.config.RetryPolicy; +import org.apache.iggy.exception.IggyConnectionException; +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyServerException; import org.junit.jupiter.api.Test; import java.io.EOFException; @@ -36,6 +40,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.List; +import java.util.Optional; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -383,4 +388,52 @@ static Response success(int operation, ByteBuf body) { private interface RequestHandler { Response handle(Request request); } + + /** + * A roster read that learned nothing must leave the last one standing: + * emptying the redial candidates when the cluster is unreachable takes them + * away exactly when they are needed. + */ + @Test + void shouldKeepTheLastRosterWhenALookupLearnedNothing() { + InetAddress loopback = InetAddress.getLoopbackAddress(); + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(8090) + .build(); + List roster = List.of(new ConnectionInfo("iggy-0", 8091), new ConnectionInfo("iggy-1", 8092)); + + client.rememberRoster(new LeaderAwareness.LeaderLookup(Optional.empty(), roster)); + assertThat(client.rosterTargets()).isEqualTo(roster); + + client.rememberRoster(LeaderAwareness.LeaderLookup.inconclusive()); + assertThat(client.rosterTargets()) + .as("an inconclusive check erased the endpoints the client still needs") + .isEqualTo(roster); + } + + /** + * Only the server answering "no" ends a redial. A channel that closed + * before the reply, or a node that refuses because it is not the primary, + * says nothing about the credentials -- treated as a rejection, they are + * dropped and the client is published on a node that is already gone. + */ + @Test + void shouldTreatOnlyANonTransientServerVerdictAsASignInRejection() { + assertThat(AsyncIggyTcpClient.isSignInRejection( + IggyServerException.fromTcpResponse(IggyErrorCode.INVALID_CREDENTIALS.getCode(), new byte[0]))) + .isTrue(); + assertThat(AsyncIggyTcpClient.isSignInRejection(new IggyConnectionException("channel closed"))) + .as("a channel that died mid sign-in is not a rejected credential") + .isFalse(); + assertThat(AsyncIggyTcpClient.isSignInRejection(new IOException("connection reset"))) + .isFalse(); + assertThat(AsyncIggyTcpClient.isSignInRejection( + IggyServerException.fromTcpResponse(AsyncTcpConnection.TRANSIENT_NOT_ACCEPTED, new byte[0]))) + .as("a node that is not the primary refuses transiently; another endpoint answers") + .isFalse(); + assertThat(AsyncIggyTcpClient.isSignInRejection( + IggyServerException.fromTcpResponse(AsyncTcpConnection.TRANSIENT_NOT_COMMITTED, new byte[0]))) + .isFalse(); + } } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java index fe32b9cece..d339cebf7b 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java @@ -35,8 +35,11 @@ import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import static org.assertj.core.api.Assertions.assertThat; @@ -159,6 +162,119 @@ void shouldReplayTransientImplicitLoginAfterEviction() throws Exception { } } + /** + * Closing is caller intent, like a logout. A sign-in still in flight when + * it happens must not put its credentials back: `connect()` clears the + * closed flag, so the next connection loss would replay a session the + * caller had ended. + */ + @Test + void shouldNotRememberASignInThatLandedAfterClose() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { + AtomicInteger registrations = new AtomicInteger(); + CompletableFuture server = serve(serverSocket, 4, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); + } + if (request.operation() == OPERATION_REGISTER) { + int attempt = registrations.incrementAndGet(); + if (attempt > 1) { + // The second sign-in is the one racing the close. + try { + Thread.sleep(300); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + return Response.success(OPERATION_REGISTER, registerBody(attempt)); + } + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(serverSocket.getLocalPort()) + .requestTimeout(Duration.ofSeconds(5)) + .build(); + client.connect().get(5, TimeUnit.SECONDS); + client.users().login("iggy", "iggy").get(5, TimeUnit.SECONDS); + + CompletableFuture racing = client.users().login("iggy", "iggy"); + Thread.sleep(50); + client.close().get(5, TimeUnit.SECONDS); + racing.handle((ignored, error) -> null).get(5, TimeUnit.SECONDS); + + assertThat(client.hasRememberedLogin()) + .as("a sign-in that landed after the close put its credentials back") + .isFalse(); + server.completeExceptionally(new IllegalStateException("test over")); + } + } + + /** + * The user an eviction re-establishes must not depend on which failure + * ran. Configured credentials are what every connect of this client signs + * in as, so they are what comes back -- not whoever the caller signed in + * as by hand, which is what the connection's captured login would replay. + */ + @Test + void shouldSignInAsTheConfiguredUserAfterAnEviction() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { + AtomicInteger registrations = new AtomicInteger(); + List registeredLogins = new CopyOnWriteArrayList<>(); + AtomicBoolean evict = new AtomicBoolean(true); + CompletableFuture server = serve(serverSocket, 6, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); + } + if (request.operation() == OPERATION_REGISTER) { + registeredLogins.add(request.bodyAsText()); + return Response.success(OPERATION_REGISTER, registerBody(registrations.incrementAndGet())); + } + if (request.operation() == OPERATION_CREATE_STREAM && evict.compareAndSet(true, false)) { + return Response.eviction(EVICTION_STALE_CLIENT); + } + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(serverSocket.getLocalPort()) + .credentials("configured", "configured") + .requestTimeout(Duration.ofSeconds(2)) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.login().get(5, TimeUnit.SECONDS); + client.users().login("handrun", "handrun").get(5, TimeUnit.SECONDS); + int registrationsBeforeEviction = registrations.get(); + + assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .hasCauseInstanceOf(IggyServerException.class); + + // The sign-in that follows the eviction runs on its own, so + // give it a moment to land. + long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos(); + while (registrations.get() == registrationsBeforeEviction && System.nanoTime() < deadline) { + Thread.sleep(25); + } + assertThat(registrations.get()) + .as("the eviction was not followed by a sign-in") + .isGreaterThan(registrationsBeforeEviction); + assertThat(registeredLogins.get(registeredLogins.size() - 1)) + .as("the eviction revived the hand-run sign-in instead of the configured one") + .contains("configured") + .doesNotContain("handrun"); + } finally { + client.close().get(5, TimeUnit.SECONDS); + } + server.completeExceptionally(new IllegalStateException("test over")); + } + } + /** * A stale-client eviction is the server ending the session * authoritatively, like a logout. A client whose credentials were @@ -207,6 +323,12 @@ void shouldNotReviveAHandRunSignInAfterAStaleClientEviction() throws Exception { assertThat(registrations) .as("the evicted session was signed back in") .hasValue(registrationsBeforeEviction); + // The connection replays the login it captured to bring up a + // replacement channel, so that copy has to go too: kept, the + // next channel revives exactly the session the server ended. + assertThat(client.currentConnection().authenticationSnapshot()) + .as("the captured login outlived the session it established") + .isEmpty(); } finally { client.close().get(5, TimeUnit.SECONDS); } @@ -290,7 +412,8 @@ private static Request readRequest(InputStream input) throws IOException { return new Request( Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), fields.getInt(REQUEST_CODE_OFFSET), - fields.getLong(REQUEST_ID_OFFSET)); + fields.getLong(REQUEST_ID_OFFSET), + body); } private static void writeResponse(OutputStream output, Request request, Response response) throws IOException { @@ -365,10 +488,15 @@ private static void writeString(ByteBuf body, String value) { body.writeBytes(bytes); } - private record Request(int operation, int commandCode, long requestId) { + private record Request(int operation, int commandCode, long requestId, byte[] body) { boolean is(int expectedCode, int expectedOperation) { return commandCode == expectedCode && operation == expectedOperation; } + + /** The request body as text, for asserting which user a login names. */ + String bodyAsText() { + return new String(body, StandardCharsets.UTF_8); + } } private record Response(int command, int operation, int status, int evictionReason, ByteBuf body) { diff --git a/foreign/node/src/client/client.connection.test.ts b/foreign/node/src/client/client.connection.test.ts index eb1b353e50..6e1dce2742 100644 --- a/foreign/node/src/client/client.connection.test.ts +++ b/foreign/node/src/client/client.connection.test.ts @@ -472,6 +472,91 @@ describe('IggyConnection', () => { } ); + it('skips the first backoff when another endpoint is known', + async () => { + // The endpoint the client is on is dead and a live one sits behind it in + // the roster: waiting out the interval before the first pass would push + // the failover past what the caller waits for, and the node just lost may + // be gone for good. + const dead = await startServer(); + const deadPort = (dead.address() as AddressInfo).port; + await new Promise((resolve) => dead.close(() => resolve())); + const live = await startServer(); + const livePort = (live.address() as AddressInfo).port; + + const interval = 3000; + const connection = new IggyConnection({ + transport: 'TCP', + options: { host: '127.0.0.1', port: deadPort }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: true, interval, maxRetries: 3 }, + maxResponseFrameSize: FRAME_LIMIT + }); + connection.on('error', () => undefined); + try { + connection.rememberRoster([{ host: '127.0.0.1', port: livePort }]); + const dialed = once(live, 'connection'); + const started = Date.now(); + void connection.connect().catch(() => undefined); + await dialed; + + assert.ok(Date.now() - started < interval, + 'the failover waited out the backoff before its first pass' + ); + } finally { + connection._destroy(); + await new Promise((resolve) => live.close(() => resolve())); + } + } + ); + + it('bounds a dial that never becomes usable when others are queued behind it', + async () => { + // Plain TCP behind a TLS client: the socket connects, so only a bound on + // the handshake ends the attempt. The endpoint behind it is dead, so the + // pass has to end on its own rather than hang on the first one. + const silent = await startServer(); + const silentPort = (silent.address() as AddressInfo).port; + const held: Socket[] = []; + silent.on('connection', (socket) => { held.push(socket); }); + const dead = await startServer(); + const deadPort = (dead.address() as AddressInfo).port; + await new Promise((resolve) => dead.close(() => resolve())); + + const connection = new IggyConnection({ + transport: 'TLS', + options: { + host: '127.0.0.1', + port: silentPort, + rejectUnauthorized: false + }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: true, interval: 10, maxRetries: 3 }, + maxResponseFrameSize: FRAME_LIMIT + }); + connection.on('error', () => undefined); + try { + connection.rememberRoster([{ host: '127.0.0.1', port: deadPort }]); + void connection.connect().catch(() => undefined); + + // Unbounded, the first dial never ends and this endpoint is dialed + // exactly once, forever. + const deadline = Date.now() + 8_000; + while (held.length < 2 && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 50)); + + assert.ok(held.length >= 2, + 'a dial that never became usable held the pass' + ); + assert.equal(connection.connected, false); + } finally { + connection._destroy(); + held.forEach((socket) => socket.destroy()); + await new Promise((resolve) => silent.close(() => resolve())); + } + } + ); + it('stops a redial pass that is destroyed part-way through', async () => { // The endpoint the client is on is dead, so every dial to it is refused diff --git a/foreign/node/src/client/client.connection.ts b/foreign/node/src/client/client.connection.ts index 16269a1538..ea5c7dcbd9 100644 --- a/foreign/node/src/client/client.connection.ts +++ b/foreign/node/src/client/client.connection.ts @@ -71,15 +71,21 @@ const getTransport = (config: ClientConfig): Socket => { export type Endpoint = { host: string, port: number }; /** - * Bound on one dial while other endpoints are queued behind it. A socket has - * no connect deadline of its own, so a node whose syns are dropped would hold - * the whole pass. Matches the Rust SDK. + * Bound on one dial while other endpoints are queued behind it. Neither the + * connect nor the TLS handshake has a deadline of its own, so a node whose syns + * are dropped -- or one that accepts TCP and never answers the ClientHello -- + * would hold the whole pass. Matches the Rust SDK. */ const FAILOVER_DIAL_TIMEOUT_MS = 2_000; /** * Default reconnection settings. - * Attempts reconnection every 5 seconds, up to 12 times. + * + * One retry is one full pass over every endpoint the client knows, so this is + * twelve passes rather than twelve dials, waiting 5 seconds between them. The + * first pass runs at once when more than one endpoint is known: the node just + * lost may be gone for good, and pausing before dialing a survivor only pushes + * the failover past the interval a caller is willing to wait. */ const DefaultReconnectOption: ReconnectOption = { enabled: true, @@ -191,7 +197,10 @@ export class IggyConnection extends EventEmitter { this.emit('error', err); }); - socket.once('connect', () => { + // The readiness event, not 'connect': on TLS the socket is only usable + // once the handshake completes, and writing a request before that would + // announce a connection the peer has not agreed to yet. + socket.once(this._readyEvent(), () => { if (this.socket !== socket) return; debug('socket/connect event'); @@ -235,7 +244,13 @@ export class IggyConnection extends EventEmitter { this.connecting = true; const socket = this.socket; - const connectPromise = this._waitForConnection(socket); + // Bounded here too when the client knows somewhere else to go: this dial + // is not part of a pass, so an endpoint that never becomes usable would + // hold it with no timer of its own and the redial pass would never start. + const connectPromise = this._dialWithin( + socket, + this._redialCandidates().length > 1 + ); this.connectPromise = connectPromise; const clearConnectPromise = () => { if (this.connectPromise === connectPromise) @@ -251,7 +266,9 @@ export class IggyConnection extends EventEmitter { * A socket has no connect deadline of its own and there is no 'timeout' * listener on it, so a node whose syns are dropped holds the pass for the * whole OS connect timeout -- and it leads every pass, because the current - * endpoint only moves on success. The bound matches the Rust SDK's. + * endpoint only moves on success. The bound covers the TLS handshake too, + * which is what `_readyEvent()` waits for and has no deadline of its own + * either. It matches the Rust, Go and C# SDKs'. */ private async _dialWithin(socket: Socket, bounded: boolean): Promise { if (!bounded) @@ -277,10 +294,23 @@ export class IggyConnection extends EventEmitter { } } + /** + * The event that says a socket can carry a request. + * + * On TLS that is 'secureConnect', not 'connect': the latter fires as soon as + * the TCP handshake completes, so waiting on it would treat a peer that + * never answers the ClientHello as connected and leave the handshake with no + * deadline at all. + */ + private _readyEvent(): 'connect' | 'secureConnect' { + return this.config.transport === 'TLS' ? 'secureConnect' : 'connect'; + } + private _waitForConnection(socket: Socket): Promise { + const ready = this._readyEvent(); return new Promise((resolve, reject) => { const cleanup = () => { - socket.removeListener('connect', resolveConnect); + socket.removeListener(ready, resolveConnect); socket.removeListener('error', rejectConnect); socket.removeListener('close', rejectClosed); }; @@ -297,7 +327,7 @@ export class IggyConnection extends EventEmitter { }; socket.once('error', rejectConnect); socket.once('close', rejectClosed); - socket.once('connect', resolveConnect); + socket.once(ready, resolveConnect); }); } diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts index 3a6719a740..e938c7603a 100644 --- a/foreign/node/src/client/client.socket.test.ts +++ b/foreign/node/src/client/client.socket.test.ts @@ -969,6 +969,38 @@ describe('VSR client socket', () => { } }); + it('keeps re-issuing a not-admitted request while the roster still names this node', + async () => { + const server = await startVsrServer((frame, socket) => { + if (frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_031) { + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + singleNodeHandler(server.port)(frame, socket); + }); + const client = new CommandResponseStream(vsrConfig(server.port)); + try { + await client.authenticate(vsrConfig(server.port).credentials); + + // A refusal the roster cannot explain is a wait, not a verdict: an + // election may still be in flight, so the request keeps going for its + // whole budget instead of failing after the first re-check window. + const pending = client.sendCommand(60_031, Buffer.alloc(0)); + const outcome = await Promise.race([ + pending.then(() => 'answered', () => 'gave up'), + new Promise((resolve) => { + setTimeout(() => resolve('still trying'), 4_000).unref(); + }) + ]); + + assert.equal(outcome, 'still trying'); + } finally { + client.destroy(); + await server.close(); + } + } + ); + it('keeps a typed transient error and session at its retry deadline', async () => { const server = await startVsrServer((frame, socket) => { diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts index 746f43e2bd..43d378b844 100644 --- a/foreign/node/src/client/client.socket.ts +++ b/foreign/node/src/client/client.socket.ts @@ -24,7 +24,7 @@ import type { } from '../client/client.type.js'; import { ResponseError, responseError } from '../wire/error.utils.js'; import { debug } from './client.debug.js'; -import { IggyConnection } from './client.connection.js'; +import { type Endpoint, IggyConnection } from './client.connection.js'; import { LOGIN, LOGIN_WITH_TOKEN, LOGOUT, PING } from '../wire/index.js'; import { GET_CLUSTER_METADATA } from '../wire/cluster/get-cluster-metadata.command.js'; import { COMMAND_CODE } from '../wire/command.code.js'; @@ -81,6 +81,8 @@ type Job = { payload: Buffer, /** Whether to parse the response */ handleResponse: boolean, + /** When the whole request gives up, however often it is re-issued */ + deadline: number, /** Promise resolve function */ resolve: (v: CommandResponse | PromiseLike) => void, /** Promise reject function */ @@ -116,6 +118,12 @@ export class CommandResponseStream extends EventEmitter { private authenticationPromise?: Promise; /** Whether a login is already being moved to the leader */ private settlingLeader: boolean; + /** + * Whether a refused request is already re-checking the leader. The roster + * read that re-check runs can be refused the same way, and answering a + * leader check with another leader check would recurse. + */ + private followingLeaderMove = false; /** How long a leaderless roster is polled before settling in place */ private leaderlessWaitBudget: number; /** Delay between roster reads while the cluster elects */ @@ -208,17 +216,37 @@ export class CommandResponseStream extends EventEmitter { // `_processVsr`. The refusal comes back out here instead, where the // queue is free, and the command is re-issued on the node that now // leads. + // + // A not-admitted refusal means the request was never applied, so it is + // re-issued for the whole request budget rather than given up on after + // one window: the roster can still name this node -- an election in + // flight, a leader that has not moved yet -- and that is a wait, not a + // verdict. + // One budget for the whole request: the transient replays on a + // connection, the leader re-checks, and the re-issues after a move all + // spend it, so a request cannot outlive it by moving. + const deadline = Date.now() + VSR_RESPONSE_TIMEOUT_MS; let response: CommandResponse; - for (let move = 0; ; move += 1) { + for (;;) { try { response = await this._queueCommand(command, payload, handleResponse, - last); + last, deadline); break; } catch (error) { if (!(error instanceof LeaderMovedError)) throw error; - if (move >= MAX_LEADER_REDIRECTS || !await this._followLeaderMove()) + // The roster read itself is refused: it runs through this same + // path, and re-checking the leader to answer a leader check would + // recurse. Its caller reads a failure as "stay where you are". + if (this.followingLeaderMove || Date.now() >= deadline) throw responseError(command, error.refusal.errorCode); + await this._followLeaderMove(); + // A move drops the session with the socket it was bound to, so the + // re-issue would otherwise go out under no session: a replicated + // command fails client-side, a non-replicated one goes out with + // session 0. + if (!this.isAuthenticated && !this.isUnloggedCommand(command)) + await this.authenticate(this.options.credentials); } } if (!isLoginCommand(command) || this.settlingLeader) @@ -240,13 +268,15 @@ export class CommandResponseStream extends EventEmitter { command: number, payload: Buffer, handleResponse: boolean, - last: boolean + last: boolean, + deadline: number ): Promise { return new Promise((resolve, reject) => { const job = { command, payload, handleResponse, + deadline, resolve, reject }; @@ -261,16 +291,30 @@ export class CommandResponseStream extends EventEmitter { /** * Re-reads the roster and moves to the leader it names. * - * @returns Whether the client moved, so the refused request is worth - * re-issuing + * Best effort: an unreadable roster, or one that still names this node, + * leaves the client where it is and the refused request is re-issued here + * anyway. Guarded against re-entry, since the roster read is itself a + * command that can be refused the same way. + * + * @returns Whether the client moved */ private async _followLeaderMove(): Promise { - const leader = await this._readLeaderEndpoint(); - if (!leader || this.connection.isConnectedTo(leader.host, leader.port)) + if (this.followingLeaderMove) + return false; + this.followingLeaderMove = true; + try { + const leader = await this._readLeaderEndpoint(); + if (!leader || this.connection.isConnectedTo(leader.host, leader.port)) + return false; + debug(`the leader moved to ${leader.host}:${leader.port}, following it`); + await this.connection.redirect(leader.host, leader.port); + return true; + } catch (error) { + debug('the leader could not be re-checked, staying on this node', error); return false; - debug(`the leader moved to ${leader.host}:${leader.port}, following it`); - await this.connection.redirect(leader.host, leader.port); - return true; + } finally { + this.followingLeaderMove = false; + } } private _rememberRoster(response: CommandResponse): void { @@ -299,9 +343,9 @@ export class CommandResponseStream extends EventEmitter { while (this._execQueue.length > 0 && this.connection.socket.writable) { const next = this._execQueue.shift(); if (!next) break; - const { command, payload, handleResponse, resolve, reject } = next; + const { command, payload, handleResponse, deadline, resolve, reject } = next; try { - resolve(await this._processNext(command, payload, handleResponse)); + resolve(await this._processNext(command, payload, handleResponse, deadline)); } catch (err) { reject(err); } @@ -325,40 +369,46 @@ export class CommandResponseStream extends EventEmitter { * @param command - Command code * @param payload - Command payload * @param handleResp - Whether to parse the response + * @param deadline - When the whole request gives up, shared with the leader + * re-checks and the re-issues after a move * @returns Promise resolving to the command response */ _processNext( command: number, payload: Buffer, - handleResp = true + handleResp = true, + deadline = Date.now() + VSR_RESPONSE_TIMEOUT_MS ): Promise { if (isLoginCommand(command) && this.isAuthenticated) - return this._processVsrLogin(command, payload, handleResp); - return this._processVsr(command, payload, handleResp); + return this._processVsrLogin(command, payload, handleResp, deadline); + return this._processVsr(command, payload, handleResp, deadline); } private async _processVsrLogin( command: number, payload: Buffer, - handleResp: boolean + handleResp: boolean, + deadline: number ): Promise { - await this._processVsr(LOGOUT.code, LOGOUT.serialize(), true); - return this._processVsr(command, payload, handleResp); + await this._processVsr(LOGOUT.code, LOGOUT.serialize(), true, deadline); + return this._processVsr(command, payload, handleResp, deadline); } private async _processVsr( command: number, payload: Buffer, - handleResp: boolean + handleResp: boolean, + deadline: number ): Promise { let requestWritten = false; try { const prepared = prepareVsrCommand(command, payload); // A transient retry must preserve all request identity fields. const frame = this.vsrSession.encode(prepared.command, prepared.payload); - const startedAt = Date.now(); - const deadline = startedAt + VSR_RESPONSE_TIMEOUT_MS; - const notAcceptedDeadline = startedAt + VSR_FAILOVER_CHECK_MS; + // Derived from the request's own budget rather than read off the clock, + // so one request spends one budget however many times it is re-issued. + const notAcceptedDeadline = + deadline - VSR_RESPONSE_TIMEOUT_MS + VSR_FAILOVER_CHECK_MS; let lastTransientError: ResponseError | undefined; let parsed: CommandResponse; while (true) { @@ -547,8 +597,7 @@ export class CommandResponseStream extends EventEmitter { * died between the login and this read), keeps the client on its current * node instead of failing a login that already succeeded. */ - private async _readLeaderEndpoint(): - Promise<{ host: string, port: number } | undefined> { + private async _readLeaderEndpoint(): Promise { // A cluster can be transiently leaderless: a restarted node cedes the // primaryship its stale view assigns it, and the roster reports no leader // until the peers' election completes. That window is roughly one heartbeat diff --git a/foreign/node/src/client/client.type.ts b/foreign/node/src/client/client.type.ts index 40c29bc2cc..63703005f2 100644 --- a/foreign/node/src/client/client.type.ts +++ b/foreign/node/src/client/client.type.ts @@ -100,9 +100,16 @@ export type TransportType = typeof Transports[number]; export type ReconnectOption = { /** Whether automatic reconnection is enabled */ enabled: boolean, - /** Interval between reconnection attempts in milliseconds */ + /** + * Milliseconds to wait between passes. The first pass runs at once when more + * than one endpoint is known. + */ interval: number, - /** Maximum number of reconnection attempts */ + /** + * Maximum number of passes over the known endpoints. One pass dials the + * endpoint the client is on, the endpoint it was configured with, and every + * node the roster named, so this counts passes rather than dials. + */ maxRetries: number } From d0a95b1cbb7966a18946ca16c26246696cf545a0 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Wed, 26 Aug 2026 09:15:42 +0200 Subject: [PATCH 09/16] address review comments --- core/sdk/src/tcp/tcp_client.rs | 128 ++++++++++++------ .../HeartbeatTests.cs | 26 ++-- .../Implementations/TcpMessageStream.Vsr.cs | 8 +- .../Implementations/TcpMessageStream.cs | 35 ++--- .../VsrTests/EndpointFailoverTests.cs | 114 +++++++++------- foreign/csharp/README.md | 27 ++-- foreign/go/client/tcp/tcp_connect_test.go | 13 +- foreign/go/client/tcp/tcp_core.go | 79 ++++++----- foreign/go/client/tcp/tcp_core_review_test.go | 13 +- foreign/go/client/tcp/tcp_failover_test.go | 27 ++-- .../go/client/tcp/tcp_session_management.go | 18 +-- .../client/async/tcp/AsyncIggyTcpClient.java | 84 ++++++------ .../client/async/tcp/AsyncTcpConnection.java | 15 -- ...yncIggyTcpClientTransientFailoverTest.java | 104 ++++---------- foreign/node/src/client/client.connection.ts | 11 +- foreign/node/src/client/client.socket.test.ts | 72 ++++++++++ foreign/node/src/client/client.socket.ts | 84 ++++++++---- foreign/node/src/client/client.type.ts | 8 +- 18 files changed, 485 insertions(+), 381 deletions(-) diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index ba3080bc59..59cb24b018 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -104,6 +104,11 @@ pub struct TcpClient { /// onto this node or, after a failover, another one -- can re-establish /// the session instead of surfacing `Unauthenticated`. Cleared on logout. session_credentials: Mutex>, + /// The password a committed change gave the user a configured `AutoLogin` + /// signs in as. The configured credentials cannot be rewritten, and the + /// password they carry is dead once the change commits, so every later + /// sign-in reads this instead. + configured_password: Mutex>, // `std::sync::Mutex` (not `tokio::sync::Mutex`): the critical section // is `encode_request_header`, which is pure CPU and never awaits. The // tokio variant would pay a waker alloc + internal semaphore on @@ -120,11 +125,6 @@ pub struct TcpClient { struct RememberedSignIn { credentials: Credentials, user_id: u32, - /// Set by a committed password change for `user_id`. The configured - /// `AutoLogin` credentials cannot be rewritten -- the config is shared and - /// immutable -- so this marks the remembered copy as the newer one for - /// that same user, and [`TcpClient::sign_in_credentials`] prefers it. - password_refreshed: bool, } /// A connection that completed every step of coming up, TLS included. @@ -364,7 +364,6 @@ impl iggy_common::VsrSessionControl for TcpClient { .replace(RememberedSignIn { credentials, user_id, - password_refreshed: false, }); } @@ -388,9 +387,27 @@ impl iggy_common::VsrSessionControl for TcpClient { .get_cow_str_value() .is_ok_and(|name| name.as_ref() == username), }; - if targets_session_user { - *password = SecretString::from(new_password.to_owned()); - sign_in.password_refreshed = true; + if !targets_session_user { + return; + } + + *password = SecretString::from(new_password.to_owned()); + // The configured credentials cannot be rewritten -- the config is + // shared and immutable -- and the password they carry will never work + // again, so the new one is kept here instead. Kept outside the + // remembered sign-in on purpose: that record is replaced wholesale by + // every later login, so a marker on it would survive exactly one + // reconnect and the one after that would replay the dead password. + let configured_user_changed = matches!( + &self.config.auto_login, + AutoLogin::Enabled(Credentials::UsernamePassword(configured, _)) + if configured == username + ); + if configured_user_changed { + self.configured_password + .lock() + .await + .replace(SecretString::from(new_password.to_owned())); } } @@ -457,6 +474,7 @@ impl TcpClient { leader_redirection_state: Mutex::new(LeaderRedirectionState::new()), current_server_address: Mutex::new(server_address), roster_endpoints: Mutex::new(Vec::new()), + configured_password: Mutex::new(None), session_credentials: Mutex::new(None), consensus_session: Arc::new(StdMutex::new(ConsensusSession::new())), skip_auto_login_once: Mutex::new(false), @@ -759,44 +777,44 @@ impl TcpClient { /// else the ones a manual sign-in on this client succeeded with. A manual /// sign-in is otherwise less reconnectable than a configured one, which /// is a surprising difference between two ways of doing the same thing. + /// + /// A password change this client committed for the configured user is + /// applied on top: the configured password will never work again, and every + /// later reconnect would otherwise fail `InvalidCredentials`. async fn sign_in_credentials(&self) -> Option { - let remembered = - self.session_credentials + match &self.config.auto_login { + AutoLogin::Enabled(Credentials::UsernamePassword(username, configured_password)) => { + let password = self + .configured_password + .lock() + .await + .clone() + .unwrap_or_else(|| configured_password.clone()); + Some(Credentials::UsernamePassword(username.clone(), password)) + } + AutoLogin::Enabled(credentials) => Some(credentials.clone()), + AutoLogin::Disabled => self + .session_credentials .lock() .await .as_ref() - .map(|remembered: &RememberedSignIn| { - ( - remembered.credentials.clone(), - remembered.password_refreshed, - ) - }); - - match (&self.config.auto_login, remembered) { - // A committed password change for the configured user outranks the - // configured password: the config cannot be rewritten, and signing - // in with the password this client itself replaced would fail - // `InvalidCredentials` on every later reconnect. Same user either - // way -- `refresh_session_password` only marks a change that - // targeted the signed-in one. - ( - AutoLogin::Enabled(Credentials::UsernamePassword(configured_username, _)), - Some((Credentials::UsernamePassword(username, password), true)), - ) if configured_username == &username => { - Some(Credentials::UsernamePassword(username, password)) - } - (AutoLogin::Enabled(configured), _) => Some(configured.clone()), - (AutoLogin::Disabled, remembered) => remembered.map(|(credentials, _)| credentials), + .map(|remembered| remembered.credentials.clone()), } } /// Endpoints to dial for one connect, likeliest first: where the client - /// currently is, then the roster it learned while connected, then the - /// configured seeds. + /// currently is, the addresses it was configured with, then the roster it + /// learned while connected. + /// + /// Configured before learned, as in the other SDKs: those are the endpoints + /// the caller vouched for, while a roster read from a cluster that has since + /// changed shape may name nodes that are gone. async fn dial_candidates(&self) -> Vec { let mut candidates = vec![self.current_server_address.lock().await.clone()]; let roster = self.roster_endpoints.lock().await.clone(); - for endpoint in roster.iter().chain(self.config.failover_addresses.iter()) { + let configured = std::iter::once(&self.config.server_address) + .chain(self.config.failover_addresses.iter()); + for endpoint in configured.chain(roster.iter()) { let mut known = false; for candidate in &candidates { if is_same_address(candidate, endpoint).await { @@ -1609,15 +1627,16 @@ mod tests { "127.0.0.1:8092".to_string(), ]; - // The current endpoint leads, the roster follows, and neither the - // roster's copy of the current endpoint nor a seed that only spells - // the same endpoint differently earns a second dial. + // The current endpoint leads, the configured ones follow, and the + // roster comes last -- the same order as the other SDKs. Neither the + // roster's copy of an endpoint already named nor a seed that only + // spells one differently earns a second dial. assert_eq!( client.dial_candidates().await, vec![ "127.0.0.1:8090".to_string(), - "127.0.0.1:8091".to_string(), "127.0.0.1:8092".to_string(), + "127.0.0.1:8091".to_string(), ] ); } @@ -1804,12 +1823,31 @@ mod tests { ) .await; - match client.sign_in_credentials().await { - Some(Credentials::UsernamePassword(username, password)) => { - assert_eq!(username, "iggy"); - assert_eq!(password.expose_secret(), "new"); + // Every later reconnect, not just the next one: each of them signs in + // and remembers that sign-in afresh, and the configured password is + // dead for good once the change commits. + for reconnect in 0..3 { + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, password)) => { + assert_eq!(username, "iggy"); + assert_eq!( + password.expose_secret(), + "new", + "the configured password came back on reconnect {reconnect}" + ); + // What the reconnect's own login does with what it signed + // in with. + client + .remember_session_credentials( + Credentials::UsernamePassword(username, password), + SESSION_USER_ID, + ) + .await; + } + other => { + panic!("expected the configured user with the new password, got {other:?}") + } } - other => panic!("expected the configured user with the new password, got {other:?}"), } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs index 7a9de7fb67..b0e71feeed 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs @@ -104,28 +104,28 @@ public async Task EvictedClient_WithPersonalAccessTokenAutoLogin_Should_Reconnec me.ConsumerGroupsCount.ShouldBe(0); } + /// + /// An eviction is the server's heartbeat verifier reacting to silence, not caller intent, so a client + /// that signed in by hand recovers from it exactly like one whose credentials were configured: the + /// sign-in it remembered re-establishes the session. Only an explicit sign-out or Dispose ends it. + /// [Test] - public async Task EvictedClient_WithoutAutoLogin_Should_FailFast_And_NotReconnect() + public async Task EvictedClient_WithoutAutoLogin_Should_ReestablishItsSession() { using var client = await CreateClient(TimeSpan.FromHours(1), false); await client.LoginUserAsync("iggy", "iggy"); var (streamName, _) = await JoinFreshGroup(client); - var reconnected = false; - client.SubscribeConnectionEvents(args => - { - reconnected |= args.CurrentState == ConnectionState.Connecting; - return Task.CompletedTask; - }); - await Task.Delay(IdleFor); - // A reconnect could not bring the session back, so the request surfaces the loss instead of coming back - // over an unauthenticated connection. + // The evicted request surfaces the loss; the one after it comes back over a session the remembered + // sign-in re-established. await Should.ThrowAsync(() => client.GetMeAsync()); - reconnected.ShouldBeFalse(); - await Should.ThrowAsync(() => - client.GetStreamByIdAsync(Identifier.String(streamName))); + + var stream = await client.GetStreamByIdAsync(Identifier.String(streamName)); + stream.ShouldNotBeNull(); + var me = await client.GetMeAsync(); + me.ShouldNotBeNull(); } private Task CreateClient(TimeSpan heartbeatInterval, bool autoLogin = true) diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs index ea41f1e0f3..e100fcc546 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs @@ -569,11 +569,9 @@ private async Task> SendRawAsync(int code, ReadOnlyMemory Volatile.Read(ref _isConnecting) != 0; @@ -1348,7 +1349,16 @@ await sslStream.AuthenticateAsClientAsync( /// private static bool IsTlsConfigurationFault(Exception e) { - return e is AuthenticationException or InvalidCertificatePathException; + if (e is InvalidCertificatePathException) + { + return true; + } + + // AuthenticationException also carries a handshake that died on the wire - a reset, a closed socket - + // and that says nothing about the configuration. Only a verdict reached without transport trouble is + // one no retry can change. + return e is AuthenticationException + && e.InnerException is not (IOException or SocketException); } private async Task SendAckAsync(int code, ReadOnlyMemory body, CancellationToken token) @@ -1367,12 +1377,9 @@ private async Task> SendWithResponseAsync(int code, ReadOnlyM { _logger.LogWarning("Connection lost"); - // A server-side eviction is the server ending this session authoritatively, like a logout: the - // remembered sign-in ends with it, so only a configured auto login may bring the session back. - // Remembered credentials exist for transport loss, where the session died with the socket rather - // than by anyone's decision. - ForgetSessionAfterEviction(e); - + // A stale-client eviction is not caller intent: the server's heartbeat verifier sends it after a + // gc pause or a laptop sleep, so the remembered sign-in survives it and the reconnect below + // re-establishes the session. Only an explicit sign-out or Dispose ends it. Same rule in every SDK. if (!_configuration.ReconnectionSettings.Enabled) { _logger.LogWarning("Reconnection is disabled"); @@ -1402,18 +1409,6 @@ private static bool IsLostConnection(Exception e) || e is IggyInvalidStatusCodeException { StatusCode: VsrError.STALE_CLIENT, FromServer: true }; } - // An eviction ends this session authoritatively, like a logout: the sign-in this client remembered goes - // with it, so only a configured auto login may bring the session back. Called from the consensus layer, - // which sees the eviction whatever it interrupted - an eviction that landed on a replicated write is - // reported as VsrRequestOutcomeUnknownException and never reaches the lost-connection path at all. - private void ForgetSessionAfterEviction(Exception verdict) - { - if (verdict is IggyInvalidStatusCodeException { StatusCode: VsrError.STALE_CLIENT, FromServer: true }) - { - _rememberedLogin = null; - } - } - private async Task> HandleReconnectionAsync(int code, ReadOnlyMemory body, bool autoLogin, CancellationToken token) { diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs index 2d88801845..b835fbe560 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs @@ -105,12 +105,12 @@ public async Task ResumesOnASurvivorAfterTheSignedInNodeDies() /// /// Mirrors the integration contract (HeartbeatTests - /// EvictedClient_WithoutAutoLogin_Should_FailFast_And_NotReconnect): a server-side eviction ends the - /// session authoritatively, so the credentials a manual sign-in remembered must not resurrect it - the - /// evicted request surfaces the loss with no reconnect attempt. + /// EvictedClient_WithoutAutoLogin_Should_ReestablishItsSession): an eviction comes off the server's + /// heartbeat timer rather than from the caller, so the sign-in this client remembered survives it and + /// the reconnect re-establishes the session. Same rule in every SDK. /// [Fact] - public async Task ServerEvictionForgetsTheRememberedSignIn() + public async Task ServerEvictionReplaysTheRememberedSignIn() { using var node = new MockNode(); var evict = false; @@ -121,11 +121,15 @@ public async Task ServerEvictionForgetsTheRememberedSignIn() return Reply(OperationRegister, RegisterBody(session: 128)); } - return evict - ? EvictionFrame(EvictionStaleClient) - : Reply(OperationNonReplicated, request.Code == GetClusterMetadataCode - ? ClusterMetadata(node.Port, node.Port, node.Port) - : []); + if (evict) + { + evict = false; + return EvictionFrame(EvictionStaleClient); + } + + return Reply(OperationNonReplicated, request.Code == GetClusterMetadataCode + ? ClusterMetadata(node.Port, node.Port, node.Port) + : []); }); var configuration = new IggyClientConfigurator @@ -144,29 +148,25 @@ public async Task ServerEvictionForgetsTheRememberedSignIn() await client.ConnectAsync(TestContext.Current.CancellationToken); await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); await client.PingAsync(TestContext.Current.CancellationToken); - var connectionsBeforeEviction = node.Connections; + var registrationsBeforeEviction = node.Registrations; + // A ping is replay-safe, so the eviction is absorbed: the reconnect signs in again with the + // remembered credentials and the request completes over the session it re-established. evict = true; - var evicted = await Assert.ThrowsAsync(() => - client.PingAsync(TestContext.Current.CancellationToken)); - Assert.Equal(VsrError.STALE_CLIENT, evicted.StatusCode); - Assert.True(evicted.FromServer); - Assert.Equal(connectionsBeforeEviction, node.Connections); - - // The dropped connection leaves the next call transport-shaped, but the eviction forgot the remembered - // sign-in, so it must fail fast instead of reconnecting into a resurrected session. - await Assert.ThrowsAsync(() => - client.PingAsync(TestContext.Current.CancellationToken)); - Assert.Equal(connectionsBeforeEviction, node.Connections); + await client.PingAsync(TestContext.Current.CancellationToken); + + Assert.True(node.Registrations > registrationsBeforeEviction, + "the reconnect signed in again with the remembered credentials"); + await client.PingAsync(TestContext.Current.CancellationToken); } /// - /// The same contract when the eviction lands on a replicated write: that request is reported as - /// outcome-unknown rather than as a lost connection, so it never passes through the lost-connection - /// path, and the session it evicted still has to be forgotten. + /// The same rule when the eviction lands on a replicated write: that request is reported as + /// outcome-unknown, because its own outcome is unknown, but the session behind it is still + /// re-established for the requests that follow. /// [Fact] - public async Task ServerEvictionDuringAReplicatedWriteForgetsTheRememberedSignIn() + public async Task ServerEvictionDuringAReplicatedWriteReplaysTheRememberedSignIn() { using var node = new MockNode(); var evict = false; @@ -177,11 +177,15 @@ public async Task ServerEvictionDuringAReplicatedWriteForgetsTheRememberedSignIn return Reply(OperationRegister, RegisterBody(session: 128)); } - return evict - ? EvictionFrame(EvictionStaleClient) - : Reply(OperationNonReplicated, request.Code == GetClusterMetadataCode - ? ClusterMetadata(node.Port, node.Port, node.Port) - : []); + if (evict) + { + evict = false; + return EvictionFrame(EvictionStaleClient); + } + + return Reply(OperationNonReplicated, request.Code == GetClusterMetadataCode + ? ClusterMetadata(node.Port, node.Port, node.Port) + : []); }); var configuration = new IggyClientConfigurator @@ -200,28 +204,27 @@ public async Task ServerEvictionDuringAReplicatedWriteForgetsTheRememberedSignIn await client.ConnectAsync(TestContext.Current.CancellationToken); await client.LoginUserAsync("iggy", "iggy", TestContext.Current.CancellationToken); await client.PingAsync(TestContext.Current.CancellationToken); - var connectionsBeforeEviction = node.Connections; + var registrationsBeforeEviction = node.Registrations; evict = true; await Assert.ThrowsAsync(() => client.CreateStreamAsync("evicted-mid-write", token: TestContext.Current.CancellationToken)); - Assert.Equal(connectionsBeforeEviction, node.Connections); - await Assert.ThrowsAsync(() => - client.PingAsync(TestContext.Current.CancellationToken)); - Assert.Equal(connectionsBeforeEviction, node.Connections); + await client.PingAsync(TestContext.Current.CancellationToken); + Assert.True(node.Registrations > registrationsBeforeEviction, + "the reconnect signed in again with the remembered credentials"); } /// - /// A survivor that only comes up after the first rotation still has to be found. The reconnection - /// budget counts rotations, not dials, so one retry is one full pass over every endpoint the client - /// knows rather than one dial of the endpoint it started from. + /// A survivor that is not listening yet when its node dies still has to be found: the client keeps + /// rotating over every endpoint it knows, so one that comes up while it is retrying is dialed on a + /// later pass rather than only on the first. /// [Fact] - public async Task ResumesOnASurvivorThatComesUpAfterTheFirstRotation() + public async Task ResumesOnASurvivorThatComesUpWhileTheClientIsRetrying() { - // A port nothing listens on yet: the survivor comes up on it only after the first rotation has - // already failed on both endpoints. + // A port nothing listens on: the survivor binds it only after the client has already failed on both + // endpoints, so the first pass cannot be the one that finds it. var probe = new TcpListener(IPAddress.Loopback, 0); probe.Start(); var survivorPort = (ushort)((IPEndPoint)probe.LocalEndpoint).Port; @@ -240,10 +243,8 @@ public async Task ResumesOnASurvivorThatComesUpAfterTheFirstRotation() ReconnectionSettings = new ReconnectionSettings { Enabled = true, - // One retry: the rotation after the first failure is the last one, which is where the - // budget check used to cut the sweep short. - MaxRetries = 1, - InitialDelay = TimeSpan.FromMilliseconds(600) + MaxRetries = 4, + InitialDelay = TimeSpan.FromMilliseconds(200) } }; using var client = new TcpMessageStream(configuration, NullLoggerFactory.Instance); @@ -253,19 +254,32 @@ public async Task ResumesOnASurvivorThatComesUpAfterTheFirstRotation() await client.PingAsync(TestContext.Current.CancellationToken); primary.Kill(); - using var survivor = new MockNode(survivorPort); + MockNode? survivor = null; + // Constructed inside the delay, because the listener starts in the constructor: built up front, the + // survivor would be answering from the very first dial and nothing about the later passes would be + // exercised. var comesUp = Task.Run(async () => { - await Task.Delay(250, TestContext.Current.CancellationToken); + await Task.Delay(300, TestContext.Current.CancellationToken); + survivor = new MockNode(survivorPort); survivor.Serve(request => request.Code == GetClusterMetadataCode ? Reply(OperationNonReplicated, ClusterMetadata(primary.Port, survivorPort, survivorPort)) : Answer(request)); }, TestContext.Current.CancellationToken); - await client.PingAsync(TestContext.Current.CancellationToken); - await comesUp; + try + { + await client.PingAsync(TestContext.Current.CancellationToken); + await comesUp; - Assert.True(survivor.Registrations >= 1, "the session was re-established on the survivor"); + Assert.NotNull(survivor); + Assert.True(survivor!.Registrations >= 1, "the session was re-established on the survivor"); + } + finally + { + await comesUp; + survivor?.Dispose(); + } } private static byte[] EvictionFrame(byte reason) diff --git a/foreign/csharp/README.md b/foreign/csharp/README.md index 316ef3cb04..3fed133b1b 100644 --- a/foreign/csharp/README.md +++ b/foreign/csharp/README.md @@ -108,8 +108,9 @@ var client = IggyClientFactory.CreateClient(new IggyClientConfigurator BackoffMultiplier = 2.0 }, - // Auto-login after connection. Reconnection needs it: without credentials to replay a reconnect cannot - // restore the session, so a lost connection fails the request instead + // Auto-login after connection. Optional for reconnection: a client that signs in with + // LoginUserAsync has that sign-in replayed on a reconnect too. Without either, a reconnect + // cannot restore the session and a lost connection fails the request AutoLoginSettings = AutoLoginSettings.For("your_username", "your_password"), // or AutoLoginSettings.ForPersonalAccessToken("your_token") @@ -196,14 +197,20 @@ The SDK replays a request whenever the server says it never admitted it. Two cas to `WithConnection`. Before, a builder-created client came back from a reconnect unauthenticated; now the credentials are held for the lifetime of the connection and replayed. - The TCP client now pings the server every `HeartbeatInterval` (5 seconds, always on) on its own, and - reconnection is on by default (it was off before) with unlimited retries, like the Rust client. Only a - failed dial is retried; a rejected certificate, bad credentials or a missing leader is thrown right away. - A dropped connection fails every in-flight request at once; they share a single reconnect and are replayed - on the connection it establishes. With the default `MaxRetries = 0` an unreachable server is retried - forever, so a request that passes no `CancellationToken` waits for as long as the server stays down - set - `MaxRetries` or pass a token to bound it. Reconnection only replays a request when `AutoLoginSettings` can - restore the session; a client that logged in by hand fails fast on a lost connection. Set - `ReconnectionSettings.Enabled = false` to opt out of reconnection. + reconnection is on by default (it was off before) with unlimited retries, like the Rust client. Bringing an + endpoint up is what gets another attempt, and one retry is a full pass over every endpoint the client knows - + where it is, the configured address, and every node the roster named - rather than one dial of the first. + Bad credentials or a missing leader is thrown right away, and so is a TLS fault no retry can fix (an + unreadable CA file, a certificate this client will never accept), once the pass has given the other + endpoints their turn. A dropped connection fails every in-flight request at once; they share a single + reconnect and are replayed on the connection it establishes. With the default `MaxRetries = 0` an + unreachable server is retried forever, so a request that passes no `CancellationToken` waits for as long as + the server stays down - set `MaxRetries` or pass a token to bound it. A reconnect restores the session from + `AutoLoginSettings` or from the sign-in a `LoginUserAsync` call succeeded with, so a client that logged in by + hand reconnects too; without either there is nothing to restore and the request fails. A server-side + eviction (the heartbeat verifier reacting to silence) is recovered from the same way; only `LogoutUserAsync` + or `Dispose` ends a session for good. Set `ReconnectionSettings.Enabled = false` to opt out of + reconnection. - `AutoLoginSettings` properties are now `init`-only, as is `IggyClientConfigurator.HeartbeatInterval`. Build them with an object initializer or the `AutoLoginSettings.For` / `AutoLoginSettings.ForPersonalAccessToken` factories instead of assigning after construction. diff --git a/foreign/go/client/tcp/tcp_connect_test.go b/foreign/go/client/tcp/tcp_connect_test.go index aa3ce3a12e..12fcd615ac 100644 --- a/foreign/go/client/tcp/tcp_connect_test.go +++ b/foreign/go/client/tcp/tcp_connect_test.go @@ -495,7 +495,18 @@ func TestExchange_DoesNotPreemptAReplayedSignIn(t *testing.T) { // The connect sign-in, the dropped explicit one, and its replay. An // automatic sign-in on the new connection would make a fourth. assert.Equal(t, 3, signIns) - assert.False(t, client.skipAutoLoginOnce, "the suppression fires exactly once") + + // The suppression rides the replay's own context, so nothing about it + // outlives that call: the next Connect signs in again. + require.NoError(t, client.disconnect()) + require.NoError(t, client.Connect(context.Background())) + signIns = 0 + for _, recorded := range server.recorded() { + if recorded.operation() == vsr.OperationRegister { + signIns++ + } + } + assert.Equal(t, 4, signIns, "the suppression leaked past the call that meant it") } func TestExchange_FailsFastWhenAutoLoginIsOff(t *testing.T) { diff --git a/foreign/go/client/tcp/tcp_core.go b/foreign/go/client/tcp/tcp_core.go index 0ebb91c312..9cad23f743 100644 --- a/foreign/go/client/tcp/tcp_core.go +++ b/foreign/go/client/tcp/tcp_core.go @@ -79,9 +79,6 @@ type IggyTcpClient struct { // session carries the consensus client identity and request watermark; // guarded by c.mtx. session *vsr.Session - // skipAutoLoginOnce suppresses the next automatic sign-in so a replayed - // login is not preempted by one the reconnect issues; guarded by c.mtx. - skipAutoLoginOnce bool // loggedOut records an explicit sign-out, so a reconnect's automatic // sign-in does not silently reverse it; guarded by c.mtx. loggedOut bool @@ -466,6 +463,19 @@ func appendCommandFrame(buf []byte, cmd command.Command) ([]byte, error) { // deadlock on that lock or recurse Connect without a bound. type connectScoped struct{} +// skipAutoLogin marks the context of a Connect whose caller owns the sign-in: +// a replayed login, or a redirect inside the sign-in transaction. Carried on +// the context rather than on the client, so it cannot outlive the call that +// meant it -- a client-wide flag leaks when Connect returns early on the +// already-connected gate, and then suppresses somebody else's auto-login. +type skipAutoLogin struct{} + +// suppressAutoLogin returns ctx marked so the Connect it drives does not sign +// in by itself. +func suppressAutoLogin(ctx context.Context) context.Context { + return context.WithValue(ctx, skipAutoLogin{}, struct{}{}) +} + // localPreconditionError marks a request that failed before its frame was // written. The connection is healthy, so exchange must not tear it down and // re-dial over what is purely local state. @@ -482,18 +492,10 @@ func (c *IggyTcpClient) exchange(ctx context.Context, code uint32, frame []byte) return response, err } - // A stale-client eviction is the server ending this session - // authoritatively, like a logout: the remembered sign-in ends with it, so - // only a configured auto-login may bring the session back. Remembered - // credentials exist for transport loss, where the session died with the - // socket rather than by anyone's decision. This runs before the gates - // below because they all return: with reconnection disabled the eviction - // would otherwise never be forgotten, and the next manual Connect would - // sign in with the evicted session's credentials. - if errors.Is(err, ierror.ErrStaleClient) { - c.forgetLogin() - } - + // A stale-client eviction is not caller intent: the heartbeat verifier + // sends it after a gc pause or a laptop sleep, so the remembered sign-in + // survives it and the reconnect re-establishes the session. Only an + // explicit sign-out ends it. Same rule in every SDK. var precondition *localPreconditionError if errors.As(err, &precondition) { return nil, err @@ -532,22 +534,20 @@ func (c *IggyTcpClient) exchange(ctx context.Context, code uint32, frame []byte) if disconnectErr := c.disconnect(); disconnectErr != nil { return nil, disconnectErr } + reconnectCtx := ctx if login { - c.mtx.Lock() - c.skipAutoLoginOnce = true - c.mtx.Unlock() + // The caller replays the login itself, so the reconnect must not. + reconnectCtx = suppressAutoLogin(ctx) } + c.mtx.Lock() + serverAddress := c.currentServerAddress + c.mtx.Unlock() c.logger.Info("Reconnecting to the server...", - slog.String("server_address", c.currentServerAddress), + slog.String("server_address", serverAddress), slog.Any("error", err)) - if reconnectErr := c.Connect(ctx); reconnectErr != nil { - if login { - c.mtx.Lock() - c.skipAutoLoginOnce = false - c.mtx.Unlock() - } + if reconnectErr := c.Connect(reconnectCtx); reconnectErr != nil { return nil, reconnectErr } return c.sendFrame(ctx, code, frame) @@ -648,18 +648,15 @@ func (c *IggyTcpClient) sendFrame(ctx context.Context, code uint32, frame []byte // on the reconnect path would wait on that lock forever. The // transaction signs in itself on the node it lands on, so the // reconnect must not. - connectScopedRequest := ctx.Value(connectScoped{}) != nil - if connectScopedRequest { - c.mtx.Lock() - c.skipAutoLoginOnce = true - c.mtx.Unlock() + redirectCtx := ctx + if ctx.Value(connectScoped{}) != nil { + // Issued from inside the sign-in transaction, which holds + // registerMtx: the automatic sign-in on the reconnect path + // would wait on that lock forever. The transaction signs in + // itself on the node it lands on, so the reconnect must not. + redirectCtx = suppressAutoLogin(ctx) } - if connectErr := c.Connect(ctx); connectErr != nil { - if connectScopedRequest { - c.mtx.Lock() - c.skipAutoLoginOnce = false - c.mtx.Unlock() - } + if connectErr := c.Connect(redirectCtx); connectErr != nil { return nil, connectErr } stamped = false @@ -1029,12 +1026,14 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { // The server fence does not survive the old socket, so the new connection // starts from a fresh client identity. c.session.Reset() - skipAutoLogin := c.skipAutoLoginOnce - c.skipAutoLoginOnce = false - c.logger.Info("Iggy client has connected to the Iggy server", slog.String("client_address", c.clientAddress), slog.String("server_address", c.currentServerAddress)) + clientAddress := c.clientAddress + serverAddress := c.currentServerAddress c.mtx.Unlock() + c.logger.Info("Iggy client has connected to the Iggy server", + slog.String("client_address", clientAddress), + slog.String("server_address", serverAddress)) - if err := c.establishSession(ctx, skipAutoLogin); err != nil { + if err := c.establishSession(ctx, ctx.Value(skipAutoLogin{}) != nil); err != nil { _ = c.disconnect() return err } diff --git a/foreign/go/client/tcp/tcp_core_review_test.go b/foreign/go/client/tcp/tcp_core_review_test.go index f9deff4f76..4d0c2e5cdd 100644 --- a/foreign/go/client/tcp/tcp_core_review_test.go +++ b/foreign/go/client/tcp/tcp_core_review_test.go @@ -136,14 +136,19 @@ func TestConnect_SuppressedSignInSendsNothing(t *testing.T) { client := newDialingClient(t, server.address(), WithAutoLogin(NewUsernamePasswordCredentials("iggy", "iggy"))) - // The state a replayed login leaves behind before its reconnect. - client.skipAutoLoginOnce = true - require.NoError(t, client.Connect(context.Background())) + // The context a replayed login drives its reconnect with. + require.NoError(t, client.Connect(suppressAutoLogin(context.Background()))) assert.Empty(t, server.recorded(), "the replayed login owns the sign-in; Connect must not preempt it") - assert.False(t, client.skipAutoLoginOnce, "the suppression is consumed exactly once") + + // And it is that context, not the client, that carries the suppression: + // a Connect without it signs in. + require.NoError(t, client.disconnect()) + require.NoError(t, client.Connect(context.Background())) + assert.NotEmpty(t, server.recorded(), + "the suppression outlived the call that meant it") } func TestClose_InterruptsAnInFlightReplayWait(t *testing.T) { diff --git a/foreign/go/client/tcp/tcp_failover_test.go b/foreign/go/client/tcp/tcp_failover_test.go index 30badd8a34..020e8b2b89 100644 --- a/foreign/go/client/tcp/tcp_failover_test.go +++ b/foreign/go/client/tcp/tcp_failover_test.go @@ -124,17 +124,20 @@ func TestFailover_FailsFastWhenNothingEverSignedIn(t *testing.T) { "a client that never signed in cannot restore a session by reconnecting") } -// A stale-client eviction is the server ending the session authoritatively, -// like a logout: the remembered sign-in must not resurrect it, so the evicted -// request surfaces the loss instead of reconnecting into a fresh session. -func TestFailover_ServerEvictionForgetsTheRememberedSignIn(t *testing.T) { +// A stale-client eviction is not caller intent: the heartbeat verifier sends it +// after a gc pause or a laptop sleep, and a client that signed in by hand has +// to recover from it exactly like one with a configured auto-login. Same rule +// in every SDK. +func TestFailover_ServerEvictionReplaysTheRememberedSignIn(t *testing.T) { var server *testListener var evict atomic.Bool + var registers atomic.Int32 server = listenVSR(t, nil, func(_, _ int, read request) []byte { if read.operation() == vsr.OperationRegister { + registers.Add(1) return registerReplyFrame(7, 128) } - if evict.Load() { + if evict.CompareAndSwap(true, false) { return evictionFrame(vsr.EvictionStaleClient, 0, 0) } if read.code() == uint32(command.GetClusterMetadataCode) { @@ -148,15 +151,19 @@ func TestFailover_ServerEvictionForgetsTheRememberedSignIn(t *testing.T) { require.NoError(t, client.Connect(ctx)) _, err := client.LoginUser(ctx, "iggy", "iggy") require.NoError(t, err) - connectionsBefore := server.connections() + registersBefore := registers.Load() evict.Store(true) - require.Error(t, client.Ping(ctx), "the evicted request surfaces the loss") + // The evicted request is answered by the eviction, and the reconnect it + // triggers signs in again with the credentials the sign-in remembered. + _ = client.Ping(ctx) _, remembered := client.signInCredentials() - assert.False(t, remembered, "the eviction forgot the remembered sign-in") - assert.Equal(t, connectionsBefore, server.connections(), - "no reconnect dial resurrected the evicted session") + assert.True(t, remembered, "an eviction is not a sign-out; the credentials stay") + require.NoError(t, client.Ping(ctx), "the session came back on its own") + assert.Greater(t, registers.Load(), registersBefore, + "the reconnect re-established the session") + assert.True(t, client.session.Bound()) } // An explicit sign-out is caller intent: the reconnect must not sign back in diff --git a/foreign/go/client/tcp/tcp_session_management.go b/foreign/go/client/tcp/tcp_session_management.go index d2e5341161..4da1fb23b5 100644 --- a/foreign/go/client/tcp/tcp_session_management.go +++ b/foreign/go/client/tcp/tcp_session_management.go @@ -76,7 +76,10 @@ func (c *IggyTcpClient) register( c.registerMtx.Lock() defer c.registerMtx.Unlock() - c.logger.Info("Iggy client is signing in...", slog.String("client_address", c.clientAddress)) + c.mtx.Lock() + clientAddress := c.clientAddress + c.mtx.Unlock() + c.logger.Info("Iggy client is signing in...", slog.String("client_address", clientAddress)) if err := c.endBoundSession(ctx); err != nil { return nil, err @@ -136,8 +139,11 @@ func (c *IggyTcpClient) signIn(ctx context.Context, code uint32, body []byte) (* return nil, err } + c.mtx.Lock() + signedInAddress := c.clientAddress + c.mtx.Unlock() c.logger.Info("Iggy client has signed in successfully.", - slog.String("client_address", c.clientAddress), + slog.String("client_address", signedInAddress), slog.String("server_version", registered.ServerVersion)) return &iggcon.IdentityInfo{UserId: registered.UserID}, nil } @@ -169,13 +175,7 @@ func (c *IggyTcpClient) settleOnLeader(ctx context.Context, code uint32, body [] // The replayed sign-in below owns the session; the redirected Connect // must not sign in on its own, or the replay commits a second Register. - c.mtx.Lock() - c.skipAutoLoginOnce = true - c.mtx.Unlock() - if err := c.Connect(ctx); err != nil { - c.mtx.Lock() - c.skipAutoLoginOnce = false - c.mtx.Unlock() + if err := c.Connect(suppressAutoLogin(ctx)); err != nil { return nil, err } settled, err = c.signIn(ctx, code, body) diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java index ef2baa5ed7..2bec95acb9 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java @@ -500,42 +500,21 @@ private AsyncTcpConnection openConnection(ConnectionInfo target) { /** * A server-side eviction reached this client. The routing state it cached - * belonged to the evicted session, and a stale-client eviction is the - * server ending that session authoritatively, like a logout: nothing may - * revive it behind the caller's back. + * belonged to the evicted session, so it goes. * - * The captured login always goes, whatever is configured. The connection - * replays it to bring up a replacement channel, so keeping it would revive - * exactly the session the server ended -- and on a client whose caller - * signed in as somebody other than the configured user, it would revive a - * different user than a redial replays, making the outcome depend on which - * failure ran. - * - * What may re-establish the session is what every connect of this client - * signs in as: credentials configured on the builder. A sign-in the caller - * ran is theirs to repeat. + * The sign-in stays. A stale-client eviction is not caller intent: the + * server's heartbeat verifier sends it after a gc pause or a laptop sleep, + * and a client that signed in by hand has to recover from it exactly like + * one whose credentials were configured. The connection re-authenticates + * the replacement channel from the login it captured, which is the same + * sign-in a redial would replay. Same rule in every SDK; only an explicit + * sign-out or close ends a session for good. */ private void onSessionReset(int errorCode) { routingState.clearAssignments(); - if (errorCode != IggyErrorCode.STALE_CLIENT.getCode()) { - return; - } - AsyncTcpConnection currentConnection = connection.get(); - if (currentConnection != null) { - currentConnection.forgetCapturedLogin(); + if (errorCode == IggyErrorCode.STALE_CLIENT.getCode()) { + log.debug("The server evicted this session as stale; the next request re-establishes it"); } - if (username.isEmpty() || password.isEmpty()) { - log.warn("The server evicted this session as stale; the sign-in it ran will not be replayed"); - rememberedLogin = null; - return; - } - - log.info("The server evicted this session as stale; signing in again with the configured credentials"); - replayLogin().whenComplete((ignored, error) -> { - if (error != null) { - log.warn("Signing in again after the eviction failed: {}", error.getMessage()); - } - }); } /** @@ -660,11 +639,18 @@ private CompletableFuture redialAttempt(int attempt, RetryPolicy policy) { if (closed) { return CompletableFuture.completedFuture(null); } - if (attempt > policy.getMaxRetries()) { + List candidates = redialCandidates(); + // The retry budget bounds the rotations, not the endpoints. A policy of + // zero retries with several endpoints known still gets one rotation: + // those endpoints - the address the client was configured with, the + // nodes the roster named - were made known in order to be tried, and + // the other SDKs sweep them once too. With one endpoint known, zero + // retries redials nothing, which is what it asked for. + boolean sweepOnce = attempt == 1 && candidates.size() > 1; + if (attempt > policy.getMaxRetries() && !sweepOnce) { log.error("Redial gave up after {} attempts, next request will fail fast", policy.getMaxRetries()); return CompletableFuture.completedFuture(null); } - List candidates = redialCandidates(); // The delay paces rotations, not dials. The first rotation runs at once // when there is somewhere else to go: pausing before dialing a survivor // only pushes the failover past the window the caller waits in, and the @@ -767,27 +753,33 @@ private static Throwable unwrap(Throwable error) { } /** - * Re-establishes the session on the freshly published connection: with the - * credentials configured on the builder, or else the sign-in a caller ran - * by hand. Configured credentials win, as in the Rust and Go SDKs -- they - * are what every connect of this client is meant to sign in as, and a - * remembered sign-in exists to make a hand-run login as reconnectable as - * a configured one, not to override it. + * Re-establishes the session on the freshly published connection with the + * sign-in that last succeeded, falling back to the credentials configured + * on the builder when no login has run yet. + * + * The last sign-in rather than the configured one, which is where this + * differs from the Rust and Go SDKs: the connection re-authenticates a + * replacement channel from the login payload it captured, which is that + * same last sign-in. Replaying a different user here would make the same + * eviction land on a different session depending on whether the channel + * or the redial got there first. A client that only ever used its + * configured credentials remembers exactly those, so nothing changes for + * it. * * The login runs through the users client, so leader discovery retargets * again before Register when the redialed node is not the leader. */ private CompletableFuture replayLogin() { + Supplier> replay = rememberedLogin; + if (replay != null) { + // Runs through loginOnLeader, so a redial that landed on a backup + // still settles on the leader before the session is used. + return loginOnLeader(replay).thenApply(identity -> null); + } if (username.isPresent() && password.isPresent() && usersClient != null) { return usersClient.login(username.get(), password.get()).thenApply(identity -> null); } - Supplier> replay = rememberedLogin; - if (replay == null) { - return CompletableFuture.completedFuture(null); - } - // Runs through loginOnLeader, so a redial that landed on a backup - // still settles on the leader before the session is used. - return loginOnLeader(replay).thenApply(identity -> null); + return CompletableFuture.completedFuture(null); } /** diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java index 94515314cf..71a0bca733 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java @@ -841,21 +841,6 @@ private void onSessionEvicted(int errorCode) { sessionResetListener.accept(errorCode); } - /** - * Drops the captured sign-in, so the next channel comes up - * unauthenticated instead of replaying it. - * - * The channel replays the payload it captured to re-authenticate a - * replacement channel, which is right for a lost connection and wrong - * after an eviction the server decided on: that would resurrect the very - * session the server ended. - */ - void forgetCapturedLogin() { - authenticated = false; - authGeneration.incrementAndGet(); - releaseLoginPayload(); - } - private void captureLoginPayloadIfNeeded(int commandCode, ByteBuf payload) { if (isLoginCode(commandCode)) { updateLoginPayload(commandCode, payload); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java index d339cebf7b..f67405969e 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java @@ -213,13 +213,14 @@ void shouldNotRememberASignInThatLandedAfterClose() throws Exception { } /** - * The user an eviction re-establishes must not depend on which failure - * ran. Configured credentials are what every connect of this client signs - * in as, so they are what comes back -- not whoever the caller signed in - * as by hand, which is what the connection's captured login would replay. + * A stale-client eviction is not caller intent: the server's heartbeat + * verifier sends it after a gc pause or a laptop sleep. A client that + * signed in by hand recovers from it exactly like one whose credentials + * were configured (the test above), and the sign-in it recovers with is the + * one that last succeeded. Same rule in every SDK. */ @Test - void shouldSignInAsTheConfiguredUserAfterAnEviction() throws Exception { + void shouldReviveTheSignInAfterAStaleClientEviction() throws Exception { InetAddress loopback = InetAddress.getLoopbackAddress(); try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { AtomicInteger registrations = new AtomicInteger(); @@ -233,69 +234,13 @@ void shouldSignInAsTheConfiguredUserAfterAnEviction() throws Exception { registeredLogins.add(request.bodyAsText()); return Response.success(OPERATION_REGISTER, registerBody(registrations.incrementAndGet())); } - if (request.operation() == OPERATION_CREATE_STREAM && evict.compareAndSet(true, false)) { - return Response.eviction(EVICTION_STALE_CLIENT); - } - return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); - }); - - AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() - .host(loopback.getHostAddress()) - .port(serverSocket.getLocalPort()) - .credentials("configured", "configured") - .requestTimeout(Duration.ofSeconds(2)) - .build(); - try { - client.connect().get(5, TimeUnit.SECONDS); - client.login().get(5, TimeUnit.SECONDS); - client.users().login("handrun", "handrun").get(5, TimeUnit.SECONDS); - int registrationsBeforeEviction = registrations.get(); - - assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) - .get(5, TimeUnit.SECONDS)) - .hasCauseInstanceOf(IggyServerException.class); - - // The sign-in that follows the eviction runs on its own, so - // give it a moment to land. - long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos(); - while (registrations.get() == registrationsBeforeEviction && System.nanoTime() < deadline) { - Thread.sleep(25); - } - assertThat(registrations.get()) - .as("the eviction was not followed by a sign-in") - .isGreaterThan(registrationsBeforeEviction); - assertThat(registeredLogins.get(registeredLogins.size() - 1)) - .as("the eviction revived the hand-run sign-in instead of the configured one") - .contains("configured") - .doesNotContain("handrun"); - } finally { - client.close().get(5, TimeUnit.SECONDS); - } - server.completeExceptionally(new IllegalStateException("test over")); - } - } - - /** - * A stale-client eviction is the server ending the session - * authoritatively, like a logout. A client whose credentials were - * configured signs in again on every connect and recovers (the test - * above); one whose session came from a caller's own sign-in must not have - * that session revived behind the caller's back. - */ - @Test - void shouldNotReviveAHandRunSignInAfterAStaleClientEviction() throws Exception { - InetAddress loopback = InetAddress.getLoopbackAddress(); - try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { - AtomicInteger registrations = new AtomicInteger(); - CompletableFuture server = serve(serverSocket, 4, request -> { - if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { - return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); - } - if (request.operation() == OPERATION_REGISTER) { - return Response.success(OPERATION_REGISTER, registerBody(registrations.incrementAndGet())); - } if (request.operation() == OPERATION_CREATE_STREAM) { - return Response.eviction(EVICTION_STALE_CLIENT); + if (evict.compareAndSet(true, false)) { + return Response.eviction(EVICTION_STALE_CLIENT); + } + ByteBuf body = Unpooled.buffer(Integer.BYTES); + body.writeIntLE(0); + return Response.success(OPERATION_CREATE_STREAM, body); } return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); }); @@ -308,27 +253,26 @@ void shouldNotReviveAHandRunSignInAfterAStaleClientEviction() throws Exception { .build(); try { client.connect().get(5, TimeUnit.SECONDS); - client.users().login("iggy", "iggy").get(5, TimeUnit.SECONDS); + client.users().login("handrun", "handrun").get(5, TimeUnit.SECONDS); int registrationsBeforeEviction = registrations.get(); assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) .get(5, TimeUnit.SECONDS)) .hasCauseInstanceOf(IggyServerException.class); - // Whatever the caller does next, nothing may sign this session - // back in on its own. - assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + assertThat(client.hasRememberedLogin()) + .as("an eviction is not a sign-out; the credentials stay") + .isTrue(); + + // The next request brings the session back, under the sign-in + // that last succeeded. + assertThat(client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) .get(5, TimeUnit.SECONDS)) - .isNotNull(); - assertThat(registrations) - .as("the evicted session was signed back in") - .hasValue(registrationsBeforeEviction); - // The connection replays the login it captured to bring up a - // replacement channel, so that copy has to go too: kept, the - // next channel revives exactly the session the server ended. - assertThat(client.currentConnection().authenticationSnapshot()) - .as("the captured login outlived the session it established") .isEmpty(); + assertThat(registrations.get()) + .as("the evicted session was not re-established") + .isGreaterThan(registrationsBeforeEviction); + assertThat(registeredLogins.get(registeredLogins.size() - 1)).contains("handrun"); } finally { client.close().get(5, TimeUnit.SECONDS); } diff --git a/foreign/node/src/client/client.connection.ts b/foreign/node/src/client/client.connection.ts index ea5c7dcbd9..0226147e12 100644 --- a/foreign/node/src/client/client.connection.ts +++ b/foreign/node/src/client/client.connection.ts @@ -381,7 +381,14 @@ export class IggyConnection extends EventEmitter { let lastError = initialError; let expectedSocket = this.socket; let firstPass = true; - while (enabled && this.reconnectCount < maxRetries) { + // Reconnection settings bound the retries, not the endpoints. With them off + // and several endpoints known - the address the client was configured with, + // the nodes the roster named - those endpoints were made known in order to + // be tried, so they get one pass and no backoff, as in the other SDKs. A + // client that knows one endpoint and turned reconnection off redials + // nothing, which is what it asked for. + const sweepOnce = !enabled && this._redialCandidates().length > 1; + while (this.reconnectCount < maxRetries || (sweepOnce && firstPass)) { this.connecting = true; this.reconnectCount += 1; const candidates = this._redialCandidates(); @@ -389,7 +396,7 @@ export class IggyConnection extends EventEmitter { // endpoints known there is somewhere else to go, and pausing first only // pushes the failover past the interval a caller is willing to wait; // later passes still back off. - if (!firstPass || candidates.length === 1) + if (enabled && (!firstPass || candidates.length === 1)) await waitForReconnect(interval); firstPass = false; if (this.ending) diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts index e938c7603a..2da50fdcd8 100644 --- a/foreign/node/src/client/client.socket.test.ts +++ b/foreign/node/src/client/client.socket.test.ts @@ -209,6 +209,12 @@ const vsrConfig = (port: number): ClientConfig => ({ }); /** Shrinks the leaderless poll so a test observes it without waiting on it. */ +/** Drives the leader re-check a refused request runs. */ +const followLeaderMove = (client: CommandResponseStream): Promise => + (client as unknown as { + _followLeaderMove: () => Promise + })._followLeaderMove(); + const compressLeaderlessPoll = ( client: CommandResponseStream, budget: number @@ -969,6 +975,72 @@ describe('VSR client socket', () => { } }); + it('re-issues every request refused by a demoted node, not just the first', + async () => { + // One demotion, several refused requests: each of them re-checking on + // its own would move the client once per request, and the first + // redirect's drop would fail the others' roster reads - reporting a + // refusal they never had to. + const leader = await startVsrServer((frame, socket) => { + singleNodeHandler(leader.port)(frame, socket); + }); + // Leader at login, so the settlement leaves the client here, then + // demoted: the refusals below are what tells the client to look again. + let demotedYet = false; + const demoted = await startVsrServer((frame, socket) => { + const code = frame.readUInt32LE(REQUEST_OFFSET.reserved); + if (code === COMMAND_CODE.GetClusterMetadata) { + socket.write(replyFrame( + Operation.NonReplicated, + demotedYet + ? twoNodeMetadataBody(demoted.port, leader.port) + : twoNodeMetadataBody(leader.port, demoted.port) + )); + return; + } + if (code === 60_032) { + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + singleNodeHandler(demoted.port)(frame, socket); + }); + + const client = new CommandResponseStream(vsrConfig(demoted.port)); + try { + await client.authenticate(vsrConfig(demoted.port).credentials); + demotedYet = true; + const rosterReadsBefore = demoted.frames.filter( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === + COMMAND_CODE.GetClusterMetadata + ).length; + + // Two refusals, one re-check: the second caller shares the move the + // first started rather than starting its own or being told 58. + const moves = await Promise.all([ + followLeaderMove(client), + followLeaderMove(client) + ]); + + assert.deepEqual(moves, [true, true]); + const rosterReads = demoted.frames.filter( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === + COMMAND_CODE.GetClusterMetadata + ).length - rosterReadsBefore; + assert.equal(rosterReads, 1, + 'each refusal re-read the roster on its own' + ); + const connection = (client as unknown as { + connection: { isConnectedTo: (host: string, port: number) => boolean } + }).connection; + assert.equal(connection.isConnectedTo('127.0.0.1', leader.port), true); + } finally { + client.destroy(); + await leader.close(); + await demoted.close(); + } + } + ); + it('keeps re-issuing a not-admitted request while the roster still names this node', async () => { const server = await startVsrServer((frame, socket) => { diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts index 43d378b844..2251418750 100644 --- a/foreign/node/src/client/client.socket.ts +++ b/foreign/node/src/client/client.socket.ts @@ -119,11 +119,10 @@ export class CommandResponseStream extends EventEmitter { /** Whether a login is already being moved to the leader */ private settlingLeader: boolean; /** - * Whether a refused request is already re-checking the leader. The roster - * read that re-check runs can be refused the same way, and answering a - * leader check with another leader check would recurse. + * The leader re-check a refused request started, shared with every other + * request refused by the same node so one demotion moves the client once. */ - private followingLeaderMove = false; + private leaderMoveInFlight?: Promise; /** How long a leaderless roster is polled before settling in place */ private leaderlessWaitBudget: number; /** Delay between roster reads while the cluster elects */ @@ -202,7 +201,8 @@ export class CommandResponseStream extends EventEmitter { try { const { handleResponse = true, - last = true + last = true, + followsLeaderMoves = true } = options; if (!this.connection.connected) @@ -222,6 +222,7 @@ export class CommandResponseStream extends EventEmitter { // one window: the roster can still name this node -- an election in // flight, a leader that has not moved yet -- and that is a wait, not a // verdict. + // // One budget for the whole request: the transient replays on a // connection, the leader re-checks, and the re-issues after a move all // spend it, so a request cannot outlive it by moving. @@ -235,12 +236,24 @@ export class CommandResponseStream extends EventEmitter { } catch (error) { if (!(error instanceof LeaderMovedError)) throw error; - // The roster read itself is refused: it runs through this same - // path, and re-checking the leader to answer a leader check would - // recurse. Its caller reads a failure as "stay where you are". - if (this.followingLeaderMove || Date.now() >= deadline) + // The roster read that a re-check runs is itself a command that can + // be refused this way, and answering a leader check with another + // leader check would recurse. Its caller reads a failure as "stay + // where you are". + if (!followsLeaderMoves || Date.now() >= deadline) throw responseError(command, error.refusal.errorCode); - await this._followLeaderMove(); + + const moved = await this._followLeaderMove(); + if (!moved) { + // Nowhere else to go yet: the roster still names this node, or it + // could not be read. Paced, because the in-connection replay + // window belongs to the request's budget and has already been + // spent -- re-issuing straight away would spin. + const remaining = deadline - Date.now(); + if (remaining <= 0) + throw responseError(command, error.refusal.errorCode); + await delay(Math.min(VSR_FAILOVER_CHECK_MS, remaining)); + } // A move drops the session with the socket it was bound to, so the // re-issue would otherwise go out under no session: a replicated // command fails client-side, a non-replicated one goes out with @@ -292,29 +305,40 @@ export class CommandResponseStream extends EventEmitter { * Re-reads the roster and moves to the leader it names. * * Best effort: an unreadable roster, or one that still names this node, - * leaves the client where it is and the refused request is re-issued here - * anyway. Guarded against re-entry, since the roster read is itself a - * command that can be refused the same way. + * leaves the client where it is and the refused request is re-issued anyway. + * + * Single-flighted, and concurrent callers share the outcome instead of + * failing: several commands are refused by the same demoted node, and each + * starting its own redirect would move the client once per command. The + * first redirect's `'disconnected'` also fails the others' roster reads, so + * a caller that raced one would report a refusal it never had to. * * @returns Whether the client moved */ - private async _followLeaderMove(): Promise { - if (this.followingLeaderMove) - return false; - this.followingLeaderMove = true; - try { - const leader = await this._readLeaderEndpoint(); - if (!leader || this.connection.isConnectedTo(leader.host, leader.port)) + private _followLeaderMove(): Promise { + const inFlight = this.leaderMoveInFlight; + if (inFlight) + return inFlight; + + const move = (async () => { + try { + const leader = await this._readLeaderEndpoint(); + if (!leader || this.connection.isConnectedTo(leader.host, leader.port)) + return false; + debug(`the leader moved to ${leader.host}:${leader.port}, following it`); + await this.connection.redirect(leader.host, leader.port); + return true; + } catch (error) { + debug('the leader could not be re-checked, staying on this node', error); return false; - debug(`the leader moved to ${leader.host}:${leader.port}, following it`); - await this.connection.redirect(leader.host, leader.port); - return true; - } catch (error) { - debug('the leader could not be re-checked, staying on this node', error); - return false; - } finally { - this.followingLeaderMove = false; - } + } + })(); + this.leaderMoveInFlight = move; + void move.finally(() => { + if (this.leaderMoveInFlight === move) + this.leaderMoveInFlight = undefined; + }); + return move; } private _rememberRoster(response: CommandResponse): void { @@ -617,7 +641,7 @@ export class CommandResponseStream extends EventEmitter { const response = await this.sendCommand( GET_CLUSTER_METADATA.code, GET_CLUSTER_METADATA.serialize(), - { last: false } + { last: false, followsLeaderMoves: false } ); // The redial candidates are fed by `_processVsr` for every roster // read, leaderless ones included: a roster with no leader still names diff --git a/foreign/node/src/client/client.type.ts b/foreign/node/src/client/client.type.ts index 63703005f2..bb9f89f0fe 100644 --- a/foreign/node/src/client/client.type.ts +++ b/foreign/node/src/client/client.type.ts @@ -47,7 +47,13 @@ export type SendCommandOptions = { /** Whether the response uses the standard command response decoder */ handleResponse?: boolean, /** Whether to append rather than prepend the command to the queue */ - last?: boolean + last?: boolean, + /** + * Whether a not-admitted refusal re-checks the leader and re-issues the + * command. False for the roster read a re-check itself runs: answering a + * leader check with another leader check would recurse. + */ + followsLeaderMoves?: boolean }; /** From 5bda045a6b0a300d1b6cabdefa561097851dea53 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Wed, 26 Aug 2026 10:38:49 +0200 Subject: [PATCH 10/16] address review --- .../tcp_client_reconnection_config.rs | 15 +- .../HeartbeatTests.cs | 11 +- foreign/go/client/tcp/tcp_core.go | 62 +++++++- foreign/go/client/tcp/tcp_failover_test.go | 54 ++++++- .../client/async/tcp/AsyncIggyTcpClient.java | 24 ++- ...syncIggyTcpClientEndpointFailoverTest.java | 30 +++- ...yncIggyTcpClientTransientFailoverTest.java | 1 + .../node/src/client/client.connection.test.ts | 76 ++++++++++ foreign/node/src/client/client.connection.ts | 41 ++++-- foreign/node/src/client/client.socket.test.ts | 139 +++++++++++++++++- foreign/node/src/client/client.socket.ts | 65 +++++++- foreign/python/apache_iggy.pyi | 16 +- foreign/python/src/config.rs | 16 +- 13 files changed, 496 insertions(+), 54 deletions(-) diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs b/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs index 89d8644e8b..a2365e5917 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs @@ -20,10 +20,23 @@ use std::str::FromStr; #[derive(Debug, Clone)] pub struct TcpClientReconnectionConfig { + /// Whether a lost connection is redialed at all. With this off the + /// endpoints the client knows still get one pass, since they were + /// configured to be tried, but nothing is retried after it. pub enabled: bool, + /// How many passes over the known endpoints, or `None` for unlimited. + /// + /// Passes, not dials: one pass tries the endpoint the client is on, the + /// addresses it was configured with, and every node the roster named, so a + /// survivor is reached inside the first pass rather than one delay per + /// endpoint. pub max_retries: Option, - /// Delay between connection attempts. + /// Delay between passes. The first pass runs at once when the client knows + /// more than one endpoint. pub interval: NonZeroIggyDuration, + /// Cooldown before redialing the endpoint that was just lost. It is owed to + /// that endpoint alone: the others are dialed without waiting, and the + /// paced one goes last in the pass. pub reestablish_after: IggyDuration, } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs index b0e71feeed..882c89f881 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/HeartbeatTests.cs @@ -118,14 +118,17 @@ public async Task EvictedClient_WithoutAutoLogin_Should_ReestablishItsSession() await Task.Delay(IdleFor); - // The evicted request surfaces the loss; the one after it comes back over a session the remembered - // sign-in re-established. - await Should.ThrowAsync(() => client.GetMeAsync()); - + // A read is replay-safe, so the eviction is absorbed: the reconnect signs in again with the + // credentials the hand-run login remembered, and the request completes over the session it + // re-established. var stream = await client.GetStreamByIdAsync(Identifier.String(streamName)); stream.ShouldNotBeNull(); + + // The session is a new one, though: what the server evicted stays evicted, so the group membership + // that belonged to it is gone. var me = await client.GetMeAsync(); me.ShouldNotBeNull(); + me.ConsumerGroupsCount.ShouldBe(0); } private Task CreateClient(TimeSpan heartbeatInterval, bool autoLogin = true) diff --git a/foreign/go/client/tcp/tcp_core.go b/foreign/go/client/tcp/tcp_core.go index 9cad23f743..c2145f7035 100644 --- a/foreign/go/client/tcp/tcp_core.go +++ b/foreign/go/client/tcp/tcp_core.go @@ -82,6 +82,11 @@ type IggyTcpClient struct { // loggedOut records an explicit sign-out, so a reconnect's automatic // sign-in does not silently reverse it; guarded by c.mtx. loggedOut bool + // connectInFlight is the attempt a Connect is running, shared with every + // caller that arrives while it is in progress: closed when the attempt + // settles, with connectErr carrying its outcome. Guarded by c.mtx. + connectInFlight chan struct{} + connectErr error // rememberedLogin holds the credentials a manual sign-in succeeded with, // so a reconnect -- on this node or, after a failover, another one -- can // re-establish the session instead of surfacing an unauthenticated error. @@ -908,7 +913,12 @@ func (c *IggyTcpClient) GetConnectionInfo() *iggcon.ConnectionInfo { } // Connect establishes the TCP connection to the server. -func (c *IggyTcpClient) Connect(ctx context.Context) error { +// +// Single-flighted: one attempt dials, and every caller that arrives while it +// runs waits for it and shares its outcome. Reporting success to those callers +// instead would hand them a client with no connection yet, and their next +// request would fail ErrNotConnected for no reason of its own. +func (c *IggyTcpClient) Connect(ctx context.Context) (err error) { c.mtx.Lock() switch c.transportState { case iggcon.TransportStateShutdown: @@ -921,15 +931,42 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { c.logger.Debug("Client is already connected.", slog.String("client_address", clientAddress)) return nil case iggcon.TransportStateConnecting: + inFlight := c.connectInFlight c.mtx.Unlock() - c.logger.Debug("Client is already connecting.") - return nil + c.logger.Debug("Client is already connecting; waiting for that attempt.") + if inFlight == nil { + return nil + } + select { + case <-inFlight: + case <-ctx.Done(): + return ctx.Err() + case <-c.closed: + return ierror.ErrClientShutdown + } + c.mtx.Lock() + attemptErr := c.connectErr + c.mtx.Unlock() + return attemptErr default: c.transportState = iggcon.TransportStateConnecting + c.connectInFlight = make(chan struct{}) + c.connectErr = nil } connectedAt := c.connectedAt c.mtx.Unlock() + // Settles the attempt for whoever is waiting on it, whichever way it ends. + defer func() { + c.mtx.Lock() + c.connectErr = err + if c.connectInFlight != nil { + close(c.connectInFlight) + c.connectInFlight = nil + } + c.mtx.Unlock() + }() + candidates := c.connectionCandidates() if len(candidates) == 0 { // Nowhere to dial: a client configured with an empty server address @@ -1019,6 +1056,25 @@ func (c *IggyTcpClient) Connect(ctx context.Context) error { } c.mtx.Lock() + if c.transportState != iggcon.TransportStateConnecting { + // Disconnected or shut down while this attempt was dialing: the + // connection it just made is not wanted, and installing it would + // resurrect a client somebody asked to stop. + state := c.transportState + c.mtx.Unlock() + _ = conn.Close() + c.logger.Debug("The connect was superseded while dialing; dropping the connection.") + if state == iggcon.TransportStateShutdown { + return ierror.ErrClientShutdown + } + return ierror.ErrNotConnected + } + // A connection installed over another one leaks its socket: two Connects + // can race here, and the loser's conn would otherwise stay open with + // nothing left holding it. + if previous := c.conn; previous != nil { + _ = previous.Close() + } c.conn = conn c.reader = bufio.NewReaderSize(conn, 64*1024) c.transportState = iggcon.TransportStateConnected diff --git a/foreign/go/client/tcp/tcp_failover_test.go b/foreign/go/client/tcp/tcp_failover_test.go index 020e8b2b89..6ad47d16af 100644 --- a/foreign/go/client/tcp/tcp_failover_test.go +++ b/foreign/go/client/tcp/tcp_failover_test.go @@ -154,9 +154,10 @@ func TestFailover_ServerEvictionReplaysTheRememberedSignIn(t *testing.T) { registersBefore := registers.Load() evict.Store(true) - // The evicted request is answered by the eviction, and the reconnect it - // triggers signs in again with the credentials the sign-in remembered. - _ = client.Ping(ctx) + // A ping is non-replicated, so the eviction is absorbed: the reconnect it + // triggers signs in again with the credentials the sign-in remembered and + // the request completes over the session it re-established. + require.NoError(t, client.Ping(ctx), "the evicted request was not recovered") _, remembered := client.signInCredentials() assert.True(t, remembered, "an eviction is not a sign-out; the credentials stay") @@ -553,6 +554,53 @@ func TestFailover_RejectsAConnectWithNoEndpointToDial(t *testing.T) { assert.Error(t, client.Ping(context.Background())) } +// Concurrent Connects are one attempt, and a caller that did not run it still +// gets a client it can use the moment its Connect returns. Told "connected" +// while the attempt is still signing in, its next request fails +// ErrNotConnected for no reason of its own. +func TestConnect_ConcurrentCallersShareOneAttempt(t *testing.T) { + var server *testListener + server = listenVSR(t, nil, func(_, _ int, read request) []byte { + if read.operation() == vsr.OperationRegister { + // The sign-in is the slow part of an attempt, and it runs after the + // dial: a caller that returned early would use the client here. + time.Sleep(300 * time.Millisecond) + } + return singleNodeHandler(t, func() string { return server.address() })(0, 0, read) + }) + + client := newDialingClient(t, server.address(), + WithAutoLogin(NewUsernamePasswordCredentials("iggy", "iggy"))) + + const callers = 8 + results := make(chan error, callers) + start := make(chan struct{}) + for range callers { + go func() { + <-start + if err := client.Connect(context.Background()); err != nil { + results <- err + return + } + // Usable right away, or the Connect that returned was a lie. + results <- client.Ping(context.Background()) + }() + } + close(start) + for range callers { + require.NoError(t, <-results, "a caller was handed a client it could not use") + } + + assert.Equal(t, 1, server.connections(), "the callers dialed more than once") + var registers int + for _, read := range server.recorded() { + if read.operation() == vsr.OperationRegister { + registers++ + } + } + assert.Equal(t, 1, registers, "one attempt, one sign-in") +} + // deadAddress returns an address nothing listens on, so a dial to it is // refused at once. func deadAddress(t *testing.T) string { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java index 2bec95acb9..c0344a50f2 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java @@ -119,6 +119,12 @@ public class AsyncIggyTcpClient { private static final int INVALID_COMMAND_ERROR_CODE = 3; private static final Logger log = LoggerFactory.getLogger(AsyncIggyTcpClient.class); private static final RetryPolicy DEFAULT_RECONNECT_POLICY = RetryPolicy.fixedDelay(12, Duration.ofSeconds(5)); + /** + * Bound on one dial while other endpoints are queued behind it, matching + * the Rust, Go, C# and Node SDKs. Netty's own connect timeout would + * otherwise let a node whose syns are dropped hold the whole rotation. + */ + private static final Duration FAILOVER_DIAL_TIMEOUT = Duration.ofSeconds(2); private final ConnectionInfo seedConnectionInfo; private final AtomicBoolean reconnecting = new AtomicBoolean(); @@ -489,7 +495,7 @@ private AsyncTcpConnection openConnection(ConnectionInfo target) { enableTls, tlsCertificate, poolConfig, - connectionTimeout, + dialTimeout(), requestTimeout, heartbeatInterval, maxVsrFrameSize, @@ -498,6 +504,22 @@ private AsyncTcpConnection openConnection(ConnectionInfo target) { this::onConnectionFailure); } + /** + * How long one dial may take. + * + * With other endpoints queued behind this one, a node whose syns are + * dropped must not hold the rotation, so the wait is capped at + * {@link #FAILOVER_DIAL_TIMEOUT} - the bound the other SDKs use. A caller + * who configured a connection timeout gets exactly that, and a client that + * knows one endpoint keeps the ordinary default. + */ + private Optional dialTimeout() { + if (connectionTimeout.isPresent() || redialCandidates().size() < 2) { + return connectionTimeout; + } + return Optional.of(FAILOVER_DIAL_TIMEOUT); + } + /** * A server-side eviction reached this client. The routing state it cached * belonged to the evicted session, so it goes. diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java index fd8080755a..a2bc8e11ae 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java @@ -82,6 +82,7 @@ void shouldResumeOnASurvivorAfterTheSignedInNodeDies() throws Exception { int survivorPort = survivorSocket.getLocalPort(); AtomicInteger survivorRegistrations = new AtomicInteger(); AtomicInteger survivorPings = new AtomicInteger(); + List survivorLogins = new CopyOnWriteArrayList<>(); // The primary leads, so the sign-in settles there and the roster is // only remembered -- not acted on -- until the node dies. @@ -102,6 +103,7 @@ void shouldResumeOnASurvivorAfterTheSignedInNodeDies() throws Exception { } if (request.operation() == OPERATION_REGISTER) { survivorRegistrations.incrementAndGet(); + survivorLogins.add(request.bodyAsText()); return Response.success(OPERATION_REGISTER, registerBody(2)); } if (request.is(PING_CODE, OPERATION_NON_REPLICATED)) { @@ -110,14 +112,16 @@ void shouldResumeOnASurvivorAfterTheSignedInNodeDies() throws Exception { return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER); }); - // No builder credentials: the replay can only come from the - // sign-in the caller ran, which is the shape that could not - // reconnect at all before. With credentials configured, the replay - // falls back to them and the test would pass without a remembered - // sign-in at all. + // Credentials on the builder and a hand-run login for somebody + // else. The redial replays the sign-in that last succeeded, which + // is also what the connection replays from the login it captured + // when the pool swaps a channel: replaying a different user here + // would make the same failure land on a different session depending + // on which path got there first. AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() .host(loopback.getHostAddress()) .port(primaryPort) + .credentials("configured", "configured") .requestTimeout(Duration.ofSeconds(2)) // A whole rotation dials every endpoint the client knows, so // the survivor is reached before this delay is ever spent. @@ -126,7 +130,7 @@ void shouldResumeOnASurvivorAfterTheSignedInNodeDies() throws Exception { .build(); try { client.connect().get(5, TimeUnit.SECONDS); - client.users().login("iggy", "iggy").get(5, TimeUnit.SECONDS); + client.users().login("handrun", "handrun").get(5, TimeUnit.SECONDS); client.sendBinaryRequest(PING_CODE, new byte[0]).get(5, TimeUnit.SECONDS); assertThat(client.getConnectionInfo().port()).isEqualTo(primaryPort); @@ -148,6 +152,10 @@ void shouldResumeOnASurvivorAfterTheSignedInNodeDies() throws Exception { assertThat(survivorPings) .as("the request landed on the survivor") .hasValueGreaterThanOrEqualTo(1); + assertThat(survivorLogins.get(survivorLogins.size() - 1)) + .as("the redial replayed the configured user instead of the last sign-in") + .contains("handrun") + .doesNotContain("configured"); } finally { client.close().get(5, TimeUnit.SECONDS); survivor.close(); @@ -353,7 +361,8 @@ private static Request readRequest(InputStream input) throws IOException { return new Request( Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), fields.getInt(REQUEST_CODE_OFFSET), - fields.getLong(REQUEST_ID_OFFSET)); + fields.getLong(REQUEST_ID_OFFSET), + body); } private static void writeResponse(OutputStream output, Request request, Response response) throws IOException { @@ -372,10 +381,15 @@ private static void writeResponse(OutputStream output, Request request, Response output.flush(); } - private record Request(int operation, int commandCode, long requestId) { + private record Request(int operation, int commandCode, long requestId, byte[] body) { boolean is(int expectedCode, int expectedOperation) { return commandCode == expectedCode && operation == expectedOperation; } + + /** The request body as text, for asserting which user a login names. */ + String bodyAsText() { + return new String(body, StandardCharsets.UTF_8); + } } private record Response(int operation, ByteBuf body) { diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java index f67405969e..95abbc29a9 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java @@ -64,6 +64,7 @@ class AsyncIggyTcpClientTransientFailoverTest { private static final int OPERATION_CREATE_STREAM = 128; private static final int GET_CLUSTER_METADATA_CODE = 12; private static final int CREATE_STREAM_CODE = 202; + private static final int PING_CODE = 1; private static final int TRANSIENT_NOT_ACCEPTED = 58; private static final int EVICTION_STALE_CLIENT = 13; diff --git a/foreign/node/src/client/client.connection.test.ts b/foreign/node/src/client/client.connection.test.ts index 6e1dce2742..250c712385 100644 --- a/foreign/node/src/client/client.connection.test.ts +++ b/foreign/node/src/client/client.connection.test.ts @@ -472,6 +472,82 @@ describe('IggyConnection', () => { } ); + it('does not redial at all when reconnection is disabled', + async () => { + // `enabled: false` is what a caller says to opt out. The retry budget is + // whatever the defaults hold, so a loop that reads it without checking + // this flag would run every one of those passes - and with the backoff + // gated on the same flag, back to back. + // + // The endpoint accepts and hangs up, so the drop that would start a + // redial happens and every dial of it is counted. + const hangup = await startServer(); + const hangupPort = (hangup.address() as AddressInfo).port; + let accepted = 0; + hangup.on('connection', (socket) => { + accepted += 1; + socket.destroy(); + }); + + const connection = new IggyConnection({ + transport: 'TCP', + options: { host: '127.0.0.1', port: hangupPort }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: false, interval: 10, maxRetries: 12 }, + maxResponseFrameSize: FRAME_LIMIT + }); + connection.on('error', () => undefined); + try { + await connection.connect().catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 300)); + + assert.equal(accepted, 1, + 'a client that turned reconnection off redialed anyway' + ); + assert.equal(connection.connected, false); + } finally { + connection._destroy(); + await new Promise((resolve) => hangup.close(() => resolve())); + } + } + ); + + it('sweeps the endpoints it knows once when reconnection is disabled', + async () => { + // Opting out of retries is not opting out of the endpoints: with more + // than one known, they get exactly one pass and no backoff, as in the + // other SDKs. + const dead = await startServer(); + const deadPort = (dead.address() as AddressInfo).port; + await new Promise((resolve) => dead.close(() => resolve())); + const live = await startServer(); + const livePort = (live.address() as AddressInfo).port; + let accepted = 0; + live.on('connection', () => { accepted += 1; }); + + const connection = new IggyConnection({ + transport: 'TCP', + options: { host: '127.0.0.1', port: deadPort }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: false, interval: 10, maxRetries: 12 }, + maxResponseFrameSize: FRAME_LIMIT + }); + connection.on('error', () => undefined); + try { + connection.rememberRoster([{ host: '127.0.0.1', port: livePort }]); + await connection.connect().catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 200)); + + assert.equal(accepted, 1, + 'the known endpoints got either no pass or more than one' + ); + } finally { + connection._destroy(); + await new Promise((resolve) => live.close(() => resolve())); + } + } + ); + it('skips the first backoff when another endpoint is known', async () => { // The endpoint the client is on is dead and a live one sits behind it in diff --git a/foreign/node/src/client/client.connection.ts b/foreign/node/src/client/client.connection.ts index 0226147e12..f4e3cbfb33 100644 --- a/foreign/node/src/client/client.connection.ts +++ b/foreign/node/src/client/client.connection.ts @@ -124,6 +124,13 @@ export class IggyConnection extends EventEmitter { public connecting: boolean; /** Whether the connection is being intentionally closed */ public ending: boolean; + /** + * Whether the socket is being replaced by a deliberate leader redirect + * rather than lost. The drop looks the same from the outside, but nothing a + * caller submitted is in doubt: work waiting to be sent belongs on the node + * the client moves to, not in an error. + */ + public redirecting: boolean; /** Reconnection configuration */ private reconnectOption: ReconnectOption; /** Number of reconnection attempts made */ @@ -156,6 +163,7 @@ export class IggyConnection extends EventEmitter { this.connected = false; this.connecting = false; this.ending = false; + this.redirecting = false; this.reconnectOption = { ...DefaultReconnectOption, ...config.reconnect }; this.seedOptions = { ...config.options }; this.rosterEndpoints = []; @@ -388,7 +396,7 @@ export class IggyConnection extends EventEmitter { // client that knows one endpoint and turned reconnection off redials // nothing, which is what it asked for. const sweepOnce = !enabled && this._redialCandidates().length > 1; - while (this.reconnectCount < maxRetries || (sweepOnce && firstPass)) { + while ((enabled && this.reconnectCount < maxRetries) || (sweepOnce && firstPass)) { this.connecting = true; this.reconnectCount += 1; const candidates = this._redialCandidates(); @@ -502,19 +510,24 @@ export class IggyConnection extends EventEmitter { ...this.config, options: redirectedOptions }; - // Destroying the old socket settles any dial still waiting on it. Its - // lifecycle listeners stay attached but go inert once the socket is - // replaced below, so surface the drop to in-flight exchanges ourselves. - this.socket.destroy(); - this.connected = false; - this.connecting = false; - this.connectPromise = undefined; - this.reconnectPromise = undefined; - this._endResponseWait(); - this.socket = this._installSocket(getTransport(redirectedConfig)); - this.emit('disconnected', false); - await this.connect(); - this.config.options = redirectedOptions; + this.redirecting = true; + try { + // Destroying the old socket settles any dial still waiting on it. Its + // lifecycle listeners stay attached but go inert once the socket is + // replaced below, so surface the drop to in-flight exchanges ourselves. + this.socket.destroy(); + this.connected = false; + this.connecting = false; + this.connectPromise = undefined; + this.reconnectPromise = undefined; + this._endResponseWait(); + this.socket = this._installSocket(getTransport(redirectedConfig)); + this.emit('disconnected', false); + await this.connect(); + this.config.options = redirectedOptions; + } finally { + this.redirecting = false; + } } abort(): void { diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts index 2da50fdcd8..9a0591a987 100644 --- a/foreign/node/src/client/client.socket.test.ts +++ b/foreign/node/src/client/client.socket.test.ts @@ -35,7 +35,7 @@ import { import { Operation } from '../wire/vsr/operation.js'; import { VsrEvictionError } from '../wire/vsr/reply.js'; import { CommandResponseStream } from './client.socket.js'; -import type { ClientConfig } from './client.type.js'; +import type { ClientConfig, CommandResponse } from './client.type.js'; const TEST_SESSION = 42n; const TLS_CERTIFICATE = readFileSync( @@ -209,6 +209,23 @@ const vsrConfig = (port: number): ClientConfig => ({ }); /** Shrinks the leaderless poll so a test observes it without waiting on it. */ +/** The queue a command waits in, for parking one the way the client does. */ +const execQueue = (client: CommandResponseStream): { + command: number, + payload: Buffer, + handleResponse: boolean, + deadline: number, + resolve: (v: CommandResponse | PromiseLike) => void, + reject: (e: unknown) => void +}[] => (client as unknown as { _execQueue: never[] })._execQueue; + +/** The connection under a stream, for driving a redirect the way a move does. */ +const connectionOf = (client: CommandResponseStream): { + redirect: (host: string, port: number) => Promise +} => (client as unknown as { + connection: { redirect: (host: string, port: number) => Promise } +}).connection; + /** Drives the leader re-check a refused request runs. */ const followLeaderMove = (client: CommandResponseStream): Promise => (client as unknown as { @@ -1041,6 +1058,126 @@ describe('VSR client socket', () => { } ); + it('re-issues a command queued behind a leader move instead of failing it', + async () => { + // A move replaces the socket, which looks like a drop to everything + // waiting in the queue. Nothing queued was written, though, so it belongs + // on the node the client moves to rather than in a lost-connection error + // the caller can do nothing about. + const leader = await startVsrServer((frame, socket) => { + singleNodeHandler(leader.port)(frame, socket); + }); + const demoted = await startVsrServer((frame, socket) => { + singleNodeHandler(demoted.port)(frame, socket); + }); + + const client = new CommandResponseStream(vsrConfig(demoted.port)); + try { + await client.authenticate(vsrConfig(demoted.port).credentials); + + // Parked the way a command is while something else holds the queue. + const queued = new Promise((resolve, reject) => { + execQueue(client).push({ + command: 60_034, + payload: Buffer.alloc(0), + handleResponse: true, + deadline: Date.now() + 30_000, + resolve, + reject + }); + }); + + await connectionOf(client).redirect('127.0.0.1', leader.port); + await queued; + + const landedOnLeader = leader.frames.some( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_034 + ); + assert.ok(landedOnLeader, + 'the queued command never reached the node the client moved to' + ); + } finally { + client.destroy(); + await leader.close(); + await demoted.close(); + } + } + ); + + it('surfaces the refusal rather than a timeout when the budget runs out', + async () => { + const server = await startVsrServer((frame, socket) => { + if (frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_036) { + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + singleNodeHandler(server.port)(frame, socket); + }); + const client = new CommandResponseStream(vsrConfig(server.port)); + const realNow = Date.now; + try { + await client.authenticate(vsrConfig(server.port).credentials); + // The request's own budget, then the clock jumped past it: what is left + // cannot carry another attempt, so the caller has to see the answer the + // server gave and not the timeout a doomed re-issue would produce. + // The request's budget, the first exchange, the window that hands the + // refusal out, and then a clock 10ms short of the deadline: too little + // to carry another attempt, so the caller has to see the answer the + // server gave rather than the timeout a doomed re-issue would produce. + const times = [0, 1, 2_001, 29_990]; + Date.now = () => times.shift() ?? 30_050; + + await assert.rejects( + () => client.sendCommand(60_036, Buffer.alloc(0)), + (error: unknown) => + error instanceof ResponseError && + error.commandCode === 60_036 && + error.errorCode === 58 + ); + } finally { + Date.now = realNow; + client.destroy(); + await server.close(); + } + } + ); + + it('paces the re-issues while the roster still names this node', + async () => { + // Re-issuing is right, spinning is not: the in-connection replay window + // belongs to the request's budget and is spent after the first pass, so + // without a wait the client would hammer the node for the whole budget. + let refusals = 0; + const server = await startVsrServer((frame, socket) => { + if (frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_035) { + refusals += 1; + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + singleNodeHandler(server.port)(frame, socket); + }); + const client = new CommandResponseStream(vsrConfig(server.port)); + try { + await client.authenticate(vsrConfig(server.port).credentials); + + const pending = client.sendCommand(60_035, Buffer.alloc(0)); + pending.catch(() => undefined); + await new Promise((resolve) => { + setTimeout(resolve, 5_000).unref(); + }); + + // The first 2s window replays on the connection at its own interval; + // every window after it costs one refusal per pace. + assert.ok(refusals < 200, + `the re-issues were not paced: ${refusals} refusals in 5s` + ); + } finally { + client.destroy(); + await server.close(); + } + } + ); + it('keeps re-issuing a not-admitted request while the roster still names this node', async () => { const server = await startVsrServer((frame, socket) => { diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts index 2251418750..bafd4a878c 100644 --- a/foreign/node/src/client/client.socket.ts +++ b/foreign/node/src/client/client.socket.ts @@ -51,6 +51,15 @@ const TRANSIENT_NOT_ACCEPTED = 58; */ const VSR_FAILOVER_CHECK_MS = 2_000; +/** + * Whether a request's budget still holds enough for another attempt. One + * exchange needs at least a replay interval to be worth starting; below that + * the attempt can only end in a timeout, which would hide the refusal that + * actually came back. + */ +const worthAnotherAttempt = (deadline: number): boolean => + deadline - Date.now() > VSR_RETRY_INTERVAL_MS; + /** * A request the current node keeps refusing as not-admitted. Carries the * refusal so the caller can surface it when the roster turns out to still name @@ -177,12 +186,40 @@ export class CommandResponseStream extends EventEmitter { }); this.connection.on('disconnected', () => { this._resetSession(); + if (this.connection.redirecting) { + // The client is moving to the leader, which is its own doing: a queued + // command has not been written, so it belongs on the node being moved + // to rather than in an error. + this._reissueQueue(); + return; + } this._failQueue( new Error('connection closed before queued commands were sent') ); }); } + /** + * Re-submits queued commands through the full send path, so each one + * reconnects, re-authenticates and re-checks the leader as if it had just + * been called. + * + * Only for a drop the client caused. Nothing here was written, so there is no + * outcome in doubt: a command still in the queue when the socket is replaced + * would otherwise fail with a lost-connection error the caller can do nothing + * about. + */ + private _reissueQueue(): void { + const queued = this._execQueue; + this._execQueue = []; + for (const job of queued) { + debug('re-issuing a queued command after a leader move', job.command); + this.sendCommand(job.command, job.payload, { + handleResponse: job.handleResponse + }).then(job.resolve, job.reject); + } + } + /** * Sends a command to the server. * Automatically handles connection, authentication and leader settlement. @@ -240,7 +277,12 @@ export class CommandResponseStream extends EventEmitter { // be refused this way, and answering a leader check with another // leader check would recurse. Its caller reads a failure as "stay // where you are". - if (!followsLeaderMoves || Date.now() >= deadline) + // + // A budget too small to carry another attempt ends it here, with the + // refusal the server actually gave: re-issued into what is left, the + // request would time out instead and the caller would see a timeout + // where the answer was "not admitted". + if (!followsLeaderMoves || !worthAnotherAttempt(deadline)) throw responseError(command, error.refusal.errorCode); const moved = await this._followLeaderMove(); @@ -249,11 +291,13 @@ export class CommandResponseStream extends EventEmitter { // could not be read. Paced, because the in-connection replay // window belongs to the request's budget and has already been // spent -- re-issuing straight away would spin. - const remaining = deadline - Date.now(); - if (remaining <= 0) - throw responseError(command, error.refusal.errorCode); - await delay(Math.min(VSR_FAILOVER_CHECK_MS, remaining)); + await delay(Math.min( + VSR_FAILOVER_CHECK_MS, + Math.max(0, deadline - Date.now()) + )); } + if (!worthAnotherAttempt(deadline)) + throw responseError(command, error.refusal.errorCode); // A move drops the session with the socket it was bound to, so the // re-issue would otherwise go out under no session: a replicated // command fails client-side, a non-replicated one goes out with @@ -374,8 +418,15 @@ export class CommandResponseStream extends EventEmitter { reject(err); } } - if (this._execQueue.length > 0) - this._failQueue(new Error('connection is not writable')); + if (this._execQueue.length > 0) { + // The same distinction as on 'disconnected': the socket a leader move + // replaced stops being writable, and what is still queued belongs on the + // node being moved to. + if (this.connection.redirecting) + this._reissueQueue(); + else + this._failQueue(new Error('connection is not writable')); + } this.busy = false; this._emitFinishQueue(); } diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index e93a8ebfab..87cfafd466 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi @@ -2016,14 +2016,18 @@ class TcpReconnectionConfig: Args: enabled: Whether to reconnect at all. Defaults to enabled. - max_retries: Attempts before giving up, or `None` for unlimited. - Defaults to unlimited, which means a call awaited while the server - is down never returns: `connect()`, `send_messages()` and + max_retries: Passes over the known endpoints before giving up, or + `None` for unlimited. One pass tries the endpoint the client is + on, the address it was configured with, and every node the + roster named, so this counts passes rather than dials. Defaults + to unlimited, which means a call awaited while the server is + down never returns: `connect()`, `send_messages()` and `poll_messages()` all wait inside the retry loop. Set a finite number for request/reply style usage, so a call fails instead. - interval: Delay between attempts. Defaults to 1 second. - reestablish_after: Cooldown before reconnecting after a previously - successful connection. Defaults to 5 seconds. + interval: Delay between passes. Defaults to 1 second. The first pass + runs at once when more than one endpoint is known. + reestablish_after: Cooldown before redialing the endpoint that was + just lost, owed to that endpoint alone. Defaults to 5 seconds. Raises: ValueError: If a duration is negative, if `max_retries` is outside the diff --git a/foreign/python/src/config.rs b/foreign/python/src/config.rs index 79eb285b16..09a7dc90d4 100644 --- a/foreign/python/src/config.rs +++ b/foreign/python/src/config.rs @@ -121,14 +121,18 @@ impl TcpReconnectionConfig { /// /// Args: /// enabled: Whether to reconnect at all. Defaults to enabled. - /// max_retries: Attempts before giving up, or `None` for unlimited. - /// Defaults to unlimited, which means a call awaited while the server - /// is down never returns: `connect()`, `send_messages()` and + /// max_retries: Passes over the known endpoints before giving up, or + /// `None` for unlimited. One pass tries the endpoint the client is + /// on, the address it was configured with, and every node the + /// roster named, so this counts passes rather than dials. Defaults + /// to unlimited, which means a call awaited while the server is + /// down never returns: `connect()`, `send_messages()` and /// `poll_messages()` all wait inside the retry loop. Set a finite /// number for request/reply style usage, so a call fails instead. - /// interval: Delay between attempts. Defaults to 1 second. - /// reestablish_after: Cooldown before reconnecting after a previously - /// successful connection. Defaults to 5 seconds. + /// interval: Delay between passes. Defaults to 1 second. The first pass + /// runs at once when more than one endpoint is known. + /// reestablish_after: Cooldown before redialing the endpoint that was + /// just lost, owed to that endpoint alone. Defaults to 5 seconds. /// /// Raises: /// ValueError: If a duration is negative, if `max_retries` is outside the From b2b9c0e693e04720e8c2a2c0062798e075fd5441 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Wed, 26 Aug 2026 13:02:13 +0200 Subject: [PATCH 11/16] address review comments --- core/common/src/types/args/mod.rs | 14 ++ .../auth_config/connection_string.rs | 29 +++ .../tcp_config/tcp_client_config.rs | 5 +- .../tcp_client_reconnection_config.rs | 15 +- .../tcp_connection_string_options.rs | 18 ++ core/sdk/src/client_provider.rs | 4 +- core/sdk/src/clients/client_builder.rs | 7 + core/sdk/src/tcp/tcp_client.rs | 142 +++++++++++---- examples/rust/src/shared/args.rs | 5 + .../VsrTests/EndpointFailoverTests.cs | 8 +- foreign/go/client/tcp/tcp_core.go | 112 +++++++----- foreign/go/client/tcp/tcp_failover_test.go | 136 ++++++++++++++ .../client/async/tcp/AsyncIggyTcpClient.java | 59 ++++--- .../client/async/tcp/AsyncTcpConnection.java | 33 +++- .../async/tcp/vsr/VsrResponseHandler.java | 8 +- ...syncIggyTcpClientEndpointFailoverTest.java | 109 ++++++++++++ ...yncIggyTcpClientTransientFailoverTest.java | 67 ++++++- .../node/src/client/client.connection.test.ts | 56 ++++++ foreign/node/src/client/client.connection.ts | 14 +- foreign/node/src/client/client.socket.test.ts | 167 ++++++++++++++++-- foreign/node/src/client/client.socket.ts | 100 +++++++++-- foreign/node/src/client/client.type.ts | 9 +- foreign/python/apache_iggy.pyi | 28 ++- foreign/python/src/config.rs | 40 ++++- 24 files changed, 1016 insertions(+), 169 deletions(-) diff --git a/core/common/src/types/args/mod.rs b/core/common/src/types/args/mod.rs index d44b223ced..ecd529096b 100644 --- a/core/common/src/types/args/mod.rs +++ b/core/common/src/types/args/mod.rs @@ -76,6 +76,12 @@ pub struct ArgsOptional { #[serde(skip_serializing_if = "Option::is_none")] pub tcp_server_address: Option, + /// The optional addresses of other nodes of the same cluster, dialed in + /// order when the server address cannot be reached + #[arg(long, value_delimiter = ',')] + #[serde(skip_serializing_if = "Option::is_none")] + pub tcp_failover_addresses: Option>, + /// The optional number of max reconnect retries for the TCP transport /// /// [default: 10] @@ -244,6 +250,10 @@ pub struct Args { /// The optional client address for the TCP transport pub tcp_server_address: String, + /// The optional addresses of other nodes of the same cluster, dialed in + /// order when the server address cannot be reached + pub tcp_failover_addresses: Vec, + /// The optional number of maximum reconnect retries for the TCP transport pub tcp_reconnection_enabled: bool, @@ -380,6 +390,7 @@ impl Default for Args { username: DEFAULT_ROOT_USERNAME.to_string(), password: DEFAULT_ROOT_PASSWORD.to_string(), tcp_server_address: "127.0.0.1:8090".to_string(), + tcp_failover_addresses: Vec::new(), tcp_reconnection_enabled: true, tcp_reconnection_max_retries: Some(10), tcp_reconnection_interval: "1s".to_string(), @@ -446,6 +457,9 @@ impl From> for Args { if let Some(tcp_server_address) = optional_args.tcp_server_address { args.tcp_server_address = tcp_server_address; } + if let Some(tcp_failover_addresses) = optional_args.tcp_failover_addresses { + args.tcp_failover_addresses = tcp_failover_addresses; + } if let Some(tcp_reconnection_retries) = optional_args.tcp_reconnection_max_retries { args.tcp_reconnection_max_retries = Some(tcp_reconnection_retries); } diff --git a/core/common/src/types/configuration/auth_config/connection_string.rs b/core/common/src/types/configuration/auth_config/connection_string.rs index 82691812f0..96491693d1 100644 --- a/core/common/src/types/configuration/auth_config/connection_string.rs +++ b/core/common/src/types/configuration/auth_config/connection_string.rs @@ -159,6 +159,7 @@ impl ConnectionStringUtils { mod tests { use super::*; use crate::NonZeroIggyDuration; + use crate::TcpClientConfig; use crate::TcpConnectionStringOptions; use secrecy::ExposeSecret; @@ -320,4 +321,32 @@ mod tests { NonZeroIggyDuration::from_str("5s").unwrap() ); } + + #[test] + fn should_carry_failover_addresses_into_the_config() { + let value = format!( + "{DEFAULT_CONNECTION_STRING_PREFIX}user:secret@127.0.0.1:1234?failover_addresses=127.0.0.2:1234, 127.0.0.3:1234" + ); + let connection_string = + ConnectionString::::new(&value).unwrap(); + assert_eq!( + connection_string.options.failover_addresses(), + ["127.0.0.2:1234", "127.0.0.3:1234"] + ); + + let config = TcpClientConfig::from(connection_string); + assert_eq!(config.server_address, "127.0.0.1:1234"); + assert_eq!( + config.failover_addresses, + ["127.0.0.2:1234", "127.0.0.3:1234"] + ); + } + + #[test] + fn should_leave_the_failover_addresses_empty_when_the_option_is_absent() { + let value = format!("{DEFAULT_CONNECTION_STRING_PREFIX}user:secret@127.0.0.1:1234"); + let connection_string = + ConnectionString::::new(&value).unwrap(); + assert!(connection_string.options.failover_addresses().is_empty()); + } } diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_config.rs b/core/common/src/types/configuration/tcp_config/tcp_client_config.rs index 44480c3377..288337af65 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_config.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_config.rs @@ -72,10 +72,7 @@ impl From> for TcpClientConfig { fn from(connection_string: ConnectionString) -> Self { TcpClientConfig { server_address: connection_string.server_address().into(), - // The connection-string grammar names a single host, so a client - // built from one starts with no seeds and learns the roster once - // it is connected. - failover_addresses: Vec::new(), + failover_addresses: connection_string.options().failover_addresses().to_vec(), auto_login: connection_string.auto_login().to_owned(), tls_enabled: connection_string.options().tls_enabled(), tls_domain: connection_string.options().tls_domain().into(), diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs b/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs index a2365e5917..3d4c2957b1 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs @@ -24,7 +24,9 @@ pub struct TcpClientReconnectionConfig { /// endpoints the client knows still get one pass, since they were /// configured to be tried, but nothing is retried after it. pub enabled: bool, - /// How many passes over the known endpoints, or `None` for unlimited. + /// How many passes over the known endpoints *after the first*, or `None` + /// for unlimited. `Some(0)` still makes that one pass, since the endpoints + /// were configured to be tried. /// /// Passes, not dials: one pass tries the endpoint the client is on, the /// addresses it was configured with, and every node the roster named, so a @@ -34,9 +36,14 @@ pub struct TcpClientReconnectionConfig { /// Delay between passes. The first pass runs at once when the client knows /// more than one endpoint. pub interval: NonZeroIggyDuration, - /// Cooldown before redialing the endpoint that was just lost. It is owed to - /// that endpoint alone: the others are dialed without waiting, and the - /// paced one goes last in the pass. + /// Cooldown before redialing the endpoint of the last successful + /// connection, measured from when that connection was established rather + /// than from when it was lost: a session that outlived this interval is + /// redialed with no wait at all, which is the point -- the pace limit is + /// there for connections that keep dropping straight away. + /// + /// Owed to that endpoint alone: the others are dialed without waiting, and + /// the paced one goes last in the pass. pub reestablish_after: IggyDuration, } diff --git a/core/common/src/types/configuration/tcp_config/tcp_connection_string_options.rs b/core/common/src/types/configuration/tcp_config/tcp_connection_string_options.rs index 1c957f1c37..ebd5042857 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_connection_string_options.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_connection_string_options.rs @@ -23,6 +23,7 @@ use std::str::FromStr; #[derive(Debug)] pub struct TcpConnectionStringOptions { + failover_addresses: Vec, tls_enabled: bool, tls_domain: String, tls_ca_file: Option, @@ -32,6 +33,10 @@ pub struct TcpConnectionStringOptions { } impl TcpConnectionStringOptions { + pub fn failover_addresses(&self) -> &[String] { + &self.failover_addresses + } + pub fn tls_enabled(&self) -> bool { self.tls_enabled } @@ -64,6 +69,7 @@ impl ConnectionStringOptions for TcpConnectionStringOptions { fn parse_options(options: &str) -> Result { let options = options.split('&').collect::>(); + let mut failover_addresses = Vec::new(); let mut tls_enabled = false; let mut tls_domain = "".to_string(); let mut tls_ca_file = None; @@ -79,6 +85,14 @@ impl ConnectionStringOptions for TcpConnectionStringOptions { return Err(IggyError::InvalidConnectionString); } match option_parts[0] { + "failover_addresses" => { + failover_addresses = option_parts[1] + .split(',') + .map(str::trim) + .filter(|address| !address.is_empty()) + .map(str::to_string) + .collect(); + } "tls" => { tls_enabled = option_parts[1] == "true"; } @@ -129,6 +143,7 @@ impl ConnectionStringOptions for TcpConnectionStringOptions { .map_err(|_| IggyError::InvalidConnectionString)?; let connection_string_options = TcpConnectionStringOptions::new( + failover_addresses, tls_enabled, tls_domain, tls_ca_file, @@ -143,6 +158,7 @@ impl ConnectionStringOptions for TcpConnectionStringOptions { impl TcpConnectionStringOptions { pub fn new( + failover_addresses: Vec, tls_enabled: bool, tls_domain: String, tls_ca_file: Option, @@ -151,6 +167,7 @@ impl TcpConnectionStringOptions { nodelay: bool, ) -> Self { Self { + failover_addresses, tls_enabled, tls_domain, tls_ca_file, @@ -164,6 +181,7 @@ impl TcpConnectionStringOptions { impl Default for TcpConnectionStringOptions { fn default() -> Self { TcpConnectionStringOptions { + failover_addresses: Vec::new(), tls_enabled: false, tls_domain: "".to_string(), tls_ca_file: None, diff --git a/core/sdk/src/client_provider.rs b/core/sdk/src/client_provider.rs index 82d6d82041..a107c881de 100644 --- a/core/sdk/src/client_provider.rs +++ b/core/sdk/src/client_provider.rs @@ -135,9 +135,7 @@ impl ClientProviderConfig { TransportProtocol::Tcp => { config.tcp = Some(Arc::new(TcpClientConfig { server_address: args.tcp_server_address, - // Command-line arguments name a single server; the roster - // is learned once the client is connected. - failover_addresses: Vec::new(), + failover_addresses: args.tcp_failover_addresses, tls_enabled: args.tcp_tls_enabled, tls_domain: args.tcp_tls_domain, tls_ca_file: args.tcp_tls_ca_file, diff --git a/core/sdk/src/clients/client_builder.rs b/core/sdk/src/clients/client_builder.rs index 20a960c6d3..5cfa317b48 100644 --- a/core/sdk/src/clients/client_builder.rs +++ b/core/sdk/src/clients/client_builder.rs @@ -160,6 +160,13 @@ impl TcpClientBuilder { self } + /// Sets the addresses of other nodes of the same cluster, dialed in order + /// when the server address cannot be reached. + pub fn with_failover_addresses(mut self, failover_addresses: Vec) -> Self { + self.config = self.config.with_failover_addresses(failover_addresses); + self + } + /// Sets the auto sign in during connection. pub fn with_auto_sign_in(mut self, auto_sign_in: AutoLogin) -> Self { self.config = self.config.with_auto_sign_in(auto_sign_in); diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index 59cb24b018..2b54f8aaa9 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -372,38 +372,59 @@ impl iggy_common::VsrSessionControl for TcpClient { } async fn refresh_session_password(&self, user: &Identifier, new_password: &str) { - let mut remembered = self.session_credentials.lock().await; - let Some(sign_in) = remembered.as_mut() else { - return; - }; - // A personal access token is not derived from the password. - let Credentials::UsernamePassword(username, password) = &mut sign_in.credentials else { - return; + // Two independent copies of a password can go stale here, and a change + // may target either user: the sign-in this client remembers, and the + // one a configured `AutoLogin` carries. A caller signed in as somebody + // else -- by hand, or with a token -- can still be the one who changed + // the configured user's password. + let remembered_username = { + let mut remembered = self.session_credentials.lock().await; + remembered.as_mut().and_then(|sign_in| { + // A personal access token is not derived from the password. + let Credentials::UsernamePassword(username, password) = &mut sign_in.credentials + else { + return None; + }; + let targets_session_user = match user.kind { + IdKind::Numeric => user.get_u32_value().is_ok_and(|id| id == sign_in.user_id), + IdKind::String => user + .get_cow_str_value() + .is_ok_and(|name| name.as_ref() == username), + }; + if targets_session_user { + *password = SecretString::from(new_password.to_owned()); + } + Some((username.clone(), targets_session_user)) + }) }; - let targets_session_user = match user.kind { - IdKind::Numeric => user.get_u32_value().is_ok_and(|id| id == sign_in.user_id), - IdKind::String => user - .get_cow_str_value() - .is_ok_and(|name| name.as_ref() == username), - }; - if !targets_session_user { + let AutoLogin::Enabled(Credentials::UsernamePassword(configured, _)) = + &self.config.auto_login + else { return; - } + }; - *password = SecretString::from(new_password.to_owned()); // The configured credentials cannot be rewritten -- the config is // shared and immutable -- and the password they carry will never work - // again, so the new one is kept here instead. Kept outside the + // again, so the new one is kept beside them. Kept outside the // remembered sign-in on purpose: that record is replaced wholesale by // every later login, so a marker on it would survive exactly one // reconnect and the one after that would replay the dead password. - let configured_user_changed = matches!( - &self.config.auto_login, - AutoLogin::Enabled(Credentials::UsernamePassword(configured, _)) - if configured == username - ); - if configured_user_changed { + // + // A numeric identifier can only be recognised as the configured user + // through the id the signed-in user's own login reported, so a change + // made from another user's session has to name the user for the + // configured copy to be refreshed. Naming it is what an administrator + // doing this from elsewhere does anyway. + let targets_configured_user = match user.kind { + IdKind::String => user + .get_cow_str_value() + .is_ok_and(|name| name.as_ref() == configured), + IdKind::Numeric => remembered_username.is_some_and(|(username, is_session_user)| { + is_session_user && &username == configured + }), + }; + if targets_configured_user { self.configured_password .lock() .await @@ -782,7 +803,20 @@ impl TcpClient { /// applied on top: the configured password will never work again, and every /// later reconnect would otherwise fail `InvalidCredentials`. async fn sign_in_credentials(&self) -> Option { + // The sign-in that last succeeded, whoever ran it. One rule in every + // SDK: a client is whoever it last signed in as, so the same failure + // restores the same session everywhere. A configured `AutoLogin` signs + // in through this very path, so for a client that never signed in by + // hand the remembered credentials *are* the configured ones. + if let Some(remembered) = self.session_credentials.lock().await.as_ref() { + return Some(remembered.credentials.clone()); + } + match &self.config.auto_login { + // Before the first sign-in, or after a logout dropped what was + // remembered. A password change this client committed for the + // configured user is applied on top: the configured password will + // never work again, and the config cannot be rewritten. AutoLogin::Enabled(Credentials::UsernamePassword(username, configured_password)) => { let password = self .configured_password @@ -793,12 +827,7 @@ impl TcpClient { Some(Credentials::UsernamePassword(username.clone(), password)) } AutoLogin::Enabled(credentials) => Some(credentials.clone()), - AutoLogin::Disabled => self - .session_credentials - .lock() - .await - .as_ref() - .map(|remembered| remembered.credentials.clone()), + AutoLogin::Disabled => None, } } @@ -1877,6 +1906,10 @@ mod tests { ) .await; + // A sign-out drops what was remembered, so what the configured + // credentials carry is what the next connect signs in with -- and this + // change was somebody else's. + client.forget_session_credentials().await; match client.sign_in_credentials().await { Some(Credentials::UsernamePassword(username, password)) => { assert_eq!(username, "configured"); @@ -1886,8 +1919,49 @@ mod tests { } } + // A change made from a session signed in as somebody else still kills the + // configured password, so the next connect must not replay it. Named rather + // than numbered, since only the signed-in user's own id is known here. #[tokio::test] - async fn configured_credentials_outrank_the_ones_a_sign_in_remembered() { + async fn a_password_change_naming_the_configured_user_reaches_it_from_another_session() { + let client = TcpClient::create(Arc::new(TcpClientConfig { + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "configured".to_string(), + "old".into(), + )), + ..TcpClientConfig::default() + })) + .expect("create the client"); + client + .remember_session_credentials( + Credentials::PersonalAccessToken("token".into()), + SESSION_USER_ID, + ) + .await; + + client + .refresh_session_password( + &Identifier::named("configured").expect("named identifier"), + "new", + ) + .await; + + client.forget_session_credentials().await; + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, password)) => { + assert_eq!(username, "configured"); + assert_eq!(password.expose_secret(), "new"); + } + other => panic!("expected the configured user with the new password, got {other:?}"), + } + } + + // One rule in every SDK: a client is whoever it last signed in as, so the + // same failure restores the same session in each of them. The connection + // re-authenticates from the login it captured, and a redial that replayed + // somebody else would make the outcome depend on which got there first. + #[tokio::test] + async fn the_last_sign_in_outranks_the_configured_credentials() { let client = TcpClient::create(Arc::new(TcpClientConfig { auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( "configured".to_string(), @@ -1903,6 +1977,14 @@ mod tests { ) .await; + match client.sign_in_credentials().await { + Some(Credentials::UsernamePassword(username, _)) => assert_eq!(username, "signed-in"), + other => panic!("expected the sign-in that last succeeded, got {other:?}"), + } + + // A sign-out leaves no session to restore, and the configured + // credentials are what every connect of this client signs in as. + client.forget_session_credentials().await; match client.sign_in_credentials().await { Some(Credentials::UsernamePassword(username, _)) => assert_eq!(username, "configured"), other => panic!("expected the configured credentials, got {other:?}"), diff --git a/examples/rust/src/shared/args.rs b/examples/rust/src/shared/args.rs index ef43f5fb40..6d8ba09cce 100644 --- a/examples/rust/src/shared/args.rs +++ b/examples/rust/src/shared/args.rs @@ -98,6 +98,9 @@ pub struct Args { #[arg(long, default_value = "127.0.0.1:8090")] pub tcp_server_address: String, + #[arg(long, value_delimiter = ',')] + pub tcp_failover_addresses: Vec, + #[arg(long, default_value = "false")] pub tcp_tls_enabled: bool, @@ -231,6 +234,7 @@ impl Default for Args { tcp_reconnection_reestablish_after: "5s".to_string(), tcp_heartbeat_interval: "5s".to_string(), tcp_server_address: "127.0.0.1:8090".to_string(), + tcp_failover_addresses: Vec::new(), tcp_tls_enabled: false, tcp_tls_domain: "localhost".to_string(), tcp_tls_ca_file: "".to_string(), @@ -331,6 +335,7 @@ impl Args { username: self.username.clone(), password: self.password.clone(), tcp_server_address: self.tcp_server_address.clone(), + tcp_failover_addresses: self.tcp_failover_addresses.clone(), tcp_reconnection_enabled: self.tcp_reconnection_enabled, tcp_reconnection_max_retries: self.tcp_reconnection_max_retries, tcp_reconnection_interval: self.tcp_reconnection_interval.clone(), diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs index b835fbe560..e0e0203976 100644 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/EndpointFailoverTests.cs @@ -219,6 +219,10 @@ await Assert.ThrowsAsync(() => /// A survivor that is not listening yet when its node dies still has to be found: the client keeps /// rotating over every endpoint it knows, so one that comes up while it is retrying is dialed on a /// later pass rather than only on the first. + /// + /// The retry budget counts rotations, not dials: a single retry buys a whole second pass over + /// both endpoints. Spent per dial, the budget would be gone before the survivor came up. + /// /// [Fact] public async Task ResumesOnASurvivorThatComesUpWhileTheClientIsRetrying() @@ -243,7 +247,9 @@ public async Task ResumesOnASurvivorThatComesUpWhileTheClientIsRetrying() ReconnectionSettings = new ReconnectionSettings { Enabled = true, - MaxRetries = 4, + // One retry, so the pass that finds the survivor is the one the budget pays for. A larger + // budget would find it whether the budget counts rotations or dials. + MaxRetries = 1, InitialDelay = TimeSpan.FromMilliseconds(200) } }; diff --git a/foreign/go/client/tcp/tcp_core.go b/foreign/go/client/tcp/tcp_core.go index c2145f7035..057907d61b 100644 --- a/foreign/go/client/tcp/tcp_core.go +++ b/foreign/go/client/tcp/tcp_core.go @@ -52,6 +52,21 @@ func GetDefaultOptions() Options { } } +// connectAttempt is one run of Connect, shared with the callers waiting on it. +// +// The outcome is kept per attempt rather than in a field on the client: a +// waiter that read a shared field would read whatever the attempt after the one +// it waited on had written there, and a fresh attempt has no outcome yet. +type connectAttempt struct { + // done is closed once the attempt settles, whichever way it ends. + done chan struct{} + // err is the attempt's outcome, written before done is closed. + err error + // suppressesLogin records that the owner does not sign in, which is what + // makes the attempt safe to wait on from inside the sign-in transaction. + suppressesLogin bool +} + type IggyTcpClient struct { conn net.Conn // reader buffers reads off conn, so a reply costs one syscall instead of @@ -82,11 +97,9 @@ type IggyTcpClient struct { // loggedOut records an explicit sign-out, so a reconnect's automatic // sign-in does not silently reverse it; guarded by c.mtx. loggedOut bool - // connectInFlight is the attempt a Connect is running, shared with every - // caller that arrives while it is in progress: closed when the attempt - // settles, with connectErr carrying its outcome. Guarded by c.mtx. - connectInFlight chan struct{} - connectErr error + // connectAttempt is the attempt a Connect is running, shared with every + // caller that arrives while it is in progress. Guarded by c.mtx. + connectAttempt *connectAttempt // rememberedLogin holds the credentials a manual sign-in succeeded with, // so a reconnect -- on this node or, after a failover, another one -- can // re-establish the session instead of surfacing an unauthenticated error. @@ -648,11 +661,6 @@ func (c *IggyTcpClient) sendFrame(ctx context.Context, code uint32, frame []byte return nil, redirectErr } if redirect { - // A connect-scoped request is issued from inside the sign-in - // transaction, which holds registerMtx: the automatic sign-in - // on the reconnect path would wait on that lock forever. The - // transaction signs in itself on the node it lands on, so the - // reconnect must not. redirectCtx := ctx if ctx.Value(connectScoped{}) != nil { // Issued from inside the sign-in transaction, which holds @@ -919,6 +927,7 @@ func (c *IggyTcpClient) GetConnectionInfo() *iggcon.ConnectionInfo { // instead would hand them a client with no connection yet, and their next // request would fail ErrNotConnected for no reason of its own. func (c *IggyTcpClient) Connect(ctx context.Context) (err error) { + suppressesLogin := ctx.Value(skipAutoLogin{}) != nil c.mtx.Lock() switch c.transportState { case iggcon.TransportStateShutdown: @@ -931,40 +940,48 @@ func (c *IggyTcpClient) Connect(ctx context.Context) (err error) { c.logger.Debug("Client is already connected.", slog.String("client_address", clientAddress)) return nil case iggcon.TransportStateConnecting: - inFlight := c.connectInFlight + attempt := c.connectAttempt c.mtx.Unlock() - c.logger.Debug("Client is already connecting; waiting for that attempt.") - if inFlight == nil { + if attempt == nil { return nil } + if suppressesLogin && !attempt.suppressesLogin { + // Only the sign-in transaction suppresses the automatic sign-in, + // and it holds registerMtx while it does. The attempt in flight + // ends in a sign-in that needs that same lock, so waiting here + // would close a cycle: the owner blocked on registerMtx, this + // goroutine blocked on the owner, and neither context cancelled. + c.logger.Debug("Another connect is signing in; not waiting for it.") + return ierror.ErrCannotEstablishConnection + } + c.logger.Debug("Client is already connecting; waiting for that attempt.") select { - case <-inFlight: + case <-attempt.done: case <-ctx.Done(): return ctx.Err() case <-c.closed: return ierror.ErrClientShutdown } - c.mtx.Lock() - attemptErr := c.connectErr - c.mtx.Unlock() - return attemptErr - default: - c.transportState = iggcon.TransportStateConnecting - c.connectInFlight = make(chan struct{}) - c.connectErr = nil + return attempt.err + } + attempt := &connectAttempt{ + done: make(chan struct{}), + suppressesLogin: suppressesLogin, } + c.transportState = iggcon.TransportStateConnecting + c.connectAttempt = attempt connectedAt := c.connectedAt c.mtx.Unlock() // Settles the attempt for whoever is waiting on it, whichever way it ends. defer func() { + attempt.err = err c.mtx.Lock() - c.connectErr = err - if c.connectInFlight != nil { - close(c.connectInFlight) - c.connectInFlight = nil + if c.connectAttempt == attempt { + c.connectAttempt = nil } c.mtx.Unlock() + close(attempt.done) }() candidates := c.connectionCandidates() @@ -1056,24 +1073,23 @@ func (c *IggyTcpClient) Connect(ctx context.Context) (err error) { } c.mtx.Lock() - if c.transportState != iggcon.TransportStateConnecting { - // Disconnected or shut down while this attempt was dialing: the - // connection it just made is not wanted, and installing it would - // resurrect a client somebody asked to stop. - state := c.transportState + if state := c.transportState; state != iggcon.TransportStateConnecting { + // Superseded while this attempt was dialing. The connection it just + // made is surplus either way, but what to report differs: a client + // another attempt already connected is connected, and saying + // otherwise would fail a caller whose client is up. c.mtx.Unlock() _ = conn.Close() - c.logger.Debug("The connect was superseded while dialing; dropping the connection.") - if state == iggcon.TransportStateShutdown { + c.logger.Debug("The connect was superseded while dialing; dropping the connection.", + slog.Any("transport_state", state)) + switch state { + case iggcon.TransportStateShutdown: return ierror.ErrClientShutdown + case iggcon.TransportStateConnected: + return nil + default: + return ierror.ErrNotConnected } - return ierror.ErrNotConnected - } - // A connection installed over another one leaks its socket: two Connects - // can race here, and the loser's conn would otherwise stay open with - // nothing left holding it. - if previous := c.conn; previous != nil { - _ = previous.Close() } c.conn = conn c.reader = bufio.NewReaderSize(conn, 64*1024) @@ -1100,6 +1116,10 @@ func (c *IggyTcpClient) Connect(ctx context.Context) (err error) { // client's own TLS configuration is wrong -- an unreadable or unparsable CA // file, a domain that yields no server name, or a certificate this client will // never accept. None of those change on a retry. +// +// A peer that answered the ClientHello in plaintext is not one of them: that +// says something about the endpoint, not about this client, and the endpoints +// behind it in the roster may be speaking TLS perfectly well. func isTLSConfigFault(err error) bool { if errors.Is(err, ierror.ErrInvalidTlsCertificatePath) || errors.Is(err, ierror.ErrInvalidTlsCertificate) || @@ -1108,8 +1128,7 @@ func isTLSConfigFault(err error) bool { } var certificateError *tls.CertificateVerificationError - var recordError tls.RecordHeaderError - return errors.As(err, &certificateError) || errors.As(err, &recordError) + return errors.As(err, &certificateError) } // awaitReestablish waits out what is left of the reestablishAfter window since @@ -1332,6 +1351,15 @@ func (c *IggyTcpClient) disconnect() error { if c.transportState == iggcon.TransportStateDisconnected || c.transportState == iggcon.TransportStateShutdown { return nil } + if c.transportState == iggcon.TransportStateConnecting { + // An attempt is already dialing. Every caller here is tearing the + // connection down to reconnect, which that attempt is doing anyway: + // resetting the state under it would let the next Connect start a + // second attempt, and the two would fight over which socket ends up + // installed and which error the waiters are told about. + c.logger.Debug("Not disconnecting; a connect is already in flight.") + return nil + } c.logger.Info("Iggy client is disconnecting from server...", slog.String("client_address", c.clientAddress)) c.transportState = iggcon.TransportStateDisconnected diff --git a/foreign/go/client/tcp/tcp_failover_test.go b/foreign/go/client/tcp/tcp_failover_test.go index 6ad47d16af..e7946a462e 100644 --- a/foreign/go/client/tcp/tcp_failover_test.go +++ b/foreign/go/client/tcp/tcp_failover_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + iggcon "github.com/apache/iggy/foreign/go/contracts" ierror "github.com/apache/iggy/foreign/go/errors" "github.com/apache/iggy/foreign/go/internal/command" "github.com/apache/iggy/foreign/go/internal/vsr" @@ -543,6 +544,49 @@ func TestFailover_AConfigFaultEndsTheConnectInsteadOfRetryingForever(t *testing. } } +// A peer that answers the handshake in plaintext says something about that +// endpoint, not about this client's TLS configuration. Ended the whole connect, +// one misconfigured node in the roster costs the client every endpoint behind +// it, including the ones that are only down for a moment. +func TestFailover_APlaintextEndpointDoesNotEndTheSweep(t *testing.T) { + certificate, _ := selfSignedCert(t) + var accepted atomic.Int32 + var survivor *testListener + survivor = listenVSR(t, + func(conn net.Conn) net.Conn { + if accepted.Add(1) == 1 { + // Down for the first pass, up for the second: without it the + // sweep reaches this node and the pass succeeds either way. + _ = conn.Close() + return conn + } + return tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{certificate}}) + }, + singleNodeHandler(t, func() string { return survivor.address() })) + + plaintext, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = plaintext.Close() }) + go func() { + for { + conn, err := plaintext.Accept() + if err != nil { + return + } + _, _ = conn.Write([]byte("not a TLS record\n")) + _ = conn.Close() + } + }() + + client := newDialingClient(t, plaintext.Addr().String(), + WithTLS(WithTLSValidateCertificate(false))) + client.config.reconnection.maxRetries = 2 + client.knownServerAddresses = []string{survivor.address()} + + require.NoError(t, client.Connect(context.Background())) + assert.Equal(t, survivor.address(), client.currentServerAddress) +} + // A client with nothing to dial must say so: reporting success would leave // every request answering ErrNotConnected while Connect keeps claiming a // connection. @@ -601,6 +645,98 @@ func TestConnect_ConcurrentCallersShareOneAttempt(t *testing.T) { assert.Equal(t, 1, registers, "one attempt, one sign-in") } +// Two requests failing at once are one reconnect. Each tears the connection +// down before reconnecting, and a teardown that resets the state under an +// attempt already dialing lets the second caller start a second attempt: the +// two then fight over which socket is installed and which outcome the waiters +// are handed. +func TestConnect_ConcurrentReconnectsThroughExchangeShareOneAttempt(t *testing.T) { + var server *testListener + server = listenVSR(t, nil, func(connection, index int, read request) []byte { + if connection == 0 && read.code() == uint32(command.PingCode) { + // Ends the socket under both in-flight requests at once. + return nil + } + if connection > 0 && read.operation() == vsr.OperationRegister { + // The reconnect's sign-in is the slow part, so the second caller + // reliably arrives while the first attempt is still running. + time.Sleep(300 * time.Millisecond) + } + return singleNodeHandler(t, func() string { return server.address() })(connection, index, read) + }) + + client := newDialingClient(t, server.address(), + WithAutoLogin(NewUsernamePasswordCredentials("iggy", "iggy"))) + require.NoError(t, client.Connect(context.Background())) + + const callers = 2 + results := make(chan error, callers) + start := make(chan struct{}) + for range callers { + go func() { + <-start + results <- client.Ping(context.Background()) + }() + } + close(start) + for range callers { + require.NoError(t, <-results, "a request did not survive the reconnect") + } + + assert.Equal(t, 2, server.connections(), + "the two failing requests reconnected separately") +} + +// The sign-in transaction holds registerMtx across its reconnect, and an +// attempt started by a plain request ends in a sign-in that needs that same +// lock. Waiting for that attempt closes a cycle -- the owner blocked on +// registerMtx, the transaction blocked on the owner -- and callers pass a +// context with no deadline, so nothing breaks it. +func TestConnect_DoesNotWaitOnAnAttemptThatSignsIn(t *testing.T) { + certificate, _ := selfSignedCert(t) + var survivor *testListener + survivor = listenVSR(t, + func(conn net.Conn) net.Conn { + return tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{certificate}}) + }, + singleNodeHandler(t, func() string { return survivor.address() })) + + // A listener that accepts and never answers the ClientHello: the attempt + // spends the whole dial bound here, which is the window a second caller + // arrives in. + silent, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = silent.Close() }) + + client := newDialingClient(t, silent.Addr().String(), + WithTLS(WithTLSValidateCertificate(false)), + WithAutoLogin(NewUsernamePasswordCredentials("iggy", "iggy"))) + client.knownServerAddresses = []string{survivor.address()} + + // Stands in for the sign-in transaction: register holds this across the + // disconnect and the reconnect that follow it. + client.registerMtx.Lock() + owner := make(chan error, 1) + go func() { owner <- client.Connect(context.Background()) }() + require.Eventually(t, func() bool { + client.mtx.Lock() + defer client.mtx.Unlock() + return client.transportState == iggcon.TransportStateConnecting + }, time.Second, time.Millisecond, "the attempt never started dialing") + + suppressed := make(chan error, 1) + go func() { suppressed <- client.Connect(suppressAutoLogin(context.Background())) }() + select { + case err := <-suppressed: + require.Error(t, err, "the transaction was told a connection it does not have is up") + case <-time.After(2 * failoverDialTimeout): + t.Fatal("the sign-in transaction waited on an attempt that cannot finish without it") + } + + client.registerMtx.Unlock() + require.NoError(t, <-owner, "the attempt the transaction left alone did not finish") +} + // deadAddress returns an address nothing listens on, so a dial to it is // refused at once. func deadAddress(t *testing.T) string { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java index c0344a50f2..3f6ebd0d49 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java @@ -116,15 +116,18 @@ */ public class AsyncIggyTcpClient { - private static final int INVALID_COMMAND_ERROR_CODE = 3; - private static final Logger log = LoggerFactory.getLogger(AsyncIggyTcpClient.class); - private static final RetryPolicy DEFAULT_RECONNECT_POLICY = RetryPolicy.fixedDelay(12, Duration.ofSeconds(5)); /** * Bound on one dial while other endpoints are queued behind it, matching * the Rust, Go, C# and Node SDKs. Netty's own connect timeout would * otherwise let a node whose syns are dropped hold the whole rotation. + * + * Package-private: the tests pin the bound against the other SDKs'. */ - private static final Duration FAILOVER_DIAL_TIMEOUT = Duration.ofSeconds(2); + static final Duration FAILOVER_DIAL_TIMEOUT = Duration.ofSeconds(2); + + private static final int INVALID_COMMAND_ERROR_CODE = 3; + private static final Logger log = LoggerFactory.getLogger(AsyncIggyTcpClient.class); + private static final RetryPolicy DEFAULT_RECONNECT_POLICY = RetryPolicy.fixedDelay(12, Duration.ofSeconds(5)); private final ConnectionInfo seedConnectionInfo; private final AtomicBoolean reconnecting = new AtomicBoolean(); @@ -509,15 +512,20 @@ private AsyncTcpConnection openConnection(ConnectionInfo target) { * * With other endpoints queued behind this one, a node whose syns are * dropped must not hold the rotation, so the wait is capped at - * {@link #FAILOVER_DIAL_TIMEOUT} - the bound the other SDKs use. A caller - * who configured a connection timeout gets exactly that, and a client that - * knows one endpoint keeps the ordinary default. + * {@link #FAILOVER_DIAL_TIMEOUT} - the bound the other SDKs use. A + * configured connection timeout is capped too rather than exempted: it says + * how long one endpoint may take, and a rotation that spends it on every + * endpoint reaches the survivor long after the caller gave up. A client that + * knows one endpoint keeps whatever it configured, since there is nothing + * queued behind that dial. */ - private Optional dialTimeout() { - if (connectionTimeout.isPresent() || redialCandidates().size() < 2) { + Optional dialTimeout() { + if (redialCandidates().size() < 2) { return connectionTimeout; } - return Optional.of(FAILOVER_DIAL_TIMEOUT); + return Optional.of(connectionTimeout + .filter(configured -> configured.compareTo(FAILOVER_DIAL_TIMEOUT) < 0) + .orElse(FAILOVER_DIAL_TIMEOUT)); } /** @@ -670,7 +678,9 @@ private CompletableFuture redialAttempt(int attempt, RetryPolicy policy) { // retries redials nothing, which is what it asked for. boolean sweepOnce = attempt == 1 && candidates.size() > 1; if (attempt > policy.getMaxRetries() && !sweepOnce) { - log.error("Redial gave up after {} attempts, next request will fail fast", policy.getMaxRetries()); + // The rotations actually run, not the configured budget: a policy of + // zero retries still sweeps once when it knows several endpoints. + log.error("Redial gave up after {} rotations, next request will fail fast", attempt - 1); return CompletableFuture.completedFuture(null); } // The delay paces rotations, not dials. The first rotation runs at once @@ -699,10 +709,14 @@ private CompletableFuture sweepCandidates( return redialAttempt(attempt + 1, policy); } ConnectionInfo target = candidates.get(index); + // A policy of zero retries that knows several endpoints still gets the + // one rotation they were made known for, so the budget shown here is + // what will actually run rather than what was configured. + int rotations = Math.max(policy.getMaxRetries(), candidates.size() > 1 ? 1 : 0); log.info( "Redial attempt {}/{} to {} ({}/{})", attempt, - policy.getMaxRetries(), + rotations, target.serverAddress(), index + 1, candidates.size()); @@ -779,14 +793,14 @@ private static Throwable unwrap(Throwable error) { * sign-in that last succeeded, falling back to the credentials configured * on the builder when no login has run yet. * - * The last sign-in rather than the configured one, which is where this - * differs from the Rust and Go SDKs: the connection re-authenticates a - * replacement channel from the login payload it captured, which is that - * same last sign-in. Replaying a different user here would make the same - * eviction land on a different session depending on whether the channel - * or the redial got there first. A client that only ever used its - * configured credentials remembers exactly those, so nothing changes for - * it. + * The last sign-in outranks the configured credentials, the same rule as in + * every other SDK: a client is whoever it last signed in as. The connection + * also re-authenticates a replacement channel from the login payload it + * captured, which is that same last sign-in, so replaying a different user + * here would make one eviction land on a different session depending on + * whether the channel or the redial got there first. A client that only ever + * used its configured credentials remembers exactly those, so nothing + * changes for it. * * The login runs through the users client, so leader discovery retargets * again before Register when the redialed node is not the leader. @@ -954,11 +968,6 @@ boolean hasRememberedLogin() { return rememberedLogin != null; } - /** The live connection, for tests in this package. */ - AsyncTcpConnection currentConnection() { - return connection.get(); - } - /** * Endpoints a redial rotates through, likeliest first: where the client * currently is, the address it was configured with, then the roster it diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java index 71a0bca733..48d70abb4e 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java @@ -36,6 +36,7 @@ import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslHandler; import io.netty.util.concurrent.FutureListener; import io.netty.util.concurrent.ScheduledFuture; import org.apache.iggy.client.async.tcp.vsr.ConsensusSession; @@ -169,12 +170,13 @@ public AsyncTcpConnection( this.vsrEncoder = new VsrRequestEncoder(consensusSession); this.eventLoopGroup = new MultiThreadIoEventLoopGroup(NioIoHandler.newFactory()); + long dialTimeoutMillis = + connectionTimeout.orElse(DEFAULT_CONNECTION_TIMEOUT).toMillis(); var bootstrap = new Bootstrap() .group(eventLoopGroup) .channel(NioSocketChannel.class) .option(ChannelOption.TCP_NODELAY, true) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) - connectionTimeout.orElse(DEFAULT_CONNECTION_TIMEOUT).toMillis()) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) dialTimeoutMillis) .option(ChannelOption.SO_KEEPALIVE, true) .remoteAddress(host, port); @@ -184,7 +186,14 @@ public AsyncTcpConnection( this.channelPool = new FixedChannelPool( bootstrap, new PoolChannelHandler( - host, port, enableTls, sslContext, consensusSession, maxVsrFrameSize, this::onSessionEvicted), + host, + port, + enableTls, + sslContext, + dialTimeoutMillis, + consensusSession, + maxVsrFrameSize, + this::onSessionEvicted), ChannelHealthChecker.ACTIVE, FixedChannelPool.AcquireTimeoutAction.FAIL, poolConfig.getAcquireTimeoutMillis(), @@ -832,9 +841,10 @@ private void handlePostResponse(Channel channel, int commandCode, boolean isLogi * login and Register. The fresh session invalidates cached routing state * such as consumer-group assignments. * - * The reason travels to the listener, which owns the question of whether - * the session may be re-established at all: only it knows whether the - * sign-in was configured on the client or run by a caller. + * The reason travels to the listener so it can drop what belonged to the + * evicted session and log what happened. The session itself is kept + * whichever way the sign-in was made: only an explicit sign-out or close + * ends one. */ private void onSessionEvicted(int errorCode) { authGeneration.incrementAndGet(); @@ -900,15 +910,18 @@ private static final class PoolChannelHandler extends AbstractChannelPoolHandler private final int port; private final boolean enableTls; private final SslContext sslContext; + private final long dialTimeoutMillis; private final ConsensusSession consensusSession; private final int maxVsrFrameSize; private final IntConsumer onEviction; + @SuppressWarnings("checkstyle:ParameterNumber") PoolChannelHandler( String host, int port, boolean enableTls, SslContext sslContext, + long dialTimeoutMillis, ConsensusSession consensusSession, int maxVsrFrameSize, IntConsumer onEviction) { @@ -916,6 +929,7 @@ private static final class PoolChannelHandler extends AbstractChannelPoolHandler this.port = port; this.enableTls = enableTls; this.sslContext = sslContext; + this.dialTimeoutMillis = dialTimeoutMillis; this.consensusSession = consensusSession; this.maxVsrFrameSize = maxVsrFrameSize; this.onEviction = onEviction; @@ -925,7 +939,12 @@ private static final class PoolChannelHandler extends AbstractChannelPoolHandler public void channelCreated(Channel ch) { ChannelPipeline pipeline = ch.pipeline(); if (enableTls) { - pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port)); + SslHandler ssl = sslContext.newHandler(ch.alloc(), host, port); + // A peer that accepts TCP and then never answers the + // ClientHello would otherwise hold the dial for Netty's own + // 10s default, well past the bound the rotation dials under. + ssl.setHandshakeTimeoutMillis(dialTimeoutMillis); + pipeline.addLast("ssl", ssl); } pipeline.addLast("frameDecoder", new VsrFrameDecoder(maxVsrFrameSize)); pipeline.addLast("responseHandler", new VsrResponseHandler(consensusSession, onEviction)); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java index ac182227f9..16a8f47474 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java @@ -163,10 +163,10 @@ private void handleEviction(ChannelHandlerContext ctx, ByteBuf frame) { IggyServerException error = VsrHeaders.evictionToException(frame); session.reset(); try { - // The reason travels with the notification: an eviction the server - // decided on (a stale client) ends the session authoritatively, - // while the rest are transport-shaped, and the listener has to - // tell them apart. + // The reason travels with the notification so the listener can drop + // what belonged to the evicted session and say which eviction it + // was. None of them ends the sign-in: the next request + // re-establishes the session. onEviction.accept(error.getRawErrorCode()); } catch (RuntimeException listenerError) { log.warn("Eviction listener failed: {}", listenerError.getMessage()); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java index a2bc8e11ae..e9a09de1ad 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientEndpointFailoverTest.java @@ -403,6 +403,115 @@ private interface RequestHandler { Response handle(Request request); } + /** + * A retry budget of zero still gets one rotation when several endpoints are + * known: those endpoints -- the address the client was configured with, the + * nodes the roster named -- were made known in order to be tried, and every + * other SDK sweeps them once too. With one endpoint known, zero retries + * redials nothing, which is what it asked for. + */ + @Test + void shouldSweepTheKnownEndpointsOnceWithNoRetries() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket primarySocket = new ServerSocket(0, 4, loopback); + ServerSocket survivorSocket = new ServerSocket(0, 4, loopback)) { + int primaryPort = primarySocket.getLocalPort(); + int survivorPort = survivorSocket.getLocalPort(); + AtomicInteger survivorRegistrations = new AtomicInteger(); + + MockNode primary = MockNode.serve(primarySocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, primaryPort)); + } + if (request.operation() == OPERATION_REGISTER) { + return Response.success(OPERATION_REGISTER, registerBody(1)); + } + return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER); + }); + MockNode survivor = MockNode.serve(survivorSocket, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(primaryPort, survivorPort, survivorPort)); + } + if (request.operation() == OPERATION_REGISTER) { + survivorRegistrations.incrementAndGet(); + return Response.success(OPERATION_REGISTER, registerBody(2)); + } + return Response.success(OPERATION_NON_REPLICATED, Unpooled.EMPTY_BUFFER); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(primaryPort) + .credentials("iggy", "iggy") + .requestTimeout(Duration.ofSeconds(2)) + .retryPolicy(RetryPolicy.noRetry()) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.login().get(5, TimeUnit.SECONDS); + client.sendBinaryRequest(PING_CODE, new byte[0]).get(5, TimeUnit.SECONDS); + + primary.kill(); + + assertThat(resumeWithin(client, Duration.ofSeconds(4))) + .as("zero retries skipped the one rotation the known endpoints are for") + .isTrue(); + assertThat(client.getConnectionInfo().port()).isEqualTo(survivorPort); + assertThat(survivorRegistrations) + .as("the sign-in was replayed on the survivor") + .hasValueGreaterThanOrEqualTo(1); + } finally { + client.close().get(5, TimeUnit.SECONDS); + survivor.close(); + } + } + } + + /** + * Every dial is capped once more than one endpoint is known, a configured + * connection timeout included: it says how long one endpoint may take, and a + * rotation that spends it on each of them reaches the survivor long after + * the caller gave up. A timeout shorter than the cap is what the caller + * asked for and stands; with one endpoint known nothing is queued behind + * the dial, so the configured value stands there too. + */ + @Test + void shouldCapEveryDialWhenMoreThanOneEndpointIsKnown() { + InetAddress loopback = InetAddress.getLoopbackAddress(); + List roster = List.of(new ConnectionInfo("iggy-1", 8091)); + + AsyncIggyTcpClient patient = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(8090) + .connectionTimeout(Duration.ofSeconds(30)) + .build(); + assertThat(patient.dialTimeout()).contains(Duration.ofSeconds(30)); + patient.rememberRoster(new LeaderAwareness.LeaderLookup(Optional.empty(), roster)); + assertThat(patient.dialTimeout()) + .as("a configured timeout exempted the dial from the failover cap") + .contains(AsyncIggyTcpClient.FAILOVER_DIAL_TIMEOUT); + + AsyncIggyTcpClient impatient = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(8090) + .connectionTimeout(Duration.ofMillis(500)) + .build(); + impatient.rememberRoster(new LeaderAwareness.LeaderLookup(Optional.empty(), roster)); + assertThat(impatient.dialTimeout()) + .as("a timeout shorter than the cap is what the caller asked for") + .contains(Duration.ofMillis(500)); + + AsyncIggyTcpClient unconfigured = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(8090) + .build(); + assertThat(unconfigured.dialTimeout()).isEmpty(); + unconfigured.rememberRoster(new LeaderAwareness.LeaderLookup(Optional.empty(), roster)); + assertThat(unconfigured.dialTimeout()).contains(AsyncIggyTcpClient.FAILOVER_DIAL_TIMEOUT); + } + /** * A roster read that learned nothing must leave the last one standing: * emptying the redial candidates when the cluster is unreachable takes them diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java index 95abbc29a9..897c5e476e 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java @@ -64,7 +64,6 @@ class AsyncIggyTcpClientTransientFailoverTest { private static final int OPERATION_CREATE_STREAM = 128; private static final int GET_CLUSTER_METADATA_CODE = 12; private static final int CREATE_STREAM_CODE = 202; - private static final int PING_CODE = 1; private static final int TRANSIENT_NOT_ACCEPTED = 58; private static final int EVICTION_STALE_CLIENT = 13; @@ -281,6 +280,72 @@ void shouldReviveTheSignInAfterAStaleClientEviction() throws Exception { } } + /** + * Credentials on the builder and a hand-run sign-in for somebody else: the + * revived session is the last sign-in, the same rule as on a redial and in + * every other SDK. The connection re-authenticates a replacement channel + * from the login it captured, which is that same sign-in, so replaying the + * configured user here would make one eviction land on a different session + * depending on which path got there first. + */ + @Test + void shouldReviveTheLastSignInRatherThanTheConfiguredOne() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 4, loopback)) { + AtomicInteger registrations = new AtomicInteger(); + List registeredLogins = new CopyOnWriteArrayList<>(); + AtomicBoolean evict = new AtomicBoolean(true); + CompletableFuture server = serve(serverSocket, 6, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); + } + if (request.operation() == OPERATION_REGISTER) { + registeredLogins.add(request.bodyAsText()); + return Response.success(OPERATION_REGISTER, registerBody(registrations.incrementAndGet())); + } + if (request.operation() == OPERATION_CREATE_STREAM) { + if (evict.compareAndSet(true, false)) { + return Response.eviction(EVICTION_STALE_CLIENT); + } + ByteBuf body = Unpooled.buffer(Integer.BYTES); + body.writeIntLE(0); + return Response.success(OPERATION_CREATE_STREAM, body); + } + return Response.success(request.operation(), Unpooled.EMPTY_BUFFER); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(serverSocket.getLocalPort()) + .credentials("configured", "configured") + .requestTimeout(Duration.ofSeconds(2)) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.users().login("handrun", "handrun").get(5, TimeUnit.SECONDS); + int registrationsBeforeEviction = registrations.get(); + + assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .hasCauseInstanceOf(IggyServerException.class); + + assertThat(client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .isEmpty(); + assertThat(registrations.get()) + .as("the evicted session was not re-established") + .isGreaterThan(registrationsBeforeEviction); + assertThat(registeredLogins.get(registeredLogins.size() - 1)) + .as("the revived session signed in as the configured user") + .contains("handrun") + .doesNotContain("configured"); + } finally { + client.close().get(5, TimeUnit.SECONDS); + } + server.completeExceptionally(new IllegalStateException("test over")); + } + } + private static Response handleOldLeader( Request request, int oldLeaderPort, int newLeaderPort, AtomicInteger denials) { if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { diff --git a/foreign/node/src/client/client.connection.test.ts b/foreign/node/src/client/client.connection.test.ts index 250c712385..b1269d19d0 100644 --- a/foreign/node/src/client/client.connection.test.ts +++ b/foreign/node/src/client/client.connection.test.ts @@ -548,6 +548,62 @@ describe('IggyConnection', () => { } ); + it('makes one pass when every endpoint is down and reconnection is disabled', + async () => { + // One pass, not the whole retry budget: with the budget read but the + // flag ignored, a client that opted out of retries dials every endpoint + // once per pass for all of them -- and with the backoff gated on the same + // flag, back to back. + // + // Plain TCP behind a TLS client, closed at once: the dial fails, so the + // pass moves on, and every dial is counted where it lands. + const first = await startServer(); + const firstPort = (first.address() as AddressInfo).port; + let firstDials = 0; + first.on('connection', (socket) => { + firstDials += 1; + socket.destroy(); + }); + const second = await startServer(); + const secondPort = (second.address() as AddressInfo).port; + let secondDials = 0; + second.on('connection', (socket) => { + secondDials += 1; + socket.destroy(); + }); + + const connection = new IggyConnection({ + transport: 'TLS', + options: { + host: '127.0.0.1', + port: firstPort, + rejectUnauthorized: false + }, + credentials: { username: 'iggy', password: 'iggy' }, + reconnect: { enabled: false, interval: 10, maxRetries: 12 }, + maxResponseFrameSize: FRAME_LIMIT + }); + connection.on('error', () => undefined); + try { + connection.rememberRoster([{ host: '127.0.0.1', port: secondPort }]); + await connection.connect().catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 300)); + + assert.equal(connection.connected, false); + // The connect's own dial of the configured endpoint, then one pass over + // both: the endpoint the client starts on is dialed twice, the one + // behind it once. + assert.deepEqual([firstDials, secondDials], [2, 1], + 'a client that opted out of retries swept more than once' + ); + } finally { + connection._destroy(); + await new Promise((resolve) => first.close(() => resolve())); + await new Promise((resolve) => second.close(() => resolve())); + } + } + ); + it('skips the first backoff when another endpoint is known', async () => { // The endpoint the client is on is dead and a live one sits behind it in diff --git a/foreign/node/src/client/client.connection.ts b/foreign/node/src/client/client.connection.ts index f4e3cbfb33..49f7752f89 100644 --- a/foreign/node/src/client/client.connection.ts +++ b/foreign/node/src/client/client.connection.ts @@ -133,7 +133,11 @@ export class IggyConnection extends EventEmitter { public redirecting: boolean; /** Reconnection configuration */ private reconnectOption: ReconnectOption; - /** Number of reconnection attempts made */ + /** + * Number of passes made over the known endpoints. One pass dials the + * endpoint the client is on, the endpoint it was configured with, and every + * node the roster named. + */ private reconnectCount: number; /** Shared promise for concurrent callers waiting on one connection attempt */ private connectPromise?: Promise; @@ -395,8 +399,14 @@ export class IggyConnection extends EventEmitter { // be tried, so they get one pass and no backoff, as in the other SDKs. A // client that knows one endpoint and turned reconnection off redials // nothing, which is what it asked for. + // Counted rather than tracked within this call: every dial the pass fails + // closes a socket, and a close starts a reconnect of its own. Bounded by + // `firstPass` alone, each of those closes would open another sweep and the + // pass would repeat for as long as the endpoints stay down. The count is + // reset when a connection is established, so a later loss sweeps again. const sweepOnce = !enabled && this._redialCandidates().length > 1; - while ((enabled && this.reconnectCount < maxRetries) || (sweepOnce && firstPass)) { + while ((enabled && this.reconnectCount < maxRetries) || + (sweepOnce && this.reconnectCount < 1)) { this.connecting = true; this.reconnectCount += 1; const candidates = this._redialCandidates(); diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts index 9a0591a987..2b1aae15f6 100644 --- a/foreign/node/src/client/client.socket.test.ts +++ b/foreign/node/src/client/client.socket.test.ts @@ -226,12 +226,6 @@ const connectionOf = (client: CommandResponseStream): { connection: { redirect: (host: string, port: number) => Promise } }).connection; -/** Drives the leader re-check a refused request runs. */ -const followLeaderMove = (client: CommandResponseStream): Promise => - (client as unknown as { - _followLeaderMove: () => Promise - })._followLeaderMove(); - const compressLeaderlessPoll = ( client: CommandResponseStream, budget: number @@ -1031,14 +1025,14 @@ describe('VSR client socket', () => { COMMAND_CODE.GetClusterMetadata ).length; - // Two refusals, one re-check: the second caller shares the move the - // first started rather than starting its own or being told 58. - const moves = await Promise.all([ - followLeaderMove(client), - followLeaderMove(client) + // Two commands, both refused by the demoted node: one re-check between + // them, and both answered on the node it moved to. + const answers = await Promise.all([ + client.sendCommand(60_032, Buffer.alloc(0)), + client.sendCommand(60_032, Buffer.alloc(0)) ]); - assert.deepEqual(moves, [true, true]); + assert.deepEqual(answers.map((answer) => answer.status), [0, 0]); const rosterReads = demoted.frames.filter( (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === COMMAND_CODE.GetClusterMetadata @@ -1046,6 +1040,12 @@ describe('VSR client socket', () => { assert.equal(rosterReads, 1, 'each refusal re-read the roster on its own' ); + const reissued = leader.frames.filter( + (frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_032 + ).length; + assert.equal(reissued, 2, + 'a refused command was not re-issued on the node the move landed on' + ); const connection = (client as unknown as { connection: { isConnectedTo: (host: string, port: number) => boolean } }).connection; @@ -1058,6 +1058,79 @@ describe('VSR client socket', () => { } ); + it('holds a queued command instead of writing it to the node being left', + async () => { + // A refusal sends its caller to re-read the roster, and the drain that + // handed it out keeps going. Written in that window, the next queued + // command goes to the socket the move is about to replace: in flight when + // that happens, it dies with a lost-connection error nobody can act on + // instead of being re-issued on the node the move lands on. + const leader = await startVsrServer((frame, socket) => { + singleNodeHandler(leader.port)(frame, socket); + }); + let demotedYet = false; + const demoted = await startVsrServer((frame, socket) => { + const code = frame.readUInt32LE(REQUEST_OFFSET.reserved); + if (code === COMMAND_CODE.GetClusterMetadata) { + socket.write(replyFrame( + Operation.NonReplicated, + demotedYet + ? twoNodeMetadataBody(demoted.port, leader.port) + : twoNodeMetadataBody(leader.port, demoted.port) + )); + return; + } + if (code === 60_037) { + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + if (code === 60_038) { + // Accepted and never answered: a command written here is stuck until + // the move replaces the socket under it. + return; + } + singleNodeHandler(demoted.port)(frame, socket); + }); + + const client = new CommandResponseStream(vsrConfig(demoted.port)); + try { + await client.authenticate(vsrConfig(demoted.port).credentials); + demotedYet = true; + + // The second command is queued while the first is in flight, which is + // where a command caught in a move comes from. + const refused = client.sendCommand(60_037, Buffer.alloc(0)); + const behind = client.sendCommand(60_038, Buffer.alloc(0)); + refused.catch(() => undefined); + behind.catch(() => undefined); + + const settled = await Promise.race([ + Promise.all([refused, behind]).then(() => 'answered'), + new Promise((resolve) => { + setTimeout(() => resolve('stalled'), 10_000).unref(); + }) + ]); + assert.equal(settled, 'answered', + 'the command behind the refusal went out on the node being left' + ); + assert.ok( + !demoted.frames.some((frame) => + frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_038), + 'the command behind the refusal was written to the node being left' + ); + assert.ok( + leader.frames.some((frame) => + frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_038), + 'the command behind the refusal never reached the node moved to' + ); + } finally { + client.destroy(); + await leader.close(); + await demoted.close(); + } + } + ); + it('re-issues a command queued behind a leader move instead of failing it', async () => { // A move replaces the socket, which looks like a drop to everything @@ -1117,9 +1190,6 @@ describe('VSR client socket', () => { const realNow = Date.now; try { await client.authenticate(vsrConfig(server.port).credentials); - // The request's own budget, then the clock jumped past it: what is left - // cannot carry another attempt, so the caller has to see the answer the - // server gave and not the timeout a doomed re-issue would produce. // The request's budget, the first exchange, the window that hands the // refusal out, and then a clock 10ms short of the deadline: too little // to carry another attempt, so the caller has to see the answer the @@ -1142,6 +1212,73 @@ describe('VSR client socket', () => { } ); + it('surfaces the refusal when the move left too little of the budget', + async () => { + // The move itself costs budget: a roster read, an election it waited out, + // a redial. What is left can be positive and still too small to carry + // another exchange, and re-issued into it the request times out -- the + // caller then sees a timeout where the answer was "not admitted". + const leader = await startVsrServer((frame, socket) => { + singleNodeHandler(leader.port)(frame, socket); + }); + let demotedYet = false; + const demoted = await startVsrServer((frame, socket) => { + const code = frame.readUInt32LE(REQUEST_OFFSET.reserved); + if (code === COMMAND_CODE.GetClusterMetadata) { + socket.write(replyFrame( + Operation.NonReplicated, + demotedYet + ? twoNodeMetadataBody(demoted.port, leader.port) + : twoNodeMetadataBody(leader.port, demoted.port) + )); + return; + } + if (code === 60_039) { + socket.write(replyFrame(Operation.NonReplicated, Buffer.alloc(0), 58)); + return; + } + singleNodeHandler(demoted.port)(frame, socket); + }); + + const client = new CommandResponseStream(vsrConfig(demoted.port)); + const realNow = Date.now; + let offset = 0; + try { + await client.authenticate(vsrConfig(demoted.port).credentials); + demotedYet = true; + + const deadline = realNow() + 30_000; + const connection = connectionOf(client); + const move = connection.redirect.bind(connection); + connection.redirect = async (host: string, port: number) => { + await move(host, port); + // 30ms of budget left the moment the client lands: positive, and + // below the interval one exchange needs. + offset = deadline - realNow() - 30; + }; + Date.now = () => realNow() + offset; + + await assert.rejects( + () => client.sendCommand(60_039, Buffer.alloc(0), { deadline }), + (error: unknown) => + error instanceof ResponseError && + error.commandCode === 60_039 && + error.errorCode === 58 + ); + assert.ok( + !leader.frames.some((frame) => + frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_039), + 'the request was re-issued into a budget too small to answer it' + ); + } finally { + Date.now = realNow; + client.destroy(); + await leader.close(); + await demoted.close(); + } + } + ); + it('paces the re-issues while the roster still names this node', async () => { // Re-issuing is right, spinning is not: the in-connection replay window diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts index bafd4a878c..3924ba14ad 100644 --- a/foreign/node/src/client/client.socket.ts +++ b/foreign/node/src/client/client.socket.ts @@ -90,6 +90,10 @@ type Job = { payload: Buffer, /** Whether to parse the response */ handleResponse: boolean, + /** Whether the command is appended rather than prepended to the queue */ + last: boolean, + /** Whether a not-admitted refusal re-checks the leader and re-issues */ + followsLeaderMoves: boolean, /** When the whole request gives up, however often it is re-issued */ deadline: number, /** Promise resolve function */ @@ -132,6 +136,13 @@ export class CommandResponseStream extends EventEmitter { * request refused by the same node so one demotion moves the client once. */ private leaderMoveInFlight?: Promise; + /** + * Refusals handed out to callers that have not decided what to do with them + * yet. The queue holds while any are outstanding: the caller of a refused + * command re-checks the leader, and a command written in the meantime goes + * out on the socket that check is about to replace. + */ + private leaderMovesUndecided: number; /** How long a leaderless roster is polled before settling in place */ private leaderlessWaitBudget: number; /** Delay between roster reads while the cluster elects */ @@ -165,6 +176,7 @@ export class CommandResponseStream extends EventEmitter { this.vsrSession = new VsrSession(); this.authenticationPromise = undefined; this.settlingLeader = false; + this.leaderMovesUndecided = 0; this.leaderlessWaitBudget = LEADERLESS_WAIT_BUDGET_MS; this.leaderlessPollInterval = LEADERLESS_POLL_INTERVAL_MS; this.pendingSubmissions = 0; @@ -214,8 +226,15 @@ export class CommandResponseStream extends EventEmitter { this._execQueue = []; for (const job of queued) { debug('re-issuing a queued command after a leader move', job.command); + // The whole job, not just the payload: a fresh budget would let a command + // caught in a move take twice the response timeout, and a roster read + // re-issued as leader-following would answer a leader check with another + // leader check. this.sendCommand(job.command, job.payload, { - handleResponse: job.handleResponse + handleResponse: job.handleResponse, + last: job.last, + followsLeaderMoves: job.followsLeaderMoves, + deadline: job.deadline }).then(job.resolve, job.reject); } } @@ -262,13 +281,15 @@ export class CommandResponseStream extends EventEmitter { // // One budget for the whole request: the transient replays on a // connection, the leader re-checks, and the re-issues after a move all - // spend it, so a request cannot outlive it by moving. - const deadline = Date.now() + VSR_RESPONSE_TIMEOUT_MS; + // spend it, so a request cannot outlive it by moving. A command re-issued + // after a move keeps the budget it was first submitted with, rather than + // opening a second one. + const deadline = options.deadline ?? Date.now() + VSR_RESPONSE_TIMEOUT_MS; let response: CommandResponse; for (;;) { try { response = await this._queueCommand(command, payload, handleResponse, - last, deadline); + last, followsLeaderMoves, deadline); break; } catch (error) { if (!(error instanceof LeaderMovedError)) @@ -282,10 +303,19 @@ export class CommandResponseStream extends EventEmitter { // refusal the server actually gave: re-issued into what is left, the // request would time out instead and the caller would see a timeout // where the answer was "not admitted". - if (!followsLeaderMoves || !worthAnotherAttempt(deadline)) - throw responseError(command, error.refusal.errorCode); - - const moved = await this._followLeaderMove(); + let moved = false; + try { + if (!followsLeaderMoves || !worthAnotherAttempt(deadline)) + throw responseError(command, error.refusal.errorCode); + moved = await this._followLeaderMove(); + } finally { + // Released as soon as the move is decided, before the pace below + // and before any re-authentication: those go through the queue + // themselves, and a queue still held for this refusal would never + // reach them. + if (followsLeaderMoves) + this._releaseUndecidedMove(); + } if (!moved) { // Nowhere else to go yet: the roster still names this node, or it // could not be read. Paced, because the in-connection replay @@ -326,13 +356,16 @@ export class CommandResponseStream extends EventEmitter { payload: Buffer, handleResponse: boolean, last: boolean, + followsLeaderMoves: boolean, deadline: number ): Promise { return new Promise((resolve, reject) => { - const job = { + const job: Job = { command, payload, handleResponse, + last, + followsLeaderMoves, deadline, resolve, reject @@ -379,12 +412,35 @@ export class CommandResponseStream extends EventEmitter { })(); this.leaderMoveInFlight = move; void move.finally(() => { - if (this.leaderMoveInFlight === move) - this.leaderMoveInFlight = undefined; + if (this.leaderMoveInFlight !== move) + return; + this.leaderMoveInFlight = undefined; + // The drain stopped while the move was being decided. A move that + // happened re-issues what was held back on the new socket; one that did + // not leaves it here, with nothing else due to pick it up. + if (!this.connection.redirecting) + void this._processQueue(); }); return move; } + /** Whether a leader move is being decided or carried out. */ + private _movePending(): boolean { + return this.leaderMovesUndecided > 0 || this.leaderMoveInFlight !== undefined; + } + + /** + * Releases the queue hold one refusal took, and drains what was held back + * once the last of them is decided. + */ + private _releaseUndecidedMove(): void { + if (this.leaderMovesUndecided > 0) + this.leaderMovesUndecided -= 1; + if (this._movePending() || this.connection.redirecting) + return; + void this._processQueue(); + } + private _rememberRoster(response: CommandResponse): void { try { const metadata = GET_CLUSTER_METADATA.deserialize(response); @@ -409,12 +465,27 @@ export class CommandResponseStream extends EventEmitter { return; this.busy = true; while (this._execQueue.length > 0 && this.connection.socket.writable) { - const next = this._execQueue.shift(); + // While a leader move is being decided, only the roster read the move + // itself runs goes out -- it is what decides where the client lands, and + // it is the one command that does not follow moves. Draining the rest + // would write them to the socket `redirect()` is about to replace, and a + // command in flight when that happens dies with a lost-connection error + // instead of being re-issued on the node the move lands on. + const index = this._movePending() + ? this._execQueue.findIndex((job) => !job.followsLeaderMoves) + : 0; + if (index < 0) break; + const [next] = this._execQueue.splice(index, 1); if (!next) break; const { command, payload, handleResponse, deadline, resolve, reject } = next; try { resolve(await this._processNext(command, payload, handleResponse, deadline)); } catch (err) { + if (err instanceof LeaderMovedError && next.followsLeaderMoves) + // Counted before the rejection is handed out, not after: the caller + // resumes as a microtask, so this loop would otherwise write the next + // command before the re-check it is about to start has begun. + this.leaderMovesUndecided += 1; reject(err); } } @@ -424,8 +495,11 @@ export class CommandResponseStream extends EventEmitter { // node being moved to. if (this.connection.redirecting) this._reissueQueue(); - else + else if (!this._movePending()) this._failQueue(new Error('connection is not writable')); + // Otherwise the move is still being decided: these commands were never + // written, and they are drained again once it settles -- here if the + // client stays, on the new socket if it moves. } this.busy = false; this._emitFinishQueue(); diff --git a/foreign/node/src/client/client.type.ts b/foreign/node/src/client/client.type.ts index bb9f89f0fe..42ae39239d 100644 --- a/foreign/node/src/client/client.type.ts +++ b/foreign/node/src/client/client.type.ts @@ -53,7 +53,14 @@ export type SendCommandOptions = { * command. False for the roster read a re-check itself runs: answering a * leader check with another leader check would recurse. */ - followsLeaderMoves?: boolean + followsLeaderMoves?: boolean, + /** + * When the whole request gives up, as an epoch timestamp in milliseconds. + * Set when a command already carries a budget -- one re-issued after a + * leader move keeps the budget it was first submitted with, rather than + * opening a second one on top of it. Defaults to a fresh response timeout. + */ + deadline?: number }; /** diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index 87cfafd466..cf3d847ce4 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi @@ -1933,6 +1933,8 @@ class TcpConfig: @property def server_address(self) -> builtins.str: ... @property + def failover_addresses(self) -> builtins.list[builtins.str]: ... + @property def auto_login(self) -> AutoLogin: ... @property def reconnection(self) -> TcpReconnectionConfig: ... @@ -1952,6 +1954,7 @@ class TcpConfig: cls, *, server_address: builtins.str | None = None, + failover_addresses: typing.Sequence[builtins.str] | None = None, auto_login: AutoLogin | None = None, reconnection: TcpReconnectionConfig | None = None, heartbeat_interval: datetime.timedelta | None = None, @@ -1966,6 +1969,11 @@ class TcpConfig: Args: server_address: `host:port` of the Iggy server. Defaults to `127.0.0.1:8090`. + failover_addresses: `host:port` of other nodes of the same cluster, dialed + in order when `server_address` cannot be reached. The roster the server + reports is remembered while the client is connected and dialed first, + so these seeds only have to be enough to reach the cluster once. + Defaults to none. auto_login: Credentials replayed on every connect. Defaults to `AutoLogin.disabled()`. reconnection: Reconnection policy. Defaults to `TcpReconnectionConfig()`. heartbeat_interval: Interval of heartbeats sent by the client. Defaults to 5 seconds. @@ -1985,8 +1993,9 @@ class TcpConfig: leaving it on. Raises: - ValueError: If `server_address` is not a valid `host:port` pair, if a - duration is negative, or if `heartbeat_interval` is zero. + ValueError: If `server_address` or one of `failover_addresses` is not a + valid `host:port` pair, if a duration is negative, or if + `heartbeat_interval` is zero. """ def __repr__(self) -> builtins.str: ... @@ -2016,18 +2025,21 @@ class TcpReconnectionConfig: Args: enabled: Whether to reconnect at all. Defaults to enabled. - max_retries: Passes over the known endpoints before giving up, or - `None` for unlimited. One pass tries the endpoint the client is - on, the address it was configured with, and every node the - roster named, so this counts passes rather than dials. Defaults + max_retries: Passes over the known endpoints after the first, or + `None` for unlimited; `0` still makes that first pass. One pass + tries the endpoint the client is on, the address it was + configured with, and every node the roster named, so this counts + passes rather than dials. Defaults to unlimited, which means a call awaited while the server is down never returns: `connect()`, `send_messages()` and `poll_messages()` all wait inside the retry loop. Set a finite number for request/reply style usage, so a call fails instead. interval: Delay between passes. Defaults to 1 second. The first pass runs at once when more than one endpoint is known. - reestablish_after: Cooldown before redialing the endpoint that was - just lost, owed to that endpoint alone. Defaults to 5 seconds. + reestablish_after: Cooldown before redialing the endpoint of the last + successful connection, measured from when it was established, so + a session that outlived the interval is redialed at once. Owed to + that endpoint alone. Defaults to 5 seconds. Raises: ValueError: If a duration is negative, if `max_retries` is outside the diff --git a/foreign/python/src/config.rs b/foreign/python/src/config.rs index 09a7dc90d4..a132f2878b 100644 --- a/foreign/python/src/config.rs +++ b/foreign/python/src/config.rs @@ -121,18 +121,21 @@ impl TcpReconnectionConfig { /// /// Args: /// enabled: Whether to reconnect at all. Defaults to enabled. - /// max_retries: Passes over the known endpoints before giving up, or - /// `None` for unlimited. One pass tries the endpoint the client is - /// on, the address it was configured with, and every node the - /// roster named, so this counts passes rather than dials. Defaults + /// max_retries: Passes over the known endpoints after the first, or + /// `None` for unlimited; `0` still makes that first pass. One pass + /// tries the endpoint the client is on, the address it was + /// configured with, and every node the roster named, so this counts + /// passes rather than dials. Defaults /// to unlimited, which means a call awaited while the server is /// down never returns: `connect()`, `send_messages()` and /// `poll_messages()` all wait inside the retry loop. Set a finite /// number for request/reply style usage, so a call fails instead. /// interval: Delay between passes. Defaults to 1 second. The first pass /// runs at once when more than one endpoint is known. - /// reestablish_after: Cooldown before redialing the endpoint that was - /// just lost, owed to that endpoint alone. Defaults to 5 seconds. + /// reestablish_after: Cooldown before redialing the endpoint of the last + /// successful connection, measured from when it was established, so + /// a session that outlived the interval is redialed at once. Owed to + /// that endpoint alone. Defaults to 5 seconds. /// /// Raises: /// ValueError: If a duration is negative, if `max_retries` is outside the @@ -241,6 +244,11 @@ impl TcpConfig { /// /// Args: /// server_address: `host:port` of the Iggy server. Defaults to `127.0.0.1:8090`. + /// failover_addresses: `host:port` of other nodes of the same cluster, dialed + /// in order when `server_address` cannot be reached. The roster the server + /// reports is remembered while the client is connected and dialed first, + /// so these seeds only have to be enough to reach the cluster once. + /// Defaults to none. /// auto_login: Credentials replayed on every connect. Defaults to `AutoLogin.disabled()`. /// reconnection: Reconnection policy. Defaults to `TcpReconnectionConfig()`. /// heartbeat_interval: Interval of heartbeats sent by the client. Defaults to 5 seconds. @@ -260,12 +268,14 @@ impl TcpConfig { /// leaving it on. /// /// Raises: - /// ValueError: If `server_address` is not a valid `host:port` pair, if a - /// duration is negative, or if `heartbeat_interval` is zero. + /// ValueError: If `server_address` or one of `failover_addresses` is not a + /// valid `host:port` pair, if a duration is negative, or if + /// `heartbeat_interval` is zero. #[new] #[pyo3(signature = ( *, server_address=None, + failover_addresses=None, auto_login=None, reconnection=None, heartbeat_interval=None, @@ -280,6 +290,8 @@ impl TcpConfig { #[gen_stub(override_type(type_repr = "builtins.str | None"))] server_address: Option< String, >, + #[gen_stub(override_type(type_repr = "typing.Sequence[builtins.str] | None", imports=("typing")))] + failover_addresses: Option>, #[gen_stub(override_type(type_repr = "AutoLogin | None"))] auto_login: Option, #[gen_stub(override_type(type_repr = "TcpReconnectionConfig | None"))] reconnection: Option< TcpReconnectionConfig, @@ -299,6 +311,9 @@ impl TcpConfig { if let Some(server_address) = server_address { builder = builder.with_server_address(server_address); } + if let Some(failover_addresses) = failover_addresses { + builder = builder.with_failover_addresses(failover_addresses); + } let mut inner = builder .build() .map_err(|e| PyValueError::new_err(e.to_string()))?; @@ -340,6 +355,12 @@ impl TcpConfig { self.inner.server_address.clone() } + #[gen_stub(override_return_type(type_repr = "builtins.list[builtins.str]"))] + #[getter] + fn failover_addresses(&self) -> Vec { + self.inner.failover_addresses.clone() + } + #[getter] fn auto_login(&self) -> AutoLogin { AutoLogin { @@ -392,8 +413,9 @@ impl TcpConfig { None => "None".to_owned(), }; format!( - "TcpConfig(server_address={:?}, auto_login={}, reconnection={}, heartbeat_interval={}, tls_enabled={}, tls_domain={:?}, tls_ca_file={tls_ca_file}, tls_validate_certificate={}, nodelay={})", + "TcpConfig(server_address={:?}, failover_addresses={:?}, auto_login={}, reconnection={}, heartbeat_interval={}, tls_enabled={}, tls_domain={:?}, tls_ca_file={tls_ca_file}, tls_validate_certificate={}, nodelay={})", self.inner.server_address, + self.inner.failover_addresses, self.auto_login().__repr__(), self.reconnection().__repr__(), duration_repr(self.inner.heartbeat_interval.get()), From 46c33fd169d8548b05c26029bad9d185972f040c Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Wed, 26 Aug 2026 13:47:28 +0200 Subject: [PATCH 12/16] fix CI --- core/common/src/types/args/mod.rs | 5 +++-- core/integration/tests/cli/general/test_help_command.rs | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/core/common/src/types/args/mod.rs b/core/common/src/types/args/mod.rs index ecd529096b..0f8ced668c 100644 --- a/core/common/src/types/args/mod.rs +++ b/core/common/src/types/args/mod.rs @@ -76,8 +76,9 @@ pub struct ArgsOptional { #[serde(skip_serializing_if = "Option::is_none")] pub tcp_server_address: Option, - /// The optional addresses of other nodes of the same cluster, dialed in - /// order when the server address cannot be reached + /// The optional addresses of other cluster nodes for the TCP transport + /// + /// Dialed in order when the server address cannot be reached. #[arg(long, value_delimiter = ',')] #[serde(skip_serializing_if = "Option::is_none")] pub tcp_failover_addresses: Option>, diff --git a/core/integration/tests/cli/general/test_help_command.rs b/core/integration/tests/cli/general/test_help_command.rs index a7ea9e2a0c..1642bb44fa 100644 --- a/core/integration/tests/cli/general/test_help_command.rs +++ b/core/integration/tests/cli/general/test_help_command.rs @@ -90,6 +90,11 @@ Options: {CLAP_INDENT} [default: 127.0.0.1:8090] + --tcp-failover-addresses + The optional addresses of other cluster nodes for the TCP transport +{CLAP_INDENT} + Dialed in order when the server address cannot be reached. + --tcp-reconnection-max-retries The optional number of max reconnect retries for the TCP transport {CLAP_INDENT} From f3ea3f774c12fd653e7ecd7f08710ba7c780761d Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Wed, 26 Aug 2026 15:24:40 +0200 Subject: [PATCH 13/16] update stale docs --- .../configuration/tcp_config/tcp_client_config.rs | 12 +++++++----- .../tcp_config/tcp_client_config_builder.rs | 4 ++-- core/sdk/src/clients/client_builder.rs | 4 ++-- foreign/python/apache_iggy.pyi | 7 ++++--- foreign/python/src/config.rs | 7 ++++--- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_config.rs b/core/common/src/types/configuration/tcp_config/tcp_client_config.rs index 288337af65..9ba32f6111 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_config.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_config.rs @@ -26,11 +26,13 @@ use std::str::FromStr; pub struct TcpClientConfig { /// The address of the Iggy server. pub server_address: String, - /// Addresses of other nodes of the same cluster, dialed in order when - /// `server_address` cannot be reached. The roster the server reports is - /// remembered while the client is connected and dialed first, so these - /// seeds only have to be enough to reach the cluster once -- at the very - /// first connect, when nothing has been learned yet. + /// Addresses of other nodes of the same cluster, dialed when + /// `server_address` cannot be reached. + /// + /// Part of every failover pass, ahead of the roster the client learned while + /// connected, since the caller vouched for these. Before the first connect -- + /// a fresh process included -- they are the only other addresses there are, + /// because reading the roster needs a connection. pub failover_addresses: Vec, /// Whether to use TLS when connecting to the server. pub tls_enabled: bool, diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs b/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs index 08e50a9b6e..06dd6ae5d4 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs @@ -45,8 +45,8 @@ impl TcpClientConfigBuilder { self } - /// Sets the addresses of other nodes of the same cluster, dialed in order - /// when `server_address` cannot be reached. + /// Sets the addresses of other nodes of the same cluster, dialed when `server_address` + /// cannot be reached, ahead of the roster the client learns while connected. pub fn with_failover_addresses(mut self, failover_addresses: Vec) -> Self { self.config.failover_addresses = failover_addresses; self diff --git a/core/sdk/src/clients/client_builder.rs b/core/sdk/src/clients/client_builder.rs index 5cfa317b48..6177d73ffa 100644 --- a/core/sdk/src/clients/client_builder.rs +++ b/core/sdk/src/clients/client_builder.rs @@ -160,8 +160,8 @@ impl TcpClientBuilder { self } - /// Sets the addresses of other nodes of the same cluster, dialed in order - /// when the server address cannot be reached. + /// Sets the addresses of other nodes of the same cluster, dialed when the server address + /// cannot be reached, ahead of the roster the client learns while connected. pub fn with_failover_addresses(mut self, failover_addresses: Vec) -> Self { self.config = self.config.with_failover_addresses(failover_addresses); self diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index cf3d847ce4..d44435930e 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi @@ -1970,9 +1970,10 @@ class TcpConfig: Args: server_address: `host:port` of the Iggy server. Defaults to `127.0.0.1:8090`. failover_addresses: `host:port` of other nodes of the same cluster, dialed - in order when `server_address` cannot be reached. The roster the server - reports is remembered while the client is connected and dialed first, - so these seeds only have to be enough to reach the cluster once. + when `server_address` cannot be reached. Part of every failover pass, + ahead of the roster the client learned while connected, and the only + other addresses it has before its first connect, a fresh process + included, since reading the roster needs a connection. Defaults to none. auto_login: Credentials replayed on every connect. Defaults to `AutoLogin.disabled()`. reconnection: Reconnection policy. Defaults to `TcpReconnectionConfig()`. diff --git a/foreign/python/src/config.rs b/foreign/python/src/config.rs index a132f2878b..05d2a7d413 100644 --- a/foreign/python/src/config.rs +++ b/foreign/python/src/config.rs @@ -245,9 +245,10 @@ impl TcpConfig { /// Args: /// server_address: `host:port` of the Iggy server. Defaults to `127.0.0.1:8090`. /// failover_addresses: `host:port` of other nodes of the same cluster, dialed - /// in order when `server_address` cannot be reached. The roster the server - /// reports is remembered while the client is connected and dialed first, - /// so these seeds only have to be enough to reach the cluster once. + /// when `server_address` cannot be reached. Part of every failover pass, + /// ahead of the roster the client learned while connected, and the only + /// other addresses it has before its first connect, a fresh process + /// included, since reading the roster needs a connection. /// Defaults to none. /// auto_login: Credentials replayed on every connect. Defaults to `AutoLogin.disabled()`. /// reconnection: Reconnection policy. Defaults to `TcpReconnectionConfig()`. From d2a777cfdc5c8447bd449abc83fea1ceddfac315 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Wed, 26 Aug 2026 15:45:29 +0200 Subject: [PATCH 14/16] remove failover_address --- core/common/src/types/args/mod.rs | 15 ---- .../auth_config/connection_string.rs | 29 ------- .../tcp_config/tcp_client_config.rs | 10 --- .../tcp_config/tcp_client_config_builder.rs | 38 --------- .../tcp_connection_string_options.rs | 18 ---- .../tests/cli/general/test_help_command.rs | 5 -- core/sdk/src/client_provider.rs | 1 - core/sdk/src/clients/client_builder.rs | 7 -- core/sdk/src/tcp/tcp_client.rs | 82 +++++++++---------- examples/rust/src/shared/args.rs | 5 -- foreign/python/apache_iggy.pyi | 14 +--- foreign/python/src/config.rs | 26 +----- 12 files changed, 46 insertions(+), 204 deletions(-) diff --git a/core/common/src/types/args/mod.rs b/core/common/src/types/args/mod.rs index 0f8ced668c..d44b223ced 100644 --- a/core/common/src/types/args/mod.rs +++ b/core/common/src/types/args/mod.rs @@ -76,13 +76,6 @@ pub struct ArgsOptional { #[serde(skip_serializing_if = "Option::is_none")] pub tcp_server_address: Option, - /// The optional addresses of other cluster nodes for the TCP transport - /// - /// Dialed in order when the server address cannot be reached. - #[arg(long, value_delimiter = ',')] - #[serde(skip_serializing_if = "Option::is_none")] - pub tcp_failover_addresses: Option>, - /// The optional number of max reconnect retries for the TCP transport /// /// [default: 10] @@ -251,10 +244,6 @@ pub struct Args { /// The optional client address for the TCP transport pub tcp_server_address: String, - /// The optional addresses of other nodes of the same cluster, dialed in - /// order when the server address cannot be reached - pub tcp_failover_addresses: Vec, - /// The optional number of maximum reconnect retries for the TCP transport pub tcp_reconnection_enabled: bool, @@ -391,7 +380,6 @@ impl Default for Args { username: DEFAULT_ROOT_USERNAME.to_string(), password: DEFAULT_ROOT_PASSWORD.to_string(), tcp_server_address: "127.0.0.1:8090".to_string(), - tcp_failover_addresses: Vec::new(), tcp_reconnection_enabled: true, tcp_reconnection_max_retries: Some(10), tcp_reconnection_interval: "1s".to_string(), @@ -458,9 +446,6 @@ impl From> for Args { if let Some(tcp_server_address) = optional_args.tcp_server_address { args.tcp_server_address = tcp_server_address; } - if let Some(tcp_failover_addresses) = optional_args.tcp_failover_addresses { - args.tcp_failover_addresses = tcp_failover_addresses; - } if let Some(tcp_reconnection_retries) = optional_args.tcp_reconnection_max_retries { args.tcp_reconnection_max_retries = Some(tcp_reconnection_retries); } diff --git a/core/common/src/types/configuration/auth_config/connection_string.rs b/core/common/src/types/configuration/auth_config/connection_string.rs index 96491693d1..82691812f0 100644 --- a/core/common/src/types/configuration/auth_config/connection_string.rs +++ b/core/common/src/types/configuration/auth_config/connection_string.rs @@ -159,7 +159,6 @@ impl ConnectionStringUtils { mod tests { use super::*; use crate::NonZeroIggyDuration; - use crate::TcpClientConfig; use crate::TcpConnectionStringOptions; use secrecy::ExposeSecret; @@ -321,32 +320,4 @@ mod tests { NonZeroIggyDuration::from_str("5s").unwrap() ); } - - #[test] - fn should_carry_failover_addresses_into_the_config() { - let value = format!( - "{DEFAULT_CONNECTION_STRING_PREFIX}user:secret@127.0.0.1:1234?failover_addresses=127.0.0.2:1234, 127.0.0.3:1234" - ); - let connection_string = - ConnectionString::::new(&value).unwrap(); - assert_eq!( - connection_string.options.failover_addresses(), - ["127.0.0.2:1234", "127.0.0.3:1234"] - ); - - let config = TcpClientConfig::from(connection_string); - assert_eq!(config.server_address, "127.0.0.1:1234"); - assert_eq!( - config.failover_addresses, - ["127.0.0.2:1234", "127.0.0.3:1234"] - ); - } - - #[test] - fn should_leave_the_failover_addresses_empty_when_the_option_is_absent() { - let value = format!("{DEFAULT_CONNECTION_STRING_PREFIX}user:secret@127.0.0.1:1234"); - let connection_string = - ConnectionString::::new(&value).unwrap(); - assert!(connection_string.options.failover_addresses().is_empty()); - } } diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_config.rs b/core/common/src/types/configuration/tcp_config/tcp_client_config.rs index 9ba32f6111..b60f4b3ad0 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_config.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_config.rs @@ -26,14 +26,6 @@ use std::str::FromStr; pub struct TcpClientConfig { /// The address of the Iggy server. pub server_address: String, - /// Addresses of other nodes of the same cluster, dialed when - /// `server_address` cannot be reached. - /// - /// Part of every failover pass, ahead of the roster the client learned while - /// connected, since the caller vouched for these. Before the first connect -- - /// a fresh process included -- they are the only other addresses there are, - /// because reading the roster needs a connection. - pub failover_addresses: Vec, /// Whether to use TLS when connecting to the server. pub tls_enabled: bool, /// The domain to use for TLS when connecting to the server. @@ -57,7 +49,6 @@ impl Default for TcpClientConfig { fn default() -> TcpClientConfig { TcpClientConfig { server_address: "127.0.0.1:8090".to_string(), - failover_addresses: Vec::new(), tls_enabled: false, tls_domain: "".to_string(), tls_ca_file: None, @@ -74,7 +65,6 @@ impl From> for TcpClientConfig { fn from(connection_string: ConnectionString) -> Self { TcpClientConfig { server_address: connection_string.server_address().into(), - failover_addresses: connection_string.options().failover_addresses().to_vec(), auto_login: connection_string.auto_login().to_owned(), tls_enabled: connection_string.options().tls_enabled(), tls_domain: connection_string.options().tls_domain().into(), diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs b/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs index 06dd6ae5d4..943b69f53e 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_config_builder.rs @@ -23,7 +23,6 @@ use crate::{ /// Builder for the TCP client configuration. /// Allows configuring the TCP client with custom settings or using defaults: /// - `server_address`: Default is "127.0.0.1:8090" -/// - `failover_addresses`: Default is empty. /// - `auto_login`: Default is AutoLogin::Disabled. /// - `reconnection`: Default is enabled unlimited retries and 1 second interval. /// - `tls_enabled`: Default is false. @@ -45,13 +44,6 @@ impl TcpClientConfigBuilder { self } - /// Sets the addresses of other nodes of the same cluster, dialed when `server_address` - /// cannot be reached, ahead of the roster the client learns while connected. - pub fn with_failover_addresses(mut self, failover_addresses: Vec) -> Self { - self.config.failover_addresses = failover_addresses; - self - } - /// Sets the auto sign in during connection. pub fn with_auto_sign_in(mut self, auto_sign_in: AutoLogin) -> Self { self.config.auto_login = auto_sign_in; @@ -116,10 +108,6 @@ impl TcpClientConfigBuilder { pub fn build(mut self) -> Result { self.config.server_address = self.config.server_address.trim().to_owned(); validate_server_address(&self.config.server_address)?; - for failover_address in &mut self.config.failover_addresses { - *failover_address = failover_address.trim().to_owned(); - validate_server_address(failover_address)?; - } Ok(self.config) } @@ -197,32 +185,6 @@ mod tests { )); } - #[test] - fn valid_failover_addresses_should_succeed() { - let config = builder_with_address("127.0.0.1:8090") - .with_failover_addresses(vec![ - " 127.0.0.1:8091 ".to_string(), - "iggy-server-3:8090".to_string(), - ]) - .build() - .expect("build the configuration"); - - assert_eq!( - config.failover_addresses, - vec!["127.0.0.1:8091", "iggy-server-3:8090"] - ); - } - - #[test] - fn malformed_failover_address_should_fail() { - let builder = builder_with_address("127.0.0.1:8090") - .with_failover_addresses(vec!["127.0.0.1".to_string()]); - assert!(matches!( - builder.build(), - Err(IggyError::InvalidIpAddress(_, _)) - )); - } - #[test] fn docker_compose_service_name_should_succeed() { let builder = builder_with_address("iggy-server:8090"); diff --git a/core/common/src/types/configuration/tcp_config/tcp_connection_string_options.rs b/core/common/src/types/configuration/tcp_config/tcp_connection_string_options.rs index ebd5042857..1c957f1c37 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_connection_string_options.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_connection_string_options.rs @@ -23,7 +23,6 @@ use std::str::FromStr; #[derive(Debug)] pub struct TcpConnectionStringOptions { - failover_addresses: Vec, tls_enabled: bool, tls_domain: String, tls_ca_file: Option, @@ -33,10 +32,6 @@ pub struct TcpConnectionStringOptions { } impl TcpConnectionStringOptions { - pub fn failover_addresses(&self) -> &[String] { - &self.failover_addresses - } - pub fn tls_enabled(&self) -> bool { self.tls_enabled } @@ -69,7 +64,6 @@ impl ConnectionStringOptions for TcpConnectionStringOptions { fn parse_options(options: &str) -> Result { let options = options.split('&').collect::>(); - let mut failover_addresses = Vec::new(); let mut tls_enabled = false; let mut tls_domain = "".to_string(); let mut tls_ca_file = None; @@ -85,14 +79,6 @@ impl ConnectionStringOptions for TcpConnectionStringOptions { return Err(IggyError::InvalidConnectionString); } match option_parts[0] { - "failover_addresses" => { - failover_addresses = option_parts[1] - .split(',') - .map(str::trim) - .filter(|address| !address.is_empty()) - .map(str::to_string) - .collect(); - } "tls" => { tls_enabled = option_parts[1] == "true"; } @@ -143,7 +129,6 @@ impl ConnectionStringOptions for TcpConnectionStringOptions { .map_err(|_| IggyError::InvalidConnectionString)?; let connection_string_options = TcpConnectionStringOptions::new( - failover_addresses, tls_enabled, tls_domain, tls_ca_file, @@ -158,7 +143,6 @@ impl ConnectionStringOptions for TcpConnectionStringOptions { impl TcpConnectionStringOptions { pub fn new( - failover_addresses: Vec, tls_enabled: bool, tls_domain: String, tls_ca_file: Option, @@ -167,7 +151,6 @@ impl TcpConnectionStringOptions { nodelay: bool, ) -> Self { Self { - failover_addresses, tls_enabled, tls_domain, tls_ca_file, @@ -181,7 +164,6 @@ impl TcpConnectionStringOptions { impl Default for TcpConnectionStringOptions { fn default() -> Self { TcpConnectionStringOptions { - failover_addresses: Vec::new(), tls_enabled: false, tls_domain: "".to_string(), tls_ca_file: None, diff --git a/core/integration/tests/cli/general/test_help_command.rs b/core/integration/tests/cli/general/test_help_command.rs index 1642bb44fa..a7ea9e2a0c 100644 --- a/core/integration/tests/cli/general/test_help_command.rs +++ b/core/integration/tests/cli/general/test_help_command.rs @@ -90,11 +90,6 @@ Options: {CLAP_INDENT} [default: 127.0.0.1:8090] - --tcp-failover-addresses - The optional addresses of other cluster nodes for the TCP transport -{CLAP_INDENT} - Dialed in order when the server address cannot be reached. - --tcp-reconnection-max-retries The optional number of max reconnect retries for the TCP transport {CLAP_INDENT} diff --git a/core/sdk/src/client_provider.rs b/core/sdk/src/client_provider.rs index a107c881de..6b431562a3 100644 --- a/core/sdk/src/client_provider.rs +++ b/core/sdk/src/client_provider.rs @@ -135,7 +135,6 @@ impl ClientProviderConfig { TransportProtocol::Tcp => { config.tcp = Some(Arc::new(TcpClientConfig { server_address: args.tcp_server_address, - failover_addresses: args.tcp_failover_addresses, tls_enabled: args.tcp_tls_enabled, tls_domain: args.tcp_tls_domain, tls_ca_file: args.tcp_tls_ca_file, diff --git a/core/sdk/src/clients/client_builder.rs b/core/sdk/src/clients/client_builder.rs index 6177d73ffa..20a960c6d3 100644 --- a/core/sdk/src/clients/client_builder.rs +++ b/core/sdk/src/clients/client_builder.rs @@ -160,13 +160,6 @@ impl TcpClientBuilder { self } - /// Sets the addresses of other nodes of the same cluster, dialed when the server address - /// cannot be reached, ahead of the roster the client learns while connected. - pub fn with_failover_addresses(mut self, failover_addresses: Vec) -> Self { - self.config = self.config.with_failover_addresses(failover_addresses); - self - } - /// Sets the auto sign in during connection. pub fn with_auto_sign_in(mut self, auto_sign_in: AutoLogin) -> Self { self.config = self.config.with_auto_sign_in(auto_sign_in); diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index 2b54f8aaa9..5648fffaf0 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -832,17 +832,16 @@ impl TcpClient { } /// Endpoints to dial for one connect, likeliest first: where the client - /// currently is, the addresses it was configured with, then the roster it + /// currently is, the address it was configured with, then the roster it /// learned while connected. /// - /// Configured before learned, as in the other SDKs: those are the endpoints + /// Configured before learned, as in the other SDKs: that is the endpoint /// the caller vouched for, while a roster read from a cluster that has since /// changed shape may name nodes that are gone. async fn dial_candidates(&self) -> Vec { let mut candidates = vec![self.current_server_address.lock().await.clone()]; let roster = self.roster_endpoints.lock().await.clone(); - let configured = std::iter::once(&self.config.server_address) - .chain(self.config.failover_addresses.iter()); + let configured = std::iter::once(&self.config.server_address); for endpoint in configured.chain(roster.iter()) { let mut known = false; for candidate in &candidates { @@ -1318,15 +1317,21 @@ mod tests { const SESSION_USER_ID: u32 = 7; - fn client_with(server_address: &str, failover_addresses: Vec) -> TcpClient { + fn client_with(server_address: &str) -> TcpClient { TcpClient::create(Arc::new(TcpClientConfig { server_address: server_address.to_string(), - failover_addresses, ..TcpClientConfig::default() })) .expect("create the client") } + /// A client whose roster names `endpoints`, as a leader check leaves it. + async fn client_with_roster(server_address: &str, endpoints: Vec) -> TcpClient { + let client = client_with(server_address); + *client.roster_endpoints.lock().await = endpoints; + client + } + /// A listener nothing ever accepts from: the kernel completes the TCP /// handshake out of its backlog, which is all a dial needs to succeed. async fn live_endpoint() -> (TcpListener, String) { @@ -1356,14 +1361,13 @@ mod tests { address } - // With reconnection off there are no retries, but the failover endpoints - // were configured to be tried and each still gets its one turn. + // With reconnection off there are no retries, but the endpoints the roster + // named are still there to be tried and each gets its one turn. #[tokio::test] - async fn a_client_with_reconnection_disabled_still_sweeps_its_failover_endpoints() { + async fn a_client_with_reconnection_disabled_still_sweeps_the_endpoints_it_knows() { let (_listener, survivor) = live_endpoint().await; let client = TcpClient::create(Arc::new(TcpClientConfig { server_address: dead_endpoint().await, - failover_addresses: vec![survivor.clone()], reconnection: TcpClientReconnectionConfig { enabled: false, ..TcpClientReconnectionConfig::default() @@ -1371,6 +1375,7 @@ mod tests { ..TcpClientConfig::default() })) .expect("create the client"); + *client.roster_endpoints.lock().await = vec![survivor.clone()]; TcpClient::connect(&client).await.expect("connect"); assert_eq!(*client.current_server_address.lock().await, survivor); @@ -1383,7 +1388,6 @@ mod tests { async fn a_connect_that_exhausts_every_endpoint_leaves_the_client_disconnected() { let client = TcpClient::create(Arc::new(TcpClientConfig { server_address: dead_endpoint().await, - failover_addresses: vec![dead_endpoint().await], reconnection: TcpClientReconnectionConfig { enabled: false, ..TcpClientReconnectionConfig::default() @@ -1391,6 +1395,7 @@ mod tests { ..TcpClientConfig::default() })) .expect("create the client"); + *client.roster_endpoints.lock().await = vec![dead_endpoint().await]; assert!(matches!( TcpClient::connect(&client).await, @@ -1417,7 +1422,6 @@ mod tests { let plaintext = endpoint_that_hangs_up().await; let client = TcpClient::create(Arc::new(TcpClientConfig { server_address: configured.clone(), - failover_addresses: vec![plaintext], tls_enabled: true, tls_validate_certificate: false, reconnection: TcpClientReconnectionConfig { @@ -1427,6 +1431,7 @@ mod tests { ..TcpClientConfig::default() })) .expect("create the client"); + *client.roster_endpoints.lock().await = vec![plaintext]; assert!(matches!( TcpClient::connect(&client).await, @@ -1443,7 +1448,6 @@ mod tests { let (_listener, silent) = live_endpoint().await; let client = TcpClient::create(Arc::new(TcpClientConfig { server_address: silent, - failover_addresses: vec![dead_endpoint().await], tls_enabled: true, tls_validate_certificate: false, reconnection: TcpClientReconnectionConfig { @@ -1453,6 +1457,7 @@ mod tests { ..TcpClientConfig::default() })) .expect("create the client"); + *client.roster_endpoints.lock().await = vec![dead_endpoint().await]; let sweep = tokio::time::timeout( FAILOVER_DIAL_TIMEOUT * 3, @@ -1588,7 +1593,6 @@ mod tests { let (_listener, survivor) = live_endpoint().await; let client = TcpClient::create(Arc::new(TcpClientConfig { server_address: dead_endpoint().await, - failover_addresses: vec![survivor.clone()], reconnection: TcpClientReconnectionConfig { reestablish_after: IggyDuration::from_str("10s").expect("duration"), ..TcpClientReconnectionConfig::default() @@ -1596,6 +1600,7 @@ mod tests { ..TcpClientConfig::default() })) .expect("create the client"); + *client.roster_endpoints.lock().await = vec![survivor.clone()]; client .connected_at .lock() @@ -1620,7 +1625,6 @@ mod tests { let (_listener, current) = live_endpoint().await; let client = TcpClient::create(Arc::new(TcpClientConfig { server_address: current.clone(), - failover_addresses: vec![dead_endpoint().await], reconnection: TcpClientReconnectionConfig { reestablish_after: IggyDuration::from_str("1s").expect("duration"), ..TcpClientReconnectionConfig::default() @@ -1628,6 +1632,7 @@ mod tests { ..TcpClientConfig::default() })) .expect("create the client"); + *client.roster_endpoints.lock().await = vec![dead_endpoint().await]; client .connected_at .lock() @@ -1646,37 +1651,32 @@ mod tests { #[tokio::test] async fn dial_candidates_lead_with_the_current_endpoint_and_name_each_other_one_once() { - let client = client_with( + let client = client_with_roster( "127.0.0.1:8090", - vec!["127.0.0.1:8092".to_string(), "localhost:8090".to_string()], - ); - *client.roster_endpoints.lock().await = vec![ - "127.0.0.1:8090".to_string(), - "127.0.0.1:8091".to_string(), - "127.0.0.1:8092".to_string(), - ]; - - // The current endpoint leads, the configured ones follow, and the - // roster comes last -- the same order as the other SDKs. Neither the - // roster's copy of an endpoint already named nor a seed that only - // spells one differently earns a second dial. - assert_eq!( - client.dial_candidates().await, vec![ "127.0.0.1:8090".to_string(), - "127.0.0.1:8092".to_string(), + "localhost:8090".to_string(), "127.0.0.1:8091".to_string(), - ] + ], + ) + .await; + + // The current endpoint leads and the roster follows, the same order as + // the other SDKs. An endpoint the roster names again earns no second + // dial, whether it is spelled the same way or not. + assert_eq!( + client.dial_candidates().await, + vec!["127.0.0.1:8090".to_string(), "127.0.0.1:8091".to_string()] ); } #[tokio::test] - async fn a_client_that_learned_no_roster_still_dials_its_configured_seeds() { - let client = client_with("127.0.0.1:8090", vec!["127.0.0.1:8091".to_string()]); + async fn a_client_that_learned_no_roster_dials_only_its_configured_endpoint() { + let client = client_with("127.0.0.1:8090"); assert_eq!( client.dial_candidates().await, - vec!["127.0.0.1:8090".to_string(), "127.0.0.1:8091".to_string()] + vec!["127.0.0.1:8090".to_string()] ); } @@ -1684,7 +1684,7 @@ mod tests { // the caller signs in again. Only involuntary drops keep the sign-in. #[tokio::test] async fn an_explicit_disconnect_forgets_the_remembered_sign_in() { - let client = client_with("127.0.0.1:8090", Vec::new()); + let client = client_with("127.0.0.1:8090"); client .remember_session_credentials( Credentials::UsernamePassword("iggy".to_string(), "iggy".into()), @@ -1701,7 +1701,7 @@ mod tests { #[tokio::test] async fn a_transport_drop_keeps_the_remembered_sign_in() { - let client = client_with("127.0.0.1:8090", Vec::new()); + let client = client_with("127.0.0.1:8090"); client .remember_session_credentials( Credentials::UsernamePassword("iggy".to_string(), "iggy".into()), @@ -1721,7 +1721,7 @@ mod tests { #[tokio::test] async fn a_sign_in_makes_a_client_without_auto_login_reconnectable() { - let client = client_with("127.0.0.1:8090", Vec::new()); + let client = client_with("127.0.0.1:8090"); assert!(client.sign_in_credentials().await.is_none()); client @@ -1743,7 +1743,7 @@ mod tests { // unrelated request with `InvalidCredentials`. #[tokio::test] async fn a_password_change_for_the_signed_in_user_updates_the_remembered_sign_in() { - let client = client_with("127.0.0.1:8090", Vec::new()); + let client = client_with("127.0.0.1:8090"); client .remember_session_credentials( Credentials::UsernamePassword("iggy".to_string(), "old".into()), @@ -1777,7 +1777,7 @@ mod tests { // changes say nothing about the credentials this client reconnects with. #[tokio::test] async fn a_password_change_for_another_user_leaves_the_remembered_sign_in_alone() { - let client = client_with("127.0.0.1:8090", Vec::new()); + let client = client_with("127.0.0.1:8090"); client .remember_session_credentials( Credentials::UsernamePassword("iggy".to_string(), "old".into()), @@ -1802,7 +1802,7 @@ mod tests { // A personal access token is not derived from any password. #[tokio::test] async fn a_password_change_leaves_a_remembered_personal_access_token_alone() { - let client = client_with("127.0.0.1:8090", Vec::new()); + let client = client_with("127.0.0.1:8090"); client .remember_session_credentials( Credentials::PersonalAccessToken("token".into()), diff --git a/examples/rust/src/shared/args.rs b/examples/rust/src/shared/args.rs index 6d8ba09cce..ef43f5fb40 100644 --- a/examples/rust/src/shared/args.rs +++ b/examples/rust/src/shared/args.rs @@ -98,9 +98,6 @@ pub struct Args { #[arg(long, default_value = "127.0.0.1:8090")] pub tcp_server_address: String, - #[arg(long, value_delimiter = ',')] - pub tcp_failover_addresses: Vec, - #[arg(long, default_value = "false")] pub tcp_tls_enabled: bool, @@ -234,7 +231,6 @@ impl Default for Args { tcp_reconnection_reestablish_after: "5s".to_string(), tcp_heartbeat_interval: "5s".to_string(), tcp_server_address: "127.0.0.1:8090".to_string(), - tcp_failover_addresses: Vec::new(), tcp_tls_enabled: false, tcp_tls_domain: "localhost".to_string(), tcp_tls_ca_file: "".to_string(), @@ -335,7 +331,6 @@ impl Args { username: self.username.clone(), password: self.password.clone(), tcp_server_address: self.tcp_server_address.clone(), - tcp_failover_addresses: self.tcp_failover_addresses.clone(), tcp_reconnection_enabled: self.tcp_reconnection_enabled, tcp_reconnection_max_retries: self.tcp_reconnection_max_retries, tcp_reconnection_interval: self.tcp_reconnection_interval.clone(), diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index d44435930e..9bddb81f17 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi @@ -1933,8 +1933,6 @@ class TcpConfig: @property def server_address(self) -> builtins.str: ... @property - def failover_addresses(self) -> builtins.list[builtins.str]: ... - @property def auto_login(self) -> AutoLogin: ... @property def reconnection(self) -> TcpReconnectionConfig: ... @@ -1954,7 +1952,6 @@ class TcpConfig: cls, *, server_address: builtins.str | None = None, - failover_addresses: typing.Sequence[builtins.str] | None = None, auto_login: AutoLogin | None = None, reconnection: TcpReconnectionConfig | None = None, heartbeat_interval: datetime.timedelta | None = None, @@ -1969,12 +1966,6 @@ class TcpConfig: Args: server_address: `host:port` of the Iggy server. Defaults to `127.0.0.1:8090`. - failover_addresses: `host:port` of other nodes of the same cluster, dialed - when `server_address` cannot be reached. Part of every failover pass, - ahead of the roster the client learned while connected, and the only - other addresses it has before its first connect, a fresh process - included, since reading the roster needs a connection. - Defaults to none. auto_login: Credentials replayed on every connect. Defaults to `AutoLogin.disabled()`. reconnection: Reconnection policy. Defaults to `TcpReconnectionConfig()`. heartbeat_interval: Interval of heartbeats sent by the client. Defaults to 5 seconds. @@ -1994,9 +1985,8 @@ class TcpConfig: leaving it on. Raises: - ValueError: If `server_address` or one of `failover_addresses` is not a - valid `host:port` pair, if a duration is negative, or if - `heartbeat_interval` is zero. + ValueError: If `server_address` is not a valid `host:port` pair, if a + duration is negative, or if `heartbeat_interval` is zero. """ def __repr__(self) -> builtins.str: ... diff --git a/foreign/python/src/config.rs b/foreign/python/src/config.rs index 05d2a7d413..ed6e2a14f0 100644 --- a/foreign/python/src/config.rs +++ b/foreign/python/src/config.rs @@ -244,12 +244,6 @@ impl TcpConfig { /// /// Args: /// server_address: `host:port` of the Iggy server. Defaults to `127.0.0.1:8090`. - /// failover_addresses: `host:port` of other nodes of the same cluster, dialed - /// when `server_address` cannot be reached. Part of every failover pass, - /// ahead of the roster the client learned while connected, and the only - /// other addresses it has before its first connect, a fresh process - /// included, since reading the roster needs a connection. - /// Defaults to none. /// auto_login: Credentials replayed on every connect. Defaults to `AutoLogin.disabled()`. /// reconnection: Reconnection policy. Defaults to `TcpReconnectionConfig()`. /// heartbeat_interval: Interval of heartbeats sent by the client. Defaults to 5 seconds. @@ -269,14 +263,12 @@ impl TcpConfig { /// leaving it on. /// /// Raises: - /// ValueError: If `server_address` or one of `failover_addresses` is not a - /// valid `host:port` pair, if a duration is negative, or if - /// `heartbeat_interval` is zero. + /// ValueError: If `server_address` is not a valid `host:port` pair, if a + /// duration is negative, or if `heartbeat_interval` is zero. #[new] #[pyo3(signature = ( *, server_address=None, - failover_addresses=None, auto_login=None, reconnection=None, heartbeat_interval=None, @@ -291,8 +283,6 @@ impl TcpConfig { #[gen_stub(override_type(type_repr = "builtins.str | None"))] server_address: Option< String, >, - #[gen_stub(override_type(type_repr = "typing.Sequence[builtins.str] | None", imports=("typing")))] - failover_addresses: Option>, #[gen_stub(override_type(type_repr = "AutoLogin | None"))] auto_login: Option, #[gen_stub(override_type(type_repr = "TcpReconnectionConfig | None"))] reconnection: Option< TcpReconnectionConfig, @@ -312,9 +302,6 @@ impl TcpConfig { if let Some(server_address) = server_address { builder = builder.with_server_address(server_address); } - if let Some(failover_addresses) = failover_addresses { - builder = builder.with_failover_addresses(failover_addresses); - } let mut inner = builder .build() .map_err(|e| PyValueError::new_err(e.to_string()))?; @@ -356,12 +343,6 @@ impl TcpConfig { self.inner.server_address.clone() } - #[gen_stub(override_return_type(type_repr = "builtins.list[builtins.str]"))] - #[getter] - fn failover_addresses(&self) -> Vec { - self.inner.failover_addresses.clone() - } - #[getter] fn auto_login(&self) -> AutoLogin { AutoLogin { @@ -414,9 +395,8 @@ impl TcpConfig { None => "None".to_owned(), }; format!( - "TcpConfig(server_address={:?}, failover_addresses={:?}, auto_login={}, reconnection={}, heartbeat_interval={}, tls_enabled={}, tls_domain={:?}, tls_ca_file={tls_ca_file}, tls_validate_certificate={}, nodelay={})", + "TcpConfig(server_address={:?}, auto_login={}, reconnection={}, heartbeat_interval={}, tls_enabled={}, tls_domain={:?}, tls_ca_file={tls_ca_file}, tls_validate_certificate={}, nodelay={})", self.inner.server_address, - self.inner.failover_addresses, self.auto_login().__repr__(), self.reconnection().__repr__(), duration_repr(self.inner.heartbeat_interval.get()), From f90f9dfefeed0ce6349d220ea9d64bcbfcae74ff Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Wed, 26 Aug 2026 22:20:34 +0200 Subject: [PATCH 15/16] address review comments --- core/sdk/src/leader_aware.rs | 92 ++- core/sdk/src/tcp/tcp_client.rs | 581 ++++++++++++------ .../Implementations/TcpMessageStream.cs | 28 +- 3 files changed, 481 insertions(+), 220 deletions(-) diff --git a/core/sdk/src/leader_aware.rs b/core/sdk/src/leader_aware.rs index 66cdad35f3..b5c038f349 100644 --- a/core/sdk/src/leader_aware.rs +++ b/core/sdk/src/leader_aware.rs @@ -132,11 +132,32 @@ pub async fn check_and_redirect_to_leader( } } +/// Every endpoint the roster names for `transport`, empty when the read did +/// not answer. +/// +/// No leader verdict and no waiting for an election: the caller is not moving +/// anywhere, it only wants somewhere to dial once the node it is on dies. +pub(crate) async fn read_transport_endpoints( + client: &C, + transport: TransportProtocol, +) -> Vec { + match client.get_cluster_metadata().await { + Ok(metadata) => transport_endpoints(&metadata, transport), + Err(error) => { + debug!("Failed to read the cluster roster: {error}"); + Vec::new() + } + } +} + /// How long to wait for a transiently leaderless cluster to elect before /// proceeding on the current node anyway. const LEADERLESS_WAIT_BUDGET: std::time::Duration = std::time::Duration::from_secs(5); const LEADERLESS_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(250); +/// Bound on one name lookup made to compare two addresses. +const RESOLVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); + /// One leader-check verdict from a cluster-metadata snapshot. enum Outcome { /// A healthy leader exists elsewhere; reconnect to it. @@ -156,11 +177,23 @@ fn transport_endpoints(metadata: &ClusterMetadata, transport: TransportProtocol) .iter() .filter_map(|node| { let port = transport_port(node, transport); - (port != 0).then(|| format!("{}:{port}", node.ip)) + (port != 0).then(|| node_address(node, port)) }) .collect() } +/// One node's `host:port`, bracketing a literal IPv6 address. Appending a +/// port to a bare `::1` yields a spelling no dial can parse, so an IPv6 +/// cluster would hand out a roster of undialable entries that still count as +/// endpoints to fail over to. +fn node_address(node: &ClusterNode, port: u16) -> String { + if node.ip.contains(':') && !node.ip.starts_with('[') { + format!("[{}]:{port}", node.ip) + } else { + format!("{}:{port}", node.ip) + } +} + fn transport_port(node: &ClusterNode, transport: TransportProtocol) -> u16 { match transport { TransportProtocol::Tcp => node.endpoints.tcp, @@ -193,7 +226,7 @@ async fn process_cluster_metadata( match leader { Some(leader_node) => { let leader_port = transport_port(leader_node, transport); - let leader_address = format!("{}:{}", leader_node.ip, leader_port); + let leader_address = node_address(leader_node, leader_port); info!( "Found leader node: {} at {} (using {} transport)", @@ -272,10 +305,17 @@ where } /// Every socket address a host:port spelling resolves to, `None` when the -/// resolver does not know the name (which then compares unequal, at worst -/// costing one extra dial). +/// resolver does not know the name or does not answer in time (which then +/// compares unequal, at worst costing one extra dial). async fn resolve_all(addr: String) -> Option> { - let resolved: Vec = tokio::net::lookup_host(addr).await.ok()?.collect(); + // A resolver that never answers must not own the request budget: this + // comparison runs on the connect and redirect paths, the redirect one + // inside the caller's request deadline, and `lookup_host` has no deadline + // of its own. + let lookup = tokio::time::timeout(RESOLVE_TIMEOUT, tokio::net::lookup_host(addr)) + .await + .ok()?; + let resolved: Vec = lookup.ok()?.collect(); (!resolved.is_empty()).then_some(resolved) } @@ -385,6 +425,48 @@ mod tests { assert!(transport_endpoints(&metadata, TransportProtocol::Quic).is_empty()); } + // A port appended to a bare IPv6 address parses as neither, so the roster + // of an IPv6 cluster would name endpoints no dial can use while still + // counting as somewhere to fail over to. + #[test] + fn an_ipv6_node_is_named_as_a_bracketed_address() { + let metadata = ClusterMetadata { + name: "iggy".to_string(), + nodes: vec![ + node("iggy-1", "::1", 8090, ClusterNodeRole::Leader), + node("iggy-2", "[fd00::2]", 8090, ClusterNodeRole::Follower), + ], + }; + + let endpoints = transport_endpoints(&metadata, TransportProtocol::Tcp); + assert_eq!(endpoints, vec!["[::1]:8090", "[fd00::2]:8090"]); + for endpoint in endpoints { + assert!( + SocketAddr::from_str(&endpoint).is_ok(), + "the roster named an endpoint no dial can parse: {endpoint}" + ); + } + } + + // The address a redirect hands to the next dial comes from the same + // roster entry, so it has to be spelled the same way. + #[tokio::test] + async fn a_redirect_to_an_ipv6_leader_names_a_dialable_address() { + let metadata = ClusterMetadata { + name: "iggy".to_string(), + nodes: vec![ + node("iggy-1", "fd00::1", 8090, ClusterNodeRole::Leader), + node("iggy-2", "fd00::2", 8090, ClusterNodeRole::Follower), + ], + }; + + match process_cluster_metadata(&metadata, "[fd00::2]:8090", TransportProtocol::Tcp).await { + Outcome::Redirect(leader) => assert_eq!(leader, "[fd00::1]:8090"), + Outcome::LeaderIsCurrent => panic!("the follower was taken for the leader"), + Outcome::NoLeader => panic!("the roster named a healthy leader"), + } + } + #[tokio::test] async fn test_is_same_address() { assert!(is_same_address("127.0.0.1:8090", "127.0.0.1:8090").await); diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index 5648fffaf0..a271cef8c3 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -16,8 +16,8 @@ // under the License. use crate::leader_aware::{ - LeaderRedirectionState, check_and_redirect_to_leader, is_same_address, - is_unauthenticated_metadata_probe, + LeaderRedirectionState, check_and_redirect_to_leader, is_same_spelling, + is_unauthenticated_metadata_probe, read_transport_endpoints, }; use crate::prelude::Client; use crate::prelude::TcpClientConfig; @@ -46,6 +46,7 @@ use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; use std::sync::Mutex as StdMutex; +use std::sync::atomic::{AtomicBool, Ordering}; #[cfg(test)] use tokio::net::TcpListener; use tokio::net::TcpStream; @@ -76,6 +77,11 @@ const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_ /// overall. const TRANSIENT_FAILOVER_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2); +/// Bound on the roster read that follows a sign-in the caller ran itself. The +/// read is a convenience for a failover that may never happen, so a cluster +/// that answers it slowly must not hold up the sign-in. +const ROSTER_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + /// Bound on one dial while the client has other endpoints to try. A host /// that drops the SYN -- powered off, or partitioned away -- takes the OS /// connect timeout to fail, which is minutes, and every other endpoint waits @@ -100,6 +106,9 @@ pub struct TcpClient { /// unreachable exactly when it is needed, so the client has to have /// remembered it while the connection was still healthy. roster_endpoints: Mutex>, + /// Set once a sign-in on this client has gone looking for the roster, so + /// that read happens once (see [`TcpClient::learn_roster_once`]). + roster_learned: AtomicBool, /// Credentials a sign-in on this client succeeded with, so a reconnect -- /// onto this node or, after a failover, another one -- can re-establish /// the session instead of surfacing `Unauthenticated`. Cleared on logout. @@ -134,6 +143,15 @@ struct EstablishedConnection { remote_address: SocketAddr, } +/// A sign-in that did not complete, and whether the connection it ran on went +/// with it. A connection that is gone leaves the endpoints the sweep has not +/// reached yet worth dialing; one that stands means only the session is +/// missing, which no other endpoint would answer differently. +struct SignInFailure { + error: IggyError, + connection_lost: bool, +} + impl Default for TcpClient { fn default() -> Self { TcpClient::create(Arc::new(TcpClientConfig::default())).unwrap() @@ -282,7 +300,9 @@ impl BinaryTransport for TcpClient { /// reached the log may be re-sent. /// /// - the errors raised before the frame was written, and the server's own -/// refusals, which precede execution; +/// refusals, which precede execution. A `StaleClient` eviction is neither: +/// it arrives out of band and is consumed in place of the pending reply, so +/// the request it interrupted may already have committed; /// - operations that never enter the log: a non-replicated read, and a logout, /// which ends whatever session the connection carried -- the reconnect /// brought a new one, and refusing the replay would strand @@ -298,7 +318,6 @@ fn replay_is_safe(code: u32, error: &IggyError) -> bool { IggyError::NotConnected | IggyError::CannotEstablishConnection | IggyError::Unauthenticated - | IggyError::StaleClient ) || matches!( operation_for_code(code), @@ -306,28 +325,6 @@ fn replay_is_safe(code: u32, error: &IggyError) -> bool { ) } -/// Why a TLS handshake failed, as far as retrying is concerned. -/// -/// A certificate this client will never accept -- the wrong CA, a name it does -/// not cover, a peer that answers a ClientHello with something else -- says the -/// same thing on every attempt. Reported as a configuration fault it ends the -/// connect after one sweep; reported as a lost connection it would be redialed -/// every interval forever under `max_retries = None`, which is how a wrong CA -/// looks like a flaky network. -fn classify_handshake_failure(error: &std::io::Error) -> IggyError { - match error - .get_ref() - .and_then(|inner| inner.downcast_ref::()) - { - Some( - rustls::Error::InvalidCertificate(_) - | rustls::Error::NoCertificatesPresented - | rustls::Error::InvalidMessage(_), - ) => IggyError::InvalidTlsCertificate, - _ => IggyError::CannotEstablishConnection, - } -} - impl iggy_common::VsrSessionSealed for TcpClient {} #[async_trait::async_trait] @@ -365,6 +362,7 @@ impl iggy_common::VsrSessionControl for TcpClient { credentials, user_id, }); + self.learn_roster_once().await; } async fn forget_session_credentials(&self) { @@ -495,6 +493,7 @@ impl TcpClient { leader_redirection_state: Mutex::new(LeaderRedirectionState::new()), current_server_address: Mutex::new(server_address), roster_endpoints: Mutex::new(Vec::new()), + roster_learned: AtomicBool::new(false), configured_password: Mutex::new(None), session_credentials: Mutex::new(None), consensus_session: Arc::new(StdMutex::new(ConsensusSession::new())), @@ -505,26 +504,33 @@ impl TcpClient { async fn connect(&self) -> Result<(), IggyError> { loop { - match self.get_state().await { - ClientState::Shutdown => { - trace!("Cannot connect. Client is shutdown."); - return Err(IggyError::ClientShutdown); - } - ClientState::Connected - | ClientState::Authenticating - | ClientState::Authenticated => { - let client_address = self.get_client_address_value().await; - trace!("Client: {client_address} is already connected."); - return Ok(()); - } - ClientState::Connecting => { - trace!("Client is already connecting."); - return Ok(()); + // Read and claimed under one lock acquisition. Apart, two callers + // both find `Disconnected` and both sweep: the loser's + // `reset_vsr_session` re-mints the client id under the identity the + // winner is binding, and its `replace` below drops the live + // authenticated stream. + { + let mut state = self.state.lock().await; + match *state { + ClientState::Shutdown => { + trace!("Cannot connect. Client is shutdown."); + return Err(IggyError::ClientShutdown); + } + ClientState::Connected + | ClientState::Authenticating + | ClientState::Authenticated => { + let client_address = self.get_client_address_value().await; + trace!("Client: {client_address} is already connected."); + return Ok(()); + } + ClientState::Connecting => { + trace!("Client is already connecting."); + return Ok(()); + } + _ => *state = ClientState::Connecting, } - _ => {} } - self.set_state(ClientState::Connecting).await; let mut candidates = self.dial_candidates().await; // `reestablish_after` paces reconnects to the endpoint this client // was last on, and to that one only: the other endpoints owe it no @@ -538,17 +544,25 @@ impl TcpClient { candidates.rotate_left(1); } + let skip_auto_login = { + let mut guard = self.skip_auto_login_once.lock().await; + std::mem::take(&mut *guard) + }; + let mut retry_count = 0; - let connection_stream: ConnectionStreamKind; - let remote_address; - let client_address; let mut candidate = 0; // A fault no retry can fix, remembered rather than returned at - // once: it belongs to the endpoint that raised it (a certificate - // that names another host, a domain that will not parse), and the - // endpoints behind that one may be perfectly usable. + // once: it belongs to the endpoint that raised it (an unreadable CA + // file, a domain that will not parse), and the endpoints behind + // that one may be perfectly usable. let mut config_fault: Option = None; - loop { + // A sign-in that failed together with the connection it ran on. + // The sweep carries on -- a node that answers the dial and then + // goes quiet must not own the client, and it is also the endpoint + // the next connect would lead with -- and this is the reason the + // caller gets if nothing behind it works out either. + let mut sign_in_failure: Option = None; + let should_redirect = loop { let server_address = candidates[candidate].clone(); if server_address == paced_endpoint && let Some(remaining) = self.reestablish_wait().await @@ -560,6 +574,7 @@ impl TcpClient { info!("{NAME} client is connecting to server: {server_address}..."); match self.establish_bounded(&server_address, &candidates).await { Ok(connection) => { + let dialed = server_address.clone(); // The endpoint that answered is where this client now // lives: the leader check compares against it, and the // next reconnect starts from it. Recorded only once the @@ -567,11 +582,41 @@ impl TcpClient { // fails the TLS handshake does not become sticky and // shadow the endpoints behind it. *self.current_server_address.lock().await = server_address; - client_address = connection.client_address; - remote_address = connection.remote_address; + let client_address = connection.client_address; self.client_address.lock().await.replace(client_address); - connection_stream = connection.stream; - break; + let now = IggyTimestamp::now(); + info!( + "{NAME} client: {client_address} has connected to server: {} at: {now}", + connection.remote_address, + ); + self.stream.lock().await.replace(connection.stream); + self.set_state(ClientState::Connected).await; + self.connected_at.lock().await.replace(now); + self.publish_event(DiagnosticEvent::Connected).await; + + match self + .establish_session(client_address, skip_auto_login) + .await + { + Ok(should_redirect) => break should_redirect, + Err(failure) if failure.connection_lost => { + warn!( + "The sign-in on the server: {dialed} did not complete: {}", + failure.error, + ); + sign_in_failure = Some(failure.error); + // The sweep owns the state again: the sign-in + // took the connection down with it, and left + // `Disconnected` another caller would start a + // second sweep alongside this one. + self.set_state(ClientState::Connecting).await; + } + // The connection stands and only the session is + // missing: rejected credentials say the same thing + // on every node, and no endpoint behind this one + // would answer differently. + Err(failure) => return Err(failure.error), + } } Err(IggyError::CannotEstablishConnection) => {} Err(error) => config_fault = Some(error), @@ -586,12 +631,11 @@ impl TcpClient { } candidate = 0; - // An unreadable CA file, a certificate that names another - // host, a domain that will not parse: no endpoint answered and - // at least one said why in a way that a retry cannot change, - // so the caller gets that reason instead of a retry loop that - // buries it (`max_retries = None` would otherwise redial it - // every interval forever). + // An unreadable CA file, a domain that will not parse: no + // endpoint answered and at least one said why in a way that a + // retry cannot change, so the caller gets that reason instead + // of a retry loop that buries it (`max_retries = None` would + // otherwise redial it every interval forever). if let Some(error) = config_fault { self.fail_connect().await; return Err(error); @@ -604,7 +648,7 @@ impl TcpClient { if !self.config.reconnection.enabled { warn!("Automatic reconnection is disabled."); self.fail_connect().await; - return Err(IggyError::CannotEstablishConnection); + return Err(sign_in_failure.unwrap_or(IggyError::CannotEstablishConnection)); } let unlimited_retries = self.config.reconnection.max_retries.is_none(); @@ -629,109 +673,7 @@ impl TcpClient { } self.fail_connect().await; - return Err(IggyError::CannotEstablishConnection); - } - - let now = IggyTimestamp::now(); - info!( - "{NAME} client: {client_address} has connected to server: {remote_address} at: {now}", - ); - self.stream.lock().await.replace(connection_stream); - self.set_state(ClientState::Connected).await; - self.connected_at.lock().await.replace(now); - self.publish_event(DiagnosticEvent::Connected).await; - let skip_auto_login = { - let mut guard = self.skip_auto_login_once.lock().await; - std::mem::take(&mut *guard) - }; - - // Handle auto-login - let should_redirect = match self.sign_in_credentials().await { - None => { - info!("No credentials to sign in with."); - // Only `IggyClient` redirects after a manual sign-in, so - // a raw transport can stay on a backup: its first - // replicated write gets `TransientNotAccepted`, the - // redirect drops the session, and the retry fails - // `Unauthenticated` until the caller signs in again. - false - } - Some(credentials) => { - if skip_auto_login { - info!("Skipping automatic sign-in for a retried login/register request."); - false - } else { - info!("{NAME} client: {client_address} is signing in..."); - self.set_state(ClientState::Authenticating).await; - let signed_in = match &credentials { - Credentials::UsernamePassword(username, password) => self - .login_user(username, password.expose_secret()) - .await - .map(|_| format!("the user credentials, username: {username}")), - Credentials::PersonalAccessToken(token) => self - .login_with_personal_access_token(token.expose_secret()) - .await - .map(|_| "a personal access token".to_owned()), - }; - match signed_in { - Ok(how) => { - info!("{NAME} client: {client_address} has signed in with {how}.") - } - Err(error) => { - // With the transport up and only the session - // missing, the state has to say so: left at - // `Authenticating` every gated operation fails - // client-side with `Disconnected`, `connect()` - // returns ok without dialing, and nothing short - // of an explicit `login_user` recovers. - // - // A sign-in can also fail because the socket - // died under it. Whatever is left of that - // connection cannot carry a request, so it goes - // rather than being kept behind a `Connected` - // that makes the next `connect()` a no-op and - // leaves every gated operation failing until - // someone calls `disconnect()` by hand. - if matches!( - error, - IggyError::Disconnected - | IggyError::EmptyResponse - | IggyError::NotConnected - | IggyError::CannotEstablishConnection - | IggyError::TcpError - | IggyError::StaleClient - ) { - self.disconnect_transport().await?; - } else if self.get_state().await == ClientState::Authenticating { - self.set_state(ClientState::Connected).await; - } - // A rejected credential does not become valid on - // the next reconnect, and replaying it costs an - // argon2 on the server every time. Configured - // credentials stay as configured -- they are the - // caller's to fix -- so only the remembered - // sign-in is dropped. - if matches!( - error, - IggyError::InvalidCredentials - | IggyError::InvalidUsername - | IggyError::InvalidPassword - | IggyError::Unauthenticated - ) { - self.forget_session_credentials().await; - } - return Err(error); - } - } - - // The sole leader settlement, and it runs - // authenticated. Any node completes a login now -- a - // backup forwards the register to the primary -- so - // this decides where later ops land, not whether - // sign-in works. - self.handle_leader_redirection().await? - } - } + return Err(sign_in_failure.unwrap_or(IggyError::CannotEstablishConnection)); }; if should_redirect { @@ -742,6 +684,105 @@ impl TcpClient { } } + /// Re-establish the session on a connection that just came up and settle it + /// on the leader. Reports whether the leader check asks for a redirect. + async fn establish_session( + &self, + client_address: SocketAddr, + skip_auto_login: bool, + ) -> Result { + let Some(credentials) = self.sign_in_credentials().await else { + info!("No credentials to sign in with."); + // Only `IggyClient` redirects after a manual sign-in, so a raw + // transport can stay on a backup: its first replicated write gets + // `TransientNotAccepted`, the redirect drops the session, and the + // retry fails `Unauthenticated` until the caller signs in again. + return Ok(false); + }; + + if skip_auto_login { + info!("Skipping automatic sign-in for a retried login/register request."); + return Ok(false); + } + + info!("{NAME} client: {client_address} is signing in..."); + self.set_state(ClientState::Authenticating).await; + let signed_in = match &credentials { + Credentials::UsernamePassword(username, password) => self + .login_user(username, password.expose_secret()) + .await + .map(|_| format!("the user credentials, username: {username}")), + Credentials::PersonalAccessToken(token) => self + .login_with_personal_access_token(token.expose_secret()) + .await + .map(|_| "a personal access token".to_owned()), + }; + match signed_in { + Ok(how) => info!("{NAME} client: {client_address} has signed in with {how}."), + Err(error) => return Err(self.fail_sign_in(error).await), + } + + // The sole leader settlement, and it runs authenticated. Any node + // completes a login now -- a backup forwards the register to the + // primary -- so this decides where later ops land, not whether sign-in + // works. + self.handle_leader_redirection() + .await + .map_err(|error| SignInFailure { + error, + connection_lost: false, + }) + } + + /// Put the client back into a state that describes what a failed sign-in + /// left behind, and report whether the connection survived it. + async fn fail_sign_in(&self, error: IggyError) -> SignInFailure { + // A sign-in can fail because the socket died under it. Whatever is left + // of that connection cannot carry a request, so it goes rather than + // being kept behind a `Connected` that makes the next `connect()` a + // no-op and leaves every gated operation failing until someone calls + // `disconnect()` by hand. + let connection_lost = matches!( + error, + IggyError::Disconnected + | IggyError::EmptyResponse + | IggyError::NotConnected + | IggyError::CannotEstablishConnection + | IggyError::TcpError + | IggyError::StaleClient + ); + if connection_lost { + if let Err(teardown_error) = self.disconnect_transport().await { + warn!("Failed to drop the connection of a failed sign-in: {teardown_error}"); + } + } else if self.get_state().await == ClientState::Authenticating { + // With the transport up and only the session missing, the state has + // to say so: left at `Authenticating` every gated operation fails + // client-side with `Disconnected`, `connect()` returns ok without + // dialing, and nothing short of an explicit `login_user` recovers. + self.set_state(ClientState::Connected).await; + } + + // A rejected credential does not become valid on the next reconnect, + // and replaying it costs an argon2 on the server every time. Configured + // credentials stay as configured -- they are the caller's to fix -- so + // only the remembered sign-in is dropped. + if matches!( + error, + IggyError::InvalidCredentials + | IggyError::InvalidUsername + | IggyError::InvalidPassword + | IggyError::Unauthenticated + ) { + self.forget_session_credentials().await; + } + + SignInFailure { + error, + connection_lost, + } + } + /// Checks cluster metadata and handles leader redirection if needed. /// Returns true if redirection occurred and reconnection is needed. pub(crate) async fn handle_leader_redirection(&self) -> Result { @@ -803,11 +844,12 @@ impl TcpClient { /// applied on top: the configured password will never work again, and every /// later reconnect would otherwise fail `InvalidCredentials`. async fn sign_in_credentials(&self) -> Option { - // The sign-in that last succeeded, whoever ran it. One rule in every - // SDK: a client is whoever it last signed in as, so the same failure - // restores the same session everywhere. A configured `AutoLogin` signs - // in through this very path, so for a client that never signed in by - // hand the remembered credentials *are* the configured ones. + // The sign-in that last succeeded, whoever ran it: a client is whoever + // it last signed in as, so a reconnect restores the session the caller + // last asked for rather than one it had moved off. A configured + // `AutoLogin` signs in through this very path, so for a client that + // never signed in by hand the remembered credentials *are* the + // configured ones. if let Some(remembered) = self.session_credentials.lock().await.as_ref() { return Some(remembered.credentials.clone()); } @@ -831,6 +873,48 @@ impl TcpClient { } } + /// Read the cluster roster once, on the first sign-in that succeeds on a + /// client whose caller signs it in by hand. + /// + /// `connect()` follows the sign-in it runs itself with a leader check, and + /// that check is what refreshes the roster. A client with no configured + /// `AutoLogin` is signed in by its caller instead, and only `IggyClient` + /// follows that with a leader check, so a raw transport would know exactly + /// one endpoint -- the one it was configured with -- and redial the node + /// that died for as long as it lived. + /// + /// Once per client, which is also what keeps the read from nesting: it goes + /// through the reconnect path, whose sign-in calls straight back into here. + /// Bounded for the same reason: the read is a convenience for a failover + /// that may never happen, so it must not hold up the sign-in that triggered + /// it -- unbounded retries would do exactly that. + async fn learn_roster_once(&self) { + // Only a live session can read the roster, and only the caller's own + // sign-in leaves one behind here: a connect that signs in follows it + // with a leader check of its own. + if self.auto_login_configured() + || self.get_state().await != ClientState::Authenticated + || self.roster_learned.swap(true, Ordering::SeqCst) + { + return; + } + + let read = read_transport_endpoints(self, TransportProtocol::Tcp); + let Ok(endpoints) = tokio::time::timeout(ROSTER_READ_TIMEOUT, read).await else { + warn!("Reading the cluster roster took longer than {ROSTER_READ_TIMEOUT:?}"); + return; + }; + if endpoints.is_empty() { + return; + } + + info!( + "{NAME} client learned {} endpoint(s) to fail over to.", + endpoints.len() + ); + *self.roster_endpoints.lock().await = endpoints; + } + /// Endpoints to dial for one connect, likeliest first: where the client /// currently is, the address it was configured with, then the roster it /// learned while connected. @@ -843,14 +927,13 @@ impl TcpClient { let roster = self.roster_endpoints.lock().await.clone(); let configured = std::iter::once(&self.config.server_address); for endpoint in configured.chain(roster.iter()) { - let mut known = false; - for candidate in &candidates { - if is_same_address(candidate, endpoint).await { - known = true; - break; - } - } - if !known { + // Spellings only, no name resolution: one duplicate endpoint costs + // a dial that fails on its own, while a resolver that does not + // answer would stall the failover before it dialed anything. + if !candidates + .iter() + .any(|candidate| is_same_spelling(candidate, endpoint)) + { candidates.push(endpoint.clone()); } } @@ -943,8 +1026,14 @@ impl TcpClient { IggyError::InvalidTlsDomain })?; let stream = connector.connect(domain, stream).await.map_err(|error| { + // The verdict describes the peer, not this client: a certificate + // that names another host, a peer that answers a ClientHello with + // something else. The endpoints behind it may be fine, and with + // one roster entry per node the SNI is a bare address that no + // certificate has to cover, so this ends the dial rather than the + // connect. error!("Failed to establish a TLS connection to the server: {error}"); - classify_handshake_failure(&error) + IggyError::CannotEstablishConnection })?; Ok(EstablishedConnection { @@ -1016,8 +1105,18 @@ impl TcpClient { /// failover unauthenticated. The public [`Client::disconnect`] wraps this /// and forgets them first. async fn disconnect_transport(&self) -> Result<(), IggyError> { - if self.get_state().await == ClientState::Disconnected { - return Ok(()); + match self.get_state().await { + ClientState::Disconnected => return Ok(()), + // A connect is already sweeping, and every caller here is tearing + // the connection down in order to reconnect -- which is what that + // sweep is doing. Tearing it down under the sweep would re-mint the + // client id the sign-in in flight is binding and take the stream it + // just installed. + ClientState::Connecting => { + trace!("Not disconnecting; a connect is already in flight."); + return Ok(()); + } + _ => {} } let client_address = self.get_client_address_value().await; @@ -1313,6 +1412,7 @@ const fn is_login_register_code(code: u32) -> bool { mod tests { use super::*; use iggy_binary_protocol::codes::{GET_ME_CODE, LOGOUT_USER_CODE, SEND_MESSAGES_CODE}; + use std::sync::atomic::AtomicUsize; use tokio::io::{AsyncReadExt, AsyncWriteExt}; const SESSION_USER_ID: u32 = 7; @@ -1350,15 +1450,23 @@ mod tests { } /// A peer that accepts TCP and hangs up without a byte: enough for the - /// dial, never enough for a TLS handshake. - async fn endpoint_that_hangs_up() -> String { + /// dial, never enough for a TLS handshake or a sign-in. The counter says + /// how many dials reached it. + async fn counted_endpoint_that_hangs_up() -> (String, Arc) { let (listener, address) = live_endpoint().await; + let dials = Arc::new(AtomicUsize::new(0)); + let accepted = dials.clone(); tokio::spawn(async move { while let Ok((stream, _)) = listener.accept().await { + accepted.fetch_add(1, Ordering::SeqCst); drop(stream); } }); - address + (address, dials) + } + + async fn endpoint_that_hangs_up() -> String { + counted_endpoint_that_hangs_up().await.0 } // With reconnection off there are no retries, but the endpoints the roster @@ -1469,11 +1577,15 @@ mod tests { } /// A peer that accepts TCP and then answers a ClientHello with something - /// else: the handshake fails for a reason no retry changes. - async fn endpoint_that_speaks_no_tls() -> String { + /// else, so the handshake fails on the peer's own answer. The counter says + /// how many dials reached it. + async fn counted_endpoint_that_speaks_no_tls() -> (String, Arc) { let (listener, address) = live_endpoint().await; + let dials = Arc::new(AtomicUsize::new(0)); + let accepted = dials.clone(); tokio::spawn(async move { while let Ok((mut stream, _)) = listener.accept().await { + accepted.fetch_add(1, Ordering::SeqCst); tokio::spawn(async move { let _ = stream.write_all(b"this is not a TLS record\n").await; // Held open, so the failure is the handshake's verdict @@ -1483,18 +1595,58 @@ mod tests { }); } }); - address + (address, dials) } - // A certificate this client will never accept says the same thing on every - // attempt, so it has to reach the caller instead of being redialed every - // interval forever -- which is what `max_retries = None` did with it. + // A handshake verdict describes the peer -- a certificate that names + // another host, an answer that is not TLS at all -- and not this client's + // configuration, so it ends the dial rather than the connect: the endpoints + // behind it are untried, and a redial can find a repaired node. #[tokio::test] - async fn a_handshake_no_retry_can_fix_ends_the_connect() { + async fn a_handshake_the_peer_failed_is_dialed_again() { + let (plaintext, dials) = counted_endpoint_that_speaks_no_tls().await; let client = TcpClient::create(Arc::new(TcpClientConfig { - server_address: endpoint_that_speaks_no_tls().await, + server_address: plaintext, tls_enabled: true, tls_validate_certificate: false, + reconnection: TcpClientReconnectionConfig { + // One retry, so the pass runs twice and the connect still ends + // on its own. + max_retries: Some(1), + interval: NonZeroIggyDuration::from_str("100ms").expect("duration"), + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + + let connect = tokio::time::timeout( + std::time::Duration::from_secs(10), + std::pin::pin!(TcpClient::connect(&client)), + ) + .await + .expect("the connect has to end on its own"); + assert!(matches!(connect, Err(IggyError::CannotEstablishConnection))); + assert_eq!( + dials.load(Ordering::SeqCst), + 2, + "a handshake the peer failed ended the connect instead of the dial" + ); + assert_eq!(client.get_state().await, ClientState::Disconnected); + } + + // A CA file that cannot be read is this client's own configuration, and it + // says the same thing on every attempt: reported as a lost connection it + // would be redialed every interval forever under `max_retries = None`, + // which is how a wrong CA path looks like a flaky network. + #[tokio::test] + async fn a_ca_file_that_cannot_be_read_ends_the_connect() { + let (_listener, endpoint) = live_endpoint().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: endpoint, + tls_enabled: true, + tls_validate_certificate: true, + tls_ca_file: Some("no-such-ca-file.pem".to_string()), reconnection: TcpClientReconnectionConfig { // Unlimited retries, so a transient classification never // returns and this test times out instead of failing. @@ -1512,7 +1664,39 @@ mod tests { ) .await .expect("the connect has to end on its own"); - assert!(matches!(connect, Err(IggyError::InvalidTlsCertificate))); + assert!(matches!(connect, Err(IggyError::InvalidTlsCertificatePath))); + assert_eq!(client.get_state().await, ClientState::Disconnected); + } + + // A node that answers the dial and then cannot carry the sign-in has to + // hand the sweep on. Ending it there leaves that node the one the client is + // recorded on, so every later connect leads with it and the endpoints + // behind it are never reached. + #[tokio::test] + async fn a_sign_in_that_failed_hands_the_sweep_on_to_the_next_endpoint() { + let (dialed_first, _) = counted_endpoint_that_hangs_up().await; + let (survivor, survivor_dials) = counted_endpoint_that_hangs_up().await; + let client = TcpClient::create(Arc::new(TcpClientConfig { + server_address: dialed_first, + auto_login: AutoLogin::Enabled(Credentials::UsernamePassword( + "iggy".to_string(), + "iggy".into(), + )), + reconnection: TcpClientReconnectionConfig { + enabled: false, + ..TcpClientReconnectionConfig::default() + }, + ..TcpClientConfig::default() + })) + .expect("create the client"); + *client.roster_endpoints.lock().await = vec![survivor]; + + assert!(TcpClient::connect(&client).await.is_err()); + assert_eq!( + survivor_dials.load(Ordering::SeqCst), + 1, + "the endpoint behind the one whose sign-in failed was never dialed" + ); assert_eq!(client.get_state().await, ClientState::Disconnected); } @@ -1563,9 +1747,11 @@ mod tests { SEND_MESSAGES_CODE, &IggyError::CannotEstablishConnection )); - assert!(replay_is_safe(SEND_MESSAGES_CODE, &IggyError::StaleClient)); // Written, and its outcome unknown: a replicated write must not be - // re-sent under a session the fence cannot match it against. + // re-sent under a session the fence cannot match it against. An + // eviction is consumed in place of the reply, so it says nothing about + // whether the write committed. + assert!(!replay_is_safe(SEND_MESSAGES_CODE, &IggyError::StaleClient)); assert!(!replay_is_safe( SEND_MESSAGES_CODE, &IggyError::Disconnected @@ -1956,10 +2142,9 @@ mod tests { } } - // One rule in every SDK: a client is whoever it last signed in as, so the - // same failure restores the same session in each of them. The connection - // re-authenticates from the login it captured, and a redial that replayed - // somebody else would make the outcome depend on which got there first. + // A client is whoever it last signed in as: the connection re-authenticates + // from the login it captured, and a redial that replayed somebody else + // would make the outcome depend on which of the two got there first. #[tokio::test] async fn the_last_sign_in_outranks_the_configured_credentials() { let client = TcpClient::create(Arc::new(TcpClientConfig { diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index 38bbfebb20..245299d099 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -21,7 +21,6 @@ using System.Net.Security; using System.Net.Sockets; using System.Runtime.InteropServices; -using System.Security.Authentication; using System.Security.Cryptography.X509Certificates; using Apache.Iggy.Configuration; using Apache.Iggy.Contracts; @@ -1157,8 +1156,8 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken if (IsTlsConfigurationFault(e)) { // A fault no retry can fix, kept aside rather than thrown at once: it belongs to the - // endpoint that raised it - a certificate that names another host - and the endpoints - // behind that one may be perfectly usable. + // endpoint that raised it - a CA file that cannot be read - and the endpoints behind that + // one may be perfectly usable. configurationFault = e; } @@ -1174,8 +1173,8 @@ private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken _currentAddress = candidates[0]; // No endpoint answered and at least one said why in a way no retry changes: an unreadable CA - // file, a certificate this client will never accept. The caller gets that reason instead of a - // retry loop that buries it - unlimited retries would otherwise redial it forever. + // file. The caller gets that reason instead of a retry loop that buries it - unlimited retries + // would otherwise redial it forever. if (configurationFault is not null) { SetConnectionState(ConnectionState.Disconnected); @@ -1344,21 +1343,16 @@ await sslStream.AuthenticateAsClientAsync( /// /// Whether bringing an endpoint up failed for a reason that says this client's own TLS configuration is - /// wrong: a CA file that cannot be read, or a certificate it will never accept. Neither changes on a - /// retry, so the sweep reports it instead of redialing forever. + /// wrong: a CA file that cannot be read. It does not change on a retry, so the sweep reports it instead + /// of redialing forever. /// private static bool IsTlsConfigurationFault(Exception e) { - if (e is InvalidCertificatePathException) - { - return true; - } - - // AuthenticationException also carries a handshake that died on the wire - a reset, a closed socket - - // and that says nothing about the configuration. Only a verdict reached without transport trouble is - // one no retry can change. - return e is AuthenticationException - && e.InnerException is not (IOException or SocketException); + // A handshake verdict describes the peer, not this client: a certificate that names another host, a + // peer that answers a ClientHello with something else. The endpoints behind it may be fine, and with + // one roster entry per node the target host is a bare address no certificate has to cover, so a failed + // handshake ends the dial rather than the connect. + return e is InvalidCertificatePathException; } private async Task SendAckAsync(int code, ReadOnlyMemory body, CancellationToken token) From 3bf74dfe8a26a64b4fcb5e2fa5de99dd4739f2bc Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Wed, 26 Aug 2026 22:25:09 +0200 Subject: [PATCH 16/16] fix docs --- .../tcp_config/tcp_client_reconnection_config.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs b/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs index 3d4c2957b1..497db19211 100644 --- a/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs +++ b/core/common/src/types/configuration/tcp_config/tcp_client_reconnection_config.rs @@ -32,6 +32,19 @@ pub struct TcpClientReconnectionConfig { /// addresses it was configured with, and every node the roster named, so a /// survivor is reached inside the first pass rather than one delay per /// endpoint. + /// + /// The number is not portable across SDKs. Each counts the same setting in + /// its own terms, and `0` means something different in every one of them, so + /// a deployment that runs several has to set this per SDK rather than copy + /// one value across: + /// + /// | SDK | `N` | `0` | unlimited | + /// | ---- | -------------------------------------- | ----------------------------------------- | ---------------- | + /// | Rust | `N` passes after a first, unpaced one | that first pass alone | `None` | + /// | C# | as Rust | unlimited | `0` | + /// | Go | `N` passes, the first one of them | unlimited | `0` | + /// | Java | `N` passes, the first one of them | one pass, and only with several endpoints | a large `N` | + /// | Node | `N` passes, the first one of them | no pass at all | a large `N` | pub max_retries: Option, /// Delay between passes. The first pass runs at once when the client knows /// more than one endpoint.