From 9e2619bbb27b258b325215e2d3b73daea0fece3a Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:08:43 +0800 Subject: [PATCH 01/11] fix #446: fence FsyncCoordinator against truncation, clamp durable/persisted index, fix purge order - FsyncCoordinator generation-fences against truncation races - remove_range clamps durable_index/persisted_index post-truncation - fix purge ordering relative to durable_index advance - fix flaky snapshot_transfer_does_not_block_apply_embedded test: `since` baseline was captured after the 80-entry write loop, racing against the async snapshot+purge task that can complete mid-loop --- .dockerignore | 4 +- .../src/storage/buffered_raft_log.rs | 180 +++++++++++------ .../concurrent_fsync_test.rs | 4 + .../drain_fsync_test.rs | 161 +++++++++++---- .../persisted_index_clamp_test.rs | 188 ++++++++++++++++++ .../process_crash_safety_test.rs | 102 ++++++++++ .../replace_range_fsync_test.rs | 88 ++++++++ .../truncation_fsync_fence_test.rs | 101 ++++++++++ .../src/storage/fsync_coordinator.rs | 27 ++- .../src/storage/fsync_coordinator_test.rs | 52 +++++ .../test_utils/mock/mock_storage_engine.rs | 51 +++++ d-engine-core/src/watch/mod.rs | 1 + ..._transfer_does_not_block_apply_embedded.rs | 36 +++- .../performance_test.rs | 14 +- .../storage_buffered_raft_log/stress_test.rs | 9 +- examples/single-node-expansion/Makefile | 25 ++- examples/single-node-expansion/config/n1.toml | 52 ++++- .../three-nodes-standalone/docker/Dockerfile | 4 +- examples/three-nodes-standalone/src/main.rs | 9 +- 19 files changed, 978 insertions(+), 130 deletions(-) create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs diff --git a/.dockerignore b/.dockerignore index 2a8dfc95..f20b5623 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ **/target/ .git/ -examples/ +examples/* +!examples/three-nodes-standalone +!examples/client-usage-standalone diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index 713286e9..b841238e 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -197,6 +197,15 @@ impl TermSegments { /// on tokio worker threads or the inbound event loop. #[derive(Debug)] pub enum IOTask { + /// Persist entries on the IO thread. `append_entries()` sends this and + /// awaits `done` — replaces the old inline `persist_entries()` call that + /// ran on the caller's own task (raft-core-loop), which could block + /// behind the IO thread's own concurrent fsync. + Persist { + entries: Vec, + done: oneshot::Sender>, + }, + /// Atomically truncate from `truncate_from` then persist `new_entries`. /// Conflict-resolution path: truncate + write are a single atomic IO unit. /// `done` is signalled after the IO thread finishes the replace so callers @@ -258,6 +267,10 @@ where // Raft must not tell a client or a peer a write is safe ahead of this point, // regardless of what's already visible in `entries`. pub(crate) durable_index: AtomicU64, + // Highest index handed to the storage engine (page cache), not yet + // fsynced. Set by append_entries()'s synchronous persist_entries() call. + // Lets the IO thread know what to fsync without re-scanning/re-writing. + persisted_index: AtomicU64, // The next index to be allocated pub(crate) next_id: AtomicU64, @@ -465,10 +478,20 @@ where } self.insert_to_memory(&entries); - // Signal IO thread to persist. Multiple concurrent notify_one() calls - // while the IO thread is busy coalesce into one wakeup — no per-write - // kernel cond_signal. IO thread reads from SkipMap via max_index. - self.write_notify.notify_one(); + + // Route the actual write through the IO thread — never call + // persist_entries() inline from this task. + // Still blocks the caller until truly persisted. + let (done_tx, done_rx) = oneshot::channel(); + self.command_sender + .send(IOTask::Persist { + entries, + done: done_tx, + }) + .map_err(|e| NetworkError::SingalSendFailed(format!("Persist send failed: {e:?}")))?; + done_rx + .await + .map_err(|_| NetworkError::SingalSendFailed("Persist done channel closed".into()))??; Ok(()) } @@ -657,9 +680,8 @@ where self.purge_prefix(cutoff_index); // Purged entries are backed by the snapshot; treat cutoff as durable. - // fetch_max is monotonic — avoids racing fsync_coordinator's concurrent - // advance on the raft-io thread — and this fires LogFlushed consistently - // with every other durable_index advancement in this file. + // Must run after purge_prefix() — advance_durable_and_notify() validates + // against last_purged_index, which purge_prefix() just established. self.advance_durable_and_notify(cutoff_index.index); // Route purge through the IO thread so it never blocks the inbound event loop. @@ -839,6 +861,7 @@ where last_purged_index: AtomicU64::new(last_purged_index_val), last_purged_term: AtomicU64::new(last_purged_term_val), durable_index: AtomicU64::new(disk_len), + persisted_index: AtomicU64::new(disk_len), next_id: AtomicU64::new(disk_len + 1), write_notify: Arc::new(Notify::new()), command_sender: command_sender.clone(), @@ -956,9 +979,7 @@ where if should_break { break; } } _ = safety_timer.tick() => { - let start = this.durable_index.load(Ordering::Acquire) + 1; - let end = this.max_index.load(Ordering::Acquire); - let _ = Self::persist_pending_range(&this, start, end, &mut pending_max, "safety-net").await; + Self::fold_persisted_watermark(&this, &mut pending_max); if pending_max > 0 { this.fsync_coordinator.submit(&this, pending_max, vec![]); @@ -969,38 +990,14 @@ where } } - /// Writes entries in `(from, to]` that haven't reached page cache yet - /// (no fsync). Advances `pending_max` on success; propagates the error - /// as-is on failure — whether to notify any waiting `Flush` caller is - /// left to the caller. - async fn persist_pending_range( + /// Folds `persisted_index` (set by `append_entries()`'s synchronous + /// write) into `pending_max`, so the IO thread still dispatches fsync + /// for it even though writing is no longer this thread's job. + fn fold_persisted_watermark( this: &Arc, - from: u64, - to: u64, pending_max: &mut u64, - ctx: &str, - ) -> Result<()> { - if this.is_poisoned() { - return Err(Error::Fatal("raft log storage is poisoned".to_string())); - } - - if from > to { - return Ok(()); - } - let entries = this.get_entries_range(from..=to)?; - if entries.is_empty() { - return Ok(()); - } - this.log_store - .persist_entries(entries) - .await - .inspect(|_| { - *pending_max = (*pending_max).max(to); - }) - .inspect_err(|e| { - error!("{ctx} persist_entries failed: {e:?}"); - this.mark_poisoned_and_notify(format!("{ctx}: persist_entries failed: {e:?}")); - }) + ) { + *pending_max = (*pending_max).max(this.persisted_index.load(Ordering::Acquire)); } async fn run_batch_turn( @@ -1010,15 +1007,7 @@ where mut replies: Vec>>, mut seen_shutdown: bool, ) -> bool { - let start = this.durable_index.load(Ordering::Acquire) + 1; - let end = this.max_index.load(Ordering::Acquire); - let mut persist_failed = false; - if let Err(e) = Self::persist_pending_range(this, start, end, pending_max, "batch").await { - for reply in replies.drain(..) { - let _ = reply.send(Err(Error::Fatal(format!("persist_entries failed: {e:?}")))); - } - persist_failed = true; - } + Self::fold_persisted_watermark(this, pending_max); // `seen_shutdown` is not a gate here — regardless of whether the // caller already knows shutdown is happening, any commands still @@ -1043,13 +1032,6 @@ where } } - if !replies.is_empty() && !persist_failed { - let start = *pending_max + 1; - let end = this.max_index.load(Ordering::Acquire); - let _ = - Self::persist_pending_range(this, start, end, pending_max, "batch catch-up").await; - } - this.fsync_coordinator.submit(this, *pending_max, replies); *pending_max = 0; if seen_shutdown { @@ -1078,6 +1060,33 @@ where IOTask::Shutdown => { unreachable!("Shutdown is always filtered out before reaching handle_non_write_cmd") } + IOTask::Persist { entries, done } => { + if this.is_poisoned() { + let _ = done.send(Err(Error::Fatal("raft log storage is poisoned".into()))); + return true; // signal batch_processor to exit — disk state is untrusted + } + let max_idx = entries.last().map(|e| e.index).unwrap_or(0); + let result = this.log_store.persist_entries(entries).await; + if let Err(ref e) = result { + error!("IOTask::Persist failed (fatal): {e:?}"); + this.mark_poisoned_and_notify(format!("Persist failed: {e:?}")); + let _ = done.send(result); + return true; // signal batch_processor to exit — disk state is corrupted + } + if max_idx > 0 { + let current_bound = this + .max_index + .load(Ordering::Acquire) + .max(this.last_purged_index.load(Ordering::Acquire)); + let safe_max_idx = max_idx.min(current_bound); + if safe_max_idx > 0 { + this.persisted_index.fetch_max(safe_max_idx, Ordering::AcqRel); + this.fsync_coordinator.submit(this, safe_max_idx, vec![]); + } + } + let _ = done.send(result); + false // write succeeded, storage still trustworthy — keep the IO thread running + } IOTask::ReplaceRange { truncate_from, new_entries, @@ -1101,6 +1110,7 @@ where } if max_idx > 0 { *pending_max = (*pending_max).max(max_idx); + this.fsync_coordinator.submit(this, max_idx, vec![]); } let _ = done.send(result); false @@ -1148,6 +1158,7 @@ where self.entries.write().clear(); self.durable_index.store(0, Ordering::Release); + self.persisted_index.store(0, Ordering::Release); self.next_id.store(1, Ordering::Release); // Reset boundaries @@ -1222,17 +1233,31 @@ where } } - /// Advance `durable_index` to `new_durable` (monotonically) and send `LogFlushed`. + // The single choke point every reported max_index must pass through — + // re-validates against the current log boundary regardless of how many + // upstream call sites raced to produce this value. pub(super) fn advance_durable_and_notify( &self, - new_durable: u64, + reported_max: u64, ) { - let prev = self.durable_index.fetch_max(new_durable, Ordering::AcqRel); - if new_durable > prev + let current_max = self + .max_index + .load(Ordering::Acquire) + .max(self.last_purged_index.load(Ordering::Acquire)); + let safe_max = reported_max.min(current_max); + debug_assert!( + safe_max == reported_max, + "advance_durable_and_notify: reported_max {reported_max} exceeded current bound {current_max}, clamped" + ); + if safe_max == 0 { + return; + } + let prev = self.durable_index.fetch_max(safe_max, Ordering::AcqRel); + if safe_max > prev && let Some(ref tx) = self.log_flush_tx { let _ = tx.send(crate::InternalEvent::LogFlushed { - durable_index: new_durable, + durable_index: safe_max, }); } } @@ -1283,6 +1308,12 @@ where let (new_min, new_max) = self.remove_range_locked(&entries, range); self.min_index.store(new_min, Ordering::Release); self.max_index.store(new_max, Ordering::Release); + + self.persisted_index.fetch_min(new_max, Ordering::AcqRel); + self.durable_index.fetch_min(new_max, Ordering::AcqRel); + // Clamps pending_max and bumps generation, in that order — see + // fence_truncation()'s doc comment for why the order matters. + self.fsync_coordinator.fence_truncation(new_max); // `entries` guard drops here (end of scope) — write lock released. } @@ -1382,9 +1413,6 @@ where // then term (Acquire) always observe a consistent pair. self.last_purged_term.store(cutoff.term, Ordering::Release); self.last_purged_index.store(cutoff.index, Ordering::Release); - - // `entries` guard drops here — everything above is now visible together - // to any reader acquiring the read lock or loading these atomics after. } // Update the term index (completely lock-free) @@ -1439,6 +1467,14 @@ where pub fn is_empty(&self) -> bool { self.entries.read().is_empty() } + + #[cfg(test)] + pub(super) fn set_max_index_for_test( + &self, + value: u64, + ) { + self.max_index.store(value, Ordering::Release); + } } impl Drop for BufferedRaftLog @@ -1503,10 +1539,18 @@ mod id_allocation_test; #[path = "buffered_raft_log_test/performance_test.rs"] mod performance_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/persisted_index_clamp_test.rs"] +mod persisted_index_clamp_test; + #[cfg(test)] #[path = "buffered_raft_log_test/pipeline_overlap_test.rs"] mod pipeline_overlap_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/process_crash_safety_test.rs"] +mod process_crash_safety_test; + #[cfg(test)] #[path = "buffered_raft_log_test/quorum_durability_test.rs"] mod quorum_durability_test; @@ -1519,6 +1563,10 @@ mod raft_properties_test; #[path = "buffered_raft_log_test/remove_range_test.rs"] mod remove_range_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/replace_range_fsync_test.rs"] +mod replace_range_fsync_test; + #[cfg(test)] #[path = "buffered_raft_log_test/shutdown_test.rs"] mod shutdown_test; @@ -1531,6 +1579,10 @@ mod term_index_test; #[path = "buffered_raft_log_test/term_segments_test.rs"] mod term_segments_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/truncation_fsync_fence_test.rs"] +mod truncation_fsync_fence_test; + #[cfg(test)] #[path = "buffered_raft_log_test/worker_test.rs"] mod worker_test; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs index 178a9328..fce20bf3 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs @@ -313,6 +313,10 @@ async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready + // advance_durable_and_notify() clamps against max_index — simulate a log + // that already has 150 entries, matching the highest value used below. + raft_log.set_max_index_for_test(150); + // Simulates a fsync task completing with index 150, then a second, older // fsync task (dispatched earlier, finishing later) completing with 100. raft_log.advance_durable_and_notify(150); diff --git a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs index 838cc386..daf9a096 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs @@ -828,6 +828,9 @@ async fn test_poisoned_skips_purge() { let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); + // advance_durable_and_notify() clamps against max_index — simulate a log + // that already has the entry this test purges up to. + raft_log.set_max_index_for_test(1); raft_log.poisoned.store(true, Ordering::SeqCst); let result = raft_log.purge_logs_up_to(LogId { term: 1, index: 1 }).await; @@ -843,13 +846,21 @@ async fn test_poisoned_skips_purge() { /// 2026-07-19 — `run_batch_turn`'s drain loop now replies before returning, /// instead of silently dropping the oneshot sender). /// -/// Ordering is made deterministic (not timing-sensitive) by gating the IO -/// thread inside its first `persist_entries()` call. While it's blocked, an +/// Ordering is made deterministic (not timing-sensitive) by gating the base +/// entries' `persist_entries()` call — `append_entries()` now calls it +/// synchronously, so the base append is spawned as its own task and blocks +/// there instead of returning immediately. While it's blocked, an /// `IOTask::Flush` is sent directly (guaranteed FIFO-first) followed by a /// conflict-triggering `filter_out_conflicts_and_append` call (sends -/// `IOTask::ReplaceRange` second). Releasing the gate lets `run_batch_turn` -/// drain both in one pass, in that order. -#[tokio::test] +/// `IOTask::ReplaceRange` second). Releasing the gate lets the base append +/// finish and `run_batch_turn` drain both queued commands in one pass, in +/// that order. +/// +/// Needs `flavor = "multi_thread"`: the gate blocks on a synchronous +/// `std::sync::mpsc::Receiver::recv()`, which would otherwise freeze the +/// single default executor thread that the spawned base-append task, the +/// conflict task, and this test body all need to share. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() { let (gate_tx, gate_rx) = std::sync::mpsc::channel::<()>(); let gate_rx = std::sync::Mutex::new(Some(gate_rx)); @@ -900,30 +911,34 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); - // Base entries land in memory synchronously; the IO thread wakes and - // immediately blocks inside the gated persist_entries() call, before it - // ever drains the command queue. - raft_log - .append_entries(vec![ - Entry { - index: 1, - term: 1, - payload: None, - }, - Entry { - index: 2, - term: 1, - payload: None, - }, - Entry { - index: 3, - term: 1, - payload: None, - }, - ]) - .await - .unwrap(); - sleep(Duration::from_millis(20)).await; // let the IO thread reach the gate + // Base entries land in memory synchronously (before the gate), then + // append_entries() blocks inside its own gated persist_entries() call — + // spawned so the rest of this test can proceed while it's stuck there. + let base_append_task = tokio::spawn({ + let raft_log = raft_log.clone(); + async move { + raft_log + .append_entries(vec![ + Entry { + index: 1, + term: 1, + payload: None, + }, + Entry { + index: 2, + term: 1, + payload: None, + }, + Entry { + index: 3, + term: 1, + payload: None, + }, + ]) + .await + } + }); + sleep(Duration::from_millis(20)).await; // let it reach the gate // Send Flush directly — guarantees it's enqueued before the ReplaceRange // sent below, so it's the one already sitting in `replies` when the @@ -955,6 +970,12 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() sleep(Duration::from_millis(50)).await; // let the ReplaceRange send land gate_tx.send(()).unwrap(); + timeout(Duration::from_secs(2), base_append_task) + .await + .expect("base append task must not hang") + .expect("base append task must not panic") + .expect("base append must succeed once the gate releases"); + let flush_result = timeout(Duration::from_secs(2), flush_rx) .await .expect("flush reply must not hang"); @@ -1045,7 +1066,7 @@ async fn test_poisoned_survives_reset() { /// A `persist_entries()` (page-cache write) failure poisons the log, exactly /// like an fsync failure does — these are two independent failure surfaces -/// (see `persist_pending_range` vs `FsyncCoordinator::run_until_caught_up`) +/// (see `IOTask::Persist` vs `FsyncCoordinator::run_until_caught_up`) /// and both must reach the same fatal outcome. /// /// Without this test, a bug that only wires up ONE of the two poisoning @@ -1073,18 +1094,20 @@ async fn test_persist_entries_failure_poisons() { let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - // Triggers the IO thread's persist_pending_range call, which hits the - // mock's first (failing) persist_entries() — this is the - // persist_pending_range poisoning path, NOT FsyncCoordinator's. - raft_log + // append_entries() routes the write through IOTask::Persist and awaits + // the IO thread's reply — a persist failure now surfaces synchronously, + // right here, not discovered later by some other task. + let result = raft_log .append_entries(vec![Entry { index: 1, term: 1, payload: None, }]) - .await - .unwrap(); - sleep(Duration::from_millis(20)).await; // let the IO thread process it + .await; + assert!( + result.is_err(), + "a persist_entries() failure must surface synchronously from append_entries()" + ); assert!( raft_log.is_poisoned(), @@ -1106,6 +1129,70 @@ async fn test_persist_entries_failure_poisons() { ); } +/// `IOTask::Persist`'s own `is_poisoned()` guard (top of its handler, on the +/// IO thread) is a *different* check from `append_entries()`'s caller-side +/// fast-fail (line ~471) — that one only protects writes submitted *after* +/// poisoning already happened. This test targets the IO-thread-side guard +/// specifically, for a `Persist` task that was already queued *before* the +/// log got poisoned by something else (e.g. a concurrent ReplaceRange/Purge +/// failure): send `IOTask::Persist` directly through `command_sender`, +/// bypassing `append_entries()` entirely. Uses a plain always-succeeds mock +/// (`with_id`, no call-count requirement) — if the IO-thread-side guard is +/// missing or removed, `persist_entries()` would run and `done` would carry +/// `Ok(())` instead of the expected "...poisoned..." error, which the +/// `other => panic!` arm below catches either way. +#[tokio::test] +async fn test_poisoned_rejects_queued_persist_task() { + let storage = Arc::new(MockStorageEngine::with_id( + "poisoned_rejects_queued_persist_task".into(), + )); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + strategy: PersistenceStrategy::MemFirst, + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + storage, + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + // Poisoned by something unrelated to this Persist task — simulated + // directly, same as the other `test_poisoned_skips_*` tests in this file. + raft_log.poisoned.store(true, Ordering::SeqCst); + + let (done_tx, done_rx) = tokio::sync::oneshot::channel(); + raft_log + .command_sender + .send(IOTask::Persist { + entries: vec![Entry { + index: 1, + term: 1, + payload: None, + }], + done: done_tx, + }) + .expect("IO thread must still be alive to receive the task"); + + let result = done_rx.await.expect("IO thread must reply, not drop the sender"); + match result { + Err(Error::Fatal(msg)) => assert!( + msg.contains("poisoned"), + "expected the poisoned short-circuit to fire before persist_entries() \ + was ever called, got: {msg}" + ), + other => panic!( + "expected Err(Fatal(\"...poisoned...\")), got: {other:?} — this means \ + the IO-thread-side is_poisoned() guard did not fire and \ + persist_entries() ran anyway", + ), + } +} + /// If `notify_fatal`'s underlying channel is already closed when a failure /// happens, the node must not fail *silently* — poisoned must still end up /// `true`, and the failure must be visible somewhere (log line), even though diff --git a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs new file mode 100644 index 00000000..e91d3cb5 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs @@ -0,0 +1,188 @@ +//! `persisted_index` must never claim a follower has written more to its +//! storage engine than what its log actually contains right now. +//! +//! Scenario: a follower has replicated entries 1-10 from an old leader and +//! synchronously written them to its storage engine (page cache), but hasn't +//! fsynced yet. A new leader is elected, finds entries 2-10 don't match its +//! own history, and tells the follower to truncate everything from index=2 +//! onward — the follower's real log now only has index=1. The new leader then +//! sends one brand-new entry that happens to land at index=2 again (different +//! content, new term). +//! +//! `persisted_index` only ever moves up (`fetch_max`), so without clamping it +//! on truncation, it would still remember "wrote up to 10" from before the +//! truncation — a stale high-water mark that the small index=2 write can't +//! pull back down. The next disk sync would then advertise `durable_index=10` +//! to the rest of the engine, even though the follower's log — and its +//! storage engine — genuinely only holds entries 1 and 2. A power loss at +//! that moment would prove the claim false: the follower reboots with only +//! [1, 2], not [1..=10], yet something upstream may already have acted on +//! "this follower is durable through 10" (e.g. deciding it's safe to purge +//! earlier log entries elsewhere in the cluster). + +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use crate::{ + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, + PersistenceStrategy, +}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// `durable_index()` must never exceed `last_entry_id()` — it must never +/// claim durability for an index that doesn't exist in the log anymore. +#[tokio::test] +async fn test_durable_index_never_exceeds_log_after_truncation_and_resync() { + let ctx = BufferedRaftLogTestContext::new( + PersistenceStrategy::MemFirst, + FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, // isolate from the safety-net timer + }, + "durable_index_never_exceeds_log_after_truncation_and_resync", + ); + + // Old leader (term=1) replicates entries 1..=10. append_entries() persists + // them to the storage engine synchronously, but nothing fsyncs them yet. + ctx.append_entries(1, 10, 1).await; + assert_eq!(ctx.raft_log.last_entry_id(), 10); + assert_eq!(ctx.raft_log.durable_index(), 0, "nothing fsynced yet"); + + // New leader (term=2) finds index=2 doesn't match its history (term=1 + // there, should be term=2) and truncates from index=2 onward, replacing + // it with one brand-new entry — real log becomes just [1, 2]. This goes + // through filter_out_conflicts_and_append's term-conflict slow path: + // remove_range(2..=MAX) (the clamp under test fires here, since it drops + // max_index from 10 down to 1) followed by inserting the new index=2. + ctx.raft_log + .filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]) + .await + .unwrap(); + assert_eq!( + ctx.raft_log.last_entry_id(), + 2, + "log truncated and replaced down to [1, 2]" + ); + + // Trigger a disk sync and give it time to complete. + ctx.raft_log.flush().await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + // The follower must never advertise durability for an index it doesn't + // actually have. If persisted_index wasn't clamped down during the + // truncation, this would report 10 here — a lie. + assert!( + ctx.raft_log.durable_index() <= ctx.raft_log.last_entry_id(), + "durable_index ({}) must never exceed last_entry_id ({}) — it must not \ + claim durability for entries the truncation already discarded", + ctx.raft_log.durable_index(), + ctx.raft_log.last_entry_id() + ); + assert_eq!( + ctx.raft_log.durable_index(), + 2, + "durable_index must reach the log's true end (2), not a stale pre-truncation watermark" + ); +} + +/// Different ordering from the test above: there, `remove_range`'s clamp ran +/// *before* anything else touched `persisted_index`. Here, a `Persist` task +/// dispatched *before* the truncation is still stuck on the IO thread (write +/// not yet reached the storage engine) when the truncation's own clamp runs — +/// and only *afterward* does that stale `Persist` complete and call +/// `persisted_index.fetch_max(10, ..)` (the line under review in +/// `handle_write_cmd`'s `IOTask::Persist` arm), using an index from entries +/// the truncation already discarded. `fetch_max` only ever moves up, so if +/// this call isn't fenced the same way `advance_durable_and_notify` fences a +/// stale fsync (see `truncation_fsync_fence_test.rs`), it silently +/// resurrects the clamp remove_range just applied. +/// +/// Scenario: +/// 1. Old leader (term=1) replicates entries 1..=10. `append_entries()` +/// inserts them into memory immediately, then blocks inside +/// `persist_entries()` on a gate — the write hasn't reached the storage +/// engine yet. +/// 2. New leader (term=2): index=2 conflicts. `filter_out_conflicts_and_append` +/// runs — `remove_range(2..=MAX)` (in-memory, synchronous, not routed +/// through the IO thread) executes and clamps immediately; the task then +/// blocks on the IO thread for its own queued `ReplaceRange`, which can't +/// run yet because the IO thread is still stuck on step 1's gate. +/// 3. Release the gate. The stale `Persist` for entries 1..=10 completes and +/// calls `persisted_index.fetch_max(10, ..)` — after the truncation's +/// clamp already ran, using entries that no longer exist. The queued +/// `ReplaceRange` runs next but does not re-clamp (its clamp already +/// fired once, in step 2, at truncation time). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_persisted_index_does_not_adopt_a_stale_persist_after_truncation() { + let (storage, persist_gate) = MockStorageEngine::not_durable_gated_persist( + "persisted_index_does_not_adopt_a_stale_persist_after_truncation".into(), + ); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + strategy: PersistenceStrategy::MemFirst, + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + let entries: Vec = (1..=10).map(|i| entry(i, 1)).collect(); + let append_task = { + let raft_log = raft_log.clone(); + tokio::spawn(async move { raft_log.append_entries(entries).await }) + }; + // Let append_task reach the gate inside persist_entries(). + tokio::time::sleep(Duration::from_millis(50)).await; + + let truncate_task = { + let raft_log = raft_log.clone(); + tokio::spawn(async move { + raft_log.filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]).await + }) + }; + // Let truncate_task run remove_range()'s synchronous clamp and reach + // its own await point (queued behind the still-gated Persist). + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + raft_log.last_entry_id(), + 2, + "remove_range()'s in-memory truncation must be visible immediately, \ + without waiting for the gated Persist or the queued ReplaceRange" + ); + + // Release the stale Persist — it completes and calls + // persisted_index.fetch_max(10, ..) using now-discarded entries. + persist_gate.send(()).expect("IO thread should still be waiting on the gate"); + append_task.await.unwrap().unwrap(); + truncate_task.await.unwrap().unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + assert!( + raft_log.persisted_index.load(Ordering::Acquire) <= raft_log.last_entry_id(), + "persisted_index ({}) must never exceed last_entry_id ({}) — the stale \ + Persist for entries 1..=10 must not be adopted after truncation shrank \ + the log to [1, 2]", + raft_log.persisted_index.load(Ordering::Acquire), + raft_log.last_entry_id() + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs new file mode 100644 index 00000000..3819e101 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs @@ -0,0 +1,102 @@ +//! Process-crash safety of entries counted toward quorum. +//! +//! `calculate_majority_matched_index` counts a leader's own write via +//! `last_entry_id()` — the in-memory SkipMap — as soon as `append_entries()` +//! returns. That's fine for power-loss safety (fsync is deliberately async, +//! see quorum_durability_test.rs) as long as the entry has at least reached the +//! storage engine (OS-managed page cache / WAL), which survives an ordinary +//! process crash even without fsync. +//! +//! These tests pin down whether `append_entries()` actually waits for the +//! storage engine (`LogStore::persist_entries`) before returning. Today it does +//! not — persistence happens later, asynchronously, on the IO thread — so an +//! entry can be quorum-eligible while a process crash between `append_entries()` +//! returning and the IO thread's next wakeup would lose it. + +use std::sync::Arc; +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::{ + BufferedRaftLog, FlushPolicy, LogStore, MockStorageEngine, MockTypeConfig, PersistenceConfig, + PersistenceStrategy, RaftLog, StorageEngine, +}; + +/// `append_entries()` must not return before the entry reaches the storage +/// engine — otherwise a quorum-eligible write exists only in memory and is +/// lost on an ordinary process crash (not just power loss). +/// +/// Gates `LogStore::persist_entries()` so it never completes during the test. +/// Today, `append_entries()` only inserts into the in-memory SkipMap and +/// notifies the IO thread — it does not call `persist_entries()` itself — so +/// it returns immediately regardless of the gate, and the storage engine never +/// sees the entry. After the fix, `append_entries()` must call +/// `persist_entries()` synchronously before returning, so with the gate closed +/// it must still be pending. +/// +/// Needs `flavor = "multi_thread"`: the gate blocks on a synchronous +/// `std::sync::mpsc::Receiver::recv()` inside `persist_entries()`, which now +/// runs directly on whichever task calls `append_entries()`. On the default +/// single-threaded runtime that would freeze the only executor thread — +/// including this test's own `sleep()` below — for the gate's entire +/// lifetime, an unrelated deadlock, not the behavior under test. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_append_entries_waits_for_storage_engine_before_returning() { + let (storage, persist_gate) = + MockStorageEngine::not_durable_gated_persist("append_waits_for_storage_engine".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + strategy: PersistenceStrategy::MemFirst, + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage.clone()), + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready + + let entry = Entry { + index: 1, + term: 1, + payload: None, + }; + + let append_task = tokio::spawn({ + let raft_log = raft_log.clone(); + async move { raft_log.append_entries(vec![entry]).await } + }); + + // Long enough that, if append_entries() were waiting on persist_entries(), + // it would still be pending; short enough to keep the suite fast. + tokio::time::sleep(Duration::from_millis(100)).await; + + // FIXED: append_entries() now routes the write through the IO thread + // (IOTask::Persist + oneshot) and does not return until it completes — + // with the gate closed, it must still be pending. + assert!( + !append_task.is_finished(), + "append_entries() must not return before persist_entries() completes" + ); + + // Ground truth: query the storage engine directly, not raft_log's own + // SkipMap-backed accessor (which would show the entry regardless). + assert!( + storage.log_store().entry(1).await.unwrap().is_none(), + "entry must not be visible in the storage engine while persist_entries() is gated" + ); + + persist_gate.send(()).expect("IO thread should still be waiting on the gate"); + append_task.await.unwrap().unwrap(); + + // append_entries() only returns after persist_entries() completes now, so + // the entry must already be visible — no polling needed. + assert!( + storage.log_store().entry(1).await.unwrap().is_some(), + "entry must be in the storage engine once append_entries() returns" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs new file mode 100644 index 00000000..4f0beb2d --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs @@ -0,0 +1,88 @@ +//! `IOTask::ReplaceRange` (term-conflict truncation, see +//! `filter_out_conflicts_and_append`'s slow path) writes to the storage engine +//! synchronously and bumps `pending_max`, but is dispatched through the +//! `receiver.recv()` => `cmd => { handle_non_write_cmd(...) }` arm of the IO +//! thread's select loop — a branch that, unlike `run_batch_turn`, never calls +//! `fsync_coordinator.submit()`. If no further `append_entries()` call arrives +//! afterward (which would separately trigger a `run_batch_turn` via +//! `write_notify`), the replaced entries sit "written but never fsync-submitted" +//! indefinitely — nothing but the idle-timer safety net would ever flush them. +//! +//! This test pins down that gap: it disables the safety net (a very long +//! `idle_flush_interval_ms`) so only the normal notify-driven path could +//! possibly advance `durable_index`, then proves it never does after a +//! term-conflict truncation with no subsequent append. + +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use crate::{FlushPolicy, PersistenceStrategy}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// A term-conflict truncation (`ReplaceRange`) must eventually become durable +/// even if no `append_entries()` call follows it. +/// +/// Today it does not: `ReplaceRange` is handled outside `run_batch_turn`, so +/// nothing submits fsync for it. Only the idle-timer safety net would catch +/// this — and this test disables that timer (60s interval, well beyond the +/// test's wait window) to isolate the notify-driven path from the safety net. +/// +/// RED (today): `durable_index()` never reaches `last_entry_id()` after the +/// truncation, because the replaced entries' fsync was never submitted. +#[tokio::test] +async fn test_replace_range_becomes_durable_without_a_following_append() { + let ctx = BufferedRaftLogTestContext::new( + PersistenceStrategy::MemFirst, + FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, // effectively disabled for this test's timeframe + }, + "replace_range_becomes_durable_without_a_following_append", + ); + + // Arrange: log [1,2,3] all term=1, explicitly flushed durable. + ctx.append_entries(1, 3, 1).await; + ctx.raft_log.flush().await.unwrap(); + assert_eq!(ctx.raft_log.durable_index(), 3, "baseline must be durable"); + + // Act: leader (term=2) sends entries that conflict at index=2 and extend + // the log to index=4. filter_out_conflicts_and_append's slow path detects + // the term mismatch at index=2, truncates [2,3], and replaces with + // [2,3,4] (term=2) via IOTask::ReplaceRange — with no append_entries() + // call afterward. + let result = ctx + .raft_log + .filter_out_conflicts_and_append(1, 1, vec![entry(2, 2), entry(3, 2), entry(4, 2)]) + .await + .unwrap(); + assert_eq!(result.unwrap().index, 4); + assert_eq!( + ctx.raft_log.last_entry_id(), + 4, + "memory must reflect the replace" + ); + + // Give the IO thread ample time to have submitted fsync, if anything + // besides the (disabled) safety net were going to do it. + tokio::time::sleep(Duration::from_millis(200)).await; + + // FIXED: ReplaceRange's handler now submits fsync directly instead of + // relying on a following append/notify or the safety net. + assert_eq!( + ctx.raft_log.durable_index(), + 4, + "ReplaceRange must submit fsync itself, without needing a following append" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs new file mode 100644 index 00000000..c2532224 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs @@ -0,0 +1,101 @@ +//! `FsyncCoordinator`'s `generation` fence protects against a physical fsync +//! whose result arrives after the world it was syncing no longer exists — but +//! today only `reset()` (full wipe) bumps `generation` via `fence_reset()`. +//! Term-conflict truncation (`filter_out_conflicts_and_append`'s slow path, +//! `remove_range` + `IOTask::ReplaceRange`) does not. +//! +//! Scenario: a follower has 10 entries synchronously written to its storage +//! engine but not yet fsynced — a physical fsync for "up to index 10" is +//! already dispatched and running in the background. Before that fsync +//! returns, a new leader tells the follower its log from index=2 onward is +//! wrong; the follower truncates and replaces it, ending up with only +//! entries [1, 2]. The in-flight fsync — which has no way to know any of +//! this happened — then completes and reports "index 10 is durable" anyway. +//! `durable_index` only moves up (`fetch_max`), so nothing afterward can +//! correct this: `durable_index()` gets stuck claiming durability for +//! entries [3..=10], which don't exist in this follower's log anymore. + +use std::sync::Arc; +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::storage::raft_log::RaftLog; +use crate::{ + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, + PersistenceStrategy, +}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// `durable_index()` must never exceed `last_entry_id()` — a follower must +/// never claim durability for log entries a truncation has already discarded. +/// +/// RED (today): the stale in-flight fsync (dispatched for index=10, before +/// the truncation) is not fenced, and blindly advances `durable_index` to 10 +/// after the truncation has already shrunk the log to [1, 2]. +#[tokio::test] +async fn test_durable_index_does_not_adopt_a_stale_fsync_after_truncation() { + // Gate closed: the first flush() call — for the original 10-entry batch — + // blocks here until we release it, letting us deterministically truncate + // the log while that fsync is still "in flight". + let (storage, flush_gate) = MockStorageEngine::not_durable_gated_flush( + "durable_index_does_not_adopt_a_stale_fsync_after_truncation".into(), + ); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + strategy: PersistenceStrategy::MemFirst, + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready + + // Old leader (term=1) replicates entries 1..=10. append_entries() persists + // them synchronously; write_notify then wakes the IO thread, which + // dispatches a physical fsync for "up to index=10" — that fsync is now + // running in the background, blocked on flush_gate. + let entries: Vec = (1..=10).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(entries).await.unwrap(); + + // Give the IO thread + blocking task time to reach the gated flush() call. + tokio::time::sleep(Duration::from_millis(50)).await; + + // New leader (term=2): index=2 conflicts, truncate and replace — the + // stale fsync (still blocked on the gate) has no way to observe this. + raft_log.filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]).await.unwrap(); + assert_eq!( + raft_log.last_entry_id(), + 2, + "log must be truncated and replaced down to [1, 2] before the stale fsync completes" + ); + + // Release the gate — the stale fsync (dispatched for index=10, before the + // truncation) now completes. + flush_gate.send(()).unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + assert!( + raft_log.durable_index() <= raft_log.last_entry_id(), + "durable_index ({}) must never exceed last_entry_id ({}) — the stale \ + fsync for index=10 must not be adopted after truncation shrank the \ + log to [1, 2]", + raft_log.durable_index(), + raft_log.last_entry_id() + ); +} diff --git a/d-engine-core/src/storage/fsync_coordinator.rs b/d-engine-core/src/storage/fsync_coordinator.rs index 47b74313..603a0c01 100644 --- a/d-engine-core/src/storage/fsync_coordinator.rs +++ b/d-engine-core/src/storage/fsync_coordinator.rs @@ -19,7 +19,11 @@ pub(super) struct FsyncCoordinator { inflight: AtomicBool, pending_max: AtomicU64, pending_replies: Mutex>>>, - generation: AtomicU64, // Bumped on every reset; fences out stale in-flight fsync results. + + // Fencing token (like Raft's `term`) for in-flight fsync results. Private — + // only bump via a fence_*() verb below, one per invalidating event. Never a + // value-passing variant (index math can under-fence, see fence_truncation()). + generation: AtomicU64, } impl FsyncCoordinator { @@ -156,13 +160,16 @@ impl FsyncCoordinator { } } + fn bump_generation(&self) { + self.generation.fetch_add(1, Ordering::AcqRel); + } + /// Called from reset_internal() before clearing in-memory state. /// Bumps generation to fence the in-flight physical flush (if any), /// AND drains anything already queued but not yet picked up by a /// flush round — that queued data was submitted before reset and /// must not be silently adopted by the next round. pub(super) fn fence_reset(&self) { - self.generation.fetch_add(1, Ordering::AcqRel); self.pending_max.store(0, Ordering::Release); let stale = std::mem::take(&mut *self.pending_replies.lock().unwrap()); for reply in stale { @@ -170,6 +177,22 @@ impl FsyncCoordinator { "stale fsync generation, superseded by reset".into(), ))); } + self.bump_generation(); + } + + /// Called from `remove_range()` before a truncation is applied. Bumps + /// `generation` to fence any fsync already in flight for data this + /// truncation is about to discard — mirrors `fence_reset()`, but does + /// NOT touch `pending_max`/`pending_replies`: unlike a full reset, + /// a truncation's own `IOTask::ReplaceRange` handler submits a fresh, + /// correct `max_index` for the surviving log right after this runs, + /// so there is nothing stale left to drain. + pub(super) fn fence_truncation( + &self, + new_max: u64, + ) { + self.pending_max.fetch_min(new_max, Ordering::AcqRel); + self.bump_generation(); } } diff --git a/d-engine-core/src/storage/fsync_coordinator_test.rs b/d-engine-core/src/storage/fsync_coordinator_test.rs index 76cf0bf7..32f21ffd 100644 --- a/d-engine-core/src/storage/fsync_coordinator_test.rs +++ b/d-engine-core/src/storage/fsync_coordinator_test.rs @@ -341,6 +341,9 @@ fn test_run_until_caught_up_advances_durable_index_on_success() { ); let coord = FsyncCoordinator::new(); let raft_log = minimal_raft_log(storage); + // advance_durable_and_notify() clamps against max_index — this test + // simulates a log that already has 5 entries, matching pending_max below. + raft_log.set_max_index_for_test(5); coord.inflight.store(true, Ordering::Release); coord.pending_max.store(5, Ordering::Release); @@ -461,6 +464,53 @@ fn test_run_until_caught_up_discards_stale_generation_result_without_advancing() ); } +/// The other half of the fence: when `generation` at completion still +/// matches `generation` at the round's start (nothing fenced it while the +/// physical flush was running), the result must be accepted normally — +/// `durable_index` advances and queued replies resolve to `Ok`. +/// +/// Bumps `generation` twice before the round starts (so this isn't just +/// "stays at the default 0"), proving it's the *match*, not the specific +/// value, that matters. +/// +/// Expected: +/// - `durable_index()` advances to the round's `max_index`. +/// - The queued reply resolves to `Ok(())`. +#[test] +fn test_run_until_caught_up_accepts_result_when_generation_unchanged() { + let (storage, _flush_call_count) = MockStorageEngine::not_durable( + "run_until_caught_up_accepts_result_when_generation_unchanged".into(), + ); + let coord = FsyncCoordinator::new(); + let raft_log = minimal_raft_log(storage); + // advance_durable_and_notify() clamps against max_index — matches pending_max below. + raft_log.set_max_index_for_test(5); + + // Two unrelated fences happened earlier — generation is 2, not 0 — before + // this round is even recorded as in flight. + coord.fence_reset(); + coord.fence_reset(); + assert_eq!(coord.generation.load(Ordering::Acquire), 2); + + let (tx, mut rx) = oneshot::channel::>(); + coord.inflight.store(true, Ordering::Release); + coord.pending_max.store(5, Ordering::Release); + coord.pending_replies.lock().unwrap().push(tx); + + // Nothing fences this round while it runs — generation stays at 2. + coord.run_until_caught_up(&raft_log); + + assert_eq!( + raft_log.durable_index.load(Ordering::Acquire), + 5, + "a matching generation must let the result advance durable_index normally" + ); + assert!( + rx.try_recv().expect("reply must have been answered").is_ok(), + "a matching generation must resolve queued replies as Ok, not Err" + ); +} + /// Multiple `submit()` calls made while a round is in flight are coalesced /// into a single subsequent physical `flush()` call by the same task — not /// one physical flush per `submit()` call. @@ -476,6 +526,8 @@ fn test_run_until_caught_up_coalesces_queued_submits_into_one_flush() { ); let coord = FsyncCoordinator::new(); let raft_log = minimal_raft_log(storage); + // advance_durable_and_notify() clamps against max_index — matches pending_max below. + raft_log.set_max_index_for_test(10); // Simulate two submit() calls that both lost the CAS while a round was // in flight — both just accumulated into the same pending state. diff --git a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs index b21f91da..7990cf58 100644 --- a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs +++ b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs @@ -635,6 +635,57 @@ impl MockStorageEngine { (engine, tx) } + /// Create a MockStorageEngine where the first `persist_entries()` call blocks + /// until the returned sender fires. `flush()`/`is_write_durable()` are left at + /// their always-succeeds default (`configure_durable`) — this gate is only + /// about the write-to-storage-engine step, not fsync. + /// + /// Use this to make the ordering between `append_entries()` returning and the + /// entry actually reaching the storage engine deterministic (no sleep/race) — + /// see `process_crash_safety_test.rs`. + pub fn not_durable_gated_persist(id: String) -> (Self, std::sync::mpsc::Sender<()>) { + let (tx, rx) = std::sync::mpsc::channel::<()>(); + let rx = Mutex::new(Some(rx)); + + let mut mock_log_store = MockLogStore::new(); + let mut mock_meta_store = MockMetaStore::new(); + + Self::configure_mocks(&mut mock_log_store, &mut mock_meta_store, &id); + // persist_entries is gated below instead of via configure_persist_entries_success. + Self::configure_replace_range_success(&mut mock_log_store, &id); + Self::configure_purge_success(&mut mock_log_store); + Self::configure_reset_success(&mut mock_log_store, &id); + Self::configure_save_hard_state_success(&mut mock_meta_store, &id); + Self::configure_durable(&mut mock_log_store); + + let instance_id_ref = id.clone(); + mock_log_store.expect_persist_entries().returning(move |entries| { + // Only the first call blocks — take() leaves None for subsequent calls. + if let Some(gate) = rx.lock().unwrap().take() { + let _ = gate.recv(); // blocks until the test sends () + } + let mut data = MOCK_STORAGE_DATA.lock().unwrap(); + for entry in &entries { + let key = format!("{instance_id_ref}_entry_{}", entry.index); + let value = bincode::serialize(entry).unwrap(); + data.insert(key, value); + } + if let Some(last_entry) = entries.last() { + let key = format!("{instance_id_ref}_last_index"); + data.insert(key, last_entry.index.to_be_bytes().to_vec()); + } + Ok(()) + }); + + let engine = Self { + log_store: Arc::new(mock_log_store), + meta_store: Arc::new(mock_meta_store), + instance_id: id, + }; + + (engine, tx) + } + /// Configure `is_write_durable=true` and no-op flush (durable mock). fn configure_durable(log_store: &mut MockLogStore) { log_store.expect_is_write_durable().returning(|| true); diff --git a/d-engine-core/src/watch/mod.rs b/d-engine-core/src/watch/mod.rs index 0998f2c0..32fcc4ba 100644 --- a/d-engine-core/src/watch/mod.rs +++ b/d-engine-core/src/watch/mod.rs @@ -87,6 +87,7 @@ //! watcher_buffer_size: 256, //! enable_metrics: true, //! max_watcher_count: 5000, +//! heartbeat_interval_ms: 30_000, //! }; //! ``` //! diff --git a/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs b/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs index b54a1080..666f1146 100644 --- a/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs +++ b/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs @@ -91,9 +91,20 @@ async fn test_snapshot_transfer_does_not_block_apply() -> Result<(), Box RETAINED_LOGS=8 (checked below), no node's log can cross a - // purge boundary before these writes land, so nothing earlier in the buffer can - // satisfy the match below — no extra gate is needed to make `since` safe. - // // This used to gate `since` on a `wait_for_snapshot` directory scan for the leader's // `.gz` file first, on the theory that "file exists" proves the snapshot is built. // It doesn't: `compress_directory` (default_state_machine_handler.rs) calls @@ -196,7 +202,6 @@ push_queue_size = 1 // made this test flake under CI load: RocksDB checkpoint export + tar/gzip // compression + metadata persist is genuinely sequential disk+CPU work that slows // down under contention. - let since = logs.lock().unwrap().len(); // The retained-log purge boundary — the actual signal this test needs, not just "a // snapshot file exists" (see comment above for why those differ). Emitted by @@ -208,14 +213,29 @@ push_queue_size = 1 // SNAPSHOT_THRESHOLD=64 > RETAINED_LOGS=8, the earliest possible snapshot on this // cluster already has last_included.index >= 64, so purge_upto_index is always > 0 // by the time this log line can appear at all. + // 60 x 500ms = 30s, not 15s: this test lives in the `multi-node-cluster-local` + // nextest group (throttled but not serialized, see .config/nextest.toml), and the + // log line polled below shares a process-global Mutex> with every other + // concurrently-running test in this binary (see log_capture.rs). Under full-suite + // load, RocksDB checkpoint export + tar/gzip (genuinely sequential CPU+disk work, + // see comment above) plus that shared-mutex contention can push real completion past + // 15s even though nothing is actually wrong — same root cause already documented in + // stress_test.rs's 30s bound. let mut purged = false; - for _ in 0..30 { + for _ in 0..60 { if logs_contain_globally_since(&logs, since, "purge_upto_index=") { purged = true; break; } tokio::time::sleep(Duration::from_millis(500)).await; } + if !purged { + eprintln!("=== DEBUG: captured logs since baseline writes ==="); + for line in logs.lock().unwrap()[since..].iter() { + eprintln!("{line}"); + } + eprintln!("=== END DEBUG ==="); + } assert!( purged, "Leader never logged a completed log purge — node 4 joining now would prove \ diff --git a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs index 0368f99a..453e17e7 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs @@ -225,16 +225,26 @@ async fn test_performance_benchmarks() { // Adjust test parameters according to the environment let operations = if is_ci { // CI environment uses a more relaxed threshold + // + // append_entries now round-trips through a dedicated IO thread via + // oneshot (see #444: leader's own write must reach the storage + // engine before counting toward quorum) — this is an intentional + // correctness/speed tradeoff, not a regression. The old threshold + // (500) predates that fix. New floor leaves ~2x headroom below the + // observed ~318-328 ops/sec on a modern dev machine, keeping the + // 2:1 local:CI ratio from before. [ - ("append_entries", 500, 500.0), + ("append_entries", 500, 100.0), ("get_entries_range", 2500, 25000.0), ("entry_lookup", 5000, 100000.0), ("term_queries", 4000, 25000.0), ] } else { // Local environment uses a stricter threshold + // + // See CI-branch comment above — same #444 rationale. [ - ("append_entries", 1000, 1000.0), + ("append_entries", 1000, 200.0), ("get_entries_range", 5000, 50000.0), ("entry_lookup", 10000, 200000.0), ("term_queries", 8000, 50000.0), diff --git a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs index f89ae55a..44972c8a 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs @@ -123,8 +123,15 @@ async fn test_high_concurrency_mixed_operations() { // Verify data integrity assert_eq!(ctx.raft_log.len(), 10000); + // append_entries() now round-trips through a dedicated IO thread via + // oneshot (see #444: leader's own write must reach the storage engine + // before counting toward quorum) — an intentional correctness/speed + // tradeoff, not a regression. The old 10s bound predates that fix; + // observed wall-clock for this test's 10k concurrent writes is now + // 13-18s depending on machine load. New bound leaves real headroom + // above that range rather than chasing the exact number. assert!( - duration < Duration::from_secs(10), + duration < Duration::from_secs(30), "Operations took too long: {duration:?}" ); } diff --git a/examples/single-node-expansion/Makefile b/examples/single-node-expansion/Makefile index a31b2775..a00218c4 100644 --- a/examples/single-node-expansion/Makefile +++ b/examples/single-node-expansion/Makefile @@ -8,12 +8,35 @@ # =============================== LOG_LEVEL ?= debug + +# On macOS with Homebrew: auto-detect compression lib paths to skip bundled C++ +# compilation of RocksDB dependencies, which fails under macOS 26 + Xcode 26 +# (Clang 16 lacks __builtin_ctzg/__builtin_clzg from LLVM 18+ SDK headers). +# brew --prefix resolves correctly on both Apple Silicon (/opt/homebrew) and +# Intel Mac (/usr/local). Silently no-ops when brew or a lib is absent. +SNAPPY_PREFIX := $(shell brew --prefix snappy 2>/dev/null) +LZ4_PREFIX := $(shell brew --prefix lz4 2>/dev/null) +ZSTD_PREFIX := $(shell brew --prefix zstd 2>/dev/null) +BREW_ROCKSDB_ENV := + +ifneq ($(SNAPPY_PREFIX),) +ifneq ($(wildcard $(SNAPPY_PREFIX)/lib),) + BREW_ROCKSDB_ENV += SNAPPY_LIB_DIR=$(SNAPPY_PREFIX)/lib +endif +endif +ifneq ($(LZ4_PREFIX),) + BREW_ROCKSDB_ENV += LZ4_LIB_DIR=$(LZ4_PREFIX)/lib +endif +ifneq ($(ZSTD_PREFIX),) + BREW_ROCKSDB_ENV += ZSTD_LIB_DIR=$(ZSTD_PREFIX)/lib +endif + # =============================== # Build Targets # =============================== build: @echo "Building release binary..." - cargo build --release --jobs 4 + $(BREW_ROCKSDB_ENV) cargo build --release --jobs 4 # =============================== # Single Node Bootstrap diff --git a/examples/single-node-expansion/config/n1.toml b/examples/single-node-expansion/config/n1.toml index bdb78715..712a3935 100644 --- a/examples/single-node-expansion/config/n1.toml +++ b/examples/single-node-expansion/config/n1.toml @@ -14,6 +14,8 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 +cmd_channel_capacity = 1024 +ordered_channel_capacity = 1024 [raft.election] election_timeout_min = 1000 @@ -23,27 +25,54 @@ election_timeout_max = 2000 default_policy = "LeaseRead" lease_duration_ms = 500 +[raft.read_actor] +channel_capacity = 10240 +max_drain = 2000 + +[raft.batching] +# Maximum number of commands to accumulate in a single batch during drain operations +max_batch_size = 200 + +[raft.metrics] +enable_backpressure = false +enable_batch = false + +[raft.backpressure] +max_pending_writes = 1000 +max_pending_reads = 500 + + [raft.persistence] +# strategy = "DiskFirst" strategy = "MemFirst" -flush_policy = { Batch = { idle_flush_interval_ms = 20 } } +flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } +# Maximum number of log entries to buffer in memory +# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] -enable = false -max_log_entries_before_snapshot = 10000 -retained_log_entries = 3 +enable = true +max_log_entries_before_snapshot = 5000 +retained_log_entries = 100 +cleanup_retain_count = 100 -# == Network Control Plane == +# == TTL Lease Configuration == +[raft.state_machine.lease] +cleanup_interval_ms = 1000 +max_cleanup_duration_ms = 1 + +# == Network Control Plane (voting, heartbeat, etc.) == [network.control] connection_window_size = 4_194_304 stream_window_size = 2_097_152 tcp_keepalive_in_secs = 60 -http2_keep_alive_interval_in_secs = 15 -http2_keep_alive_timeout_in_secs = 10 +http2_keep_alive_interval_in_secs = 15 # Slightly increase to reduce frequent keep-alives +http2_keep_alive_timeout_in_secs = 10 # Increase timeout +# New performance tuning parameters -# == Network Data Plane == +# == Network Data Plane (append_entries, etc.) == [network.data] connect_timeout_in_ms = 100 request_timeout_in_ms = 300 @@ -51,13 +80,16 @@ connection_window_size = 8_388_608 stream_window_size = 4_194_304 tcp_keepalive_in_secs = 60 -http2_keep_alive_interval_in_secs = 15 +http2_keep_alive_interval_in_secs = 15 # Same as control plane http2_keep_alive_timeout_in_secs = 10 - +# New data plane optimizations # == Server Transport (single listener serving every RPC type) == [network.server] concurrency_limit_per_connection = 100 # Increased for higher concurrent replication load max_concurrent_streams = 4096 # Increased to reduce stream creation overhead max_pending_accept_reset_streams = 2000 # Higher pending stream limit for Rapid Reset mitigation + +[storage] +unified_db = false diff --git a/examples/three-nodes-standalone/docker/Dockerfile b/examples/three-nodes-standalone/docker/Dockerfile index df52fc4b..e971495e 100644 --- a/examples/three-nodes-standalone/docker/Dockerfile +++ b/examples/three-nodes-standalone/docker/Dockerfile @@ -55,6 +55,8 @@ RUN apt-get update && \ iptables \ iproute2 \ libc6 \ + libfuse3-3 \ + fuse3 \ tzdata && \ rm -rf /var/lib/apt/lists/* && \ mkdir -p /var/run/sshd && \ @@ -85,4 +87,4 @@ COPY examples/three-nodes-standalone/docker/monitoring/promtail/config.yml /etc/ WORKDIR /app -CMD ["sh", "-c", "/usr/sbin/sshd -D & CONFIG_PATH=$CONFIG_PATH LOG_DIR=$LOG_DIR METRICS_PORT=$METRICS_PORT RUST_LOG=demo=$LOG_LEVEL,d_engine=$LOG_LEVEL,hyper=warn,sled=warn demo & promtail --config.file=/etc/promtail/config.yml > /app/logs/promtail.log 2>&1"] +CMD ["sh", "-c", "/usr/sbin/sshd -D & CONFIG_PATH=$CONFIG_PATH LOG_DIR=$LOG_DIR METRICS_PORT=$METRICS_PORT DB_PATH=/app/db/$ID RUST_LOG=demo=$LOG_LEVEL,d_engine=$LOG_LEVEL,hyper=warn,sled=warn demo & promtail --config.file=/etc/promtail/config.yml > /app/logs/promtail.log 2>&1"] diff --git a/examples/three-nodes-standalone/src/main.rs b/examples/three-nodes-standalone/src/main.rs index 4a0fd403..de1e1f83 100644 --- a/examples/three-nodes-standalone/src/main.rs +++ b/examples/three-nodes-standalone/src/main.rs @@ -17,7 +17,7 @@ use tracing_subscriber::Layer; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; -#[tokio::main(flavor = "multi_thread", worker_threads = 4)] +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] async fn main() { let log_dir = env::var("LOG_DIR") .map_err(|_| "LOG_DIR environment variable not set") @@ -58,8 +58,11 @@ async fn main() { let (graceful_tx, graceful_rx) = watch::channel(()); // Start the server (wait for its initialization to complete) - let server_handler = - tokio::spawn(start_dengine_server(data_dir, config_path, graceful_rx.clone())); + let server_handler = tokio::spawn(start_dengine_server( + data_dir, + config_path, + graceful_rx.clone(), + )); // Wait for the server to initialize (adjust the waiting time according to the actual logic) tokio::time::sleep(Duration::from_secs(1)).await; From 340b518912151d4f5971b1961663ae4f0aefcb33 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:36:55 +0800 Subject: [PATCH 02/11] fix #446: gate commit quorum and follower ACKs on fsync-durable index (RPO=0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Leader's quorum contribution now uses durable_index, not last_entry_id (including single-voter clusters, which previously fell back to last_entry_id per fix #329 — RPO=0 is now mandatory there too). - Follower/learner AppendEntries ACKs are withheld until the node's own durable_index catches up (new PendingAck, released on LogFlushed). - Rewrote the gRPC AppendEntries forwarder (FuturesUnordered, no strict FIFO) to remove the head-of-line blocking that withheld ACKs would otherwise cause; added stuck-send detection (error log + metric). - Renamed → ; removed the dead single-variant / config and its example/bench TOML references. - Test coverage: quorum-durability unit tests, pending-ack dedup/boundary/ role-transition-drop-safety, forwarder ordering end-to-end, and a real- disk crash + quorum composition test. - Updated CHANGELOG and the throughput-optimization-guide for the new ack-latency-not-data-loss framing. --- CHANGELOG.md | 23 ++ benches/embedded-bench/config/n1.toml | 2 - benches/embedded-bench/config/n2.toml | 2 - benches/embedded-bench/config/n3.toml | 2 - benches/reports/v0.2.5/bench_report_v0.2.5.md | 3 +- d-engine-core/src/config/raft.rs | 53 +-- d-engine-core/src/lib.rs | 5 + d-engine-core/src/raft_role/follower_state.rs | 16 + .../src/raft_role/follower_state_test.rs | 335 +++++++++++++++++- d-engine-core/src/raft_role/leader_state.rs | 15 +- .../single_voter_commit_test.rs | 75 ++-- d-engine-core/src/raft_role/learner_state.rs | 13 + .../src/raft_role/learner_state_test.rs | 102 +++++- d-engine-core/src/raft_role/role_state.rs | 85 ++++- .../src/storage/buffered_raft_log.rs | 16 +- .../basic_operations_test.rs | 28 +- .../concurrent_fsync_test.rs | 72 ++-- .../concurrent_operations_test.rs | 5 +- .../drain_fsync_test.rs | 18 - .../durable_index_test.rs | 8 +- .../buffered_raft_log_test/edge_cases_test.rs | 6 +- .../flush_strategy_test.rs | 11 +- .../id_allocation_test.rs | 4 +- .../performance_test.rs | 51 +-- .../persisted_index_clamp_test.rs | 7 +- .../pipeline_overlap_test.rs | 4 +- .../process_crash_safety_test.rs | 3 +- .../quorum_durability_test.rs | 235 +++++++++--- .../raft_properties_test.rs | 10 +- .../remove_range_test.rs | 11 +- .../replace_range_fsync_test.rs | 3 +- .../buffered_raft_log_test/shutdown_test.rs | 8 +- .../buffered_raft_log_test/term_index_test.rs | 8 +- .../term_segments_test.rs | 3 +- .../truncation_fsync_fence_test.rs | 6 +- .../buffered_raft_log_test/worker_test.rs | 3 +- .../src/storage/fsync_coordinator_test.rs | 2 - d-engine-core/src/storage/raft_log.rs | 12 +- .../buffered_raft_log_test_helpers.rs | 11 +- .../src/network/grpc/grpc_raft_service.rs | 107 ++++-- .../network/grpc/grpc_raft_service_test.rs | 175 +++++++++ d-engine-server/src/node/builder_test.rs | 2 - .../src/test_utils/integration/mod.rs | 2 - d-engine-server/tests/common/mod.rs | 4 - .../crash_recovery_test.rs | 25 +- .../tests/storage_buffered_raft_log/mod.rs | 11 +- .../performance_test.rs | 9 +- .../quorum_crash_recovery_test.rs | 108 ++++++ .../storage_integration_test.rs | 3 +- .../storage_buffered_raft_log/stress_test.rs | 9 +- .../watch_membership_embedded.rs | 2 - .../docs/examples/three-nodes-standalone.md | 3 +- .../throughput-optimization-guide.md | 48 +-- .../server_guide/customize-storage-engine.md | 2 +- examples/single-node-expansion/config/n1.toml | 5 +- examples/single-node-expansion/config/n2.toml | 1 - examples/single-node-expansion/config/n3.toml | 1 - examples/sled-cluster/config/n1.toml | 2 - examples/sled-cluster/config/n2.toml | 2 - examples/sled-cluster/config/n3.toml | 2 - examples/three-nodes-embedded/README.md | 1 - .../three-nodes-standalone/config/n1.toml | 5 +- .../three-nodes-standalone/config/n2.toml | 5 +- .../three-nodes-standalone/config/n3.toml | 5 +- .../docker/config/n1.toml | 5 +- .../docker/config/n2.toml | 5 +- .../docker/config/n3.toml | 5 +- 67 files changed, 1269 insertions(+), 561 deletions(-) create mode 100644 d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d9012493..3299062d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,18 @@ All notable changes to this project will be documented in this file. returns immediately, and entries arriving during an in-flight fsync are coalesced into the same physical disk flush. Storage-level group commit is restored without artificial batching windows. +- **🛑 Client-acknowledged writes could be lost on correlated power loss (#446)**: Raft commit quorum + counted the leader's own log contribution using its in-memory tail (`last_entry_id()`), not its + fsync-confirmed position (`durable_index()`) — a write could reach a majority-looking commit index, + and be acknowledged to the client, before enough replicas had actually synced it to disk. If those + nodes then lost power before their next fsync, the acknowledged write was gone. Fixed: leader quorum + calculation, follower `AppendEntries` ACK timing (a follower now withholds its response until its own + `durable_index` reaches the acknowledged entry), and single-voter clusters (previously exempted from + this class of fix, see #329) all gate on `durable_index`. RPO=0 for acknowledged writes is now a + mandatory invariant. Net effect: write acknowledgment latency now includes fsync time on a quorum of + replicas — see [Throughput Optimization Guide](./d-engine/src/docs/performance/throughput-optimization-guide.md) + for tuning `idle_flush_interval_ms`. + ### Changed - **MSRV raised to Rust 1.89**: The `data_dir` startup lock (prevents two node processes from @@ -65,6 +77,17 @@ All notable changes to this project will be documented in this file. - **`NodeBuilder` is no longer public** — use `EmbeddedEngine::start_custom`/`StandaloneEngine::run_custom` to plug in a custom storage engine or state machine. See [Migration Guide](./MIGRATION_GUIDE.md) for details. +- **⚠️ `[raft] ordered_channel_capacity` renamed to `max_pending_append_responses`** (#446): Follows the + gRPC `AppendEntries` forwarder rewrite (`FuturesUnordered`-based, no longer strict-FIFO) that shipped + alongside the durability fix above. Old field name is silently ignored, not an error — update existing + configs to the new name to keep the setting in effect. + +- **⚠️ `[raft.persistence] strategy` removed** (#446): `PersistenceStrategy` was a single-variant enum + (`MemFirst`) left over from #268; its only meaning now lives in whether an entry has reached + `durable_index`, which is no longer a configurable choice. Existing configs setting `strategy = + "MemFirst"` or `"DiskFirst"` are silently ignored, not an error — remove the field, `flush_policy` + is the only persistence knob now. + --- ## [v0.2.4] - 2026-05-23 diff --git a/benches/embedded-bench/config/n1.toml b/benches/embedded-bench/config/n1.toml index 52055b93..8dec8c84 100644 --- a/benches/embedded-bench/config/n1.toml +++ b/benches/embedded-bench/config/n1.toml @@ -16,10 +16,8 @@ max_drain = 1024 max_batch_size = 200 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.metrics] diff --git a/benches/embedded-bench/config/n2.toml b/benches/embedded-bench/config/n2.toml index 56c9cf13..455effff 100644 --- a/benches/embedded-bench/config/n2.toml +++ b/benches/embedded-bench/config/n2.toml @@ -16,10 +16,8 @@ max_drain = 1024 max_batch_size = 200 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.metrics] diff --git a/benches/embedded-bench/config/n3.toml b/benches/embedded-bench/config/n3.toml index 701a1551..1296b03d 100644 --- a/benches/embedded-bench/config/n3.toml +++ b/benches/embedded-bench/config/n3.toml @@ -16,10 +16,8 @@ max_drain = 1024 max_batch_size = 200 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.metrics] diff --git a/benches/reports/v0.2.5/bench_report_v0.2.5.md b/benches/reports/v0.2.5/bench_report_v0.2.5.md index 8e2bcb69..a6e08570 100644 --- a/benches/reports/v0.2.5/bench_report_v0.2.5.md +++ b/benches/reports/v0.2.5/bench_report_v0.2.5.md @@ -59,7 +59,7 @@ _(v0.2.5: 6-round average; v0.2.4: 4-round average; v0.2.3: 4-round average (Lea _(v0.2.5: 5-round average; v0.2.4: 5-round average; v0.2.3: 5-round average. All manually collected. 2026-07-12: 4-round average (conns=200, clients=200, Docker monitoring stack stopped).)_ | **Scenario** | **Metric** | **v0.2.3** | **v0.2.4** | **v0.2.5** | **Δ (v0.2.4→v0.2.5)** | **0712** | **Δ (v0.2.5→0712)** | -| ------------------- | ----------- | ------------ | ------------ | ------------ | --------------------- | ------------ | ------------------- | +| ------------------- | ----------- | ------------ | ------------ | ------------ | --------------------- | ------------- | ------------------- | | Single Client Write | Throughput | 6,421 ops/s | 5,245 ops/s | 5,234 ops/s | stable | 9,450 ops/s | **+80.5%** ✅ | | | Avg Latency | 0.155 ms | 0.190 ms | 0.190 ms | stable | 0.105 ms | **-44.6%** ✅ | | | p99 Latency | 0.200 ms | 0.235 ms | 0.237 ms | stable | 0.223 ms | -6.0% → | @@ -214,7 +214,6 @@ read_actor_channel_capacity = 10240 read_actor_max_drain = 2000 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [raft.batching] diff --git a/d-engine-core/src/config/raft.rs b/d-engine-core/src/config/raft.rs index 78e3a333..8e03f2f5 100644 --- a/d-engine-core/src/config/raft.rs +++ b/d-engine-core/src/config/raft.rs @@ -79,11 +79,15 @@ pub struct RaftConfig { #[serde(default = "default_cmd_channel_capacity")] pub cmd_channel_capacity: usize, - /// Ordered channel capacity for stream_append_entries ordering - /// Controls buffering of response receivers in FIFO order - /// Default value is set via default_ordered_channel_capacity() function - #[serde(default = "default_ordered_channel_capacity")] - pub ordered_channel_capacity: usize, + /// Max in-flight AppendEntries requests on `stream_append_entries` that can be + /// dispatched to the Raft loop and awaiting their response at once. Once this many + /// are pending, the stream stops reading new requests until one completes — this + /// bounds memory/task growth if this node's own durable_index stalls (RPO=0, #446). + /// Also used directly as the output channel's buffer size, since completed + /// responses can never outnumber in-flight requests. + /// Default value is set via default_max_pending_append_responses() function + #[serde(default = "default_max_pending_append_responses")] + pub max_pending_append_responses: usize, /// ReadActor configuration — tuning for the dedicated Eventual/LeaseRead fast path. #[serde(default)] @@ -141,7 +145,7 @@ impl Default for RaftConfig { auto_join: AutoJoinConfig::default(), snapshot_rpc_timeout_ms: default_snapshot_rpc_timeout_ms(), cmd_channel_capacity: default_cmd_channel_capacity(), - ordered_channel_capacity: default_ordered_channel_capacity(), + max_pending_append_responses: default_max_pending_append_responses(), read_actor: ReadActorConfig::default(), read_consistency: ReadConsistencyConfig::default(), backpressure: BackpressureConfig::default(), @@ -201,7 +205,7 @@ fn default_cmd_channel_capacity() -> usize { 1024 } -fn default_ordered_channel_capacity() -> usize { +fn default_max_pending_append_responses() -> usize { 1024 } @@ -817,27 +821,6 @@ impl Default for PromotionConfig { fn default_stale_learner_threshold() -> Duration { Duration::from_secs(300) } -/// Defines how Raft log entries are persisted and accessed. -/// -/// All strategies use a configurable [`FlushPolicy`] to control when memory contents -/// are flushed to disk, affecting write latency and durability guarantees. -/// -/// **Note:** Both strategies now fully load all log entries from disk into memory at startup. -/// The in-memory `SkipMap` serves as the primary data structure for reads in all modes. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub enum PersistenceStrategy { - /// Memory-first persistence strategy. - /// - /// - **Write path**: On append, the log entry is first written to the in-memory `SkipMap` and - /// acknowledged immediately. Disk persistence happens asynchronously in the background, - /// governed by [`FlushPolicy`]. - /// - /// - **Read path**: Reads are always served from the in-memory `SkipMap`. - /// - /// - **Startup behavior**: All log entries are loaded from disk into memory at startup. - /// - MemFirst, -} /// Controls when in-memory logs should be flushed to disk. /// @@ -857,14 +840,6 @@ pub enum FlushPolicy { /// Configuration parameters for log persistence behavior #[derive(Serialize, Deserialize, Clone, Debug)] pub struct PersistenceConfig { - /// Strategy for persisting Raft logs - /// - /// This controls the trade-off between durability guarantees and performance - /// characteristics. The choice impacts both write throughput and recovery - /// behavior after node failures. - #[serde(default = "default_persistence_strategy")] - pub strategy: PersistenceStrategy, - /// Flush policy for asynchronous strategies /// /// This controls when log entries are flushed to disk. The choice impacts @@ -886,11 +861,6 @@ pub struct PersistenceConfig { pub shutdown_timeout_ms: u64, } -/// Default persistence strategy (optimized for balanced workloads) -fn default_persistence_strategy() -> PersistenceStrategy { - PersistenceStrategy::MemFirst -} - /// Default flush policy for asynchronous strategies /// /// This controls when log entries are flushed to disk. The choice impacts @@ -933,7 +903,6 @@ impl PersistenceConfig { impl Default for PersistenceConfig { fn default() -> Self { Self { - strategy: default_persistence_strategy(), flush_policy: default_flush_policy(), max_buffered_entries: default_max_buffered_entries(), shutdown_timeout_ms: default_shutdown_timeout_ms(), diff --git a/d-engine-core/src/lib.rs b/d-engine-core/src/lib.rs index 93333741..387324b2 100644 --- a/d-engine-core/src/lib.rs +++ b/d-engine-core/src/lib.rs @@ -173,6 +173,11 @@ pub(crate) fn if_higher_term_found( /// entries in the logs. If the logs have last entries with different terms, then the log with the /// later term is more up-to-date. If the logs end with the same term, then whichever log is longer /// is more up-to-date. +/// +/// #446: callers must pass the in-memory last-log-id (last_entry_id), never durable_index. +/// A node with an un-fsynced tail must still be able to reject a candidate whose log is +/// genuinely less up to date — voting eligibility and commit-durability are separate +/// concerns and must not share the same index source. pub(crate) fn is_target_log_more_recent( my_last_log_index: u64, my_last_log_term: u64, diff --git a/d-engine-core/src/raft_role/follower_state.rs b/d-engine-core/src/raft_role/follower_state.rs index c1e30ebf..f0dfde4e 100644 --- a/d-engine-core/src/raft_role/follower_state.rs +++ b/d-engine-core/src/raft_role/follower_state.rs @@ -7,6 +7,7 @@ use d_engine_proto::server::cluster::ClusterConfUpdateResponse; use d_engine_proto::server::cluster::LeaderDiscoveryResponse; use d_engine_proto::server::election::VoteResponse; use d_engine_proto::server::storage::SnapshotMetadata; +use std::collections::BTreeMap; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; @@ -43,6 +44,7 @@ use crate::RaftNodeConfig; use crate::Result; use crate::StateTransitionError; use crate::TypeConfig; +use crate::role_state::PendingAck; use crate::role_state::schedule_and_execute_purge; use crate::utils::cluster::error; use crate::utils::cluster_printer::print_role_transition_line; @@ -73,6 +75,10 @@ pub struct FollowerState { /// Last physically purged log index (inclusive) pub last_purged_index: Option, + /// AppendEntries responses withheld pending this node's own durable_index. + /// See `role_state::PendingAck`. + pending_append_acks: BTreeMap, + // -- Snapshot Management -- /// Prevents concurrent snapshot creation /// @@ -463,6 +469,12 @@ impl RaftRoleState for FollowerState { fn pending_purge_upto_mut(&mut self) -> Option<&mut Option> { Some(&mut self.pending_purge_upto) } + + fn pending_append_acks_mut( + &mut self + ) -> Option<&mut std::collections::BTreeMap> { + Some(&mut self.pending_append_acks) + } } impl FollowerState { @@ -484,6 +496,7 @@ impl FollowerState { node_config.raft.election.election_timeout_max, )), node_config, + pending_append_acks: BTreeMap::new(), snapshot_in_progress: AtomicBool::new(false), _marker: PhantomData, last_purged_index: None, @@ -511,6 +524,7 @@ impl From<&CandidateState> for FollowerState { )), node_config: candidate_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: candidate_state.last_purged_index, // scheduled_purge_upto: None, _marker: PhantomData, @@ -527,6 +541,7 @@ impl From<&LeaderState> for FollowerState { leader_state.node_config.raft.election.election_timeout_max, )), node_config: leader_state.node_config.clone(), + pending_append_acks: BTreeMap::new(), snapshot_in_progress: AtomicBool::new( leader_state.snapshot_in_progress.load(Ordering::SeqCst), ), @@ -548,6 +563,7 @@ impl From<&LearnerState> for FollowerState { )), node_config: learner_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: learner_state.last_purged_index, pending_purge_upto: learner_state.pending_purge_upto, _marker: PhantomData, diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index d6ab39a8..2516c7fd 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -994,6 +994,16 @@ async fn test_handle_append_entries_success_from_new_leader() { "Should update commit_index" ); + // RPO=0 (#446): the success ACK is withheld until durable_index reaches the claimed index. + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 1, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(1, &context, &flush_tx).await; + // Verify: Response with success=true let response = resp_rx.recv().await.expect("should receive response").unwrap(); assert!(response.is_success(), "Response should indicate success"); @@ -2885,12 +2895,12 @@ async fn test_follower_rejects_strong_consistency_reads() { // MemFirst ACK Tests // ============================================================================ -/// Follower ACKs leader immediately after memory write (MemFirst). +/// Follower withholds the AppendEntries ACK until its own durable_index catches up. /// -/// The IO thread continues to fsync asynchronously. Safety: before commit, -/// the leader's durable_index >= N (quorum uses durable_index). +/// RPO=0 (#446): an ACK asserts durability, so it must not go out before the +/// claimed index is fsynced. LogFlushed releases the withheld response. #[tokio::test] -async fn test_follower_acks_immediately_after_memory_write() { +async fn test_follower_withholds_ack_until_durable() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); @@ -2937,9 +2947,306 @@ async fn test_follower_acks_immediately_after_memory_write() { .is_ok() ); - // MemFirst: ACK sent immediately, no waiting for fsync - let response = resp_rx.try_recv().expect("ACK must be sent immediately after memory write"); - assert!(response.unwrap().is_success()); + // RPO=0: the ACK is withheld while durable_index < claimed index (5). + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 5, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(appended_index, &context, &flush_tx).await; + + let response = resp_rx.try_recv().expect("ACK must be released once durable").unwrap(); + assert!(response.is_success()); +} + +/// #446: if `FollowerState` is dropped (role transition — e.g. a higher-term +/// AppendEntries or becoming a candidate) while it still holds a withheld ACK, the +/// pending sender must be dropped with it — the caller waiting on `resp_rx` must see +/// the channel close, not hang forever and not receive a stale success. +/// +/// This is the actual mechanism #446's design relies on for role-transition safety +/// (see the design doc / ADR-042 discussion): a real role transition replaces +/// `self.role` wholesale, which drops the old `FollowerState` — including +/// `pending_append_acks` and every sender inside it. This test drops the struct +/// directly rather than driving a full role-transition workflow, because that's +/// exactly what a role transition does to it; nothing here relies on any other part +/// of the transition machinery. +#[tokio::test] +async fn test_dropping_follower_state_releases_pending_ack_senders_as_closed() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let leader_term = 2u64; + let appended_index = 5u64; + + let mut replication_handler = MockReplicationCore::new(); + replication_handler.expect_handle_append_entries().returning(move |_, _, _| { + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success( + 1, + leader_term, + Some(LogId { + term: leader_term, + index: appended_index, + }), + ), + commit_index_update: None, + }) + }); + context.handlers.replication_handler = replication_handler; + context.membership = Arc::new(MockMembership::new()); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state_mut().update_current_term(leader_term); + + let append_request = AppendEntriesRequest { + term: leader_term, + leader_id: 2, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); + + // Confirm the ACK is genuinely withheld (durable_index hasn't caught up) before + // dropping — otherwise this test wouldn't be exercising the pending-ack path at all. + assert!( + resp_rx.try_recv().is_err(), + "precondition: the ACK must still be withheld before the role transition" + ); + + // Simulates a real role transition: `self.role = self.role.become_xxx()?` drops + // the old FollowerState (and everything it owns) the same way this explicit + // drop does. + drop(state); + + // The withheld ACK's sender is gone — resp_rx must observe the channel closing, + // not hang forever and not receive a stale success response. + let result = tokio::time::timeout(std::time::Duration::from_secs(1), resp_rx.recv()) + .await + .expect("recv() must resolve promptly once the sender is dropped, not hang"); + assert!( + result.is_err(), + "dropping FollowerState must close the pending ACK's channel, not deliver a \ + stale response" + ); +} + +/// #446: two independent AppendEntries requests (e.g. a leader retry) that both claim +/// the same threshold index must both eventually receive a response — the second one +/// landing on `pending_append_acks` must not silently overwrite the first. +#[tokio::test] +async fn test_multiple_requests_at_same_threshold_all_receive_response() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let leader_term = 2u64; + let claimed_index = 5u64; + + let mut replication_handler = MockReplicationCore::new(); + replication_handler + .expect_handle_append_entries() + .times(2) + .returning(move |_, _, _| { + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success( + 1, + leader_term, + Some(LogId { + term: leader_term, + index: claimed_index, + }), + ), + commit_index_update: None, + }) + }); + context.handlers.replication_handler = replication_handler; + context.membership = Arc::new(MockMembership::new()); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state_mut().update_current_term(leader_term); + + let append_request = AppendEntriesRequest { + term: leader_term, + leader_id: 2, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + // First request lands on threshold=5 and gets withheld. + let (resp_tx1, mut resp_rx1) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(append_request.clone(), vec![resp_tx1]), + &context, + internal_event_tx.clone(), + ) + .await + .unwrap(); + + // A second, independent request (e.g. leader retry) also claims index 5. + let (resp_tx2, mut resp_rx2) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(append_request, vec![resp_tx2]), + &context, + internal_event_tx, + ) + .await + .unwrap(); + + assert!( + resp_rx1.try_recv().is_err(), + "first request must still be withheld" + ); + assert!( + resp_rx2.try_recv().is_err(), + "second request must still be withheld" + ); + + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(claimed_index, &context, &flush_tx).await; + + // Both senders — not just one — must receive a response. A single-value + // `BTreeMap` without `.entry().or_insert_with(...)` merging would + // let the second insert silently overwrite the first, dropping this ACK forever. + let response1 = resp_rx1.try_recv().expect("first sender must receive a response").unwrap(); + let response2 = resp_rx2 + .try_recv() + .expect("second sender must also receive a response, not be silently overwritten") + .unwrap(); + assert!(response1.is_success()); + assert!(response2.is_success()); +} + +/// #446: `LogFlushed` must release exactly the pending ACKs whose threshold is `<=` +/// durable_index — not `<` (off-by-one), and not all-or-nothing. +#[tokio::test] +async fn test_log_flushed_releases_only_thresholds_at_or_below_durable() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let leader_term = 2u64; + let call_count = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let call_count_clone = call_count.clone(); + + let mut replication_handler = MockReplicationCore::new(); + replication_handler + .expect_handle_append_entries() + .times(3) + .returning(move |_, _, _| { + let claimed = match call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst) { + 0 => 10, + 1 => 12, + _ => 15, + }; + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success( + 1, + leader_term, + Some(LogId { + term: leader_term, + index: claimed, + }), + ), + commit_index_update: None, + }) + }); + context.handlers.replication_handler = replication_handler; + context.membership = Arc::new(MockMembership::new()); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state_mut().update_current_term(leader_term); + + let base_request = AppendEntriesRequest { + term: leader_term, + leader_id: 2, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + let (tx10, mut rx10) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(base_request.clone(), vec![tx10]), + &context, + internal_event_tx.clone(), + ) + .await + .unwrap(); + let (tx12, mut rx12) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(base_request.clone(), vec![tx12]), + &context, + internal_event_tx.clone(), + ) + .await + .unwrap(); + let (tx15, mut rx15) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(base_request, vec![tx15]), + &context, + internal_event_tx, + ) + .await + .unwrap(); + + assert!(rx10.try_recv().is_err()); + assert!(rx12.try_recv().is_err()); + assert!(rx15.try_recv().is_err()); + + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + + // durable_index advances to 12 — releases 10 and 12 (boundary is <=, not <), 15 stays. + state.handle_log_flushed(12, &context, &flush_tx).await; + assert!( + rx10.try_recv() + .expect("threshold 10 <= durable 12, must be released") + .unwrap() + .is_success() + ); + assert!( + rx12.try_recv() + .expect("threshold 12 <= durable 12 (boundary case), must be released") + .unwrap() + .is_success() + ); + assert!( + rx15.try_recv().is_err(), + "threshold 15 > durable 12, must still be withheld" + ); + + // durable_index advances to 15 — releases the rest. + state.handle_log_flushed(15, &context, &flush_tx).await; + assert!( + rx15.try_recv() + .expect("threshold 15 <= durable 15, must now be released") + .unwrap() + .is_success() + ); } /// Follower sends ACK immediately for heartbeat (no entries). @@ -3042,8 +3349,18 @@ async fn test_follower_commit_index_and_ack_both_sent_immediately() { new_commit, "commit_index must advance immediately" ); - let response = resp_rx.try_recv().expect("ACK must be sent immediately"); - assert!(response.unwrap().is_success()); + // RPO=0: commit_index advances immediately, but the ACK is withheld until durable. + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 5, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(appended_index, &context, &flush_tx).await; + + let response = resp_rx.try_recv().expect("ACK must be released once durable").unwrap(); + assert!(response.is_success()); } /// Spawns a fake Worker that answers exactly one `InstallSnapshot` command with `result`, diff --git a/d-engine-core/src/raft_role/leader_state.rs b/d-engine-core/src/raft_role/leader_state.rs index 06b21cb8..f607bb06 100644 --- a/d-engine-core/src/raft_role/leader_state.rs +++ b/d-engine-core/src/raft_role/leader_state.rs @@ -1344,17 +1344,10 @@ impl RaftRoleState for LeaderState { internal_event_tx: &mpsc::UnboundedSender, ) { let new_commit_index = if self.cluster_metadata.single_voter { - // MemFirst single-voter: LogFlushed(durable) is the IO checkpoint. - // Commit to last_entry_id() — not just durable — to allow pipelining - // across IO batch boundaries. Matches multi-voter MemFirst where leader - // contributes last_entry_id() to quorum (not durable_index). - let last_log_index = ctx.raft_log().last_entry_id(); - debug_assert!( - last_log_index >= durable, - "last_entry_id ({last_log_index}) must be >= durable ({durable})" - ); - if last_log_index > self.commit_index() { - Some(last_log_index) + // RPO=0 (#446): single-voter has no majority to fall back on — commit + // must not advance past what this node has itself fsynced. + if durable > self.commit_index() { + Some(durable) } else { None } diff --git a/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs b/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs index 3b9b8d07..2f6e7ec1 100644 --- a/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs @@ -1,17 +1,16 @@ //! Single-Voter Commit Path Tests //! -//! Regression tests for the MemFirst single-voter commit path in `handle_log_flushed`. +//! RPO=0 (#446): `handle_log_flushed` single-voter branch must commit to `durable`, not +//! `last_entry_id()` — a single-voter cluster has no majority to fall back on, so if the +//! leader itself hasn't fsynced an entry, there is no copy anywhere safe from power loss. //! -//! ## Bug History -//! `fix #329` changed `handle_log_flushed` single-voter branch to commit to `durable` -//! instead of `last_entry_id()`. This placed IO thread latency on the commit critical -//! path, causing a ~3x latency regression in 3-node embedded bench (1731µs vs ~566µs). -//! -//! ## MemFirst Single-Voter Invariant -//! `LogFlushed(durable)` is an IO checkpoint. Commit must advance to `last_entry_id()` -//! — not just `durable` — to allow pipelining across IO batch boundaries. -//! This matches the multi-voter path where the leader contributes `last_entry_id()` to -//! quorum (not `durable_index`). +//! ## Superseded design (kept as history, do not resurrect) +//! `fix #329` changed this branch to commit to `durable` instead of `last_entry_id()`, +//! then reverted it after measuring a ~3x latency regression in 3-node embedded bench +//! (1731µs vs ~566µs) — IO thread latency landed on the commit critical path. That +//! regression is real and will resurface here. RPO=0 makes paying it mandatory for +//! single-voter clusters — there is no majority to absorb the risk the old design +//! accepted. use crate::MockMembership; use crate::MockRaftLog; @@ -61,16 +60,16 @@ async fn setup_single_voter( (state, ctx, last_entry_id) } -/// MemFirst single-voter: `handle_log_flushed` must commit to `last_entry_id`, not `durable`. +/// RPO=0: `handle_log_flushed` must commit to `durable`, not `last_entry_id`. /// -/// Simulates: IO batch flushed entries 1-3 (`durable=3`), but entries 4-5 arrived -/// in memory during the flush (`last_entry_id=5`). MemFirst: commit must advance -/// to 5 (all in-memory entries), not stall at 3 (only persisted entries). +/// Simulates: entries 4-5 arrived in memory (`last_entry_id=5`) but the IO batch has +/// only flushed entries 1-3 so far (`durable=3`). Commit must stay at 3 — entries 4-5 +/// aren't crash-safe yet, and a single-voter cluster has no other copy to fall back on. /// -/// This test FAILS if `handle_log_flushed` uses `durable` for commit -/// (the `fix #329` regression that caused +617µs avg latency in 3-node embedded bench). +/// This test FAILS if `handle_log_flushed` still uses `last_entry_id` for commit (the +/// old MemFirst behavior, since revoked — RPO=0 makes single-voter durability mandatory). #[tokio::test] -async fn test_single_voter_commit_uses_last_entry_id_not_durable() { +async fn test_single_voter_commit_uses_durable_not_last_entry_id() { // last_entry_id=5: entries 4-5 arrived in memory during the IO flush of 1-3 let (mut state, ctx, _last_entry_id) = setup_single_voter(5).await; let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); @@ -80,13 +79,13 @@ async fn test_single_voter_commit_uses_last_entry_id_not_durable() { assert_eq!( state.commit_index(), - 5, - "MemFirst single-voter: commit must use last_entry_id=5, not durable=3. \ - Using durable puts IO latency on the commit critical path." + 3, + "RPO=0: commit must use durable=3, not last_entry_id=5 — entries 4-5 aren't \ + fsynced yet, and single-voter has no majority to fall back on" ); } -/// After IO catches up (durable == last_entry_id), commit equals last_entry_id. +/// After IO catches up (durable == last_entry_id), commit equals durable. #[tokio::test] async fn test_single_voter_commit_when_durable_equals_last_entry_id() { let (mut state, ctx, _last_entry_id) = setup_single_voter(5).await; @@ -94,20 +93,16 @@ async fn test_single_voter_commit_when_durable_equals_last_entry_id() { state.handle_log_flushed(5, &ctx, &internal_event_tx).await; - assert_eq!( - state.commit_index(), - 5, - "commit must advance to last_entry_id=5 when durable=5" - ); + assert_eq!(state.commit_index(), 5, "commit must advance to durable=5"); } -/// Pipelining across multiple IO batches: each flush triggers commit to current last_entry_id. +/// Commit tracks `durable` across IO batches, not the in-memory tail. /// -/// Simulates rapid writes where IO batches lag behind in-memory log: -/// - Flush 1: IO flushed 1-3, log has 1-7 in memory → commit=7 -/// - Flush 2: IO flushed 4-7, log has 1-10 in memory → commit=10 +/// Simulates rapid writes where the in-memory log runs ahead of what's fsynced: +/// - Flush 1: IO flushed 1-3 (durable=3), memory has 1-7 → commit=3, not 7 +/// - Flush 2: IO flushed 4-7 (durable=7), memory now has 1-10 → commit=7, not 10 #[tokio::test] -async fn test_single_voter_pipelining_across_io_batches() { +async fn test_single_voter_commit_tracks_durable_not_memory_tail() { let (mut state, ctx, last_entry_id) = setup_single_voter(7).await; let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); @@ -115,8 +110,8 @@ async fn test_single_voter_pipelining_across_io_batches() { state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), - 7, - "commit must advance to last_entry_id=7" + 3, + "commit must stay at durable=3 — entries 4-7 aren't fsynced yet" ); // IO batch 2: flushed 4-7, memory now has 1-10 @@ -124,14 +119,14 @@ async fn test_single_voter_pipelining_across_io_batches() { state.handle_log_flushed(7, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), - 10, - "commit must advance to last_entry_id=10" + 7, + "commit must advance to durable=7, not the in-memory tail (10)" ); } -/// No-op flush: last_entry_id == commit_index means nothing new to commit. +/// No-op flush: durable == commit_index means nothing new is safe to commit yet. #[tokio::test] -async fn test_single_voter_no_commit_when_nothing_new() { +async fn test_single_voter_no_commit_when_nothing_new_durable() { let (mut state, ctx, _last_entry_id) = setup_single_voter(3).await; let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); @@ -139,11 +134,11 @@ async fn test_single_voter_no_commit_when_nothing_new() { state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!(state.commit_index(), 3); - // Second flush with same last_entry_id=3: no new entries → no commit advance + // Second flush with same durable=3: nothing new is fsynced → no commit advance state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), 3, - "commit must not advance when last_entry_id == commit_index" + "commit must not advance when durable == commit_index" ); } diff --git a/d-engine-core/src/raft_role/learner_state.rs b/d-engine-core/src/raft_role/learner_state.rs index 01c735b1..f465b3e8 100644 --- a/d-engine-core/src/raft_role/learner_state.rs +++ b/d-engine-core/src/raft_role/learner_state.rs @@ -22,6 +22,7 @@ use crate::alias::MOF; use crate::cluster_printer::print_learner_join_success; use crate::cluster_printer::print_learner_promoted_to_voter; use crate::cluster_printer::print_role_transition_line; +use crate::role_state::PendingAck; use crate::role_state::schedule_and_execute_purge; use async_trait::async_trait; use d_engine_proto::common::LogId; @@ -35,6 +36,7 @@ use d_engine_proto::server::cluster::LeaderDiscoveryResponse; use d_engine_proto::server::election::VoteResponse; use d_engine_proto::server::election::VotedFor; use d_engine_proto::server::storage::SnapshotMetadata; +use std::collections::BTreeMap; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; @@ -88,6 +90,10 @@ pub struct LearnerState { /// reflected in the latest snapshot. pub last_purged_index: Option, + /// AppendEntries responses withheld pending this node's own durable_index. + /// See `role_state::PendingAck`. + pending_append_acks: BTreeMap, + // -- Snapshot Management -- /// Prevents concurrent snapshot creation /// @@ -514,6 +520,10 @@ impl RaftRoleState for LearnerState { fn pending_purge_upto_mut(&mut self) -> Option<&mut Option> { Some(&mut self.pending_purge_upto) } + + fn pending_append_acks_mut(&mut self) -> Option<&mut BTreeMap> { + Some(&mut self.pending_append_acks) + } } impl LearnerState { @@ -537,6 +547,7 @@ impl LearnerState { shared_state: SharedState::new(node_id, None, None), last_purged_index: None, snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), node_config, _marker: PhantomData, pending_purge_upto: None, @@ -646,6 +657,7 @@ impl From<&FollowerState> for LearnerState { shared_state: follower_state.shared_state.clone(), node_config: follower_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: follower_state.last_purged_index, pending_purge_upto: follower_state.pending_purge_upto, _marker: PhantomData, @@ -658,6 +670,7 @@ impl From<&CandidateState> for LearnerState { shared_state: candidate_state.shared_state.clone(), node_config: candidate_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: candidate_state.last_purged_index, pending_purge_upto: None, _marker: PhantomData, diff --git a/d-engine-core/src/raft_role/learner_state_test.rs b/d-engine-core/src/raft_role/learner_state_test.rs index 7eaf7bc7..ce68c796 100644 --- a/d-engine-core/src/raft_role/learner_state_test.rs +++ b/d-engine-core/src/raft_role/learner_state_test.rs @@ -363,6 +363,16 @@ async fn test_learner_handles_append_entries_success() { assert_eq!(state.current_term(), leader_term); assert_eq!(state.commit_index(), expected_commit); + // RPO=0 (#446): the success ACK is withheld until durable_index reaches the claimed index. + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 1, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(1, &context, &flush_tx).await; + let response = resp_rx.recv().await.unwrap().unwrap(); assert!(response.is_success()); } @@ -1730,9 +1740,78 @@ async fn test_apply_completed_respects_snapshot_disabled_config() { // MemFirst ACK Tests // ============================================================================ -/// Learner ACKs leader immediately after memory write (MemFirst). +/// Learner withholds the AppendEntries ACK until its own durable_index catches up. +/// +/// RPO=0 (#446): an ACK asserts durability, so it must not go out before the +/// claimed index is fsynced. LogFlushed releases the withheld response. +#[tokio::test] +async fn test_learner_withholds_ack_until_durable() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let leader_term = 2u64; + let appended_index = 5u64; + + let mut replication_handler = crate::MockReplicationCore::new(); + replication_handler.expect_handle_append_entries().returning(move |_, _, _| { + Ok(crate::AppendResponseWithUpdates { + response: d_engine_proto::server::replication::AppendEntriesResponse::success( + 1, + leader_term, + Some(LogId { + term: leader_term, + index: appended_index, + }), + ), + commit_index_update: None, + }) + }); + context.handlers.replication_handler = replication_handler; + context.membership = Arc::new(MockMembership::new()); + + let mut state = LearnerState::::new(1, context.node_config.clone()); + state.update_current_term(leader_term); + + let append_request = d_engine_proto::server::replication::AppendEntriesRequest { + term: leader_term, + leader_id: 2, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok() + ); + + // RPO=0: the ACK is withheld while durable_index < claimed index (5). + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 5, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(appended_index, &context, &flush_tx).await; + + let response = resp_rx.try_recv().expect("ACK must be released once durable").unwrap(); + assert!(response.is_success()); +} + +/// #446: if `LearnerState` is dropped (role transition — e.g. promotion to voter, or a +/// higher-term AppendEntries) while it still holds a withheld ACK, the pending sender +/// must be dropped with it — the caller waiting on `resp_rx` must see the channel +/// close, not hang forever and not receive a stale success. See the equivalent +/// Follower test for the full rationale — same mechanism, same reasoning. #[tokio::test] -async fn test_learner_acks_immediately_after_memory_write() { +async fn test_dropping_learner_state_releases_pending_ack_senders_as_closed() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); @@ -1778,9 +1857,22 @@ async fn test_learner_acks_immediately_after_memory_write() { .is_ok() ); - // MemFirst: ACK sent immediately - let response = resp_rx.try_recv().expect("ACK must be sent immediately after memory write"); - assert!(response.unwrap().is_success()); + assert!( + resp_rx.try_recv().is_err(), + "precondition: the ACK must still be withheld before the role transition" + ); + + // Simulates a real role transition dropping the old LearnerState. + drop(state); + + let result = tokio::time::timeout(std::time::Duration::from_secs(1), resp_rx.recv()) + .await + .expect("recv() must resolve promptly once the sender is dropped, not hang"); + assert!( + result.is_err(), + "dropping LearnerState must close the pending ACK's channel, not deliver a \ + stale response" + ); } /// Spawns a fake Worker that answers exactly one `InstallSnapshot` command with `result`, diff --git a/d-engine-core/src/raft_role/role_state.rs b/d-engine-core/src/raft_role/role_state.rs index 89c6181e..f5948150 100644 --- a/d-engine-core/src/raft_role/role_state.rs +++ b/d-engine-core/src/raft_role/role_state.rs @@ -31,10 +31,13 @@ use d_engine_proto::common::LogId; use d_engine_proto::server::election::VotedFor; use d_engine_proto::server::replication::AppendEntriesRequest; use d_engine_proto::server::replication::AppendEntriesResponse; +use d_engine_proto::server::replication::SuccessResult; +use d_engine_proto::server::replication::append_entries_response; use d_engine_proto::server::storage::SnapshotAck; use d_engine_proto::server::storage::SnapshotChunk; use d_engine_proto::server::storage::SnapshotMetadata; use d_engine_proto::server::storage::SnapshotResponse; +use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::sync::mpsc; use tokio::time::Instant; @@ -56,6 +59,17 @@ pub(crate) enum PeerReplicationState { Snapshot, } +/// An AppendEntries response withheld because this node's own `durable_index` hasn't +/// caught up to what it would claim yet (RPO=0, #446). Keyed by the claimed index in +/// `pending_append_acks` (BTreeMap) — `senders` accumulates via +/// `entry().or_insert_with()` if more than one request lands on the same threshold +/// (retry, or a heartbeat landing on the same tail). +pub(crate) struct PendingAck { + pub(super) response: AppendEntriesResponse, + pub(super) senders: + Vec>>, +} + #[async_trait] pub(crate) trait RaftRoleState: Send + Sync + 'static { type T: TypeConfig; @@ -402,11 +416,25 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { /// Default: no-op for Candidate/Follower/Learner (ACK already sent on memory write). async fn handle_log_flushed( &mut self, - _durable: u64, + durable: u64, _ctx: &RaftContext, _internal_event_tx: &mpsc::UnboundedSender, ) { - // Candidate: no-op + // RPO=0 (#446): release any withheld AppendEntries responses whose claimed + // index is now durable. No-op for Candidate/Leader (pending_append_acks_mut + // returns None for them; Leader overrides this whole method anyway). + let Some(pending) = self.pending_append_acks_mut() else { + return; + }; + let later = pending.split_off(&(durable.saturating_add(1))); + let ready = std::mem::replace(pending, later); + for (_, ack) in ready { + for sender in ack.senders { + if let Err(e) = sender.send(Ok(ack.response)) { + error!("Failed to send released AppendEntriesResponse: {:?}", e); + } + } + } } /// Handle AppendEntries result from a per-follower ReplicationWorker. @@ -560,13 +588,48 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { } debug!("AppendEntriesResponse: {:?}", response); - // MemFirst: ACK immediately after memory write. IO thread fsyncs async. - // Safety: quorum uses last_entry_id (in-memory); crash safety is guaranteed by - // majority replication, not per-follower durability. + // RPO=0 (#446): a success response must not go out until this node's + // own durable_index has caught up to what it claims — an ACK asserts + // durability, and it must not lie about that. Conflict/higher-term + // responses don't claim any durable state, so they're never withheld. + let claimed_index = match &response.result { + Some(append_entries_response::Result::Success(SuccessResult { + last_match: Some(log_id), + })) => Some(log_id.index), + _ => None, + }; - for sender in senders { - if let Err(e) = sender.send(Ok(response)) { - error!("Failed to send: {:?}", e); + let withhold = + claimed_index.is_some_and(|idx| ctx.storage.raft_log.durable_index() < idx); + + if withhold { + let idx = claimed_index.unwrap(); + match self.pending_append_acks_mut() { + Some(pending) => { + pending + .entry(idx) + .or_insert_with(|| PendingAck { + response, + senders: Vec::new(), + }) + .senders + .extend(senders); + } + None => { + // Should never happen — only Follower/Learner reach this + // branch. Fail loud rather than silently dropping an ACK + // the leader is waiting on. + error!("no pending_append_acks slot on a role that should have one"); + for sender in senders { + let _ = sender.send(Ok(response)); + } + } + } + } else { + for sender in senders { + if let Err(e) = sender.send(Ok(response)) { + error!("Failed to send: {:?}", e); + } } } } @@ -940,6 +1003,12 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { _state: PeerReplicationState, ) { } + + /// Follower/Learner's withheld-response queue. `None` for Candidate/Leader — + /// same pattern as `pending_purge_upto_mut` below. + fn pending_append_acks_mut(&mut self) -> Option<&mut BTreeMap> { + None + } } /// Attempts to execute whatever purge target is currently pending, if any. Shared by both diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index b841238e..0fb7960d 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -359,6 +359,9 @@ where } } + // #446: this is what election-eligibility comparisons (is_target_log_more_recent) + // read. It must keep reflecting the in-memory tail, never durable_index — a node + // with an un-fsynced entry must still be able to reject a less-up-to-date candidate. fn last_log_id(&self) -> Option { let last_index = self.last_entry_id(); if last_index > 0 { @@ -640,11 +643,10 @@ where mut peer_matched_ids: Vec, ) -> Option { let _timer = ScopedTimer::new("calculate_majority_matched_index"); - // Leader's contribution: last_entry_id (in-memory). With MemFirst (Level 2), db.write() - // returns once data reaches OS page cache — durable_index advances immediately. - // Followers also ACK after OS page cache write (no fsync wait). Crash safety is - // OS page cache level: process crash is recoverable, power loss is not. - peer_matched_ids.push(self.last_entry_id()); + // RPO=0 (#446): leader's own contribution must be its own durable (fsynced) + // position, not the in-memory tail — otherwise a majority-looking commit can + // still lose data on correlated power loss. + peer_matched_ids.push(self.durable_index()); // Sort in descending order peer_matched_ids.sort_unstable_by(|a, b| b.cmp(a)); @@ -775,8 +777,8 @@ where idle_flush_interval_ms, } = persistence_config.flush_policy; debug!( - "Creating BufferedRaftLog with node_id: {}, strategy: {:?}, idle_flush_interval_ms: {:?}, disk_len: {:?}", - node_id, persistence_config.strategy, idle_flush_interval_ms, disk_len + "Creating BufferedRaftLog with node_id: {}, idle_flush_interval_ms: {:?}, disk_len: {:?}", + node_id, idle_flush_interval_ms, disk_len ); let shutdown_timeout_ms = persistence_config.shutdown_timeout_ms; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs index 6790f539..cb12c6d7 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs @@ -13,7 +13,7 @@ use crate::test_utils::{ BufferedRaftLogTestContext, mock_empty_entries, simulate_delete_command, simulate_insert_command, }; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; /// Test get_entries_range returns correct subset /// @@ -24,7 +24,6 @@ use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; #[tokio::test] async fn test_get_entries_range_returns_correct_subset() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -52,7 +51,6 @@ async fn test_get_entries_range_returns_correct_subset() { #[tokio::test] async fn test_get_entries_range_handles_large_range() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -86,7 +84,6 @@ async fn test_get_entries_range_handles_large_range() { #[tokio::test] async fn test_filter_conflicts_removes_entries_with_different_term() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -133,7 +130,6 @@ async fn test_filter_conflicts_removes_entries_with_different_term() { #[tokio::test] async fn test_filter_conflicts_handles_multiple_scenarios() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -193,7 +189,6 @@ async fn test_filter_conflicts_handles_multiple_scenarios() { #[tokio::test] async fn test_last_entry_returns_highest_index() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -218,7 +213,6 @@ async fn test_last_entry_returns_highest_index() { #[tokio::test] async fn test_last_entry_matches_buffer_length() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -247,7 +241,6 @@ async fn test_last_entry_matches_buffer_length() { #[tokio::test] async fn test_last_entry_with_large_payload_id() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -272,7 +265,6 @@ async fn test_last_entry_with_large_payload_id() { #[tokio::test] async fn test_insert_batch_appends_entries_in_order() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -305,7 +297,6 @@ async fn test_insert_batch_appends_entries_in_order() { #[tokio::test] async fn test_get_entries_range_multiple_bounds() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -360,7 +351,6 @@ async fn test_get_entries_range_multiple_bounds() { #[tokio::test] async fn test_insert_duplicate_commands_as_separate_events() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -395,7 +385,6 @@ async fn test_insert_duplicate_commands_as_separate_events() { #[tokio::test] async fn test_purge_after_insert_maintains_consistency() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -428,7 +417,6 @@ async fn test_purge_after_insert_maintains_consistency() { #[tokio::test] async fn test_purge_logs_removes_entries_up_to_index() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -476,7 +464,6 @@ async fn test_purge_logs_removes_entries_up_to_index() { #[tokio::test] async fn test_concurrent_purge_operations_are_safe() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -515,7 +502,6 @@ async fn test_concurrent_purge_operations_are_safe() { #[tokio::test] async fn test_first_entry_id_after_purge_updates() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -547,7 +533,6 @@ async fn test_first_entry_id_after_purge_updates() { #[tokio::test] async fn test_single_entry_insert_succeeds() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -569,7 +554,6 @@ async fn test_single_entry_insert_succeeds() { #[tokio::test] async fn test_is_empty_returns_true_for_new_log() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -588,7 +572,6 @@ async fn test_is_empty_returns_true_for_new_log() { #[tokio::test] async fn test_is_empty_returns_false_after_append() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -613,7 +596,6 @@ async fn test_is_empty_returns_false_after_append() { #[tokio::test] async fn test_last_log_id_for_empty_log() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -636,7 +618,6 @@ async fn test_last_log_id_for_empty_log() { #[tokio::test] async fn test_last_log_id_after_appends() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -662,7 +643,6 @@ async fn test_last_log_id_after_appends() { #[tokio::test] async fn test_drop_shuts_down_workers_gracefully() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -686,14 +666,12 @@ async fn test_drop_shuts_down_workers_gracefully() { #[tokio::test] async fn test_same_index_and_term_implies_identical_prefix() { let ctx1 = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, "test_log_matching_1", ); let ctx2 = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -726,7 +704,6 @@ async fn test_same_index_and_term_implies_identical_prefix() { #[tokio::test] async fn test_committed_entry_present_in_future_leaders() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -758,7 +735,6 @@ async fn test_committed_entry_present_in_future_leaders() { #[tokio::test] async fn test_append_updates_last_entry() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -784,7 +760,6 @@ async fn test_append_updates_last_entry() { #[tokio::test] async fn test_insert_batch_with_empty_list() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -807,7 +782,6 @@ async fn test_insert_batch_with_empty_list() { #[tokio::test] async fn test_insert_batch_updates_metadata() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs index fce20bf3..8cde96b9 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs @@ -8,7 +8,7 @@ use crate::{ BufferedRaftLog, FlushPolicy, InternalEvent, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, RaftLog, + PersistenceConfig, RaftLog, }; use d_engine_proto::common::Entry; use std::sync::Arc; @@ -34,7 +34,6 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -85,34 +84,35 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { ); } -/// `calculate_majority_matched_index` uses the in-memory SkipMap (`last_entry_id`), -/// not `durable_index` — even when all fsyncs are stalled indefinitely. +/// `calculate_majority_matched_index` uses `durable_index` (fsync-confirmed), not the +/// in-memory `last_entry_id` — even when a follower already reports the index, the +/// leader's own contribution must not count toward quorum until it has itself fsynced. /// -/// Stall every flush() call via a MockLogStore barrier, append entries, then verify -/// that majority-matched calculation returns the correct in-memory index. +/// Stall every flush() call via a MockLogStore barrier, append entries, then verify that +/// majority-matched calculation does NOT advance while fsync is stalled, and does advance +/// once fsync completes. /// -/// Regression guard: if majority calculation ever changes to depend on `durable_index`, -/// this test will catch it before it reaches production. +/// Regression guard: RPO=0 (#446) requires the leader's own copy to be durable before it +/// counts toward commit — if this ever reverts to using `last_entry_id`, this test will +/// catch it before it reaches production. /// /// Expected: -/// - Append entries so `last_entry_id()` reaches N (e.g. 5) while fsync is -/// permanently stalled — `durable_index()` stays at its pre-write value -/// (0) throughout. -/// - Feed `calculate_majority_matched_index` a `match_index` map where enough -/// followers already report N to form a majority. -/// - Assert the returned majority-matched index equals N (matching -/// `last_entry_id()`) — NOT 0 (what it would return if it mistakenly used -/// `durable_index()` instead). +/// - Append entries so `last_entry_id()` reaches N (e.g. 2) while fsync is permanently +/// stalled — `durable_index()` stays at its pre-write value (0) throughout. +/// - Feed `calculate_majority_matched_index` a `match_index` map where one follower +/// already reports N=2 (majority IF the leader's own un-fsynced entry counted) — +/// assert the result is `None` while durable_index is still 0. +/// - Release the gate. Once `durable_index` reaches 2, the same call must return +/// `Some(2)`. #[tokio::test] -async fn test_majority_matched_index_uses_memory_not_durable_index() { +async fn test_majority_matched_index_uses_durable_not_memory() { // Gate closed: the first flush() call will block until we send () on `flush_gate`. let (storage, flush_gate) = MockStorageEngine::not_durable_gated_flush( - "majority_matched_index_uses_memory_not_durable_index".into(), + "majority_matched_index_uses_durable_not_memory".into(), ); let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify should trigger fsync here. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, @@ -160,22 +160,22 @@ async fn test_majority_matched_index_uses_memory_not_durable_index() { assert_eq!( raft_log.last_entry_id(), pre_last_entry_id + size, - "durable_index must not advance before fsync completes" + "in-memory tail should still advance even while fsync is stalled" ); // One follower already matched index 2; the other is still behind at 0 — asymmetric - // on purpose. With only ONE follower at 2, the leader's own contribution decides - // whether the majority (2 out of 3 voters) reaches 2. If this ever regresses to use - // `durable_index()` (0, since fsync is still gated) instead of `last_entry_id()` (2), - // the median drops to 0 and the call returns `None` instead of `Some(2)`. + // on purpose. If the leader's own un-fsynced entry counted (the old MemFirst + // behavior), 2 out of 3 voters would reach index 2 — but RPO=0 requires the + // leader's own copy to be durable first, so this must return None while fsync + // is still gated. let result = raft_log.calculate_majority_matched_index(1, 1, vec![2, 0]); assert_eq!( - result, - Some(2), - "majority index must use last_entry_id (2), not durable_index (0)" + result, None, + "RPO=0: the leader's own un-fsynced entry must not count toward quorum, even \ + when a follower already reports it" ); - // Release the gate — flush() returns, advance_durable_and_notify(1) fires. + // Release the gate — flush() returns, advance_durable_and_notify(2) fires. flush_gate.send(()).unwrap(); // Pick a polling/backoff strategy instead of a fixed sleep, @@ -187,6 +187,14 @@ async fn test_majority_matched_index_uses_memory_not_durable_index() { pre_write_durable_index + size, "durable_index must reach the expected index after fsync completes" ); + + // Now the leader's own contribution is durable (2), so the same call must succeed. + let result_after_fsync = raft_log.calculate_majority_matched_index(1, 1, vec![2, 0]); + assert_eq!( + result_after_fsync, + Some(2), + "once the leader's own entry is durable, majority index must advance to 2" + ); } /// `entry_term()` returns the correct term during high-concurrency writes @@ -212,7 +220,6 @@ async fn test_entry_term_correct_during_concurrent_fsync_delay() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -300,7 +307,6 @@ async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -368,7 +374,6 @@ async fn test_flush_caller_blocked_until_fsync_completes() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -442,7 +447,6 @@ async fn test_flush_callers_arriving_during_inflight_fsync_are_coalesced() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -526,7 +530,6 @@ async fn test_shutdown_with_pending_flush_caller_still_receives_ok_reply() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -600,7 +603,6 @@ async fn test_shutdown_with_pending_flush_caller_still_receives_err_reply() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -681,7 +683,6 @@ async fn test_reset_during_inflight_fsync_does_not_resurrect_stale_durable_index let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -763,7 +764,6 @@ async fn test_post_reset_writes_are_not_discarded_by_stale_fence() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs index 2b345e79..d3201d2f 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs @@ -3,15 +3,14 @@ use std::time::Duration; use futures::future::join_all; use tokio; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, simulate_insert_command}; -use crate::{FlushPolicy, PersistenceStrategy}; use d_engine_proto::common::{Entry, LogId}; #[tokio::test] async fn test_remove_range_with_concurrent_reads() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -54,7 +53,6 @@ async fn test_remove_range_with_concurrent_reads() { #[tokio::test] async fn test_concurrent_append_and_purge() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -128,7 +126,6 @@ async fn test_get_entries_range_never_returns_torn_result_during_concurrent_purg const ITERATIONS: usize = 500; let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs index daf9a096..bf4e3996 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs @@ -20,7 +20,6 @@ use crate::MockMetaStore; use crate::MockStorageEngine; use crate::MockTypeConfig; use crate::PersistenceConfig; -use crate::PersistenceStrategy; use d_engine_proto::common::Entry; use d_engine_proto::common::LogId; use std::sync::Arc; @@ -156,7 +155,6 @@ async fn test_pending_max_zeroed_on_reset_preventing_durable_index_corruption() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify triggers fsync. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, @@ -288,7 +286,6 @@ async fn test_flush_propagates_io_error() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -350,7 +347,6 @@ async fn test_fsync_failure_poisons_and_rejects_writes_after_reset() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -407,7 +403,6 @@ async fn test_replace_range_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -497,7 +492,6 @@ async fn test_purge_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -547,7 +541,6 @@ async fn test_reset_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -596,7 +589,6 @@ async fn test_save_hard_state_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -645,7 +637,6 @@ async fn test_poisoned_rejects_save_hard_state() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -693,7 +684,6 @@ async fn test_poisoned_skips_replace_range() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -775,7 +765,6 @@ async fn test_poisoned_does_not_skip_reset() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -816,7 +805,6 @@ async fn test_poisoned_skips_purge() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -899,7 +887,6 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -1010,7 +997,6 @@ async fn test_new_buffered_raft_log_starts_unpoisoned() { let (raft_log, _receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify triggers fsync. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, @@ -1042,7 +1028,6 @@ async fn test_poisoned_survives_reset() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify triggers fsync. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, @@ -1082,7 +1067,6 @@ async fn test_persist_entries_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -1149,7 +1133,6 @@ async fn test_poisoned_rejects_queued_persist_task() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -1216,7 +1199,6 @@ async fn test_notify_fatal_channel_closed_still_poisons_and_logs() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs index 521338b3..2c4f0c02 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs @@ -6,16 +6,12 @@ use tokio::sync::mpsc; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, MockStorageEngine, simulate_insert_command}; -use crate::{ - BufferedRaftLog, FlushPolicy, InternalEvent, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, -}; +use crate::{BufferedRaftLog, FlushPolicy, InternalEvent, MockTypeConfig, PersistenceConfig}; use d_engine_proto::common::{Entry, LogId}; #[tokio::test] async fn test_durable_index_monotonic_under_concurrency() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -55,7 +51,6 @@ async fn test_durable_index_monotonic_under_concurrency() { #[tokio::test] async fn test_durable_index_with_non_contiguous_entries() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -126,7 +121,6 @@ async fn test_purge_does_not_regress_durable_index_already_ahead() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs index 60f00876..15ff8451 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs @@ -1,14 +1,13 @@ use bytes::Bytes; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; use d_engine_proto::common::{Entry, EntryPayload, LogId}; #[tokio::test] async fn test_empty_log_operations() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -27,7 +26,6 @@ async fn test_empty_log_operations() { #[tokio::test] async fn test_single_entry_operations() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -56,7 +54,6 @@ async fn test_single_entry_operations() { #[tokio::test] async fn test_gap_handling_in_indexes() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -99,7 +96,6 @@ async fn test_gap_handling_in_indexes() { #[tokio::test] async fn test_extreme_boundary_conditions() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs index 44fd0040..95e40d8b 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs @@ -13,7 +13,7 @@ use d_engine_proto::common::{Entry, EntryPayload}; use tokio::time::{Duration, sleep}; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; /// Test MemFirst with threshold=1 persists entries after flush /// @@ -23,7 +23,6 @@ use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; #[tokio::test] async fn test_mem_first_entries_durable_after_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -52,7 +51,6 @@ async fn test_mem_first_entries_durable_after_flush() { #[tokio::test] async fn test_mem_first_concurrent_writes_durable_after_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -102,7 +100,6 @@ async fn test_mem_first_concurrent_writes_durable_after_flush() { #[tokio::test] async fn test_mem_first_crash_recovery_restores_flushed_entries() { let original_ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -149,7 +146,6 @@ async fn test_mem_first_crash_recovery_restores_flushed_entries() { #[tokio::test] async fn test_mem_first_buffers_entries_before_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1000, }, @@ -173,7 +169,6 @@ async fn test_mem_first_buffers_entries_before_flush() { #[tokio::test] async fn test_mem_first_flushes_asynchronously() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -202,7 +197,6 @@ async fn test_mem_first_flushes_asynchronously() { #[tokio::test] async fn test_mem_first_concurrent_buffering() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 5000, }, @@ -243,7 +237,6 @@ async fn test_mem_first_concurrent_buffering() { #[tokio::test] async fn test_batched_flushes_at_threshold() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 10000, // High interval to test threshold trigger }, @@ -269,7 +262,6 @@ async fn test_batched_flushes_at_threshold() { #[tokio::test] async fn test_batched_flushes_at_interval() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -295,7 +287,6 @@ async fn test_batched_flushes_at_interval() { #[tokio::test] async fn test_batched_partial_flush_recovery() { let original_ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs index 095db56d..508c3bd5 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs @@ -11,8 +11,7 @@ use std::sync::Arc; use std::sync::atomic::Ordering; use crate::{ - BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, RaftLog, + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, RaftLog, }; fn setup_memory() -> Arc> { @@ -20,7 +19,6 @@ fn setup_memory() -> Arc> { let (raft_log, _receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs index a9530bf9..52826bfb 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs @@ -12,7 +12,7 @@ use tokio::time::Instant; use crate::{ BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, RaftLog, + PersistenceConfig, RaftLog, }; use d_engine_proto::common::{Entry, EntryPayload}; @@ -47,24 +47,17 @@ async fn test_reset_performance_during_active_flush() { let max_reset_duration_ms = FLUSH_DELAY_MS * 3; // 600ms: accounts for IO thread overhead let test_cases = vec![ - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1000, - }, - ), - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1, - }, - ), + FlushPolicy::Batch { + idle_flush_interval_ms: 1000, + }, + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, ]; - for (strategy, flush_policy) in test_cases { + for flush_policy in test_cases { let storage = create_delayed_storage(FLUSH_DELAY_MS); let config = PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), max_buffered_entries: 1000, shutdown_timeout_ms: 5000, @@ -100,9 +93,8 @@ async fn test_reset_performance_during_active_flush() { assert!( duration.as_millis() < max_reset_duration_ms as u128, - "Reset took {}ms during active flush ({:?}/{:?})", + "Reset took {}ms during active flush ({:?})", duration.as_millis(), - strategy, flush_policy ); } @@ -124,7 +116,6 @@ async fn test_filter_conflicts_performance_during_flush() { for (idle_flush_interval_ms, max_duration_ms) in test_cases { let storage = create_delayed_storage(FLUSH_DELAY_MS); let config = PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, @@ -190,21 +181,15 @@ async fn test_fresh_cluster_performance_consistency() { let max_duration_ms = if is_ci { 50 } else { 5 }; let test_cases = vec![ - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1000, - }, - ), - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1, - }, - ), + FlushPolicy::Batch { + idle_flush_interval_ms: 1000, + }, + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, ]; - for (strategy, flush_policy) in test_cases { + for flush_policy in test_cases { let mut log_store = MockLogStore::new(); log_store.expect_is_write_durable().returning(|| true); log_store.expect_flush().return_once(|| Ok(())); @@ -215,7 +200,6 @@ async fn test_fresh_cluster_performance_consistency() { log_store.expect_reset().returning(|| Ok(())); let config = PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), max_buffered_entries: 1000, shutdown_timeout_ms: 5000, @@ -235,9 +219,8 @@ async fn test_fresh_cluster_performance_consistency() { assert!( duration.as_millis() < max_duration_ms as u128, - "Fresh cluster reset took {}ms ({:?}/{:?})", + "Fresh cluster reset took {}ms ({:?})", duration.as_millis(), - strategy, flush_policy ); } diff --git a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs index e91d3cb5..95f49106 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs @@ -28,10 +28,7 @@ use d_engine_proto::common::Entry; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{ - BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, -}; +use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; fn entry( index: u64, @@ -49,7 +46,6 @@ fn entry( #[tokio::test] async fn test_durable_index_never_exceeds_log_after_truncation_and_resync() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // isolate from the safety-net timer }, @@ -134,7 +130,6 @@ async fn test_persisted_index_does_not_adopt_a_stale_persist_after_truncation() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs index efb54b80..a0d689fa 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs @@ -13,12 +13,11 @@ use d_engine_proto::common::Entry; use crate::test_utils::BufferedRaftLogTestContext; use crate::{ BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, RaftLog, + PersistenceConfig, RaftLog, }; fn ctx(name: &str) -> BufferedRaftLogTestContext { BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -472,7 +471,6 @@ async fn test_io_task_replace_range_delegates_to_replace_range_not_truncate() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs index 3819e101..1a206f84 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs @@ -20,7 +20,7 @@ use d_engine_proto::common::Entry; use crate::{ BufferedRaftLog, FlushPolicy, LogStore, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, RaftLog, StorageEngine, + RaftLog, StorageEngine, }; /// `append_entries()` must not return before the entry reaches the storage @@ -48,7 +48,6 @@ async fn test_append_entries_waits_for_storage_engine_before_returning() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs index 0429eebc..36b88ce3 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs @@ -1,62 +1,194 @@ //! Quorum Durability Tests //! -//! MemFirst design: leader contributes `last_entry_id` (in-memory) to quorum. -//! IO thread persistence is async and NOT on the commit critical path. +//! RPO=0 (#446): leader contributes `durable_index` (not `last_entry_id`) to quorum — +//! commit must not advance past what the leader itself has survived fsync for. //! -//! Follower ACK path: followers ACK immediately after memory write (no `wait_durable`). -//! IO thread fsyncs asynchronously; crash safety is guaranteed by quorum, not per-follower durability. +//! Superseded design (kept here as history, do not resurrect): the old MemFirst model had +//! the leader contribute `last_entry_id` (in-memory) so IO persistence never sat on the +//! commit critical path. That traded away RPO=0 — a majority-acked write could still be +//! lost on correlated power loss before fsync. This file's tests now lock in the new +//! behavior instead of the old one. +//! +//! Follower ACK path (tracked separately, not yet landed): followers will ACK only after +//! their own durable_index catches up — so a follower's reported match_index is inherently +//! already durable by the time the leader sees it. +//! +//! Election-eligibility comparison must keep reading the in-memory log, never +//! `durable_index` — a separate, independent invariant from the durable-quorum change +//! above, but one a majority-count safety argument for #446 depends on. See +//! `test_election_eligibility_reads_memory_log_not_durable_index`. +//! +//! Note: these tests rely on `BufferedRaftLog`'s in-memory layer existing (they force a +//! gap between `last_entry_id` and `durable_index` via a gated mock flush). If that layer +//! is ever removed, this file's setup assumptions need revisiting — not a decided plan, +//! just a known dependency to check first. +//! +//! Tests that need a genuine, un-fsynced gap between `last_entry_id` and `durable_index` +//! use `MockStorageEngine::not_durable_gated_flush` — a real channel-based gate, not a +//! timing guess. An earlier version of this file relied on a long `idle_flush_interval_ms` +//! and assumed the dedicated `raft-io-*` OS thread just wouldn't get scheduled before the +//! assertions ran; that's a real race (the IO thread is independent of the test's own +//! runtime), and it was intermittently losing under load — flaky, not broken logic. Do not +//! reintroduce that pattern here. +use crate::BufferedRaftLog; +use crate::FlushPolicy; +use crate::MockStorageEngine; +use crate::MockTypeConfig; +use crate::PersistenceConfig; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; +use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; +use std::sync::Arc; use std::time::Duration; -/// Flush policy with a far-future safety timer — IO thread only fsyncs on WriteNotify. -/// In current_thread test runtime, durable_index stays at 0 immediately after append_entries -/// because the IO thread task has no chance to run until the test yields. -fn no_auto_flush_policy() -> FlushPolicy { - FlushPolicy::Batch { - idle_flush_interval_ms: 999_999, - } +/// Entries `1..=n`, all at `term`, no payload — the shape these tests need. +fn entries( + n: u64, + term: u64, +) -> Vec { + (1..=n) + .map(|index| Entry { + index, + term, + payload: None, + }) + .collect() } -// ── Leader quorum uses last_entry_id (in-memory), not durable_index ── +// ── Leader quorum uses durable_index, not last_entry_id (RPO=0) ── -/// MemFirst: leader's quorum contribution is last_entry_id (in-memory), not durable_index. +/// RPO=0: leader's quorum contribution is durable_index (fsync-confirmed), not +/// last_entry_id (in-memory). /// -/// Even when durable_index=0 (IO thread has not flushed), quorum must be satisfied -/// as soon as last_entry_id + follower ACKs form a majority. IO persistence is async -/// and must NOT block commit. +/// Even when a follower has already ACKed an index, the leader must not count its own +/// un-fsynced entry toward quorum — otherwise a majority-looking commit can still lose +/// data on correlated power loss (the leader's own copy was never actually durable). /// -/// This test FAILS if calculate_majority_matched_index uses durable_index (the bug -/// introduced by fix #329 which incorrectly put IO thread latency on the commit -/// critical path, causing +617µs avg latency regression in 3-node embedded bench). +/// This test FAILS if calculate_majority_matched_index still uses last_entry_id (the old +/// MemFirst behavior, since revoked). It replaces +/// `test_memfirst_quorum_uses_last_entry_id_not_durable_index`, which asserted the exact +/// opposite of this on purpose — that assertion documented a since-revoked design decision. #[tokio::test] -async fn test_memfirst_quorum_uses_last_entry_id_not_durable_index() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, - no_auto_flush_policy(), // durable_index stays 0 — IO thread won't run - "test_memfirst_quorum_last_entry_id", +async fn test_quorum_uses_durable_index_not_last_entry_id() { + // Gate closed: the first flush() call blocks until we send () on `flush_gate` — fsync + // deterministically never completes until we say so, no timing involved. + let (storage, flush_gate) = + MockStorageEngine::not_durable_gated_flush("test_quorum_durable_index".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - // Entry written to SkipMap (in memory). IO thread has not flushed yet. - ctx.append_entries(1, 1, 1).await; + raft_log.append_entries(entries(1, 1)).await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; // let it reach the gate - assert_eq!(ctx.raft_log.last_entry_id(), 1); - assert_eq!(ctx.raft_log.durable_index(), 0); // IO thread hasn't run + assert_eq!(raft_log.last_entry_id(), 1); + assert_eq!(raft_log.durable_index(), 0); // gate never released — fsync hasn't completed - let result = ctx.raft_log.calculate_majority_matched_index( + let result = raft_log.calculate_majority_matched_index( 1, 0, - vec![1], // one follower acked index=1; together with leader = majority of 3 + vec![1], // one follower reports match=1 (already durable, post-Stage2 semantics) + ); + + // RPO=0: leader contributes durable_index=0, not last_entry_id=1. + // peer_matched_ids = [follower=1, leader=0], sorted desc = [1,0], median(len/2=1) = 0. + // majority_index=0 is not < commit_index=0, so falls through to the term check on + // entry(0) — index 0 is not a real entry (log is 1-indexed) — Ok(None) — result is None. + assert_eq!( + result, None, + "RPO=0: the leader's own un-fsynced entry must not count toward quorum, even when \ + a follower has already acked it — one follower alone isn't majority without the \ + leader's own durable contribution" + ); + + let _ = flush_gate.send(()); // release so the blocked IO thread doesn't linger +} + +/// Election-eligibility comparison (`last_log_id`, consumed by +/// `election_handler::handle_vote_request`) must read the in-memory log, never +/// `durable_index`. A follower with an un-fsynced tail must still be able to correctly +/// reject a candidate whose log is genuinely less up to date — voting eligibility and +/// commit-durability are two separate concerns and must not be conflated by sharing the +/// same index source. +#[tokio::test] +async fn test_election_eligibility_reads_memory_log_not_durable_index() { + let (storage, flush_gate) = + MockStorageEngine::not_durable_gated_flush("test_election_eligibility_memory_log".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + raft_log.append_entries(entries(10, 1)).await.unwrap(); // entries 1..=10, term=1 + tokio::time::sleep(Duration::from_millis(50)).await; + + assert_eq!(raft_log.durable_index(), 0, "nothing fsynced yet"); + assert_eq!( + raft_log.last_log_id(), + Some(LogId { index: 10, term: 1 }), + "election-eligibility comparison must see the un-fsynced tail, not fall back to \ + durable_index=0 — a candidate with a truly-shorter log must still be rejected" + ); + + let _ = flush_gate.send(()); +} + +/// `calculate_majority_matched_index`'s median-based calculation requires an actual +/// majority of `peer_matched_ids` to reach an index before it counts toward commit — a +/// minority (here: 2 of 5) reporting a higher index cannot move the result past what the +/// rest of the cluster last confirmed. +/// +/// This is a pure property of the median calculation itself — the function has no way to +/// know whether any of its inputs are stale or expired, so this test does not by itself +/// prove anything about stale reports being harmless. It only pins down the arithmetic +/// that a separate, broader safety argument for #446 relies on. +#[tokio::test] +async fn test_majority_matched_index_requires_actual_majority_of_reports() { + let ctx = BufferedRaftLogTestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, + "test_majority_requires_actual_majority", ); - // MemFirst: leader contributes last_entry_id=1. - // quorum = [leader=1, follower=1] → majority of {leader, f1, f2} satisfied → Some(1). + ctx.append_entries(1, 10, 1).await; + ctx.raft_log.flush().await.unwrap(); // leader's own entries now durable through 10 + + assert_eq!(ctx.raft_log.durable_index(), 10); + + // 1 of 4 followers reports index 10; the other 3 are still at their last-known + // value, 9. + let result = ctx.raft_log.calculate_majority_matched_index(1, 9, vec![10, 9, 9, 9]); + + // peer_matched_ids after leader's own contribution = [10, 9, 9, 9, 10] + // sorted desc = [10,10,9,9,9], median(len/2=2) = 9 — majority stays at 9, entry(9) + // exists with term=1=current_term, so the result is the previously-safe Some(9), not 10. assert_eq!( result, - Some(1), - "MemFirst: quorum must use last_entry_id, not durable_index — IO must not block commit" + Some(9), + "a minority (2 of 5) reporting a higher index cannot move majority past what the \ + other 3 nodes last confirmed" ); } @@ -67,7 +199,6 @@ async fn test_memfirst_quorum_uses_last_entry_id_not_durable_index() { #[tokio::test] async fn test_quorum_succeeds_after_leader_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 999_999, // only threshold trigger, no timer }, @@ -103,35 +234,47 @@ async fn test_quorum_succeeds_after_leader_flush() { // ── Bug 2: gap between last_entry_id and durable_index ── -/// Demonstrates that after append_entries with MemFirst + no-auto-flush, -/// last_entry_id and durable_index diverge. +/// Demonstrates that after append_entries with a stalled fsync, last_entry_id and +/// durable_index diverge. /// /// This is the root condition enabling the bug: both values exist, /// but quorum calculation only uses the unsafe one. #[tokio::test] async fn test_last_entry_id_diverges_from_durable_index_with_mem_first() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, - no_auto_flush_policy(), - "test_diverge_mem_first", + let (storage, flush_gate) = + MockStorageEngine::not_durable_gated_flush("test_diverge_mem_first".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + max_buffered_entries: 1000, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); - ctx.append_entries(1, 5, 1).await; // entries 1..=5, no flush + raft_log.append_entries(entries(5, 1)).await.unwrap(); // entries 1..=5, no flush + tokio::time::sleep(Duration::from_millis(50)).await; - assert_eq!(ctx.raft_log.last_entry_id(), 5, "memory index should be 5"); + assert_eq!(raft_log.last_entry_id(), 5, "memory index should be 5"); assert_eq!( - ctx.raft_log.durable_index(), + raft_log.durable_index(), 0, "durable_index must remain 0: no flush has run" ); // This gap (5 vs 0) is exactly what the quorum bug exploits. + + let _ = flush_gate.send(()); } /// After explicit flush, durable_index must equal last_entry_id. #[tokio::test] async fn test_durable_index_equals_last_entry_id_after_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs index f7acfe7f..373aeffd 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs @@ -1,12 +1,11 @@ +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, simulate_insert_command}; -use crate::{FlushPolicy, PersistenceStrategy}; use d_engine_proto::common::Entry; #[tokio::test] async fn test_log_matching_property() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -46,7 +45,6 @@ async fn test_log_matching_property() { #[tokio::test] async fn test_leader_completeness_property() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -84,7 +82,6 @@ async fn test_leader_completeness_property() { #[tokio::test] async fn test_calculate_majority_matched_index_case0() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -108,7 +105,6 @@ async fn test_calculate_majority_matched_index_case0() { #[tokio::test] async fn test_calculate_majority_matched_index_case1() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -132,7 +128,6 @@ async fn test_calculate_majority_matched_index_case1() { #[tokio::test] async fn test_calculate_majority_matched_index_case2() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -157,7 +152,6 @@ async fn test_calculate_majority_matched_index_case2() { #[tokio::test] async fn test_calculate_majority_matched_index_case3() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -180,7 +174,6 @@ async fn test_calculate_majority_matched_index_case3() { #[tokio::test] async fn test_calculate_majority_matched_index_case4() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -204,7 +197,6 @@ async fn test_calculate_majority_matched_index_case4() { #[tokio::test] async fn test_calculate_majority_matched_index_case5() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs index 656e2c25..c62dc1eb 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs @@ -1,8 +1,8 @@ use d_engine_proto::common::{Entry, LogId}; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, simulate_insert_command}; -use crate::{FlushPolicy, PersistenceStrategy}; fn entry( index: u64, @@ -18,7 +18,6 @@ fn entry( #[tokio::test] async fn test_remove_middle_range() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -50,7 +49,6 @@ async fn test_remove_middle_range() { #[tokio::test] async fn test_remove_from_start() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -77,7 +75,6 @@ async fn test_remove_from_start() { #[tokio::test] async fn test_remove_to_end() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -105,7 +102,6 @@ async fn test_remove_to_end() { #[tokio::test] async fn test_remove_empty_range() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -126,7 +122,6 @@ async fn test_remove_empty_range() { #[tokio::test] async fn test_remove_entire_log() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -150,7 +145,6 @@ async fn test_remove_entire_log() { #[tokio::test] async fn test_remove_single_entry() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -185,7 +179,6 @@ async fn test_remove_single_entry() { #[tokio::test] async fn test_remove_range_clears_term_indexes_for_removed_entries() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -252,7 +245,6 @@ async fn test_remove_range_clears_term_indexes_for_removed_entries() { #[tokio::test] async fn test_purge_prefix_removes_entries_and_records_boundary_together() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -301,7 +293,6 @@ async fn test_purge_prefix_removes_entries_and_records_boundary_together() { #[tokio::test] async fn test_purge_prefix_multi_term_cutoff_updates_term_indexes_and_boundary() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs index 4f0beb2d..5f0358c7 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs @@ -17,9 +17,9 @@ use std::time::Duration; use d_engine_proto::common::Entry; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; fn entry( index: u64, @@ -45,7 +45,6 @@ fn entry( #[tokio::test] async fn test_replace_range_becomes_durable_without_a_following_append() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // effectively disabled for this test's timeframe }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs index c14d7c4a..ad909049 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs @@ -7,7 +7,7 @@ use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; use crate::{ BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, + PersistenceConfig, }; use d_engine_proto::common::{Entry, EntryPayload}; @@ -47,7 +47,6 @@ fn test_io_thread_survives_runtime_drop() { let (log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -84,7 +83,6 @@ fn test_io_thread_survives_runtime_drop() { #[tokio::test] async fn test_shutdown_closes_channel_properly() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -113,7 +111,6 @@ async fn test_shutdown_closes_channel_properly() { #[tokio::test] async fn test_shutdown_awaits_worker_completion() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 5000, }, @@ -159,7 +156,6 @@ async fn test_shutdown_handles_slow_workers() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -203,7 +199,6 @@ async fn test_shutdown_handles_slow_workers() { #[tokio::test] async fn test_shutdown_with_multiple_flushes() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -289,7 +284,6 @@ async fn test_replace_range_failure_propagates_error_and_shuts_down_io_thread() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // no auto-flush }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs index 5e37ef0d..cec818e4 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs @@ -8,12 +8,11 @@ use d_engine_proto::common::Entry; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; #[tokio::test] async fn test_first_index_for_term() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -88,7 +87,6 @@ async fn test_first_index_for_term() { #[tokio::test] async fn test_last_index_for_term() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -162,7 +160,6 @@ async fn test_last_index_for_term() { #[tokio::test] async fn test_term_index_functions_with_purged_logs() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -214,7 +211,6 @@ async fn test_term_index_functions_with_purged_logs() { #[tokio::test] async fn test_term_index_sequential_multi_term_insertion() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1000, }, @@ -263,7 +259,6 @@ async fn test_term_index_sequential_multi_term_insertion() { #[tokio::test] async fn test_term_indexes_rebuilt_correctly_after_restart() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -304,7 +299,6 @@ async fn test_term_indexes_rebuilt_correctly_after_restart() { #[tokio::test] async fn test_term_index_performance_large_dataset() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 5000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs index a385cd5d..5f36a0f6 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs @@ -16,7 +16,7 @@ use d_engine_proto::common::Entry; use crate::storage::buffered_raft_log::TermSegments; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; // --------------------------------------------------------------------------- // Helpers @@ -37,7 +37,6 @@ fn entries( fn ctx(name: &str) -> BufferedRaftLogTestContext { BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs index c2532224..16714b50 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs @@ -21,10 +21,7 @@ use std::time::Duration; use d_engine_proto::common::Entry; use crate::storage::raft_log::RaftLog; -use crate::{ - BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, -}; +use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; fn entry( index: u64, @@ -54,7 +51,6 @@ async fn test_durable_index_does_not_adopt_a_stale_fsync_after_truncation() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs index c2181802..965e8cf2 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs @@ -1,15 +1,14 @@ use std::time::Duration; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; /// Verifies that the flush worker continues operating normally after processing a large number /// of flush tasks — the worker does not exit or become unresponsive under sustained load. #[tokio::test] async fn test_flush_worker_sustains_throughput_under_load() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, diff --git a/d-engine-core/src/storage/fsync_coordinator_test.rs b/d-engine-core/src/storage/fsync_coordinator_test.rs index 32f21ffd..3b2a371e 100644 --- a/d-engine-core/src/storage/fsync_coordinator_test.rs +++ b/d-engine-core/src/storage/fsync_coordinator_test.rs @@ -19,7 +19,6 @@ use crate::MockMetaStore; use crate::MockStorageEngine; use crate::MockTypeConfig; use crate::PersistenceConfig; -use crate::PersistenceStrategy; use crate::Result; use std::sync::Arc; @@ -31,7 +30,6 @@ fn minimal_raft_log(storage: MockStorageEngine) -> Arc::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, diff --git a/d-engine-core/src/storage/raft_log.rs b/d-engine-core/src/storage/raft_log.rs index b21778bc..0b9b1496 100644 --- a/d-engine-core/src/storage/raft_log.rs +++ b/d-engine-core/src/storage/raft_log.rs @@ -209,14 +209,14 @@ pub trait RaftLog: Send + Sync + 'static { /// - Persist entries to durable storage BEFORE updating in-memory state /// - Call fsync/flush before returning Ok(()) /// - Ensures entries survive crashes immediately - /// - Example: BufferedRaftLog with PersistenceStrategy::DiskFirst + /// - Example: a store that fsyncs before returning /// /// 2. **Memory-First (Performance-optimized, Acceptable for Followers)**: /// - Update in-memory state first /// - Enqueue entries for asynchronous durability /// - MUST guarantee eventual durability via background flush /// - MUST call flush() before acknowledging commits - /// - Example: BufferedRaftLog with PersistenceStrategy::MemFirst + /// - Example: a store that fsyncs asynchronously /// - WARNING: Leader MUST wait_durable() before responding to AppendEntries RPCs /// /// # Safety Invariants @@ -228,10 +228,10 @@ pub trait RaftLog: Send + Sync + 'static { /// - MUST update term indexes (first/last_index_for_term) atomically /// /// # Raft Protocol Integration - /// - Leaders using MemFirst MUST call wait_durable(index) before: + /// - Leaders using async fsync MUST call wait_durable(index) before: /// * Responding success to AppendEntries RPC /// * Advancing commit index - /// - Followers can use MemFirst safely because leader durability guarantees safety + /// - Followers can use async fsync safely because leader durability guarantees safety /// /// # Failure Semantics /// - On error, implementer MAY roll back partial writes @@ -252,7 +252,7 @@ pub trait RaftLog: Send + Sync + 'static { /// /// # Usage Pattern /// ```rust,ignore - /// // Leader with MemFirst strategy + /// // Leader with async fsync /// raft_log.append_entries(new_entries).await?; /// raft_log.wait_durable(max_index).await?; // MUST wait before RPC response /// respond_to_client(Ok(())); @@ -261,7 +261,7 @@ pub trait RaftLog: Send + Sync + 'static { /// # Safety Invariants /// - MUST NOT return until flush() for this index completes successfully /// - If implementation doesn't support async durability, return Ok(()) immediately - /// - Critical for MemFirst strategy correctness + /// - Critical for async-fsync correctness async fn wait_durable( &self, index: u64, diff --git a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs index 9bcfaa07..43e955e9 100644 --- a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs +++ b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs @@ -12,15 +12,13 @@ use bytes::Bytes; use d_engine_proto::common::{Entry, EntryPayload}; use crate::{ - BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, RaftLog, + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, RaftLog, }; /// Test context for BufferedRaftLog tests pub struct BufferedRaftLogTestContext { pub raft_log: Arc>, pub storage: Arc, - pub strategy: PersistenceStrategy, pub flush_policy: FlushPolicy, pub instance_id: String, } @@ -28,7 +26,6 @@ pub struct BufferedRaftLogTestContext { impl BufferedRaftLogTestContext { /// Create a new test context with specified strategy and flush policy pub fn new( - strategy: PersistenceStrategy, flush_policy: FlushPolicy, instance_id: &str, ) -> Self { @@ -37,7 +34,6 @@ impl BufferedRaftLogTestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), max_buffered_entries: 1000, shutdown_timeout_ms: 5000, @@ -52,7 +48,6 @@ impl BufferedRaftLogTestContext { Self { raft_log, storage, - strategy, flush_policy, instance_id: instance_id.to_string(), } @@ -92,7 +87,6 @@ impl BufferedRaftLogTestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: flush_policy.clone(), max_buffered_entries: 1000, shutdown_timeout_ms: 5000, @@ -105,7 +99,6 @@ impl BufferedRaftLogTestContext { let ctx = Self { raft_log, storage, - strategy: PersistenceStrategy::MemFirst, flush_policy, instance_id: instance_id.to_string(), }; @@ -120,7 +113,6 @@ impl BufferedRaftLogTestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), max_buffered_entries: 1000, shutdown_timeout_ms: 5000, @@ -135,7 +127,6 @@ impl BufferedRaftLogTestContext { Self { raft_log, storage, - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), instance_id: self.instance_id.clone(), } diff --git a/d-engine-server/src/network/grpc/grpc_raft_service.rs b/d-engine-server/src/network/grpc/grpc_raft_service.rs index 15940d48..5b66126c 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service.rs @@ -6,7 +6,6 @@ use crate::Node; use crate::proto_convert; use d_engine_core::InboundEvent; use d_engine_core::MaybeCloneOneshot; -use d_engine_core::MaybeCloneOneshotReceiver; use d_engine_core::RaftOneshot; use d_engine_core::TypeConfig; #[cfg(feature = "watch")] @@ -136,12 +135,24 @@ where Pin> + Send>>; /// Processes a persistent bidirectional AppendEntries stream from the cluster leader. + /// #446: responses are forwarded as soon as each one is ready, not in strict arrival + /// order — leader-side match_index/next_index updates are already designed to + /// tolerate out-of-order pipeline responses (`leader_state.rs`, "only advance, + /// never retreat"), so nothing downstream needs strict ordering. Strict FIFO would + /// let one response still waiting on this node's own durable_index (RPO=0) block + /// every later, already-ready response on the same connection — including + /// unrelated ones like heartbeats. /// - /// Decouples request ingestion from response emission: - /// - recv task: reads batches from the stream, dispatches each as a `InboundEvent::AppendEntries` - /// (non-blocking between batches) - /// - forwarder task: drains ordered response handles sequentially; ordering is guaranteed - /// by the Raft single-threaded event loop + /// Single task, bounded concurrency: reads a new request only while fewer than + /// `max_pending_append_responses` requests are still in flight, so a stalled fsync + /// bounds memory/task growth instead of growing without limit. + /// + /// Known tradeoff, not an oversight: this is a single task, so a slow/stuck + /// network write (`out_tx.send().await` blocking because the peer isn't reading) + /// also delays reading new requests AND processing the shutdown signal, until the + /// write unblocks or the connection dies. Accepted deliberately — if the peer + /// isn't reading responses, there's no useful work to do by reading more requests + /// either; this is legitimate backpressure, not a bug. async fn stream_append_entries( &self, request: tonic::Request>, @@ -157,30 +168,31 @@ where let mut in_stream = request.into_inner(); let event_tx = self.event_tx.clone(); - let ordered_channel_capacity = self.node_config.raft.ordered_channel_capacity; + let max_pending = self.node_config.raft.max_pending_append_responses; let mut shutdown = self.shutdown_signal.clone(); + let node_id = self.node_id; - // Output: ordered ACKs sent back to the leader over the bidi stream - let (out_tx, out_rx) = mpsc::channel::>(128); - - // Ordered queue: response oneshot receivers in FIFO arrival order - let (ordered_tx, mut ordered_rx) = mpsc::channel::< - MaybeCloneOneshotReceiver>, - >(ordered_channel_capacity); + // Output: ACKs sent back to the leader over the bidi stream, in completion order. + // Capacity matches max_pending — completed responses can never outnumber + // in-flight requests, so there's no separate number to reason about here. + let (out_tx, out_rx) = mpsc::channel::>(max_pending); - // Recv task: read batches, dispatch to Raft loop without waiting for each ACK. - // Selects on shutdown signal so the task exits immediately on node stop, rather - // than waiting for the next message from the leader. This unblocks serve_with_shutdown - // and allows Arc (and Arc) to be released promptly after stop(). + // Single task: read requests, dispatch to the Raft loop, and forward whichever + // response becomes ready first — bounded by `max_pending` in-flight responses. tokio::spawn(async move { use futures::StreamExt; + use futures::stream::FuturesUnordered; + + let mut pending = FuturesUnordered::new(); + let mut inbound_open = true; + loop { tokio::select! { biased; _ = shutdown.changed() => { break; } - result = in_stream.next() => { + result = in_stream.next(), if inbound_open && pending.len() < max_pending => { match result { Some(Ok(req)) => { let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); @@ -188,32 +200,51 @@ where debug!("[stream_append_entries|recv] event_tx closed"); break; } - if ordered_tx.send(resp_rx).await.is_err() { - break; - } + pending.push(async move { + match resp_rx.await { + Ok(Ok(resp)) => Ok(resp), + Ok(Err(status)) => Err(status), + Err(_) => Err(Status::internal("Response channel closed")), + } + }); } Some(Err(e)) => { // Debug: expected when the peer goes away (crash/restart/shutdown), self-heals. debug!("[stream_append_entries|recv] stream error: {:?}", e); - break; + inbound_open = false; } - None => break, + None => inbound_open = false, + } + } + Some(result) = pending.next(), if !pending.is_empty() => { + // Observability only — behavior doesn't change, the send still + // runs to completion normally. If the peer isn't reading (network + // stall, dead connection with the TCP timeout not yet fired), this + // surfaces it instead of silently blocking with zero signal. + let mut send_fut = std::pin::pin!(out_tx.send(result)); + let mut stuck_logged = false; + let closed = loop { + tokio::select! { + res = &mut send_fut => break res.is_err(), + _ = tokio::time::sleep(Duration::from_secs(5)), if !stuck_logged => { + stuck_logged = true; + error!( + node_id, + "stream_append_entries forwarder stuck sending a \ + response for >5s — peer may not be reading \ + (network stall or dead connection)" + ); + metrics::counter!( + "server.grpc.stream_append_entries.forwarder_stuck" + ) + .increment(1); + } + } + }; + if closed { + break; } } - } - } - }); - - // Forwarder task: drain ordered queue sequentially (FIFO guaranteed by Raft loop) - tokio::spawn(async move { - while let Some(resp_rx) = ordered_rx.recv().await { - let result = match resp_rx.await { - Ok(Ok(resp)) => Ok(resp), - Ok(Err(status)) => Err(status), - Err(_) => Err(Status::internal("Response channel closed")), - }; - if out_tx.send(result).await.is_err() { - break; } } }); diff --git a/d-engine-server/src/network/grpc/grpc_raft_service_test.rs b/d-engine-server/src/network/grpc/grpc_raft_service_test.rs index 97d5583f..32f8de17 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service_test.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service_test.rs @@ -3,12 +3,15 @@ use std::time::Duration; use crate::ApplyResult; use d_engine_core::AppendResponseWithUpdates; use d_engine_core::InternalEvent; +use d_engine_core::MaybeCloneOneshot; +use d_engine_core::MaybeCloneOneshotReceiver; use d_engine_core::MockElectionCore; use d_engine_core::MockMembership; use d_engine_core::MockRaftLog; use d_engine_core::MockReplicationCore; use d_engine_core::MockTypeConfig; use d_engine_core::RaftNodeConfig; +use d_engine_core::RaftOneshot; use d_engine_core::convert::safe_kv_bytes; use d_engine_proto::client::ClientReadRequest; use d_engine_proto::client::ClientWriteRequest; @@ -605,3 +608,175 @@ async fn test_handle_client_scan_not_leader_carries_leader_hint_in_metadata() { Some("http://127.0.0.1:9082") ); } + +/// Historical record, not a regression guard: this is the strict-FIFO forwarder +/// pattern `stream_append_entries` used *before* #446 (one withheld response blocked +/// every later response on the same connection). It reconstructs the old primitives +/// rather than calling production code, because that code no longer exists — +/// `stream_append_entries` was rewritten to a bounded, order-tolerant forwarder (see +/// `test_stream_append_entries_does_not_block_ready_response_behind_pending_one` for +/// the real, current behavior). Kept only so a future reader can see what the old +/// failure mode looked like; do not treat this as coverage of current code. +#[tokio::test] +async fn test_ordered_forwarder_head_of_line_blocking() { + let (out_tx, mut out_rx) = mpsc::channel::>(128); + let (ordered_tx, mut ordered_rx) = mpsc::channel::< + MaybeCloneOneshotReceiver>, + >(128); + + // Mirrors grpc_raft_service.rs's forwarder loop. + tokio::spawn(async move { + while let Some(resp_rx) = ordered_rx.recv().await { + let result = match resp_rx.await { + Ok(Ok(resp)) => Ok(resp), + Ok(Err(status)) => Err(status), + Err(_) => Err(tonic::Status::internal("Response channel closed")), + }; + if out_tx.send(result).await.is_err() { + break; + } + } + }); + + // First item: never resolved — stands in for a response withheld pending durable_index. + let (_stuck_tx, stuck_rx) = MaybeCloneOneshot::new(); + ordered_tx.send(stuck_rx).await.unwrap(); + + // Second item: already resolved — stands in for an unrelated, ready-to-send response + // (e.g. a heartbeat) that arrived right after. + let (ready_tx, ready_rx) = MaybeCloneOneshot::new(); + ordered_tx.send(ready_rx).await.unwrap(); + ready_tx.send(Ok(AppendEntriesResponse::success(1, 1, None))).unwrap(); + + // The second, already-ready response must not be observable yet — it's stuck + // behind the first, unresolved one in strict FIFO order. + let blocked = time::timeout(Duration::from_millis(50), out_rx.recv()).await; + assert!( + blocked.is_err(), + "an already-ready response was blocked behind an earlier unresolved one — \ + confirms the forwarder is strict FIFO" + ); +} + +/// #446: `stream_append_entries` must not let a response still withheld (durable_index +/// hasn't caught up to what it claims) block a later, unrelated response that's already +/// answerable. Drives the real production method end-to-end — not a reconstruction — +/// via a synthetic 2-item input stream. +/// +/// request 1 claims index 10 while `durable_index()` is fixed at 5 — withheld, +/// queued in `pending_append_acks`, never released in this test. +/// request 2 claims index 3, which is `<= durable_index` — answerable immediately. +/// +/// If the forwarder is still strict FIFO, the first item out of the response stream +/// would have to be request 1's (never arrives) — this test would time out. If it's +/// the new bounded/order-tolerant forwarder, request 2's response (identifiable by its +/// distinct `last_match.term` marker) comes out first. +#[tokio::test] +async fn test_stream_append_entries_does_not_block_ready_response_behind_pending_one() { + tokio::time::pause(); + let settings = RaftNodeConfig::new().expect("Should succeed to init RaftNodeConfig."); + let mut settings = settings.validate().expect("Validate RaftNodeConfig successfully"); + settings.raft.general_raft_timeout_duration_in_ms = 200; + settings.raft.batching.max_batch_size = 1; + + let mut membership = MockMembership::::new(); + membership.expect_voters().returning(Vec::new); + membership.expect_members().returning(Vec::new); + membership.expect_replication_peers().returning(Vec::new); + membership.expect_get_peers_id_with_condition().returning(|_| vec![]); + + let mut raft_log = MockRaftLog::new(); + raft_log.expect_last_entry_id().returning(|| 0); + raft_log.expect_flush().returning(|| Ok(())); + raft_log.expect_load_hard_state().returning(|| Ok(None)); + raft_log.expect_save_hard_state().returning(|_| Ok(())); + raft_log.expect_last_log_id().returning(|| None); + // Fixed durable frontier: only request 2's claimed index (3) clears it. + raft_log.expect_durable_index().returning(|| 5); + + let call_count = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let call_count_clone = call_count.clone(); + let mut replication_handler = MockReplicationCore::::new(); + replication_handler + .expect_check_append_entries_request_is_legal() + .returning(|my_term, _, _| AppendEntriesResponse::success(1, my_term, None)); + replication_handler.expect_handle_append_entries().returning(move |_, _, _| { + let is_first = call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0; + let (claimed_index, term_marker) = if is_first { (10, 111) } else { (3, 222) }; + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success( + 1, + term_marker, + Some(LogId { + term: term_marker, + index: claimed_index, + }), + ), + commit_index_update: None, + }) + }); + + let (_graceful_tx, graceful_rx) = watch::channel(()); + let builder = MockBuilder::new(graceful_rx); + let node = builder + .with_raft_log(raft_log) + .with_membership(membership) + .with_replication_handler(replication_handler) + .with_node_config(settings) + .build_node(); + node.set_rpc_ready(true); + + let raft_lock = node.raft_core.clone(); + let _raft_handle = tokio::spawn(async move { + let mut raft = raft_lock.lock().await; + let _ = time::timeout(Duration::from_secs(5), raft.run()).await; + }); + + tokio::time::advance(Duration::from_millis(2)).await; + tokio::time::sleep(Duration::from_millis(2)).await; + + // request 1: prev_log_index=0. request 2: prev_log_index=99 — deliberately different + // from request 1's, so merge_append_entries (which only merges contiguous requests) + // can never combine them into a single handle_append_entries call. + let req1 = AppendEntriesRequest { + term: 1, + leader_id: 1, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let req2 = AppendEntriesRequest { + prev_log_index: 99, + ..req1.clone() + }; + let stream = crate::test_utils::create_test_snapshot_stream(vec![req1, req2]); + + let response = node + .stream_append_entries(Request::new(stream)) + .await + .expect("stream_append_entries must accept the request"); + use futures::StreamExt; + let mut out_stream = response.into_inner(); + + let first_out = time::timeout(Duration::from_secs(2), out_stream.next()) + .await + .expect( + "the response for request 2 (already durable) must arrive without waiting for \ + request 1 (withheld) — if this times out, the forwarder is still strict FIFO", + ) + .expect("stream must yield an item") + .expect("must be Ok, not a transport error"); + + let last_match = match first_out.result { + Some(d_engine_proto::server::replication::append_entries_response::Result::Success( + success, + )) => success.last_match.expect("success response must carry last_match"), + other => panic!("expected a success response, got {other:?}"), + }; + assert_eq!( + last_match.term, 222, + "the first response observed must be request 2's (marker term=222) — request 1 \ + (marker term=111) is still withheld and must not be observed yet, nor block this one" + ); +} diff --git a/d-engine-server/src/node/builder_test.rs b/d-engine-server/src/node/builder_test.rs index 6c3f85b5..0556b0e2 100644 --- a/d-engine-server/src/node/builder_test.rs +++ b/d-engine-server/src/node/builder_test.rs @@ -7,7 +7,6 @@ use d_engine_core::LogStore; use d_engine_core::MockStateMachine; use d_engine_core::MockStorageEngine; use d_engine_core::PersistenceConfig; -use d_engine_core::PersistenceStrategy; use d_engine_core::RaftNodeConfig; use d_engine_core::StateMachine; use d_engine_core::StorageEngine; @@ -58,7 +57,6 @@ async fn test_set_raft_log_replaces_default() { BufferedRaftLog::>::new( id, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/src/test_utils/integration/mod.rs b/d-engine-server/src/test_utils/integration/mod.rs index 838f4c9c..00365525 100644 --- a/d-engine-server/src/test_utils/integration/mod.rs +++ b/d-engine-server/src/test_utils/integration/mod.rs @@ -49,7 +49,6 @@ use d_engine_core::FlushPolicy; use d_engine_core::LogSizePolicy; use d_engine_core::MockStateMachine; use d_engine_core::PersistenceConfig; -use d_engine_core::PersistenceStrategy; use d_engine_core::RaftLog; use d_engine_core::RaftNodeConfig; use d_engine_core::ReplicationHandler; @@ -181,7 +180,6 @@ pub fn setup_raft_components( let (buffered_raft_log, receiver) = BufferedRaftLog::new( id, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/common/mod.rs b/d-engine-server/tests/common/mod.rs index 2d10210a..7dfd1b17 100644 --- a/d-engine-server/tests/common/mod.rs +++ b/d-engine-server/tests/common/mod.rs @@ -13,7 +13,6 @@ use d_engine_core::config::BackoffPolicy; use d_engine_core::config::ElectionConfig; use d_engine_core::config::FlushPolicy; use d_engine_core::config::PersistenceConfig; -use d_engine_core::config::PersistenceStrategy; use d_engine_core::config::RaftConfig; use d_engine_core::config::RaftNodeConfig; use d_engine_core::config::SnapshotConfig; @@ -124,7 +123,6 @@ pub async fn create_node_config( ] [raft.persistence] - strategy = "MemFirst" flush_policy = {{ Batch = {{ threshold = 100, idle_flush_interval_ms = 1 }} }} [raft.election] @@ -174,7 +172,6 @@ pub async fn create_node_config_with_role( ] [raft.persistence] - strategy = "MemFirst" flush_policy = {{ Batch = {{ threshold = 1, idle_flush_interval_ms = 1 }} }} [raft.election] @@ -218,7 +215,6 @@ pub fn node_config(cluster_toml: &str) -> RaftNodeConfig { ..Default::default() }, persistence: PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs b/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs index de85eebd..ef0f53e2 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs @@ -8,9 +8,7 @@ use std::sync::Arc; use std::time::Duration; use bytes::Bytes; -use d_engine_core::{ - BufferedRaftLog, FlushPolicy, PersistenceConfig, PersistenceStrategy, RaftLog, -}; +use d_engine_core::{BufferedRaftLog, FlushPolicy, PersistenceConfig, RaftLog}; use d_engine_proto::common::{Entry, EntryPayload}; use d_engine_server::{FileStateMachine, FileStorageEngine, node::RaftTypeConfig}; use tokio::time::sleep; @@ -21,7 +19,6 @@ use super::TestContext; async fn test_crash_recovery() { // Create and populate storage let original_ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -65,7 +62,6 @@ async fn test_crash_recovery() { async fn test_crash_recovery_with_multiple_entries() { // Create and populate storage let original_ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -128,7 +124,6 @@ async fn test_partial_flush_with_graceful_shutdown() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -165,7 +160,6 @@ async fn test_partial_flush_with_graceful_shutdown() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -210,7 +204,6 @@ async fn test_partial_flush_after_crash() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -259,7 +252,6 @@ async fn test_partial_flush_after_crash() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -299,21 +291,18 @@ async fn test_recovery_under_different_scenarios() { // drain cycle, so all 100 entries are always durable after explicit flush(). let scenarios = vec![ ( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, 100usize, ), ( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 10, }, 100, ), ( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1000, }, @@ -321,9 +310,9 @@ async fn test_recovery_under_different_scenarios() { ), ]; - for (strategy, flush_policy, expected_recovery) in scenarios { - let instance_id = format!("recovery_test_{strategy:?}_{flush_policy:?}"); - let original_ctx = TestContext::new(strategy.clone(), flush_policy.clone(), &instance_id); + for (flush_policy, expected_recovery) in scenarios { + let instance_id = format!("recovery_test_{flush_policy:?}"); + let original_ctx = TestContext::new(flush_policy.clone(), &instance_id); // Add test data for i in 1..=100 { @@ -351,7 +340,7 @@ async fn test_recovery_under_different_scenarios() { assert_eq!( recovered_ctx.raft_log.len(), expected_recovery, - "Recovery mismatch for strategy {strategy:?} policy {flush_policy:?}" + "Recovery mismatch for policy {flush_policy:?}" ); recovered_ctx.close().await; } @@ -363,7 +352,6 @@ async fn test_memfirst_crash_recovery_durability() { let recovered_path = { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 10000, }, @@ -390,7 +378,6 @@ async fn test_memfirst_crash_recovery_durability() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -424,7 +411,6 @@ async fn test_diskfirst_crash_recovery_durability() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -458,7 +444,6 @@ async fn test_diskfirst_crash_recovery_durability() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/storage_buffered_raft_log/mod.rs b/d-engine-server/tests/storage_buffered_raft_log/mod.rs index ad11cddb..8687f2a4 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/mod.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/mod.rs @@ -15,15 +15,14 @@ use std::sync::Arc; use std::time::Duration; use bytes::Bytes; -use d_engine_core::{ - BufferedRaftLog, FlushPolicy, PersistenceConfig, PersistenceStrategy, RaftLog, alias::ROF, -}; +use d_engine_core::{BufferedRaftLog, FlushPolicy, PersistenceConfig, RaftLog, alias::ROF}; use d_engine_proto::common::{Entry, EntryPayload}; use d_engine_server::{FileStateMachine, FileStorageEngine, node::RaftTypeConfig}; use tempfile::tempdir; mod crash_recovery_test; mod performance_test; +mod quorum_crash_recovery_test; mod storage_integration_test; mod stress_test; @@ -32,7 +31,6 @@ pub struct TestContext { pub raft_log: Arc>>, pub storage: Arc, pub _temp_dir: Option, - pub strategy: PersistenceStrategy, pub flush_policy: FlushPolicy, pub path: String, } @@ -40,7 +38,6 @@ pub struct TestContext { impl TestContext { /// Create new test context with FileStorageEngine pub fn new( - strategy: PersistenceStrategy, flush_policy: FlushPolicy, instance_id: &str, ) -> Self { @@ -51,7 +48,6 @@ impl TestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), max_buffered_entries: 10000, shutdown_timeout_ms: 5000, @@ -67,7 +63,6 @@ impl TestContext { path: path.to_str().unwrap().to_string(), raft_log, storage, - strategy, flush_policy, _temp_dir: Some(temp_dir), } @@ -92,7 +87,6 @@ impl TestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), max_buffered_entries: 10000, shutdown_timeout_ms: 5000, @@ -106,7 +100,6 @@ impl TestContext { Self { raft_log, storage, - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), _temp_dir: Some(temp_dir), path: self.path.clone(), diff --git a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs index 453e17e7..1d050f53 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs @@ -6,9 +6,7 @@ use super::TestContext; use bytes::Bytes; -use d_engine_core::{ - BufferedRaftLog, FlushPolicy, PersistenceConfig, PersistenceStrategy, RaftLog, -}; +use d_engine_core::{BufferedRaftLog, FlushPolicy, PersistenceConfig, RaftLog}; use d_engine_proto::common::{Entry, EntryPayload}; use d_engine_server::{FileStateMachine, FileStorageEngine, node::RaftTypeConfig}; use std::collections::HashMap; @@ -33,7 +31,6 @@ mod filter_out_conflicts_and_append_performance_tests { for (idle_flush_interval_ms, max_duration_ms) in test_cases { // Create MemFirst storage with batch policy let config = PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, @@ -104,7 +101,6 @@ mod filter_out_conflicts_and_append_performance_tests { for (idle_flush_interval_ms, max_duration_ms) in test_cases { // Create MemFirst storage with batch policy let config = PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, @@ -171,7 +167,6 @@ mod filter_out_conflicts_and_append_performance_tests { async fn test_last_entry_id_performance() { // Set up test context let test_context = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 360_000, }, @@ -252,7 +247,6 @@ async fn test_performance_benchmarks() { }; let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -342,7 +336,6 @@ async fn test_read_performance_under_concurrent_write_load() { }; let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, diff --git a/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs b/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs new file mode 100644 index 00000000..351e5a8d --- /dev/null +++ b/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs @@ -0,0 +1,108 @@ +//! Quorum + real-disk crash recovery integration test (#446 gap 4). +//! +//! Composes two pieces that are each already covered in isolation elsewhere, but never +//! together: `calculate_majority_matched_index` (RPO=0 quorum arithmetic, unit-tested +//! against a gated mock in `buffered_raft_log_test/quorum_durability_test.rs`) and real +//! `FileStorageEngine` crash/reopen (unit-tested without any quorum math in +//! `crash_recovery_test.rs`). This file proves they actually compose: an index that the +//! quorum calculation says is safe to acknowledge to the client is still there after a +//! real crash + reopen from the same on-disk path. +//! +//! Followers are represented as reported match_index values, same as in +//! `quorum_durability_test.rs` — this file's job is the leader-side real-disk durability +//! boundary, not follower ACK withholding (covered by follower_state_test.rs / +//! learner_state_test.rs). +//! +//! Deliberately NOT attempted here, and now CONFIRMED impossible with this engine's +//! architecture (not just a flakiness risk — an actual dead end, verified by building and +//! deadlocking it): proving that an entry which never reached quorum-durable is genuinely +//! absent from a real crash + reopen. `BufferedRaftLog::append_entries` +//! (`d-engine-core/src/storage/buffered_raft_log.rs:467-500`) is documented and +//! implemented to block the caller until `persist_entries()` returns — "still blocks the +//! caller until truly persisted" — and `FileLogStore::persist_entries` +//! (`d-engine-server/src/storage/adaptors/file/file_storage_engine.rs:254`) already writes +//! the entry to the real OS-visible file as an unconditional part of its body, before it +//! can return. So by the time `append_entries().await` ever resolves at all, the entry is +//! already on the file — there is no window where it's "acknowledged as appended" yet +//! "recoverably absent." A gate on `persist_entries()` was built and tried here; it did +//! not create the intended window, it just deadlocked `append_entries()` forever (the +//! call this file's other test depends on to make progress at all). Reverted. +//! What IS real and already correctly tested (see below): the gap between +//! `last_entry_id` and `durable_index` — `persist_entries()` writes the bytes, but a +//! *separate* `flush()` call (`sync_all()`) is what advances `durable_index`, and that one +//! genuinely runs later/independently. What's NOT reachable by a same-process test is +//! observing that an un-`sync_all`'d write doesn't survive — on the same OS instance, +//! `write()` alone (which `persist_entries` already does) is enough for a freshly-opened +//! handle to see the bytes, real crash or not. Proving the un-fsynced case would need an +//! actual power-loss simulation (dropped page cache / real reboot) — which prior sessions +//! already found to be a poor fit for this class of bug (see mempalace notes on the +//! Jepsen/lazyfs work for #444). + +use super::TestContext; +use d_engine_core::FlushPolicy; +use d_engine_core::RaftLog; + +#[tokio::test] +async fn test_quorum_acknowledged_index_survives_real_crash_and_reopen() { + let ctx = TestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, + "test_quorum_ack_survives_crash", + ); + + // First 5 entries, explicitly flushed: genuinely durable, deterministic. + ctx.append_entries(1, 5, 1).await; + ctx.raft_log.flush().await.unwrap(); + assert_eq!(ctx.raft_log.durable_index(), 5); + + // 3-node cluster: both followers already report match_index=5 (post-Stage2 + // semantics — a follower only reports a match_index once its own durable_index + // reaches it). This is the index the leader would actually acknowledge to the + // client. + let commit = ctx.raft_log.calculate_majority_matched_index(1, 0, vec![5, 5]); + assert_eq!( + commit, + Some(5), + "index 5 is durable on the leader and acked by both followers" + ); + + // A follower report of 10 must not move commit past what the leader itself has + // fsynced — restates the Stage1 invariant as this test's own setup precondition + // rather than assuming it silently. + let would_be_wrong = ctx.raft_log.calculate_majority_matched_index(1, 0, vec![10, 5]); + assert_eq!( + would_be_wrong, + Some(5), + "leader's own un-fsynced tail must not leak into the client-visible commit index" + ); + + // Second batch, also explicitly flushed, so the whole log is durable before the + // simulated crash — keeps this test's crash/recovery assertions exact, not bounded. + ctx.append_entries(6, 5, 1).await; + ctx.raft_log.flush().await.unwrap(); + assert_eq!(ctx.raft_log.durable_index(), 10); + + let recovered = ctx.recover_from_crash(); + ctx.close().await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + // The index that was actually acknowledged to the client (5) — and everything else + // that was durably flushed (up to 10) — survives the real crash + reopen. + assert_eq!(recovered.raft_log.durable_index(), 10); + for i in 1..=10 { + assert!( + recovered.raft_log.entry(i).unwrap().is_some(), + "entry {i} must survive real crash + reopen" + ); + } + + // Re-running the same quorum calculation against the recovered log reaches the same + // conclusion — the leader's durability contribution to quorum is stable across a + // real restart, not just in the pre-crash in-memory view. + let commit_after_recovery = + recovered.raft_log.calculate_majority_matched_index(1, 0, vec![5, 5]); + assert_eq!(commit_after_recovery, Some(5)); + + recovered.close().await; +} diff --git a/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs b/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs index 8cd4ab85..f9c9dfe6 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs @@ -3,7 +3,7 @@ //! These tests verify BufferedRaftLog integration with FileStorageEngine //! at the storage layer, including compaction and storage-specific operations. -use d_engine_core::{FlushPolicy, PersistenceStrategy, RaftLog}; +use d_engine_core::{FlushPolicy, RaftLog}; use d_engine_proto::common::LogId; use super::TestContext; @@ -14,7 +14,6 @@ use super::TestContext; #[tokio::test] async fn test_log_compaction() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs index 44972c8a..3f41a4c0 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs @@ -6,7 +6,7 @@ use std::time::Duration; use bytes::Bytes; -use d_engine_core::{FlushPolicy, LogStore, PersistenceStrategy, RaftLog, StorageEngine}; +use d_engine_core::{FlushPolicy, LogStore, RaftLog, StorageEngine}; use d_engine_proto::common::{Entry, EntryPayload}; use futures::future::join_all; use tokio::time::Instant; @@ -23,7 +23,6 @@ use super::TestContext; #[tokio::test] async fn test_high_concurrency() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -63,7 +62,6 @@ async fn test_high_concurrency() { #[traced_test] async fn test_high_concurrency_mixed_operations() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -142,7 +140,6 @@ mod mem_first_tests { #[tokio::test] async fn test_basic_write_before_persist() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -158,7 +155,6 @@ mod mem_first_tests { #[tokio::test] async fn test_async_persistence() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -177,7 +173,6 @@ mod mem_first_tests { #[tokio::test] async fn test_power_loss_data_loss() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -195,7 +190,6 @@ mod mem_first_tests { #[tokio::test] async fn test_high_concurrency_memory_only() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -232,7 +226,6 @@ mod mem_first_tests { #[tokio::test] async fn test_term_index_correctness_under_load() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs b/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs index b5edc3e4..051bd020 100644 --- a/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs +++ b/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs @@ -125,8 +125,6 @@ learner_check_throttle_ms = 100 election_timeout_min = 300 election_timeout_max = 3000 -[raft.persistence] -strategy = "MemFirst" [retry.election] max_retries = 5 diff --git a/d-engine/src/docs/examples/three-nodes-standalone.md b/d-engine/src/docs/examples/three-nodes-standalone.md index 96cbe01a..b5432c04 100644 --- a/d-engine/src/docs/examples/three-nodes-standalone.md +++ b/d-engine/src/docs/examples/three-nodes-standalone.md @@ -59,7 +59,6 @@ default_policy = "LeaseRead" lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" # Only strategy in v0.2.4+ (DiskFirst removed) flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } max_buffered_entries = 10000 ``` @@ -123,7 +122,7 @@ All performance reports in `/benches/standalone-bench/reports` use this exact co **Raft settings:** -- Persistence: `MemFirst` (only strategy in v0.2.4+) with 1000ms idle flush interval +- Persistence: batched fsync (Level 3, fdatasync) with 1000ms idle flush interval - Read consistency: `LeaseRead` (500ms lease duration) - Replication: Batched append entries (5000 threshold, 0ms delay) - Network: Tuned for high throughput (see `config/n1.toml` for details) diff --git a/d-engine/src/docs/performance/throughput-optimization-guide.md b/d-engine/src/docs/performance/throughput-optimization-guide.md index 03bc4efc..1fe3341e 100644 --- a/d-engine/src/docs/performance/throughput-optimization-guide.md +++ b/d-engine/src/docs/performance/throughput-optimization-guide.md @@ -23,18 +23,6 @@ pub(crate) enum ConnectionType { ## Persistence Strategy & Throughput/Latency Trade-offs -`MemFirst` is the only persistence strategy in v0.2.4+. It batches writes to OS page cache and flushes with fsync asynchronously — committing data to disk before notifying Raft. - -### Strategy Configuration - -```toml -[raft.persistence] -strategy = "MemFirst" -flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -``` - -### `MemFirst` Strategy - - **Write Path**: Entries are written to OS page cache via `db.write()` / `file.write()`; the IO thread batches them and calls fsync (`flush_wal(true)` / `sync_all()`) before advancing `durable_index`. Raft only counts an entry toward quorum after fsync completes. @@ -54,8 +42,7 @@ flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } - Lower values reduce the unflushed batch window but increase IO pressure. - Default `1000` ms is suitable for most workloads. -> **Note**: `DiskFirst` strategy was removed in v0.2.4. `MemFirst` replaced it with batched -> fsync — multiple writes share one fsync call, reducing IO overhead while still providing +> **Note**: Writes are batched into a single fsync, reducing IO overhead while still providing > disk-level durability for all client-acknowledged (committed) writes. ## Batching Configuration @@ -179,7 +166,7 @@ tonic::transport::Server::builder() ``` -Inbound message size is the one setting that *is* per-service rather than +Inbound message size is the one setting that _is_ per-service rather than transport-wide, so it's applied on each `XxxServiceServer` individually: ```rust,ignore @@ -197,7 +184,7 @@ RaftReplicationServiceServer::from_arc(node.clone()) | p99.9 Latency | 14015 µs | 11279 µs | -19.5% | > **Key improvement**: 15% reduction in tail latency - critical for consensus stability -> **Note**: These metrics show the impact of connection pooling optimization. These results can be further improved by tuning the PersistenceStrategy for your specific workload. +> **Note**: These metrics show the impact of connection pooling optimization. These results can be further improved by tuning `FlushPolicy` for your specific workload. > > For absolute performance benchmarks, see [v0.2.4 Performance Report](https://github.com/deventlab/d-engine/tree/main/benches/reports/v0.2.4/bench_report_v0.2.4.md) @@ -230,7 +217,7 @@ RaftReplicationServiceServer::from_arc(node.clone()) ``` -5. **Monitor Flush Lag**: When using `MemFirst`, monitor the difference between `last_log_index` and `durable_index`. A growing gap indicates the disk is not keeping up with writes, increasing potential data loss. +5. **Monitor Flush Lag**: Monitor the difference between `last_log_index` and `durable_index`. Raft only counts an entry toward quorum and acknowledges it to the client after fsync — so a growing gap does not put acknowledged writes at risk. It does mean client-facing write latency is growing, and (if the gap keeps growing) the amount of work an unflushed batch would need to redo on restart is growing too. ## Anti-Patterns to Avoid @@ -244,11 +231,9 @@ get_peer_channel(peer_id, ConnectionType::Control).await?; client.request_vote(...) // DON'T: Set idle_flush_interval_ms too low — defeats batching. -[strategy = "MemFirst"] flush_policy = { Batch = { idle_flush_interval_ms = 1 } } // Near-synchronous; low throughput // DO: Use a generous idle interval to amortize disk I/O cost. -[strategy = "MemFirst"] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } ``` @@ -261,8 +246,8 @@ flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } Control: Low latency ↔ Data: High throughput ↔ Bulk: Bandwidth 3. **Improves fault containment** Connection issues affect only one operation type -4. **Decouples Performance from Durability** - `MemFirst` with tunable `idle_flush_interval_ms` lets you balance write throughput against flush frequency. +4. **Decouples Performance from Ack Latency** + Client-acknowledged writes are always fsync-durable — that's not tunable. `idle_flush_interval_ms` lets you balance write throughput against how long a client waits for that fsync. ## Reference Deployment Configurations @@ -277,7 +262,6 @@ Adjust values based on snapshot size, log append rate, and cluster size. ```toml [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [network.control] @@ -306,7 +290,6 @@ max_concurrent_streams = 128 ```toml [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [network.control] @@ -325,7 +308,7 @@ max_concurrent_streams = 256 ``` -**Tip**: For public cloud, moderate concurrency and 32MB bulk windows ensure stable snapshot streaming without affecting heartbeats. The batch policy is tuned for high throughput with a reasonable data loss window. +**Tip**: For public cloud, moderate concurrency and 32MB bulk windows ensure stable snapshot streaming without affecting heartbeats. The batch policy is tuned for high throughput; acknowledged writes are never at risk regardless of the interval, only ack latency and unflushed-batch replay time on restart scale with it. ### 3. 5-Node High-Durability Cluster (Production) @@ -335,8 +318,7 @@ max_concurrent_streams = 256 ```toml [raft.persistence] -strategy = "MemFirst" -flush_policy = { Batch = { idle_flush_interval_ms = 100 } } # More frequent flush for durability +flush_policy = { Batch = { idle_flush_interval_ms = 100 } } # More frequent flush, shorter ack latency [network.control] connection_window_size = 4_194_304 # 4MB @@ -354,7 +336,7 @@ max_concurrent_streams = 512 ``` -**Tip**: For higher write persistence within a process lifecycle, lower `idle_flush_interval_ms` (e.g., 100ms). Note: `MemFirst` is not power-loss safe regardless of flush interval. +**Tip**: Lower `idle_flush_interval_ms` (e.g., 100ms) shortens client-facing write latency and shrinks the unflushed-batch window an IO thread has to redo on restart. Acknowledged writes are power-loss safe regardless of this setting — Raft only counts an entry toward quorum, and acknowledges it to the client, after fsync completes. ## Network Environment Tuning Recommendations @@ -362,12 +344,12 @@ These parameters are primarily **network-dependent**, not CPU/memory dependent. Adjust them based on latency, packet loss, and connection stability. -| **Environment** | **tcp_keepalive_in_secs** | **http2_keep_alive_interval_in_secs** | **http2_keep_alive_timeout_in_secs** | **Notes** | -| -------------------------------- | ------------------------- | ------------------------------------- | ------------------------------------ | ------------------------------------------------------- | -| **Local / In-Cluster (LAN)** | 60 | 10 | 5 | Low latency & stable; defaults are fine | -| **Cross-Region / Stable WAN** | 60 | 15 | 8 | Slightly longer keep-alive to avoid false disconnects | -| **Public Cloud / Moderate Loss** | 60 | 20 | 10 | Higher interval & timeout for lossy links | -| **High Latency / Unstable WAN** | 120 | 30 | 15 | Longer timeouts prevent spurious drops | +| **Environment** | **tcp_keepalive_in_secs** | **http2_keep_alive_interval_in_secs** | **http2_keep_alive_timeout_in_secs** | **Notes** | +| -------------------------------- | ------------------------- | ------------------------------------- | ------------------------------------ | ----------------------------------------------------- | +| **Local / In-Cluster (LAN)** | 60 | 10 | 5 | Low latency & stable; defaults are fine | +| **Cross-Region / Stable WAN** | 60 | 15 | 8 | Slightly longer keep-alive to avoid false disconnects | +| **Public Cloud / Moderate Loss** | 60 | 20 | 10 | Higher interval & timeout for lossy links | +| **High Latency / Unstable WAN** | 120 | 30 | 15 | Longer timeouts prevent spurious drops | **Guidelines:** diff --git a/d-engine/src/docs/server_guide/customize-storage-engine.md b/d-engine/src/docs/server_guide/customize-storage-engine.md index 0874acb0..ef9b088b 100644 --- a/d-engine/src/docs/server_guide/customize-storage-engine.md +++ b/d-engine/src/docs/server_guide/customize-storage-engine.md @@ -91,7 +91,7 @@ impl StorageEngine for CustomStorageEngine { - **Consistency**: Maintain exactly-once semantics for log entries - **Performance**: Target >100k ops/sec for log persistence. Do not call `fsync` inside `persist_entries()`—the framework batches entries and calls `flush()` once per batch - (`MemFirst + FlushPolicy::Batch`), which amortises the `fsync` cost across many entries. + (`FlushPolicy::Batch`), which amortises the `fsync` cost across many entries. - **Resource Management**: Clean up resources in `Drop` implementation ## 3. StorageEngine API Reference diff --git a/examples/single-node-expansion/config/n1.toml b/examples/single-node-expansion/config/n1.toml index 712a3935..d036239d 100644 --- a/examples/single-node-expansion/config/n1.toml +++ b/examples/single-node-expansion/config/n1.toml @@ -15,7 +15,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -43,11 +43,8 @@ max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/single-node-expansion/config/n2.toml b/examples/single-node-expansion/config/n2.toml index 5e5356c3..bad8ce26 100644 --- a/examples/single-node-expansion/config/n2.toml +++ b/examples/single-node-expansion/config/n2.toml @@ -28,7 +28,6 @@ lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 20 } } max_buffered_entries = 10000 diff --git a/examples/single-node-expansion/config/n3.toml b/examples/single-node-expansion/config/n3.toml index 75a585e3..67cf1fd2 100644 --- a/examples/single-node-expansion/config/n3.toml +++ b/examples/single-node-expansion/config/n3.toml @@ -30,7 +30,6 @@ lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 20 } } max_buffered_entries = 10000 diff --git a/examples/sled-cluster/config/n1.toml b/examples/sled-cluster/config/n1.toml index a5b391e8..5902a6ad 100644 --- a/examples/sled-cluster/config/n1.toml +++ b/examples/sled-cluster/config/n1.toml @@ -16,8 +16,6 @@ batch_size = 5000 [raft.persistence] -strategy = "MemFirst" -# strategy = "DiskFirst" flush_policy = { Batch = { idle_flush_interval_ms = 100 } } [raft.snapshot] diff --git a/examples/sled-cluster/config/n2.toml b/examples/sled-cluster/config/n2.toml index 87a5f29a..c7a89b70 100644 --- a/examples/sled-cluster/config/n2.toml +++ b/examples/sled-cluster/config/n2.toml @@ -16,8 +16,6 @@ batch_size = 5000 [raft.persistence] -strategy = "MemFirst" -# strategy = "DiskFirst" flush_policy = { Batch = { idle_flush_interval_ms = 100 } } [raft.snapshot] diff --git a/examples/sled-cluster/config/n3.toml b/examples/sled-cluster/config/n3.toml index 5fe5b628..c099227a 100644 --- a/examples/sled-cluster/config/n3.toml +++ b/examples/sled-cluster/config/n3.toml @@ -16,8 +16,6 @@ batch_size = 5000 [raft.persistence] -strategy = "MemFirst" -# strategy = "DiskFirst" flush_policy = { Batch = { idle_flush_interval_ms = 100 } } diff --git a/examples/three-nodes-embedded/README.md b/examples/three-nodes-embedded/README.md index 25255a59..b6f2dda2 100644 --- a/examples/three-nodes-embedded/README.md +++ b/examples/three-nodes-embedded/README.md @@ -84,7 +84,6 @@ default_policy = "LeaseRead" lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { threshold = 100, interval_ms = 20 } } ``` diff --git a/examples/three-nodes-standalone/config/n1.toml b/examples/three-nodes-standalone/config/n1.toml index a040e1ae..bb6db45a 100644 --- a/examples/three-nodes-standalone/config/n1.toml +++ b/examples/three-nodes-standalone/config/n1.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/three-nodes-standalone/config/n2.toml b/examples/three-nodes-standalone/config/n2.toml index 20e816f2..95280d76 100644 --- a/examples/three-nodes-standalone/config/n2.toml +++ b/examples/three-nodes-standalone/config/n2.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/three-nodes-standalone/config/n3.toml b/examples/three-nodes-standalone/config/n3.toml index 0a29fe2d..4aa67b3b 100644 --- a/examples/three-nodes-standalone/config/n3.toml +++ b/examples/three-nodes-standalone/config/n3.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/three-nodes-standalone/docker/config/n1.toml b/examples/three-nodes-standalone/docker/config/n1.toml index efa8e941..8deff325 100644 --- a/examples/three-nodes-standalone/docker/config/n1.toml +++ b/examples/three-nodes-standalone/docker/config/n1.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/three-nodes-standalone/docker/config/n2.toml b/examples/three-nodes-standalone/docker/config/n2.toml index 78f66899..7e7eb644 100644 --- a/examples/three-nodes-standalone/docker/config/n2.toml +++ b/examples/three-nodes-standalone/docker/config/n2.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] diff --git a/examples/three-nodes-standalone/docker/config/n3.toml b/examples/three-nodes-standalone/docker/config/n3.toml index 674109a2..73d6bc59 100644 --- a/examples/three-nodes-standalone/docker/config/n3.toml +++ b/examples/three-nodes-standalone/docker/config/n3.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,11 +39,8 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } # Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) max_buffered_entries = 10000 [raft.snapshot] From d9a5b1730cbfc7d244cf6043cde7116bfe8dc231 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:12:56 +0800 Subject: [PATCH 03/11] chore #446: bump grpc-go and x/net to resolve Dependabot alerts (#86-91) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Upgrade google.golang.org/grpc v1.80.0→v1.83.2 (d-engine-proto/go, examples/quick-start-standalone) - Upgrade golang.org/x/net v0.53.0→v0.58.0 in both modules - Fixes: xDS RBAC authz bypass, HTTP/2 rapid-reset DoS, RBAC parser panic, HTTP/2 DATA frame OOM, x/net HTML parser DoS --- d-engine-proto/go/go.mod | 10 +++---- d-engine-proto/go/go.sum | 40 +++++++++++++------------- examples/quick-start-standalone/go.mod | 10 +++---- examples/quick-start-standalone/go.sum | 40 +++++++++++++------------- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/d-engine-proto/go/go.mod b/d-engine-proto/go/go.mod index 68f93120..b5dd4aa2 100644 --- a/d-engine-proto/go/go.mod +++ b/d-engine-proto/go/go.mod @@ -3,13 +3,13 @@ module github.com/deventlab/d-engine/proto go 1.25.0 require ( - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.11 ) require ( - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect ) diff --git a/d-engine-proto/go/go.sum b/d-engine-proto/go/go.sum index adb3ad1c..2d3ad8d1 100644 --- a/d-engine-proto/go/go.sum +++ b/d-engine-proto/go/go.sum @@ -12,27 +12,27 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/examples/quick-start-standalone/go.mod b/examples/quick-start-standalone/go.mod index 79b1ef9c..9865cf61 100644 --- a/examples/quick-start-standalone/go.mod +++ b/examples/quick-start-standalone/go.mod @@ -6,13 +6,13 @@ replace github.com/deventlab/d-engine/proto => ../../d-engine-proto/go require ( github.com/deventlab/d-engine/proto v0.0.0 - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.83.2 ) require ( - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/examples/quick-start-standalone/go.sum b/examples/quick-start-standalone/go.sum index adb3ad1c..2d3ad8d1 100644 --- a/examples/quick-start-standalone/go.sum +++ b/examples/quick-start-standalone/go.sum @@ -12,27 +12,27 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= From 97eb1c0337006c4e838ecfbecef89b1d42978477 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:46:01 +0800 Subject: [PATCH 04/11] fix #446: single-writer durable_index/persisted_index, close truncation TOCTOU race - durable_index: sole writer is raft.rs's event loop, content-validated via try_advance_durable_index(index, term) against entry_term(index) - persisted_index: sole writer is the IO thread, clamp moved from remove_range into IOTask::ReplaceRange - remove_range keeps its synchronous durable_index clamp (flush() short-circuit depends on it) - rename handle_non_write_cmd -> run_storage_tasks, max_index -> memory_max_index - remove dead config max_buffered_entries + 12 example/bench TOML configs - test: 22 d-engine-core + 5 d-engine-server tests updated for the new drain-fsync-completions pattern; new content_validated_watermark_test.rs --- benches/embedded-bench/config/n1.toml | 2 - benches/embedded-bench/config/n2.toml | 2 - benches/embedded-bench/config/n3.toml | 2 - d-engine-core/src/config/raft.rs | 13 -- d-engine-core/src/event.rs | 8 + d-engine-core/src/raft.rs | 9 + .../src/storage/buffered_raft_log.rs | 142 ++++++------ .../concurrent_fsync_test.rs | 99 ++++----- .../content_validated_watermark_test.rs | 204 ++++++++++++++++++ .../drain_fsync_test.rs | 33 +-- .../durable_index_test.rs | 5 +- .../flush_strategy_test.rs | 15 +- .../id_allocation_test.rs | 1 - .../performance_test.rs | 3 - .../persisted_index_clamp_test.rs | 4 +- .../pipeline_overlap_test.rs | 1 - .../process_crash_safety_test.rs | 1 - .../quorum_durability_test.rs | 12 +- .../raft_properties_test.rs | 12 +- .../replace_range_fsync_test.rs | 6 +- .../buffered_raft_log_test/shutdown_test.rs | 3 - .../truncation_fsync_fence_test.rs | 1 - .../src/storage/fsync_coordinator.rs | 27 +-- .../src/storage/fsync_coordinator_test.rs | 76 ++++++- d-engine-core/src/storage/raft_log.rs | 12 ++ .../buffered_raft_log_test_helpers.rs | 48 ++++- .../test_utils/mock/mock_storage_engine.rs | 4 +- d-engine-server/src/node/builder_test.rs | 1 - .../src/test_utils/integration/mod.rs | 1 - .../crash_recovery_test.rs | 10 +- .../tests/storage_buffered_raft_log/mod.rs | 27 ++- .../performance_test.rs | 2 - .../quorum_crash_recovery_test.rs | 4 +- .../storage_integration_test.rs | 3 +- .../storage_buffered_raft_log/stress_test.rs | 6 +- .../docs/examples/three-nodes-standalone.md | 1 - examples/single-node-expansion/config/n1.toml | 2 - examples/single-node-expansion/config/n2.toml | 1 - examples/single-node-expansion/config/n3.toml | 1 - .../three-nodes-standalone/config/n1.toml | 2 - .../three-nodes-standalone/config/n2.toml | 2 - .../three-nodes-standalone/config/n3.toml | 2 - .../docker/config/n1.toml | 2 - .../docker/config/n2.toml | 2 - .../docker/config/n3.toml | 2 - 45 files changed, 561 insertions(+), 255 deletions(-) create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs diff --git a/benches/embedded-bench/config/n1.toml b/benches/embedded-bench/config/n1.toml index 8dec8c84..da9af610 100644 --- a/benches/embedded-bench/config/n1.toml +++ b/benches/embedded-bench/config/n1.toml @@ -17,8 +17,6 @@ max_batch_size = 200 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.metrics] enable_backpressure = false diff --git a/benches/embedded-bench/config/n2.toml b/benches/embedded-bench/config/n2.toml index 455effff..1e10d048 100644 --- a/benches/embedded-bench/config/n2.toml +++ b/benches/embedded-bench/config/n2.toml @@ -17,8 +17,6 @@ max_batch_size = 200 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.metrics] enable_backpressure = false diff --git a/benches/embedded-bench/config/n3.toml b/benches/embedded-bench/config/n3.toml index 1296b03d..96f32af8 100644 --- a/benches/embedded-bench/config/n3.toml +++ b/benches/embedded-bench/config/n3.toml @@ -17,8 +17,6 @@ max_batch_size = 200 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.metrics] enable_backpressure = false diff --git a/d-engine-core/src/config/raft.rs b/d-engine-core/src/config/raft.rs index 8e03f2f5..60a7456a 100644 --- a/d-engine-core/src/config/raft.rs +++ b/d-engine-core/src/config/raft.rs @@ -847,13 +847,6 @@ pub struct PersistenceConfig { #[serde(default = "default_flush_policy")] pub flush_policy: FlushPolicy, - /// Maximum number of in-memory log entries to buffer when using async strategies - /// - /// This acts as a safety valve to prevent memory exhaustion during periods of - /// high write throughput or when disk persistence is slow. - #[serde(default = "default_max_buffered_entries")] - pub max_buffered_entries: usize, - /// Maximum time to wait, on shutdown, for an in-flight fsync task to finish /// before giving up. Bounds close() against a stuck/slow disk — the task /// itself is not cancelled, it keeps running in the background regardless. @@ -871,11 +864,6 @@ fn default_flush_policy() -> FlushPolicy { } } -/// Default maximum buffered log entries -fn default_max_buffered_entries() -> usize { - 10_000 -} - fn default_shutdown_timeout_ms() -> u64 { 5_000 } @@ -904,7 +892,6 @@ impl Default for PersistenceConfig { fn default() -> Self { Self { flush_policy: default_flush_policy(), - max_buffered_entries: default_max_buffered_entries(), shutdown_timeout_ms: default_shutdown_timeout_ms(), } } diff --git a/d-engine-core/src/event.rs b/d-engine-core/src/event.rs index 3f35dbf3..9c756914 100644 --- a/d-engine-core/src/event.rs +++ b/d-engine-core/src/event.rs @@ -71,6 +71,14 @@ pub enum InternalEvent { durable_index: u64, }, + /// Raw fsync-completion signal — NOT yet validated. Consumer must call + /// `raft_log().try_advance_durable_index(index, term)`, which re-checks + /// content before actually advancing `durable_index`. + FsyncCompleted { + index: u64, + term: u64, + }, + /// AppendEntries result from a per-follower ReplicationWorker back to the Raft loop. /// Leader processes this in handle_append_result: updates match_index, re-calculates commit, /// and drains pending_client_writes when quorum is achieved. diff --git a/d-engine-core/src/raft.rs b/d-engine-core/src/raft.rs index c8e2eac6..853c81eb 100644 --- a/d-engine-core/src/raft.rs +++ b/d-engine-core/src/raft.rs @@ -609,6 +609,15 @@ where .handle_log_flushed(durable_index, &self.ctx, &self.internal_event_tx) .await; } + InternalEvent::FsyncCompleted { index, term } => { + if let Some(new_durable) = + self.ctx.raft_log().try_advance_durable_index(index, term) + { + self.role + .handle_log_flushed(new_durable, &self.ctx, &self.internal_event_tx) + .await; + } + } InternalEvent::AppendResult { follower_id, result, diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index 0fb7960d..2a0519aa 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -18,7 +18,7 @@ //! ## IO thread (notify-then-spawn-fsync) //! //! On wakeup from `write_notify`: -//! 1. **Read** — scan SkipMap range `(durable_index, max_index]` +//! 1. **Read** — scan SkipMap range `(durable_index, memory_max_index]` //! 2. **Persist** — write range to OS page cache via `persist_entries` //! 3. **Spawn fsync** — dispatch fdatasync to `spawn_blocking` pool via `spawn_fsync`, return immediately //! 4. **Loop** — back to `select!` for next wakeup; prior fsync runs concurrently in pool @@ -277,8 +277,8 @@ where // --- In-memory index --- // O(1) answer to "is this index currently held in memory" — lets callers // (e.g. entry_term()) reject an out-of-range index without touching `entries`. - min_index: AtomicU64, // Smallest log index (0 if empty) - max_index: AtomicU64, // Largest log index (0 if empty) + min_index: AtomicU64, // Smallest log index (0 if empty) + memory_max_index: AtomicU64, // Largest log index held in memory (0 if empty) — may be ahead of what's persisted/durable // The term of the last entry ever purged (compacted away after a snapshot). // Raft's AppendEntries consistency check needs the term at prev_log_index @@ -286,7 +286,7 @@ where // this, a follower can't tell "purged, but we agree" apart from "conflict". // // Must be published in the same critical section as the entries removal - // and the min_index/max_index advance it corresponds to — a reader must + // and the min_index/memory_max_index advance it corresponds to — a reader must // never be able to observe the entry gone but this boundary not yet set. last_purged_index: AtomicU64, last_purged_term: AtomicU64, @@ -342,7 +342,7 @@ where } fn last_entry_id(&self) -> u64 { - self.max_index.load(Ordering::Acquire) + self.memory_max_index.load(Ordering::Acquire) } fn durable_index(&self) -> u64 { @@ -394,7 +394,7 @@ where entry_id: u64, ) -> Option { // Bounds check: skip TermSegments entirely for out-of-range queries. - let max = self.max_index.load(Ordering::Acquire); + let max = self.memory_max_index.load(Ordering::Acquire); let min = self.min_index.load(Ordering::Acquire); if max == 0 || entry_id < min || entry_id > max { // Cold path: check purge boundary so that AppendEntries built with @@ -599,7 +599,7 @@ where if diverge_index <= last_current_index { // Real term conflict: truncate from diverge_index, replace with tail. // Await the done channel so callers can flush() knowing the truncation - // is durable — durable_index may exceed max_index after truncation, + // is durable — durable_index may exceed memory_max_index after truncation, // which would cause flush() to short-circuit before the replace lands. self.remove_range(diverge_index..=u64::MAX); self.insert_to_memory(tail); @@ -682,9 +682,16 @@ where self.purge_prefix(cutoff_index); // Purged entries are backed by the snapshot; treat cutoff as durable. - // Must run after purge_prefix() — advance_durable_and_notify() validates - // against last_purged_index, which purge_prefix() just established. - self.advance_durable_and_notify(cutoff_index.index); + // Already running on the single owner (called from role_state.rs, same + // thread as remove_range) — safe to apply directly, no message hop needed. + if let Some(new_durable) = + self.try_advance_durable_index(cutoff_index.index, cutoff_index.term) + && let Some(ref tx) = self.log_flush_tx + { + let _ = tx.send(crate::InternalEvent::LogFlushed { + durable_index: new_durable, + }); + } // Route purge through the IO thread so it never blocks the inbound event loop. // Also writes the purge boundary to META_CF in the RocksDB implementation. @@ -702,12 +709,36 @@ where Ok(()) } + fn try_advance_durable_index( + &self, + index: u64, + term: u64, + ) -> Option { + let prev = self.durable_index.load(Ordering::Acquire); + if index <= prev { + return None; + } + if self.entry_term(index) != Some(term) { + return None; + } + let safe = index.min( + self.memory_max_index + .load(Ordering::Acquire) + .max(self.last_purged_index.load(Ordering::Acquire)), + ); + if safe <= prev { + return None; + } + self.durable_index.fetch_max(safe, Ordering::AcqRel); + Some(safe) + } + async fn flush(&self) -> Result<()> { - let max_index = self.max_index.load(Ordering::Acquire); - if max_index == 0 { + let memory_max_index = self.memory_max_index.load(Ordering::Acquire); + if memory_max_index == 0 { return Ok(()); } - if self.durable_index.load(Ordering::Acquire) >= max_index { + if self.durable_index.load(Ordering::Acquire) >= memory_max_index { return Ok(()); } let (tx, rx) = oneshot::channel(); @@ -830,7 +861,7 @@ where // Initialize atomic boundaries let min_index = entries.front().map(|e| *e.key()).unwrap_or(0); - let max_index = entries.back().map(|e| *e.key()).unwrap_or(0); + let memory_max_index = entries.back().map(|e| *e.key()).unwrap_or(0); if disk_len > 0 && loaded_count == 0 { warn!( @@ -859,7 +890,7 @@ where shutdown_timeout_ms, entries: RwLock::new(entries), min_index: AtomicU64::new(min_index), - max_index: AtomicU64::new(max_index), + memory_max_index: AtomicU64::new(memory_max_index), last_purged_index: AtomicU64::new(last_purged_index_val), last_purged_term: AtomicU64::new(last_purged_term_val), durable_index: AtomicU64::new(disk_len), @@ -937,7 +968,7 @@ where /// from one-per-write to one-per-burst. /// /// On each wakeup: - /// 1. Read entries in `(durable_index, max_index]` from SkipMap. + /// 1. Read entries in `(durable_index, memory_max_index]` from SkipMap. /// 2. persist_entries to OS page cache (no fsync). /// 3. Drain any pending control commands from the mpsc channel. /// 4. fsync once — advance durable_index, wake WaitDurable callers. @@ -972,7 +1003,7 @@ where IOTask::Shutdown => Self::run_batch_turn(&this, &mut receiver, &mut pending_max, Vec::new(), true).await, IOTask::Flush(reply) => Self::run_batch_turn(&this, &mut receiver, &mut pending_max, vec![reply], false).await, cmd => { - if Self::handle_non_write_cmd(cmd, &this, &mut pending_max).await { + if Self::run_storage_tasks(cmd, &this, &mut pending_max).await { break; } continue; @@ -1023,7 +1054,7 @@ where } IOTask::Flush(reply) => replies.push(reply), cmd => { - if Self::handle_non_write_cmd(cmd, this, pending_max).await { + if Self::run_storage_tasks(cmd, this, pending_max).await { for reply in replies { let _ = reply .send(Err(Error::Fatal("fatal IO error, batch aborted".into()))); @@ -1042,25 +1073,22 @@ where seen_shutdown } - /// Handle IOTask variants that are NOT `Flush` or `Shutdown`. - /// - /// Callers (`batch_processor`) dispatch `Flush` and `Shutdown` directly in the outer - /// `match` before this function is ever called — those two arms are unreachable here. + /// Runs one storage-mutating IOTask (Persist/ReplaceRange/Purge/Reset) + /// against log_store. Flush/Shutdown are intercepted by the caller + /// (`batch_processor`) before this is called — unreachable here. /// /// Returns `true` if `batch_processor` must exit immediately (fatal IO error). - async fn handle_non_write_cmd( + async fn run_storage_tasks( cmd: IOTask, this: &Arc, pending_max: &mut u64, ) -> bool { match cmd { IOTask::Flush(_) => { - unreachable!( - "Flush must be intercepted in the drain loop before handle_non_write_cmd" - ) + unreachable!("Flush must be intercepted in the drain loop before run_storage_tasks") } IOTask::Shutdown => { - unreachable!("Shutdown is always filtered out before reaching handle_non_write_cmd") + unreachable!("Shutdown is always filtered out before reaching run_storage_tasks") } IOTask::Persist { entries, done } => { if this.is_poisoned() { @@ -1077,7 +1105,7 @@ where } if max_idx > 0 { let current_bound = this - .max_index + .memory_max_index .load(Ordering::Acquire) .max(this.last_purged_index.load(Ordering::Acquire)); let safe_max_idx = max_idx.min(current_bound); @@ -1110,8 +1138,14 @@ where let _ = done.send(result); return true; // signal batch_processor to exit — disk state is corrupted } + // persisted_index moved here from remove_range — this handler + // is the sole writer now, single-threaded, no content check needed. + this.persisted_index + .fetch_min(truncate_from.saturating_sub(1), Ordering::AcqRel); + if max_idx > 0 { *pending_max = (*pending_max).max(max_idx); + this.persisted_index.fetch_max(max_idx, Ordering::AcqRel); this.fsync_coordinator.submit(this, max_idx, vec![]); } let _ = done.send(result); @@ -1165,7 +1199,7 @@ where // Reset boundaries self.min_index.store(0, Ordering::Release); - self.max_index.store(0, Ordering::Release); + self.memory_max_index.store(0, Ordering::Release); // Clear term indexes to ensure consistency after reset self.term_first_index.clear(); @@ -1220,9 +1254,9 @@ where } if let Some(last_entry) = entries.last() { - let mut current_max = self.max_index.load(Ordering::Relaxed); + let mut current_max = self.memory_max_index.load(Ordering::Relaxed); while last_entry.index > current_max { - match self.max_index.compare_exchange_weak( + match self.memory_max_index.compare_exchange_weak( current_max, last_entry.index, Ordering::AcqRel, @@ -1235,32 +1269,15 @@ where } } - // The single choke point every reported max_index must pass through — - // re-validates against the current log boundary regardless of how many - // upstream call sites raced to produce this value. - pub(super) fn advance_durable_and_notify( + /// Fire-and-forget signal to the single owner (raft.rs's event loop). + /// Called by fsync_coordinator (C) — never writes `durable_index` itself. + pub(super) fn notify_fsync_completed( &self, - reported_max: u64, + index: u64, + term: u64, ) { - let current_max = self - .max_index - .load(Ordering::Acquire) - .max(self.last_purged_index.load(Ordering::Acquire)); - let safe_max = reported_max.min(current_max); - debug_assert!( - safe_max == reported_max, - "advance_durable_and_notify: reported_max {reported_max} exceeded current bound {current_max}, clamped" - ); - if safe_max == 0 { - return; - } - let prev = self.durable_index.fetch_max(safe_max, Ordering::AcqRel); - if safe_max > prev - && let Some(ref tx) = self.log_flush_tx - { - let _ = tx.send(crate::InternalEvent::LogFlushed { - durable_index: safe_max, - }); + if let Some(ref tx) = self.log_flush_tx { + let _ = tx.send(crate::InternalEvent::FsyncCompleted { index, term }); } } @@ -1309,9 +1326,8 @@ where let entries = self.entries.write(); let (new_min, new_max) = self.remove_range_locked(&entries, range); self.min_index.store(new_min, Ordering::Release); - self.max_index.store(new_max, Ordering::Release); + self.memory_max_index.store(new_max, Ordering::Release); - self.persisted_index.fetch_min(new_max, Ordering::AcqRel); self.durable_index.fetch_min(new_max, Ordering::AcqRel); // Clamps pending_max and bumps generation, in that order — see // fence_truncation()'s doc comment for why the order matters. @@ -1398,7 +1414,7 @@ where /// Purge entries at/below `cutoff.index`, publishing `last_purged_index`/ /// `last_purged_term` in the SAME critical section as the entries removal - /// and the min_index/max_index advance. Only place that should ever write + /// and the min_index/memory_max_index advance. Only place that should ever write /// `last_purged_*` — a reader must never observe the entries gone but the /// boundary not yet recorded (#442). pub fn purge_prefix( @@ -1409,7 +1425,7 @@ where let (new_min, new_max) = self.remove_range_locked(&entries, 0..=cutoff.index); self.min_index.store(new_min, Ordering::Release); - self.max_index.store(new_max, Ordering::Release); + self.memory_max_index.store(new_max, Ordering::Release); // Write term before index (Release) so readers that load index first // then term (Acquire) always observe a consistent pair. @@ -1471,11 +1487,11 @@ where } #[cfg(test)] - pub(super) fn set_max_index_for_test( + pub(super) fn set_memory_max_index_for_test( &self, value: u64, ) { - self.max_index.store(value, Ordering::Release); + self.memory_max_index.store(value, Ordering::Release); } } @@ -1585,6 +1601,10 @@ mod term_segments_test; #[path = "buffered_raft_log_test/truncation_fsync_fence_test.rs"] mod truncation_fsync_fence_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/content_validated_watermark_test.rs"] +mod content_validated_watermark_test; + #[cfg(test)] #[path = "buffered_raft_log_test/worker_test.rs"] mod worker_test; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs index 8cde96b9..495aad69 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs @@ -6,9 +6,9 @@ //! advance_durable_and_notify contract //! - **Concurrency**: Reset races, out-of-order completion, crash recovery +use crate::test_utils::drain_and_apply_fsync_completions; use crate::{ - BufferedRaftLog, FlushPolicy, InternalEvent, MockStorageEngine, MockTypeConfig, - PersistenceConfig, RaftLog, + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, RaftLog, }; use d_engine_proto::common::Entry; use std::sync::Arc; @@ -37,12 +37,15 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), ); - let raft_log = raft_log.start(receiver, None); + // A real log_flush_tx is required now: durable_index only advances when + // something drains InternalEvent::FsyncCompleted and calls + // try_advance_durable_index — see drain_and_apply_fsync_completions. + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready let pre_write_durable_index = raft_log.durable_index(); @@ -70,12 +73,13 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { "durable_index must not advance before fsync completes" ); - // Release the gate — flush() returns, advance_durable_and_notify(1) fires. + // Release the gate — flush() returns, notify_fsync_completed(1, 1) fires. flush_gate.send(()).unwrap(); // Pick a polling/backoff strategy instead of a fixed sleep, // to avoid flakiness under CI load. tokio::time::sleep(Duration::from_millis(50)).await; + drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!( raft_log.durable_index(), @@ -117,12 +121,15 @@ async fn test_majority_matched_index_uses_durable_not_memory() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), ); - let raft_log = raft_log.start(receiver, None); + // A real log_flush_tx is required now: durable_index only advances when + // something drains InternalEvent::FsyncCompleted and calls + // try_advance_durable_index — see drain_and_apply_fsync_completions. + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready let pre_write_durable_index = raft_log.durable_index(); @@ -175,12 +182,13 @@ async fn test_majority_matched_index_uses_durable_not_memory() { when a follower already reports it" ); - // Release the gate — flush() returns, advance_durable_and_notify(2) fires. + // Release the gate — flush() returns, notify_fsync_completed(2, 1) fires. flush_gate.send(()).unwrap(); // Pick a polling/backoff strategy instead of a fixed sleep, // to avoid flakiness under CI load. tokio::time::sleep(Duration::from_millis(50)).await; + drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!( raft_log.durable_index(), @@ -223,7 +231,6 @@ async fn test_entry_term_correct_during_concurrent_fsync_delay() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -282,24 +289,24 @@ async fn test_entry_term_correct_during_concurrent_fsync_delay() { // ── Logic correctness ───────────────────────────────────────────────────────── -/// `advance_durable_and_notify` is monotonic: a late-arriving lower index is a no-op. +/// `try_advance_durable_index` is monotonic: a late-arriving lower index is a no-op. /// -/// Directly call `advance_durable_and_notify(150)`, then `advance_durable_and_notify(100)`. +/// Directly call `try_advance_durable_index(150, 1)`, then `try_advance_durable_index(100, 1)`. /// Assert: /// - final `durable_index() == 150` (not 100) -/// - `LogFlushed` event fired exactly once (for 150), not twice +/// - the 150 call returns `Some(150)` (it fired), the 100 call returns `None` (no-op) /// /// Verifies the `fetch_max` invariant that makes out-of-order concurrent fsyncs safe. /// -/// Expected: -/// - After `advance_durable_and_notify(150)`: `durable_index() == 150`. -/// - After the subsequent `advance_durable_and_notify(100)`: `durable_index()` -/// is STILL `150` (unchanged — 100 < 150 must be a no-op, not a regression). -/// - The flush-completion notification fires exactly once, carrying 150 — the -/// discarded 100 call must not fire a second notification. +/// Test changed from #446/#447's original (which checked an `InternalEvent::LogFlushed` +/// on a channel): `try_advance_durable_index` no longer sends that notification itself — +/// the caller (raft.rs's `FsyncCompleted` handler) decides whether to fire +/// `handle_log_flushed`, based on this method's `Option` return value. So "fired +/// exactly once, only for 150" is now asserted directly on the return values instead of +/// on a channel — same intent, moved to match where the behavior actually lives now. #[tokio::test] async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { - // Storage engine choice doesn't matter here — advance_durable_and_notify is + // Storage engine choice doesn't matter here — try_advance_durable_index is // called directly, bypassing the real fsync pipeline entirely. let storage = Arc::new(MockStorageEngine::with_id( "durable_index_monotonic_when_fsyncs_complete_out_of_order".into(), @@ -310,44 +317,44 @@ async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, ); - let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel::(); - let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - // advance_durable_and_notify() clamps against max_index — simulate a log - // that already has 150 entries, matching the highest value used below. - raft_log.set_max_index_for_test(150); + // try_advance_durable_index() content-validates against entry_term(index) — + // needs real entries in memory, not just a raw max_index poke. + let entries: Vec = (1..=150) + .map(|index| Entry { + index, + term: 1, + payload: None, + }) + .collect(); + raft_log.append_entries(entries).await.unwrap(); // Simulates a fsync task completing with index 150, then a second, older // fsync task (dispatched earlier, finishing later) completing with 100. - raft_log.advance_durable_and_notify(150); - raft_log.advance_durable_and_notify(100); + let result_150 = raft_log.try_advance_durable_index(150, 1); + let result_100 = raft_log.try_advance_durable_index(100, 1); + assert_eq!( + result_150, + Some(150), + "the 150 call must fire — it's the first advance" + ); + assert_eq!( + result_100, None, + "the later, lower 100 call must be a no-op (None), not a regression" + ); assert_eq!( raft_log.durable_index(), 150, "durable_index must reflect the highest index seen (150), not the \ later-arriving lower one (100)" ); - - // Exactly one LogFlushed event must have fired, carrying 150 — the - // no-op 100 call must not have sent a second event. - let event = log_flush_rx.try_recv().expect("LogFlushed must fire for the 150 call"); - match event { - InternalEvent::LogFlushed { durable_index } => { - assert_eq!(durable_index, 150, "LogFlushed must carry 150, not 100"); - } - other => panic!("expected InternalEvent::LogFlushed, got {other:?}"), - } - assert!( - log_flush_rx.try_recv().is_err(), - "no second LogFlushed event should have fired for the no-op 100 call" - ); } /// A `flush()` caller receives `Ok(())` only after its batch is physically on disk. @@ -377,7 +384,6 @@ async fn test_flush_caller_blocked_until_fsync_completes() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -450,7 +456,6 @@ async fn test_flush_callers_arriving_during_inflight_fsync_are_coalesced() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -533,7 +538,6 @@ async fn test_shutdown_with_pending_flush_caller_still_receives_ok_reply() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 100, }, Arc::new(storage), @@ -606,7 +610,6 @@ async fn test_shutdown_with_pending_flush_caller_still_receives_err_reply() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 100, }, Arc::new(storage), @@ -686,7 +689,6 @@ async fn test_reset_during_inflight_fsync_does_not_resurrect_stale_durable_index flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -767,12 +769,12 @@ async fn test_post_reset_writes_are_not_discarded_by_stale_fence() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready // Append triggers the automatic round (round 1), which wins the CAS and @@ -823,6 +825,7 @@ async fn test_post_reset_writes_are_not_discarded_by_stale_fence() { result.is_ok(), "Y's flush() must succeed — its data was written after reset, not stale" ); + drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!( raft_log.durable_index(), 1, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs new file mode 100644 index 00000000..68a38051 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs @@ -0,0 +1,204 @@ +//! Content-validated `durable_index` advance (#446/#447 single-owner +//! redesign). Tests `try_advance_durable_index(&self, index: u64, term: u64) +//! -> Option` — `Some(new_value)` only when it actually advanced, +//! `None` when rejected as stale (`entry_term(index) != Some(term)`) or +//! already applied. +//! +//! Not wired into the mod tree yet — add +//! `#[path = "buffered_raft_log_test/content_validated_watermark_test.rs"] +//! mod content_validated_watermark_test;` to `buffered_raft_log.rs` next to +//! the other test module declarations. +//! +//! Why these tests don't need thread races or timing gates (unlike +//! `truncation_fsync_fence_test.rs`): under single ownership, the report and +//! the truncation are just two sequential calls in whatever order they +//! happen to arrive — no interleaving *inside* a function body is possible +//! because there's only one caller. Each test below drives one arrival order +//! directly. +//! +//! `persisted_index` has no equivalent test here — it doesn't need content +//! validation. Its only writer is now the IO thread (B), processing +//! `IOTask::Persist`/`ReplaceRange` strictly in the order the single owner +//! (A) issued them (each `await`ed before the next is sent), so there's no +//! stale-message window the way there is for `durable_index`'s async, +//! un-awaited fsync-completion report. + +use std::sync::Arc; +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +async fn new_raft_log() -> Arc> { + let storage = Arc::new(MockStorageEngine::with_id( + "content_validated_watermark_test".into(), + )); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + raft_log.start(receiver, None) +} + +/// Business scenario: follower has entries 1..=100 under term 1. A physical +/// fsync for "up to 100" is still in flight when a new leader (term 2) +/// truncates 81..=100 and replaces it with its own entries. The in-flight +/// fsync's completion — a report for (index=100, term=1) — arrives after the +/// replacement. Index 100 still exists, but it's term 2 now: the report +/// describes content that's gone. +/// +/// Expected: rejected. `durable_index` must not move to 100. +#[tokio::test] +async fn test_stale_durable_report_rejected_when_term_no_longer_matches() { + let raft_log = new_raft_log().await; + + let term1_entries: Vec = (1..=100).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(term1_entries).await.unwrap(); + + // New leader (term 2) truncates 81..=100 and replaces with its own tail. + let term2_tail: Vec = (81..=100).map(|i| entry(i, 2)).collect(); + raft_log.filter_out_conflicts_and_append(80, 1, term2_tail).await.unwrap(); + + // The stale in-flight fsync's report, generated before the truncation. + let result = raft_log.try_advance_durable_index(100, 1); + + assert_eq!( + result, None, + "a durable report for term=1 must be rejected once index 100 belongs to term=2" + ); + assert!( + raft_log.durable_index() < 81, + "durable_index ({}) must not advance into the replaced [81,100] range \ + on a rejected report", + raft_log.durable_index() + ); +} + +/// Sanity check: an unremarkable report (no truncation involved) must still +/// be applied. The new validation must not reject everything. +#[tokio::test] +async fn test_durable_report_accepted_when_term_still_matches() { + let raft_log = new_raft_log().await; + + let entries: Vec = (1..=100).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(entries).await.unwrap(); + + let result = raft_log.try_advance_durable_index(100, 1); + + assert_eq!( + result, + Some(100), + "a report matching current log content must be applied" + ); + assert_eq!(raft_log.durable_index(), 100); +} + +/// Same scenario as `test_stale_durable_report_rejected_when_term_no_longer_matches`, +/// but the report arrives BEFORE the truncation instead of after — the other +/// possible arrival order. Under single ownership both orders must land on +/// the same final state, because the owner processes one event at a time +/// rather than racing a background write against a live update. +#[tokio::test] +async fn test_durable_report_then_truncation_is_order_independent() { + let raft_log = new_raft_log().await; + + let term1_entries: Vec = (1..=100).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(term1_entries).await.unwrap(); + + // Report arrives first, while the log is still all term 1 — legitimately + // applied at this point in time. + let result = raft_log.try_advance_durable_index(100, 1); + assert_eq!(result, Some(100)); + assert_eq!(raft_log.durable_index(), 100); + + // Truncation arrives after — must still clamp durable_index down, + // exactly as it does today via `remove_range`'s existing fetch_min. + let term2_tail: Vec = (81..=100).map(|i| entry(i, 2)).collect(); + raft_log.filter_out_conflicts_and_append(80, 1, term2_tail).await.unwrap(); + + assert!( + raft_log.durable_index() < 81, + "truncation must clamp durable_index down to 80 regardless of the \ + earlier report having advanced it to 100, durable_index is {}", + raft_log.durable_index() + ); +} + +/// Regression test for the `flush()` short-circuit (`durable_index >= +/// memory_max_index` at `buffered_raft_log.rs:710`). This isn't proving a +/// live bug in the current design (`remove_range` clamps `durable_index` +/// synchronously, so the short-circuit's precondition always holds) — it's +/// pinning down that invariant so a future change that defers the clamp +/// (e.g. copying openraft's "don't touch the watermark on truncation, rely +/// on term comparison instead") doesn't silently reopen the RPO=0 violation +/// this whole fix was for: `flush()` returning `Ok(())` before the real, +/// post-truncation tail has actually been fsynced. +/// +/// Scenario: entries 1..=100 durable. New leader truncates 81..=100 (term 2 +/// tail 81..=85 replaces it) — `durable_index` clamps to 80, +/// `memory_max_index` becomes 85. Calling `flush()` right after must NOT +/// take the short-circuit (80 < 85) — it must dispatch a real physical +/// flush for the new, not-yet-synced tail. +#[tokio::test] +async fn test_flush_does_not_short_circuit_after_truncation_regrows_the_log() { + let (mut ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( + FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + "flush_no_short_circuit_after_truncation", + ); + + ctx.append_entries(1, 100, 1).await; + ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); + assert_eq!( + ctx.raft_log.durable_index(), + 100, + "baseline must be fully durable" + ); + + let flushes_before_truncation = flush_count.load(std::sync::atomic::Ordering::Relaxed); + + // New leader (term 2) truncates 81..=100, replaces with its own tail + // 81..=85 — durable_index clamps to 80, memory_max_index becomes 85. + let term2_tail: Vec = (81..=85).map(|i| entry(i, 2)).collect(); + ctx.raft_log.filter_out_conflicts_and_append(80, 1, term2_tail).await.unwrap(); + assert!(ctx.raft_log.durable_index() < 81, "clamp must have fired"); + assert_eq!(ctx.raft_log.last_entry_id(), 85); + + ctx.raft_log.flush().await.unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + ctx.drain_fsync_completions(); + + let flushes_after = flush_count.load(std::sync::atomic::Ordering::Relaxed); + assert!( + flushes_after > flushes_before_truncation, + "flush() must dispatch a real physical flush for the new tail, not \ + short-circuit on a stale-looking durable_index — before={flushes_before_truncation}, after={flushes_after}" + ); + assert_eq!( + ctx.raft_log.durable_index(), + 85, + "the new tail must actually become durable, not just claimed so" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs index bf4e3996..9a3300c0 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs @@ -38,7 +38,7 @@ use crate::{FlushPolicy, RaftLog}; /// automatically — no explicit `flush()` required. #[tokio::test] async fn test_writes_become_durable_via_io_thread() { - let (ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( + let (mut ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -68,6 +68,7 @@ async fn test_writes_become_durable_via_io_thread() { // Give IO thread time to process write_notify wakeup and fsync. sleep(Duration::from_millis(50)).await; + ctx.drain_fsync_completions(); // durable_index must have advanced via IO thread auto-fsync (no explicit flush). assert_eq!( @@ -94,7 +95,7 @@ async fn test_writes_become_durable_via_io_thread() { /// N entries in one call → ≤2 fsyncs (not N), regardless of storage speed. #[tokio::test] async fn test_batch_append_produces_one_flush() { - let (ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( + let (mut ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -112,6 +113,7 @@ async fn test_batch_append_produces_one_flush() { ctx.raft_log.append_entries(entries).await.unwrap(); ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 100); @@ -134,7 +136,7 @@ async fn test_batch_append_produces_one_flush() { /// `else { pending_max = 0 }` branch is skipped. /// /// ## Original bug (fixed pre-#422) -/// `handle_non_write_cmd(IOTask::Reset)` wiped the on-disk log but did NOT zero +/// `run_storage_tasks(IOTask::Reset)` wiped the on-disk log but did NOT zero /// `pending_max`. On the next `write_notify` wakeup the IO thread would compute: /// ``` /// pending_max = pending_max.max(new_end) // stale 10 wins over new 3 @@ -159,7 +161,6 @@ async fn test_pending_max_zeroed_on_reset_preventing_durable_index_corruption() flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -219,7 +220,7 @@ async fn test_pending_max_zeroed_on_reset_preventing_durable_index_corruption() /// flush() call must be durable when flush() returns, regardless of internal batching. #[tokio::test] async fn test_flush_is_strict_durability_barrier() { - let (ctx, _flush_count) = BufferedRaftLogTestContext::new_not_durable( + let (mut ctx, _flush_count) = BufferedRaftLogTestContext::new_not_durable( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -238,6 +239,7 @@ async fn test_flush_is_strict_durability_barrier() { .unwrap(); } ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!( ctx.raft_log.durable_index(), 20, @@ -256,6 +258,7 @@ async fn test_flush_is_strict_durability_barrier() { .unwrap(); } ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!( ctx.raft_log.durable_index(), 50, @@ -289,7 +292,6 @@ async fn test_flush_propagates_io_error() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -350,7 +352,6 @@ async fn test_fsync_failure_poisons_and_rejects_writes_after_reset() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -406,7 +407,6 @@ async fn test_replace_range_failure_poisons() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -495,7 +495,6 @@ async fn test_purge_failure_poisons() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -544,7 +543,6 @@ async fn test_reset_failure_poisons() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -592,7 +590,6 @@ async fn test_save_hard_state_failure_poisons() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -640,7 +637,6 @@ async fn test_poisoned_rejects_save_hard_state() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -665,7 +661,7 @@ async fn test_poisoned_rejects_save_hard_state() { } // ============================================================================ -// Gap fix: handle_non_write_cmd now checks is_poisoned() before executing +// Gap fix: run_storage_tasks now checks is_poisoned() before executing // ReplaceRange/Purge/Reset, instead of only checking it in run_batch_turn's // drain loop (which missed the direct-dispatch path in batch_processor's // top-level select, and the "just poisoned mid-turn" race). @@ -687,7 +683,6 @@ async fn test_poisoned_skips_replace_range() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -768,7 +763,6 @@ async fn test_poisoned_does_not_skip_reset() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -808,7 +802,6 @@ async fn test_poisoned_skips_purge() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -818,7 +811,7 @@ async fn test_poisoned_skips_purge() { // advance_durable_and_notify() clamps against max_index — simulate a log // that already has the entry this test purges up to. - raft_log.set_max_index_for_test(1); + raft_log.set_memory_max_index_for_test(1); raft_log.poisoned.store(true, Ordering::SeqCst); let result = raft_log.purge_logs_up_to(LogId { term: 1, index: 1 }).await; @@ -890,7 +883,6 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -1001,7 +993,6 @@ async fn test_new_buffered_raft_log_starts_unpoisoned() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -1032,7 +1023,6 @@ async fn test_poisoned_survives_reset() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -1070,7 +1060,6 @@ async fn test_persist_entries_failure_poisons() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -1136,7 +1125,6 @@ async fn test_poisoned_rejects_queued_persist_task() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -1202,7 +1190,6 @@ async fn test_notify_fatal_channel_closed_still_poisons_and_logs() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), diff --git a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs index 2c4f0c02..f6602e40 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs @@ -11,7 +11,7 @@ use d_engine_proto::common::{Entry, LogId}; #[tokio::test] async fn test_durable_index_monotonic_under_concurrency() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -39,6 +39,7 @@ async fn test_durable_index_monotonic_under_concurrency() { // Wait for flush to complete tokio::time::sleep(Duration::from_millis(200)).await; + ctx.drain_fsync_completions(); // Verify monotonicity let durable = ctx.raft_log.durable_index(); @@ -124,7 +125,6 @@ async fn test_purge_does_not_regress_durable_index_already_ahead() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -135,6 +135,7 @@ async fn test_purge_does_not_regress_durable_index_already_ahead() { // Arrange: entries 1..=100, all flushed — durable_index reaches 100 and // fires LogFlushed(100). simulate_insert_command(&raft_log, (1..=100).collect(), 1).await; + crate::test_utils::drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!(raft_log.durable_index(), 100); // Drain the LogFlushed(100) from the insert+flush above — not what this diff --git a/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs index 95e40d8b..d3cbfab7 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs @@ -22,7 +22,7 @@ use crate::{FlushPolicy, RaftLog}; /// - Expected: durable_index == 5 after explicit flush() #[tokio::test] async fn test_mem_first_entries_durable_after_flush() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -32,6 +32,7 @@ async fn test_mem_first_entries_durable_after_flush() { // Act: Append entries then wait for durability ctx.append_entries(1, 5, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Assert: All entries durable after flush assert_eq!( @@ -50,7 +51,7 @@ async fn test_mem_first_entries_durable_after_flush() { /// - Expected: All 1000 entries durable after flush(), no data loss #[tokio::test] async fn test_mem_first_concurrent_writes_durable_after_flush() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -81,6 +82,7 @@ async fn test_mem_first_concurrent_writes_durable_after_flush() { // Wait for all entries to become durable ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Assert: All entries durable after flush assert_eq!( @@ -168,7 +170,7 @@ async fn test_mem_first_buffers_entries_before_flush() { /// - Expected: Entries become durable after flush #[tokio::test] async fn test_mem_first_flushes_asynchronously() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -181,6 +183,7 @@ async fn test_mem_first_flushes_asynchronously() { // Act: Explicit flush ctx.raft_log.flush().await.unwrap(); sleep(Duration::from_millis(100)).await; // Allow async flush + ctx.drain_fsync_completions(); // Assert: Entries now durable assert!( @@ -236,7 +239,7 @@ async fn test_mem_first_concurrent_buffering() { /// - Expected: Flush triggered at threshold #[tokio::test] async fn test_batched_flushes_at_threshold() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 10000, // High interval to test threshold trigger }, @@ -246,6 +249,7 @@ async fn test_batched_flushes_at_threshold() { // Act: Append exactly threshold entries ctx.append_entries(1, 5, 1).await; sleep(Duration::from_millis(100)).await; // Allow flush + ctx.drain_fsync_completions(); // Assert: Entries should be flushed assert!( @@ -261,7 +265,7 @@ async fn test_batched_flushes_at_threshold() { /// - Expected: Flush triggered by timer #[tokio::test] async fn test_batched_flushes_at_interval() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -271,6 +275,7 @@ async fn test_batched_flushes_at_interval() { // Act: Append few entries and wait for interval ctx.append_entries(1, 2, 1).await; sleep(Duration::from_millis(200)).await; // Wait for interval flush + ctx.drain_fsync_completions(); // Assert: Entries flushed by timer assert!( diff --git a/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs index 508c3bd5..ab405f0c 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs @@ -22,7 +22,6 @@ fn setup_memory() -> Arc> { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs index 52826bfb..62bc61ff 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs @@ -59,7 +59,6 @@ async fn test_reset_performance_during_active_flush() { let storage = create_delayed_storage(FLUSH_DELAY_MS); let config = PersistenceConfig { flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }; @@ -119,7 +118,6 @@ async fn test_filter_conflicts_performance_during_flush() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }; @@ -201,7 +199,6 @@ async fn test_fresh_cluster_performance_consistency() { let config = PersistenceConfig { flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs index 95f49106..6efc9e95 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs @@ -45,7 +45,7 @@ fn entry( /// claim durability for an index that doesn't exist in the log anymore. #[tokio::test] async fn test_durable_index_never_exceeds_log_after_truncation_and_resync() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // isolate from the safety-net timer }, @@ -77,6 +77,7 @@ async fn test_durable_index_never_exceeds_log_after_truncation_and_resync() { // Trigger a disk sync and give it time to complete. ctx.raft_log.flush().await.unwrap(); tokio::time::sleep(Duration::from_millis(50)).await; + ctx.drain_fsync_completions(); // The follower must never advertise durability for an index it doesn't // actually have. If persisted_index wasn't clamped down during the @@ -133,7 +134,6 @@ async fn test_persisted_index_does_not_adopt_a_stale_persist_after_truncation() flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), diff --git a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs index a0d689fa..d9e9876a 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs @@ -474,7 +474,6 @@ async fn test_io_task_replace_range_delegates_to_replace_range_not_truncate() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs index 1a206f84..2f495a53 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs @@ -51,7 +51,6 @@ async fn test_append_entries_waits_for_storage_engine_before_returning() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage.clone()), diff --git a/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs index 36b88ce3..4d45b7ac 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs @@ -82,7 +82,6 @@ async fn test_quorum_uses_durable_index_not_last_entry_id() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -132,7 +131,6 @@ async fn test_election_eligibility_reads_memory_log_not_durable_index() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -165,7 +163,7 @@ async fn test_election_eligibility_reads_memory_log_not_durable_index() { /// that a separate, broader safety argument for #446 relies on. #[tokio::test] async fn test_majority_matched_index_requires_actual_majority_of_reports() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -174,6 +172,7 @@ async fn test_majority_matched_index_requires_actual_majority_of_reports() { ctx.append_entries(1, 10, 1).await; ctx.raft_log.flush().await.unwrap(); // leader's own entries now durable through 10 + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 10); @@ -198,7 +197,7 @@ async fn test_majority_matched_index_requires_actual_majority_of_reports() { /// quorum should proceed normally. #[tokio::test] async fn test_quorum_succeeds_after_leader_flush() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 999_999, // only threshold trigger, no timer }, @@ -210,6 +209,7 @@ async fn test_quorum_succeeds_after_leader_flush() { // Wait for flush to complete tokio::time::sleep(Duration::from_millis(50)).await; + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.last_entry_id(), 1); assert_eq!( @@ -249,7 +249,6 @@ async fn test_last_entry_id_diverges_from_durable_index_with_mem_first() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -274,7 +273,7 @@ async fn test_last_entry_id_diverges_from_durable_index_with_mem_first() { /// After explicit flush, durable_index must equal last_entry_id. #[tokio::test] async fn test_durable_index_equals_last_entry_id_after_flush() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -283,6 +282,7 @@ async fn test_durable_index_equals_last_entry_id_after_flush() { ctx.append_entries(1, 5, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); let last = ctx.raft_log.last_entry_id(); let durable = ctx.raft_log.durable_index(); diff --git a/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs index 373aeffd..bbec2429 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs @@ -44,7 +44,7 @@ async fn test_log_matching_property() { #[tokio::test] async fn test_leader_completeness_property() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -54,6 +54,7 @@ async fn test_leader_completeness_property() { // Leader writes entries and flushes so durable_index = 10 ctx.append_entries(1, 10, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Simulate majority replication scenario: // Leader has: [1,2,3,4,5,6,7,8,9,10], durable_index=10 @@ -81,7 +82,7 @@ async fn test_leader_completeness_property() { #[tokio::test] async fn test_calculate_majority_matched_index_case0() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -94,6 +95,7 @@ async fn test_calculate_majority_matched_index_case0() { simulate_insert_command(&ctx.raft_log, vec![1], 1).await; simulate_insert_command(&ctx.raft_log, vec![2, 3], 2).await; + ctx.drain_fsync_completions(); assert_eq!( Some(3), @@ -127,7 +129,7 @@ async fn test_calculate_majority_matched_index_case1() { #[tokio::test] async fn test_calculate_majority_matched_index_case2() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -143,6 +145,7 @@ async fn test_calculate_majority_matched_index_case2() { simulate_insert_command(&ctx.raft_log, vec![1], 1).await; simulate_insert_command(&ctx.raft_log, vec![2], 2).await; simulate_insert_command(&ctx.raft_log, vec![3], 3).await; + ctx.drain_fsync_completions(); assert_eq!( Some(3), ctx.raft_log.calculate_majority_matched_index(ct, ci, vec![4, 2]) @@ -196,7 +199,7 @@ async fn test_calculate_majority_matched_index_case4() { #[tokio::test] async fn test_calculate_majority_matched_index_case5() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -215,6 +218,7 @@ async fn test_calculate_majority_matched_index_case5() { let raft_log_entry_ids: Vec = (1..=raft_log_length).collect(); simulate_insert_command(&ctx.raft_log, raft_log_entry_ids, 1).await; + ctx.drain_fsync_completions(); assert_eq!( Some(peer2_match), diff --git a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs index 5f0358c7..c6259867 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs @@ -1,7 +1,7 @@ //! `IOTask::ReplaceRange` (term-conflict truncation, see //! `filter_out_conflicts_and_append`'s slow path) writes to the storage engine //! synchronously and bumps `pending_max`, but is dispatched through the -//! `receiver.recv()` => `cmd => { handle_non_write_cmd(...) }` arm of the IO +//! `receiver.recv()` => `cmd => { run_storage_tasks(...) }` arm of the IO //! thread's select loop — a branch that, unlike `run_batch_turn`, never calls //! `fsync_coordinator.submit()`. If no further `append_entries()` call arrives //! afterward (which would separately trigger a `run_batch_turn` via @@ -44,7 +44,7 @@ fn entry( /// truncation, because the replaced entries' fsync was never submitted. #[tokio::test] async fn test_replace_range_becomes_durable_without_a_following_append() { - let ctx = BufferedRaftLogTestContext::new( + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // effectively disabled for this test's timeframe }, @@ -54,6 +54,7 @@ async fn test_replace_range_becomes_durable_without_a_following_append() { // Arrange: log [1,2,3] all term=1, explicitly flushed durable. ctx.append_entries(1, 3, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 3, "baseline must be durable"); // Act: leader (term=2) sends entries that conflict at index=2 and extend @@ -76,6 +77,7 @@ async fn test_replace_range_becomes_durable_without_a_following_append() { // Give the IO thread ample time to have submitted fsync, if anything // besides the (disabled) safety net were going to do it. tokio::time::sleep(Duration::from_millis(200)).await; + ctx.drain_fsync_completions(); // FIXED: ReplaceRange's handler now submits fsync directly instead of // relying on a following append/notify or the safety net. diff --git a/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs index ad909049..79ebce89 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs @@ -50,7 +50,6 @@ fn test_io_thread_survives_runtime_drop() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 50, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -159,7 +158,6 @@ async fn test_shutdown_handles_slow_workers() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -287,7 +285,6 @@ async fn test_replace_range_failure_propagates_error_and_shuts_down_io_thread() flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // no auto-flush }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs index 16714b50..6e5fe913 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs @@ -54,7 +54,6 @@ async fn test_durable_index_does_not_adopt_a_stale_fsync_after_truncation() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), diff --git a/d-engine-core/src/storage/fsync_coordinator.rs b/d-engine-core/src/storage/fsync_coordinator.rs index 603a0c01..93e05145 100644 --- a/d-engine-core/src/storage/fsync_coordinator.rs +++ b/d-engine-core/src/storage/fsync_coordinator.rs @@ -1,6 +1,7 @@ use crate::BufferedRaftLog; use crate::Error; use crate::LogStore; +use crate::RaftLog; use crate::Result; use crate::TypeConfig; use std::sync::Arc; @@ -9,20 +10,15 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use tokio::sync::oneshot; use tracing::error; -/// Tracks whether a fsync task is currently running on the blocking pool. -/// Ensures at most one physical `flush_wal` call is in flight at any time, -/// restoring natural batching: entries that arrive while a fsync is running -/// accumulate in `pending_max`/`pending_replies`, and are picked up by the -/// SAME task once it finishes its current round — rather than spawning a -/// new competing task per `write_notify` wakeup. +/// Schedules physical fsync calls — batches concurrent requests into one +/// flush() at a time. Does not judge whether results are still valid; see +/// BufferedRaftLog::apply_durable_report. pub(super) struct FsyncCoordinator { inflight: AtomicBool, pending_max: AtomicU64, pending_replies: Mutex>>>, - // Fencing token (like Raft's `term`) for in-flight fsync results. Private — - // only bump via a fence_*() verb below, one per invalidating event. Never a - // value-passing variant (index math can under-fence, see fence_truncation()). + // Lets a stale round skip its reply early. Optional — not required for correctness. generation: AtomicU64, } @@ -78,6 +74,7 @@ impl FsyncCoordinator { let gen_at_start = self.generation.load(Ordering::Acquire); let max_index = self.pending_max.swap(0, Ordering::AcqRel); + let max_term = this.entry_term(max_index).unwrap_or(0); let replies = std::mem::take(&mut *self.pending_replies.lock().unwrap()); if this.is_poisoned() { @@ -126,8 +123,7 @@ impl FsyncCoordinator { r }; - // Fence check: if a reset happened while this batch was in flight, - // its result is for data that no longer exists — discard. + // Skip replying if this round is already known stale. if self.generation.load(Ordering::Acquire) != gen_at_start { for reply in replies { let _ = reply.send(Err(crate::Error::Fatal( @@ -138,7 +134,7 @@ impl FsyncCoordinator { } match &result { - Ok(()) => this.advance_durable_and_notify(max_index), + Ok(()) => this.notify_fsync_completed(max_index, max_term), Err(e) => { // One fsync failure = fatal, no threshold, no retry-and-hope. // Durability state is now unknown, this node @@ -180,13 +176,6 @@ impl FsyncCoordinator { self.bump_generation(); } - /// Called from `remove_range()` before a truncation is applied. Bumps - /// `generation` to fence any fsync already in flight for data this - /// truncation is about to discard — mirrors `fence_reset()`, but does - /// NOT touch `pending_max`/`pending_replies`: unlike a full reset, - /// a truncation's own `IOTask::ReplaceRange` handler submits a fresh, - /// correct `max_index` for the surviving log right after this runs, - /// so there is nothing stale left to drain. pub(super) fn fence_truncation( &self, new_max: u64, diff --git a/d-engine-core/src/storage/fsync_coordinator_test.rs b/d-engine-core/src/storage/fsync_coordinator_test.rs index 3b2a371e..93fdf8c0 100644 --- a/d-engine-core/src/storage/fsync_coordinator_test.rs +++ b/d-engine-core/src/storage/fsync_coordinator_test.rs @@ -14,13 +14,16 @@ use super::*; use crate::FlushPolicy; +use crate::InternalEvent; use crate::MockLogStore; use crate::MockMetaStore; use crate::MockStorageEngine; use crate::MockTypeConfig; use crate::PersistenceConfig; use crate::Result; +use d_engine_proto::common::Entry; use std::sync::Arc; +use tokio::sync::mpsc; /// Build a `BufferedRaftLog` for direct `FsyncCoordinator` method calls — /// never `.start()`-ed, no IO thread, no channel plumbing. Only `log_store`/ @@ -33,7 +36,6 @@ fn minimal_raft_log(storage: MockStorageEngine) -> Arc::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + raft_log.entries.write().insert( + 5, + Entry { + index: 5, + term: 1, + payload: None, + }, + ); + raft_log.set_memory_max_index_for_test(5); coord.inflight.store(true, Ordering::Release); coord.pending_max.store(5, Ordering::Release); coord.run_until_caught_up(&raft_log); + // Stand in for raft.rs's InternalEvent::FsyncCompleted handler. + while let Ok(InternalEvent::FsyncCompleted { index, term }) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(index, term); + } + assert_eq!( raft_log.durable_index.load(Ordering::Acquire), 5, @@ -480,9 +511,31 @@ fn test_run_until_caught_up_accepts_result_when_generation_unchanged() { "run_until_caught_up_accepts_result_when_generation_unchanged".into(), ); let coord = FsyncCoordinator::new(); - let raft_log = minimal_raft_log(storage); - // advance_durable_and_notify() clamps against max_index — matches pending_max below. - raft_log.set_max_index_for_test(5); + // See test_run_until_caught_up_advances_durable_index_on_success for why + // this doesn't use minimal_raft_log(): needs a real backing entry for + // try_advance_durable_index's content check, and a registered + // log_flush_tx to receive the FsyncCompleted report. + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + raft_log.entries.write().insert( + 5, + Entry { + index: 5, + term: 1, + payload: None, + }, + ); + raft_log.set_memory_max_index_for_test(5); // Two unrelated fences happened earlier — generation is 2, not 0 — before // this round is even recorded as in flight. @@ -498,6 +551,11 @@ fn test_run_until_caught_up_accepts_result_when_generation_unchanged() { // Nothing fences this round while it runs — generation stays at 2. coord.run_until_caught_up(&raft_log); + // Stand in for raft.rs's InternalEvent::FsyncCompleted handler. + while let Ok(InternalEvent::FsyncCompleted { index, term }) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(index, term); + } + assert_eq!( raft_log.durable_index.load(Ordering::Acquire), 5, @@ -525,7 +583,7 @@ fn test_run_until_caught_up_coalesces_queued_submits_into_one_flush() { let coord = FsyncCoordinator::new(); let raft_log = minimal_raft_log(storage); // advance_durable_and_notify() clamps against max_index — matches pending_max below. - raft_log.set_max_index_for_test(10); + raft_log.set_memory_max_index_for_test(10); // Simulate two submit() calls that both lost the CAS while a round was // in flight — both just accumulated into the same pending state. diff --git a/d-engine-core/src/storage/raft_log.rs b/d-engine-core/src/storage/raft_log.rs index 0b9b1496..c5560be8 100644 --- a/d-engine-core/src/storage/raft_log.rs +++ b/d-engine-core/src/storage/raft_log.rs @@ -77,6 +77,18 @@ pub trait RaftLog: Send + Sync + 'static { /// - DiskFirst: equals `last_entry_id()` (every append blocks until durable). fn durable_index(&self) -> u64; + /// Content-validated durable-watermark advance. `index`/`term` describe + /// what a completed fsync claims is now safe — rejected (`None`) if + /// `entry_term(index) != Some(term)`, meaning the log content at that + /// index has changed (truncated + replaced) since fsync started on it. + /// `Some(new_value)` only when it actually advanced — callers use this + /// to decide whether to fire `handle_log_flushed`. + fn try_advance_durable_index( + &self, + index: u64, + term: u64, + ) -> Option; + /// Returns the LogId (term + index) of the last entry. /// /// # Returns diff --git a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs index 43e955e9..a9376c82 100644 --- a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs +++ b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs @@ -21,6 +21,7 @@ pub struct BufferedRaftLogTestContext { pub storage: Arc, pub flush_policy: FlushPolicy, pub instance_id: String, + log_flush_rx: tokio::sync::mpsc::UnboundedReceiver, } impl BufferedRaftLogTestContext { @@ -35,12 +36,12 @@ impl BufferedRaftLogTestContext { 1, PersistenceConfig { flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); // Small delay to ensure processor is ready std::thread::sleep(std::time::Duration::from_millis(10)); @@ -50,9 +51,21 @@ impl BufferedRaftLogTestContext { storage, flush_policy, instance_id: instance_id.to_string(), + log_flush_rx, } } + /// Stands in for `raft.rs`'s `InternalEvent::FsyncCompleted` handler, + /// which isn't running in these `BufferedRaftLog`-only unit tests. Call + /// after any operation that should make `durable_index` advance + /// (`append_entries`, `flush`, truncation + resync, ...) and before + /// asserting on `durable_index()` — see + /// `drain_and_apply_fsync_completions` for why this is necessary since + /// #446/#447. + pub fn drain_fsync_completions(&mut self) { + drain_and_apply_fsync_completions(&self.raft_log, &mut self.log_flush_rx); + } + /// Helper to append a batch of entries with specified range and term pub async fn append_entries( &self, @@ -88,12 +101,12 @@ impl BufferedRaftLogTestContext { 1, PersistenceConfig { flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(std::time::Duration::from_millis(10)); let ctx = Self { @@ -101,6 +114,7 @@ impl BufferedRaftLogTestContext { storage, flush_policy, instance_id: instance_id.to_string(), + log_flush_rx, }; (ctx, flush_count) } @@ -114,12 +128,12 @@ impl BufferedRaftLogTestContext { 1, PersistenceConfig { flush_policy: self.flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); // Small delay to ensure processor is ready std::thread::sleep(std::time::Duration::from_millis(10)); @@ -129,6 +143,7 @@ impl BufferedRaftLogTestContext { storage, flush_policy: self.flush_policy.clone(), instance_id: self.instance_id.clone(), + log_flush_rx, } } } @@ -228,3 +243,24 @@ pub async fn simulate_delete_command( raft_log.insert_batch(entries).await.unwrap(); raft_log.flush().await.unwrap(); } + +/// Stands in for `raft.rs`'s `InternalEvent::FsyncCompleted` handler, which +/// `BufferedRaftLog`-only unit tests don't have running. Since #446/#447, +/// `durable_index` only advances when something calls +/// `try_advance_durable_index(index, term)` in response to that event — +/// `FsyncCoordinator`/`IOTask::ReplaceRange`'s `notify_fsync_completed` only +/// *sends* the event, it never writes `durable_index` itself. A test that +/// registers a `log_flush_tx` and wants to see `durable_index()` actually +/// advance must drain that channel through this helper — otherwise the +/// event sits unread and `durable_index()` never moves, no matter how long +/// you sleep. +pub fn drain_and_apply_fsync_completions( + raft_log: &Arc>, + log_flush_rx: &mut tokio::sync::mpsc::UnboundedReceiver, +) { + while let Ok(event) = log_flush_rx.try_recv() { + if let crate::InternalEvent::FsyncCompleted { index, term } = event { + raft_log.try_advance_durable_index(index, term); + } + } +} diff --git a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs index 7990cf58..b8a32bdd 100644 --- a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs +++ b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs @@ -322,7 +322,7 @@ impl MockStorageEngine { /// After a failed fsync, `batch_processor` logs the error and does NOT zero /// `pending_max` (the success branch `else { pending_max = 0 }` is not taken). /// This is the deterministic pre-condition needed to exercise the bug where - /// `handle_non_write_cmd(IOTask::Reset)` forgets to zero `pending_max`. + /// `run_storage_tasks(IOTask::Reset)` forgets to zero `pending_max`. pub fn not_durable_first_flush_fails(id: String) -> Self { let mut mock_log_store = MockLogStore::new(); let mut mock_meta_store = MockMetaStore::new(); @@ -393,7 +393,7 @@ impl MockStorageEngine { /// Create a MockStorageEngine where `replace_range()` always fails, /// simulating a fatal storage error during conflict-resolution - /// (truncate + write). `handle_non_write_cmd`'s `IOTask::ReplaceRange` + /// (truncate + write). `run_storage_tasks`'s `IOTask::ReplaceRange` /// arm treats this as unrecoverable — disk state is now uncertain. pub fn not_durable_replace_range_fails(id: String) -> Self { let mut mock_log_store = MockLogStore::new(); diff --git a/d-engine-server/src/node/builder_test.rs b/d-engine-server/src/node/builder_test.rs index 0556b0e2..eb675479 100644 --- a/d-engine-server/src/node/builder_test.rs +++ b/d-engine-server/src/node/builder_test.rs @@ -60,7 +60,6 @@ async fn test_set_raft_log_replaces_default() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, mock_storage_engine.clone(), diff --git a/d-engine-server/src/test_utils/integration/mod.rs b/d-engine-server/src/test_utils/integration/mod.rs index 00365525..e0208b75 100644 --- a/d-engine-server/src/test_utils/integration/mod.rs +++ b/d-engine-server/src/test_utils/integration/mod.rs @@ -183,7 +183,6 @@ pub fn setup_raft_components( flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage_engine.clone(), diff --git a/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs b/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs index ef0f53e2..9130f08f 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs @@ -61,7 +61,7 @@ async fn test_crash_recovery() { #[tokio::test] async fn test_crash_recovery_with_multiple_entries() { // Create and populate storage - let original_ctx = TestContext::new( + let mut original_ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -85,6 +85,7 @@ async fn test_crash_recovery_with_multiple_entries() { // Ensure all entries are persisted for DiskFirst strategy original_ctx.raft_log.flush().await.unwrap(); + original_ctx.drain_fsync_completions(); // Verify all entries are in memory and durable assert_eq!(original_ctx.raft_log.durable_index(), 5); @@ -127,7 +128,6 @@ async fn test_partial_flush_with_graceful_shutdown() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -163,7 +163,6 @@ async fn test_partial_flush_with_graceful_shutdown() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -207,7 +206,6 @@ async fn test_partial_flush_after_crash() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -255,7 +253,6 @@ async fn test_partial_flush_after_crash() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -381,7 +378,6 @@ async fn test_memfirst_crash_recovery_durability() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -414,7 +410,6 @@ async fn test_diskfirst_crash_recovery_durability() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -447,7 +442,6 @@ async fn test_diskfirst_crash_recovery_durability() { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-server/tests/storage_buffered_raft_log/mod.rs b/d-engine-server/tests/storage_buffered_raft_log/mod.rs index 8687f2a4..35b4300c 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/mod.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/mod.rs @@ -33,6 +33,7 @@ pub struct TestContext { pub _temp_dir: Option, pub flush_policy: FlushPolicy, pub path: String, + log_flush_rx: tokio::sync::mpsc::UnboundedReceiver, } impl TestContext { @@ -49,12 +50,12 @@ impl TestContext { 1, PersistenceConfig { flush_policy: flush_policy.clone(), - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); // Small delay to ensure processor is ready std::thread::sleep(Duration::from_millis(10)); @@ -65,6 +66,23 @@ impl TestContext { storage, flush_policy, _temp_dir: Some(temp_dir), + log_flush_rx, + } + } + + /// Stands in for `raft.rs`'s `InternalEvent::FsyncCompleted` handler, + /// which isn't running in these `BufferedRaftLog`-only integration + /// tests. Since #446/#447, `durable_index` only advances when something + /// drains that event and calls `try_advance_durable_index` — call this + /// after any operation that should make `durable_index` advance and + /// before asserting on it. Not needed after `recover_from_crash()`: the + /// recovered context's `durable_index` is derived directly from on-disk + /// state at construction, not from this event. + pub fn drain_fsync_completions(&mut self) { + while let Ok(event) = self.log_flush_rx.try_recv() { + if let d_engine_core::InternalEvent::FsyncCompleted { index, term } = event { + self.raft_log.try_advance_durable_index(index, term); + } } } @@ -88,12 +106,12 @@ impl TestContext { 1, PersistenceConfig { flush_policy: self.flush_policy.clone(), - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); @@ -103,6 +121,7 @@ impl TestContext { flush_policy: self.flush_policy.clone(), _temp_dir: Some(temp_dir), path: self.path.clone(), + log_flush_rx, } } diff --git a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs index 1d050f53..3e7b81c7 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs @@ -34,7 +34,6 @@ mod filter_out_conflicts_and_append_performance_tests { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }; @@ -104,7 +103,6 @@ mod filter_out_conflicts_and_append_performance_tests { flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }; diff --git a/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs b/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs index 351e5a8d..ea60cdc1 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs @@ -44,7 +44,7 @@ use d_engine_core::RaftLog; #[tokio::test] async fn test_quorum_acknowledged_index_survives_real_crash_and_reopen() { - let ctx = TestContext::new( + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -54,6 +54,7 @@ async fn test_quorum_acknowledged_index_survives_real_crash_and_reopen() { // First 5 entries, explicitly flushed: genuinely durable, deterministic. ctx.append_entries(1, 5, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 5); // 3-node cluster: both followers already report match_index=5 (post-Stage2 @@ -81,6 +82,7 @@ async fn test_quorum_acknowledged_index_survives_real_crash_and_reopen() { // simulated crash — keeps this test's crash/recovery assertions exact, not bounded. ctx.append_entries(6, 5, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 10); let recovered = ctx.recover_from_crash(); diff --git a/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs b/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs index f9c9dfe6..acf79449 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs @@ -13,7 +13,7 @@ use super::TestContext; #[tokio::test] async fn test_log_compaction() { - let ctx = TestContext::new( + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -23,6 +23,7 @@ async fn test_log_compaction() { // With MemFirst, entries are buffered and flushed asynchronously. // Wait for all entries to become durable before checking durable_index. ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Compact first 50 entries ctx.raft_log.purge_logs_up_to(LogId { index: 50, term: 1 }).await.unwrap(); diff --git a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs index 3f41a4c0..7968af39 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs @@ -22,7 +22,7 @@ use super::TestContext; #[tokio::test] async fn test_high_concurrency() { - let ctx = TestContext::new( + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -52,6 +52,7 @@ async fn test_high_concurrency() { // With MemFirst, entries are buffered; wait for all to be durable before asserting. ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Verify all entries persisted assert_eq!(ctx.raft_log.durable_index(), 1000); @@ -154,7 +155,7 @@ mod mem_first_tests { #[tokio::test] async fn test_async_persistence() { - let ctx = TestContext::new( + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -164,6 +165,7 @@ mod mem_first_tests { // Trigger flush ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Verify persistence assert_eq!(ctx.raft_log.durable_index(), 100); diff --git a/d-engine/src/docs/examples/three-nodes-standalone.md b/d-engine/src/docs/examples/three-nodes-standalone.md index b5432c04..b5ab1764 100644 --- a/d-engine/src/docs/examples/three-nodes-standalone.md +++ b/d-engine/src/docs/examples/three-nodes-standalone.md @@ -60,7 +60,6 @@ lease_duration_ms = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -max_buffered_entries = 10000 ``` **Key differences from single-node expansion:** diff --git a/examples/single-node-expansion/config/n1.toml b/examples/single-node-expansion/config/n1.toml index d036239d..1d961cb3 100644 --- a/examples/single-node-expansion/config/n1.toml +++ b/examples/single-node-expansion/config/n1.toml @@ -44,8 +44,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = true diff --git a/examples/single-node-expansion/config/n2.toml b/examples/single-node-expansion/config/n2.toml index bad8ce26..0e4bb985 100644 --- a/examples/single-node-expansion/config/n2.toml +++ b/examples/single-node-expansion/config/n2.toml @@ -29,7 +29,6 @@ lease_duration_ms = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 20 } } -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/single-node-expansion/config/n3.toml b/examples/single-node-expansion/config/n3.toml index 67cf1fd2..32dc4c3b 100644 --- a/examples/single-node-expansion/config/n3.toml +++ b/examples/single-node-expansion/config/n3.toml @@ -31,7 +31,6 @@ lease_duration_ms = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 20 } } -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/three-nodes-standalone/config/n1.toml b/examples/three-nodes-standalone/config/n1.toml index bb6db45a..1acb62d9 100644 --- a/examples/three-nodes-standalone/config/n1.toml +++ b/examples/three-nodes-standalone/config/n1.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = true diff --git a/examples/three-nodes-standalone/config/n2.toml b/examples/three-nodes-standalone/config/n2.toml index 95280d76..959b904a 100644 --- a/examples/three-nodes-standalone/config/n2.toml +++ b/examples/three-nodes-standalone/config/n2.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = true diff --git a/examples/three-nodes-standalone/config/n3.toml b/examples/three-nodes-standalone/config/n3.toml index 4aa67b3b..12c0ff72 100644 --- a/examples/three-nodes-standalone/config/n3.toml +++ b/examples/three-nodes-standalone/config/n3.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = true diff --git a/examples/three-nodes-standalone/docker/config/n1.toml b/examples/three-nodes-standalone/docker/config/n1.toml index 8deff325..a5b007dc 100644 --- a/examples/three-nodes-standalone/docker/config/n1.toml +++ b/examples/three-nodes-standalone/docker/config/n1.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/three-nodes-standalone/docker/config/n2.toml b/examples/three-nodes-standalone/docker/config/n2.toml index 7e7eb644..06c54f50 100644 --- a/examples/three-nodes-standalone/docker/config/n2.toml +++ b/examples/three-nodes-standalone/docker/config/n2.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/three-nodes-standalone/docker/config/n3.toml b/examples/three-nodes-standalone/docker/config/n3.toml index 73d6bc59..2cfe4ad9 100644 --- a/examples/three-nodes-standalone/docker/config/n3.toml +++ b/examples/three-nodes-standalone/docker/config/n3.toml @@ -40,8 +40,6 @@ max_pending_reads = 500 [raft.persistence] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -max_buffered_entries = 10000 [raft.snapshot] enable = false From 559708a5b93ed72873bc57f796d958d6eaf039db Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:13:52 +0800 Subject: [PATCH 05/11] fix #446: revert synchronous persist in append_entries (leader perf regression) I/O critical path. RPO=0 is already enforced downstream (quorum counts durable_index; followers withhold ACKs until durable), so the wait is redundant. Flame graph confirms the raft loop no longer blocks on persist. storage: - append_entries: insert_to_memory + write_notify.notify_one(), return - drop IOTask::Persist + persisted_index (nothing reads log entries back from storage, so openraft's `submitted`-style watermark has no consumer here) - IO thread: restore persist_pending_range SkipMap scan on write_notify - unchanged: durable-quorum, follower withhold, FsyncCompleted -> try_advance_durable_index, fence_truncation - module doc rewritten to match observability: - core.raft.snapshot.log_lag gauge; error! at 50x threshold (unbounded in-memory log growth = OOM) - core.raft.log.{memory_max_index,durable_index,rpo_window} gauges - startup line: in-memory Raft-log memory budget tests: - rm process_crash_safety_test (asserts the reverted "append_entries waits for storage" behavior; RPO=0 covered by quorum_durability_test + follower withhold) - persisted_index_clamp_test -> durable_index_truncation_clamp_test; assertions re-pointed to durable_index, stale-persist-after-truncation scenario preserved - rm drain_fsync_test::test_poisoned_rejects_queued_persist_task (guard already covered by test_poisoned_skips_replace_range) - test_persist_entries_failure_poisons: restore main's async-poison shape --- .../snapshot_policy/log_size.rs | 30 ++- .../src/storage/buffered_raft_log.rs | 193 +++++++++--------- .../content_validated_watermark_test.rs | 32 +-- .../drain_fsync_test.rs | 86 ++------ .../durable_index_truncation_clamp_test.rs | 162 +++++++++++++++ .../persisted_index_clamp_test.rs | 183 ----------------- .../process_crash_safety_test.rs | 100 --------- .../truncation_fsync_fence_test.rs | 4 +- .../test_utils/mock/mock_storage_engine.rs | 6 +- d-engine-server/src/node/builder.rs | 28 ++- 10 files changed, 336 insertions(+), 488 deletions(-) create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs delete mode 100644 d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs delete mode 100644 d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs diff --git a/d-engine-core/src/state_machine_handler/snapshot_policy/log_size.rs b/d-engine-core/src/state_machine_handler/snapshot_policy/log_size.rs index 96a466f7..da59406e 100644 --- a/d-engine-core/src/state_machine_handler/snapshot_policy/log_size.rs +++ b/d-engine-core/src/state_machine_handler/snapshot_policy/log_size.rs @@ -4,7 +4,7 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; - +use tracing::error; use tracing::trace; use tracing::warn; @@ -39,12 +39,28 @@ impl SnapshotPolicy for LogSizePolicy { let lag = self.calculate_lag(ctx); let threshold = self.threshold.load(Ordering::Relaxed); - if threshold > 0 && lag >= threshold.saturating_mul(10) { - warn!( - lag, - threshold, - "Log lag exceeds 10x snapshot threshold — snapshots may not be keeping up" - ); + metrics::gauge!("core.raft.snapshot.log_lag").set(lag as f64); + + // The in-memory Raft log grows until a snapshot purges it. If snapshot + // creation can't keep up with the write rate this climbs unbounded and + // eventually OOMs the node — make it loud well before that. + if threshold > 0 { + if lag >= threshold.saturating_mul(50) { + error!( + lag, + threshold, + "Raft log lag is 50x the snapshot threshold — snapshot creation is \ + NOT keeping up with writes; the in-memory log is growing unbounded \ + and will OOM this node. Check snapshot/apply throughput." + ); + } else if lag >= threshold.saturating_mul(10) { + warn!( + lag, + threshold, + "Raft log lag exceeds 10x the snapshot threshold — snapshots may \ + not be keeping up" + ); + } } let should_trigger = lag >= threshold; diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index 2a0519aa..f0d4351f 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -17,33 +17,41 @@ //! //! ## IO thread (notify-then-spawn-fsync) //! -//! On wakeup from `write_notify`: +//! On wakeup from `write_notify` (`run_batch_turn`): //! 1. **Read** — scan SkipMap range `(durable_index, memory_max_index]` -//! 2. **Persist** — write range to OS page cache via `persist_entries` -//! 3. **Spawn fsync** — dispatch fdatasync to `spawn_blocking` pool via `spawn_fsync`, return immediately -//! 4. **Loop** — back to `select!` for next wakeup; prior fsync runs concurrently in pool +//! 2. **Persist** — write range to OS page cache via `persist_entries` (no fsync) +//! 3. **Dispatch fsync** — `FsyncCoordinator::submit()` hands the fdatasync to a +//! `spawn_blocking` task and returns immediately +//! 4. **Loop** — back to `select!`; the prior fsync runs concurrently in the pool //! -//! `durable_index` is advanced inside the blocking task via `advance_durable_and_notify` -//! (`fetch_max`, AcqRel). Multiple concurrent tasks completing out of order are safe: -//! a late-arriving lower index is a no-op. +//! ## How `durable_index` advances (#446 single-owner) //! -//! ## Fsync triggers +//! The blocking fsync task does **not** write `durable_index`. On completion it +//! calls `notify_fsync_completed(index, term)`, sending +//! `InternalEvent::FsyncCompleted { index, term }` to `raft.rs`'s event loop — +//! the sole owner of `durable_index`. That loop calls +//! `try_advance_durable_index(index, term)`, which content-validates the report +//! (`entry_term(index) == Some(term)`) and clamps to `memory_max_index` before +//! `fetch_max`. A stale report — for entries a concurrent truncation already +//! discarded — is rejected. `FsyncCoordinator`'s `generation` fence +//! (`fence_truncation` / `fence_reset`) is the first line of defence: a fsync +//! round whose generation changed mid-flight never sends its completion at all. //! -//! All four triggers below funnel through `run_batch_turn` (persist pending -//! entries, drain any queued commands, then dispatch) into the single -//! `FsyncCoordinator::submit()` entry point — there is no separate inline path. +//! ## Fsync triggers //! -//! 1. **Notify-driven** (normal): `write_notify` → `run_batch_turn` (no reply) -//! 2. **Explicit** (flush API): `flush()` → `IOTask::Flush(tx)` → `run_batch_turn` with reply sender -//! 3. **Idle timer** (safety net): `idle_flush_interval_ms` elapsed → `persist_pending_range` + `submit()` -//! 4. **Shutdown**: `IOTask::Shutdown` → `run_batch_turn`, then `close()` waits (bounded by -//! `shutdown_timeout_ms`) for the IO thread's runtime to drain any in-flight fsync task +//! 1. **Notify-driven** (normal): `write_notify` → `run_batch_turn` +//! 2. **Explicit** (flush API): `flush()` → `IOTask::Flush(tx)` → `run_batch_turn` with a reply sender +//! 3. **Idle timer** (safety net): `idle_flush_interval_ms` elapsed → +//! `persist_pending_range` + `FsyncCoordinator::submit()` directly (not via `run_batch_turn`) +//! 4. **Shutdown**: `IOTask::Shutdown` → `run_batch_turn`, then `close()` waits +//! (bounded by `shutdown_timeout_ms`) for the IO thread's runtime to drain any in-flight fsync //! //! ## Durability contract //! -//! `durable_index` advances only after physical fdatasync in the blocking task. -//! Concurrent fsyncs coalesce at the storage layer: if batch B's fsync covers A's WAL -//! position, A's `flush_wal` returns fast with no extra disk IO — storage-layer group commit. +//! `durable_index` advances only after a physical fdatasync completes — and only +//! on `raft.rs`'s event loop, after content validation. Concurrent fsyncs +//! coalesce at the storage layer: if batch B's fsync covers A's WAL position, +//! A's `flush_wal` returns fast with no extra disk IO — storage-layer group commit. use super::fsync_coordinator::FsyncCoordinator; use crate::Error; @@ -197,15 +205,6 @@ impl TermSegments { /// on tokio worker threads or the inbound event loop. #[derive(Debug)] pub enum IOTask { - /// Persist entries on the IO thread. `append_entries()` sends this and - /// awaits `done` — replaces the old inline `persist_entries()` call that - /// ran on the caller's own task (raft-core-loop), which could block - /// behind the IO thread's own concurrent fsync. - Persist { - entries: Vec, - done: oneshot::Sender>, - }, - /// Atomically truncate from `truncate_from` then persist `new_entries`. /// Conflict-resolution path: truncate + write are a single atomic IO unit. /// `done` is signalled after the IO thread finishes the replace so callers @@ -267,10 +266,7 @@ where // Raft must not tell a client or a peer a write is safe ahead of this point, // regardless of what's already visible in `entries`. pub(crate) durable_index: AtomicU64, - // Highest index handed to the storage engine (page cache), not yet - // fsynced. Set by append_entries()'s synchronous persist_entries() call. - // Lets the IO thread know what to fsync without re-scanning/re-writing. - persisted_index: AtomicU64, + // The next index to be allocated pub(crate) next_id: AtomicU64, @@ -482,19 +478,13 @@ where self.insert_to_memory(&entries); - // Route the actual write through the IO thread — never call - // persist_entries() inline from this task. - // Still blocks the caller until truly persisted. - let (done_tx, done_rx) = oneshot::channel(); - self.command_sender - .send(IOTask::Persist { - entries, - done: done_tx, - }) - .map_err(|e| NetworkError::SingalSendFailed(format!("Persist send failed: {e:?}")))?; - done_rx - .await - .map_err(|_| NetworkError::SingalSendFailed("Persist done channel closed".into()))??; + // Signal the IO thread to persist. Fire-and-forget: the entry is in the + // in-memory log and quorum-visible now; the IO thread scans the SkipMap + // and persists + fsyncs off this task. Concurrent notify_one() calls + // coalesce into one wakeup. RPO=0 is enforced downstream (#446), not + // here: commit quorum counts only `durable_index()`, and followers + // withhold AppendEntries ACKs until their own `durable_index` catches up. + self.write_notify.notify_one(); Ok(()) } @@ -894,7 +884,6 @@ where last_purged_index: AtomicU64::new(last_purged_index_val), last_purged_term: AtomicU64::new(last_purged_term_val), durable_index: AtomicU64::new(disk_len), - persisted_index: AtomicU64::new(disk_len), next_id: AtomicU64::new(disk_len + 1), write_notify: Arc::new(Notify::new()), command_sender: command_sender.clone(), @@ -1012,7 +1001,9 @@ where if should_break { break; } } _ = safety_timer.tick() => { - Self::fold_persisted_watermark(&this, &mut pending_max); + let start = this.durable_index.load(Ordering::Acquire) + 1; + let end = this.memory_max_index.load(Ordering::Acquire); + let _ = Self::persist_pending_range(&this, start, end, &mut pending_max, "safety-net").await; if pending_max > 0 { this.fsync_coordinator.submit(&this, pending_max, vec![]); @@ -1023,14 +1014,38 @@ where } } - /// Folds `persisted_index` (set by `append_entries()`'s synchronous - /// write) into `pending_max`, so the IO thread still dispatches fsync - /// for it even though writing is no longer this thread's job. - fn fold_persisted_watermark( + /// Writes entries in `(from, to]` that haven't reached page cache yet (no + /// fsync). Iterates the SkipMap for the range — entries removed by a + /// concurrent truncation simply aren't returned, so a stale from/to pair is + /// self-correcting and never writes wrong data. Advances `pending_max` on + /// success; propagates the error as-is on failure. + async fn persist_pending_range( this: &Arc, + from: u64, + to: u64, pending_max: &mut u64, - ) { - *pending_max = (*pending_max).max(this.persisted_index.load(Ordering::Acquire)); + ctx: &str, + ) -> Result<()> { + if this.is_poisoned() { + return Err(Error::Fatal("raft log storage is poisoned".to_string())); + } + if from > to { + return Ok(()); + } + let entries = this.get_entries_range(from..=to)?; + if entries.is_empty() { + return Ok(()); + } + this.log_store + .persist_entries(entries) + .await + .inspect(|_| { + *pending_max = (*pending_max).max(to); + }) + .inspect_err(|e| { + error!("{ctx} persist_entries failed: {e:?}"); + this.mark_poisoned_and_notify(format!("{ctx}: persist_entries failed: {e:?}")); + }) } async fn run_batch_turn( @@ -1040,7 +1055,15 @@ where mut replies: Vec>>, mut seen_shutdown: bool, ) -> bool { - Self::fold_persisted_watermark(this, pending_max); + let start = this.durable_index.load(Ordering::Acquire) + 1; + let end = this.memory_max_index.load(Ordering::Acquire); + let mut persist_failed = false; + if let Err(e) = Self::persist_pending_range(this, start, end, pending_max, "batch").await { + for reply in replies.drain(..) { + let _ = reply.send(Err(Error::Fatal(format!("persist_entries failed: {e:?}")))); + } + persist_failed = true; + } // `seen_shutdown` is not a gate here — regardless of whether the // caller already knows shutdown is happening, any commands still @@ -1065,15 +1088,34 @@ where } } + // A Flush caller needs everything up to now durable; a command drained + // above may have advanced memory_max_index past the leading persist. + if !replies.is_empty() && !persist_failed { + let start = *pending_max + 1; + let end = this.memory_max_index.load(Ordering::Acquire); + let _ = + Self::persist_pending_range(this, start, end, pending_max, "batch catch-up").await; + } + this.fsync_coordinator.submit(this, *pending_max, replies); + *pending_max = 0; if seen_shutdown { let _ = this.meta_store.flush(); } + + // Observability — IO thread, per write burst (not per append). Where the + // log pipeline stands; `rpo_window` = in memory but not yet fsynced. + let mem = this.memory_max_index.load(Ordering::Relaxed); + let dur = this.durable_index.load(Ordering::Relaxed); + metrics::gauge!("core.raft.log.memory_max_index").set(mem as f64); + metrics::gauge!("core.raft.log.durable_index").set(dur as f64); + metrics::gauge!("core.raft.log.rpo_window").set(mem.saturating_sub(dur) as f64); + seen_shutdown } - /// Runs one storage-mutating IOTask (Persist/ReplaceRange/Purge/Reset) + /// Runs one storage-mutating IOTask (ReplaceRange/Purge/Reset) /// against log_store. Flush/Shutdown are intercepted by the caller /// (`batch_processor`) before this is called — unreachable here. /// @@ -1090,33 +1132,6 @@ where IOTask::Shutdown => { unreachable!("Shutdown is always filtered out before reaching run_storage_tasks") } - IOTask::Persist { entries, done } => { - if this.is_poisoned() { - let _ = done.send(Err(Error::Fatal("raft log storage is poisoned".into()))); - return true; // signal batch_processor to exit — disk state is untrusted - } - let max_idx = entries.last().map(|e| e.index).unwrap_or(0); - let result = this.log_store.persist_entries(entries).await; - if let Err(ref e) = result { - error!("IOTask::Persist failed (fatal): {e:?}"); - this.mark_poisoned_and_notify(format!("Persist failed: {e:?}")); - let _ = done.send(result); - return true; // signal batch_processor to exit — disk state is corrupted - } - if max_idx > 0 { - let current_bound = this - .memory_max_index - .load(Ordering::Acquire) - .max(this.last_purged_index.load(Ordering::Acquire)); - let safe_max_idx = max_idx.min(current_bound); - if safe_max_idx > 0 { - this.persisted_index.fetch_max(safe_max_idx, Ordering::AcqRel); - this.fsync_coordinator.submit(this, safe_max_idx, vec![]); - } - } - let _ = done.send(result); - false // write succeeded, storage still trustworthy — keep the IO thread running - } IOTask::ReplaceRange { truncate_from, new_entries, @@ -1138,14 +1153,9 @@ where let _ = done.send(result); return true; // signal batch_processor to exit — disk state is corrupted } - // persisted_index moved here from remove_range — this handler - // is the sole writer now, single-threaded, no content check needed. - this.persisted_index - .fetch_min(truncate_from.saturating_sub(1), Ordering::AcqRel); if max_idx > 0 { *pending_max = (*pending_max).max(max_idx); - this.persisted_index.fetch_max(max_idx, Ordering::AcqRel); this.fsync_coordinator.submit(this, max_idx, vec![]); } let _ = done.send(result); @@ -1194,7 +1204,6 @@ where self.entries.write().clear(); self.durable_index.store(0, Ordering::Release); - self.persisted_index.store(0, Ordering::Release); self.next_id.store(1, Ordering::Release); // Reset boundaries @@ -1558,17 +1567,13 @@ mod id_allocation_test; mod performance_test; #[cfg(test)] -#[path = "buffered_raft_log_test/persisted_index_clamp_test.rs"] -mod persisted_index_clamp_test; +#[path = "buffered_raft_log_test/durable_index_truncation_clamp_test.rs"] +mod durable_index_truncation_clamp_test; #[cfg(test)] #[path = "buffered_raft_log_test/pipeline_overlap_test.rs"] mod pipeline_overlap_test; -#[cfg(test)] -#[path = "buffered_raft_log_test/process_crash_safety_test.rs"] -mod process_crash_safety_test; - #[cfg(test)] #[path = "buffered_raft_log_test/quorum_durability_test.rs"] mod quorum_durability_test; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs index 68a38051..5e58fe62 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs @@ -1,27 +1,13 @@ -//! Content-validated `durable_index` advance (#446/#447 single-owner -//! redesign). Tests `try_advance_durable_index(&self, index: u64, term: u64) -//! -> Option` — `Some(new_value)` only when it actually advanced, -//! `None` when rejected as stale (`entry_term(index) != Some(term)`) or -//! already applied. +//! Content-validated `durable_index` advance (#446 single-owner redesign). +//! Tests `try_advance_durable_index(index, term) -> Option`: +//! `Some(new)` only when it actually advanced, `None` when rejected as stale +//! (`entry_term(index) != Some(term)`) or already applied. //! -//! Not wired into the mod tree yet — add -//! `#[path = "buffered_raft_log_test/content_validated_watermark_test.rs"] -//! mod content_validated_watermark_test;` to `buffered_raft_log.rs` next to -//! the other test module declarations. -//! -//! Why these tests don't need thread races or timing gates (unlike -//! `truncation_fsync_fence_test.rs`): under single ownership, the report and -//! the truncation are just two sequential calls in whatever order they -//! happen to arrive — no interleaving *inside* a function body is possible -//! because there's only one caller. Each test below drives one arrival order -//! directly. -//! -//! `persisted_index` has no equivalent test here — it doesn't need content -//! validation. Its only writer is now the IO thread (B), processing -//! `IOTask::Persist`/`ReplaceRange` strictly in the order the single owner -//! (A) issued them (each `await`ed before the next is sent), so there's no -//! stale-message window the way there is for `durable_index`'s async, -//! un-awaited fsync-completion report. +//! Why no thread races / timing gates here (unlike `truncation_fsync_fence_test.rs`): +//! `durable_index` has one owner — raft.rs's event loop. The fsync-completion +//! report and a truncation are just two sequential calls on that thread, so no +//! interleaving inside a function body is possible. Each test drives one +//! arrival order directly. use std::sync::Arc; use std::time::Duration; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs index 9a3300c0..815a9ab0 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs @@ -1041,8 +1041,12 @@ async fn test_poisoned_survives_reset() { /// A `persist_entries()` (page-cache write) failure poisons the log, exactly /// like an fsync failure does — these are two independent failure surfaces -/// (see `IOTask::Persist` vs `FsyncCoordinator::run_until_caught_up`) -/// and both must reach the same fatal outcome. +/// (`persist_pending_range` vs `FsyncCoordinator::run_until_caught_up`) and +/// both must reach the same fatal outcome. +/// +/// The persist runs on the IO thread off `append_entries`'s task, so the +/// poison lands asynchronously — the black-box guarantee is that the *next* +/// write is rejected, not that this one fails. /// /// Without this test, a bug that only wires up ONE of the two poisoning /// paths (e.g. fsync failures poison correctly, but persist_entries @@ -1067,20 +1071,18 @@ async fn test_persist_entries_failure_poisons() { let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - // append_entries() routes the write through IOTask::Persist and awaits - // the IO thread's reply — a persist failure now surfaces synchronously, - // right here, not discovered later by some other task. - let result = raft_log + // Notifies the IO thread, which runs persist_pending_range and hits the + // mock's first (failing) persist_entries() — the persist_pending_range + // poisoning path, not FsyncCoordinator's. + raft_log .append_entries(vec![Entry { index: 1, term: 1, payload: None, }]) - .await; - assert!( - result.is_err(), - "a persist_entries() failure must surface synchronously from append_entries()" - ); + .await + .unwrap(); + sleep(Duration::from_millis(20)).await; // let the IO thread process it assert!( raft_log.is_poisoned(), @@ -1102,68 +1104,6 @@ async fn test_persist_entries_failure_poisons() { ); } -/// `IOTask::Persist`'s own `is_poisoned()` guard (top of its handler, on the -/// IO thread) is a *different* check from `append_entries()`'s caller-side -/// fast-fail (line ~471) — that one only protects writes submitted *after* -/// poisoning already happened. This test targets the IO-thread-side guard -/// specifically, for a `Persist` task that was already queued *before* the -/// log got poisoned by something else (e.g. a concurrent ReplaceRange/Purge -/// failure): send `IOTask::Persist` directly through `command_sender`, -/// bypassing `append_entries()` entirely. Uses a plain always-succeeds mock -/// (`with_id`, no call-count requirement) — if the IO-thread-side guard is -/// missing or removed, `persist_entries()` would run and `done` would carry -/// `Ok(())` instead of the expected "...poisoned..." error, which the -/// `other => panic!` arm below catches either way. -#[tokio::test] -async fn test_poisoned_rejects_queued_persist_task() { - let storage = Arc::new(MockStorageEngine::with_id( - "poisoned_rejects_queued_persist_task".into(), - )); - let (raft_log, receiver) = BufferedRaftLog::::new( - 1, - PersistenceConfig { - flush_policy: FlushPolicy::Batch { - idle_flush_interval_ms: 60_000, - }, - shutdown_timeout_ms: 5000, - }, - storage, - ); - let raft_log = raft_log.start(receiver, None); - std::thread::sleep(Duration::from_millis(10)); - - // Poisoned by something unrelated to this Persist task — simulated - // directly, same as the other `test_poisoned_skips_*` tests in this file. - raft_log.poisoned.store(true, Ordering::SeqCst); - - let (done_tx, done_rx) = tokio::sync::oneshot::channel(); - raft_log - .command_sender - .send(IOTask::Persist { - entries: vec![Entry { - index: 1, - term: 1, - payload: None, - }], - done: done_tx, - }) - .expect("IO thread must still be alive to receive the task"); - - let result = done_rx.await.expect("IO thread must reply, not drop the sender"); - match result { - Err(Error::Fatal(msg)) => assert!( - msg.contains("poisoned"), - "expected the poisoned short-circuit to fire before persist_entries() \ - was ever called, got: {msg}" - ), - other => panic!( - "expected Err(Fatal(\"...poisoned...\")), got: {other:?} — this means \ - the IO-thread-side is_poisoned() guard did not fire and \ - persist_entries() ran anyway", - ), - } -} - /// If `notify_fatal`'s underlying channel is already closed when a failure /// happens, the node must not fail *silently* — poisoned must still end up /// `true`, and the failure must be visible somewhere (log line), even though diff --git a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs new file mode 100644 index 00000000..2893d55e --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs @@ -0,0 +1,162 @@ +//! `durable_index` must never claim more of the log survived to disk than the +//! log actually holds right now. The danger case: a term-conflict truncation +//! shrinks the log while a persist / fsync for the old, longer log is still in +//! flight — the stale in-flight write must not push `durable_index` past the +//! truncation point. Two guards cover this: `try_advance_durable_index`'s term +//! check, and the `FsyncCoordinator` generation fence bumped by `remove_range`. + +use std::sync::Arc; +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// After a drastic truncate-then-regrow, `durable_index` must land exactly on +/// the new tail — never above it (would claim durability for discarded +/// entries), never stuck below it (the new tail must actually become durable). +#[tokio::test] +async fn test_durable_index_lands_on_new_tail_after_truncate_and_resync() { + let mut ctx = BufferedRaftLogTestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, // isolate from the safety-net timer + }, + "durable_index_lands_on_new_tail_after_truncate_and_resync", + ); + + // Old leader (term 1) replicates 1..=10. append_entries inserts them into + // memory and notifies the IO thread; nothing is fsync-confirmed until the + // FsyncCompleted events are drained below. + ctx.append_entries(1, 10, 1).await; + assert_eq!(ctx.raft_log.last_entry_id(), 10); + assert_eq!( + ctx.raft_log.durable_index(), + 0, + "no fsync report drained yet" + ); + + // New leader (term 2): index 2 conflicts, so the log is truncated from 2 + // and replaced with a single new entry — real log becomes [1, 2]. Slow + // path: remove_range(2..) drops memory_max_index to 1 and clamps + // durable_index down, then the new index 2 is inserted. + ctx.raft_log + .filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]) + .await + .unwrap(); + assert_eq!(ctx.raft_log.last_entry_id(), 2, "log is now [1, 2]"); + + // flush() only returns after its own fsync-completion event is enqueued, + // so draining right here is deterministic — no sleep needed. + ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); + + // The stale report for index 10 must be rejected (index 10 no longer + // exists); the report for index 2 must be accepted. + assert!( + ctx.raft_log.durable_index() <= ctx.raft_log.last_entry_id(), + "durable_index ({}) must not exceed last_entry_id ({})", + ctx.raft_log.durable_index(), + ctx.raft_log.last_entry_id() + ); + assert_eq!( + ctx.raft_log.durable_index(), + 2, + "durable_index must reach the true tail (2), not a stale pre-truncation watermark" + ); +} + +/// A persist whose entry set was captured *before* a truncation but finishes +/// *after* it must not let a stale fsync-completion report advance +/// `durable_index` into the range the truncation discarded. +/// +/// Timeline (deterministic via the persist gate): +/// 1. Old leader (term 1) replicates 1..=10. append_entries returns at once; +/// the IO thread starts persisting the range and blocks on the gate — its +/// captured entry set is [1..=10]. +/// 2. New leader (term 2): index 2 conflicts. filter_out_conflicts_and_append +/// runs remove_range(2..) synchronously (log is now [1]), inserts the new +/// index 2, and queues IOTask::ReplaceRange — which can't run yet, the IO +/// thread is still on the gate. +/// 3. Release the gate: the stale persist writes [1..=10], then the queued +/// ReplaceRange fixes the disk, then a flush() drives one legitimate +/// fsync-completion for index 2. +/// +/// Expected: draining the fsync completions advances `durable_index` to 2 and +/// rejects the stale report for index 10. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_stale_persist_after_truncation_does_not_advance_durable_index() { + let (storage, persist_gate) = + MockStorageEngine::not_durable_gated_persist("stale_persist_after_truncation".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let (log_flush_tx, mut log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + std::thread::sleep(Duration::from_millis(10)); + + // Step 1: replicate 1..=10; the IO thread blocks persisting this range. + let entries: Vec = (1..=10).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(entries).await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + // Step 2: term conflict at index 2 — on a task, since its ReplaceRange + // await is stuck behind the gated persist. + let truncate = { + let raft_log = raft_log.clone(); + tokio::spawn(async move { + raft_log.filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]).await + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + raft_log.last_entry_id(), + 2, + "in-memory truncation is synchronous — visible without waiting on the IO thread" + ); + + // Step 3: release the stale persist, let ReplaceRange land, then flush. + persist_gate.send(()).expect("IO thread should still be waiting on the gate"); + truncate.await.unwrap().unwrap(); + raft_log.flush().await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + // Drain fsync completions the way raft.rs's event loop would. + while let Ok(event) = log_flush_rx.try_recv() { + if let crate::InternalEvent::FsyncCompleted { index, term } = event { + raft_log.try_advance_durable_index(index, term); + } + } + + assert!( + raft_log.durable_index() <= raft_log.last_entry_id(), + "durable_index ({}) must never exceed last_entry_id ({}) — the stale persist \ + for 1..=10 must not be reported durable after truncation shrank the log to [1, 2]", + raft_log.durable_index(), + raft_log.last_entry_id() + ); + assert_eq!( + raft_log.durable_index(), + 2, + "durable_index must land on the post-truncation tail (2), not the stale 10" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs deleted file mode 100644 index 6efc9e95..00000000 --- a/d-engine-core/src/storage/buffered_raft_log_test/persisted_index_clamp_test.rs +++ /dev/null @@ -1,183 +0,0 @@ -//! `persisted_index` must never claim a follower has written more to its -//! storage engine than what its log actually contains right now. -//! -//! Scenario: a follower has replicated entries 1-10 from an old leader and -//! synchronously written them to its storage engine (page cache), but hasn't -//! fsynced yet. A new leader is elected, finds entries 2-10 don't match its -//! own history, and tells the follower to truncate everything from index=2 -//! onward — the follower's real log now only has index=1. The new leader then -//! sends one brand-new entry that happens to land at index=2 again (different -//! content, new term). -//! -//! `persisted_index` only ever moves up (`fetch_max`), so without clamping it -//! on truncation, it would still remember "wrote up to 10" from before the -//! truncation — a stale high-water mark that the small index=2 write can't -//! pull back down. The next disk sync would then advertise `durable_index=10` -//! to the rest of the engine, even though the follower's log — and its -//! storage engine — genuinely only holds entries 1 and 2. A power loss at -//! that moment would prove the claim false: the follower reboots with only -//! [1, 2], not [1..=10], yet something upstream may already have acted on -//! "this follower is durable through 10" (e.g. deciding it's safe to purge -//! earlier log entries elsewhere in the cluster). - -use std::sync::Arc; -use std::sync::atomic::Ordering; -use std::time::Duration; - -use d_engine_proto::common::Entry; - -use crate::storage::raft_log::RaftLog; -use crate::test_utils::BufferedRaftLogTestContext; -use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; - -fn entry( - index: u64, - term: u64, -) -> Entry { - Entry { - index, - term, - payload: None, - } -} - -/// `durable_index()` must never exceed `last_entry_id()` — it must never -/// claim durability for an index that doesn't exist in the log anymore. -#[tokio::test] -async fn test_durable_index_never_exceeds_log_after_truncation_and_resync() { - let mut ctx = BufferedRaftLogTestContext::new( - FlushPolicy::Batch { - idle_flush_interval_ms: 60_000, // isolate from the safety-net timer - }, - "durable_index_never_exceeds_log_after_truncation_and_resync", - ); - - // Old leader (term=1) replicates entries 1..=10. append_entries() persists - // them to the storage engine synchronously, but nothing fsyncs them yet. - ctx.append_entries(1, 10, 1).await; - assert_eq!(ctx.raft_log.last_entry_id(), 10); - assert_eq!(ctx.raft_log.durable_index(), 0, "nothing fsynced yet"); - - // New leader (term=2) finds index=2 doesn't match its history (term=1 - // there, should be term=2) and truncates from index=2 onward, replacing - // it with one brand-new entry — real log becomes just [1, 2]. This goes - // through filter_out_conflicts_and_append's term-conflict slow path: - // remove_range(2..=MAX) (the clamp under test fires here, since it drops - // max_index from 10 down to 1) followed by inserting the new index=2. - ctx.raft_log - .filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]) - .await - .unwrap(); - assert_eq!( - ctx.raft_log.last_entry_id(), - 2, - "log truncated and replaced down to [1, 2]" - ); - - // Trigger a disk sync and give it time to complete. - ctx.raft_log.flush().await.unwrap(); - tokio::time::sleep(Duration::from_millis(50)).await; - ctx.drain_fsync_completions(); - - // The follower must never advertise durability for an index it doesn't - // actually have. If persisted_index wasn't clamped down during the - // truncation, this would report 10 here — a lie. - assert!( - ctx.raft_log.durable_index() <= ctx.raft_log.last_entry_id(), - "durable_index ({}) must never exceed last_entry_id ({}) — it must not \ - claim durability for entries the truncation already discarded", - ctx.raft_log.durable_index(), - ctx.raft_log.last_entry_id() - ); - assert_eq!( - ctx.raft_log.durable_index(), - 2, - "durable_index must reach the log's true end (2), not a stale pre-truncation watermark" - ); -} - -/// Different ordering from the test above: there, `remove_range`'s clamp ran -/// *before* anything else touched `persisted_index`. Here, a `Persist` task -/// dispatched *before* the truncation is still stuck on the IO thread (write -/// not yet reached the storage engine) when the truncation's own clamp runs — -/// and only *afterward* does that stale `Persist` complete and call -/// `persisted_index.fetch_max(10, ..)` (the line under review in -/// `handle_write_cmd`'s `IOTask::Persist` arm), using an index from entries -/// the truncation already discarded. `fetch_max` only ever moves up, so if -/// this call isn't fenced the same way `advance_durable_and_notify` fences a -/// stale fsync (see `truncation_fsync_fence_test.rs`), it silently -/// resurrects the clamp remove_range just applied. -/// -/// Scenario: -/// 1. Old leader (term=1) replicates entries 1..=10. `append_entries()` -/// inserts them into memory immediately, then blocks inside -/// `persist_entries()` on a gate — the write hasn't reached the storage -/// engine yet. -/// 2. New leader (term=2): index=2 conflicts. `filter_out_conflicts_and_append` -/// runs — `remove_range(2..=MAX)` (in-memory, synchronous, not routed -/// through the IO thread) executes and clamps immediately; the task then -/// blocks on the IO thread for its own queued `ReplaceRange`, which can't -/// run yet because the IO thread is still stuck on step 1's gate. -/// 3. Release the gate. The stale `Persist` for entries 1..=10 completes and -/// calls `persisted_index.fetch_max(10, ..)` — after the truncation's -/// clamp already ran, using entries that no longer exist. The queued -/// `ReplaceRange` runs next but does not re-clamp (its clamp already -/// fired once, in step 2, at truncation time). -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_persisted_index_does_not_adopt_a_stale_persist_after_truncation() { - let (storage, persist_gate) = MockStorageEngine::not_durable_gated_persist( - "persisted_index_does_not_adopt_a_stale_persist_after_truncation".into(), - ); - let (raft_log, receiver) = BufferedRaftLog::::new( - 1, - PersistenceConfig { - flush_policy: FlushPolicy::Batch { - idle_flush_interval_ms: 60_000, - }, - shutdown_timeout_ms: 5000, - }, - Arc::new(storage), - ); - let raft_log = raft_log.start(receiver, None); - std::thread::sleep(Duration::from_millis(10)); - - let entries: Vec = (1..=10).map(|i| entry(i, 1)).collect(); - let append_task = { - let raft_log = raft_log.clone(); - tokio::spawn(async move { raft_log.append_entries(entries).await }) - }; - // Let append_task reach the gate inside persist_entries(). - tokio::time::sleep(Duration::from_millis(50)).await; - - let truncate_task = { - let raft_log = raft_log.clone(); - tokio::spawn(async move { - raft_log.filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]).await - }) - }; - // Let truncate_task run remove_range()'s synchronous clamp and reach - // its own await point (queued behind the still-gated Persist). - tokio::time::sleep(Duration::from_millis(50)).await; - assert_eq!( - raft_log.last_entry_id(), - 2, - "remove_range()'s in-memory truncation must be visible immediately, \ - without waiting for the gated Persist or the queued ReplaceRange" - ); - - // Release the stale Persist — it completes and calls - // persisted_index.fetch_max(10, ..) using now-discarded entries. - persist_gate.send(()).expect("IO thread should still be waiting on the gate"); - append_task.await.unwrap().unwrap(); - truncate_task.await.unwrap().unwrap(); - tokio::time::sleep(Duration::from_millis(50)).await; - - assert!( - raft_log.persisted_index.load(Ordering::Acquire) <= raft_log.last_entry_id(), - "persisted_index ({}) must never exceed last_entry_id ({}) — the stale \ - Persist for entries 1..=10 must not be adopted after truncation shrank \ - the log to [1, 2]", - raft_log.persisted_index.load(Ordering::Acquire), - raft_log.last_entry_id() - ); -} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs deleted file mode 100644 index 2f495a53..00000000 --- a/d-engine-core/src/storage/buffered_raft_log_test/process_crash_safety_test.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! Process-crash safety of entries counted toward quorum. -//! -//! `calculate_majority_matched_index` counts a leader's own write via -//! `last_entry_id()` — the in-memory SkipMap — as soon as `append_entries()` -//! returns. That's fine for power-loss safety (fsync is deliberately async, -//! see quorum_durability_test.rs) as long as the entry has at least reached the -//! storage engine (OS-managed page cache / WAL), which survives an ordinary -//! process crash even without fsync. -//! -//! These tests pin down whether `append_entries()` actually waits for the -//! storage engine (`LogStore::persist_entries`) before returning. Today it does -//! not — persistence happens later, asynchronously, on the IO thread — so an -//! entry can be quorum-eligible while a process crash between `append_entries()` -//! returning and the IO thread's next wakeup would lose it. - -use std::sync::Arc; -use std::time::Duration; - -use d_engine_proto::common::Entry; - -use crate::{ - BufferedRaftLog, FlushPolicy, LogStore, MockStorageEngine, MockTypeConfig, PersistenceConfig, - RaftLog, StorageEngine, -}; - -/// `append_entries()` must not return before the entry reaches the storage -/// engine — otherwise a quorum-eligible write exists only in memory and is -/// lost on an ordinary process crash (not just power loss). -/// -/// Gates `LogStore::persist_entries()` so it never completes during the test. -/// Today, `append_entries()` only inserts into the in-memory SkipMap and -/// notifies the IO thread — it does not call `persist_entries()` itself — so -/// it returns immediately regardless of the gate, and the storage engine never -/// sees the entry. After the fix, `append_entries()` must call -/// `persist_entries()` synchronously before returning, so with the gate closed -/// it must still be pending. -/// -/// Needs `flavor = "multi_thread"`: the gate blocks on a synchronous -/// `std::sync::mpsc::Receiver::recv()` inside `persist_entries()`, which now -/// runs directly on whichever task calls `append_entries()`. On the default -/// single-threaded runtime that would freeze the only executor thread — -/// including this test's own `sleep()` below — for the gate's entire -/// lifetime, an unrelated deadlock, not the behavior under test. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_append_entries_waits_for_storage_engine_before_returning() { - let (storage, persist_gate) = - MockStorageEngine::not_durable_gated_persist("append_waits_for_storage_engine".into()); - let (raft_log, receiver) = BufferedRaftLog::::new( - 1, - PersistenceConfig { - flush_policy: FlushPolicy::Batch { - idle_flush_interval_ms: 60_000, - }, - shutdown_timeout_ms: 5000, - }, - Arc::new(storage.clone()), - ); - let raft_log = raft_log.start(receiver, None); - std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - - let entry = Entry { - index: 1, - term: 1, - payload: None, - }; - - let append_task = tokio::spawn({ - let raft_log = raft_log.clone(); - async move { raft_log.append_entries(vec![entry]).await } - }); - - // Long enough that, if append_entries() were waiting on persist_entries(), - // it would still be pending; short enough to keep the suite fast. - tokio::time::sleep(Duration::from_millis(100)).await; - - // FIXED: append_entries() now routes the write through the IO thread - // (IOTask::Persist + oneshot) and does not return until it completes — - // with the gate closed, it must still be pending. - assert!( - !append_task.is_finished(), - "append_entries() must not return before persist_entries() completes" - ); - - // Ground truth: query the storage engine directly, not raft_log's own - // SkipMap-backed accessor (which would show the entry regardless). - assert!( - storage.log_store().entry(1).await.unwrap().is_none(), - "entry must not be visible in the storage engine while persist_entries() is gated" - ); - - persist_gate.send(()).expect("IO thread should still be waiting on the gate"); - append_task.await.unwrap().unwrap(); - - // append_entries() only returns after persist_entries() completes now, so - // the entry must already be visible — no polling needed. - assert!( - storage.log_store().entry(1).await.unwrap().is_some(), - "entry must be in the storage engine once append_entries() returns" - ); -} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs index 6e5fe913..d5affa17 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs @@ -61,8 +61,8 @@ async fn test_durable_index_does_not_adopt_a_stale_fsync_after_truncation() { let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - // Old leader (term=1) replicates entries 1..=10. append_entries() persists - // them synchronously; write_notify then wakes the IO thread, which + // Old leader (term=1) replicates entries 1..=10. append_entries() inserts + // them into memory and notifies the IO thread, which persists the range and // dispatches a physical fsync for "up to index=10" — that fsync is now // running in the background, blocked on flush_gate. let entries: Vec = (1..=10).map(|i| entry(i, 1)).collect(); diff --git a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs index b8a32bdd..60a7b8aa 100644 --- a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs +++ b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs @@ -640,9 +640,9 @@ impl MockStorageEngine { /// their always-succeeds default (`configure_durable`) — this gate is only /// about the write-to-storage-engine step, not fsync. /// - /// Use this to make the ordering between `append_entries()` returning and the - /// entry actually reaching the storage engine deterministic (no sleep/race) — - /// see `process_crash_safety_test.rs`. + /// Use this to freeze the IO thread mid-persist so a concurrent truncation + /// can be driven deterministically — see + /// `durable_index_truncation_clamp_test.rs`. pub fn not_durable_gated_persist(id: String) -> (Self, std::sync::mpsc::Sender<()>) { let (tx, rx) = std::sync::mpsc::channel::<()>(); let rx = Mutex::new(Some(rx)); diff --git a/d-engine-server/src/node/builder.rs b/d-engine-server/src/node/builder.rs index 7d66360c..8c1c2ac2 100644 --- a/d-engine-server/src/node/builder.rs +++ b/d-engine-server/src/node/builder.rs @@ -367,9 +367,31 @@ where GrpcTransport::new_with_channels(node_id, peer_failure_tx, peer_success_tx) }); - let snapshot_policy = self.snapshot_policy.take().unwrap_or(LogSizePolicy::new( - node_config.raft.snapshot.max_log_entries_before_snapshot, - )); + let max_log_entries = node_config.raft.snapshot.max_log_entries_before_snapshot; + + // Startup memory-budget line — visible on stdout regardless of log setup. + { + const EST_LOG_ENTRY_BYTES: u64 = 512; // small/medium KV write + proto + SkipMap node overhead + let peak_mb = max_log_entries.saturating_mul(EST_LOG_ENTRY_BYTES) / (1024 * 1024); + tracing::info!( + node_id, + max_log_entries, + est_peak_ram_mb = peak_mb, + "Raft log memory: peak ~{peak_mb} MB in memory between snapshots \ + ({max_log_entries} entries × ~512 B/entry est.), purged after each snapshot" + ); + if peak_mb > 100 { + tracing::warn!( + node_id, + est_peak_ram_mb = peak_mb, + "in-memory Raft log budget > 100 MB — lower \ + raft.snapshot.max_log_entries_before_snapshot if RAM-constrained" + ); + } + } + + let snapshot_policy = + self.snapshot_policy.take().unwrap_or(LogSizePolicy::new(max_log_entries)); let shutdown_signal = self.shutdown_signal.clone(); From 81afa8bd7b9f2842ff8f6f3b90a591e95fd7c4e4 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:14:05 +0800 Subject: [PATCH 06/11] fix #446: keep withheld-ACK queue alive across role transitions --- d-engine-core/src/raft.rs | 20 +- .../src/raft_role/follower_state_test.rs | 209 +--------------- .../src/raft_role/learner_state_test.rs | 139 ----------- d-engine-core/src/raft_role/mod.rs | 32 +++ .../src/raft_role/pending_ack_test.rs | 223 ++++++++++++++++++ d-engine-core/src/raft_role/role_state.rs | 199 +++++++++++----- 6 files changed, 420 insertions(+), 402 deletions(-) create mode 100644 d-engine-core/src/raft_role/pending_ack_test.rs diff --git a/d-engine-core/src/raft.rs b/d-engine-core/src/raft.rs index 853c81eb..b16172f4 100644 --- a/d-engine-core/src/raft.rs +++ b/d-engine-core/src/raft.rs @@ -475,7 +475,10 @@ where let _ = self.role.drain_read_buffer(); debug!("BecomeFollower"); - self.role = self.role.become_follower()?; + let mut new_role = self.role.become_follower()?; + let withheld_acks = self.role.take_pending_acks(); + new_role.restore_pending_acks(withheld_acks); + self.role = new_role; // Reset vote when stepping down (new term, no vote yet) self.role.state_mut().commit_vote_reset(&self.ctx)?; @@ -494,7 +497,10 @@ where let _ = self.role.drain_read_buffer(); debug!("BecomeCandidate"); - self.role = self.role.become_candidate()?; + let mut new_role = self.role.become_candidate()?; + let withheld_acks = self.role.take_pending_acks(); + new_role.restore_pending_acks(withheld_acks); + self.role = new_role; // No leader during candidate state let current_term = self.role.current_term(); @@ -505,7 +511,10 @@ where } InternalEvent::BecomeLeader => { debug!("BecomeLeader"); - self.role = self.role.become_leader()?; + let mut new_role = self.role.become_leader()?; + let withheld_acks = self.role.take_pending_acks(); + new_role.restore_pending_acks(withheld_acks); + self.role = new_role; // Mark vote as committed (candidate → leader transition) let current_term = self.role.current_term(); @@ -551,7 +560,10 @@ where let _ = self.role.drain_read_buffer(); debug!("BecomeLearner"); - self.role = self.role.become_learner()?; + let mut new_role = self.role.become_learner()?; + let withheld_acks = self.role.take_pending_acks(); + new_role.restore_pending_acks(withheld_acks); + self.role = new_role; // Learner has no leader initially let current_term = self.role.current_term(); diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index 2516c7fd..e084da8f 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -2892,13 +2892,15 @@ async fn test_follower_rejects_strong_consistency_reads() { } // ============================================================================ -// MemFirst ACK Tests +// Withheld-ACK tests (RPO=0, #446) +// +// End-to-end coverage of the AppendEntries workflow deciding to withhold or send. +// The queue mechanics (release / reject / carry across a role transition) are +// unit-tested in `pending_ack_test.rs`. // ============================================================================ -/// Follower withholds the AppendEntries ACK until its own durable_index catches up. -/// -/// RPO=0 (#446): an ACK asserts durability, so it must not go out before the -/// claimed index is fsynced. LogFlushed releases the withheld response. +/// The workflow withholds a success ACK while durable_index is behind the claimed +/// index, and releases it once `handle_log_flushed` reports that index durable. #[tokio::test] async fn test_follower_withholds_ack_until_durable() { let (_graceful_tx, graceful_rx) = watch::channel(()); @@ -2961,90 +2963,6 @@ async fn test_follower_withholds_ack_until_durable() { assert!(response.is_success()); } -/// #446: if `FollowerState` is dropped (role transition — e.g. a higher-term -/// AppendEntries or becoming a candidate) while it still holds a withheld ACK, the -/// pending sender must be dropped with it — the caller waiting on `resp_rx` must see -/// the channel close, not hang forever and not receive a stale success. -/// -/// This is the actual mechanism #446's design relies on for role-transition safety -/// (see the design doc / ADR-042 discussion): a real role transition replaces -/// `self.role` wholesale, which drops the old `FollowerState` — including -/// `pending_append_acks` and every sender inside it. This test drops the struct -/// directly rather than driving a full role-transition workflow, because that's -/// exactly what a role transition does to it; nothing here relies on any other part -/// of the transition machinery. -#[tokio::test] -async fn test_dropping_follower_state_releases_pending_ack_senders_as_closed() { - let (_graceful_tx, graceful_rx) = watch::channel(()); - let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); - - let leader_term = 2u64; - let appended_index = 5u64; - - let mut replication_handler = MockReplicationCore::new(); - replication_handler.expect_handle_append_entries().returning(move |_, _, _| { - Ok(AppendResponseWithUpdates { - response: AppendEntriesResponse::success( - 1, - leader_term, - Some(LogId { - term: leader_term, - index: appended_index, - }), - ), - commit_index_update: None, - }) - }); - context.handlers.replication_handler = replication_handler; - context.membership = Arc::new(MockMembership::new()); - - let mut state = - FollowerState::::new(1, context.node_config.clone(), None, None); - state.shared_state_mut().update_current_term(leader_term); - - let append_request = AppendEntriesRequest { - term: leader_term, - leader_id: 2, - prev_log_index: 0, - prev_log_term: 0, - entries: vec![], - leader_commit_index: 0, - }; - let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); - let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - - assert!( - state - .handle_inbound_event(inbound_event, &context, internal_event_tx) - .await - .is_ok() - ); - - // Confirm the ACK is genuinely withheld (durable_index hasn't caught up) before - // dropping — otherwise this test wouldn't be exercising the pending-ack path at all. - assert!( - resp_rx.try_recv().is_err(), - "precondition: the ACK must still be withheld before the role transition" - ); - - // Simulates a real role transition: `self.role = self.role.become_xxx()?` drops - // the old FollowerState (and everything it owns) the same way this explicit - // drop does. - drop(state); - - // The withheld ACK's sender is gone — resp_rx must observe the channel closing, - // not hang forever and not receive a stale success response. - let result = tokio::time::timeout(std::time::Duration::from_secs(1), resp_rx.recv()) - .await - .expect("recv() must resolve promptly once the sender is dropped, not hang"); - assert!( - result.is_err(), - "dropping FollowerState must close the pending ACK's channel, not deliver a \ - stale response" - ); -} - /// #446: two independent AppendEntries requests (e.g. a leader retry) that both claim /// the same threshold index must both eventually receive a response — the second one /// landing on `pending_append_acks` must not silently overwrite the first. @@ -3136,119 +3054,6 @@ async fn test_multiple_requests_at_same_threshold_all_receive_response() { assert!(response2.is_success()); } -/// #446: `LogFlushed` must release exactly the pending ACKs whose threshold is `<=` -/// durable_index — not `<` (off-by-one), and not all-or-nothing. -#[tokio::test] -async fn test_log_flushed_releases_only_thresholds_at_or_below_durable() { - let (_graceful_tx, graceful_rx) = watch::channel(()); - let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); - - let leader_term = 2u64; - let call_count = Arc::new(std::sync::atomic::AtomicU64::new(0)); - let call_count_clone = call_count.clone(); - - let mut replication_handler = MockReplicationCore::new(); - replication_handler - .expect_handle_append_entries() - .times(3) - .returning(move |_, _, _| { - let claimed = match call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst) { - 0 => 10, - 1 => 12, - _ => 15, - }; - Ok(AppendResponseWithUpdates { - response: AppendEntriesResponse::success( - 1, - leader_term, - Some(LogId { - term: leader_term, - index: claimed, - }), - ), - commit_index_update: None, - }) - }); - context.handlers.replication_handler = replication_handler; - context.membership = Arc::new(MockMembership::new()); - - let mut state = - FollowerState::::new(1, context.node_config.clone(), None, None); - state.shared_state_mut().update_current_term(leader_term); - - let base_request = AppendEntriesRequest { - term: leader_term, - leader_id: 2, - prev_log_index: 0, - prev_log_term: 0, - entries: vec![], - leader_commit_index: 0, - }; - let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - - let (tx10, mut rx10) = MaybeCloneOneshot::new(); - state - .handle_inbound_event( - InboundEvent::AppendEntries(base_request.clone(), vec![tx10]), - &context, - internal_event_tx.clone(), - ) - .await - .unwrap(); - let (tx12, mut rx12) = MaybeCloneOneshot::new(); - state - .handle_inbound_event( - InboundEvent::AppendEntries(base_request.clone(), vec![tx12]), - &context, - internal_event_tx.clone(), - ) - .await - .unwrap(); - let (tx15, mut rx15) = MaybeCloneOneshot::new(); - state - .handle_inbound_event( - InboundEvent::AppendEntries(base_request, vec![tx15]), - &context, - internal_event_tx, - ) - .await - .unwrap(); - - assert!(rx10.try_recv().is_err()); - assert!(rx12.try_recv().is_err()); - assert!(rx15.try_recv().is_err()); - - let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); - - // durable_index advances to 12 — releases 10 and 12 (boundary is <=, not <), 15 stays. - state.handle_log_flushed(12, &context, &flush_tx).await; - assert!( - rx10.try_recv() - .expect("threshold 10 <= durable 12, must be released") - .unwrap() - .is_success() - ); - assert!( - rx12.try_recv() - .expect("threshold 12 <= durable 12 (boundary case), must be released") - .unwrap() - .is_success() - ); - assert!( - rx15.try_recv().is_err(), - "threshold 15 > durable 12, must still be withheld" - ); - - // durable_index advances to 15 — releases the rest. - state.handle_log_flushed(15, &context, &flush_tx).await; - assert!( - rx15.try_recv() - .expect("threshold 15 <= durable 15, must now be released") - .unwrap() - .is_success() - ); -} - /// Follower sends ACK immediately for heartbeat (no entries). #[tokio::test] async fn test_follower_acks_immediately_for_heartbeat() { diff --git a/d-engine-core/src/raft_role/learner_state_test.rs b/d-engine-core/src/raft_role/learner_state_test.rs index ce68c796..07d32edb 100644 --- a/d-engine-core/src/raft_role/learner_state_test.rs +++ b/d-engine-core/src/raft_role/learner_state_test.rs @@ -1736,145 +1736,6 @@ async fn test_apply_completed_respects_snapshot_disabled_config() { ); } -// ============================================================================ -// MemFirst ACK Tests -// ============================================================================ - -/// Learner withholds the AppendEntries ACK until its own durable_index catches up. -/// -/// RPO=0 (#446): an ACK asserts durability, so it must not go out before the -/// claimed index is fsynced. LogFlushed releases the withheld response. -#[tokio::test] -async fn test_learner_withholds_ack_until_durable() { - let (_graceful_tx, graceful_rx) = watch::channel(()); - let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); - - let leader_term = 2u64; - let appended_index = 5u64; - - let mut replication_handler = crate::MockReplicationCore::new(); - replication_handler.expect_handle_append_entries().returning(move |_, _, _| { - Ok(crate::AppendResponseWithUpdates { - response: d_engine_proto::server::replication::AppendEntriesResponse::success( - 1, - leader_term, - Some(LogId { - term: leader_term, - index: appended_index, - }), - ), - commit_index_update: None, - }) - }); - context.handlers.replication_handler = replication_handler; - context.membership = Arc::new(MockMembership::new()); - - let mut state = LearnerState::::new(1, context.node_config.clone()); - state.update_current_term(leader_term); - - let append_request = d_engine_proto::server::replication::AppendEntriesRequest { - term: leader_term, - leader_id: 2, - prev_log_index: 0, - prev_log_term: 0, - entries: vec![], - leader_commit_index: 0, - }; - let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); - let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - - assert!( - state - .handle_inbound_event(inbound_event, &context, internal_event_tx) - .await - .is_ok() - ); - - // RPO=0: the ACK is withheld while durable_index < claimed index (5). - assert!( - resp_rx.try_recv().is_err(), - "ACK must be withheld until durable_index catches up" - ); - - // Simulate LogFlushed: durable_index advances to 5, releasing the withheld ACK. - let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); - state.handle_log_flushed(appended_index, &context, &flush_tx).await; - - let response = resp_rx.try_recv().expect("ACK must be released once durable").unwrap(); - assert!(response.is_success()); -} - -/// #446: if `LearnerState` is dropped (role transition — e.g. promotion to voter, or a -/// higher-term AppendEntries) while it still holds a withheld ACK, the pending sender -/// must be dropped with it — the caller waiting on `resp_rx` must see the channel -/// close, not hang forever and not receive a stale success. See the equivalent -/// Follower test for the full rationale — same mechanism, same reasoning. -#[tokio::test] -async fn test_dropping_learner_state_releases_pending_ack_senders_as_closed() { - let (_graceful_tx, graceful_rx) = watch::channel(()); - let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); - - let leader_term = 2u64; - let appended_index = 5u64; - - let mut replication_handler = crate::MockReplicationCore::new(); - replication_handler.expect_handle_append_entries().returning(move |_, _, _| { - Ok(crate::AppendResponseWithUpdates { - response: d_engine_proto::server::replication::AppendEntriesResponse::success( - 1, - leader_term, - Some(LogId { - term: leader_term, - index: appended_index, - }), - ), - commit_index_update: None, - }) - }); - context.handlers.replication_handler = replication_handler; - context.membership = Arc::new(MockMembership::new()); - - let mut state = LearnerState::::new(1, context.node_config.clone()); - state.update_current_term(leader_term); - - let append_request = d_engine_proto::server::replication::AppendEntriesRequest { - term: leader_term, - leader_id: 2, - prev_log_index: 0, - prev_log_term: 0, - entries: vec![], - leader_commit_index: 0, - }; - let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); - let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - - assert!( - state - .handle_inbound_event(inbound_event, &context, internal_event_tx) - .await - .is_ok() - ); - - assert!( - resp_rx.try_recv().is_err(), - "precondition: the ACK must still be withheld before the role transition" - ); - - // Simulates a real role transition dropping the old LearnerState. - drop(state); - - let result = tokio::time::timeout(std::time::Duration::from_secs(1), resp_rx.recv()) - .await - .expect("recv() must resolve promptly once the sender is dropped, not hang"); - assert!( - result.is_err(), - "dropping LearnerState must close the pending ACK's channel, not deliver a \ - stale response" - ); -} - /// Spawns a fake Worker that answers exactly one `InstallSnapshot` command with `result`, /// then exits. Returns a `StateMachineCommandSender` wired to it — stands in for the real /// `StateMachineWorker`, which isn't running in these role-layer unit tests. diff --git a/d-engine-core/src/raft_role/mod.rs b/d-engine-core/src/raft_role/mod.rs index b7737611..3a570c87 100644 --- a/d-engine-core/src/raft_role/mod.rs +++ b/d-engine-core/src/raft_role/mod.rs @@ -16,6 +16,8 @@ mod follower_state_test; #[cfg(test)] mod learner_state_test; #[cfg(test)] +mod pending_ack_test; +#[cfg(test)] mod role_state_test; use std::collections::HashMap; @@ -374,6 +376,36 @@ impl RaftRole { pub(crate) fn become_learner(&self) -> Result> { self.state().become_learner() } + /// Move the withheld-ACK queue out of the current role before a transition. + /// Only Follower and Learner keep one; every other role yields an empty map. + /// + /// A withheld ACK describes this node's durable log, not its role. Dropping it + /// on a `Learner -> Follower` promotion would strand the leader waiting on a + /// response that never arrives (#446). + pub(crate) fn take_pending_acks( + &mut self + ) -> std::collections::BTreeMap { + self.state_mut() + .pending_append_acks_mut() + .map(std::mem::take) + .unwrap_or_default() + } + + /// Install a carried withheld-ACK queue into the role a transition produced. + /// Follower and Learner adopt it; any other role cannot hold it, so its + /// entries are failed with a conflict response. + pub(crate) fn restore_pending_acks( + &mut self, + acks: std::collections::BTreeMap, + ) { + let node_id = self.state().node_id(); + let current_term = self.state().current_term(); + match self.state_mut().pending_append_acks_mut() { + Some(queue) => *queue = acks, + None => role_state::reject_pending_acks(acks, node_id, current_term), + } + } + pub fn current_term(&self) -> u64 { self.state().current_term() } diff --git a/d-engine-core/src/raft_role/pending_ack_test.rs b/d-engine-core/src/raft_role/pending_ack_test.rs new file mode 100644 index 00000000..5a445502 --- /dev/null +++ b/d-engine-core/src/raft_role/pending_ack_test.rs @@ -0,0 +1,223 @@ +//! Tests for #446's withheld-AppendEntries-ACK primitives: +//! +//! - `RaftRoleState::resolve_pending_acks` — the release / reject decision; +//! - `RaftRole::{take,restore}_pending_acks` — carrying the queue across a role +//! transition instead of dropping it. +//! +//! These drive bare role structs (no `RaftContext`, no mocks, no fs), so they +//! stay cheap to run and cheap to change. + +use std::sync::Arc; + +use d_engine_proto::common::LogId; +use d_engine_proto::server::replication::AppendEntriesResponse; +use d_engine_proto::server::replication::SuccessResult; +use d_engine_proto::server::replication::append_entries_response; +use tonic::Status; + +use super::RaftRole; +use super::candidate_state::CandidateState; +use super::follower_state::FollowerState; +use super::learner_state::LearnerState; +use crate::MaybeCloneOneshot; +use crate::MaybeCloneOneshotReceiver; +use crate::RaftNodeConfig; +use crate::RaftOneshot; +use crate::raft_role::role_state::PendingAck; +use crate::raft_role::role_state::RaftRoleState; +use crate::test_utils::mock::MockTypeConfig; + +type Rx = MaybeCloneOneshotReceiver>; + +fn config() -> Arc { + Arc::new( + RaftNodeConfig::new() + .expect("RaftNodeConfig::new") + .validate() + .expect("RaftNodeConfig::validate"), + ) +} + +fn follower(term: u64) -> RaftRole { + let mut s = FollowerState::new(1, config(), None, None); + s.shared_state_mut().update_current_term(term); + RaftRole::Follower(Box::new(s)) +} + +fn learner(term: u64) -> RaftRole { + let mut s = LearnerState::new(1, config()); + s.shared_state_mut().update_current_term(term); + RaftRole::Learner(Box::new(s)) +} + +fn candidate() -> RaftRole { + RaftRole::Candidate(Box::new(CandidateState::new(1, config()))) +} + +/// Put a withheld success ACK for `index` straight into `role`'s queue, bypassing +/// the AppendEntries workflow. Returns the receiver a caller would be blocked on. +fn withhold( + role: &mut RaftRole, + index: u64, + claimed_term: u64, + term_when_withheld: u64, +) -> Rx { + let (tx, rx) = MaybeCloneOneshot::new(); + role.state_mut() + .pending_append_acks_mut() + .expect("role keeps a pending-ack queue") + .insert( + index, + PendingAck { + claimed_term, + term_when_withheld, + senders: vec![tx], + }, + ); + rx +} + +fn queue_len(role: &mut RaftRole) -> usize { + role.state_mut().pending_append_acks_mut().map_or(0, |q| q.len()) +} + +// -- resolve_pending_acks ----------------------------------------------------- + +/// A withheld ACK is released as a success once its claimed index is durable and +/// the node is still on the term it withheld under. +#[test] +fn test_resolve_releases_success_when_durable() { + let mut role = follower(5); + let mut rx = withhold(&mut role, 8, 5, 5); + + role.state_mut().resolve_pending_acks(8); + + let resp = rx.try_recv().expect("released").unwrap(); + assert!(matches!( + resp.result, + Some(append_entries_response::Result::Success(SuccessResult { + last_match: Some(LogId { index: 8, term: 5 }), + })), + )); + assert_eq!(queue_len(&mut role), 0); +} + +/// A withheld ACK stays queued while its claimed index is still beyond durable. +#[test] +fn test_resolve_keeps_waiting_until_durable() { + let mut role = follower(5); + let mut rx = withhold(&mut role, 8, 5, 5); + + role.state_mut().resolve_pending_acks(7); + + assert!(rx.try_recv().is_err()); + assert_eq!(queue_len(&mut role), 1); +} + +/// A withheld ACK is rejected with a conflict if the node moved to a newer term +/// since withholding: a higher-term leader may have overwritten the log at that +/// index, so the durability claim can no longer be trusted (#446). +#[test] +fn test_resolve_rejects_stale_term_ack() { + let mut role = follower(6); // node is now on term 6 + let mut rx = withhold(&mut role, 8, 5, 5); // ACK was withheld under term 5 + + role.state_mut().resolve_pending_acks(8); + + let resp = rx.try_recv().expect("resolved").unwrap(); + assert!( + !resp.is_success(), + "stale-term ACK must resolve to a conflict" + ); + assert_eq!(queue_len(&mut role), 0); +} + +/// Every sender queued on one index is answered — a leader retry or a heartbeat +/// can leave more than one waiter on the same index. +#[test] +fn test_resolve_answers_every_sender_on_an_index() { + let mut role = follower(5); + let (tx1, mut rx1) = MaybeCloneOneshot::new(); + let (tx2, mut rx2) = MaybeCloneOneshot::new(); + role.state_mut().pending_append_acks_mut().unwrap().insert( + 8, + PendingAck { + claimed_term: 5, + term_when_withheld: 5, + senders: vec![tx1, tx2], + }, + ); + + role.state_mut().resolve_pending_acks(8); + + assert!(rx1.try_recv().unwrap().unwrap().is_success()); + assert!(rx2.try_recv().unwrap().unwrap().is_success()); +} + +/// `resolve_pending_acks` on a role that keeps no queue (Candidate) is a no-op. +#[test] +fn test_resolve_is_noop_without_a_queue() { + let mut role = candidate(); + role.state_mut().resolve_pending_acks(10); // must not panic +} + +// -- take / restore across a role transition --------------------------------- + +/// A withheld ACK survives a Learner -> Follower promotion: `take_pending_acks` +/// moves the queue out of the old role and `restore_pending_acks` installs it in +/// the new one. Dropping it here would strand the leader on a response that never +/// arrives — the bug #446 fixed. +#[test] +fn test_pending_ack_survives_learner_promotion() { + let mut old = learner(5); + let mut rx = withhold(&mut old, 8, 5, 5); + + let carried = old.take_pending_acks(); + assert_eq!(carried.len(), 1); + assert_eq!( + queue_len(&mut old), + 0, + "take must move the queue, not copy it" + ); + + let mut new = follower(5); + new.restore_pending_acks(carried); + + new.state_mut().resolve_pending_acks(8); + assert!(rx.try_recv().expect("released after promotion").unwrap().is_success()); +} + +/// The symmetric Follower -> Learner demotion also carries the queue. +#[test] +fn test_pending_ack_survives_follower_demotion() { + let mut old = follower(5); + let mut rx = withhold(&mut old, 8, 5, 5); + + let carried = old.take_pending_acks(); + let mut new = learner(5); + new.restore_pending_acks(carried); + + new.state_mut().resolve_pending_acks(8); + assert!(rx.try_recv().expect("released after demotion").unwrap().is_success()); +} + +/// Restoring the queue into a role that cannot hold one (Candidate) fails every +/// withheld ACK with a conflict, so the leader retries rather than timing out. +#[test] +fn test_restore_into_candidate_fails_pending_acks() { + let mut old = follower(5); + let mut rx = withhold(&mut old, 8, 5, 5); + let carried = old.take_pending_acks(); + + let mut candidate = candidate(); + candidate.restore_pending_acks(carried); + + let resp = rx.try_recv().expect("failed, not dropped").unwrap(); + assert!(!resp.is_success()); +} + +/// `take_pending_acks` on a role with no queue yields an empty map, never panics. +#[test] +fn test_take_from_queueless_role_is_empty() { + assert!(candidate().take_pending_acks().is_empty()); +} diff --git a/d-engine-core/src/raft_role/role_state.rs b/d-engine-core/src/raft_role/role_state.rs index f5948150..df9e86ea 100644 --- a/d-engine-core/src/raft_role/role_state.rs +++ b/d-engine-core/src/raft_role/role_state.rs @@ -59,17 +59,66 @@ pub(crate) enum PeerReplicationState { Snapshot, } -/// An AppendEntries response withheld because this node's own `durable_index` hasn't -/// caught up to what it would claim yet (RPO=0, #446). Keyed by the claimed index in -/// `pending_append_acks` (BTreeMap) — `senders` accumulates via -/// `entry().or_insert_with()` if more than one request lands on the same threshold -/// (retry, or a heartbeat landing on the same tail). +/// A success `AppendEntriesResponse` that has been computed but not yet sent, +/// because this node's own `durable_index` had not reached the index the response +/// claims. Held until fsync catches up, so an ACK never asserts durability the +/// node cannot yet guarantee (RPO=0, #446). +/// +/// Keyed by the claimed index. `senders` accumulates when more than one request +/// claims the same index (a leader retry, or a heartbeat landing on the tail). +/// +/// The response body is not stored: it is rebuilt on release, after re-checking +/// the term it was withheld under. A response frozen under a term the node has +/// since left must never be sent. pub(crate) struct PendingAck { - pub(super) response: AppendEntriesResponse, - pub(super) senders: + pub(crate) claimed_term: u64, + pub(crate) term_when_withheld: u64, + pub(crate) senders: Vec>>, } +/// Send the terminal response for one withheld ACK — a rebuilt success if +/// `confirm`, a conflict otherwise — to every accumulated sender. +fn resolve_pending_ack( + node_id: u32, + index: u64, + ack: PendingAck, + confirm: bool, + current_term: u64, +) { + let response = if confirm { + AppendEntriesResponse::success( + node_id, + current_term, + Some(LogId { + index, + term: ack.claimed_term, + }), + ) + } else { + AppendEntriesResponse::conflict(node_id, current_term, None, None) + }; + for sender in ack.senders { + if let Err(e) = sender.send(Ok(response)) { + error!("withheld AppendEntries ACK (index {index}): send failed: {e:?}"); + } + } +} + +/// Fail every withheld ACK with a conflict response. Used when the queue passes to +/// a role that cannot hold it (Candidate or Leader): the node no longer recognises +/// the leader those ACKs were owed to, so that leader's replication worker should +/// retry now rather than wait out an RPC timeout. +pub(crate) fn reject_pending_acks( + acks: BTreeMap, + node_id: u32, + current_term: u64, +) { + for (index, ack) in acks { + resolve_pending_ack(node_id, index, ack, false, current_term); + } +} + #[async_trait] pub(crate) trait RaftRoleState: Send + Sync + 'static { type T: TypeConfig; @@ -411,32 +460,62 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { Ok(()) } - /// Handle LogFlushed(durable) event: entries up to `durable` are now crash-safe. - /// Leader: recalculates commit_index (uses durable_index in quorum calculation). - /// Default: no-op for Candidate/Follower/Learner (ACK already sent on memory write). - async fn handle_log_flushed( + /// Release withheld AppendEntries ACKs now that the log is durable through + /// `durable`. For each withheld ACK: + /// + /// - withheld under a term this node has since left → fail it with a conflict. + /// A higher-term leader may have overwritten the log at that index; within a + /// single term a follower's entries are never replaced, so the term check + /// alone is a sufficient content guard and no log lookup is needed. + /// - claimed index now `<= durable` → rebuild and send the success. + /// - otherwise → keep waiting. + /// + /// No-op for Candidate/Leader (no queue). Runs on every fsync completion, so it + /// stays limited to integer comparisons — no log lookup. (#446) + fn resolve_pending_acks( &mut self, durable: u64, - _ctx: &RaftContext, - _internal_event_tx: &mpsc::UnboundedSender, ) { - // RPO=0 (#446): release any withheld AppendEntries responses whose claimed - // index is now durable. No-op for Candidate/Leader (pending_append_acks_mut - // returns None for them; Leader overrides this whole method anyway). + let node_id = self.node_id(); + let current_term = self.current_term(); let Some(pending) = self.pending_append_acks_mut() else { return; }; - let later = pending.split_off(&(durable.saturating_add(1))); - let ready = std::mem::replace(pending, later); - for (_, ack) in ready { - for sender in ack.senders { - if let Err(e) = sender.send(Ok(ack.response)) { - error!("Failed to send released AppendEntriesResponse: {:?}", e); + if pending.is_empty() { + return; + } + let resolved: Vec<(u64, bool)> = pending + .iter() + .filter_map(|(&index, ack)| { + if ack.term_when_withheld != current_term { + Some((index, false)) // stale term -> conflict + } else if index <= durable { + Some((index, true)) // durable -> success + } else { + None // keep waiting } + }) + .collect(); + for (index, confirm) in resolved { + if let Some(ack) = pending.remove(&index) { + resolve_pending_ack(node_id, index, ack, confirm, current_term); } } } + /// A batch of log entries reached `durable` on disk (fsync complete). + /// + /// Follower/Learner: release any withheld AppendEntries ACKs this now covers. + /// Leader: overridden to recalculate `commit_index`. Candidate: no-op. + async fn handle_log_flushed( + &mut self, + durable: u64, + _ctx: &RaftContext, + _internal_event_tx: &mpsc::UnboundedSender, + ) { + self.resolve_pending_acks(durable); + } + /// Handle AppendEntries result from a per-follower ReplicationWorker. /// Leader: updates match_index, recalculates commit, drains pending_client_writes. /// Default: no-op for all non-leader roles (stale results arriving after step-down). @@ -588,47 +667,52 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { } debug!("AppendEntriesResponse: {:?}", response); - // RPO=0 (#446): a success response must not go out until this node's - // own durable_index has caught up to what it claims — an ACK asserts - // durability, and it must not lie about that. Conflict/higher-term - // responses don't claim any durable state, so they're never withheld. - let claimed_index = match &response.result { + // RPO=0 (#446): a success response asserts the claimed entry is + // fsync-durable on this node. If this node's own `durable_index` + // has not reached that index, withhold the response until it does + // (released by `resolve_pending_acks`). Conflict and higher-term + // responses assert nothing about durability and are sent at once. + let claim = match &response.result { Some(append_entries_response::Result::Success(SuccessResult { last_match: Some(log_id), - })) => Some(log_id.index), + })) => Some((log_id.index, log_id.term)), _ => None, }; - let withhold = - claimed_index.is_some_and(|idx| ctx.storage.raft_log.durable_index() < idx); - - if withhold { - let idx = claimed_index.unwrap(); - match self.pending_append_acks_mut() { - Some(pending) => { - pending - .entry(idx) - .or_insert_with(|| PendingAck { - response, - senders: Vec::new(), - }) - .senders - .extend(senders); - } - None => { - // Should never happen — only Follower/Learner reach this - // branch. Fail loud rather than silently dropping an ACK - // the leader is waiting on. - error!("no pending_append_acks slot on a role that should have one"); - for sender in senders { - let _ = sender.send(Ok(response)); + match claim { + Some((index, claimed_term)) if ctx.storage.raft_log.durable_index() < index => { + let term_when_withheld = self.current_term(); + match self.pending_append_acks_mut() { + Some(pending) => { + pending + .entry(index) + .or_insert_with(|| PendingAck { + claimed_term, + term_when_withheld, + senders: Vec::new(), + }) + .senders + .extend(senders); + } + None => { + // Only Follower and Learner produce a success + // response here, and both carry the queue. Reaching + // this arm means a role invariant broke — send the + // ACK now rather than strand the leader. + error!( + "withheld a success ACK on a role with no pending-ACK queue" + ); + for sender in senders { + let _ = sender.send(Ok(response)); + } } } } - } else { - for sender in senders { - if let Err(e) = sender.send(Ok(response)) { - error!("Failed to send: {:?}", e); + _ => { + for sender in senders { + if let Err(e) = sender.send(Ok(response)) { + error!("failed to send AppendEntries response: {e:?}"); + } } } } @@ -1004,8 +1088,9 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { ) { } - /// Follower/Learner's withheld-response queue. `None` for Candidate/Leader — - /// same pattern as `pending_purge_upto_mut` below. + /// The withheld-ACK queue, for the roles that keep one (Follower, Learner). + /// `None` for Candidate and Leader. Carried across a Follower<->Learner + /// transition by `RaftRole::take_pending_acks` / `restore_pending_acks` (#446). fn pending_append_acks_mut(&mut self) -> Option<&mut BTreeMap> { None } From 368121f37bda67f6f7e11c5b1c1284fbb33215f0 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:17:07 +0800 Subject: [PATCH 07/11] fix #446: merge IO wakeup sources, frontier-based persist, replace_range returns new tail - delete write_notify; append_entries sends IOTask::Persist on the one command channel - run_batch_turn -> run_flush_turn; IOTask::Wake -> Persist - persist_pending_range returns the highest index actually written, not the scan bound - persisted_index: highest-persisted watermark, truncation-corrected, scans use +1 - LogStore::replace_range -> Result; callers use it instead of guessing from input - FlushPolicy doc: fixed-period idle timer, not inactivity-reset --- d-engine-core/src/config/raft.rs | 14 +- .../src/storage/buffered_raft_log.rs | 265 +++++++++++------- .../pipeline_overlap_test.rs | 4 +- d-engine-core/src/storage/storage_engine.rs | 7 +- .../test_utils/mock/mock_storage_engine.rs | 2 +- .../adaptors/file/file_storage_engine.rs | 4 +- .../rocksdb/rocksdb_storage_engine.rs | 4 +- 7 files changed, 184 insertions(+), 116 deletions(-) diff --git a/d-engine-core/src/config/raft.rs b/d-engine-core/src/config/raft.rs index 60a7456a..b9b2e051 100644 --- a/d-engine-core/src/config/raft.rs +++ b/d-engine-core/src/config/raft.rs @@ -822,16 +822,12 @@ fn default_stale_learner_threshold() -> Duration { Duration::from_secs(300) } -/// Controls when in-memory logs should be flushed to disk. +/// Interval (ms) between periodic fsyncs on the IO thread. Must be > 0. /// -/// Flush is triggered by whichever comes first: -/// - An explicit `flush()` call (immediate, no wait). -/// - `append_entries` calls `write_notify.notify_one()` for an immediate persist+fsync. -/// - The idle safety-net timer fires after `idle_flush_interval_ms` of inactivity. -/// -/// `idle_flush_interval_ms` must be greater than zero. It only fires when no -/// writes have arrived for that duration; normal-path latency is determined by -/// the fsync execution time (drain-then-fsync architecture). +/// Writes fsync on their own path (`flush()`, `append_entries` → +/// `IOTask::Persist`). This timer only re-fsyncs `(durable_index, +/// memory_max_index]` when the log is idle, so `durable_index` still advances +/// if a fsync-completion notification is lost. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub enum FlushPolicy { Batch { idle_flush_interval_ms: u64 }, diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index f0d4351f..07fb9a41 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -12,14 +12,20 @@ //! //! ## Write path //! -//! `append_entries` inserts entries into in-memory SkipMap, calls `write_notify.notify_one()`. -//! Multiple concurrent writers coalesce into a single IO thread wakeup. +//! `append_entries` inserts entries into the in-memory SkipMap, then sends a +//! fire-and-forget `IOTask::Persist` on the same channel that carries `Flush` / +//! `ReplaceRange` / `Purge` / `Reset` / `Shutdown`. One ordered wakeup source +//! means the IO thread never races two independent signals for the same pending +//! work (#446). A write burst's redundant `Persist`s are cheap no-ops: each finds +//! `persisted_index` already at `memory_max_index` and persists nothing. //! //! ## IO thread (notify-then-spawn-fsync) //! -//! On wakeup from `write_notify` (`run_batch_turn`): -//! 1. **Read** — scan SkipMap range `(durable_index, memory_max_index]` -//! 2. **Persist** — write range to OS page cache via `persist_entries` (no fsync) +//! On `IOTask::Persist` (`run_storage_tasks`) / `Flush` (`run_flush_turn`): +//! 1. **Read** — scan the SkipMap range `(persisted_index, memory_max_index]`, +//! where `persisted_index` is the IO thread's own page-cache frontier (not +//! the round-tripping `durable_index`) +//! 2. **Persist** — write the range to OS page cache via `persist_entries` (no fsync) //! 3. **Dispatch fsync** — `FsyncCoordinator::submit()` hands the fdatasync to a //! `spawn_blocking` task and returns immediately //! 4. **Loop** — back to `select!`; the prior fsync runs concurrently in the pool @@ -39,11 +45,11 @@ //! //! ## Fsync triggers //! -//! 1. **Notify-driven** (normal): `write_notify` → `run_batch_turn` -//! 2. **Explicit** (flush API): `flush()` → `IOTask::Flush(tx)` → `run_batch_turn` with a reply sender +//! 1. **Write-driven** (normal): `append_entries` → `IOTask::Persist` → `run_storage_tasks` +//! 2. **Explicit** (flush API): `flush()` → `IOTask::Flush(tx)` → `run_flush_turn` with a reply sender //! 3. **Idle timer** (safety net): `idle_flush_interval_ms` elapsed → -//! `persist_pending_range` + `FsyncCoordinator::submit()` directly (not via `run_batch_turn`) -//! 4. **Shutdown**: `IOTask::Shutdown` → `run_batch_turn`, then `close()` waits +//! `persist_pending_range` + `FsyncCoordinator::submit()` directly (not via `run_flush_turn`) +//! 4. **Shutdown**: `IOTask::Shutdown` → `run_flush_turn`, then `close()` waits //! (bounded by `shutdown_timeout_ms`) for the IO thread's runtime to drain any in-flight fsync //! //! ## Durability contract @@ -80,7 +86,6 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::time::Duration; -use tokio::sync::Notify; use tokio::sync::mpsc; use tokio::sync::oneshot; use tracing::debug; @@ -226,6 +231,11 @@ pub enum IOTask { /// result (Ok or Err) back to the caller via the oneshot channel. /// Replaces the former `FlushNow` + `WaitDurable` two-message dance. Flush(oneshot::Sender>), + /// Persist the IO thread to persist newly-appended entries. Fire-and-forget, + /// carries no data — `append_entries` sends one after each insert. Redundant + /// `Persist`s from writes coalesced while the IO thread was busy are drained + /// and discarded in a single `run_flush_turn`. + Persist, /// Shutdown the IO thread Shutdown, } @@ -299,10 +309,9 @@ where term_segments: TermSegments, // --- Flush coordination --- - /// Coalesced write notification. `append_entries` calls `notify_one()` after - /// inserting into the SkipMap. Multiple concurrent writers coalesce into a - /// single IO thread wakeup, eliminating per-write kernel cond_signal overhead. - pub(crate) write_notify: Arc, + /// The IO thread's sole inbound channel: `Persist` / `Flush` / `ReplaceRange` / `Purge` / `Reset` / `Shutdown`. One ordered + /// source so the IO thread never races two independent wakeup signals for + /// the same pending work (#446). pub(crate) command_sender: mpsc::UnboundedSender, // --- P0: LogFlushed event notification --- @@ -478,13 +487,16 @@ where self.insert_to_memory(&entries); - // Signal the IO thread to persist. Fire-and-forget: the entry is in the - // in-memory log and quorum-visible now; the IO thread scans the SkipMap - // and persists + fsyncs off this task. Concurrent notify_one() calls - // coalesce into one wakeup. RPO=0 is enforced downstream (#446), not - // here: commit quorum counts only `durable_index()`, and followers - // withhold AppendEntries ACKs until their own `durable_index` catches up. - self.write_notify.notify_one(); + // Persist the IO thread. Fire-and-forget: the entries are already in the + // in-memory log and quorum-visible; the IO thread scans the SkipMap and + // persists + fsyncs off this task. `Persist` rides the same channel as + // `Flush` / `ReplaceRange` / `Shutdown` so the IO thread has one ordered + // wakeup source (#446). RPO=0 is enforced downstream — commit quorum + // counts only `durable_index()`, and followers withhold AppendEntries + // ACKs until their own `durable_index` catches up. + self.command_sender + .send(IOTask::Persist) + .map_err(|e| NetworkError::SingalSendFailed(format!("Persist send failed: {e:?}")))?; Ok(()) } @@ -885,7 +897,6 @@ where last_purged_term: AtomicU64::new(last_purged_term_val), durable_index: AtomicU64::new(disk_len), next_id: AtomicU64::new(disk_len + 1), - write_notify: Arc::new(Notify::new()), command_sender: command_sender.clone(), term_first_index, term_last_index, @@ -949,20 +960,22 @@ where arc_self } - /// Notify-driven IO loop. + /// Write-driven IO loop. /// - /// Waits on `write_notify.notified()` for new entries in the SkipMap. - /// Multiple `notify_one()` calls while the IO thread is busy (persisting or - /// fsyncing) coalesce into a single wakeup, reducing kernel cond_signal overhead - /// from one-per-write to one-per-burst. + /// A single `select!` consumes one ordered channel of `IOTask`s and a + /// backstop timer. `Persist` / `ReplaceRange` / `Purge` / `Reset` go to + /// `run_storage_tasks`; `Flush` / `Shutdown` to `run_flush_turn` (which also + /// sweeps the queue for one combined fsync). A write burst's extra `Persist`s + /// are no-ops — `persisted_index` is already past `memory_max_index`. /// - /// On each wakeup: - /// 1. Read entries in `(durable_index, memory_max_index]` from SkipMap. + /// On each `Persist` / `Flush`: + /// 1. Read entries in `(persisted_index, memory_max_index]` from the SkipMap. /// 2. persist_entries to OS page cache (no fsync). - /// 3. Drain any pending control commands from the mpsc channel. - /// 4. fsync once — advance durable_index, wake WaitDurable callers. + /// 3. Drain any pending control commands from the channel. + /// 4. Hand the range to `FsyncCoordinator` for one concurrent fsync. /// - /// Safety-net timer fires after `idle_flush_interval_ms` of inactivity. + /// Backstop timer ticks every `idle_flush_interval_ms` (fixed period, not + /// reset by writes; skipped under load — see the safety-net arm). async fn batch_processor( this: std::sync::Weak, mut receiver: mpsc::UnboundedReceiver, @@ -979,20 +992,20 @@ where // Highest index in OS page cache, awaiting fsync. Reset to 0 after each fsync. let mut pending_max: u64 = 0; + // Highest log index the IO thread has written to the OS page cache — + // B-local, the "submitted" watermark sitting between `memory_max_index` + // and `durable_index` + let mut persisted_index: u64 = this.durable_index.load(Ordering::Acquire); + loop { tokio::select! { - _ = this.write_notify.notified() => { - if Self::run_batch_turn(&this, &mut receiver, &mut pending_max, Vec::new(), false).await { - break; - } - } cmd = receiver.recv() => { let Some(cmd) = cmd else { break }; let should_break = match cmd { - IOTask::Shutdown => Self::run_batch_turn(&this, &mut receiver, &mut pending_max, Vec::new(), true).await, - IOTask::Flush(reply) => Self::run_batch_turn(&this, &mut receiver, &mut pending_max, vec![reply], false).await, + IOTask::Shutdown => Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, &mut pending_max, Vec::new(), true).await, + IOTask::Flush(reply) => Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, &mut pending_max, vec![reply], false).await, cmd => { - if Self::run_storage_tasks(cmd, &this, &mut pending_max).await { + if Self::run_storage_tasks(cmd, &this, &mut persisted_index, &mut pending_max).await { break; } continue; @@ -1001,68 +1014,80 @@ where if should_break { break; } } _ = safety_timer.tick() => { - let start = this.durable_index.load(Ordering::Acquire) + 1; + let from = this.durable_index.load(Ordering::Acquire) + 1; let end = this.memory_max_index.load(Ordering::Acquire); - let _ = Self::persist_pending_range(&this, start, end, &mut pending_max, "safety-net").await; - - if pending_max > 0 { - this.fsync_coordinator.submit(&this, pending_max, vec![]); - pending_max = 0; + if let Ok(Some(persisted_to)) = + Self::persist_pending_range(&this, from, end, "safety-net").await + { + persisted_index = persisted_index.max(persisted_to); + this.fsync_coordinator.submit(&this, persisted_to, vec![]); } } } } } - /// Writes entries in `(from, to]` that haven't reached page cache yet (no - /// fsync). Iterates the SkipMap for the range — entries removed by a - /// concurrent truncation simply aren't returned, so a stale from/to pair is - /// self-correcting and never writes wrong data. Advances `pending_max` on - /// success; propagates the error as-is on failure. + /// Persists entries in `(from, to]` that aren't in the OS page cache yet + /// (no fsync). `from` / `to` are only scan bounds — the caller passes + /// `persisted_index + 1` and a `memory_max_index` snapshot. + /// + /// The SkipMap range scan returns only entries that still exist. If a + /// concurrent term-conflict truncation removed the top of `(from, to]` + /// between the caller's `memory_max_index` read and this scan, those + /// indices are simply absent and never written. + /// + /// Returns `Some(highest index actually written)` — which may be *below* + /// `to` in that truncation-race case — or `None` when the scan found + /// nothing. Callers advance their own `persisted_index` / fsync target + /// from this value, so neither ever points past a real entry. async fn persist_pending_range( this: &Arc, from: u64, to: u64, - pending_max: &mut u64, ctx: &str, - ) -> Result<()> { + ) -> Result> { if this.is_poisoned() { return Err(Error::Fatal("raft log storage is poisoned".to_string())); } if from > to { - return Ok(()); + return Ok(None); } let entries = this.get_entries_range(from..=to)?; - if entries.is_empty() { - return Ok(()); - } - this.log_store - .persist_entries(entries) - .await - .inspect(|_| { - *pending_max = (*pending_max).max(to); - }) - .inspect_err(|e| { - error!("{ctx} persist_entries failed: {e:?}"); - this.mark_poisoned_and_notify(format!("{ctx}: persist_entries failed: {e:?}")); - }) + let Some(highest_written) = entries.last().map(|e| e.index) else { + return Ok(None); + }; + this.log_store.persist_entries(entries).await.inspect_err(|e| { + error!( + persist_path = ctx, + from, to, "persist_entries failed: {e:?}" + ); + this.mark_poisoned_and_notify(format!("{ctx}: persist_entries failed: {e:?}")); + })?; + Ok(Some(highest_written)) } - async fn run_batch_turn( + async fn run_flush_turn( this: &Arc, receiver: &mut mpsc::UnboundedReceiver, + persisted_index: &mut u64, pending_max: &mut u64, mut replies: Vec>>, mut seen_shutdown: bool, ) -> bool { - let start = this.durable_index.load(Ordering::Acquire) + 1; let end = this.memory_max_index.load(Ordering::Acquire); let mut persist_failed = false; - if let Err(e) = Self::persist_pending_range(this, start, end, pending_max, "batch").await { - for reply in replies.drain(..) { - let _ = reply.send(Err(Error::Fatal(format!("persist_entries failed: {e:?}")))); + match Self::persist_pending_range(this, *persisted_index + 1, end, "flush-turn").await { + Ok(Some(persisted_to)) => { + *persisted_index = persisted_to; + *pending_max = (*pending_max).max(persisted_to); + } + Ok(None) => {} + Err(e) => { + for reply in replies.drain(..) { + let _ = reply.send(Err(Error::Fatal(format!("persist_entries failed: {e:?}")))); + } + persist_failed = true; } - persist_failed = true; } // `seen_shutdown` is not a gate here — regardless of whether the @@ -1076,8 +1101,12 @@ where seen_shutdown = true; } IOTask::Flush(reply) => replies.push(reply), + // A coalesced write. Drop it — the unconditional tail re-scan + // below persists everything up to `memory_max_index`, and this + // turn ends with a single fsync submit. + IOTask::Persist => {} cmd => { - if Self::run_storage_tasks(cmd, this, pending_max).await { + if Self::run_storage_tasks(cmd, this, persisted_index, pending_max).await { for reply in replies { let _ = reply .send(Err(Error::Fatal("fatal IO error, batch aborted".into()))); @@ -1088,13 +1117,19 @@ where } } - // A Flush caller needs everything up to now durable; a command drained - // above may have advanced memory_max_index past the leading persist. - if !replies.is_empty() && !persist_failed { - let start = *pending_max + 1; + // Tail re-scan: a discarded `Persist` or a drained command may have moved + // memory_max_index past the leading persist. Unconditional since #446 — + // for a coalesced write the discarded `Persist` was its only signal; a + // Flush reply / final Shutdown drain also need everything up to now. + if !persist_failed { let end = this.memory_max_index.load(Ordering::Acquire); - let _ = - Self::persist_pending_range(this, start, end, pending_max, "batch catch-up").await; + if let Ok(Some(persisted_to)) = + Self::persist_pending_range(this, *persisted_index + 1, end, "flush-turn catch-up") + .await + { + *persisted_index = persisted_to; + *pending_max = (*pending_max).max(persisted_to); + } } this.fsync_coordinator.submit(this, *pending_max, replies); @@ -1115,14 +1150,16 @@ where seen_shutdown } - /// Runs one storage-mutating IOTask (ReplaceRange/Purge/Reset) - /// against log_store. Flush/Shutdown are intercepted by the caller - /// (`batch_processor`) before this is called — unreachable here. + /// Handles one non-Flush/Shutdown `IOTask`: `Persist` (persist the SkipMap tail + /// then submit it for fsync) or a storage mutation (ReplaceRange/Purge/Reset). + /// Flush/Shutdown never reach here: `batch_processor`'s `select!` routes + /// them to `run_flush_turn`, whose drain loop also handles them inline. /// /// Returns `true` if `batch_processor` must exit immediately (fatal IO error). async fn run_storage_tasks( cmd: IOTask, this: &Arc, + persisted_index: &mut u64, pending_max: &mut u64, ) -> bool { match cmd { @@ -1132,6 +1169,22 @@ where IOTask::Shutdown => { unreachable!("Shutdown is always filtered out before reaching run_storage_tasks") } + IOTask::Persist => { + // A write landed. Persist the new tail and submit it. No queue + // drain: a burst's redundant `Persist`s find `persisted_index` + // already at `memory_max_index` and no-op here (#446). + let end = this.memory_max_index.load(Ordering::Acquire); + match Self::persist_pending_range(this, *persisted_index + 1, end, "persist").await + { + Ok(Some(persisted_to)) => { + *persisted_index = persisted_to; + this.fsync_coordinator.submit(this, persisted_to, Vec::new()); + } + Ok(None) => {} + Err(_) => return true, // poisoned — same exit convention as the mutations below + } + false + } IOTask::ReplaceRange { truncate_from, new_entries, @@ -1145,20 +1198,26 @@ where return true; } - let max_idx = new_entries.last().map(|e| e.index).unwrap_or(0); - let result = this.log_store.replace_range(truncate_from, new_entries).await; - if let Err(ref e) = result { - error!("IOTask::ReplaceRange failed (fatal): {e:?}"); - this.mark_poisoned_and_notify(format!("ReplaceRange failed: {e:?}")); - let _ = done.send(result); - return true; // signal batch_processor to exit — disk state is corrupted - } - - if max_idx > 0 { - *pending_max = (*pending_max).max(max_idx); - this.fsync_coordinator.submit(this, max_idx, vec![]); + let new_tail = match this.log_store.replace_range(truncate_from, new_entries).await + { + Ok(new_tail) => new_tail, + Err(e) => { + error!("IOTask::ReplaceRange failed (fatal): {e:?}"); + this.mark_poisoned_and_notify(format!("ReplaceRange failed: {e:?}")); + let _ = done.send(Err(e)); + return true; + } + }; + // A real new tail was written — fsync it (fdatasync is whole-WAL, + // so it also covers any leading persist from this same turn; + // clear pending_max so the turn's final submit doesn't re-send a + // now-stale index). + if new_tail >= truncate_from { + this.fsync_coordinator.submit(this, new_tail, vec![]); } - let _ = done.send(result); + *pending_max = 0; + *persisted_index = new_tail; // == truncate_from - 1 when the new tail is empty + let _ = done.send(Ok(())); false } IOTask::Purge { cutoff, done } => { @@ -1175,6 +1234,9 @@ where let _ = done.send(()); return true; // signal batch_processor to exit — disk state is corrupted } + // Purged entries were already durable (only applied entries are + // purged) — let the persist frontier skip past them (#446). + *persisted_index = (*persisted_index).max(cutoff.index); let _ = done.send(()); false } @@ -1188,9 +1250,16 @@ where error!("IOTask::Reset failed (fatal): {e:?}"); this.mark_poisoned_and_notify(format!("Reset failed: {e:?}")); let _ = done.send(result); - return true; // signal batch_processor to exit — disk state is corrupted + + // signal batch_processor to exit — disk state is corrupted + return true; } - *pending_max = 0; // disk wiped — pending page-cache watermark must be zeroed + + // disk wiped — pending page-cache watermark must be zeroed + *pending_max = 0; + + // log wiped — next entry to persist is index 1 + *persisted_index = 0; let _ = done.send(result); false } diff --git a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs index d9e9876a..a53e1380 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs @@ -437,9 +437,9 @@ async fn test_io_task_replace_range_delegates_to_replace_range_not_truncate() { // replace_range() must be called exactly once for one conflict resolution let rr_counter = replace_range_count.clone(); - log_store.expect_replace_range().returning(move |_from, _entries| { + log_store.expect_replace_range().returning(move |_from, new_entries| { rr_counter.fetch_add(1, Ordering::Relaxed); - Ok(()) + Ok(new_entries.last().map(|e| e.index).unwrap_or(0)) }); // truncate() must NOT be called — IOTask::ReplaceRange owns the full operation diff --git a/d-engine-core/src/storage/storage_engine.rs b/d-engine-core/src/storage/storage_engine.rs index 29a61830..5ece2a1f 100644 --- a/d-engine-core/src/storage/storage_engine.rs +++ b/d-engine-core/src/storage/storage_engine.rs @@ -79,16 +79,19 @@ pub trait LogStore: Send + Sync + 'static { /// /// Default implementation calls `truncate` then `persist_entries` sequentially /// (non-atomic). Override with a single WriteBatch for true crash atomicity. + /// Returns the highest log index on disk after the operation: the last of + /// `new_entries`, or `from_index - 1` when `new_entries` is empty. async fn replace_range( &self, from_index: u64, new_entries: Vec, - ) -> Result<(), Error> { + ) -> Result { + let new_last = new_entries.last().map(|e| e.index).unwrap_or(from_index.saturating_sub(1)); self.truncate(from_index).await?; if !new_entries.is_empty() { self.persist_entries(new_entries).await?; } - Ok(()) + Ok(new_last) } /// Whether a single `persist_entries` call is crash-safe without an explicit `flush()`. diff --git a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs index 60a7b8aa..83a3d4f9 100644 --- a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs +++ b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs @@ -263,7 +263,7 @@ impl MockStorageEngine { } else { data.insert(last_key, new_last_index.to_be_bytes().to_vec()); } - Ok(()) + Ok(new_last_index) }); } diff --git a/d-engine-server/src/storage/adaptors/file/file_storage_engine.rs b/d-engine-server/src/storage/adaptors/file/file_storage_engine.rs index 4aec4eba..db63c514 100644 --- a/d-engine-server/src/storage/adaptors/file/file_storage_engine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_storage_engine.rs @@ -350,7 +350,7 @@ impl LogStore for FileLogStore { &self, from_index: u64, new_entries: Vec, - ) -> Result<(), Error> { + ) -> Result { let encoded: Vec> = new_entries.iter().map(|e| e.encode_to_vec()).collect(); let new_last = { @@ -378,7 +378,7 @@ impl LogStore for FileLogStore { }; self.last_index.store(new_last, Ordering::SeqCst); - Ok(()) + Ok(new_last) } fn is_write_durable(&self) -> bool { diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_storage_engine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_storage_engine.rs index 624c901e..82f4ff5b 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_storage_engine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_storage_engine.rs @@ -383,7 +383,7 @@ impl LogStore for RocksDBLogStore { &self, from_index: u64, new_entries: Vec, - ) -> Result<(), Error> { + ) -> Result { let cf = self .db .cf_handle(LOG_CF) @@ -404,7 +404,7 @@ impl LogStore for RocksDBLogStore { self.db.write(&batch).map_err(|e| StorageError::DbError(e.to_string()))?; self.last_index.store(new_last_index, Ordering::SeqCst); - Ok(()) + Ok(new_last_index) } fn is_write_durable(&self) -> bool { From 642c778b13825d1108399a4228fa5449c0f1e3d3 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:17:29 +0800 Subject: [PATCH 08/11] test #446: persist-frontier + fsync-coordinator coverage --- .../drain_fsync_test.rs | 298 ++++++++++++++++++ .../durable_index_truncation_clamp_test.rs | 151 ++++++++- .../src/storage/fsync_coordinator_test.rs | 69 ++++ 3 files changed, 517 insertions(+), 1 deletion(-) diff --git a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs index 815a9ab0..6d97601f 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs @@ -23,6 +23,8 @@ use crate::PersistenceConfig; use d_engine_proto::common::Entry; use d_engine_proto::common::LogId; use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use tokio::sync::mpsc; use tokio::time::timeout; @@ -1164,3 +1166,299 @@ async fn test_notify_fatal_channel_closed_still_poisons_and_logs() { not a silent no-op — see notify_fatal()'s error! call" ); } + +/// Efficiency: the IO thread's persist scan must start from its own page-cache +/// frontier, not from `durable_index`. Since #446 `durable_index` only advances +/// after an `FsyncCompleted` round-trips through raft.rs's event loop; under +/// load it lags far behind what the IO thread has already written. If the scan +/// restarted from `durable_index + 1` on every wakeup, each of N appends would +/// re-scan and re-`persist_entries` the whole not-yet-durable window — O(N^2) +/// total work. +/// +/// This test pins `durable_index` at 0 (no `log_flush_tx`, so no +/// `FsyncCompleted` is ever consumed) and appends N entries one at a time. The +/// total number of entries handed to `persist_entries` across all calls must +/// stay ~N, not ~N^2/2. +#[tokio::test] +async fn test_persist_scan_tracks_frontier_not_stuck_durable_index() { + let persisted_total = Arc::new(AtomicU64::new(0)); + let persisted_total_c = persisted_total.clone(); + + let mut log_store = MockLogStore::new(); + log_store.expect_last_index().returning(|| 0); + log_store.expect_persist_entries().returning(move |entries| { + persisted_total_c.fetch_add(entries.len() as u64, Ordering::Relaxed); + Ok(()) + }); + log_store.expect_entry().returning(|_| Ok(None)); + log_store.expect_get_entries().returning(|_| Ok(vec![])); + log_store.expect_purge().returning(|_| Ok(())); + log_store.expect_load_purge_boundary().returning(|| Ok(None)); + log_store.expect_reset().returning(|| Ok(())); + log_store.expect_truncate().returning(|_| Ok(())); + log_store + .expect_replace_range() + .returning(|from, new_entries| { + Ok(new_entries + .last() + .map(|e| e.index) + .unwrap_or(from.saturating_sub(1))) + }); + log_store.expect_is_write_durable().returning(|| false); + log_store.expect_flush().returning(|| Ok(())); + log_store.expect_flush_async().returning(|| Ok(())); + + let mut meta_store = MockMetaStore::new(); + meta_store.expect_save_hard_state().returning(|_| Ok(())); + meta_store.expect_load_hard_state().returning(|| Ok(None)); + meta_store.expect_flush().returning(|| Ok(())); + meta_store.expect_flush_async().returning(|| Ok(())); + + let storage = Arc::new(MockStorageEngine::from(log_store, meta_store)); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + // No log_flush_tx: FsyncCompleted is never consumed, so durable_index + // stays pinned at 0 for the whole test. + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + const N: u64 = 100; + for i in 1..=N { + raft_log + .append_entries(vec![Entry { + index: i, + term: 1, + payload: None, + }]) + .await + .unwrap(); + // flush() forces the IO thread to persist up to memory_max_index right + // now, so the scan boundary is exercised once per append — deterministic, + // no sleeps. + raft_log.flush().await.unwrap(); + } + + assert_eq!( + raft_log.durable_index(), + 0, + "durable_index must stay stuck for this test to be meaningful" + ); + let total = persisted_total.load(Ordering::Relaxed); + assert!( + total < 3 * N, + "persist_entries received {total} entries for {N} appends; a frontier-tracking \ + scan is ~{N}, a durable_index-relative scan would be ~{} (O(N^2))", + N * (N + 1) / 2 + ); +} + +/// Cold start: after a restart, `durable_index` starts at the disk length and +/// the IO thread's persist frontier must start *past* it. The first write's +/// persist scan begins at `durable_index + 1` — an already-durable entry on +/// disk must never be handed back to `persist_entries`. +/// +/// Guards the frontier initialization (`= durable_index`, scans use `+ 1`). +#[tokio::test] +async fn test_cold_start_persist_frontier_starts_past_durable_index() { + let persist_calls: Arc>>> = Arc::new(Mutex::new(Vec::new())); + + let mut log_store = MockLogStore::new(); + log_store.expect_last_index().returning(|| 5); // disk already holds 1..=5 + log_store.expect_get_entries().returning(|range| { + Ok(range + .map(|i| Entry { + index: i, + term: 1, + payload: None, + }) + .collect()) + }); + { + let calls = persist_calls.clone(); + log_store.expect_persist_entries().returning(move |entries| { + calls + .lock() + .unwrap() + .push(entries.iter().map(|e| e.index).collect()); + Ok(()) + }); + } + log_store.expect_entry().returning(|_| Ok(None)); + log_store.expect_purge().returning(|_| Ok(())); + log_store.expect_load_purge_boundary().returning(|| Ok(None)); + log_store.expect_reset().returning(|| Ok(())); + log_store.expect_truncate().returning(|_| Ok(())); + log_store.expect_replace_range().returning(|from, e| { + Ok(e.last().map(|x| x.index).unwrap_or(from.saturating_sub(1))) + }); + log_store.expect_is_write_durable().returning(|| false); + log_store.expect_flush().returning(|| Ok(())); + log_store.expect_flush_async().returning(|| Ok(())); + + let mut meta_store = MockMetaStore::new(); + meta_store.expect_save_hard_state().returning(|_| Ok(())); + meta_store.expect_load_hard_state().returning(|| Ok(None)); + meta_store.expect_flush().returning(|| Ok(())); + meta_store.expect_flush_async().returning(|| Ok(())); + + let storage = Arc::new(MockStorageEngine::from(log_store, meta_store)); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + assert_eq!( + raft_log.durable_index(), + 5, + "restart: disk length 5 is treated as durable" + ); + + // First write after restart. Its persist scan must start at 6. + raft_log + .append_entries(vec![Entry { + index: 6, + term: 1, + payload: None, + }]) + .await + .unwrap(); + sleep(Duration::from_millis(50)).await; + + let calls = persist_calls.lock().unwrap().clone(); + assert!(!calls.is_empty(), "entry 6 must have been persisted"); + assert!( + calls.iter().flatten().all(|&idx| idx >= 6), + "cold start: the first persist must scan from durable_index+1 (6), never \ + re-scan already-durable entry 5. Got: {calls:?}" + ); +} + +/// The flush turn's unconditional tail re-scan must persist writes that were +/// coalesced into the turn — a write whose `IOTask::Persist` is dropped in the +/// drain loop still becomes durable, because the catch-up re-reads +/// `memory_max_index` and persists everything past the frontier. +/// +/// Deterministic via a per-call persist gate: every `persist_entries` announces +/// its indices and blocks until released. +#[tokio::test(flavor = "multi_thread", worker_threads = 3)] +async fn test_flush_turn_catch_up_persists_writes_coalesced_during_the_turn() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel::>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let release_rx = Mutex::new(release_rx); + + let mut log_store = MockLogStore::new(); + log_store.expect_last_index().returning(|| 0); + log_store.expect_persist_entries().returning(move |entries| { + entered_tx + .send(entries.iter().map(|e| e.index).collect()) + .ok(); + release_rx.lock().unwrap().recv().ok(); + Ok(()) + }); + log_store.expect_entry().returning(|_| Ok(None)); + log_store.expect_get_entries().returning(|_| Ok(vec![])); + log_store.expect_purge().returning(|_| Ok(())); + log_store.expect_load_purge_boundary().returning(|| Ok(None)); + log_store.expect_reset().returning(|| Ok(())); + log_store.expect_truncate().returning(|_| Ok(())); + log_store.expect_replace_range().returning(|from, e| { + Ok(e.last().map(|x| x.index).unwrap_or(from.saturating_sub(1))) + }); + log_store.expect_is_write_durable().returning(|| false); + log_store.expect_flush().returning(|| Ok(())); + log_store.expect_flush_async().returning(|| Ok(())); + + let mut meta_store = MockMetaStore::new(); + meta_store.expect_save_hard_state().returning(|_| Ok(())); + meta_store.expect_load_hard_state().returning(|| Ok(None)); + meta_store.expect_flush().returning(|| Ok(())); + meta_store.expect_flush_async().returning(|| Ok(())); + + let storage = Arc::new(MockStorageEngine::from(log_store, meta_store)); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + std::thread::sleep(Duration::from_millis(10)); + + let e = |i: u64| Entry { + index: i, + term: 1, + payload: None, + }; + + // 1..=2 persisted (frontier → 2). + raft_log.append_entries(vec![e(1), e(2)]).await.unwrap(); + assert_eq!(entered_rx.recv().unwrap(), vec![1, 2]); + release_tx.send(()).unwrap(); + + // 3..=4: their Persist is in progress (blocked in persist_entries). + raft_log.append_entries(vec![e(3), e(4)]).await.unwrap(); + assert_eq!(entered_rx.recv().unwrap(), vec![3, 4]); + + // flush() enqueues IOTask::Flush behind the in-progress Persist(3,4). + let flush_task = { + let rl = raft_log.clone(); + tokio::spawn(async move { rl.flush().await }) + }; + tokio::time::sleep(Duration::from_millis(30)).await; + + // 5..=6 land while Persist(3,4) is still blocked → their Persist queues + // behind Flush. + raft_log.append_entries(vec![e(5), e(6)]).await.unwrap(); + release_tx.send(()).unwrap(); // release Persist(3,4) → frontier → 4 + + // IO thread moves to Flush → run_flush_turn. Leading persist covers 5,6. + assert_eq!(entered_rx.recv().unwrap(), vec![5, 6]); + + // 7..=8 land now — after run_flush_turn read memory_max for its leading + // persist, before its drain loop. Their Persist queues behind Flush and + // will be dropped in the drain loop. + raft_log.append_entries(vec![e(7), e(8)]).await.unwrap(); + release_tx.send(()).unwrap(); // release leading persist(5,6) → frontier → 6 + + // The drain loop drops Persist(5,6) and Persist(7,8); the unconditional + // tail re-scan then persists 7,8. + assert_eq!( + entered_rx.recv().unwrap(), + vec![7, 8], + "flush turn's tail re-scan must persist 7,8 whose Persist was dropped" + ); + release_tx.send(()).unwrap(); + + flush_task.await.unwrap().unwrap(); + while let Ok(ev) = log_flush_rx.try_recv() { + if let crate::InternalEvent::FsyncCompleted { index, term } = ev { + raft_log.try_advance_durable_index(index, term); + } + } + assert_eq!( + raft_log.durable_index(), + 8, + "7,8 (coalesced into the flush turn) must be durable via the catch-up" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs index 2893d55e..e060fed6 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs @@ -6,13 +6,17 @@ //! check, and the `FsyncCoordinator` generation fence bumped by `remove_range`. use std::sync::Arc; +use std::sync::Mutex; use std::time::Duration; use d_engine_proto::common::Entry; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; +use crate::{ + BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, + PersistenceConfig, +}; fn entry( index: u64, @@ -160,3 +164,148 @@ async fn test_stale_persist_after_truncation_does_not_advance_durable_index() { "durable_index must land on the post-truncation tail (2), not the stale 10" ); } + +/// `persist_pending_range` must report the highest index it *actually wrote*, +/// not the upper scan bound it was handed. The two differ during a truncation +/// race: the IO thread latched `memory_max_index` = 10 (an old leader had sent +/// 8, 9, 10), then a term-conflict truncation removed everything above 7 before +/// the SkipMap scan ran. Asking to persist `(4, 10]` then writes only 5, 6, 7. +/// +/// Returning the bound (10) would push the caller's `persisted_index` and the +/// fsync target past entries that never reached disk — a redundant fdatasync +/// plus a spurious `FsyncCompleted{10}` that the term check then has to reject. +/// Reporting 7 keeps every downstream watermark on real data. +#[tokio::test] +async fn test_persist_pending_range_reports_written_max_not_scan_bound() { + let storage = Arc::new(MockStorageEngine::with_id( + "persist_pending_range_reports_written_max".into(), + )); + // No `.start()` — drive `persist_pending_range` directly, no IO thread. + let (raft_log, _receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + let raft_log = Arc::new(raft_log); + + raft_log + .append_entries((1..=7).map(|i| entry(i, 1)).collect()) + .await + .unwrap(); + + // Scan bound is 10 (stale latch); the SkipMap holds only 1..=7. + let written = BufferedRaftLog::persist_pending_range(&raft_log, 5, 10, "test") + .await + .unwrap(); + + assert_eq!( + written, + Some(7), + "must report the highest index actually written (7), not the scan bound (10)" + ); +} + +/// After a term-conflict truncation, the IO thread's persist frontier must land +/// *past* the new tail — not on it. `IOTask::ReplaceRange` already wrote (and +/// fsynced) the new tail via `replace_range`; the next write's persist scan must +/// start at `new_tail + 1`. If the frontier is left *at* `new_tail`, every +/// subsequent write re-scans and re-`persist_entries` that one boundary entry +/// (and re-submits a redundant fsync for it) — the exact waste #446 removes. +/// +/// Guards the "highest-persisted" watermark semantics: `ReplaceRange` sets the +/// watermark to `new_tail`, and scans start at `watermark + 1`. +#[tokio::test] +async fn test_persist_frontier_skips_new_tail_after_truncation() { + // Records the index list of every persist_entries() call. + let persist_calls: Arc>>> = Arc::new(Mutex::new(Vec::new())); + + let mut log_store = MockLogStore::new(); + log_store.expect_last_index().returning(|| 0); + { + let calls = persist_calls.clone(); + log_store.expect_persist_entries().returning(move |entries| { + calls + .lock() + .unwrap() + .push(entries.iter().map(|e| e.index).collect()); + Ok(()) + }); + } + log_store + .expect_replace_range() + .returning(|from, new_entries| { + Ok(new_entries + .last() + .map(|e| e.index) + .unwrap_or(from.saturating_sub(1))) + }); + log_store.expect_truncate().returning(|_| Ok(())); + log_store.expect_entry().returning(|_| Ok(None)); + log_store.expect_get_entries().returning(|_| Ok(vec![])); + log_store.expect_purge().returning(|_| Ok(())); + log_store.expect_load_purge_boundary().returning(|| Ok(None)); + log_store.expect_reset().returning(|| Ok(())); + log_store.expect_is_write_durable().returning(|| false); + log_store.expect_flush().returning(|| Ok(())); + log_store.expect_flush_async().returning(|| Ok(())); + + let mut meta_store = MockMetaStore::new(); + meta_store.expect_save_hard_state().returning(|_| Ok(())); + meta_store.expect_load_hard_state().returning(|| Ok(None)); + meta_store.expect_flush().returning(|| Ok(())); + meta_store.expect_flush_async().returning(|| Ok(())); + + let storage = Arc::new(MockStorageEngine::from(log_store, meta_store)); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + // Old leader (term 1): entries 1..=10, persisted. + raft_log + .append_entries((1..=10).map(|i| entry(i, 1)).collect()) + .await + .unwrap(); + raft_log.flush().await.unwrap(); + + // New leader (term 2): conflict at index 6 → truncate [6..], replace with + // [6, 7] (term 2). `filter_out_conflicts_and_append` awaits the + // `IOTask::ReplaceRange` reply, so the frontier is at new-tail 7 on return. + raft_log + .filter_out_conflicts_and_append(5, 1, vec![entry(6, 2), entry(7, 2)]) + .await + .unwrap(); + + // Only care about persist calls from here on — no flush() in between, so the + // next append's `IOTask::Persist` is the first thing to touch the frontier. + persist_calls.lock().unwrap().clear(); + + // Next write extends the log. Its persist scan must start at 8, not 7. + raft_log + .append_entries((8..=10).map(|i| entry(i, 2)).collect()) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + let calls = persist_calls.lock().unwrap().clone(); + let re_persisted_tail = calls.iter().flatten().any(|&idx| idx <= 7); + assert!( + !re_persisted_tail, + "after ReplaceRange set the frontier at new-tail 7, the next persist must \ + start at 8 — entry 7 (or below) must not be handed to persist_entries again. \ + Got calls: {calls:?}" + ); +} diff --git a/d-engine-core/src/storage/fsync_coordinator_test.rs b/d-engine-core/src/storage/fsync_coordinator_test.rs index 93fdf8c0..bf25e336 100644 --- a/d-engine-core/src/storage/fsync_coordinator_test.rs +++ b/d-engine-core/src/storage/fsync_coordinator_test.rs @@ -609,3 +609,72 @@ fn test_run_until_caught_up_coalesces_queued_submits_into_one_flush() { "both queued replies must resolve to Ok" ); } + +/// A single fsync round can end up covering a range whose *top* index was +/// truncated away after being submitted (a stale pre-truncation persist), +/// folded together with a *lower* index that is still valid. The coordinator +/// collapses the round to one `(max_index, max_term)` pair reported as +/// `FsyncCompleted` — if that pair is the stale top, content-validation on the +/// drain side rejects it and the valid lower index gets **no report at all**, +/// stranding `durable_index`. +/// +/// Deterministic, IO-thread-free repro of the `test_stale_persist_after_ +/// truncation_does_not_advance_durable_index` failure. Post-truncation log is +/// `[1, 2]`; `pending_max` holds the stale `10` (its entry is gone) with the +/// valid `2` folded in. One round must still let `durable_index` reach 2. +/// +/// RED until `FsyncCoordinator` derives the report from the truncation-aware +/// persist frontier instead of the coalescing `pending_max` accumulator. +#[test] +fn test_run_until_caught_up_reports_valid_frontier_when_batch_top_is_stale() { + let (storage, _flush_call_count) = MockStorageEngine::not_durable( + "run_until_caught_up_reports_valid_frontier_when_batch_top_is_stale".into(), + ); + let coord = FsyncCoordinator::new(); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + + // Post-truncation log: entries 1..=2 only. Index 10 is gone. + { + let mut e = raft_log.entries.write(); + for i in 1..=2 { + e.insert( + i, + Entry { + index: i, + term: 1, + payload: None, + }, + ); + } + } + raft_log.set_memory_max_index_for_test(2); + + // A stale `submit(10)` and a valid `submit(2)` folded into one round + // (`fetch_max(10)` then `fetch_max(2)` => 10). + coord.inflight.store(true, Ordering::Release); + coord.pending_max.store(10, Ordering::Release); + + coord.run_until_caught_up(&raft_log); + + while let Ok(InternalEvent::FsyncCompleted { index, term }) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(index, term); + } + + assert_eq!( + raft_log.durable_index.load(Ordering::Acquire), + 2, + "the fsync covered index 2 (valid) and index 10 (truncated); durable_index \ + must still reach 2 — a stale batch top must not swallow the valid frontier" + ); +} From 9f8ac3fa7b131c3334bbe3bab8916c3c5d5507e1 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:34:30 +0800 Subject: [PATCH 09/11] fix #446: order fsync pending mark term-first to unstick durable_index after cross-term truncation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pending_max was a lossy fetch_max u64 — a stale pre-truncation submit (old term, high index) swallowed the valid post-truncation one (new term, low index), so the valid mark never got its FsyncCompleted and durable_index stalled (liveness, not RPO). - pending_max: AtomicU64 -> parking_lot::Mutex, compared (term, index) term-first - FsyncCompleted carries LogId; try_advance_durable_index re-checks entry term before advancing - drop fence_truncation; remove_range just bumps generation - embedded idempotency test: start_with + 3s timeout, since a put ack now waits for fdatasync --- d-engine-core/src/event.rs | 11 +- d-engine-core/src/raft.rs | 6 +- .../src/storage/buffered_raft_log.rs | 147 ++++++++--------- .../concurrent_fsync_test.rs | 11 +- .../content_validated_watermark_test.rs | 16 +- .../drain_fsync_test.rs | 36 ++--- .../durable_index_truncation_clamp_test.rs | 42 ++--- .../src/storage/fsync_coordinator.rs | 63 ++++---- .../src/storage/fsync_coordinator_test.rs | 151 ++++++++++-------- d-engine-core/src/storage/raft_log.rs | 3 +- .../buffered_raft_log_test_helpers.rs | 4 +- .../api/embedded_test/embedded_env_test.rs | 19 ++- .../tests/storage_buffered_raft_log/mod.rs | 4 +- .../sled-cluster/src/sled_storage_engine.rs | 5 +- 14 files changed, 260 insertions(+), 258 deletions(-) diff --git a/d-engine-core/src/event.rs b/d-engine-core/src/event.rs index 9c756914..cc4ec4f6 100644 --- a/d-engine-core/src/event.rs +++ b/d-engine-core/src/event.rs @@ -71,13 +71,10 @@ pub enum InternalEvent { durable_index: u64, }, - /// Raw fsync-completion signal — NOT yet validated. Consumer must call - /// `raft_log().try_advance_durable_index(index, term)`, which re-checks - /// content before actually advancing `durable_index`. - FsyncCompleted { - index: u64, - term: u64, - }, + /// Raw fsync-completion mark — NOT yet validated. Consumer must call + /// `raft_log().try_advance_durable_index(mark)`, which re-checks the entry's + /// term before advancing `durable_index`. + FsyncCompleted(LogId), /// AppendEntries result from a per-follower ReplicationWorker back to the Raft loop. /// Leader processes this in handle_append_result: updates match_index, re-calculates commit, diff --git a/d-engine-core/src/raft.rs b/d-engine-core/src/raft.rs index b16172f4..815b4c39 100644 --- a/d-engine-core/src/raft.rs +++ b/d-engine-core/src/raft.rs @@ -621,10 +621,8 @@ where .handle_log_flushed(durable_index, &self.ctx, &self.internal_event_tx) .await; } - InternalEvent::FsyncCompleted { index, term } => { - if let Some(new_durable) = - self.ctx.raft_log().try_advance_durable_index(index, term) - { + InternalEvent::FsyncCompleted(mark) => { + if let Some(new_durable) = self.ctx.raft_log().try_advance_durable_index(mark) { self.role .handle_log_flushed(new_durable, &self.ctx, &self.internal_event_tx) .await; diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index 07fb9a41..5421bc28 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -33,15 +33,15 @@ //! ## How `durable_index` advances (#446 single-owner) //! //! The blocking fsync task does **not** write `durable_index`. On completion it -//! calls `notify_fsync_completed(index, term)`, sending -//! `InternalEvent::FsyncCompleted { index, term }` to `raft.rs`'s event loop — -//! the sole owner of `durable_index`. That loop calls -//! `try_advance_durable_index(index, term)`, which content-validates the report -//! (`entry_term(index) == Some(term)`) and clamps to `memory_max_index` before -//! `fetch_max`. A stale report — for entries a concurrent truncation already -//! discarded — is rejected. `FsyncCoordinator`'s `generation` fence -//! (`fence_truncation` / `fence_reset`) is the first line of defence: a fsync -//! round whose generation changed mid-flight never sends its completion at all. +//! calls `notify_fsync_completed(mark)`, sending `InternalEvent::FsyncCompleted(LogId)` to `raft.rs`'s +//! event loop — the sole owner of `durable_index`. That loop calls +//! `try_advance_durable_index(mark)`, which rejects the report if +//! `entry_term(mark.index) != Some(mark.term)` and clamps to `memory_max_index` +//! before `fetch_max`. Upstream, `FsyncCoordinator` keeps `pending_max` as a +//! term-first `(term, index)`: a newer term's mark always wins, so a stale +//! pre-truncation submit can never swallow the valid post-truncation one. A +//! `generation` bump (`bump_generation` / `fence_reset`) additionally fences a +//! round whose log was truncated or reset mid-flight. //! //! ## Fsync triggers //! @@ -686,15 +686,13 @@ where // Purged entries are backed by the snapshot; treat cutoff as durable. // Already running on the single owner (called from role_state.rs, same // thread as remove_range) — safe to apply directly, no message hop needed. - if let Some(new_durable) = - self.try_advance_durable_index(cutoff_index.index, cutoff_index.term) + if let Some(new_durable) = self.try_advance_durable_index(cutoff_index) && let Some(ref tx) = self.log_flush_tx { let _ = tx.send(crate::InternalEvent::LogFlushed { durable_index: new_durable, }); } - // Route purge through the IO thread so it never blocks the inbound event loop. // Also writes the purge boundary to META_CF in the RocksDB implementation. let (done_tx, done_rx) = oneshot::channel(); @@ -713,17 +711,16 @@ where fn try_advance_durable_index( &self, - index: u64, - term: u64, + mark: LogId, ) -> Option { let prev = self.durable_index.load(Ordering::Acquire); - if index <= prev { + if mark.index <= prev { return None; } - if self.entry_term(index) != Some(term) { + if self.entry_term(mark.index) != Some(mark.term) { return None; } - let safe = index.min( + let safe = mark.index.min( self.memory_max_index .load(Ordering::Acquire) .max(self.last_purged_index.load(Ordering::Acquire)), @@ -989,9 +986,6 @@ where tokio::time::interval_at(start, Duration::from_millis(idle_flush_interval_ms)); safety_timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - // Highest index in OS page cache, awaiting fsync. Reset to 0 after each fsync. - let mut pending_max: u64 = 0; - // Highest log index the IO thread has written to the OS page cache — // B-local, the "submitted" watermark sitting between `memory_max_index` // and `durable_index` @@ -1002,10 +996,10 @@ where cmd = receiver.recv() => { let Some(cmd) = cmd else { break }; let should_break = match cmd { - IOTask::Shutdown => Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, &mut pending_max, Vec::new(), true).await, - IOTask::Flush(reply) => Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, &mut pending_max, vec![reply], false).await, + IOTask::Shutdown => Self::run_flush_turn(&this, &mut receiver, &mut persisted_index,Vec::new(), true).await, + IOTask::Flush(reply) => Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, vec![reply], false).await, cmd => { - if Self::run_storage_tasks(cmd, &this, &mut persisted_index, &mut pending_max).await { + if Self::run_storage_tasks(cmd, &this, &mut persisted_index).await { break; } continue; @@ -1016,11 +1010,11 @@ where _ = safety_timer.tick() => { let from = this.durable_index.load(Ordering::Acquire) + 1; let end = this.memory_max_index.load(Ordering::Acquire); - if let Ok(Some(persisted_to)) = + if let Ok(Some(mark)) = Self::persist_pending_range(&this, from, end, "safety-net").await { - persisted_index = persisted_index.max(persisted_to); - this.fsync_coordinator.submit(&this, persisted_to, vec![]); + persisted_index = persisted_index.max(mark.index); + this.fsync_coordinator.submit(&this, mark, vec![]); } } } @@ -1036,16 +1030,16 @@ where /// between the caller's `memory_max_index` read and this scan, those /// indices are simply absent and never written. /// - /// Returns `Some(highest index actually written)` — which may be *below* - /// `to` in that truncation-race case — or `None` when the scan found - /// nothing. Callers advance their own `persisted_index` / fsync target - /// from this value, so neither ever points past a real entry. + /// Returns `Some((term, index))` of the last entry written — its index may + /// be *below* `to` in that truncation-race case — or `None` when the scan + /// found nothing. Callers advance `persisted_index` and submit this mark to + /// the fsync coordinator, so neither points past a real entry. async fn persist_pending_range( this: &Arc, from: u64, to: u64, ctx: &str, - ) -> Result> { + ) -> Result> { if this.is_poisoned() { return Err(Error::Fatal("raft log storage is poisoned".to_string())); } @@ -1053,7 +1047,10 @@ where return Ok(None); } let entries = this.get_entries_range(from..=to)?; - let Some(highest_written) = entries.last().map(|e| e.index) else { + let Some(mark) = entries.last().map(|e| LogId { + term: e.term, + index: e.index, + }) else { return Ok(None); }; this.log_store.persist_entries(entries).await.inspect_err(|e| { @@ -1063,23 +1060,28 @@ where ); this.mark_poisoned_and_notify(format!("{ctx}: persist_entries failed: {e:?}")); })?; - Ok(Some(highest_written)) + Ok(Some(mark)) } async fn run_flush_turn( this: &Arc, receiver: &mut mpsc::UnboundedReceiver, persisted_index: &mut u64, - pending_max: &mut u64, mut replies: Vec>>, mut seen_shutdown: bool, ) -> bool { + // Highest (term, index) this turn persisted. Delivered with `replies` in + // one submit at the end. Any drained ReplaceRange/Persist submits its own + // mark directly; term-first ordering in the coordinator means a stale + // leading mark here loses to a newer-term drained one. + let mut mark = LogId::default(); + let end = this.memory_max_index.load(Ordering::Acquire); let mut persist_failed = false; match Self::persist_pending_range(this, *persisted_index + 1, end, "flush-turn").await { - Ok(Some(persisted_to)) => { - *persisted_index = persisted_to; - *pending_max = (*pending_max).max(persisted_to); + Ok(Some(m)) => { + *persisted_index = m.index; + mark = m; } Ok(None) => {} Err(e) => { @@ -1090,23 +1092,16 @@ where } } - // `seen_shutdown` is not a gate here — regardless of whether the - // caller already knows shutdown is happening, any commands still - // queued must be drained and replied to. Whether to drain the queue - // and whether to eventually break the loop are separate concerns - // and must not share one flag. + // `seen_shutdown` is not a gate here — any queued commands must still be + // drained and replied to. while let Ok(cmd) = receiver.try_recv() { match cmd { - IOTask::Shutdown => { - seen_shutdown = true; - } + IOTask::Shutdown => seen_shutdown = true, IOTask::Flush(reply) => replies.push(reply), - // A coalesced write. Drop it — the unconditional tail re-scan - // below persists everything up to `memory_max_index`, and this - // turn ends with a single fsync submit. + // Coalesced write — the unconditional catch-up below covers it. IOTask::Persist => {} cmd => { - if Self::run_storage_tasks(cmd, this, persisted_index, pending_max).await { + if Self::run_storage_tasks(cmd, this, persisted_index).await { for reply in replies { let _ = reply .send(Err(Error::Fatal("fatal IO error, batch aborted".into()))); @@ -1117,24 +1112,21 @@ where } } - // Tail re-scan: a discarded `Persist` or a drained command may have moved - // memory_max_index past the leading persist. Unconditional since #446 — - // for a coalesced write the discarded `Persist` was its only signal; a - // Flush reply / final Shutdown drain also need everything up to now. + // Catch-up: a dropped Persist or a drained command may have moved + // memory_max_index past the leading persist. if !persist_failed { let end = this.memory_max_index.load(Ordering::Acquire); - if let Ok(Some(persisted_to)) = + if let Ok(Some(m)) = Self::persist_pending_range(this, *persisted_index + 1, end, "flush-turn catch-up") .await { - *persisted_index = persisted_to; - *pending_max = (*pending_max).max(persisted_to); + *persisted_index = m.index; + mark = m; } } - this.fsync_coordinator.submit(this, *pending_max, replies); + this.fsync_coordinator.submit(this, mark, replies); - *pending_max = 0; if seen_shutdown { let _ = this.meta_store.flush(); } @@ -1160,7 +1152,6 @@ where cmd: IOTask, this: &Arc, persisted_index: &mut u64, - pending_max: &mut u64, ) -> bool { match cmd { IOTask::Flush(_) => { @@ -1176,9 +1167,9 @@ where let end = this.memory_max_index.load(Ordering::Acquire); match Self::persist_pending_range(this, *persisted_index + 1, end, "persist").await { - Ok(Some(persisted_to)) => { - *persisted_index = persisted_to; - this.fsync_coordinator.submit(this, persisted_to, Vec::new()); + Ok(Some(mark)) => { + *persisted_index = mark.index; + this.fsync_coordinator.submit(this, mark, Vec::new()); } Ok(None) => {} Err(_) => return true, // poisoned — same exit convention as the mutations below @@ -1198,6 +1189,8 @@ where return true; } + // Capture the new tail's term before `new_entries` is moved. + let new_tail_term = new_entries.last().map(|e| e.term).unwrap_or(0); let new_tail = match this.log_store.replace_range(truncate_from, new_entries).await { Ok(new_tail) => new_tail, @@ -1208,15 +1201,19 @@ where return true; } }; - // A real new tail was written — fsync it (fdatasync is whole-WAL, - // so it also covers any leading persist from this same turn; - // clear pending_max so the turn's final submit doesn't re-send a - // now-stale index). + // New content written — submit its (term, index). fdatasync is + // whole-WAL so it also covers any leading persist from this turn. if new_tail >= truncate_from { - this.fsync_coordinator.submit(this, new_tail, vec![]); + this.fsync_coordinator.submit( + this, + LogId { + term: new_tail_term, + index: new_tail, + }, + vec![], + ); } - *pending_max = 0; - *persisted_index = new_tail; // == truncate_from - 1 when the new tail is empty + *persisted_index = new_tail; let _ = done.send(Ok(())); false } @@ -1255,9 +1252,6 @@ where return true; } - // disk wiped — pending page-cache watermark must be zeroed - *pending_max = 0; - // log wiped — next entry to persist is index 1 *persisted_index = 0; let _ = done.send(result); @@ -1351,11 +1345,10 @@ where /// Called by fsync_coordinator (C) — never writes `durable_index` itself. pub(super) fn notify_fsync_completed( &self, - index: u64, - term: u64, + mark: LogId, ) { if let Some(ref tx) = self.log_flush_tx { - let _ = tx.send(crate::InternalEvent::FsyncCompleted { index, term }); + let _ = tx.send(crate::InternalEvent::FsyncCompleted(mark)); } } @@ -1407,9 +1400,7 @@ where self.memory_max_index.store(new_max, Ordering::Release); self.durable_index.fetch_min(new_max, Ordering::AcqRel); - // Clamps pending_max and bumps generation, in that order — see - // fence_truncation()'s doc comment for why the order matters. - self.fsync_coordinator.fence_truncation(new_max); + self.fsync_coordinator.bump_generation(); // `entries` guard drops here (end of scope) — write lock released. } diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs index 495aad69..fad64cec 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs @@ -11,6 +11,7 @@ use crate::{ BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, RaftLog, }; use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; @@ -337,8 +338,14 @@ async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { // Simulates a fsync task completing with index 150, then a second, older // fsync task (dispatched earlier, finishing later) completing with 100. - let result_150 = raft_log.try_advance_durable_index(150, 1); - let result_100 = raft_log.try_advance_durable_index(100, 1); + let result_150 = raft_log.try_advance_durable_index(LogId { + term: 1, + index: 150, + }); + let result_100 = raft_log.try_advance_durable_index(LogId { + term: 1, + index: 100, + }); assert_eq!( result_150, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs index 5e58fe62..2e55c1fb 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs @@ -13,6 +13,7 @@ use std::sync::Arc; use std::time::Duration; use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; @@ -66,7 +67,10 @@ async fn test_stale_durable_report_rejected_when_term_no_longer_matches() { raft_log.filter_out_conflicts_and_append(80, 1, term2_tail).await.unwrap(); // The stale in-flight fsync's report, generated before the truncation. - let result = raft_log.try_advance_durable_index(100, 1); + let result = raft_log.try_advance_durable_index(LogId { + term: 1, + index: 100, + }); assert_eq!( result, None, @@ -89,7 +93,10 @@ async fn test_durable_report_accepted_when_term_still_matches() { let entries: Vec = (1..=100).map(|i| entry(i, 1)).collect(); raft_log.append_entries(entries).await.unwrap(); - let result = raft_log.try_advance_durable_index(100, 1); + let result = raft_log.try_advance_durable_index(LogId { + term: 1, + index: 100, + }); assert_eq!( result, @@ -113,7 +120,10 @@ async fn test_durable_report_then_truncation_is_order_independent() { // Report arrives first, while the log is still all term 1 — legitimately // applied at this point in time. - let result = raft_log.try_advance_durable_index(100, 1); + let result = raft_log.try_advance_durable_index(LogId { + term: 1, + index: 100, + }); assert_eq!(result, Some(100)); assert_eq!(raft_log.durable_index(), 100); diff --git a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs index 6d97601f..f8dc28ae 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs @@ -1196,14 +1196,9 @@ async fn test_persist_scan_tracks_frontier_not_stuck_durable_index() { log_store.expect_load_purge_boundary().returning(|| Ok(None)); log_store.expect_reset().returning(|| Ok(())); log_store.expect_truncate().returning(|_| Ok(())); - log_store - .expect_replace_range() - .returning(|from, new_entries| { - Ok(new_entries - .last() - .map(|e| e.index) - .unwrap_or(from.saturating_sub(1))) - }); + log_store.expect_replace_range().returning(|from, new_entries| { + Ok(new_entries.last().map(|e| e.index).unwrap_or(from.saturating_sub(1))) + }); log_store.expect_is_write_durable().returning(|| false); log_store.expect_flush().returning(|| Ok(())); log_store.expect_flush_async().returning(|| Ok(())); @@ -1284,10 +1279,7 @@ async fn test_cold_start_persist_frontier_starts_past_durable_index() { { let calls = persist_calls.clone(); log_store.expect_persist_entries().returning(move |entries| { - calls - .lock() - .unwrap() - .push(entries.iter().map(|e| e.index).collect()); + calls.lock().unwrap().push(entries.iter().map(|e| e.index).collect()); Ok(()) }); } @@ -1296,9 +1288,9 @@ async fn test_cold_start_persist_frontier_starts_past_durable_index() { log_store.expect_load_purge_boundary().returning(|| Ok(None)); log_store.expect_reset().returning(|| Ok(())); log_store.expect_truncate().returning(|_| Ok(())); - log_store.expect_replace_range().returning(|from, e| { - Ok(e.last().map(|x| x.index).unwrap_or(from.saturating_sub(1))) - }); + log_store + .expect_replace_range() + .returning(|from, e| Ok(e.last().map(|x| x.index).unwrap_or(from.saturating_sub(1)))); log_store.expect_is_write_durable().returning(|| false); log_store.expect_flush().returning(|| Ok(())); log_store.expect_flush_async().returning(|| Ok(())); @@ -1365,9 +1357,7 @@ async fn test_flush_turn_catch_up_persists_writes_coalesced_during_the_turn() { let mut log_store = MockLogStore::new(); log_store.expect_last_index().returning(|| 0); log_store.expect_persist_entries().returning(move |entries| { - entered_tx - .send(entries.iter().map(|e| e.index).collect()) - .ok(); + entered_tx.send(entries.iter().map(|e| e.index).collect()).ok(); release_rx.lock().unwrap().recv().ok(); Ok(()) }); @@ -1377,9 +1367,9 @@ async fn test_flush_turn_catch_up_persists_writes_coalesced_during_the_turn() { log_store.expect_load_purge_boundary().returning(|| Ok(None)); log_store.expect_reset().returning(|| Ok(())); log_store.expect_truncate().returning(|_| Ok(())); - log_store.expect_replace_range().returning(|from, e| { - Ok(e.last().map(|x| x.index).unwrap_or(from.saturating_sub(1))) - }); + log_store + .expect_replace_range() + .returning(|from, e| Ok(e.last().map(|x| x.index).unwrap_or(from.saturating_sub(1)))); log_store.expect_is_write_durable().returning(|| false); log_store.expect_flush().returning(|| Ok(())); log_store.expect_flush_async().returning(|| Ok(())); @@ -1452,8 +1442,8 @@ async fn test_flush_turn_catch_up_persists_writes_coalesced_during_the_turn() { flush_task.await.unwrap().unwrap(); while let Ok(ev) = log_flush_rx.try_recv() { - if let crate::InternalEvent::FsyncCompleted { index, term } = ev { - raft_log.try_advance_durable_index(index, term); + if let crate::InternalEvent::FsyncCompleted(mark) = ev { + raft_log.try_advance_durable_index(mark); } } assert_eq!( diff --git a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs index e060fed6..2df3c6a3 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs @@ -10,6 +10,7 @@ use std::sync::Mutex; use std::time::Duration; use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; @@ -146,8 +147,8 @@ async fn test_stale_persist_after_truncation_does_not_advance_durable_index() { // Drain fsync completions the way raft.rs's event loop would. while let Ok(event) = log_flush_rx.try_recv() { - if let crate::InternalEvent::FsyncCompleted { index, term } = event { - raft_log.try_advance_durable_index(index, term); + if let crate::InternalEvent::FsyncCompleted(mark) = event { + raft_log.try_advance_durable_index(mark); } } @@ -193,19 +194,14 @@ async fn test_persist_pending_range_reports_written_max_not_scan_bound() { ); let raft_log = Arc::new(raft_log); - raft_log - .append_entries((1..=7).map(|i| entry(i, 1)).collect()) - .await - .unwrap(); + raft_log.append_entries((1..=7).map(|i| entry(i, 1)).collect()).await.unwrap(); // Scan bound is 10 (stale latch); the SkipMap holds only 1..=7. - let written = BufferedRaftLog::persist_pending_range(&raft_log, 5, 10, "test") - .await - .unwrap(); + let written = BufferedRaftLog::persist_pending_range(&raft_log, 5, 10, "test").await.unwrap(); assert_eq!( written, - Some(7), + Some(LogId { term: 1, index: 7 }), "must report the highest index actually written (7), not the scan bound (10)" ); } @@ -229,21 +225,13 @@ async fn test_persist_frontier_skips_new_tail_after_truncation() { { let calls = persist_calls.clone(); log_store.expect_persist_entries().returning(move |entries| { - calls - .lock() - .unwrap() - .push(entries.iter().map(|e| e.index).collect()); + calls.lock().unwrap().push(entries.iter().map(|e| e.index).collect()); Ok(()) }); } - log_store - .expect_replace_range() - .returning(|from, new_entries| { - Ok(new_entries - .last() - .map(|e| e.index) - .unwrap_or(from.saturating_sub(1))) - }); + log_store.expect_replace_range().returning(|from, new_entries| { + Ok(new_entries.last().map(|e| e.index).unwrap_or(from.saturating_sub(1))) + }); log_store.expect_truncate().returning(|_| Ok(())); log_store.expect_entry().returning(|_| Ok(None)); log_store.expect_get_entries().returning(|_| Ok(vec![])); @@ -275,10 +263,7 @@ async fn test_persist_frontier_skips_new_tail_after_truncation() { std::thread::sleep(Duration::from_millis(10)); // Old leader (term 1): entries 1..=10, persisted. - raft_log - .append_entries((1..=10).map(|i| entry(i, 1)).collect()) - .await - .unwrap(); + raft_log.append_entries((1..=10).map(|i| entry(i, 1)).collect()).await.unwrap(); raft_log.flush().await.unwrap(); // New leader (term 2): conflict at index 6 → truncate [6..], replace with @@ -294,10 +279,7 @@ async fn test_persist_frontier_skips_new_tail_after_truncation() { persist_calls.lock().unwrap().clear(); // Next write extends the log. Its persist scan must start at 8, not 7. - raft_log - .append_entries((8..=10).map(|i| entry(i, 2)).collect()) - .await - .unwrap(); + raft_log.append_entries((8..=10).map(|i| entry(i, 2)).collect()).await.unwrap(); tokio::time::sleep(Duration::from_millis(50)).await; let calls = persist_calls.lock().unwrap().clone(); diff --git a/d-engine-core/src/storage/fsync_coordinator.rs b/d-engine-core/src/storage/fsync_coordinator.rs index 93e05145..6e2bb6a0 100644 --- a/d-engine-core/src/storage/fsync_coordinator.rs +++ b/d-engine-core/src/storage/fsync_coordinator.rs @@ -1,24 +1,28 @@ use crate::BufferedRaftLog; use crate::Error; use crate::LogStore; -use crate::RaftLog; use crate::Result; use crate::TypeConfig; +use d_engine_proto::common::LogId; +use parking_lot::Mutex; use std::sync::Arc; -use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use tokio::sync::oneshot; use tracing::error; /// Schedules physical fsync calls — batches concurrent requests into one -/// flush() at a time. Does not judge whether results are still valid; see -/// BufferedRaftLog::apply_durable_report. +/// flush() at a time. The final content check is `BufferedRaftLog:: +/// try_advance_durable_index`; this only orders the pending mark term-first. pub(super) struct FsyncCoordinator { inflight: AtomicBool, - pending_max: AtomicU64, + /// Highest `(term, index)` awaiting fsync. Term-first: a newer term's mark + /// wins over an older term's higher index, so a stale pre-truncation submit + /// cannot swallow the valid post-truncation one. + pending_max: Mutex, pending_replies: Mutex>>>, - // Lets a stale round skip its reply early. Optional — not required for correctness. + /// Bumped on truncation/reset. A round whose start predates the bump errs + /// its queued flush() replies instead of reporting a superseded result. generation: AtomicU64, } @@ -26,7 +30,7 @@ impl FsyncCoordinator { pub(super) fn new() -> Self { Self { inflight: AtomicBool::new(false), - pending_max: AtomicU64::new(0), + pending_max: Mutex::new(LogId::default()), pending_replies: Mutex::new(Vec::new()), generation: AtomicU64::new(0), } @@ -39,14 +43,17 @@ impl FsyncCoordinator { pub(super) fn submit( self: &Arc, this: &Arc>, - max_index: u64, + mark: LogId, replies: Vec>>, ) { - if max_index > 0 { - self.pending_max.fetch_max(max_index, Ordering::AcqRel); + if mark.index > 0 { + let mut p = self.pending_max.lock(); + if (mark.term, mark.index) > (p.term, p.index) { + *p = mark; + } } if !replies.is_empty() { - self.pending_replies.lock().unwrap().extend(replies); + self.pending_replies.lock().extend(replies); } if self @@ -73,9 +80,8 @@ impl FsyncCoordinator { loop { let gen_at_start = self.generation.load(Ordering::Acquire); - let max_index = self.pending_max.swap(0, Ordering::AcqRel); - let max_term = this.entry_term(max_index).unwrap_or(0); - let replies = std::mem::take(&mut *self.pending_replies.lock().unwrap()); + let mark = std::mem::take(&mut *self.pending_max.lock()); + let replies = std::mem::take(&mut *self.pending_replies.lock()); if this.is_poisoned() { for reply in replies { @@ -86,13 +92,12 @@ impl FsyncCoordinator { return; } - if max_index == 0 && replies.is_empty() { + if mark.index == 0 && replies.is_empty() { self.inflight.store(false, Ordering::Release); metrics::gauge!("core.raft.fsync.inflight").set(0.0); // Re-check: something may have slipped in between the swap // above and clearing `inflight`. If so, re-arm. - if (self.pending_max.load(Ordering::Acquire) > 0 - || !self.pending_replies.lock().unwrap().is_empty()) + if (self.pending_max.lock().index > 0 || !self.pending_replies.lock().is_empty()) && self .inflight .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -104,9 +109,9 @@ impl FsyncCoordinator { return; } - if max_index > 0 { + if mark.index > 0 { let batch_size = - max_index.saturating_sub(this.durable_index.load(Ordering::Acquire)); + mark.index.saturating_sub(this.durable_index.load(Ordering::Acquire)); metrics::histogram!("core.raft.fsync.batch_entries").record(batch_size as f64); } @@ -134,7 +139,7 @@ impl FsyncCoordinator { } match &result { - Ok(()) => this.notify_fsync_completed(max_index, max_term), + Ok(()) => this.notify_fsync_completed(mark), Err(e) => { // One fsync failure = fatal, no threshold, no retry-and-hope. // Durability state is now unknown, this node @@ -142,7 +147,7 @@ impl FsyncCoordinator { this.mark_poisoned_and_notify(format!("fsync failed: {e:?}")); // mirrors advance_durable_and_notify's pattern error!( "WAL fsync failed at index {}: {:?} — node entering fatal state", - max_index, e + mark.index, e ); } } @@ -156,18 +161,14 @@ impl FsyncCoordinator { } } - fn bump_generation(&self) { - self.generation.fetch_add(1, Ordering::AcqRel); - } - /// Called from reset_internal() before clearing in-memory state. /// Bumps generation to fence the in-flight physical flush (if any), /// AND drains anything already queued but not yet picked up by a /// flush round — that queued data was submitted before reset and /// must not be silently adopted by the next round. pub(super) fn fence_reset(&self) { - self.pending_max.store(0, Ordering::Release); - let stale = std::mem::take(&mut *self.pending_replies.lock().unwrap()); + *self.pending_max.lock() = LogId::default(); + let stale = std::mem::take(&mut *self.pending_replies.lock()); for reply in stale { let _ = reply.send(Err(Error::Fatal( "stale fsync generation, superseded by reset".into(), @@ -176,12 +177,8 @@ impl FsyncCoordinator { self.bump_generation(); } - pub(super) fn fence_truncation( - &self, - new_max: u64, - ) { - self.pending_max.fetch_min(new_max, Ordering::AcqRel); - self.bump_generation(); + pub(super) fn bump_generation(&self) { + self.generation.fetch_add(1, Ordering::AcqRel); } } diff --git a/d-engine-core/src/storage/fsync_coordinator_test.rs b/d-engine-core/src/storage/fsync_coordinator_test.rs index bf25e336..94787e4d 100644 --- a/d-engine-core/src/storage/fsync_coordinator_test.rs +++ b/d-engine-core/src/storage/fsync_coordinator_test.rs @@ -20,8 +20,10 @@ use crate::MockMetaStore; use crate::MockStorageEngine; use crate::MockTypeConfig; use crate::PersistenceConfig; +use crate::RaftLog; // try_advance_durable_index is a RaftLog trait method use crate::Result; use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; use std::sync::Arc; use tokio::sync::mpsc; @@ -59,12 +61,12 @@ fn test_new_initializes_empty_state() { "inflight must start false" ); assert_eq!( - coord.pending_max.load(Ordering::Acquire), + coord.pending_max.lock().index, 0, "pending_max must start at 0" ); assert!( - coord.pending_replies.lock().unwrap().is_empty(), + coord.pending_replies.lock().is_empty(), "pending_replies must start empty" ); assert_eq!( @@ -87,12 +89,12 @@ fn test_fence_reset_zeroes_pending_max() { let coord = FsyncCoordinator::new(); // Directly seed pending_max — no need to go through submit() (which would // spawn a real background task and race with fence_reset() below). - coord.pending_max.store(10, Ordering::Release); + *coord.pending_max.lock() = LogId { term: 1, index: 10 }; coord.fence_reset(); assert_eq!( - coord.pending_max.load(Ordering::Acquire), + coord.pending_max.lock().index, 0, "fence_reset() must zero pending_max" ); @@ -113,12 +115,12 @@ fn test_fence_reset_drains_pending_replies_with_err() { let (tx1, mut rx1) = oneshot::channel(); let (tx2, mut rx2) = oneshot::channel(); - coord.pending_replies.lock().unwrap().extend([tx1, tx2]); + coord.pending_replies.lock().extend([tx1, tx2]); coord.fence_reset(); assert!( - coord.pending_replies.lock().unwrap().is_empty(), + coord.pending_replies.lock().is_empty(), "pending_replies must be empty after fence_reset()" ); assert!( @@ -178,13 +180,9 @@ fn test_fence_reset_is_safe_with_nothing_pending() { 1, "generation must still increment even with nothing pending" ); - assert_eq!( - coord.pending_max.load(Ordering::Acquire), - 0, - "pending_max must stay 0" - ); + assert_eq!(coord.pending_max.lock().index, 0, "pending_max must stay 0"); assert!( - coord.pending_replies.lock().unwrap().is_empty(), + coord.pending_replies.lock().is_empty(), "pending_replies must stay empty" ); } @@ -207,11 +205,18 @@ fn test_submit_pending_max_uses_fetch_max_not_last_write() { // deterministic, no real concurrency needed to verify fetch_max order. coord.inflight.store(true, Ordering::Release); - coord.submit(&raft_log, 100, vec![]); - coord.submit(&raft_log, 50, vec![]); + coord.submit( + &raft_log, + LogId { + term: 1, + index: 100, + }, + vec![], + ); + coord.submit(&raft_log, LogId { term: 1, index: 50 }, vec![]); assert_eq!( - coord.pending_max.load(Ordering::Acquire), + coord.pending_max.lock().index, 100, "pending_max must stay at the high-water mark (100), not regress to \ a later, smaller submit() value (50)" @@ -234,7 +239,7 @@ async fn test_submit_first_call_sets_inflight_true() { let coord = Arc::new(FsyncCoordinator::new()); let raft_log = minimal_raft_log(storage); - coord.submit(&raft_log, 1, vec![]); + coord.submit(&raft_log, LogId { term: 1, index: 1 }, vec![]); assert!( coord.inflight.load(Ordering::Acquire), @@ -269,15 +274,15 @@ fn test_submit_second_call_does_not_spawn_second_task_while_inflight() { coord.inflight.store(true, Ordering::Release); let (tx, mut rx) = oneshot::channel::>(); - coord.submit(&raft_log, 1, vec![tx]); + coord.submit(&raft_log, LogId { term: 1, index: 1 }, vec![tx]); assert_eq!( - coord.pending_max.load(Ordering::Acquire), + coord.pending_max.lock().index, 1, "submit() must still record pending_max even though it lost the CAS" ); assert_eq!( - coord.pending_replies.lock().unwrap().len(), + coord.pending_replies.lock().len(), 1, "submit() must still queue the reply even though it lost the CAS" ); @@ -370,13 +375,13 @@ fn test_run_until_caught_up_advances_durable_index_on_success() { raft_log.set_memory_max_index_for_test(5); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(5, Ordering::Release); + *coord.pending_max.lock() = LogId { term: 1, index: 5 }; coord.run_until_caught_up(&raft_log); // Stand in for raft.rs's InternalEvent::FsyncCompleted handler. - while let Ok(InternalEvent::FsyncCompleted { index, term }) = log_flush_rx.try_recv() { - raft_log.try_advance_durable_index(index, term); + while let Ok(InternalEvent::FsyncCompleted(mark)) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(mark); } assert_eq!( @@ -402,7 +407,7 @@ fn test_run_until_caught_up_does_not_advance_durable_index_on_flush_failure() { let pre = raft_log.durable_index.load(Ordering::Acquire); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(5, Ordering::Release); + *coord.pending_max.lock() = LogId { term: 1, index: 5 }; coord.run_until_caught_up(&raft_log); @@ -429,8 +434,8 @@ fn test_run_until_caught_up_sends_err_to_replies_on_flush_failure() { let (tx, mut rx) = oneshot::channel::>(); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(5, Ordering::Release); - coord.pending_replies.lock().unwrap().push(tx); + *coord.pending_max.lock() = LogId { term: 1, index: 5 }; + coord.pending_replies.lock().push(tx); coord.run_until_caught_up(&raft_log); @@ -475,8 +480,8 @@ fn test_run_until_caught_up_discards_stale_generation_result_without_advancing() let (tx, mut rx) = oneshot::channel::>(); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(5, Ordering::Release); - coord.pending_replies.lock().unwrap().push(tx); + *coord.pending_max.lock() = LogId { term: 1, index: 5 }; + coord.pending_replies.lock().push(tx); coord.run_until_caught_up(&raft_log); @@ -545,15 +550,15 @@ fn test_run_until_caught_up_accepts_result_when_generation_unchanged() { let (tx, mut rx) = oneshot::channel::>(); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(5, Ordering::Release); - coord.pending_replies.lock().unwrap().push(tx); + *coord.pending_max.lock() = LogId { term: 1, index: 5 }; + coord.pending_replies.lock().push(tx); // Nothing fences this round while it runs — generation stays at 2. coord.run_until_caught_up(&raft_log); // Stand in for raft.rs's InternalEvent::FsyncCompleted handler. - while let Ok(InternalEvent::FsyncCompleted { index, term }) = log_flush_rx.try_recv() { - raft_log.try_advance_durable_index(index, term); + while let Ok(InternalEvent::FsyncCompleted(mark)) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(mark); } assert_eq!( @@ -590,8 +595,8 @@ fn test_run_until_caught_up_coalesces_queued_submits_into_one_flush() { let (tx1, mut rx1) = oneshot::channel::>(); let (tx2, mut rx2) = oneshot::channel::>(); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(10, Ordering::Release); - coord.pending_replies.lock().unwrap().extend([tx1, tx2]); + *coord.pending_max.lock() = LogId { term: 1, index: 10 }; + coord.pending_replies.lock().extend([tx1, tx2]); coord.run_until_caught_up(&raft_log); @@ -610,27 +615,24 @@ fn test_run_until_caught_up_coalesces_queued_submits_into_one_flush() { ); } -/// A single fsync round can end up covering a range whose *top* index was -/// truncated away after being submitted (a stale pre-truncation persist), -/// folded together with a *lower* index that is still valid. The coordinator -/// collapses the round to one `(max_index, max_term)` pair reported as -/// `FsyncCompleted` — if that pair is the stale top, content-validation on the -/// drain side rejects it and the valid lower index gets **no report at all**, -/// stranding `durable_index`. +/// Two `submit()` calls race into the same pending round: a stale +/// pre-truncation persist carrying `(term 1, index 10)` (its captured entries +/// were all term 1, and index 10 has since been truncated away) and the valid +/// post-truncation `ReplaceRange` mark `(term 2, index 2)`. /// -/// Deterministic, IO-thread-free repro of the `test_stale_persist_after_ -/// truncation_does_not_advance_durable_index` failure. Post-truncation log is -/// `[1, 2]`; `pending_max` holds the stale `10` (its entry is gone) with the -/// valid `2` folded in. One round must still let `durable_index` reach 2. +/// Term-first ordering must keep `(term 2, index 2)` — a newer term wins over +/// an older term's higher index. Before this fix `pending_max` was a bare +/// `fetch_max` on the index: `10` swallowed `2`, the round reported the stale +/// `10`, content-validation rejected it, and `durable_index` never reached 2. /// -/// RED until `FsyncCoordinator` derives the report from the truncation-aware -/// persist frontier instead of the coalescing `pending_max` accumulator. +/// Deterministic, IO-thread-free repro of +/// `test_stale_persist_after_truncation_does_not_advance_durable_index`. #[test] -fn test_run_until_caught_up_reports_valid_frontier_when_batch_top_is_stale() { +fn test_submit_term_first_keeps_valid_mark_over_stale_higher_index() { let (storage, _flush_call_count) = MockStorageEngine::not_durable( - "run_until_caught_up_reports_valid_frontier_when_batch_top_is_stale".into(), + "submit_term_first_keeps_valid_mark_over_stale_higher_index".into(), ); - let coord = FsyncCoordinator::new(); + let coord = Arc::new(FsyncCoordinator::new()); let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { @@ -644,37 +646,50 @@ fn test_run_until_caught_up_reports_valid_frontier_when_batch_top_is_stale() { let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); let raft_log = raft_log.start(receiver, Some(log_flush_tx)); - // Post-truncation log: entries 1..=2 only. Index 10 is gone. + // Post-truncation log: entry 1 (term 1) survived; entry 2 is the new + // leader's replacement (term 2). Index 10 is gone. { - let mut e = raft_log.entries.write(); - for i in 1..=2 { - e.insert( - i, - Entry { - index: i, - term: 1, - payload: None, - }, - ); - } + let e = raft_log.entries.write(); + e.insert( + 1, + Entry { + index: 1, + term: 1, + payload: None, + }, + ); + e.insert( + 2, + Entry { + index: 2, + term: 2, + payload: None, + }, + ); } raft_log.set_memory_max_index_for_test(2); - // A stale `submit(10)` and a valid `submit(2)` folded into one round - // (`fetch_max(10)` then `fetch_max(2)` => 10). + // Pretend a round is in flight so submit() only records state. coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(10, Ordering::Release); + coord.submit(&raft_log, LogId { term: 1, index: 10 }, vec![]); // stale + coord.submit(&raft_log, LogId { term: 2, index: 2 }, vec![]); // valid + + assert_eq!( + *coord.pending_max.lock(), + LogId { term: 2, index: 2 }, + "term-first: the newer-term mark must win over the stale higher index" + ); coord.run_until_caught_up(&raft_log); - while let Ok(InternalEvent::FsyncCompleted { index, term }) = log_flush_rx.try_recv() { - raft_log.try_advance_durable_index(index, term); + while let Ok(InternalEvent::FsyncCompleted(mark)) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(mark); } assert_eq!( raft_log.durable_index.load(Ordering::Acquire), 2, - "the fsync covered index 2 (valid) and index 10 (truncated); durable_index \ - must still reach 2 — a stale batch top must not swallow the valid frontier" + "durable_index must reach the valid post-truncation tail (2), not stall \ + because a stale batch top swallowed it" ); } diff --git a/d-engine-core/src/storage/raft_log.rs b/d-engine-core/src/storage/raft_log.rs index c5560be8..05e76866 100644 --- a/d-engine-core/src/storage/raft_log.rs +++ b/d-engine-core/src/storage/raft_log.rs @@ -85,8 +85,7 @@ pub trait RaftLog: Send + Sync + 'static { /// to decide whether to fire `handle_log_flushed`. fn try_advance_durable_index( &self, - index: u64, - term: u64, + mark: LogId, ) -> Option; /// Returns the LogId (term + index) of the last entry. diff --git a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs index a9376c82..bad5ac2f 100644 --- a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs +++ b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs @@ -259,8 +259,8 @@ pub fn drain_and_apply_fsync_completions( log_flush_rx: &mut tokio::sync::mpsc::UnboundedReceiver, ) { while let Ok(event) = log_flush_rx.try_recv() { - if let crate::InternalEvent::FsyncCompleted { index, term } = event { - raft_log.try_advance_durable_index(index, term); + if let crate::InternalEvent::FsyncCompleted(mark) = event { + raft_log.try_advance_durable_index(mark); } } } diff --git a/d-engine-server/src/api/embedded_test/embedded_env_test.rs b/d-engine-server/src/api/embedded_test/embedded_env_test.rs index 67173209..8c43d96e 100644 --- a/d-engine-server/src/api/embedded_test/embedded_env_test.rs +++ b/d-engine-server/src/api/embedded_test/embedded_env_test.rs @@ -31,15 +31,29 @@ mod start_data_dir_tests { } /// Opening an existing data directory is idempotent (data is preserved). + /// + /// Uses `start_with` + a timeout-only config so the client's write deadline + /// is not the 50ms `general_raft_timeout_duration_in_ms` default: since + /// #446 a `put` ack waits for a physical fdatasync, which under a loaded + /// test suite (many parallel RocksDB instances) can exceed 50ms. The + /// config's `data_dir` is still ignored — the explicit arg wins — so this + /// keeps testing exactly the reopen/data-preservation path. #[tokio::test] #[serial] async fn test_start_existing_directory_is_idempotent() { let temp_dir = tempfile::tempdir().expect("tempdir"); let data_dir = temp_dir.path().join("db"); + let config_path = temp_dir.path().join("d-engine.toml"); + std::fs::write( + &config_path, + "[raft]\ngeneral_raft_timeout_duration_in_ms = 3000\n", + ) + .expect("write config"); // First start: write a key { - let engine = EmbeddedEngine::start(&data_dir).await.expect("first start"); + let engine = + EmbeddedEngine::start_with(&data_dir, &config_path).await.expect("first start"); engine.wait_ready(std::time::Duration::from_secs(5)).await.expect("ready"); engine.client().put(b"k".to_vec(), b"v".to_vec()).await.expect("put"); tokio::time::sleep(std::time::Duration::from_millis(50)).await; @@ -48,7 +62,8 @@ mod start_data_dir_tests { // Second start: data must still be there { - let engine = EmbeddedEngine::start(&data_dir).await.expect("second start"); + let engine = + EmbeddedEngine::start_with(&data_dir, &config_path).await.expect("second start"); engine.wait_ready(std::time::Duration::from_secs(5)).await.expect("ready"); let val = engine.client().get_linearizable(b"k".to_vec()).await.expect("get"); assert_eq!(val.as_deref(), Some(b"v".as_ref()), "data must persist"); diff --git a/d-engine-server/tests/storage_buffered_raft_log/mod.rs b/d-engine-server/tests/storage_buffered_raft_log/mod.rs index 35b4300c..f8701b18 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/mod.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/mod.rs @@ -80,8 +80,8 @@ impl TestContext { /// state at construction, not from this event. pub fn drain_fsync_completions(&mut self) { while let Ok(event) = self.log_flush_rx.try_recv() { - if let d_engine_core::InternalEvent::FsyncCompleted { index, term } = event { - self.raft_log.try_advance_durable_index(index, term); + if let d_engine_core::InternalEvent::FsyncCompleted(mark) = event { + self.raft_log.try_advance_durable_index(mark); } } } diff --git a/examples/sled-cluster/src/sled_storage_engine.rs b/examples/sled-cluster/src/sled_storage_engine.rs index 03948a64..1ecec903 100644 --- a/examples/sled-cluster/src/sled_storage_engine.rs +++ b/examples/sled-cluster/src/sled_storage_engine.rs @@ -147,7 +147,8 @@ impl LogStore for SledLogStore { &self, from_index: u64, new_entries: Vec, - ) -> Result<()> { + ) -> Result { + let new_last = new_entries.last().map(|e| e.index).unwrap_or(from_index.saturating_sub(1)); let mut batch = sled::Batch::default(); // collect and remove all keys >= from_index @@ -164,7 +165,7 @@ impl LogStore for SledLogStore { } self.tree.apply_batch(batch).map_err(|e| StorageError::DbError(e.to_string()))?; - Ok(()) + Ok(new_last) } fn is_write_durable(&self) -> bool { From 33793fac9b7ce44d4ea3d4418d3418a49adbe896 Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:59:01 +0800 Subject: [PATCH 10/11] fix #446: drain queued IOTask::Persist safely, add fsync/batch metrics, macOS ramdisk bench tooling --- benches/embedded-bench/Makefile | 75 ++++++++++++++++--- benches/embedded-bench/src/main.rs | 2 +- d-engine-core/src/raft.rs | 1 + .../src/storage/buffered_raft_log.rs | 41 +++++++++- .../src/storage/fsync_coordinator.rs | 2 + examples/three-nodes-standalone/Makefile | 53 ++++++++++++- .../three-nodes-standalone/config/n1.toml | 2 +- .../three-nodes-standalone/config/n2.toml | 2 +- .../three-nodes-standalone/config/n3.toml | 2 +- examples/three-nodes-standalone/src/main.rs | 2 +- 10 files changed, 162 insertions(+), 20 deletions(-) diff --git a/benches/embedded-bench/Makefile b/benches/embedded-bench/Makefile index 966619b3..281c685e 100644 --- a/benches/embedded-bench/Makefile +++ b/benches/embedded-bench/Makefile @@ -1,27 +1,25 @@ # Makefile for embedded-bench # Provides benchmark commands matching embedded-bench/reports/v0.2.0/report_v0.2.0_final.md -.PHONY: help build clean test-single-write test-high-conc-write test-linearizable-read test-lease-read test-eventual-read test-hot-key all-tests +.PHONY: help build clean clean-log-db test-single-write test-high-conc-write test-linearizable-read test-lease-read test-eventual-read test-hot-key all-tests \ + all-ram-tests ramdisk-create clean-ram-log-db ramdisk-release +# =============================== +# Global Variables +# =============================== BENCH_BIN := ./target/release/embedded-bench CONFIG_DIR := ./config +LOG_LEVEL ?= warn # Node selection (default: n1) NODE ?= n1 CONFIG_PATH := $(CONFIG_DIR)/$(NODE).toml SINGLE_NODE_CONFIG_PATH := $(CONFIG_DIR)/single_node.toml -DATA_DIR := ./data/$(NODE) -SINGLE_NODE_DATA_DIR := ./data/single-node # Metrics port per node — same scheme as examples/three-nodes-standalone # (8081/8082/8083), reused because the two setups never run concurrently. METRICS_PORT := $(if $(filter n2,$(NODE)),8082,$(if $(filter n3,$(NODE)),8083,8081)) -# =============================== -# Global Variables -# =============================== -LOG_LEVEL ?= warn - # Common parameters (matching Standalone tests) KEY_SIZE := 8 VALUE_SIZE := 256 @@ -35,6 +33,25 @@ CLIENTS ?= 1000 VERIFY ?= false VERIFY_FLAG := $(if $(filter true,$(VERIFY)),--verify-write,) +# RAM disk (macOS only, optional) — isolates this node's storage on its own +# independent RAM disk volume, to strip physical-disk latency out of a +# benchmark. Not a general fix for unrelated flakiness — see +# tickets/milestones/v0.2.5/446-perf-batching-measurement-2026-09-13.md. +# One-shot: `make all-ram-tests`. Manual/single-node: add RAMDISK=true to +# any test-* target (needs the other two nodes already running for quorum). +RAMDISK ?= false +RAMDISK_SIZE_MB ?= 1024 +RAMDISK_SECTORS := $(shell echo $$(( $(RAMDISK_SIZE_MB) * 2048 ))) +RAMDISK_VOLUMES := RAMDisk1 RAMDisk2 RAMDisk3 +RAMDISK_INDEX := $(if $(filter n2,$(NODE)),2,$(if $(filter n3,$(NODE)),3,1)) + +ifeq ($(RAMDISK),true) +DATA_DIR := /Volumes/RAMDisk$(RAMDISK_INDEX)/$(NODE) +SINGLE_NODE_DATA_DIR := /Volumes/RAMDisk1/single-node +else +DATA_DIR := ./data/$(NODE) +SINGLE_NODE_DATA_DIR := ./data/single-node +endif # On macOS with Homebrew: auto-detect compression lib paths to skip bundled C++ # compilation of RocksDB dependencies, which fails under macOS 26 + Xcode 26 @@ -58,7 +75,6 @@ ifneq ($(ZSTD_PREFIX),) BREW_ROCKSDB_ENV += ZSTD_LIB_DIR=$(ZSTD_PREFIX)/lib endif - help: @echo "Embedded-bench Makefile - Performance Testing" @echo "" @@ -83,6 +99,14 @@ help: @echo "Run All:" @echo " make all-tests Run all benchmark tests" @echo "" + @echo "RAM Disk Cluster (macOS only, optional — isolates disk latency):" + @echo " make all-ram-tests NODE=nX Create RAM disks, run full --batch suite on this node (like all-tests)" + @echo " Run in 3 terminals with NODE=n1/n2/n3 to form the cluster" + @echo " make ramdisk-create Create RAMDisk1/2/3 if not already mounted" + @echo " make clean-ram-log-db Wipe node data off the RAM disks (keeps them mounted)" + @echo " make ramdisk-release Unmount RAMDisk1/2/3, freeing the memory" + @echo " Add RAMDISK=true to any single test-* target (needs the other 2 nodes already running)" + @echo "" @echo "Examples:" @echo " make test-linearizable-read # Run on n1 (default)" @echo " make test-linearizable-read NODE=n2 # Run on n2" @@ -109,6 +133,7 @@ clean-log-db: rm -rf ./logs/* rm -rf ./data/* rm -rf ./snapshots/* + # ============================================ # Write Performance Tests # ============================================ @@ -262,4 +287,34 @@ all-tests: build put @echo "" @echo "Compare results with Standalone mode:" - @echo " Standalone report: ../../benches/standalone-bench/reports/v0.2.2/report_v0.2.2.md" + +# ============================================ +# RAM Disk Cluster (macOS only, optional) +# ============================================ +# Isolates each node's storage on its own independent RAM disk volume — use +# this to strip physical-disk latency out of a benchmark (e.g. to study +# fsync/scheduling behavior in isolation), not as a general fix for +# unrelated flakiness. See tickets/milestones/v0.2.5/446-perf-batching-measurement-2026-09-13.md. + +# Ensures the RAM disks exist, then runs the full --batch suite for this +# node on its own RAM disk volume — same scope/shape as `make all-tests +# NODE=nX`, just on RAM disk. This is single-node, like all-tests: run it in +# 3 separate terminals with NODE=n1/n2/n3 to form the cluster, e.g. +# make all-ram-tests NODE=n2 CLIENTS=100 +all-ram-tests: ramdisk-create + $(MAKE) all-tests RAMDISK=true NODE=$(NODE) CLIENTS=$(CLIENTS) + +# Idempotent — mounts any of RAMDisk1/2/3 that aren't already present. +ramdisk-create: + @[ "$$(uname)" = "Darwin" ] || { echo "RAM disk targets need macOS."; exit 1; } + @for v in $(RAMDISK_VOLUMES); do \ + [ -d "/Volumes/$$v" ] || diskutil erasevolume HFS+ $$v `hdiutil attach -nomount ram://$(RAMDISK_SECTORS)` >/dev/null; \ + done + +# Wipes node data off the RAM disks; keeps the volumes mounted. +clean-ram-log-db: + @for v in $(RAMDISK_VOLUMES); do rm -rf /Volumes/$$v/*; done + +# Unmounts the RAM disks entirely, releasing the memory back to the OS. +ramdisk-release: + @for v in $(RAMDISK_VOLUMES); do diskutil eject /Volumes/$$v 2>/dev/null || true; done diff --git a/benches/embedded-bench/src/main.rs b/benches/embedded-bench/src/main.rs index c0d699d3..93a863d3 100644 --- a/benches/embedded-bench/src/main.rs +++ b/benches/embedded-bench/src/main.rs @@ -171,7 +171,7 @@ fn generate_value(size: usize) -> Vec { (0..size).map(|_| rng.random()).collect() } -#[tokio::main] +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] async fn main() { // Initialize logging tracing_subscriber::fmt() diff --git a/d-engine-core/src/raft.rs b/d-engine-core/src/raft.rs index 815b4c39..b206bcac 100644 --- a/d-engine-core/src/raft.rs +++ b/d-engine-core/src/raft.rs @@ -354,6 +354,7 @@ where } if count > 0 { trace!("Drained {} client commands", count); + metrics::histogram!("core.raft.client_cmd.batch_size").record(count as f64); } Ok(()) } diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index 5421bc28..384642cc 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -999,7 +999,46 @@ where IOTask::Shutdown => Self::run_flush_turn(&this, &mut receiver, &mut persisted_index,Vec::new(), true).await, IOTask::Flush(reply) => Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, vec![reply], false).await, cmd => { - if Self::run_storage_tasks(cmd, &this, &mut persisted_index).await { + let mut extra = Vec::new(); + let mut control: Option = None; + while let Ok(next) = receiver.try_recv() { + match next { + IOTask::Persist => {} // redundant — the persist below covers it + f @ (IOTask::Flush(_) | IOTask::Shutdown) => { + + control = Some(f); + break; + } + other => extra.push(other), + } + } + + + + let mut fatal = Self::run_storage_tasks(cmd, &this, &mut persisted_index).await; + if !fatal { + for other in extra { + if Self::run_storage_tasks(other, &this, &mut persisted_index).await { + fatal = true; + break; + } + } + } + if fatal { + break; + } + + let should_break = match control { + Some(IOTask::Shutdown) => { + Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, Vec::new(), true).await + } + Some(IOTask::Flush(reply)) => { + Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, vec![reply], false).await + } + Some(_) => unreachable!(), + None => false, + }; + if should_break { break; } continue; diff --git a/d-engine-core/src/storage/fsync_coordinator.rs b/d-engine-core/src/storage/fsync_coordinator.rs index 6e2bb6a0..21be38f5 100644 --- a/d-engine-core/src/storage/fsync_coordinator.rs +++ b/d-engine-core/src/storage/fsync_coordinator.rs @@ -61,8 +61,10 @@ impl FsyncCoordinator { .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) .is_err() { + metrics::counter!("core.raft.fsync.coalesced_submit").increment(1); return; // Already running — it will pick up what we just recorded. } + metrics::counter!("core.raft.fsync.fresh_round").increment(1); metrics::gauge!("core.raft.fsync.inflight").set(1.0); diff --git a/examples/three-nodes-standalone/Makefile b/examples/three-nodes-standalone/Makefile index 81a14311..7f23d971 100644 --- a/examples/three-nodes-standalone/Makefile +++ b/examples/three-nodes-standalone/Makefile @@ -3,7 +3,8 @@ .PHONY: build start-cluster clean clean-log-db help \ start-node1 start-node2 start-node3 \ perf-node1 perf-node2 perf-node3 perf-cluster \ - tokio-console-node1 tokio-console-node2 tokio-console-node3 tokio-console-cluster + tokio-console-node1 tokio-console-node2 tokio-console-node3 tokio-console-cluster \ + start-ram-cluster ramdisk-create clean-ram-log-db ramdisk-release .DEFAULT_GOAL := help # =============================== @@ -43,10 +44,16 @@ build: # =============================== # Cluster Management (Normal Mode) # =============================== +# Overridable so start-ram-cluster can point these at /Volumes/RAMDiskN +# without duplicating the node targets. +DB_PATH_1 ?= ./db/1 +DB_PATH_2 ?= ./db/2 +DB_PATH_3 ?= ./db/3 + start-node1: @echo "🚀 Starting Node 1..." @CONFIG_PATH=config/n1 \ - DB_PATH="./db/1" \ + DB_PATH="$(DB_PATH_1)" \ LOG_DIR="./logs/1" \ METRICS_PORT=8081 \ RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ @@ -56,7 +63,7 @@ start-node1: start-node2: @echo "🚀 Starting Node 2..." @CONFIG_PATH=config/n2 \ - DB_PATH="./db/2" \ + DB_PATH="$(DB_PATH_2)" \ LOG_DIR="./logs/2" \ METRICS_PORT=8082 \ RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ @@ -65,7 +72,7 @@ start-node2: start-node3: @echo "🚀 Starting Node 3..." @CONFIG_PATH=config/n3 \ - DB_PATH="./db/3" \ + DB_PATH="$(DB_PATH_3)" \ LOG_DIR="./logs/3" \ METRICS_PORT=8083 \ RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ @@ -76,6 +83,40 @@ start-cluster: @echo "Starting 3-node cluster in parallel..." $(MAKE) -j3 start-node1 start-node2 start-node3 +# =============================== +# RAM Disk (macOS only, optional) +# =============================== +# Isolates each node's storage on its own independent RAM disk volume — use +# this to strip physical-disk latency out of a benchmark (e.g. to study +# fsync/scheduling behavior in isolation), not as a general fix for +# unrelated flakiness. See tickets/milestones/v0.2.5/446-perf-batching-measurement-2026-09-13.md. +RAMDISK_SIZE_MB ?= 1024 +RAMDISK_SECTORS := $(shell echo $$(( $(RAMDISK_SIZE_MB) * 2048 ))) +RAMDISK_VOLUMES := RAMDisk1 RAMDisk2 RAMDisk3 + +# One command: wipe stale node data, ensure the 3 RAM disks exist, start the +# cluster on them instead of ./db. +start-ram-cluster: clean-ram-log-db ramdisk-create + @echo "Starting 3-node cluster on RAM disk..." + $(MAKE) -j3 start-node1 start-node2 start-node3 \ + DB_PATH_1=/Volumes/RAMDisk1/n1 DB_PATH_2=/Volumes/RAMDisk2/n2 DB_PATH_3=/Volumes/RAMDisk3/n3 + +# Idempotent — mounts any of RAMDisk1/2/3 that aren't already present. +ramdisk-create: + @[ "$$(uname)" = "Darwin" ] || { echo "RAM disk targets need macOS."; exit 1; } + @for v in $(RAMDISK_VOLUMES); do \ + [ -d "/Volumes/$$v" ] || diskutil erasevolume HFS+ $$v `hdiutil attach -nomount ram://$(RAMDISK_SECTORS)` >/dev/null; \ + done + +# Wipes node data off the RAM disks; keeps the volumes mounted so the next +# start-ram-cluster doesn't pay to recreate them. +clean-ram-log-db: + @for v in $(RAMDISK_VOLUMES); do rm -rf /Volumes/$$v/*; done + +# Unmounts the RAM disks entirely, releasing the memory back to the OS. +ramdisk-release: + @for v in $(RAMDISK_VOLUMES); do diskutil eject /Volumes/$$v 2>/dev/null || true; done + # =============================== # Performance Profiling with Samply @@ -184,6 +225,10 @@ help: @echo " perf-node1..3 - Run individual nodes under samply profiler" @echo " perf-cluster - Run full 3-node cluster under samply profiling" @echo " tokio-console-cluster - Run full 3-node cluster under tokio console monitoring" + @echo " start-ram-cluster - (macOS) Clean+create RAM disks, start cluster on them" + @echo " ramdisk-create - (macOS) Create RAMDisk1/2/3 if not already mounted" + @echo " clean-ram-log-db - Wipe node data off the RAM disks (keeps them mounted)" + @echo " ramdisk-release - Unmount RAMDisk1/2/3, freeing the memory" @echo " clean - Remove build artifacts, logs, and profiles" @echo " clean-log-db - Remove only logs and database files" @echo " help - Show this help message" diff --git a/examples/three-nodes-standalone/config/n1.toml b/examples/three-nodes-standalone/config/n1.toml index 1acb62d9..431f4e09 100644 --- a/examples/three-nodes-standalone/config/n1.toml +++ b/examples/three-nodes-standalone/config/n1.toml @@ -42,7 +42,7 @@ max_pending_reads = 500 flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [raft.snapshot] -enable = true +enable = false max_log_entries_before_snapshot = 5000 retained_log_entries = 100 cleanup_retain_count = 100 diff --git a/examples/three-nodes-standalone/config/n2.toml b/examples/three-nodes-standalone/config/n2.toml index 959b904a..0d100281 100644 --- a/examples/three-nodes-standalone/config/n2.toml +++ b/examples/three-nodes-standalone/config/n2.toml @@ -42,7 +42,7 @@ max_pending_reads = 500 flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [raft.snapshot] -enable = true +enable = false max_log_entries_before_snapshot = 5000 retained_log_entries = 100 cleanup_retain_count = 100 diff --git a/examples/three-nodes-standalone/config/n3.toml b/examples/three-nodes-standalone/config/n3.toml index 12c0ff72..5e3d80d2 100644 --- a/examples/three-nodes-standalone/config/n3.toml +++ b/examples/three-nodes-standalone/config/n3.toml @@ -42,7 +42,7 @@ max_pending_reads = 500 flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [raft.snapshot] -enable = true +enable = false max_log_entries_before_snapshot = 5000 retained_log_entries = 100 cleanup_retain_count = 100 diff --git a/examples/three-nodes-standalone/src/main.rs b/examples/three-nodes-standalone/src/main.rs index de1e1f83..c61dc79e 100644 --- a/examples/three-nodes-standalone/src/main.rs +++ b/examples/three-nodes-standalone/src/main.rs @@ -17,7 +17,7 @@ use tracing_subscriber::Layer; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; -#[tokio::main(flavor = "multi_thread", worker_threads = 2)] +#[tokio::main(flavor = "multi_thread", worker_threads = 4)] async fn main() { let log_dir = env::var("LOG_DIR") .map_err(|_| "LOG_DIR environment variable not set") From dc5096e869bd9d15a569f4d596aeab9c2d90087f Mon Sep 17 00:00:00 2001 From: Joshua Chi <112539+JoshuaChi@users.noreply.github.com> Date: Wed, 16 Sep 2026 20:44:41 +0800 Subject: [PATCH 11/11] fix #446: report post-commit term in AppendEntries reply, dedup per-ACK quorum calc - role_state.rs: AppendEntriesResponse reports post-commit term, not stale snapshot - leader_state.rs/mod.rs/raft.rs: dedup majority calc to 1x/ACK; move handle_peer_stream_error to LeaderState - leader_state.rs: probe in_flight gate ignores heartbeats, clears on Conflict/None/stream-error - buffered_raft_log.rs: prev_log_index==0 idempotent on duplicate resend - worker_test.rs: teardown tolerates NotFound (race vs OwnedSnapshotDir::drop) Unrelated, same branch: - Cargo.lock: rustls bump - PR template: AI-assistance checkbox - benches/embedded-bench: put-failure visibility - scoped_timer.rs: re-add Drop histogram - snapshot_worker_test.rs / performance_test.rs: compile fix / assertion update --- .github/PULL_REQUEST_TEMPLATE.md | 6 + Cargo.lock | 8 +- benches/embedded-bench/src/main.rs | 31 +- d-engine-core/src/raft.rs | 12 +- .../src/raft_role/follower_state_test.rs | 75 +++ d-engine-core/src/raft_role/leader_state.rs | 189 ++++--- .../leader_state_test/commit_index_test.rs | 88 +++ .../probe_backpressure_test.rs | 503 ++++++++++++++++++ .../leader_state_test/snapshot_worker_test.rs | 2 +- d-engine-core/src/raft_role/mod.rs | 23 - d-engine-core/src/raft_role/role_state.rs | 25 +- .../src/raft_test/raft_comprehensive_tests.rs | 39 +- .../src/state_machine_handler/worker_test.rs | 21 +- .../src/storage/buffered_raft_log.rs | 28 +- .../prev_log_index_zero_idempotency_test.rs | 245 +++++++++ d-engine-core/src/utils/scoped_timer.rs | 2 + .../performance_test.rs | 13 +- 17 files changed, 1159 insertions(+), 151 deletions(-) create mode 100644 d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs create mode 100644 d-engine-core/src/storage/buffered_raft_log_test/prev_log_index_zero_idempotency_test.rs diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4e9f8766..aaa0692c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -61,6 +61,12 @@ --- +## AI Assistance + +- [ ] This PR was written in part with the assistance of generative AI. All ideas and architecture decisions are mine; I have fully reviewed all changes. + +--- + ## Reviewer Notes (Optional: anything reviewers should focus on) diff --git a/Cargo.lock b/Cargo.lock index a411d838..5c74fa52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1830,9 +1830,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "log", "once_cell", @@ -1854,9 +1854,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", diff --git a/benches/embedded-bench/src/main.rs b/benches/embedded-bench/src/main.rs index 93a863d3..d14482e3 100644 --- a/benches/embedded-bench/src/main.rs +++ b/benches/embedded-bench/src/main.rs @@ -256,6 +256,7 @@ async fn run_benchmark_task( let stats = Arc::new(BenchmarkStats::new()); let key_counter = Arc::new(AtomicU64::new(0)); + let failed_count = Arc::new(AtomicU64::new(0)); let start_time = Instant::now(); let mut handles = Vec::with_capacity(clients); @@ -264,6 +265,7 @@ async fn run_benchmark_task( let engine = engine.clone(); let stats = stats.clone(); let key_counter = key_counter.clone(); + let failed_count = failed_count.clone(); let command = command.clone(); let handle = tokio::spawn(async move { @@ -289,7 +291,15 @@ async fn run_benchmark_task( } } } - Err(_) => continue, + Err(e) => { + let n = failed_count.fetch_add(1, Ordering::Relaxed); + if n < 5 { + eprintln!("Put failed: {e:?}"); + } else if n == 5 { + eprintln!("Put failed: (further failures suppressed)"); + } + continue; + } } } Commands::Get { consistency } => { @@ -327,6 +337,10 @@ async fn run_benchmark_task( futures::future::join_all(handles).await; stats.summary(start_time.elapsed()); + let failed = failed_count.load(Ordering::Relaxed); + if failed > 0 { + println!("Failed requests: {failed}"); + } } /// Run all benchmark tests in batch mode @@ -563,6 +577,7 @@ async fn run_local_benchmark(cli: Cli) { let stats = Arc::new(BenchmarkStats::new()); let key_counter = Arc::new(AtomicU64::new(0)); + let failed_count = Arc::new(AtomicU64::new(0)); let start_time = Instant::now(); let mut handles = Vec::with_capacity(cli.clients); @@ -571,6 +586,7 @@ async fn run_local_benchmark(cli: Cli) { let engine = engine.clone(); let stats = stats.clone(); let key_counter = key_counter.clone(); + let failed_count = failed_count.clone(); let cli = cli.clone(); let handle = tokio::spawn(async move { @@ -625,8 +641,13 @@ async fn run_local_benchmark(cli: Cli) { } } } - Err(_) => { - // Write failed - skip recording + Err(e) => { + let n = failed_count.fetch_add(1, Ordering::Relaxed); + if n < 5 { + eprintln!("Put failed: {e:?}"); + } else if n == 5 { + eprintln!("Put failed: (further failures suppressed)"); + } continue; } } @@ -676,6 +697,10 @@ async fn run_local_benchmark(cli: Cli) { futures::future::join_all(handles).await; stats.summary(start_time.elapsed()); + let failed = failed_count.load(Ordering::Relaxed); + if failed > 0 { + println!("Failed requests: {failed}"); + } println!("\nBenchmark completed. Press Ctrl+C to shutdown."); let _ = shutdown_rx.changed().await; diff --git a/d-engine-core/src/raft.rs b/d-engine-core/src/raft.rs index b206bcac..7eb802d7 100644 --- a/d-engine-core/src/raft.rs +++ b/d-engine-core/src/raft.rs @@ -671,7 +671,9 @@ where } InternalEvent::PeerStreamError { peer_id } => { debug!(%peer_id, "PeerStreamError: bidi stream disconnected, resetting next_index"); - self.role.handle_peer_stream_error(peer_id); + if let RaftRole::Leader(leader) = &mut self.role { + leader.handle_peer_stream_error(peer_id); + } } InternalEvent::ZombieDetected(node_id) => { debug!(%node_id, "ZombieDetected: forwarding to leader for BatchRemove"); @@ -707,9 +709,11 @@ where // Peer-state guard: seeding is only valid for the peer's CURRENT // in-flight snapshot attempt. A straggler completion that arrives after // the peer has moved on must not touch next_index. - if self.role.state().peer_replication_state(peer_id) - != PeerReplicationState::Snapshot - { + let in_snapshot_state = matches!( + &self.role, + RaftRole::Leader(leader) if leader.peer_replication_state(peer_id) == PeerReplicationState::Snapshot + ); + if !in_snapshot_state { debug!(%peer_id, "dropping SnapshotPushCompleted: peer not in Snapshot state"); return Ok(()); } diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index e084da8f..5ac6e2f8 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -1009,6 +1009,81 @@ async fn test_handle_append_entries_success_from_new_leader() { assert!(response.is_success(), "Response should indicate success"); } +/// Test: the `AppendEntriesResponse` sent back to the leader must report the follower's +/// just-updated term, not whatever term was captured in the `StateSnapshot` before +/// `commit_hard_state` ran. +/// +/// # Why this needs its own test +/// `test_handle_append_entries_success_from_new_leader` above hard-codes `new_leader_term` +/// directly into the mocked response, so it never actually reads what +/// `handle_append_entries_request_workflow` (role_state.rs) passes as the `state_snapshot` +/// argument to `handle_append_entries` — it can't catch a caller that passes a stale snapshot. +/// This test's mock instead echoes back `state_snapshot.current_term`, exactly mirroring what +/// the real `ReplicationHandler::handle_append_entries` does +/// (`replication_handler.rs`: `let current_term = state_snapshot.current_term;`), so it's +/// sensitive to whether the caller passes the pre-update or post-update snapshot. +/// +/// # Scenario +/// Follower (term=1) receives AppendEntries from a leader at term=2 — its first-ever contact. +/// +/// # Expected (RED until fixed) +/// `response.term == 2` — the follower's real term was correctly updated by `commit_hard_state` +/// before responding; the response must reflect that, not the term=1 snapshot taken before it. +#[tokio::test] +async fn test_handle_append_entries_response_reports_updated_term_not_stale_snapshot() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let follower_term = 1; + let new_leader_term = follower_term + 1; + + let mut replication_handler = MockReplicationCore::new(); + replication_handler + .expect_handle_append_entries() + .returning(move |_, state_snapshot, _| { + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success(1, state_snapshot.current_term, None), + commit_index_update: None, + }) + }); + + context.membership = Arc::new(MockMembership::new()); + context.handlers.replication_handler = replication_handler; + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state_mut().update_current_term(follower_term); + + let append_entries_request = AppendEntriesRequest { + term: new_leader_term, + leader_id: 5, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), + "handle_inbound_event should succeed" + ); + + // No claimed log entry in this batch, so RPO=0 withhold doesn't apply — response is + // available immediately. + let response = resp_rx.recv().await.expect("should receive response").unwrap(); + assert_eq!( + response.term, new_leader_term, + "AppendEntriesResponse must report the follower's just-updated term ({new_leader_term}), \ + not a StateSnapshot captured before commit_hard_state updated it ({follower_term})" + ); +} + /// Test: FollowerState rejects AppendEntries with stale term /// /// Scenario: diff --git a/d-engine-core/src/raft_role/leader_state.rs b/d-engine-core/src/raft_role/leader_state.rs index f607bb06..8e79e25c 100644 --- a/d-engine-core/src/raft_role/leader_state.rs +++ b/d-engine-core/src/raft_role/leader_state.rs @@ -201,8 +201,9 @@ pub struct ClusterMetadata { /// Task routed to a per-follower replication worker. enum ReplicationTask { - /// Normal AppendEntries replication. - Append(AppendEntriesRequest), + /// Normal AppendEntries replication. Second field is the enqueue time, used to + /// measure how long the task waited in `task_tx` before the worker sent it. + Append(AppendEntriesRequest, Instant), /// Peer's next_index fell below the purge boundary; transfer the latest snapshot. Snapshot(SnapshotMetadata, u64), } @@ -265,6 +266,8 @@ pub struct LeaderState { /// keep this change minimal and consistent with existing style. pub(super) peer_replication_state: HashMap, + in_flight: HashMap, + /// === Volatile State === /// Temporary storage for no-op entry log ID during leader initialization #[doc(hidden)] @@ -1343,7 +1346,7 @@ impl RaftRoleState for LeaderState { ctx: &RaftContext, internal_event_tx: &mpsc::UnboundedSender, ) { - let new_commit_index = if self.cluster_metadata.single_voter { + let next_commit_index = if self.cluster_metadata.single_voter { // RPO=0 (#446): single-voter has no majority to fall back on — commit // must not advance past what this node has itself fsynced. if durable > self.commit_index() { @@ -1355,10 +1358,11 @@ impl RaftRoleState for LeaderState { // Multi-voter: quorum of match_index determines commit. // Only voter peers (non-Learner role) may contribute to majority. // Learners replicate entries but must never count toward commit quorum. - self.calculate_new_commit_index(ctx.raft_log()) + let (commit_index, majority) = self.majority_matched_index(ctx.raft_log()); + Self::new_commit_index(commit_index, majority) }; - if let Some(new_commit) = new_commit_index { + if let Some(new_commit) = next_commit_index { if let Err(e) = self.update_commit_index_with_signal( Leader as i32, self.current_term(), @@ -1487,6 +1491,7 @@ impl RaftRoleState for LeaderState { "AppendResult from peer {} has no result variant", follower_id ); + self.set_peer_in_flight(follower_id, false); return Ok(()); } }; @@ -1529,7 +1534,10 @@ impl RaftRoleState for LeaderState { // Re-calculate commit index after updating this voter's match_index. if peer_update.success && is_voter { - if let Some(new_commit) = self.calculate_new_commit_index(ctx.raft_log()) { + let (commit_index, majority) = self.majority_matched_index(ctx.raft_log()); + let quorum_confirmed = majority.is_some(); + + if let Some(new_commit) = Self::new_commit_index(commit_index, majority) { self.update_commit_index_with_signal( Leader as i32, self.current_term(), @@ -1543,25 +1551,9 @@ impl RaftRoleState for LeaderState { // Lease refresh and pending_lease_reads drain are triggered by quorum ACK, // independent of whether commit_index advanced. When an expired lease fires an // empty AppendEntries heartbeat, commit_index does not change (nothing new to - // commit), so calculate_new_commit_index returns None and the block above is - // skipped. We must check quorum confirmation separately here. - let quorum_confirmed = ctx - .raft_log() - .calculate_majority_matched_index( - self.current_term(), - self.commit_index(), - self.match_index - .iter() - .filter(|(id, _)| { - self.cluster_metadata.replication_targets.iter().any(|n| { - n.id == **id - && n.role != d_engine_proto::common::NodeRole::Learner as i32 - }) - }) - .map(|(_, idx)| *idx) - .collect(), - ) - .is_some(); + // commit), so new_commit_index yields None and the block above is skipped. + // We still reuse the single majority computation (majority.is_some()) to + // confirm quorum here. if quorum_confirmed { // Anchor deadline to send time (not ACK time) to eliminate the RTT/2 window. // Falls back to now_ms() only in tests that bypass execute_and_process_raft_rpc. @@ -1875,33 +1867,6 @@ impl RaftRoleState for LeaderState { Ok(()) } - - fn peer_replication_state( - &self, - node_id: u32, - ) -> PeerReplicationState { - self.peer_replication_state - .get(&node_id) - .copied() - .unwrap_or(PeerReplicationState::Probe) - } - - fn set_peer_replication_state( - &mut self, - node_id: u32, - state: PeerReplicationState, - ) { - metrics::gauge!( - "core.raft.peer.replication_state", - "peer_id" => node_id.to_string() - ) - .set(match state { - PeerReplicationState::Probe => 0.0, - PeerReplicationState::Replicate => 1.0, - PeerReplicationState::Snapshot => 2.0, - }); - self.peer_replication_state.insert(node_id, state); - } } /// Computes the exponential backoff delay for a snapshot push failure. @@ -2049,7 +2014,12 @@ impl LeaderState { } match task { - ReplicationTask::Append(request) => { + ReplicationTask::Append(request, enqueued_at) => { + metrics::histogram!( + "core.raft.replication_worker.queue_wait_ms", + "peer_id" => peer_id.to_string() + ) + .record(enqueued_at.elapsed().as_secs_f64() * 1_000.0); // Push batch directly into the persistent bidi stream (non-blocking) if stream_sender.send(request).await.is_err() { warn!(peer_id, "Bidi stream sender closed, reconnecting"); @@ -2575,6 +2545,7 @@ impl LeaderState { follower_id: u32, update: &PeerUpdate, ) { + self.set_peer_in_flight(follower_id, false); if update.success { // Success: trust speculative advance — never regress next_index below what // the leader has already pipeline-sent. ACK confirms a lower bound only; @@ -2819,12 +2790,16 @@ impl LeaderState { Ok(()) } - /// Calculate new submission index - fn calculate_new_commit_index( + /// Compute the majority-matched index for the current term. + /// + /// Returns `(commit_index, majority)` where `commit_index` is the base used as + /// the quorum floor, so callers apply the advance rule against the *same* base + /// the majority was computed with. + fn majority_matched_index( &self, raft_log: &Arc>, - ) -> Option { - let old_commit_index = self.commit_index(); + ) -> (u64, Option) { + let commit_index = self.commit_index(); let current_term = self.current_term(); let replication_targets = &self.cluster_metadata.replication_targets; let learner_role = d_engine_proto::common::NodeRole::Learner as i32; @@ -2839,14 +2814,21 @@ impl LeaderState { .map(|(_, idx)| *idx) .collect(); - let new_commit_index = - raft_log.calculate_majority_matched_index(current_term, old_commit_index, matched_ids); + let majority = + raft_log.calculate_majority_matched_index(current_term, commit_index, matched_ids); + (commit_index, majority) + } - if new_commit_index.is_some() && new_commit_index.unwrap() > old_commit_index { - new_commit_index - } else { - None - } + /// Turn a majority-matched index into the new commit index. + /// + /// Pure function: compares against the same `commit_index` that `majority` was + /// computed with. Callers must not hand-write the `>` — this is the single + /// source of truth for strict advance. + fn new_commit_index( + commit_index: u64, + majority: Option, + ) -> Option { + majority.filter(|m| *m > commit_index) } /// Calculate safe read index for linearizable reads. @@ -3099,6 +3081,7 @@ impl LeaderState { write_propose_times: HashMap::new(), write_commit_times: HashMap::new(), peer_replication_state: HashMap::new(), + in_flight: HashMap::new(), _marker: PhantomData, } } @@ -3310,6 +3293,12 @@ impl LeaderState { continue; } + let is_probe = self.peer_replication_state(peer_id) == PeerReplicationState::Probe; + let is_heartbeat = request.entries.is_empty(); + if !is_heartbeat && is_probe && self.peer_in_flight(peer_id) { + continue; + } + // #436: write next_index once. When Replicate, trust speculative advance // (always >= effective_next_index, so no separate write is needed for it). let next_index = @@ -3322,9 +3311,13 @@ impl LeaderState { error!("failed to update next_index peer={}: {:?}", peer_id, e); } + if !is_heartbeat && is_probe { + self.set_peer_in_flight(peer_id, true); + } + self.send_to_worker_or_spawn( peer_id, - ReplicationTask::Append(request), + ReplicationTask::Append(request, Instant::now()), ReplicationWorkerConfig { transport: transport.clone(), membership: membership.clone(), @@ -3800,6 +3793,67 @@ impl LeaderState { let response = ClientResponse::read_results(results); let _ = sender.send(Ok(response)); } + + pub(crate) fn peer_replication_state( + &self, + node_id: u32, + ) -> PeerReplicationState { + self.peer_replication_state + .get(&node_id) + .copied() + .unwrap_or(PeerReplicationState::Probe) + } + + pub(crate) fn set_peer_replication_state( + &mut self, + node_id: u32, + state: PeerReplicationState, + ) { + metrics::gauge!( + "core.raft.peer.replication_state", + "peer_id" => node_id.to_string() + ) + .set(match state { + PeerReplicationState::Probe => 0.0, + PeerReplicationState::Replicate => 1.0, + PeerReplicationState::Snapshot => 2.0, + }); + self.peer_replication_state.insert(node_id, state); + } + + /// Whether `peer_id` has an unacknowledged AppendEntries outstanding. + pub(super) fn peer_in_flight( + &self, + peer_id: u32, + ) -> bool { + self.in_flight.get(&peer_id).copied().unwrap_or(false) + } + + pub(super) fn set_peer_in_flight( + &mut self, + peer_id: u32, + in_flight: bool, + ) { + self.in_flight.insert(peer_id, in_flight); + } + + /// Reset `next_index[peer] = match_index[peer] + 1` after a bidi stream disconnect. + /// Ensures the next heartbeat re-sends any unACKed in-flight entries. + /// Moved here from `raft_role/mod.rs` — this is leader-only, the generic + /// `RaftRole`-level dispatch was the anti-pattern that hid the `in_flight` gap. + pub(crate) fn handle_peer_stream_error( + &mut self, + peer_id: u32, + ) { + if self.peer_replication_state(peer_id) == PeerReplicationState::Snapshot { + return; + } + let match_idx = self.match_index(peer_id).unwrap_or(0); + let _ = self.update_next_index(peer_id, match_idx + 1); + + self.set_peer_replication_state(peer_id, PeerReplicationState::Probe); + self.set_peer_in_flight(peer_id, false); + } } impl From<&CandidateState> for LeaderState { @@ -3854,6 +3908,7 @@ impl From<&CandidateState> for LeaderState { write_propose_times: HashMap::new(), write_commit_times: HashMap::new(), peer_replication_state: HashMap::new(), + in_flight: HashMap::new(), _marker: PhantomData, } } @@ -3992,3 +4047,7 @@ mod state_management_test; #[cfg(test)] #[path = "leader_state_test/worker_lifecycle_test.rs"] mod worker_lifecycle_test; + +#[cfg(test)] +#[path = "leader_state_test/probe_backpressure_test.rs"] +mod probe_backpressure_test; diff --git a/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs b/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs index a7811973..748a1c84 100644 --- a/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs @@ -20,6 +20,7 @@ use d_engine_proto::server::replication::{AppendEntriesResponse, SuccessResult}; use rand::distr::SampleString; use std::collections::VecDeque; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use tokio::sync::{mpsc, watch}; fn success_response( @@ -38,6 +39,15 @@ fn success_response( } } +fn voter_meta(id: u32) -> NodeMeta { + NodeMeta { + id, + address: "".into(), + status: NodeStatus::Active as i32, + role: Follower.into(), + } +} + // ── helpers ────────────────────────────────────────────────────────────────── fn write_request() -> ( @@ -262,6 +272,84 @@ async fn test_multi_voter_commit_respects_quorum_result() { ); } +// ── handle_append_result: quorum computed exactly once per ACK ─────────────── + +/// A multi-voter leader must compute the majority-matched index exactly once per +/// AppendEntries ACK. The pre-fix code called `calculate_majority_matched_index` +/// twice in a single `handle_append_result` — once inside `calculate_new_commit_index` +/// and once more for the `quorum_confirmed` lease check — with identical inputs +/// (`current_term`, `commit_index`, and the same voter-filtered `match_index`). +/// +/// The single result must drive both outcomes: commit_index advance AND lease +/// (quorum) confirmation. This test fails if the method runs twice, or if either +/// consumer stops receiving the result. +#[tokio::test] +async fn test_handle_append_result_computes_quorum_once_per_ack() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + + let mut ctx = mock_raft_context( + "/tmp/test_handle_append_result_quorum_once", + graceful_rx, + None, + ); + + // Two follower voters → multi-voter; `replication_peers` must list them so + // `handle_append_result` treats peer 2 as a voter (drives the quorum path). + let mut membership = crate::MockMembership::::new(); + membership.expect_voters().returning(|| vec![voter_meta(2), voter_meta(3)]); + membership + .expect_replication_peers() + .returning(|| vec![voter_meta(2), voter_meta(3)]); + ctx.membership = Arc::new(membership); + + // Count every quorum computation; return a majority index of 1 so the single + // result is exercised by BOTH the commit path and the lease-confirmation path. + let call_count = Arc::new(AtomicU64::new(0)); + let call_count_clone = call_count.clone(); + let mut raft_log = MockRaftLog::new(); + raft_log.expect_calculate_majority_matched_index().returning(move |_, _, _| { + call_count_clone.fetch_add(1, Ordering::Relaxed); + Some(1) + }); + ctx.storage.raft_log = Arc::new(raft_log); + + let mut state = LeaderState::::new(1, ctx.node_config.clone()); + state.init_cluster_metadata(&ctx.membership).await.unwrap(); + assert!(!state.cluster_metadata.single_voter); + + ctx.handlers + .replication_handler + .expect_handle_success_response() + .returning(|_, _, _, _| { + Ok(PeerUpdate { + match_index: Some(1), + next_index: 2, + success: true, + }) + }); + + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let resp = success_response(1, 1); + state.handle_append_result(2, Ok(resp), &ctx, &internal_event_tx).await.unwrap(); + + // Root-cause assertion: one ACK must trigger exactly one majority computation. + assert_eq!( + call_count.load(Ordering::Relaxed), + 1, + "calculate_majority_matched_index must run exactly once per AppendEntries ACK" + ); + // Both consumers of that single result must still observe it. + assert_eq!( + state.commit_index(), + 1, + "commit must advance from the single majority result" + ); + assert!( + state.is_lease_valid(), + "quorum confirmation (lease) must reuse the single majority result" + ); +} + // ── handle_log_flushed: Leader commit advances on LogFlushed (#313 P0) ──────── /// Single-voter leader: LogFlushed(1) triggers commit_index to advance to 1. diff --git a/d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs b/d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs new file mode 100644 index 00000000..1641ac3b --- /dev/null +++ b/d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs @@ -0,0 +1,503 @@ +//! Test for the missing per-peer in-flight gate during `PeerReplicationState::Probe`. +//! +//! Background (see `446-expert-q-probe-backpressure-fix-8020.md` in the product-design repo): +//! every reference Raft implementation (etcd/raft, tikv/raft-rs, openraft) limits a `Probe`-state +//! peer to at most one outstanding (unacknowledged) `AppendEntries` request. d-engine's +//! `PeerReplicationState::Probe`/`Replicate` only controls whether `next_index` is optimistically +//! advanced before sending — it never checks whether the peer already has a request in flight. +//! Left unchecked, the leader keeps re-sending `prev_log_index=0` probes to a peer whose first +//! attempt hasn't been acknowledged yet, and each one forces the follower to wipe and rebuild its +//! entire log (`buffered_raft_log::reset`), which is the root cause of the throughput collapse +//! this ticket investigated. +//! +//! This test is intentionally RED until the in-flight gate is implemented in +//! `execute_and_process_raft_rpc` (Phase 5, `leader_state.rs`). It does not assert *how* the gate +//! is implemented — only the externally observable contract: a peer with one unacknowledged +//! request must not receive a second one. + +use std::collections::VecDeque; +use std::sync::Arc; + +use bytes::Bytes; +use d_engine_proto::common::{Entry, EntryPayload, NodeRole::Follower, NodeStatus}; +use d_engine_proto::server::cluster::NodeMeta; +use d_engine_proto::server::replication::{ + AppendEntriesRequest, AppendEntriesResponse, ConflictResult, append_entries_response, +}; +use tokio::sync::{mpsc, watch}; +use tracing_test::traced_test; + +use crate::MockMembership; +use crate::MockRaftLog; +use crate::RaftRequestWithSignal; +use crate::event::InternalEvent; +use crate::maybe_clone_oneshot::{MaybeCloneOneshot, RaftOneshot}; +use crate::network::PeerUpdate; +use crate::raft_role::leader_state::LeaderState; +use crate::raft_role::role_state::{PeerReplicationState, RaftRoleState}; +use crate::test_utils::mock::{MockTypeConfig, mock_raft_context}; + +/// Two-voter membership (peers 2 & 3) so the cluster is multi-voter — a single-voter leader +/// short-circuits Phase 5 entirely (no peer work to gate), which would make this test vacuous. +fn two_peer_membership() -> MockMembership { + let peers = vec![ + NodeMeta { + id: 2, + address: String::new(), + status: NodeStatus::Active as i32, + role: Follower.into(), + }, + NodeMeta { + id: 3, + address: String::new(), + status: NodeStatus::Active as i32, + role: Follower.into(), + }, + ]; + let peers2 = peers.clone(); + let mut m = MockMembership::new(); + m.expect_is_single_node_cluster().returning(|| false); + m.expect_voters().returning(move || peers.clone()); + m.expect_replication_peers().returning(move || peers2.clone()); + m +} + +/// A minimal `AppendEntriesRequest` stub — its contents don't matter, only whether Phase 5 +/// forwards a request to the peer's worker channel at all. +fn stub_request() -> AppendEntriesRequest { + AppendEntriesRequest::default() +} + +/// A non-empty probe (one entry) — used where the empty-heartbeat vs non-empty-probe +/// distinction matters for the gate: a heartbeat must neither block nor set the gate. +fn stub_probe_request() -> AppendEntriesRequest { + AppendEntriesRequest { + entries: vec![Entry { + index: 1, + term: 1, + payload: None, + }], + ..AppendEntriesRequest::default() + } +} + +/// A single one-entry write batch, matching the shape `process_batch` expects. +fn one_entry_batch() -> VecDeque { + let (tx, _rx) = >::new(); + let req = RaftRequestWithSignal { + id: "test".into(), + payloads: vec![EntryPayload::command(Bytes::from_static(b"cmd"))], + senders: vec![tx], + wait_for_apply_event: false, + }; + VecDeque::from(vec![req]) +} + +/// Scenario (release direction — the half the gate must also get right): +/// - Batch 1 is dispatched to peer 2: its first `Probe`, correctly limited to one outstanding +/// request. +/// - The follower answers that probe with a CONFLICT (reject), not a success. A reject is not +/// evidence the peer is caught up, so `update_peer_index`'s conflict branch retreats +/// `next_index` to the conflict hint and leaves peer 2 in `Probe`. +/// - Batch 2 is processed afterwards. The response to batch 1 has *arrived*, so peer 2 no longer +/// has an outstanding request and the gate must release: the corrected probe must be +/// dispatched. +/// +/// # Expected +/// `Probe` means "at most one unacknowledged `AppendEntries` at a time" (etcd/raft +/// `MsgAppFlowPaused`, openraft `Inflight::is_none()`), not "at most one ever". A response must +/// re-arm the gate, never latch it shut: a latched `Probe` peer receives nothing further — not +/// even heartbeats, since they share this dispatch path — while the frozen follower times out +/// into candidacy and the leader cannot reach quorum. +/// +/// # Current behavior (why this test is RED) +/// The gate is set on dispatch and only cleared by `handle_peer_stream_error` (a bidi stream +/// disconnect). A peer whose probe was rejected stays `Probe` with the latch closed forever, so +/// batch 2 dispatches nothing. +#[tokio::test] +#[traced_test] +async fn test_probe_peer_dispatches_next_probe_after_reject() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut ctx = mock_raft_context( + "/tmp/test_probe_peer_dispatches_next_probe_after_reject", + graceful_rx, + None, + ); + + ctx.membership = Arc::new(two_peer_membership()); + + // Both batches offer a request for peer 2, so every dispatch decision is Phase 5's own. + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(|_, _, _, _, _| { + Ok(crate::PrepareResult { + append_requests: vec![(2, stub_probe_request(), 1)], + snapshot_targets: vec![], + }) + }); + ctx.handlers + .replication_handler + .expect_handle_conflict_response() + .returning(|_, _, _, _| { + Ok(PeerUpdate { + match_index: None, + next_index: 1, + success: false, + }) + }); + + let mut raft_log = MockRaftLog::new(); + raft_log.expect_last_entry_id().returning(|| 0); + raft_log.expect_flush().returning(|| Ok(())); + raft_log.expect_save_hard_state().returning(|_| Ok(())); + ctx.storage.raft_log = Arc::new(raft_log); + + let mut state = LeaderState::::new(1, ctx.node_config.clone()); + state.init_cluster_metadata(&ctx.membership).await.unwrap(); + + let (task_tx, mut task_rx) = mpsc::unbounded_channel(); + state.replication_workers.insert( + 2, + super::ReplicationWorkerHandle { + task_tx, + snapshot_failure_count: 0, + snapshot_next_retry_at: None, + }, + ); + + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel::(); + + // Batch 1: peer 2's first probe — nothing outstanding, so it must be dispatched. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "first batch must reach peer 2's worker — it had no outstanding request" + ); + + // Peer 2 rejects that probe: the leader retreats next_index and keeps the peer in `Probe`, + // because a reject says nothing about the peer being caught up. + let reject = AppendEntriesResponse { + node_id: 2, + term: 1, + result: Some(append_entries_response::Result::Conflict(ConflictResult { + conflict_term: None, + conflict_index: Some(1), + })), + }; + state + .handle_append_result(2, Ok(reject), &ctx, &internal_event_tx) + .await + .unwrap(); + assert_eq!( + state.peer_replication_state(2), + PeerReplicationState::Probe, + "a rejected probe must leave the peer in `Probe` — it is not caught up" + ); + + // Batch 2: the response to batch 1 already arrived, so the gate must release and the + // corrected probe must go out. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "the probe was answered (reject), so the peer has nothing in flight — the leader must \ + re-probe with the corrected next_index instead of latching the peer shut forever" + ); +} + +/// Scenario: +/// - Peer 2 has a worker whose channel this test holds directly (no real transport/network +/// involved), so what actually got dispatched can be checked synchronously — no dependency on +/// background task scheduling, so this test cannot flake on timing. +/// - `prepare_batch_requests` is mocked to unconditionally offer a request for peer 2 on every +/// call, simulating "there's always more to replicate" regardless of ack status — this isolates +/// the assertion to Phase 5's own dispatch decision, which is where the missing gate belongs. +/// - Batch 1 is processed and dispatched — this is correct: peer 2 starts in `Probe` with nothing +/// outstanding, so it must receive its first probe. +/// - Batch 2 is processed *without* `handle_append_result` ever being called for peer 2's first +/// request — i.e. the leader has not (and cannot have) learned whether the first probe was +/// acknowledged. `next_index`/`match_index`/`peer_replication_state` are therefore still exactly +/// what they were after batch 1. +/// +/// # Expected (once the fix lands) +/// Batch 2 must NOT produce a second dispatch to peer 2's worker: a `Probe`-state peer with an +/// unacknowledged request in flight must wait for that response (etcd/raft `MsgAppFlowPaused`, +/// openraft `Inflight::is_none()`) before being sent to again. +/// +/// # Current behavior (why this test is RED today) +/// `execute_and_process_raft_rpc`'s Phase 5 loop sends to every peer in `append_requests` +/// unconditionally — `PeerReplicationState` only gates whether `next_index` is optimistically +/// advanced beforehand, not whether sending is allowed at all. So batch 2 dispatches anyway. +#[tokio::test] +#[traced_test] +async fn test_probe_peer_with_pending_ack_receives_no_second_dispatch() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut ctx = mock_raft_context( + "/tmp/test_probe_peer_with_pending_ack_receives_no_second_dispatch", + graceful_rx, + None, + ); + + ctx.membership = Arc::new(two_peer_membership()); + + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(|_, _, _, _, _| { + Ok(crate::PrepareResult { + append_requests: vec![(2, stub_probe_request(), 1)], + snapshot_targets: vec![], + }) + }); + + let mut raft_log = MockRaftLog::new(); + raft_log.expect_last_entry_id().returning(|| 0); + raft_log.expect_flush().returning(|| Ok(())); + raft_log.expect_save_hard_state().returning(|_| Ok(())); + ctx.storage.raft_log = Arc::new(raft_log); + + let mut state = LeaderState::::new(1, ctx.node_config.clone()); + state.init_cluster_metadata(&ctx.membership).await.unwrap(); + + // Inject peer 2's worker handle directly and keep the channel's receiver in this test — this + // is what makes the dispatch count observable synchronously, without any async worker task or + // transport mock (`send_to_worker_or_spawn` finds this handle and reuses it, so the real + // worker-spawn path — the only place that would touch `ctx.transport` — is never exercised). + // `ReplicationWorkerHandle`/`ReplicationTask` are private to `leader_state`, visible here only + // because this test module nests under it — same access pattern `inject_dead_worker_for_test` + // already relies on for the sibling `worker_lifecycle_test.rs` file. + let (task_tx, mut task_rx) = mpsc::unbounded_channel(); + state.replication_workers.insert( + 2, + super::ReplicationWorkerHandle { + task_tx, + snapshot_failure_count: 0, + snapshot_next_retry_at: None, + }, + ); + + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel::(); + + // Batch 1: peer 2 starts in `Probe` with nothing in flight — must be dispatched. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "first batch must reach peer 2's worker — it had no outstanding request" + ); + + // Batch 2: peer 2's first request has not been acknowledged (handle_append_result was never + // called), so it is still awaiting a response. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_err(), + "peer 2 already has an unacknowledged request in flight — a Probe-state peer must not \ + receive a second AppendEntries until the first is acked. See \ + 446-expert-q-probe-backpressure-fix-8020.md for the etcd/raft and openraft references." + ); +} + +/// Shared setup for the gate open/close tests below: a two-peer cluster with peer 2's worker +/// channel handed to the test, so dispatch is observable synchronously without a real transport. +async fn setup_gate_harness( + path: &str +) -> ( + crate::raft_context::RaftContext, + LeaderState, + mpsc::UnboundedReceiver, + mpsc::UnboundedSender, +) { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut ctx = mock_raft_context(path, graceful_rx, None); + ctx.membership = Arc::new(two_peer_membership()); + + let mut raft_log = MockRaftLog::new(); + raft_log.expect_last_entry_id().returning(|| 0); + raft_log.expect_flush().returning(|| Ok(())); + raft_log.expect_save_hard_state().returning(|_| Ok(())); + ctx.storage.raft_log = Arc::new(raft_log); + + let mut state = LeaderState::::new(1, ctx.node_config.clone()); + state.init_cluster_metadata(&ctx.membership).await.unwrap(); + + let (task_tx, task_rx) = mpsc::unbounded_channel(); + state.replication_workers.insert( + 2, + super::ReplicationWorkerHandle { + task_tx, + snapshot_failure_count: 0, + snapshot_next_retry_at: None, + }, + ); + + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel::(); + (ctx, state, task_rx, internal_event_tx) +} + +/// An unparseable response (no `result` variant) must still reopen the gate: the request is no +/// longer in flight, even though the leader learned nothing usable from it. Latching here would +/// freeze the peer exactly like the reject case. +#[tokio::test] +#[traced_test] +async fn test_probe_peer_dispatches_next_probe_after_unparseable_response() { + let (mut ctx, mut state, mut task_rx, internal_event_tx) = + setup_gate_harness("/tmp/test_probe_peer_dispatches_next_probe_after_unparseable_response") + .await; + + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(|_, _, _, _, _| { + Ok(crate::PrepareResult { + append_requests: vec![(2, stub_probe_request(), 1)], + snapshot_targets: vec![], + }) + }); + + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!(task_rx.try_recv().is_ok(), "first probe must be dispatched"); + + state + .handle_append_result( + 2, + Ok(AppendEntriesResponse { + node_id: 2, + term: 1, + result: None, + }), + &ctx, + &internal_event_tx, + ) + .await + .unwrap(); + + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "an unparseable response still resolves the outstanding probe — the leader must re-probe" + ); +} + +/// An empty (heartbeat) dispatch must NOT set the gate, otherwise a heartbeat would occupy the +/// "probe in flight" slot and block the next real probe. +#[tokio::test] +#[traced_test] +async fn test_heartbeat_does_not_latch_gate() { + let (mut ctx, mut state, mut task_rx, internal_event_tx) = + setup_gate_harness("/tmp/test_heartbeat_does_not_latch_gate").await; + + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(move |_, _, _, _, _| { + let n = calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let req = if n == 0 { + stub_request() + } else { + stub_probe_request() + }; + Ok(crate::PrepareResult { + append_requests: vec![(2, req, 1)], + snapshot_targets: vec![], + }) + }); + + // Batch 1 is an empty heartbeat — it must be dispatched but leave the gate open. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!(task_rx.try_recv().is_ok(), "heartbeat must be dispatched"); + + // Batch 2 is a real probe — it must not be blocked by a latch the heartbeat never set. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "an empty heartbeat must not latch the gate — the following probe must go out" + ); +} + +/// While a non-empty probe is in flight, an empty heartbeat must still be dispatched: the gate +/// throttles probes only, never heartbeats — this is the liveness backstop that unfreezes a peer +/// whose probe response was lost or unparseable. +#[tokio::test] +#[traced_test] +async fn test_heartbeat_bypasses_gate_while_probe_in_flight() { + let (mut ctx, mut state, mut task_rx, internal_event_tx) = + setup_gate_harness("/tmp/test_heartbeat_bypasses_gate_while_probe_in_flight").await; + + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(move |_, _, _, _, _| { + let n = calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let req = if n == 0 { + stub_probe_request() + } else { + stub_request() + }; + Ok(crate::PrepareResult { + append_requests: vec![(2, req, 1)], + snapshot_targets: vec![], + }) + }); + + // Batch 1 is a non-empty probe — it latches the gate. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!(task_rx.try_recv().is_ok(), "first probe must be dispatched"); + + // Batch 2 is an empty heartbeat — it must bypass the gate and still be dispatched. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "a heartbeat must bypass the probe gate — it is the liveness backstop, not a probe" + ); +} + +/// A stale-term response belongs to an older request, not the outstanding probe, so it must NOT +/// reopen the gate. Liveness is instead recovered by the heartbeat backstop (see the bypass test), +/// mirroring etcd's `MaybeUpdate(n <= Match)` early-return-without-resume. +#[tokio::test] +#[traced_test] +async fn test_stale_term_response_keeps_gate_latched() { + let (mut ctx, mut state, mut task_rx, internal_event_tx) = + setup_gate_harness("/tmp/test_stale_term_response_keeps_gate_latched").await; + + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(|_, _, _, _, _| { + Ok(crate::PrepareResult { + append_requests: vec![(2, stub_probe_request(), 1)], + snapshot_targets: vec![], + }) + }); + + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!(task_rx.try_recv().is_ok(), "first probe must be dispatched"); + + // term 0 < leader_term 1 → stale, ignored without clearing the gate. + state + .handle_append_result( + 2, + Ok(AppendEntriesResponse { + node_id: 2, + term: 0, + result: None, + }), + &ctx, + &internal_event_tx, + ) + .await + .unwrap(); + + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_err(), + "a stale-term response must not reopen the gate — the outstanding probe is still in flight" + ); +} diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index a2f31812..f4d5c91b 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -429,7 +429,7 @@ async fn test_worker_forwards_any_append_task_it_is_given_without_inspecting_sta // Bypasses process_batch/Phase 5 on purpose — see doc comment above. state.send_to_worker_or_spawn( 2, - super::ReplicationTask::Append(stub_append_request()), + super::ReplicationTask::Append(stub_append_request(), tokio::time::Instant::now()), super::ReplicationWorkerConfig { transport: Arc::new(transport), membership: ctx.membership.clone(), diff --git a/d-engine-core/src/raft_role/mod.rs b/d-engine-core/src/raft_role/mod.rs index 3a570c87..cea6639e 100644 --- a/d-engine-core/src/raft_role/mod.rs +++ b/d-engine-core/src/raft_role/mod.rs @@ -50,7 +50,6 @@ use super::InternalEvent; use super::RaftContext; use crate::Result; use crate::TypeConfig; -use crate::role_state::PeerReplicationState; /// The role state focuses solely on its own logic /// and does not directly manipulate the underlying storage or network. @@ -418,28 +417,6 @@ impl RaftRole { self.state_mut().init_peers_next_index_and_match_index(last_entry_id, peer_ids) } - /// Reset `next_index[peer] = match_index[peer] + 1` after a bidi stream disconnect. - /// Ensures the next heartbeat re-sends any unACKed in-flight entries. - pub(crate) fn handle_peer_stream_error( - &mut self, - peer_id: u32, - ) { - // The bidi stream only carries AppendEntries. While this peer is in Snapshot - // state, an error on this stream says nothing about the independent - // connection the snapshot transfer runs on, so it has no authority to act - // (mirrors etcd raft.go MsgUnreachable: only BecomeProbe() when StateReplicate). - if self.state().peer_replication_state(peer_id) == PeerReplicationState::Snapshot { - return; - } - let match_idx = self.state().match_index(peer_id).unwrap_or(0); - let _ = self.state_mut().update_next_index(peer_id, match_idx + 1); - - // #436: stream is down, we don't know what (if anything) the peer received — - // stop trusting speculative advance (etcd: BecomeProbe on MsgUnreachable). - self.state_mut() - .set_peer_replication_state(peer_id, PeerReplicationState::Probe); - } - pub(crate) async fn handle_zombie_detected( &mut self, node_id: u32, diff --git a/d-engine-core/src/raft_role/role_state.rs b/d-engine-core/src/raft_role/role_state.rs index df9e86ea..8b542503 100644 --- a/d-engine-core/src/raft_role/role_state.rs +++ b/d-engine-core/src/raft_role/role_state.rs @@ -641,10 +641,18 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { // My term might be updated, has to fetch it again let my_term = self.current_term(); + // `state_snapshot` was captured at the top of `handle_inbound_event`, before + // `commit_hard_state` above may have advanced our term. Patch it here so the + // AppendEntriesResponse reports the real, just-updated term — not the stale + // snapshot. + let state_snapshot = StateSnapshot { + current_term: my_term, + ..state_snapshot.clone() + }; // Handle replication request match ctx .replication_handler() - .handle_append_entries(append_entries_request, state_snapshot, ctx.raft_log()) + .handle_append_entries(append_entries_request, &state_snapshot, ctx.raft_log()) .await { Ok(AppendResponseWithUpdates { @@ -1073,21 +1081,6 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { .into()) } - fn peer_replication_state( - &self, - _node_id: u32, - ) -> PeerReplicationState { - // Default: unknown peer, be conservative. Also the default for non-leader roles. - PeerReplicationState::Probe - } - - fn set_peer_replication_state( - &mut self, - _node_id: u32, - _state: PeerReplicationState, - ) { - } - /// The withheld-ACK queue, for the roles that keep one (Follower, Learner). /// `None` for Candidate and Leader. Carried across a Follower<->Learner /// transition by `RaftRole::take_pending_acks` / `restore_pending_acks` (#446). diff --git a/d-engine-core/src/raft_test/raft_comprehensive_tests.rs b/d-engine-core/src/raft_test/raft_comprehensive_tests.rs index 6f169270..5062a8c5 100644 --- a/d-engine-core/src/raft_test/raft_comprehensive_tests.rs +++ b/d-engine-core/src/raft_test/raft_comprehensive_tests.rs @@ -1437,9 +1437,10 @@ async fn test_snapshot_push_completed_uses_snapshot_boundary_not_leader_tip() { let current_term = raft.current_term(); // Establish an active snapshot transfer — the handler seeds next_index only for a // peer that is actually mid-snapshot. - raft.role - .state_mut() - .set_peer_replication_state(peer_id, crate::role_state::PeerReplicationState::Snapshot); + let crate::RaftRole::Leader(leader) = &mut raft.role else { + panic!("expected Leader role after BecomeLeader"); + }; + leader.set_peer_replication_state(peer_id, crate::role_state::PeerReplicationState::Snapshot); raft.handle_internal_event(InternalEvent::SnapshotPushCompleted { peer_id, success: true, @@ -1660,17 +1661,24 @@ async fn test_peer_stream_error_does_not_touch_peer_in_snapshot_state() { raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); let peer_id = 42; - raft.role - .state_mut() - .set_peer_replication_state(peer_id, crate::role_state::PeerReplicationState::Snapshot); + { + let crate::RaftRole::Leader(leader) = &mut raft.role else { + panic!("expected Leader role after BecomeLeader"); + }; + leader + .set_peer_replication_state(peer_id, crate::role_state::PeerReplicationState::Snapshot); + } let next_index_before = raft.role.state().next_index(peer_id); raft.handle_internal_event(InternalEvent::PeerStreamError { peer_id }) .await .unwrap(); + let crate::RaftRole::Leader(leader) = &raft.role else { + panic!("expected Leader role after BecomeLeader"); + }; assert_eq!( - raft.role.state().peer_replication_state(peer_id), + leader.peer_replication_state(peer_id), crate::role_state::PeerReplicationState::Snapshot, "a bidi stream error must not downgrade a peer that is mid-snapshot-transfer" ); @@ -1696,16 +1704,25 @@ async fn test_peer_stream_error_downgrades_non_snapshot_peer_to_probe() { raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); let peer_id = 42; - raft.role - .state_mut() - .set_peer_replication_state(peer_id, crate::role_state::PeerReplicationState::Replicate); + { + let crate::RaftRole::Leader(leader) = &mut raft.role else { + panic!("expected Leader role after BecomeLeader"); + }; + leader.set_peer_replication_state( + peer_id, + crate::role_state::PeerReplicationState::Replicate, + ); + } raft.handle_internal_event(InternalEvent::PeerStreamError { peer_id }) .await .unwrap(); + let crate::RaftRole::Leader(leader) = &raft.role else { + panic!("expected Leader role after BecomeLeader"); + }; assert_eq!( - raft.role.state().peer_replication_state(peer_id), + leader.peer_replication_state(peer_id), crate::role_state::PeerReplicationState::Probe, "a bidi stream error for a non-snapshotting peer must still downgrade it to Probe" ); diff --git a/d-engine-core/src/state_machine_handler/worker_test.rs b/d-engine-core/src/state_machine_handler/worker_test.rs index 6c082abe..018e1806 100644 --- a/d-engine-core/src/state_machine_handler/worker_test.rs +++ b/d-engine-core/src/state_machine_handler/worker_test.rs @@ -1337,12 +1337,21 @@ async fn test_local_snapshot_ready_reports_operation_failed_when_superseded_clea let result = response_rx.await.unwrap(); - // Restore permissions before any assertion can panic and skip this — otherwise - // the tempdir is left behind, unremovable by the test harness's own cleanup. - let mut perms = std::fs::metadata(&dir_path).unwrap().permissions(); - perms.set_mode(0o700); - std::fs::set_permissions(&dir_path, perms).unwrap(); - std::fs::remove_dir_all(&dir_path).unwrap(); + // The tempdir might already be gone: `OwnedSnapshotDir::drop`'s detached cleanup + // thread (command.rs) races this teardown and, under load, can win — that's a + // benign outcome (goal is just "no leftover dir"), not a test failure. + if let Ok(meta) = std::fs::metadata(&dir_path) { + let mut perms = meta.permissions(); + perms.set_mode(0o700); + std::fs::set_permissions(&dir_path, perms).unwrap(); + if let Err(e) = std::fs::remove_dir_all(&dir_path) { + assert_eq!( + e.kind(), + std::io::ErrorKind::NotFound, + "unexpected teardown error: {e}" + ); + } + } assert!( matches!( diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index 384642cc..16ec557d 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -516,22 +516,16 @@ where new_entries: Vec, ) -> Result> { let _timer = ScopedTimer::new("filter_out_conflicts_and_append"); - // prev_log_index == 0 means the leader wants the follower to start from scratch - // (e.g. new follower joining, or follower log fully diverged). Reset and replace. - if prev_log_index == 0 && prev_log_term == 0 { - self.reset().await?; - self.append_entries(new_entries.clone()).await?; - return Ok(new_entries.last().map(|e| LogId { - term: e.term, - index: e.index, - })); - } - // Check log consistency: use entry_term() so purge-boundary entries - // (entries removed from the SkipMap but recorded in last_purged_index/term) - // are still recognised as valid prev_log positions after snapshot install. - if self.entry_term(prev_log_index) != Some(prev_log_term) { - return Ok(self.last_log_id()); + // prev_log_index==0 has no real entry to compare against, not a reset signal + let is_virtual_log_start = prev_log_index == 0 && prev_log_term == 0; + if !is_virtual_log_start { + // Check log consistency: use entry_term() so purge-boundary entries + // (entries removed from the SkipMap but recorded in last_purged_index/term) + // are still recognised as valid prev_log positions after snapshot install. + if self.entry_term(prev_log_index) != Some(prev_log_term) { + return Ok(self.last_log_id()); + } } let last_current_index = self.last_entry_id(); @@ -1712,3 +1706,7 @@ mod content_validated_watermark_test; #[cfg(test)] #[path = "buffered_raft_log_test/worker_test.rs"] mod worker_test; + +#[cfg(test)] +#[path = "buffered_raft_log_test/prev_log_index_zero_idempotency_test.rs"] +mod prev_log_index_zero_idempotency_test; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/prev_log_index_zero_idempotency_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/prev_log_index_zero_idempotency_test.rs new file mode 100644 index 00000000..8a277552 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/prev_log_index_zero_idempotency_test.rs @@ -0,0 +1,245 @@ +//! Tests for `filter_out_conflicts_and_append` when `prev_log_index == 0`. +//! +//! Background: `prev_log_index == 0` is Raft's sentinel for "nothing before the start of the +//! log" — Rule 2 (term-at-prev-index check) is trivially satisfied because there is no real +//! entry 0 to compare. That does NOT license skipping Rules 3/4: the receiver must still +//! compare incoming entries against whatever it already has, starting at index 1, and only +//! touch the entries that actually conflict (differing term at the same index). A batch that +//! fully matches existing content must be a no-op — this is exactly what +//! `pipeline_overlap_test.rs` already proves for `prev_log_index > 0`. +//! +//! The current implementation special-cases `prev_log_index == 0` to unconditionally +//! `reset()` (wipe the whole log, `durable_index` included) before re-appending — regardless +//! of whether the incoming entries are a pure duplicate of what's already durably stored. A +//! leader that resends a `prev_log_index=0` probe (no backpressure, a retry, a reconnect) before +//! learning the follower already caught up will repeatedly destroy real, already-durable +//! progress. These tests are RED until `prev_log_index == 0` is folded into the same +//! overlap/conflict comparison used for `prev_log_index > 0`. + +use crate::FlushPolicy; +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use d_engine_proto::common::Entry; +use std::time::Duration; + +fn ctx(name: &str) -> BufferedRaftLogTestContext { + BufferedRaftLogTestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, + name, + ) +} + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// A genuinely fresh follower (no prior `append_entries` calls at all) receiving its very +/// first `prev_log_index=0` probe must accept and append normally. +/// +/// # Why this needs its own test, not just implicit coverage +/// The other tests in this file all pre-populate the log before calling +/// `filter_out_conflicts_and_append`, so none of them exercise the case the +/// `is_virtual_log_start` guard exists to protect: without it, `entry_term(0)` returns `None` +/// unconditionally (there is no real entry 0 to look up), so `entry_term(0) != Some(0)` would +/// be true and this — the single most basic, legitimate case — would be wrongly rejected as a +/// conflict. This test pins that guard directly. +/// +/// # Expected (holds both before and after the fix — this is a regression guard for the +/// `is_virtual_log_start` skip, not a RED/GREEN discriminator for the reset() removal) +#[tokio::test] +async fn test_filter_conflicts_zero_prev_on_genuinely_empty_log_appends_all() { + let ctx = ctx("zero_prev_genuinely_empty_log_appends_all"); + assert_eq!( + ctx.raft_log.last_entry_id(), + 0, + "precondition: log must be untouched" + ); + + // Act: the very first AppendEntries this follower ever receives. + let result = ctx + .raft_log + .filter_out_conflicts_and_append( + 0, + 0, + vec![ + entry(1, 1), + entry(2, 1), + entry(3, 1), + entry(4, 1), + entry(5, 1), + ], + ) + .await + .unwrap(); + + assert_eq!(result.unwrap().index, 5); + assert_eq!(ctx.raft_log.last_entry_id(), 5); + for i in 1u64..=5 { + assert_eq!(ctx.raft_log.entry(i).unwrap().unwrap().term, 1); + } +} + +/// A leader resending `prev_log_index=0` with entries the follower already has — durably — +/// must be a no-op. This is the exact T4/T5 scenario from the #446 investigation: the +/// follower's first response to a `prev_log_index=0` probe is withheld pending its own +/// `durable_index` catching up (RPO=0); if the leader resends the identical probe before that +/// withheld ACK is released, the follower must not throw away the progress it already made. +/// +/// # Expected (RED until fixed) +/// `durable_index()` and `last_entry_id()` stay at 5 — the duplicate probe changes nothing. +#[tokio::test] +async fn test_filter_conflicts_zero_prev_duplicate_resend_preserves_durable_index() { + let mut ctx = ctx("zero_prev_duplicate_preserves_durable_index"); + + // Arrange: follower already durably has [1..5], all term=1 — simulating a prior + // `prev_log_index=0` probe that succeeded and finished fsyncing. + for i in 1u64..=5 { + ctx.raft_log.append_entries(vec![entry(i, 1)]).await.unwrap(); + } + tokio::time::sleep(Duration::from_millis(50)).await; + ctx.drain_fsync_completions(); + assert_eq!( + ctx.raft_log.durable_index(), + 5, + "precondition: [1..5] must be durable" + ); + + // Act: leader resends the identical prev_log_index=0 probe — same entries, same terms. + // This is what happens when the leader's next_index[peer] never advanced (the leader + // hasn't processed a response yet), not a genuinely new peer. + let result = ctx + .raft_log + .filter_out_conflicts_and_append( + 0, + 0, + vec![ + entry(1, 1), + entry(2, 1), + entry(3, 1), + entry(4, 1), + entry(5, 1), + ], + ) + .await + .unwrap(); + + // Assert: no-op — durable progress must survive a duplicate zero-prev probe. + assert_eq!(result.unwrap().index, 5); + assert_eq!( + ctx.raft_log.last_entry_id(), + 5, + "last_entry_id must be unchanged" + ); + assert_eq!( + ctx.raft_log.durable_index(), + 5, + "a duplicate prev_log_index=0 resend must not regress durable_index — this is what \ + re-arms the withheld-ACK deadlock (RPO=0 withhold never resolves once durable_index \ + is wiped out from under it)" + ); + for i in 1u64..=5 { + assert_eq!( + ctx.raft_log.entry(i).unwrap().unwrap().term, + 1, + "index={i} must not be touched by a duplicate zero-prev probe" + ); + } +} + +/// A `prev_log_index=0` batch that overlaps existing content but also carries genuinely new +/// entries beyond it must append only the new tail — mirrors +/// `pipeline_overlap_test::test_filter_conflicts_pipeline_overlap_no_truncation`, anchored at +/// prev=0 instead of prev>0, to prove the same comparison logic applies uniformly regardless +/// of which branch computed `prev_log_index`. +/// +/// # Expected (RED until fixed) +/// [1..5] untouched, [6,7] appended. +#[tokio::test] +async fn test_filter_conflicts_zero_prev_overlap_appends_only_new_tail() { + let ctx = ctx("zero_prev_overlap_appends_only_new_tail"); + + // Arrange: follower has [1..5], term=1 (not necessarily durable yet — overlap detection + // must work purely off in-memory content, independent of durability). + for i in 1u64..=5 { + ctx.raft_log.append_entries(vec![entry(i, 1)]).await.unwrap(); + } + assert_eq!(ctx.raft_log.last_entry_id(), 5); + + // Act: leader sends prev_log_index=0 with [1..7] — [1..5] match, [6,7] are new. + let new_entries: Vec<_> = (1u64..=7).map(|i| entry(i, 1)).collect(); + let result = ctx.raft_log.filter_out_conflicts_and_append(0, 0, new_entries).await.unwrap(); + + // Assert: existing [1..5] untouched, new tail [6,7] appended. + assert_eq!(result.unwrap().index, 7); + assert_eq!(ctx.raft_log.last_entry_id(), 7); + for i in 1u64..=5 { + assert_eq!( + ctx.raft_log.entry(i).unwrap().unwrap().term, + 1, + "index={i} was already present and must not be truncated" + ); + } + assert_eq!(ctx.raft_log.entry(6).unwrap().unwrap().term, 1); + assert_eq!(ctx.raft_log.entry(7).unwrap().unwrap().term, 1); +} + +/// A genuine conflict at index 1 (different term than what the follower already has) must +/// still truncate and replace — proves the fix is a precise reuse of the existing +/// overlap/conflict comparison, not "prev_log_index=0 always becomes a no-op." +/// +/// # Scenario +/// Follower has [1..5] term=1 (stale, from a since-superseded leader). A new leader with no +/// prior knowledge of this follower (or after a purge/snapshot boundary reset) sends +/// prev_log_index=0 with [1..3] all term=2 — a real conflict at index=1. +/// +/// # Expected (should already hold both before and after the fix — this is the control case) +/// [1..3] replaced with term=2; nothing beyond index=3 survives. +#[tokio::test] +async fn test_filter_conflicts_zero_prev_real_conflict_truncates_and_replaces() { + let ctx = ctx("zero_prev_real_conflict_truncates_and_replaces"); + + // Arrange: follower has [1..5], all term=1. + for i in 1u64..=5 { + ctx.raft_log.append_entries(vec![entry(i, 1)]).await.unwrap(); + } + assert_eq!(ctx.raft_log.last_entry_id(), 5); + + // Act: prev_log_index=0, entries=[1(t2), 2(t2), 3(t2)] — conflicts at index=1 immediately. + let result = ctx + .raft_log + .filter_out_conflicts_and_append(0, 0, vec![entry(1, 2), entry(2, 2), entry(3, 2)]) + .await + .unwrap(); + + // Assert: [1..3] replaced with term=2; stale [4,5] from the old leader must not survive. + assert_eq!(result.unwrap().index, 3); + assert_eq!( + ctx.raft_log.last_entry_id(), + 3, + "stale tail beyond the new leader's log must be gone" + ); + for i in 1u64..=3 { + assert_eq!( + ctx.raft_log.entry(i).unwrap().unwrap().term, + 2, + "index={i} must be term=2" + ); + } + assert!( + ctx.raft_log.entry(4).unwrap().is_none(), + "stale index=4 must not survive" + ); + assert!( + ctx.raft_log.entry(5).unwrap().is_none(), + "stale index=5 must not survive" + ); +} diff --git a/d-engine-core/src/utils/scoped_timer.rs b/d-engine-core/src/utils/scoped_timer.rs index e4c7d9a5..dbfa2f5a 100644 --- a/d-engine-core/src/utils/scoped_timer.rs +++ b/d-engine-core/src/utils/scoped_timer.rs @@ -19,5 +19,7 @@ impl Drop for ScopedTimer { fn drop(&mut self) { let elapsed = self.start.elapsed(); trace!(target: "timing", "[TIMING] {} took {} ms", self.name, elapsed.as_millis()); + metrics::histogram!("core.timing.scoped_duration_ms", "phase" => self.name) + .record(elapsed.as_secs_f64() * 1_000.0); } } diff --git a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs index 3e7b81c7..c007c105 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs @@ -81,10 +81,17 @@ mod filter_out_conflicts_and_append_performance_tests { "Duration {duration}ms exceeds max {max_duration_ms}ms for {idle_flush_interval_ms}ms interval" ); - // Verify correctness - assert!(log.entry(500).unwrap().is_none()); + // Verify correctness: index=501 term=1 already exists (populated above), so this + // prev_log_index=0 resend is a pure duplicate — must be a no-op, not a reset. + // See 446-expert-q-probe-backpressure-fix-8020.md. + assert_eq!( + log.last_entry_id(), + 1000, + "duplicate resend must not touch the log" + ); + assert!(log.entry(500).unwrap().is_some()); assert!(log.entry(501).unwrap().is_some()); - assert!(log.entry(502).unwrap().is_none()); + assert!(log.entry(502).unwrap().is_some()); } }