diff --git a/.dockerignore b/.dockerignore index 2a8dfc95..f20b5623 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ **/target/ .git/ -examples/ +examples/* +!examples/three-nodes-standalone +!examples/client-usage-standalone diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4e9f8766..aaa0692c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -61,6 +61,12 @@ --- +## AI Assistance + +- [ ] This PR was written in part with the assistance of generative AI. All ideas and architecture decisions are mine; I have fully reviewed all changes. + +--- + ## Reviewer Notes (Optional: anything reviewers should focus on) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9012493..3299062d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,18 @@ All notable changes to this project will be documented in this file. returns immediately, and entries arriving during an in-flight fsync are coalesced into the same physical disk flush. Storage-level group commit is restored without artificial batching windows. +- **πŸ›‘ Client-acknowledged writes could be lost on correlated power loss (#446)**: Raft commit quorum + counted the leader's own log contribution using its in-memory tail (`last_entry_id()`), not its + fsync-confirmed position (`durable_index()`) β€” a write could reach a majority-looking commit index, + and be acknowledged to the client, before enough replicas had actually synced it to disk. If those + nodes then lost power before their next fsync, the acknowledged write was gone. Fixed: leader quorum + calculation, follower `AppendEntries` ACK timing (a follower now withholds its response until its own + `durable_index` reaches the acknowledged entry), and single-voter clusters (previously exempted from + this class of fix, see #329) all gate on `durable_index`. RPO=0 for acknowledged writes is now a + mandatory invariant. Net effect: write acknowledgment latency now includes fsync time on a quorum of + replicas β€” see [Throughput Optimization Guide](./d-engine/src/docs/performance/throughput-optimization-guide.md) + for tuning `idle_flush_interval_ms`. + ### Changed - **MSRV raised to Rust 1.89**: The `data_dir` startup lock (prevents two node processes from @@ -65,6 +77,17 @@ All notable changes to this project will be documented in this file. - **`NodeBuilder` is no longer public** β€” use `EmbeddedEngine::start_custom`/`StandaloneEngine::run_custom` to plug in a custom storage engine or state machine. See [Migration Guide](./MIGRATION_GUIDE.md) for details. +- **⚠️ `[raft] ordered_channel_capacity` renamed to `max_pending_append_responses`** (#446): Follows the + gRPC `AppendEntries` forwarder rewrite (`FuturesUnordered`-based, no longer strict-FIFO) that shipped + alongside the durability fix above. Old field name is silently ignored, not an error β€” update existing + configs to the new name to keep the setting in effect. + +- **⚠️ `[raft.persistence] strategy` removed** (#446): `PersistenceStrategy` was a single-variant enum + (`MemFirst`) left over from #268; its only meaning now lives in whether an entry has reached + `durable_index`, which is no longer a configurable choice. Existing configs setting `strategy = + "MemFirst"` or `"DiskFirst"` are silently ignored, not an error β€” remove the field, `flush_policy` + is the only persistence knob now. + --- ## [v0.2.4] - 2026-05-23 diff --git a/Cargo.lock b/Cargo.lock index a411d838..5c74fa52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1830,9 +1830,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "log", "once_cell", @@ -1854,9 +1854,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", diff --git a/benches/embedded-bench/Makefile b/benches/embedded-bench/Makefile index 966619b3..281c685e 100644 --- a/benches/embedded-bench/Makefile +++ b/benches/embedded-bench/Makefile @@ -1,27 +1,25 @@ # Makefile for embedded-bench # Provides benchmark commands matching embedded-bench/reports/v0.2.0/report_v0.2.0_final.md -.PHONY: help build clean test-single-write test-high-conc-write test-linearizable-read test-lease-read test-eventual-read test-hot-key all-tests +.PHONY: help build clean clean-log-db test-single-write test-high-conc-write test-linearizable-read test-lease-read test-eventual-read test-hot-key all-tests \ + all-ram-tests ramdisk-create clean-ram-log-db ramdisk-release +# =============================== +# Global Variables +# =============================== BENCH_BIN := ./target/release/embedded-bench CONFIG_DIR := ./config +LOG_LEVEL ?= warn # Node selection (default: n1) NODE ?= n1 CONFIG_PATH := $(CONFIG_DIR)/$(NODE).toml SINGLE_NODE_CONFIG_PATH := $(CONFIG_DIR)/single_node.toml -DATA_DIR := ./data/$(NODE) -SINGLE_NODE_DATA_DIR := ./data/single-node # Metrics port per node β€” same scheme as examples/three-nodes-standalone # (8081/8082/8083), reused because the two setups never run concurrently. METRICS_PORT := $(if $(filter n2,$(NODE)),8082,$(if $(filter n3,$(NODE)),8083,8081)) -# =============================== -# Global Variables -# =============================== -LOG_LEVEL ?= warn - # Common parameters (matching Standalone tests) KEY_SIZE := 8 VALUE_SIZE := 256 @@ -35,6 +33,25 @@ CLIENTS ?= 1000 VERIFY ?= false VERIFY_FLAG := $(if $(filter true,$(VERIFY)),--verify-write,) +# RAM disk (macOS only, optional) β€” isolates this node's storage on its own +# independent RAM disk volume, to strip physical-disk latency out of a +# benchmark. Not a general fix for unrelated flakiness β€” see +# tickets/milestones/v0.2.5/446-perf-batching-measurement-2026-09-13.md. +# One-shot: `make all-ram-tests`. Manual/single-node: add RAMDISK=true to +# any test-* target (needs the other two nodes already running for quorum). +RAMDISK ?= false +RAMDISK_SIZE_MB ?= 1024 +RAMDISK_SECTORS := $(shell echo $$(( $(RAMDISK_SIZE_MB) * 2048 ))) +RAMDISK_VOLUMES := RAMDisk1 RAMDisk2 RAMDisk3 +RAMDISK_INDEX := $(if $(filter n2,$(NODE)),2,$(if $(filter n3,$(NODE)),3,1)) + +ifeq ($(RAMDISK),true) +DATA_DIR := /Volumes/RAMDisk$(RAMDISK_INDEX)/$(NODE) +SINGLE_NODE_DATA_DIR := /Volumes/RAMDisk1/single-node +else +DATA_DIR := ./data/$(NODE) +SINGLE_NODE_DATA_DIR := ./data/single-node +endif # On macOS with Homebrew: auto-detect compression lib paths to skip bundled C++ # compilation of RocksDB dependencies, which fails under macOS 26 + Xcode 26 @@ -58,7 +75,6 @@ ifneq ($(ZSTD_PREFIX),) BREW_ROCKSDB_ENV += ZSTD_LIB_DIR=$(ZSTD_PREFIX)/lib endif - help: @echo "Embedded-bench Makefile - Performance Testing" @echo "" @@ -83,6 +99,14 @@ help: @echo "Run All:" @echo " make all-tests Run all benchmark tests" @echo "" + @echo "RAM Disk Cluster (macOS only, optional β€” isolates disk latency):" + @echo " make all-ram-tests NODE=nX Create RAM disks, run full --batch suite on this node (like all-tests)" + @echo " Run in 3 terminals with NODE=n1/n2/n3 to form the cluster" + @echo " make ramdisk-create Create RAMDisk1/2/3 if not already mounted" + @echo " make clean-ram-log-db Wipe node data off the RAM disks (keeps them mounted)" + @echo " make ramdisk-release Unmount RAMDisk1/2/3, freeing the memory" + @echo " Add RAMDISK=true to any single test-* target (needs the other 2 nodes already running)" + @echo "" @echo "Examples:" @echo " make test-linearizable-read # Run on n1 (default)" @echo " make test-linearizable-read NODE=n2 # Run on n2" @@ -109,6 +133,7 @@ clean-log-db: rm -rf ./logs/* rm -rf ./data/* rm -rf ./snapshots/* + # ============================================ # Write Performance Tests # ============================================ @@ -262,4 +287,34 @@ all-tests: build put @echo "" @echo "Compare results with Standalone mode:" - @echo " Standalone report: ../../benches/standalone-bench/reports/v0.2.2/report_v0.2.2.md" + +# ============================================ +# RAM Disk Cluster (macOS only, optional) +# ============================================ +# Isolates each node's storage on its own independent RAM disk volume β€” use +# this to strip physical-disk latency out of a benchmark (e.g. to study +# fsync/scheduling behavior in isolation), not as a general fix for +# unrelated flakiness. See tickets/milestones/v0.2.5/446-perf-batching-measurement-2026-09-13.md. + +# Ensures the RAM disks exist, then runs the full --batch suite for this +# node on its own RAM disk volume β€” same scope/shape as `make all-tests +# NODE=nX`, just on RAM disk. This is single-node, like all-tests: run it in +# 3 separate terminals with NODE=n1/n2/n3 to form the cluster, e.g. +# make all-ram-tests NODE=n2 CLIENTS=100 +all-ram-tests: ramdisk-create + $(MAKE) all-tests RAMDISK=true NODE=$(NODE) CLIENTS=$(CLIENTS) + +# Idempotent β€” mounts any of RAMDisk1/2/3 that aren't already present. +ramdisk-create: + @[ "$$(uname)" = "Darwin" ] || { echo "RAM disk targets need macOS."; exit 1; } + @for v in $(RAMDISK_VOLUMES); do \ + [ -d "/Volumes/$$v" ] || diskutil erasevolume HFS+ $$v `hdiutil attach -nomount ram://$(RAMDISK_SECTORS)` >/dev/null; \ + done + +# Wipes node data off the RAM disks; keeps the volumes mounted. +clean-ram-log-db: + @for v in $(RAMDISK_VOLUMES); do rm -rf /Volumes/$$v/*; done + +# Unmounts the RAM disks entirely, releasing the memory back to the OS. +ramdisk-release: + @for v in $(RAMDISK_VOLUMES); do diskutil eject /Volumes/$$v 2>/dev/null || true; done diff --git a/benches/embedded-bench/config/n1.toml b/benches/embedded-bench/config/n1.toml index 52055b93..da9af610 100644 --- a/benches/embedded-bench/config/n1.toml +++ b/benches/embedded-bench/config/n1.toml @@ -16,11 +16,7 @@ max_drain = 1024 max_batch_size = 200 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) -max_buffered_entries = 10000 [raft.metrics] enable_backpressure = false diff --git a/benches/embedded-bench/config/n2.toml b/benches/embedded-bench/config/n2.toml index 56c9cf13..1e10d048 100644 --- a/benches/embedded-bench/config/n2.toml +++ b/benches/embedded-bench/config/n2.toml @@ -16,11 +16,7 @@ max_drain = 1024 max_batch_size = 200 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) -max_buffered_entries = 10000 [raft.metrics] enable_backpressure = false diff --git a/benches/embedded-bench/config/n3.toml b/benches/embedded-bench/config/n3.toml index 701a1551..96f32af8 100644 --- a/benches/embedded-bench/config/n3.toml +++ b/benches/embedded-bench/config/n3.toml @@ -16,11 +16,7 @@ max_drain = 1024 max_batch_size = 200 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) -max_buffered_entries = 10000 [raft.metrics] enable_backpressure = false diff --git a/benches/embedded-bench/src/main.rs b/benches/embedded-bench/src/main.rs index c0d699d3..d14482e3 100644 --- a/benches/embedded-bench/src/main.rs +++ b/benches/embedded-bench/src/main.rs @@ -171,7 +171,7 @@ fn generate_value(size: usize) -> Vec { (0..size).map(|_| rng.random()).collect() } -#[tokio::main] +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] async fn main() { // Initialize logging tracing_subscriber::fmt() @@ -256,6 +256,7 @@ async fn run_benchmark_task( let stats = Arc::new(BenchmarkStats::new()); let key_counter = Arc::new(AtomicU64::new(0)); + let failed_count = Arc::new(AtomicU64::new(0)); let start_time = Instant::now(); let mut handles = Vec::with_capacity(clients); @@ -264,6 +265,7 @@ async fn run_benchmark_task( let engine = engine.clone(); let stats = stats.clone(); let key_counter = key_counter.clone(); + let failed_count = failed_count.clone(); let command = command.clone(); let handle = tokio::spawn(async move { @@ -289,7 +291,15 @@ async fn run_benchmark_task( } } } - Err(_) => continue, + Err(e) => { + let n = failed_count.fetch_add(1, Ordering::Relaxed); + if n < 5 { + eprintln!("Put failed: {e:?}"); + } else if n == 5 { + eprintln!("Put failed: (further failures suppressed)"); + } + continue; + } } } Commands::Get { consistency } => { @@ -327,6 +337,10 @@ async fn run_benchmark_task( futures::future::join_all(handles).await; stats.summary(start_time.elapsed()); + let failed = failed_count.load(Ordering::Relaxed); + if failed > 0 { + println!("Failed requests: {failed}"); + } } /// Run all benchmark tests in batch mode @@ -563,6 +577,7 @@ async fn run_local_benchmark(cli: Cli) { let stats = Arc::new(BenchmarkStats::new()); let key_counter = Arc::new(AtomicU64::new(0)); + let failed_count = Arc::new(AtomicU64::new(0)); let start_time = Instant::now(); let mut handles = Vec::with_capacity(cli.clients); @@ -571,6 +586,7 @@ async fn run_local_benchmark(cli: Cli) { let engine = engine.clone(); let stats = stats.clone(); let key_counter = key_counter.clone(); + let failed_count = failed_count.clone(); let cli = cli.clone(); let handle = tokio::spawn(async move { @@ -625,8 +641,13 @@ async fn run_local_benchmark(cli: Cli) { } } } - Err(_) => { - // Write failed - skip recording + Err(e) => { + let n = failed_count.fetch_add(1, Ordering::Relaxed); + if n < 5 { + eprintln!("Put failed: {e:?}"); + } else if n == 5 { + eprintln!("Put failed: (further failures suppressed)"); + } continue; } } @@ -676,6 +697,10 @@ async fn run_local_benchmark(cli: Cli) { futures::future::join_all(handles).await; stats.summary(start_time.elapsed()); + let failed = failed_count.load(Ordering::Relaxed); + if failed > 0 { + println!("Failed requests: {failed}"); + } println!("\nBenchmark completed. Press Ctrl+C to shutdown."); let _ = shutdown_rx.changed().await; diff --git a/benches/reports/v0.2.5/bench_report_v0.2.5.md b/benches/reports/v0.2.5/bench_report_v0.2.5.md index 8e2bcb69..a6e08570 100644 --- a/benches/reports/v0.2.5/bench_report_v0.2.5.md +++ b/benches/reports/v0.2.5/bench_report_v0.2.5.md @@ -59,7 +59,7 @@ _(v0.2.5: 6-round average; v0.2.4: 4-round average; v0.2.3: 4-round average (Lea _(v0.2.5: 5-round average; v0.2.4: 5-round average; v0.2.3: 5-round average. All manually collected. 2026-07-12: 4-round average (conns=200, clients=200, Docker monitoring stack stopped).)_ | **Scenario** | **Metric** | **v0.2.3** | **v0.2.4** | **v0.2.5** | **Ξ” (v0.2.4β†’v0.2.5)** | **0712** | **Ξ” (v0.2.5β†’0712)** | -| ------------------- | ----------- | ------------ | ------------ | ------------ | --------------------- | ------------ | ------------------- | +| ------------------- | ----------- | ------------ | ------------ | ------------ | --------------------- | ------------- | ------------------- | | Single Client Write | Throughput | 6,421 ops/s | 5,245 ops/s | 5,234 ops/s | stable | 9,450 ops/s | **+80.5%** βœ… | | | Avg Latency | 0.155 ms | 0.190 ms | 0.190 ms | stable | 0.105 ms | **-44.6%** βœ… | | | p99 Latency | 0.200 ms | 0.235 ms | 0.237 ms | stable | 0.223 ms | -6.0% β†’ | @@ -214,7 +214,6 @@ read_actor_channel_capacity = 10240 read_actor_max_drain = 2000 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [raft.batching] diff --git a/d-engine-core/src/config/raft.rs b/d-engine-core/src/config/raft.rs index 78e3a333..b9b2e051 100644 --- a/d-engine-core/src/config/raft.rs +++ b/d-engine-core/src/config/raft.rs @@ -79,11 +79,15 @@ pub struct RaftConfig { #[serde(default = "default_cmd_channel_capacity")] pub cmd_channel_capacity: usize, - /// Ordered channel capacity for stream_append_entries ordering - /// Controls buffering of response receivers in FIFO order - /// Default value is set via default_ordered_channel_capacity() function - #[serde(default = "default_ordered_channel_capacity")] - pub ordered_channel_capacity: usize, + /// Max in-flight AppendEntries requests on `stream_append_entries` that can be + /// dispatched to the Raft loop and awaiting their response at once. Once this many + /// are pending, the stream stops reading new requests until one completes β€” this + /// bounds memory/task growth if this node's own durable_index stalls (RPO=0, #446). + /// Also used directly as the output channel's buffer size, since completed + /// responses can never outnumber in-flight requests. + /// Default value is set via default_max_pending_append_responses() function + #[serde(default = "default_max_pending_append_responses")] + pub max_pending_append_responses: usize, /// ReadActor configuration β€” tuning for the dedicated Eventual/LeaseRead fast path. #[serde(default)] @@ -141,7 +145,7 @@ impl Default for RaftConfig { auto_join: AutoJoinConfig::default(), snapshot_rpc_timeout_ms: default_snapshot_rpc_timeout_ms(), cmd_channel_capacity: default_cmd_channel_capacity(), - ordered_channel_capacity: default_ordered_channel_capacity(), + max_pending_append_responses: default_max_pending_append_responses(), read_actor: ReadActorConfig::default(), read_consistency: ReadConsistencyConfig::default(), backpressure: BackpressureConfig::default(), @@ -201,7 +205,7 @@ fn default_cmd_channel_capacity() -> usize { 1024 } -fn default_ordered_channel_capacity() -> usize { +fn default_max_pending_append_responses() -> usize { 1024 } @@ -817,38 +821,13 @@ impl Default for PromotionConfig { fn default_stale_learner_threshold() -> Duration { Duration::from_secs(300) } -/// Defines how Raft log entries are persisted and accessed. -/// -/// All strategies use a configurable [`FlushPolicy`] to control when memory contents -/// are flushed to disk, affecting write latency and durability guarantees. -/// -/// **Note:** Both strategies now fully load all log entries from disk into memory at startup. -/// The in-memory `SkipMap` serves as the primary data structure for reads in all modes. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub enum PersistenceStrategy { - /// Memory-first persistence strategy. - /// - /// - **Write path**: On append, the log entry is first written to the in-memory `SkipMap` and - /// acknowledged immediately. Disk persistence happens asynchronously in the background, - /// governed by [`FlushPolicy`]. - /// - /// - **Read path**: Reads are always served from the in-memory `SkipMap`. - /// - /// - **Startup behavior**: All log entries are loaded from disk into memory at startup. - /// - MemFirst, -} -/// Controls when in-memory logs should be flushed to disk. -/// -/// Flush is triggered by whichever comes first: -/// - An explicit `flush()` call (immediate, no wait). -/// - `append_entries` calls `write_notify.notify_one()` for an immediate persist+fsync. -/// - The idle safety-net timer fires after `idle_flush_interval_ms` of inactivity. +/// Interval (ms) between periodic fsyncs on the IO thread. Must be > 0. /// -/// `idle_flush_interval_ms` must be greater than zero. It only fires when no -/// writes have arrived for that duration; normal-path latency is determined by -/// the fsync execution time (drain-then-fsync architecture). +/// Writes fsync on their own path (`flush()`, `append_entries` β†’ +/// `IOTask::Persist`). This timer only re-fsyncs `(durable_index, +/// memory_max_index]` when the log is idle, so `durable_index` still advances +/// if a fsync-completion notification is lost. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub enum FlushPolicy { Batch { idle_flush_interval_ms: u64 }, @@ -857,14 +836,6 @@ pub enum FlushPolicy { /// Configuration parameters for log persistence behavior #[derive(Serialize, Deserialize, Clone, Debug)] pub struct PersistenceConfig { - /// Strategy for persisting Raft logs - /// - /// This controls the trade-off between durability guarantees and performance - /// characteristics. The choice impacts both write throughput and recovery - /// behavior after node failures. - #[serde(default = "default_persistence_strategy")] - pub strategy: PersistenceStrategy, - /// Flush policy for asynchronous strategies /// /// This controls when log entries are flushed to disk. The choice impacts @@ -872,13 +843,6 @@ pub struct PersistenceConfig { #[serde(default = "default_flush_policy")] pub flush_policy: FlushPolicy, - /// Maximum number of in-memory log entries to buffer when using async strategies - /// - /// This acts as a safety valve to prevent memory exhaustion during periods of - /// high write throughput or when disk persistence is slow. - #[serde(default = "default_max_buffered_entries")] - pub max_buffered_entries: usize, - /// Maximum time to wait, on shutdown, for an in-flight fsync task to finish /// before giving up. Bounds close() against a stuck/slow disk β€” the task /// itself is not cancelled, it keeps running in the background regardless. @@ -886,11 +850,6 @@ pub struct PersistenceConfig { pub shutdown_timeout_ms: u64, } -/// Default persistence strategy (optimized for balanced workloads) -fn default_persistence_strategy() -> PersistenceStrategy { - PersistenceStrategy::MemFirst -} - /// Default flush policy for asynchronous strategies /// /// This controls when log entries are flushed to disk. The choice impacts @@ -901,11 +860,6 @@ fn default_flush_policy() -> FlushPolicy { } } -/// Default maximum buffered log entries -fn default_max_buffered_entries() -> usize { - 10_000 -} - fn default_shutdown_timeout_ms() -> u64 { 5_000 } @@ -933,9 +887,7 @@ impl PersistenceConfig { impl Default for PersistenceConfig { fn default() -> Self { Self { - strategy: default_persistence_strategy(), flush_policy: default_flush_policy(), - max_buffered_entries: default_max_buffered_entries(), shutdown_timeout_ms: default_shutdown_timeout_ms(), } } diff --git a/d-engine-core/src/event.rs b/d-engine-core/src/event.rs index 3f35dbf3..cc4ec4f6 100644 --- a/d-engine-core/src/event.rs +++ b/d-engine-core/src/event.rs @@ -71,6 +71,11 @@ pub enum InternalEvent { durable_index: u64, }, + /// Raw fsync-completion mark β€” NOT yet validated. Consumer must call + /// `raft_log().try_advance_durable_index(mark)`, which re-checks the entry's + /// term before advancing `durable_index`. + FsyncCompleted(LogId), + /// AppendEntries result from a per-follower ReplicationWorker back to the Raft loop. /// Leader processes this in handle_append_result: updates match_index, re-calculates commit, /// and drains pending_client_writes when quorum is achieved. diff --git a/d-engine-core/src/lib.rs b/d-engine-core/src/lib.rs index 93333741..387324b2 100644 --- a/d-engine-core/src/lib.rs +++ b/d-engine-core/src/lib.rs @@ -173,6 +173,11 @@ pub(crate) fn if_higher_term_found( /// entries in the logs. If the logs have last entries with different terms, then the log with the /// later term is more up-to-date. If the logs end with the same term, then whichever log is longer /// is more up-to-date. +/// +/// #446: callers must pass the in-memory last-log-id (last_entry_id), never durable_index. +/// A node with an un-fsynced tail must still be able to reject a candidate whose log is +/// genuinely less up to date β€” voting eligibility and commit-durability are separate +/// concerns and must not share the same index source. pub(crate) fn is_target_log_more_recent( my_last_log_index: u64, my_last_log_term: u64, diff --git a/d-engine-core/src/raft.rs b/d-engine-core/src/raft.rs index c8e2eac6..7eb802d7 100644 --- a/d-engine-core/src/raft.rs +++ b/d-engine-core/src/raft.rs @@ -354,6 +354,7 @@ where } if count > 0 { trace!("Drained {} client commands", count); + metrics::histogram!("core.raft.client_cmd.batch_size").record(count as f64); } Ok(()) } @@ -475,7 +476,10 @@ where let _ = self.role.drain_read_buffer(); debug!("BecomeFollower"); - self.role = self.role.become_follower()?; + let mut new_role = self.role.become_follower()?; + let withheld_acks = self.role.take_pending_acks(); + new_role.restore_pending_acks(withheld_acks); + self.role = new_role; // Reset vote when stepping down (new term, no vote yet) self.role.state_mut().commit_vote_reset(&self.ctx)?; @@ -494,7 +498,10 @@ where let _ = self.role.drain_read_buffer(); debug!("BecomeCandidate"); - self.role = self.role.become_candidate()?; + let mut new_role = self.role.become_candidate()?; + let withheld_acks = self.role.take_pending_acks(); + new_role.restore_pending_acks(withheld_acks); + self.role = new_role; // No leader during candidate state let current_term = self.role.current_term(); @@ -505,7 +512,10 @@ where } InternalEvent::BecomeLeader => { debug!("BecomeLeader"); - self.role = self.role.become_leader()?; + let mut new_role = self.role.become_leader()?; + let withheld_acks = self.role.take_pending_acks(); + new_role.restore_pending_acks(withheld_acks); + self.role = new_role; // Mark vote as committed (candidate β†’ leader transition) let current_term = self.role.current_term(); @@ -551,7 +561,10 @@ where let _ = self.role.drain_read_buffer(); debug!("BecomeLearner"); - self.role = self.role.become_learner()?; + let mut new_role = self.role.become_learner()?; + let withheld_acks = self.role.take_pending_acks(); + new_role.restore_pending_acks(withheld_acks); + self.role = new_role; // Learner has no leader initially let current_term = self.role.current_term(); @@ -609,6 +622,13 @@ where .handle_log_flushed(durable_index, &self.ctx, &self.internal_event_tx) .await; } + InternalEvent::FsyncCompleted(mark) => { + if let Some(new_durable) = self.ctx.raft_log().try_advance_durable_index(mark) { + self.role + .handle_log_flushed(new_durable, &self.ctx, &self.internal_event_tx) + .await; + } + } InternalEvent::AppendResult { follower_id, result, @@ -651,7 +671,9 @@ where } InternalEvent::PeerStreamError { peer_id } => { debug!(%peer_id, "PeerStreamError: bidi stream disconnected, resetting next_index"); - self.role.handle_peer_stream_error(peer_id); + if let RaftRole::Leader(leader) = &mut self.role { + leader.handle_peer_stream_error(peer_id); + } } InternalEvent::ZombieDetected(node_id) => { debug!(%node_id, "ZombieDetected: forwarding to leader for BatchRemove"); @@ -687,9 +709,11 @@ where // Peer-state guard: seeding is only valid for the peer's CURRENT // in-flight snapshot attempt. A straggler completion that arrives after // the peer has moved on must not touch next_index. - if self.role.state().peer_replication_state(peer_id) - != PeerReplicationState::Snapshot - { + let in_snapshot_state = matches!( + &self.role, + RaftRole::Leader(leader) if leader.peer_replication_state(peer_id) == PeerReplicationState::Snapshot + ); + if !in_snapshot_state { debug!(%peer_id, "dropping SnapshotPushCompleted: peer not in Snapshot state"); return Ok(()); } diff --git a/d-engine-core/src/raft_role/follower_state.rs b/d-engine-core/src/raft_role/follower_state.rs index c1e30ebf..f0dfde4e 100644 --- a/d-engine-core/src/raft_role/follower_state.rs +++ b/d-engine-core/src/raft_role/follower_state.rs @@ -7,6 +7,7 @@ use d_engine_proto::server::cluster::ClusterConfUpdateResponse; use d_engine_proto::server::cluster::LeaderDiscoveryResponse; use d_engine_proto::server::election::VoteResponse; use d_engine_proto::server::storage::SnapshotMetadata; +use std::collections::BTreeMap; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; @@ -43,6 +44,7 @@ use crate::RaftNodeConfig; use crate::Result; use crate::StateTransitionError; use crate::TypeConfig; +use crate::role_state::PendingAck; use crate::role_state::schedule_and_execute_purge; use crate::utils::cluster::error; use crate::utils::cluster_printer::print_role_transition_line; @@ -73,6 +75,10 @@ pub struct FollowerState { /// Last physically purged log index (inclusive) pub last_purged_index: Option, + /// AppendEntries responses withheld pending this node's own durable_index. + /// See `role_state::PendingAck`. + pending_append_acks: BTreeMap, + // -- Snapshot Management -- /// Prevents concurrent snapshot creation /// @@ -463,6 +469,12 @@ impl RaftRoleState for FollowerState { fn pending_purge_upto_mut(&mut self) -> Option<&mut Option> { Some(&mut self.pending_purge_upto) } + + fn pending_append_acks_mut( + &mut self + ) -> Option<&mut std::collections::BTreeMap> { + Some(&mut self.pending_append_acks) + } } impl FollowerState { @@ -484,6 +496,7 @@ impl FollowerState { node_config.raft.election.election_timeout_max, )), node_config, + pending_append_acks: BTreeMap::new(), snapshot_in_progress: AtomicBool::new(false), _marker: PhantomData, last_purged_index: None, @@ -511,6 +524,7 @@ impl From<&CandidateState> for FollowerState { )), node_config: candidate_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: candidate_state.last_purged_index, // scheduled_purge_upto: None, _marker: PhantomData, @@ -527,6 +541,7 @@ impl From<&LeaderState> for FollowerState { leader_state.node_config.raft.election.election_timeout_max, )), node_config: leader_state.node_config.clone(), + pending_append_acks: BTreeMap::new(), snapshot_in_progress: AtomicBool::new( leader_state.snapshot_in_progress.load(Ordering::SeqCst), ), @@ -548,6 +563,7 @@ impl From<&LearnerState> for FollowerState { )), node_config: learner_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: learner_state.last_purged_index, pending_purge_upto: learner_state.pending_purge_upto, _marker: PhantomData, diff --git a/d-engine-core/src/raft_role/follower_state_test.rs b/d-engine-core/src/raft_role/follower_state_test.rs index d6ab39a8..5ac6e2f8 100644 --- a/d-engine-core/src/raft_role/follower_state_test.rs +++ b/d-engine-core/src/raft_role/follower_state_test.rs @@ -994,11 +994,96 @@ async fn test_handle_append_entries_success_from_new_leader() { "Should update commit_index" ); + // RPO=0 (#446): the success ACK is withheld until durable_index reaches the claimed index. + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 1, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(1, &context, &flush_tx).await; + // Verify: Response with success=true let response = resp_rx.recv().await.expect("should receive response").unwrap(); assert!(response.is_success(), "Response should indicate success"); } +/// Test: the `AppendEntriesResponse` sent back to the leader must report the follower's +/// just-updated term, not whatever term was captured in the `StateSnapshot` before +/// `commit_hard_state` ran. +/// +/// # Why this needs its own test +/// `test_handle_append_entries_success_from_new_leader` above hard-codes `new_leader_term` +/// directly into the mocked response, so it never actually reads what +/// `handle_append_entries_request_workflow` (role_state.rs) passes as the `state_snapshot` +/// argument to `handle_append_entries` β€” it can't catch a caller that passes a stale snapshot. +/// This test's mock instead echoes back `state_snapshot.current_term`, exactly mirroring what +/// the real `ReplicationHandler::handle_append_entries` does +/// (`replication_handler.rs`: `let current_term = state_snapshot.current_term;`), so it's +/// sensitive to whether the caller passes the pre-update or post-update snapshot. +/// +/// # Scenario +/// Follower (term=1) receives AppendEntries from a leader at term=2 β€” its first-ever contact. +/// +/// # Expected (RED until fixed) +/// `response.term == 2` β€” the follower's real term was correctly updated by `commit_hard_state` +/// before responding; the response must reflect that, not the term=1 snapshot taken before it. +#[tokio::test] +async fn test_handle_append_entries_response_reports_updated_term_not_stale_snapshot() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let follower_term = 1; + let new_leader_term = follower_term + 1; + + let mut replication_handler = MockReplicationCore::new(); + replication_handler + .expect_handle_append_entries() + .returning(move |_, state_snapshot, _| { + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success(1, state_snapshot.current_term, None), + commit_index_update: None, + }) + }); + + context.membership = Arc::new(MockMembership::new()); + context.handlers.replication_handler = replication_handler; + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state_mut().update_current_term(follower_term); + + let append_entries_request = AppendEntriesRequest { + term: new_leader_term, + leader_id: 5, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); + let inbound_event = InboundEvent::AppendEntries(append_entries_request, vec![resp_tx]); + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + assert!( + state + .handle_inbound_event(inbound_event, &context, internal_event_tx) + .await + .is_ok(), + "handle_inbound_event should succeed" + ); + + // No claimed log entry in this batch, so RPO=0 withhold doesn't apply β€” response is + // available immediately. + let response = resp_rx.recv().await.expect("should receive response").unwrap(); + assert_eq!( + response.term, new_leader_term, + "AppendEntriesResponse must report the follower's just-updated term ({new_leader_term}), \ + not a StateSnapshot captured before commit_hard_state updated it ({follower_term})" + ); +} + /// Test: FollowerState rejects AppendEntries with stale term /// /// Scenario: @@ -2882,15 +2967,17 @@ async fn test_follower_rejects_strong_consistency_reads() { } // ============================================================================ -// MemFirst ACK Tests +// Withheld-ACK tests (RPO=0, #446) +// +// End-to-end coverage of the AppendEntries workflow deciding to withhold or send. +// The queue mechanics (release / reject / carry across a role transition) are +// unit-tested in `pending_ack_test.rs`. // ============================================================================ -/// Follower ACKs leader immediately after memory write (MemFirst). -/// -/// The IO thread continues to fsync asynchronously. Safety: before commit, -/// the leader's durable_index >= N (quorum uses durable_index). +/// The workflow withholds a success ACK while durable_index is behind the claimed +/// index, and releases it once `handle_log_flushed` reports that index durable. #[tokio::test] -async fn test_follower_acks_immediately_after_memory_write() { +async fn test_follower_withholds_ack_until_durable() { let (_graceful_tx, graceful_rx) = watch::channel(()); let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); @@ -2937,9 +3024,109 @@ async fn test_follower_acks_immediately_after_memory_write() { .is_ok() ); - // MemFirst: ACK sent immediately, no waiting for fsync - let response = resp_rx.try_recv().expect("ACK must be sent immediately after memory write"); - assert!(response.unwrap().is_success()); + // RPO=0: the ACK is withheld while durable_index < claimed index (5). + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 5, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(appended_index, &context, &flush_tx).await; + + let response = resp_rx.try_recv().expect("ACK must be released once durable").unwrap(); + assert!(response.is_success()); +} + +/// #446: two independent AppendEntries requests (e.g. a leader retry) that both claim +/// the same threshold index must both eventually receive a response β€” the second one +/// landing on `pending_append_acks` must not silently overwrite the first. +#[tokio::test] +async fn test_multiple_requests_at_same_threshold_all_receive_response() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); + + let leader_term = 2u64; + let claimed_index = 5u64; + + let mut replication_handler = MockReplicationCore::new(); + replication_handler + .expect_handle_append_entries() + .times(2) + .returning(move |_, _, _| { + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success( + 1, + leader_term, + Some(LogId { + term: leader_term, + index: claimed_index, + }), + ), + commit_index_update: None, + }) + }); + context.handlers.replication_handler = replication_handler; + context.membership = Arc::new(MockMembership::new()); + + let mut state = + FollowerState::::new(1, context.node_config.clone(), None, None); + state.shared_state_mut().update_current_term(leader_term); + + let append_request = AppendEntriesRequest { + term: leader_term, + leader_id: 2, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + + // First request lands on threshold=5 and gets withheld. + let (resp_tx1, mut resp_rx1) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(append_request.clone(), vec![resp_tx1]), + &context, + internal_event_tx.clone(), + ) + .await + .unwrap(); + + // A second, independent request (e.g. leader retry) also claims index 5. + let (resp_tx2, mut resp_rx2) = MaybeCloneOneshot::new(); + state + .handle_inbound_event( + InboundEvent::AppendEntries(append_request, vec![resp_tx2]), + &context, + internal_event_tx, + ) + .await + .unwrap(); + + assert!( + resp_rx1.try_recv().is_err(), + "first request must still be withheld" + ); + assert!( + resp_rx2.try_recv().is_err(), + "second request must still be withheld" + ); + + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(claimed_index, &context, &flush_tx).await; + + // Both senders β€” not just one β€” must receive a response. A single-value + // `BTreeMap` without `.entry().or_insert_with(...)` merging would + // let the second insert silently overwrite the first, dropping this ACK forever. + let response1 = resp_rx1.try_recv().expect("first sender must receive a response").unwrap(); + let response2 = resp_rx2 + .try_recv() + .expect("second sender must also receive a response, not be silently overwritten") + .unwrap(); + assert!(response1.is_success()); + assert!(response2.is_success()); } /// Follower sends ACK immediately for heartbeat (no entries). @@ -3042,8 +3229,18 @@ async fn test_follower_commit_index_and_ack_both_sent_immediately() { new_commit, "commit_index must advance immediately" ); - let response = resp_rx.try_recv().expect("ACK must be sent immediately"); - assert!(response.unwrap().is_success()); + // RPO=0: commit_index advances immediately, but the ACK is withheld until durable. + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 5, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(appended_index, &context, &flush_tx).await; + + let response = resp_rx.try_recv().expect("ACK must be released once durable").unwrap(); + assert!(response.is_success()); } /// Spawns a fake Worker that answers exactly one `InstallSnapshot` command with `result`, diff --git a/d-engine-core/src/raft_role/leader_state.rs b/d-engine-core/src/raft_role/leader_state.rs index 06b21cb8..8e79e25c 100644 --- a/d-engine-core/src/raft_role/leader_state.rs +++ b/d-engine-core/src/raft_role/leader_state.rs @@ -201,8 +201,9 @@ pub struct ClusterMetadata { /// Task routed to a per-follower replication worker. enum ReplicationTask { - /// Normal AppendEntries replication. - Append(AppendEntriesRequest), + /// Normal AppendEntries replication. Second field is the enqueue time, used to + /// measure how long the task waited in `task_tx` before the worker sent it. + Append(AppendEntriesRequest, Instant), /// Peer's next_index fell below the purge boundary; transfer the latest snapshot. Snapshot(SnapshotMetadata, u64), } @@ -265,6 +266,8 @@ pub struct LeaderState { /// keep this change minimal and consistent with existing style. pub(super) peer_replication_state: HashMap, + in_flight: HashMap, + /// === Volatile State === /// Temporary storage for no-op entry log ID during leader initialization #[doc(hidden)] @@ -1343,18 +1346,11 @@ impl RaftRoleState for LeaderState { ctx: &RaftContext, internal_event_tx: &mpsc::UnboundedSender, ) { - let new_commit_index = if self.cluster_metadata.single_voter { - // MemFirst single-voter: LogFlushed(durable) is the IO checkpoint. - // Commit to last_entry_id() β€” not just durable β€” to allow pipelining - // across IO batch boundaries. Matches multi-voter MemFirst where leader - // contributes last_entry_id() to quorum (not durable_index). - let last_log_index = ctx.raft_log().last_entry_id(); - debug_assert!( - last_log_index >= durable, - "last_entry_id ({last_log_index}) must be >= durable ({durable})" - ); - if last_log_index > self.commit_index() { - Some(last_log_index) + let next_commit_index = if self.cluster_metadata.single_voter { + // RPO=0 (#446): single-voter has no majority to fall back on β€” commit + // must not advance past what this node has itself fsynced. + if durable > self.commit_index() { + Some(durable) } else { None } @@ -1362,10 +1358,11 @@ impl RaftRoleState for LeaderState { // Multi-voter: quorum of match_index determines commit. // Only voter peers (non-Learner role) may contribute to majority. // Learners replicate entries but must never count toward commit quorum. - self.calculate_new_commit_index(ctx.raft_log()) + let (commit_index, majority) = self.majority_matched_index(ctx.raft_log()); + Self::new_commit_index(commit_index, majority) }; - if let Some(new_commit) = new_commit_index { + if let Some(new_commit) = next_commit_index { if let Err(e) = self.update_commit_index_with_signal( Leader as i32, self.current_term(), @@ -1494,6 +1491,7 @@ impl RaftRoleState for LeaderState { "AppendResult from peer {} has no result variant", follower_id ); + self.set_peer_in_flight(follower_id, false); return Ok(()); } }; @@ -1536,7 +1534,10 @@ impl RaftRoleState for LeaderState { // Re-calculate commit index after updating this voter's match_index. if peer_update.success && is_voter { - if let Some(new_commit) = self.calculate_new_commit_index(ctx.raft_log()) { + let (commit_index, majority) = self.majority_matched_index(ctx.raft_log()); + let quorum_confirmed = majority.is_some(); + + if let Some(new_commit) = Self::new_commit_index(commit_index, majority) { self.update_commit_index_with_signal( Leader as i32, self.current_term(), @@ -1550,25 +1551,9 @@ impl RaftRoleState for LeaderState { // Lease refresh and pending_lease_reads drain are triggered by quorum ACK, // independent of whether commit_index advanced. When an expired lease fires an // empty AppendEntries heartbeat, commit_index does not change (nothing new to - // commit), so calculate_new_commit_index returns None and the block above is - // skipped. We must check quorum confirmation separately here. - let quorum_confirmed = ctx - .raft_log() - .calculate_majority_matched_index( - self.current_term(), - self.commit_index(), - self.match_index - .iter() - .filter(|(id, _)| { - self.cluster_metadata.replication_targets.iter().any(|n| { - n.id == **id - && n.role != d_engine_proto::common::NodeRole::Learner as i32 - }) - }) - .map(|(_, idx)| *idx) - .collect(), - ) - .is_some(); + // commit), so new_commit_index yields None and the block above is skipped. + // We still reuse the single majority computation (majority.is_some()) to + // confirm quorum here. if quorum_confirmed { // Anchor deadline to send time (not ACK time) to eliminate the RTT/2 window. // Falls back to now_ms() only in tests that bypass execute_and_process_raft_rpc. @@ -1882,33 +1867,6 @@ impl RaftRoleState for LeaderState { Ok(()) } - - fn peer_replication_state( - &self, - node_id: u32, - ) -> PeerReplicationState { - self.peer_replication_state - .get(&node_id) - .copied() - .unwrap_or(PeerReplicationState::Probe) - } - - fn set_peer_replication_state( - &mut self, - node_id: u32, - state: PeerReplicationState, - ) { - metrics::gauge!( - "core.raft.peer.replication_state", - "peer_id" => node_id.to_string() - ) - .set(match state { - PeerReplicationState::Probe => 0.0, - PeerReplicationState::Replicate => 1.0, - PeerReplicationState::Snapshot => 2.0, - }); - self.peer_replication_state.insert(node_id, state); - } } /// Computes the exponential backoff delay for a snapshot push failure. @@ -2056,7 +2014,12 @@ impl LeaderState { } match task { - ReplicationTask::Append(request) => { + ReplicationTask::Append(request, enqueued_at) => { + metrics::histogram!( + "core.raft.replication_worker.queue_wait_ms", + "peer_id" => peer_id.to_string() + ) + .record(enqueued_at.elapsed().as_secs_f64() * 1_000.0); // Push batch directly into the persistent bidi stream (non-blocking) if stream_sender.send(request).await.is_err() { warn!(peer_id, "Bidi stream sender closed, reconnecting"); @@ -2582,6 +2545,7 @@ impl LeaderState { follower_id: u32, update: &PeerUpdate, ) { + self.set_peer_in_flight(follower_id, false); if update.success { // Success: trust speculative advance β€” never regress next_index below what // the leader has already pipeline-sent. ACK confirms a lower bound only; @@ -2826,12 +2790,16 @@ impl LeaderState { Ok(()) } - /// Calculate new submission index - fn calculate_new_commit_index( + /// Compute the majority-matched index for the current term. + /// + /// Returns `(commit_index, majority)` where `commit_index` is the base used as + /// the quorum floor, so callers apply the advance rule against the *same* base + /// the majority was computed with. + fn majority_matched_index( &self, raft_log: &Arc>, - ) -> Option { - let old_commit_index = self.commit_index(); + ) -> (u64, Option) { + let commit_index = self.commit_index(); let current_term = self.current_term(); let replication_targets = &self.cluster_metadata.replication_targets; let learner_role = d_engine_proto::common::NodeRole::Learner as i32; @@ -2846,14 +2814,21 @@ impl LeaderState { .map(|(_, idx)| *idx) .collect(); - let new_commit_index = - raft_log.calculate_majority_matched_index(current_term, old_commit_index, matched_ids); + let majority = + raft_log.calculate_majority_matched_index(current_term, commit_index, matched_ids); + (commit_index, majority) + } - if new_commit_index.is_some() && new_commit_index.unwrap() > old_commit_index { - new_commit_index - } else { - None - } + /// Turn a majority-matched index into the new commit index. + /// + /// Pure function: compares against the same `commit_index` that `majority` was + /// computed with. Callers must not hand-write the `>` β€” this is the single + /// source of truth for strict advance. + fn new_commit_index( + commit_index: u64, + majority: Option, + ) -> Option { + majority.filter(|m| *m > commit_index) } /// Calculate safe read index for linearizable reads. @@ -3106,6 +3081,7 @@ impl LeaderState { write_propose_times: HashMap::new(), write_commit_times: HashMap::new(), peer_replication_state: HashMap::new(), + in_flight: HashMap::new(), _marker: PhantomData, } } @@ -3317,6 +3293,12 @@ impl LeaderState { continue; } + let is_probe = self.peer_replication_state(peer_id) == PeerReplicationState::Probe; + let is_heartbeat = request.entries.is_empty(); + if !is_heartbeat && is_probe && self.peer_in_flight(peer_id) { + continue; + } + // #436: write next_index once. When Replicate, trust speculative advance // (always >= effective_next_index, so no separate write is needed for it). let next_index = @@ -3329,9 +3311,13 @@ impl LeaderState { error!("failed to update next_index peer={}: {:?}", peer_id, e); } + if !is_heartbeat && is_probe { + self.set_peer_in_flight(peer_id, true); + } + self.send_to_worker_or_spawn( peer_id, - ReplicationTask::Append(request), + ReplicationTask::Append(request, Instant::now()), ReplicationWorkerConfig { transport: transport.clone(), membership: membership.clone(), @@ -3807,6 +3793,67 @@ impl LeaderState { let response = ClientResponse::read_results(results); let _ = sender.send(Ok(response)); } + + pub(crate) fn peer_replication_state( + &self, + node_id: u32, + ) -> PeerReplicationState { + self.peer_replication_state + .get(&node_id) + .copied() + .unwrap_or(PeerReplicationState::Probe) + } + + pub(crate) fn set_peer_replication_state( + &mut self, + node_id: u32, + state: PeerReplicationState, + ) { + metrics::gauge!( + "core.raft.peer.replication_state", + "peer_id" => node_id.to_string() + ) + .set(match state { + PeerReplicationState::Probe => 0.0, + PeerReplicationState::Replicate => 1.0, + PeerReplicationState::Snapshot => 2.0, + }); + self.peer_replication_state.insert(node_id, state); + } + + /// Whether `peer_id` has an unacknowledged AppendEntries outstanding. + pub(super) fn peer_in_flight( + &self, + peer_id: u32, + ) -> bool { + self.in_flight.get(&peer_id).copied().unwrap_or(false) + } + + pub(super) fn set_peer_in_flight( + &mut self, + peer_id: u32, + in_flight: bool, + ) { + self.in_flight.insert(peer_id, in_flight); + } + + /// Reset `next_index[peer] = match_index[peer] + 1` after a bidi stream disconnect. + /// Ensures the next heartbeat re-sends any unACKed in-flight entries. + /// Moved here from `raft_role/mod.rs` β€” this is leader-only, the generic + /// `RaftRole`-level dispatch was the anti-pattern that hid the `in_flight` gap. + pub(crate) fn handle_peer_stream_error( + &mut self, + peer_id: u32, + ) { + if self.peer_replication_state(peer_id) == PeerReplicationState::Snapshot { + return; + } + let match_idx = self.match_index(peer_id).unwrap_or(0); + let _ = self.update_next_index(peer_id, match_idx + 1); + + self.set_peer_replication_state(peer_id, PeerReplicationState::Probe); + self.set_peer_in_flight(peer_id, false); + } } impl From<&CandidateState> for LeaderState { @@ -3861,6 +3908,7 @@ impl From<&CandidateState> for LeaderState { write_propose_times: HashMap::new(), write_commit_times: HashMap::new(), peer_replication_state: HashMap::new(), + in_flight: HashMap::new(), _marker: PhantomData, } } @@ -3999,3 +4047,7 @@ mod state_management_test; #[cfg(test)] #[path = "leader_state_test/worker_lifecycle_test.rs"] mod worker_lifecycle_test; + +#[cfg(test)] +#[path = "leader_state_test/probe_backpressure_test.rs"] +mod probe_backpressure_test; diff --git a/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs b/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs index a7811973..748a1c84 100644 --- a/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/commit_index_test.rs @@ -20,6 +20,7 @@ use d_engine_proto::server::replication::{AppendEntriesResponse, SuccessResult}; use rand::distr::SampleString; use std::collections::VecDeque; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use tokio::sync::{mpsc, watch}; fn success_response( @@ -38,6 +39,15 @@ fn success_response( } } +fn voter_meta(id: u32) -> NodeMeta { + NodeMeta { + id, + address: "".into(), + status: NodeStatus::Active as i32, + role: Follower.into(), + } +} + // ── helpers ────────────────────────────────────────────────────────────────── fn write_request() -> ( @@ -262,6 +272,84 @@ async fn test_multi_voter_commit_respects_quorum_result() { ); } +// ── handle_append_result: quorum computed exactly once per ACK ─────────────── + +/// A multi-voter leader must compute the majority-matched index exactly once per +/// AppendEntries ACK. The pre-fix code called `calculate_majority_matched_index` +/// twice in a single `handle_append_result` β€” once inside `calculate_new_commit_index` +/// and once more for the `quorum_confirmed` lease check β€” with identical inputs +/// (`current_term`, `commit_index`, and the same voter-filtered `match_index`). +/// +/// The single result must drive both outcomes: commit_index advance AND lease +/// (quorum) confirmation. This test fails if the method runs twice, or if either +/// consumer stops receiving the result. +#[tokio::test] +async fn test_handle_append_result_computes_quorum_once_per_ack() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + + let mut ctx = mock_raft_context( + "/tmp/test_handle_append_result_quorum_once", + graceful_rx, + None, + ); + + // Two follower voters β†’ multi-voter; `replication_peers` must list them so + // `handle_append_result` treats peer 2 as a voter (drives the quorum path). + let mut membership = crate::MockMembership::::new(); + membership.expect_voters().returning(|| vec![voter_meta(2), voter_meta(3)]); + membership + .expect_replication_peers() + .returning(|| vec![voter_meta(2), voter_meta(3)]); + ctx.membership = Arc::new(membership); + + // Count every quorum computation; return a majority index of 1 so the single + // result is exercised by BOTH the commit path and the lease-confirmation path. + let call_count = Arc::new(AtomicU64::new(0)); + let call_count_clone = call_count.clone(); + let mut raft_log = MockRaftLog::new(); + raft_log.expect_calculate_majority_matched_index().returning(move |_, _, _| { + call_count_clone.fetch_add(1, Ordering::Relaxed); + Some(1) + }); + ctx.storage.raft_log = Arc::new(raft_log); + + let mut state = LeaderState::::new(1, ctx.node_config.clone()); + state.init_cluster_metadata(&ctx.membership).await.unwrap(); + assert!(!state.cluster_metadata.single_voter); + + ctx.handlers + .replication_handler + .expect_handle_success_response() + .returning(|_, _, _, _| { + Ok(PeerUpdate { + match_index: Some(1), + next_index: 2, + success: true, + }) + }); + + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); + let resp = success_response(1, 1); + state.handle_append_result(2, Ok(resp), &ctx, &internal_event_tx).await.unwrap(); + + // Root-cause assertion: one ACK must trigger exactly one majority computation. + assert_eq!( + call_count.load(Ordering::Relaxed), + 1, + "calculate_majority_matched_index must run exactly once per AppendEntries ACK" + ); + // Both consumers of that single result must still observe it. + assert_eq!( + state.commit_index(), + 1, + "commit must advance from the single majority result" + ); + assert!( + state.is_lease_valid(), + "quorum confirmation (lease) must reuse the single majority result" + ); +} + // ── handle_log_flushed: Leader commit advances on LogFlushed (#313 P0) ──────── /// Single-voter leader: LogFlushed(1) triggers commit_index to advance to 1. diff --git a/d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs b/d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs new file mode 100644 index 00000000..1641ac3b --- /dev/null +++ b/d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs @@ -0,0 +1,503 @@ +//! Test for the missing per-peer in-flight gate during `PeerReplicationState::Probe`. +//! +//! Background (see `446-expert-q-probe-backpressure-fix-8020.md` in the product-design repo): +//! every reference Raft implementation (etcd/raft, tikv/raft-rs, openraft) limits a `Probe`-state +//! peer to at most one outstanding (unacknowledged) `AppendEntries` request. d-engine's +//! `PeerReplicationState::Probe`/`Replicate` only controls whether `next_index` is optimistically +//! advanced before sending β€” it never checks whether the peer already has a request in flight. +//! Left unchecked, the leader keeps re-sending `prev_log_index=0` probes to a peer whose first +//! attempt hasn't been acknowledged yet, and each one forces the follower to wipe and rebuild its +//! entire log (`buffered_raft_log::reset`), which is the root cause of the throughput collapse +//! this ticket investigated. +//! +//! This test is intentionally RED until the in-flight gate is implemented in +//! `execute_and_process_raft_rpc` (Phase 5, `leader_state.rs`). It does not assert *how* the gate +//! is implemented β€” only the externally observable contract: a peer with one unacknowledged +//! request must not receive a second one. + +use std::collections::VecDeque; +use std::sync::Arc; + +use bytes::Bytes; +use d_engine_proto::common::{Entry, EntryPayload, NodeRole::Follower, NodeStatus}; +use d_engine_proto::server::cluster::NodeMeta; +use d_engine_proto::server::replication::{ + AppendEntriesRequest, AppendEntriesResponse, ConflictResult, append_entries_response, +}; +use tokio::sync::{mpsc, watch}; +use tracing_test::traced_test; + +use crate::MockMembership; +use crate::MockRaftLog; +use crate::RaftRequestWithSignal; +use crate::event::InternalEvent; +use crate::maybe_clone_oneshot::{MaybeCloneOneshot, RaftOneshot}; +use crate::network::PeerUpdate; +use crate::raft_role::leader_state::LeaderState; +use crate::raft_role::role_state::{PeerReplicationState, RaftRoleState}; +use crate::test_utils::mock::{MockTypeConfig, mock_raft_context}; + +/// Two-voter membership (peers 2 & 3) so the cluster is multi-voter β€” a single-voter leader +/// short-circuits Phase 5 entirely (no peer work to gate), which would make this test vacuous. +fn two_peer_membership() -> MockMembership { + let peers = vec![ + NodeMeta { + id: 2, + address: String::new(), + status: NodeStatus::Active as i32, + role: Follower.into(), + }, + NodeMeta { + id: 3, + address: String::new(), + status: NodeStatus::Active as i32, + role: Follower.into(), + }, + ]; + let peers2 = peers.clone(); + let mut m = MockMembership::new(); + m.expect_is_single_node_cluster().returning(|| false); + m.expect_voters().returning(move || peers.clone()); + m.expect_replication_peers().returning(move || peers2.clone()); + m +} + +/// A minimal `AppendEntriesRequest` stub β€” its contents don't matter, only whether Phase 5 +/// forwards a request to the peer's worker channel at all. +fn stub_request() -> AppendEntriesRequest { + AppendEntriesRequest::default() +} + +/// A non-empty probe (one entry) β€” used where the empty-heartbeat vs non-empty-probe +/// distinction matters for the gate: a heartbeat must neither block nor set the gate. +fn stub_probe_request() -> AppendEntriesRequest { + AppendEntriesRequest { + entries: vec![Entry { + index: 1, + term: 1, + payload: None, + }], + ..AppendEntriesRequest::default() + } +} + +/// A single one-entry write batch, matching the shape `process_batch` expects. +fn one_entry_batch() -> VecDeque { + let (tx, _rx) = >::new(); + let req = RaftRequestWithSignal { + id: "test".into(), + payloads: vec![EntryPayload::command(Bytes::from_static(b"cmd"))], + senders: vec![tx], + wait_for_apply_event: false, + }; + VecDeque::from(vec![req]) +} + +/// Scenario (release direction β€” the half the gate must also get right): +/// - Batch 1 is dispatched to peer 2: its first `Probe`, correctly limited to one outstanding +/// request. +/// - The follower answers that probe with a CONFLICT (reject), not a success. A reject is not +/// evidence the peer is caught up, so `update_peer_index`'s conflict branch retreats +/// `next_index` to the conflict hint and leaves peer 2 in `Probe`. +/// - Batch 2 is processed afterwards. The response to batch 1 has *arrived*, so peer 2 no longer +/// has an outstanding request and the gate must release: the corrected probe must be +/// dispatched. +/// +/// # Expected +/// `Probe` means "at most one unacknowledged `AppendEntries` at a time" (etcd/raft +/// `MsgAppFlowPaused`, openraft `Inflight::is_none()`), not "at most one ever". A response must +/// re-arm the gate, never latch it shut: a latched `Probe` peer receives nothing further β€” not +/// even heartbeats, since they share this dispatch path β€” while the frozen follower times out +/// into candidacy and the leader cannot reach quorum. +/// +/// # Current behavior (why this test is RED) +/// The gate is set on dispatch and only cleared by `handle_peer_stream_error` (a bidi stream +/// disconnect). A peer whose probe was rejected stays `Probe` with the latch closed forever, so +/// batch 2 dispatches nothing. +#[tokio::test] +#[traced_test] +async fn test_probe_peer_dispatches_next_probe_after_reject() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut ctx = mock_raft_context( + "/tmp/test_probe_peer_dispatches_next_probe_after_reject", + graceful_rx, + None, + ); + + ctx.membership = Arc::new(two_peer_membership()); + + // Both batches offer a request for peer 2, so every dispatch decision is Phase 5's own. + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(|_, _, _, _, _| { + Ok(crate::PrepareResult { + append_requests: vec![(2, stub_probe_request(), 1)], + snapshot_targets: vec![], + }) + }); + ctx.handlers + .replication_handler + .expect_handle_conflict_response() + .returning(|_, _, _, _| { + Ok(PeerUpdate { + match_index: None, + next_index: 1, + success: false, + }) + }); + + let mut raft_log = MockRaftLog::new(); + raft_log.expect_last_entry_id().returning(|| 0); + raft_log.expect_flush().returning(|| Ok(())); + raft_log.expect_save_hard_state().returning(|_| Ok(())); + ctx.storage.raft_log = Arc::new(raft_log); + + let mut state = LeaderState::::new(1, ctx.node_config.clone()); + state.init_cluster_metadata(&ctx.membership).await.unwrap(); + + let (task_tx, mut task_rx) = mpsc::unbounded_channel(); + state.replication_workers.insert( + 2, + super::ReplicationWorkerHandle { + task_tx, + snapshot_failure_count: 0, + snapshot_next_retry_at: None, + }, + ); + + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel::(); + + // Batch 1: peer 2's first probe β€” nothing outstanding, so it must be dispatched. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "first batch must reach peer 2's worker β€” it had no outstanding request" + ); + + // Peer 2 rejects that probe: the leader retreats next_index and keeps the peer in `Probe`, + // because a reject says nothing about the peer being caught up. + let reject = AppendEntriesResponse { + node_id: 2, + term: 1, + result: Some(append_entries_response::Result::Conflict(ConflictResult { + conflict_term: None, + conflict_index: Some(1), + })), + }; + state + .handle_append_result(2, Ok(reject), &ctx, &internal_event_tx) + .await + .unwrap(); + assert_eq!( + state.peer_replication_state(2), + PeerReplicationState::Probe, + "a rejected probe must leave the peer in `Probe` β€” it is not caught up" + ); + + // Batch 2: the response to batch 1 already arrived, so the gate must release and the + // corrected probe must go out. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "the probe was answered (reject), so the peer has nothing in flight β€” the leader must \ + re-probe with the corrected next_index instead of latching the peer shut forever" + ); +} + +/// Scenario: +/// - Peer 2 has a worker whose channel this test holds directly (no real transport/network +/// involved), so what actually got dispatched can be checked synchronously β€” no dependency on +/// background task scheduling, so this test cannot flake on timing. +/// - `prepare_batch_requests` is mocked to unconditionally offer a request for peer 2 on every +/// call, simulating "there's always more to replicate" regardless of ack status β€” this isolates +/// the assertion to Phase 5's own dispatch decision, which is where the missing gate belongs. +/// - Batch 1 is processed and dispatched β€” this is correct: peer 2 starts in `Probe` with nothing +/// outstanding, so it must receive its first probe. +/// - Batch 2 is processed *without* `handle_append_result` ever being called for peer 2's first +/// request β€” i.e. the leader has not (and cannot have) learned whether the first probe was +/// acknowledged. `next_index`/`match_index`/`peer_replication_state` are therefore still exactly +/// what they were after batch 1. +/// +/// # Expected (once the fix lands) +/// Batch 2 must NOT produce a second dispatch to peer 2's worker: a `Probe`-state peer with an +/// unacknowledged request in flight must wait for that response (etcd/raft `MsgAppFlowPaused`, +/// openraft `Inflight::is_none()`) before being sent to again. +/// +/// # Current behavior (why this test is RED today) +/// `execute_and_process_raft_rpc`'s Phase 5 loop sends to every peer in `append_requests` +/// unconditionally β€” `PeerReplicationState` only gates whether `next_index` is optimistically +/// advanced beforehand, not whether sending is allowed at all. So batch 2 dispatches anyway. +#[tokio::test] +#[traced_test] +async fn test_probe_peer_with_pending_ack_receives_no_second_dispatch() { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut ctx = mock_raft_context( + "/tmp/test_probe_peer_with_pending_ack_receives_no_second_dispatch", + graceful_rx, + None, + ); + + ctx.membership = Arc::new(two_peer_membership()); + + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(|_, _, _, _, _| { + Ok(crate::PrepareResult { + append_requests: vec![(2, stub_probe_request(), 1)], + snapshot_targets: vec![], + }) + }); + + let mut raft_log = MockRaftLog::new(); + raft_log.expect_last_entry_id().returning(|| 0); + raft_log.expect_flush().returning(|| Ok(())); + raft_log.expect_save_hard_state().returning(|_| Ok(())); + ctx.storage.raft_log = Arc::new(raft_log); + + let mut state = LeaderState::::new(1, ctx.node_config.clone()); + state.init_cluster_metadata(&ctx.membership).await.unwrap(); + + // Inject peer 2's worker handle directly and keep the channel's receiver in this test β€” this + // is what makes the dispatch count observable synchronously, without any async worker task or + // transport mock (`send_to_worker_or_spawn` finds this handle and reuses it, so the real + // worker-spawn path β€” the only place that would touch `ctx.transport` β€” is never exercised). + // `ReplicationWorkerHandle`/`ReplicationTask` are private to `leader_state`, visible here only + // because this test module nests under it β€” same access pattern `inject_dead_worker_for_test` + // already relies on for the sibling `worker_lifecycle_test.rs` file. + let (task_tx, mut task_rx) = mpsc::unbounded_channel(); + state.replication_workers.insert( + 2, + super::ReplicationWorkerHandle { + task_tx, + snapshot_failure_count: 0, + snapshot_next_retry_at: None, + }, + ); + + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel::(); + + // Batch 1: peer 2 starts in `Probe` with nothing in flight β€” must be dispatched. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "first batch must reach peer 2's worker β€” it had no outstanding request" + ); + + // Batch 2: peer 2's first request has not been acknowledged (handle_append_result was never + // called), so it is still awaiting a response. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_err(), + "peer 2 already has an unacknowledged request in flight β€” a Probe-state peer must not \ + receive a second AppendEntries until the first is acked. See \ + 446-expert-q-probe-backpressure-fix-8020.md for the etcd/raft and openraft references." + ); +} + +/// Shared setup for the gate open/close tests below: a two-peer cluster with peer 2's worker +/// channel handed to the test, so dispatch is observable synchronously without a real transport. +async fn setup_gate_harness( + path: &str +) -> ( + crate::raft_context::RaftContext, + LeaderState, + mpsc::UnboundedReceiver, + mpsc::UnboundedSender, +) { + let (_graceful_tx, graceful_rx) = watch::channel(()); + let mut ctx = mock_raft_context(path, graceful_rx, None); + ctx.membership = Arc::new(two_peer_membership()); + + let mut raft_log = MockRaftLog::new(); + raft_log.expect_last_entry_id().returning(|| 0); + raft_log.expect_flush().returning(|| Ok(())); + raft_log.expect_save_hard_state().returning(|_| Ok(())); + ctx.storage.raft_log = Arc::new(raft_log); + + let mut state = LeaderState::::new(1, ctx.node_config.clone()); + state.init_cluster_metadata(&ctx.membership).await.unwrap(); + + let (task_tx, task_rx) = mpsc::unbounded_channel(); + state.replication_workers.insert( + 2, + super::ReplicationWorkerHandle { + task_tx, + snapshot_failure_count: 0, + snapshot_next_retry_at: None, + }, + ); + + let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel::(); + (ctx, state, task_rx, internal_event_tx) +} + +/// An unparseable response (no `result` variant) must still reopen the gate: the request is no +/// longer in flight, even though the leader learned nothing usable from it. Latching here would +/// freeze the peer exactly like the reject case. +#[tokio::test] +#[traced_test] +async fn test_probe_peer_dispatches_next_probe_after_unparseable_response() { + let (mut ctx, mut state, mut task_rx, internal_event_tx) = + setup_gate_harness("/tmp/test_probe_peer_dispatches_next_probe_after_unparseable_response") + .await; + + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(|_, _, _, _, _| { + Ok(crate::PrepareResult { + append_requests: vec![(2, stub_probe_request(), 1)], + snapshot_targets: vec![], + }) + }); + + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!(task_rx.try_recv().is_ok(), "first probe must be dispatched"); + + state + .handle_append_result( + 2, + Ok(AppendEntriesResponse { + node_id: 2, + term: 1, + result: None, + }), + &ctx, + &internal_event_tx, + ) + .await + .unwrap(); + + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "an unparseable response still resolves the outstanding probe β€” the leader must re-probe" + ); +} + +/// An empty (heartbeat) dispatch must NOT set the gate, otherwise a heartbeat would occupy the +/// "probe in flight" slot and block the next real probe. +#[tokio::test] +#[traced_test] +async fn test_heartbeat_does_not_latch_gate() { + let (mut ctx, mut state, mut task_rx, internal_event_tx) = + setup_gate_harness("/tmp/test_heartbeat_does_not_latch_gate").await; + + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(move |_, _, _, _, _| { + let n = calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let req = if n == 0 { + stub_request() + } else { + stub_probe_request() + }; + Ok(crate::PrepareResult { + append_requests: vec![(2, req, 1)], + snapshot_targets: vec![], + }) + }); + + // Batch 1 is an empty heartbeat β€” it must be dispatched but leave the gate open. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!(task_rx.try_recv().is_ok(), "heartbeat must be dispatched"); + + // Batch 2 is a real probe β€” it must not be blocked by a latch the heartbeat never set. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "an empty heartbeat must not latch the gate β€” the following probe must go out" + ); +} + +/// While a non-empty probe is in flight, an empty heartbeat must still be dispatched: the gate +/// throttles probes only, never heartbeats β€” this is the liveness backstop that unfreezes a peer +/// whose probe response was lost or unparseable. +#[tokio::test] +#[traced_test] +async fn test_heartbeat_bypasses_gate_while_probe_in_flight() { + let (mut ctx, mut state, mut task_rx, internal_event_tx) = + setup_gate_harness("/tmp/test_heartbeat_bypasses_gate_while_probe_in_flight").await; + + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(move |_, _, _, _, _| { + let n = calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let req = if n == 0 { + stub_probe_request() + } else { + stub_request() + }; + Ok(crate::PrepareResult { + append_requests: vec![(2, req, 1)], + snapshot_targets: vec![], + }) + }); + + // Batch 1 is a non-empty probe β€” it latches the gate. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!(task_rx.try_recv().is_ok(), "first probe must be dispatched"); + + // Batch 2 is an empty heartbeat β€” it must bypass the gate and still be dispatched. + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_ok(), + "a heartbeat must bypass the probe gate β€” it is the liveness backstop, not a probe" + ); +} + +/// A stale-term response belongs to an older request, not the outstanding probe, so it must NOT +/// reopen the gate. Liveness is instead recovered by the heartbeat backstop (see the bypass test), +/// mirroring etcd's `MaybeUpdate(n <= Match)` early-return-without-resume. +#[tokio::test] +#[traced_test] +async fn test_stale_term_response_keeps_gate_latched() { + let (mut ctx, mut state, mut task_rx, internal_event_tx) = + setup_gate_harness("/tmp/test_stale_term_response_keeps_gate_latched").await; + + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(2) + .returning(|_, _, _, _, _| { + Ok(crate::PrepareResult { + append_requests: vec![(2, stub_probe_request(), 1)], + snapshot_targets: vec![], + }) + }); + + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!(task_rx.try_recv().is_ok(), "first probe must be dispatched"); + + // term 0 < leader_term 1 β†’ stale, ignored without clearing the gate. + state + .handle_append_result( + 2, + Ok(AppendEntriesResponse { + node_id: 2, + term: 0, + result: None, + }), + &ctx, + &internal_event_tx, + ) + .await + .unwrap(); + + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + assert!( + task_rx.try_recv().is_err(), + "a stale-term response must not reopen the gate β€” the outstanding probe is still in flight" + ); +} diff --git a/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs b/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs index 3b9b8d07..2f6e7ec1 100644 --- a/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/single_voter_commit_test.rs @@ -1,17 +1,16 @@ //! Single-Voter Commit Path Tests //! -//! Regression tests for the MemFirst single-voter commit path in `handle_log_flushed`. +//! RPO=0 (#446): `handle_log_flushed` single-voter branch must commit to `durable`, not +//! `last_entry_id()` β€” a single-voter cluster has no majority to fall back on, so if the +//! leader itself hasn't fsynced an entry, there is no copy anywhere safe from power loss. //! -//! ## Bug History -//! `fix #329` changed `handle_log_flushed` single-voter branch to commit to `durable` -//! instead of `last_entry_id()`. This placed IO thread latency on the commit critical -//! path, causing a ~3x latency regression in 3-node embedded bench (1731Β΅s vs ~566Β΅s). -//! -//! ## MemFirst Single-Voter Invariant -//! `LogFlushed(durable)` is an IO checkpoint. Commit must advance to `last_entry_id()` -//! β€” not just `durable` β€” to allow pipelining across IO batch boundaries. -//! This matches the multi-voter path where the leader contributes `last_entry_id()` to -//! quorum (not `durable_index`). +//! ## Superseded design (kept as history, do not resurrect) +//! `fix #329` changed this branch to commit to `durable` instead of `last_entry_id()`, +//! then reverted it after measuring a ~3x latency regression in 3-node embedded bench +//! (1731Β΅s vs ~566Β΅s) β€” IO thread latency landed on the commit critical path. That +//! regression is real and will resurface here. RPO=0 makes paying it mandatory for +//! single-voter clusters β€” there is no majority to absorb the risk the old design +//! accepted. use crate::MockMembership; use crate::MockRaftLog; @@ -61,16 +60,16 @@ async fn setup_single_voter( (state, ctx, last_entry_id) } -/// MemFirst single-voter: `handle_log_flushed` must commit to `last_entry_id`, not `durable`. +/// RPO=0: `handle_log_flushed` must commit to `durable`, not `last_entry_id`. /// -/// Simulates: IO batch flushed entries 1-3 (`durable=3`), but entries 4-5 arrived -/// in memory during the flush (`last_entry_id=5`). MemFirst: commit must advance -/// to 5 (all in-memory entries), not stall at 3 (only persisted entries). +/// Simulates: entries 4-5 arrived in memory (`last_entry_id=5`) but the IO batch has +/// only flushed entries 1-3 so far (`durable=3`). Commit must stay at 3 β€” entries 4-5 +/// aren't crash-safe yet, and a single-voter cluster has no other copy to fall back on. /// -/// This test FAILS if `handle_log_flushed` uses `durable` for commit -/// (the `fix #329` regression that caused +617Β΅s avg latency in 3-node embedded bench). +/// This test FAILS if `handle_log_flushed` still uses `last_entry_id` for commit (the +/// old MemFirst behavior, since revoked β€” RPO=0 makes single-voter durability mandatory). #[tokio::test] -async fn test_single_voter_commit_uses_last_entry_id_not_durable() { +async fn test_single_voter_commit_uses_durable_not_last_entry_id() { // last_entry_id=5: entries 4-5 arrived in memory during the IO flush of 1-3 let (mut state, ctx, _last_entry_id) = setup_single_voter(5).await; let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); @@ -80,13 +79,13 @@ async fn test_single_voter_commit_uses_last_entry_id_not_durable() { assert_eq!( state.commit_index(), - 5, - "MemFirst single-voter: commit must use last_entry_id=5, not durable=3. \ - Using durable puts IO latency on the commit critical path." + 3, + "RPO=0: commit must use durable=3, not last_entry_id=5 β€” entries 4-5 aren't \ + fsynced yet, and single-voter has no majority to fall back on" ); } -/// After IO catches up (durable == last_entry_id), commit equals last_entry_id. +/// After IO catches up (durable == last_entry_id), commit equals durable. #[tokio::test] async fn test_single_voter_commit_when_durable_equals_last_entry_id() { let (mut state, ctx, _last_entry_id) = setup_single_voter(5).await; @@ -94,20 +93,16 @@ async fn test_single_voter_commit_when_durable_equals_last_entry_id() { state.handle_log_flushed(5, &ctx, &internal_event_tx).await; - assert_eq!( - state.commit_index(), - 5, - "commit must advance to last_entry_id=5 when durable=5" - ); + assert_eq!(state.commit_index(), 5, "commit must advance to durable=5"); } -/// Pipelining across multiple IO batches: each flush triggers commit to current last_entry_id. +/// Commit tracks `durable` across IO batches, not the in-memory tail. /// -/// Simulates rapid writes where IO batches lag behind in-memory log: -/// - Flush 1: IO flushed 1-3, log has 1-7 in memory β†’ commit=7 -/// - Flush 2: IO flushed 4-7, log has 1-10 in memory β†’ commit=10 +/// Simulates rapid writes where the in-memory log runs ahead of what's fsynced: +/// - Flush 1: IO flushed 1-3 (durable=3), memory has 1-7 β†’ commit=3, not 7 +/// - Flush 2: IO flushed 4-7 (durable=7), memory now has 1-10 β†’ commit=7, not 10 #[tokio::test] -async fn test_single_voter_pipelining_across_io_batches() { +async fn test_single_voter_commit_tracks_durable_not_memory_tail() { let (mut state, ctx, last_entry_id) = setup_single_voter(7).await; let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); @@ -115,8 +110,8 @@ async fn test_single_voter_pipelining_across_io_batches() { state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), - 7, - "commit must advance to last_entry_id=7" + 3, + "commit must stay at durable=3 β€” entries 4-7 aren't fsynced yet" ); // IO batch 2: flushed 4-7, memory now has 1-10 @@ -124,14 +119,14 @@ async fn test_single_voter_pipelining_across_io_batches() { state.handle_log_flushed(7, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), - 10, - "commit must advance to last_entry_id=10" + 7, + "commit must advance to durable=7, not the in-memory tail (10)" ); } -/// No-op flush: last_entry_id == commit_index means nothing new to commit. +/// No-op flush: durable == commit_index means nothing new is safe to commit yet. #[tokio::test] -async fn test_single_voter_no_commit_when_nothing_new() { +async fn test_single_voter_no_commit_when_nothing_new_durable() { let (mut state, ctx, _last_entry_id) = setup_single_voter(3).await; let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); @@ -139,11 +134,11 @@ async fn test_single_voter_no_commit_when_nothing_new() { state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!(state.commit_index(), 3); - // Second flush with same last_entry_id=3: no new entries β†’ no commit advance + // Second flush with same durable=3: nothing new is fsynced β†’ no commit advance state.handle_log_flushed(3, &ctx, &internal_event_tx).await; assert_eq!( state.commit_index(), 3, - "commit must not advance when last_entry_id == commit_index" + "commit must not advance when durable == commit_index" ); } diff --git a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs index a2f31812..f4d5c91b 100644 --- a/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/snapshot_worker_test.rs @@ -429,7 +429,7 @@ async fn test_worker_forwards_any_append_task_it_is_given_without_inspecting_sta // Bypasses process_batch/Phase 5 on purpose β€” see doc comment above. state.send_to_worker_or_spawn( 2, - super::ReplicationTask::Append(stub_append_request()), + super::ReplicationTask::Append(stub_append_request(), tokio::time::Instant::now()), super::ReplicationWorkerConfig { transport: Arc::new(transport), membership: ctx.membership.clone(), diff --git a/d-engine-core/src/raft_role/learner_state.rs b/d-engine-core/src/raft_role/learner_state.rs index 01c735b1..f465b3e8 100644 --- a/d-engine-core/src/raft_role/learner_state.rs +++ b/d-engine-core/src/raft_role/learner_state.rs @@ -22,6 +22,7 @@ use crate::alias::MOF; use crate::cluster_printer::print_learner_join_success; use crate::cluster_printer::print_learner_promoted_to_voter; use crate::cluster_printer::print_role_transition_line; +use crate::role_state::PendingAck; use crate::role_state::schedule_and_execute_purge; use async_trait::async_trait; use d_engine_proto::common::LogId; @@ -35,6 +36,7 @@ use d_engine_proto::server::cluster::LeaderDiscoveryResponse; use d_engine_proto::server::election::VoteResponse; use d_engine_proto::server::election::VotedFor; use d_engine_proto::server::storage::SnapshotMetadata; +use std::collections::BTreeMap; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; @@ -88,6 +90,10 @@ pub struct LearnerState { /// reflected in the latest snapshot. pub last_purged_index: Option, + /// AppendEntries responses withheld pending this node's own durable_index. + /// See `role_state::PendingAck`. + pending_append_acks: BTreeMap, + // -- Snapshot Management -- /// Prevents concurrent snapshot creation /// @@ -514,6 +520,10 @@ impl RaftRoleState for LearnerState { fn pending_purge_upto_mut(&mut self) -> Option<&mut Option> { Some(&mut self.pending_purge_upto) } + + fn pending_append_acks_mut(&mut self) -> Option<&mut BTreeMap> { + Some(&mut self.pending_append_acks) + } } impl LearnerState { @@ -537,6 +547,7 @@ impl LearnerState { shared_state: SharedState::new(node_id, None, None), last_purged_index: None, snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), node_config, _marker: PhantomData, pending_purge_upto: None, @@ -646,6 +657,7 @@ impl From<&FollowerState> for LearnerState { shared_state: follower_state.shared_state.clone(), node_config: follower_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: follower_state.last_purged_index, pending_purge_upto: follower_state.pending_purge_upto, _marker: PhantomData, @@ -658,6 +670,7 @@ impl From<&CandidateState> for LearnerState { shared_state: candidate_state.shared_state.clone(), node_config: candidate_state.node_config.clone(), snapshot_in_progress: AtomicBool::new(false), + pending_append_acks: BTreeMap::new(), last_purged_index: candidate_state.last_purged_index, pending_purge_upto: None, _marker: PhantomData, diff --git a/d-engine-core/src/raft_role/learner_state_test.rs b/d-engine-core/src/raft_role/learner_state_test.rs index 7eaf7bc7..07d32edb 100644 --- a/d-engine-core/src/raft_role/learner_state_test.rs +++ b/d-engine-core/src/raft_role/learner_state_test.rs @@ -363,6 +363,16 @@ async fn test_learner_handles_append_entries_success() { assert_eq!(state.current_term(), leader_term); assert_eq!(state.commit_index(), expected_commit); + // RPO=0 (#446): the success ACK is withheld until durable_index reaches the claimed index. + assert!( + resp_rx.try_recv().is_err(), + "ACK must be withheld until durable_index catches up" + ); + + // Simulate LogFlushed: durable_index advances to 1, releasing the withheld ACK. + let (flush_tx, _flush_rx) = mpsc::unbounded_channel(); + state.handle_log_flushed(1, &context, &flush_tx).await; + let response = resp_rx.recv().await.unwrap().unwrap(); assert!(response.is_success()); } @@ -1726,63 +1736,6 @@ async fn test_apply_completed_respects_snapshot_disabled_config() { ); } -// ============================================================================ -// MemFirst ACK Tests -// ============================================================================ - -/// Learner ACKs leader immediately after memory write (MemFirst). -#[tokio::test] -async fn test_learner_acks_immediately_after_memory_write() { - let (_graceful_tx, graceful_rx) = watch::channel(()); - let (mut context, _temp_dir) = mock_raft_context_with_temp(graceful_rx, None); - - let leader_term = 2u64; - let appended_index = 5u64; - - let mut replication_handler = crate::MockReplicationCore::new(); - replication_handler.expect_handle_append_entries().returning(move |_, _, _| { - Ok(crate::AppendResponseWithUpdates { - response: d_engine_proto::server::replication::AppendEntriesResponse::success( - 1, - leader_term, - Some(LogId { - term: leader_term, - index: appended_index, - }), - ), - commit_index_update: None, - }) - }); - context.handlers.replication_handler = replication_handler; - context.membership = Arc::new(MockMembership::new()); - - let mut state = LearnerState::::new(1, context.node_config.clone()); - state.update_current_term(leader_term); - - let append_request = d_engine_proto::server::replication::AppendEntriesRequest { - term: leader_term, - leader_id: 2, - prev_log_index: 0, - prev_log_term: 0, - entries: vec![], - leader_commit_index: 0, - }; - let (resp_tx, mut resp_rx) = MaybeCloneOneshot::new(); - let inbound_event = InboundEvent::AppendEntries(append_request, vec![resp_tx]); - let (internal_event_tx, _internal_event_rx) = mpsc::unbounded_channel(); - - assert!( - state - .handle_inbound_event(inbound_event, &context, internal_event_tx) - .await - .is_ok() - ); - - // MemFirst: ACK sent immediately - let response = resp_rx.try_recv().expect("ACK must be sent immediately after memory write"); - assert!(response.unwrap().is_success()); -} - /// Spawns a fake Worker that answers exactly one `InstallSnapshot` command with `result`, /// then exits. Returns a `StateMachineCommandSender` wired to it β€” stands in for the real /// `StateMachineWorker`, which isn't running in these role-layer unit tests. diff --git a/d-engine-core/src/raft_role/mod.rs b/d-engine-core/src/raft_role/mod.rs index b7737611..cea6639e 100644 --- a/d-engine-core/src/raft_role/mod.rs +++ b/d-engine-core/src/raft_role/mod.rs @@ -16,6 +16,8 @@ mod follower_state_test; #[cfg(test)] mod learner_state_test; #[cfg(test)] +mod pending_ack_test; +#[cfg(test)] mod role_state_test; use std::collections::HashMap; @@ -48,7 +50,6 @@ use super::InternalEvent; use super::RaftContext; use crate::Result; use crate::TypeConfig; -use crate::role_state::PeerReplicationState; /// The role state focuses solely on its own logic /// and does not directly manipulate the underlying storage or network. @@ -374,6 +375,36 @@ impl RaftRole { pub(crate) fn become_learner(&self) -> Result> { self.state().become_learner() } + /// Move the withheld-ACK queue out of the current role before a transition. + /// Only Follower and Learner keep one; every other role yields an empty map. + /// + /// A withheld ACK describes this node's durable log, not its role. Dropping it + /// on a `Learner -> Follower` promotion would strand the leader waiting on a + /// response that never arrives (#446). + pub(crate) fn take_pending_acks( + &mut self + ) -> std::collections::BTreeMap { + self.state_mut() + .pending_append_acks_mut() + .map(std::mem::take) + .unwrap_or_default() + } + + /// Install a carried withheld-ACK queue into the role a transition produced. + /// Follower and Learner adopt it; any other role cannot hold it, so its + /// entries are failed with a conflict response. + pub(crate) fn restore_pending_acks( + &mut self, + acks: std::collections::BTreeMap, + ) { + let node_id = self.state().node_id(); + let current_term = self.state().current_term(); + match self.state_mut().pending_append_acks_mut() { + Some(queue) => *queue = acks, + None => role_state::reject_pending_acks(acks, node_id, current_term), + } + } + pub fn current_term(&self) -> u64 { self.state().current_term() } @@ -386,28 +417,6 @@ impl RaftRole { self.state_mut().init_peers_next_index_and_match_index(last_entry_id, peer_ids) } - /// Reset `next_index[peer] = match_index[peer] + 1` after a bidi stream disconnect. - /// Ensures the next heartbeat re-sends any unACKed in-flight entries. - pub(crate) fn handle_peer_stream_error( - &mut self, - peer_id: u32, - ) { - // The bidi stream only carries AppendEntries. While this peer is in Snapshot - // state, an error on this stream says nothing about the independent - // connection the snapshot transfer runs on, so it has no authority to act - // (mirrors etcd raft.go MsgUnreachable: only BecomeProbe() when StateReplicate). - if self.state().peer_replication_state(peer_id) == PeerReplicationState::Snapshot { - return; - } - let match_idx = self.state().match_index(peer_id).unwrap_or(0); - let _ = self.state_mut().update_next_index(peer_id, match_idx + 1); - - // #436: stream is down, we don't know what (if anything) the peer received β€” - // stop trusting speculative advance (etcd: BecomeProbe on MsgUnreachable). - self.state_mut() - .set_peer_replication_state(peer_id, PeerReplicationState::Probe); - } - pub(crate) async fn handle_zombie_detected( &mut self, node_id: u32, diff --git a/d-engine-core/src/raft_role/pending_ack_test.rs b/d-engine-core/src/raft_role/pending_ack_test.rs new file mode 100644 index 00000000..5a445502 --- /dev/null +++ b/d-engine-core/src/raft_role/pending_ack_test.rs @@ -0,0 +1,223 @@ +//! Tests for #446's withheld-AppendEntries-ACK primitives: +//! +//! - `RaftRoleState::resolve_pending_acks` β€” the release / reject decision; +//! - `RaftRole::{take,restore}_pending_acks` β€” carrying the queue across a role +//! transition instead of dropping it. +//! +//! These drive bare role structs (no `RaftContext`, no mocks, no fs), so they +//! stay cheap to run and cheap to change. + +use std::sync::Arc; + +use d_engine_proto::common::LogId; +use d_engine_proto::server::replication::AppendEntriesResponse; +use d_engine_proto::server::replication::SuccessResult; +use d_engine_proto::server::replication::append_entries_response; +use tonic::Status; + +use super::RaftRole; +use super::candidate_state::CandidateState; +use super::follower_state::FollowerState; +use super::learner_state::LearnerState; +use crate::MaybeCloneOneshot; +use crate::MaybeCloneOneshotReceiver; +use crate::RaftNodeConfig; +use crate::RaftOneshot; +use crate::raft_role::role_state::PendingAck; +use crate::raft_role::role_state::RaftRoleState; +use crate::test_utils::mock::MockTypeConfig; + +type Rx = MaybeCloneOneshotReceiver>; + +fn config() -> Arc { + Arc::new( + RaftNodeConfig::new() + .expect("RaftNodeConfig::new") + .validate() + .expect("RaftNodeConfig::validate"), + ) +} + +fn follower(term: u64) -> RaftRole { + let mut s = FollowerState::new(1, config(), None, None); + s.shared_state_mut().update_current_term(term); + RaftRole::Follower(Box::new(s)) +} + +fn learner(term: u64) -> RaftRole { + let mut s = LearnerState::new(1, config()); + s.shared_state_mut().update_current_term(term); + RaftRole::Learner(Box::new(s)) +} + +fn candidate() -> RaftRole { + RaftRole::Candidate(Box::new(CandidateState::new(1, config()))) +} + +/// Put a withheld success ACK for `index` straight into `role`'s queue, bypassing +/// the AppendEntries workflow. Returns the receiver a caller would be blocked on. +fn withhold( + role: &mut RaftRole, + index: u64, + claimed_term: u64, + term_when_withheld: u64, +) -> Rx { + let (tx, rx) = MaybeCloneOneshot::new(); + role.state_mut() + .pending_append_acks_mut() + .expect("role keeps a pending-ack queue") + .insert( + index, + PendingAck { + claimed_term, + term_when_withheld, + senders: vec![tx], + }, + ); + rx +} + +fn queue_len(role: &mut RaftRole) -> usize { + role.state_mut().pending_append_acks_mut().map_or(0, |q| q.len()) +} + +// -- resolve_pending_acks ----------------------------------------------------- + +/// A withheld ACK is released as a success once its claimed index is durable and +/// the node is still on the term it withheld under. +#[test] +fn test_resolve_releases_success_when_durable() { + let mut role = follower(5); + let mut rx = withhold(&mut role, 8, 5, 5); + + role.state_mut().resolve_pending_acks(8); + + let resp = rx.try_recv().expect("released").unwrap(); + assert!(matches!( + resp.result, + Some(append_entries_response::Result::Success(SuccessResult { + last_match: Some(LogId { index: 8, term: 5 }), + })), + )); + assert_eq!(queue_len(&mut role), 0); +} + +/// A withheld ACK stays queued while its claimed index is still beyond durable. +#[test] +fn test_resolve_keeps_waiting_until_durable() { + let mut role = follower(5); + let mut rx = withhold(&mut role, 8, 5, 5); + + role.state_mut().resolve_pending_acks(7); + + assert!(rx.try_recv().is_err()); + assert_eq!(queue_len(&mut role), 1); +} + +/// A withheld ACK is rejected with a conflict if the node moved to a newer term +/// since withholding: a higher-term leader may have overwritten the log at that +/// index, so the durability claim can no longer be trusted (#446). +#[test] +fn test_resolve_rejects_stale_term_ack() { + let mut role = follower(6); // node is now on term 6 + let mut rx = withhold(&mut role, 8, 5, 5); // ACK was withheld under term 5 + + role.state_mut().resolve_pending_acks(8); + + let resp = rx.try_recv().expect("resolved").unwrap(); + assert!( + !resp.is_success(), + "stale-term ACK must resolve to a conflict" + ); + assert_eq!(queue_len(&mut role), 0); +} + +/// Every sender queued on one index is answered β€” a leader retry or a heartbeat +/// can leave more than one waiter on the same index. +#[test] +fn test_resolve_answers_every_sender_on_an_index() { + let mut role = follower(5); + let (tx1, mut rx1) = MaybeCloneOneshot::new(); + let (tx2, mut rx2) = MaybeCloneOneshot::new(); + role.state_mut().pending_append_acks_mut().unwrap().insert( + 8, + PendingAck { + claimed_term: 5, + term_when_withheld: 5, + senders: vec![tx1, tx2], + }, + ); + + role.state_mut().resolve_pending_acks(8); + + assert!(rx1.try_recv().unwrap().unwrap().is_success()); + assert!(rx2.try_recv().unwrap().unwrap().is_success()); +} + +/// `resolve_pending_acks` on a role that keeps no queue (Candidate) is a no-op. +#[test] +fn test_resolve_is_noop_without_a_queue() { + let mut role = candidate(); + role.state_mut().resolve_pending_acks(10); // must not panic +} + +// -- take / restore across a role transition --------------------------------- + +/// A withheld ACK survives a Learner -> Follower promotion: `take_pending_acks` +/// moves the queue out of the old role and `restore_pending_acks` installs it in +/// the new one. Dropping it here would strand the leader on a response that never +/// arrives β€” the bug #446 fixed. +#[test] +fn test_pending_ack_survives_learner_promotion() { + let mut old = learner(5); + let mut rx = withhold(&mut old, 8, 5, 5); + + let carried = old.take_pending_acks(); + assert_eq!(carried.len(), 1); + assert_eq!( + queue_len(&mut old), + 0, + "take must move the queue, not copy it" + ); + + let mut new = follower(5); + new.restore_pending_acks(carried); + + new.state_mut().resolve_pending_acks(8); + assert!(rx.try_recv().expect("released after promotion").unwrap().is_success()); +} + +/// The symmetric Follower -> Learner demotion also carries the queue. +#[test] +fn test_pending_ack_survives_follower_demotion() { + let mut old = follower(5); + let mut rx = withhold(&mut old, 8, 5, 5); + + let carried = old.take_pending_acks(); + let mut new = learner(5); + new.restore_pending_acks(carried); + + new.state_mut().resolve_pending_acks(8); + assert!(rx.try_recv().expect("released after demotion").unwrap().is_success()); +} + +/// Restoring the queue into a role that cannot hold one (Candidate) fails every +/// withheld ACK with a conflict, so the leader retries rather than timing out. +#[test] +fn test_restore_into_candidate_fails_pending_acks() { + let mut old = follower(5); + let mut rx = withhold(&mut old, 8, 5, 5); + let carried = old.take_pending_acks(); + + let mut candidate = candidate(); + candidate.restore_pending_acks(carried); + + let resp = rx.try_recv().expect("failed, not dropped").unwrap(); + assert!(!resp.is_success()); +} + +/// `take_pending_acks` on a role with no queue yields an empty map, never panics. +#[test] +fn test_take_from_queueless_role_is_empty() { + assert!(candidate().take_pending_acks().is_empty()); +} diff --git a/d-engine-core/src/raft_role/role_state.rs b/d-engine-core/src/raft_role/role_state.rs index 89c6181e..8b542503 100644 --- a/d-engine-core/src/raft_role/role_state.rs +++ b/d-engine-core/src/raft_role/role_state.rs @@ -31,10 +31,13 @@ use d_engine_proto::common::LogId; use d_engine_proto::server::election::VotedFor; use d_engine_proto::server::replication::AppendEntriesRequest; use d_engine_proto::server::replication::AppendEntriesResponse; +use d_engine_proto::server::replication::SuccessResult; +use d_engine_proto::server::replication::append_entries_response; use d_engine_proto::server::storage::SnapshotAck; use d_engine_proto::server::storage::SnapshotChunk; use d_engine_proto::server::storage::SnapshotMetadata; use d_engine_proto::server::storage::SnapshotResponse; +use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::sync::mpsc; use tokio::time::Instant; @@ -56,6 +59,66 @@ pub(crate) enum PeerReplicationState { Snapshot, } +/// A success `AppendEntriesResponse` that has been computed but not yet sent, +/// because this node's own `durable_index` had not reached the index the response +/// claims. Held until fsync catches up, so an ACK never asserts durability the +/// node cannot yet guarantee (RPO=0, #446). +/// +/// Keyed by the claimed index. `senders` accumulates when more than one request +/// claims the same index (a leader retry, or a heartbeat landing on the tail). +/// +/// The response body is not stored: it is rebuilt on release, after re-checking +/// the term it was withheld under. A response frozen under a term the node has +/// since left must never be sent. +pub(crate) struct PendingAck { + pub(crate) claimed_term: u64, + pub(crate) term_when_withheld: u64, + pub(crate) senders: + Vec>>, +} + +/// Send the terminal response for one withheld ACK β€” a rebuilt success if +/// `confirm`, a conflict otherwise β€” to every accumulated sender. +fn resolve_pending_ack( + node_id: u32, + index: u64, + ack: PendingAck, + confirm: bool, + current_term: u64, +) { + let response = if confirm { + AppendEntriesResponse::success( + node_id, + current_term, + Some(LogId { + index, + term: ack.claimed_term, + }), + ) + } else { + AppendEntriesResponse::conflict(node_id, current_term, None, None) + }; + for sender in ack.senders { + if let Err(e) = sender.send(Ok(response)) { + error!("withheld AppendEntries ACK (index {index}): send failed: {e:?}"); + } + } +} + +/// Fail every withheld ACK with a conflict response. Used when the queue passes to +/// a role that cannot hold it (Candidate or Leader): the node no longer recognises +/// the leader those ACKs were owed to, so that leader's replication worker should +/// retry now rather than wait out an RPC timeout. +pub(crate) fn reject_pending_acks( + acks: BTreeMap, + node_id: u32, + current_term: u64, +) { + for (index, ack) in acks { + resolve_pending_ack(node_id, index, ack, false, current_term); + } +} + #[async_trait] pub(crate) trait RaftRoleState: Send + Sync + 'static { type T: TypeConfig; @@ -397,16 +460,60 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { Ok(()) } - /// Handle LogFlushed(durable) event: entries up to `durable` are now crash-safe. - /// Leader: recalculates commit_index (uses durable_index in quorum calculation). - /// Default: no-op for Candidate/Follower/Learner (ACK already sent on memory write). + /// Release withheld AppendEntries ACKs now that the log is durable through + /// `durable`. For each withheld ACK: + /// + /// - withheld under a term this node has since left β†’ fail it with a conflict. + /// A higher-term leader may have overwritten the log at that index; within a + /// single term a follower's entries are never replaced, so the term check + /// alone is a sufficient content guard and no log lookup is needed. + /// - claimed index now `<= durable` β†’ rebuild and send the success. + /// - otherwise β†’ keep waiting. + /// + /// No-op for Candidate/Leader (no queue). Runs on every fsync completion, so it + /// stays limited to integer comparisons β€” no log lookup. (#446) + fn resolve_pending_acks( + &mut self, + durable: u64, + ) { + let node_id = self.node_id(); + let current_term = self.current_term(); + let Some(pending) = self.pending_append_acks_mut() else { + return; + }; + if pending.is_empty() { + return; + } + let resolved: Vec<(u64, bool)> = pending + .iter() + .filter_map(|(&index, ack)| { + if ack.term_when_withheld != current_term { + Some((index, false)) // stale term -> conflict + } else if index <= durable { + Some((index, true)) // durable -> success + } else { + None // keep waiting + } + }) + .collect(); + for (index, confirm) in resolved { + if let Some(ack) = pending.remove(&index) { + resolve_pending_ack(node_id, index, ack, confirm, current_term); + } + } + } + + /// A batch of log entries reached `durable` on disk (fsync complete). + /// + /// Follower/Learner: release any withheld AppendEntries ACKs this now covers. + /// Leader: overridden to recalculate `commit_index`. Candidate: no-op. async fn handle_log_flushed( &mut self, - _durable: u64, + durable: u64, _ctx: &RaftContext, _internal_event_tx: &mpsc::UnboundedSender, ) { - // Candidate: no-op + self.resolve_pending_acks(durable); } /// Handle AppendEntries result from a per-follower ReplicationWorker. @@ -534,10 +641,18 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { // My term might be updated, has to fetch it again let my_term = self.current_term(); + // `state_snapshot` was captured at the top of `handle_inbound_event`, before + // `commit_hard_state` above may have advanced our term. Patch it here so the + // AppendEntriesResponse reports the real, just-updated term β€” not the stale + // snapshot. + let state_snapshot = StateSnapshot { + current_term: my_term, + ..state_snapshot.clone() + }; // Handle replication request match ctx .replication_handler() - .handle_append_entries(append_entries_request, state_snapshot, ctx.raft_log()) + .handle_append_entries(append_entries_request, &state_snapshot, ctx.raft_log()) .await { Ok(AppendResponseWithUpdates { @@ -560,13 +675,53 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { } debug!("AppendEntriesResponse: {:?}", response); - // MemFirst: ACK immediately after memory write. IO thread fsyncs async. - // Safety: quorum uses last_entry_id (in-memory); crash safety is guaranteed by - // majority replication, not per-follower durability. + // RPO=0 (#446): a success response asserts the claimed entry is + // fsync-durable on this node. If this node's own `durable_index` + // has not reached that index, withhold the response until it does + // (released by `resolve_pending_acks`). Conflict and higher-term + // responses assert nothing about durability and are sent at once. + let claim = match &response.result { + Some(append_entries_response::Result::Success(SuccessResult { + last_match: Some(log_id), + })) => Some((log_id.index, log_id.term)), + _ => None, + }; - for sender in senders { - if let Err(e) = sender.send(Ok(response)) { - error!("Failed to send: {:?}", e); + match claim { + Some((index, claimed_term)) if ctx.storage.raft_log.durable_index() < index => { + let term_when_withheld = self.current_term(); + match self.pending_append_acks_mut() { + Some(pending) => { + pending + .entry(index) + .or_insert_with(|| PendingAck { + claimed_term, + term_when_withheld, + senders: Vec::new(), + }) + .senders + .extend(senders); + } + None => { + // Only Follower and Learner produce a success + // response here, and both carry the queue. Reaching + // this arm means a role invariant broke β€” send the + // ACK now rather than strand the leader. + error!( + "withheld a success ACK on a role with no pending-ACK queue" + ); + for sender in senders { + let _ = sender.send(Ok(response)); + } + } + } + } + _ => { + for sender in senders { + if let Err(e) = sender.send(Ok(response)) { + error!("failed to send AppendEntries response: {e:?}"); + } + } } } } @@ -926,19 +1081,11 @@ pub(crate) trait RaftRoleState: Send + Sync + 'static { .into()) } - fn peer_replication_state( - &self, - _node_id: u32, - ) -> PeerReplicationState { - // Default: unknown peer, be conservative. Also the default for non-leader roles. - PeerReplicationState::Probe - } - - fn set_peer_replication_state( - &mut self, - _node_id: u32, - _state: PeerReplicationState, - ) { + /// The withheld-ACK queue, for the roles that keep one (Follower, Learner). + /// `None` for Candidate and Leader. Carried across a Follower<->Learner + /// transition by `RaftRole::take_pending_acks` / `restore_pending_acks` (#446). + fn pending_append_acks_mut(&mut self) -> Option<&mut BTreeMap> { + None } } diff --git a/d-engine-core/src/raft_test/raft_comprehensive_tests.rs b/d-engine-core/src/raft_test/raft_comprehensive_tests.rs index 6f169270..5062a8c5 100644 --- a/d-engine-core/src/raft_test/raft_comprehensive_tests.rs +++ b/d-engine-core/src/raft_test/raft_comprehensive_tests.rs @@ -1437,9 +1437,10 @@ async fn test_snapshot_push_completed_uses_snapshot_boundary_not_leader_tip() { let current_term = raft.current_term(); // Establish an active snapshot transfer β€” the handler seeds next_index only for a // peer that is actually mid-snapshot. - raft.role - .state_mut() - .set_peer_replication_state(peer_id, crate::role_state::PeerReplicationState::Snapshot); + let crate::RaftRole::Leader(leader) = &mut raft.role else { + panic!("expected Leader role after BecomeLeader"); + }; + leader.set_peer_replication_state(peer_id, crate::role_state::PeerReplicationState::Snapshot); raft.handle_internal_event(InternalEvent::SnapshotPushCompleted { peer_id, success: true, @@ -1660,17 +1661,24 @@ async fn test_peer_stream_error_does_not_touch_peer_in_snapshot_state() { raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); let peer_id = 42; - raft.role - .state_mut() - .set_peer_replication_state(peer_id, crate::role_state::PeerReplicationState::Snapshot); + { + let crate::RaftRole::Leader(leader) = &mut raft.role else { + panic!("expected Leader role after BecomeLeader"); + }; + leader + .set_peer_replication_state(peer_id, crate::role_state::PeerReplicationState::Snapshot); + } let next_index_before = raft.role.state().next_index(peer_id); raft.handle_internal_event(InternalEvent::PeerStreamError { peer_id }) .await .unwrap(); + let crate::RaftRole::Leader(leader) = &raft.role else { + panic!("expected Leader role after BecomeLeader"); + }; assert_eq!( - raft.role.state().peer_replication_state(peer_id), + leader.peer_replication_state(peer_id), crate::role_state::PeerReplicationState::Snapshot, "a bidi stream error must not downgrade a peer that is mid-snapshot-transfer" ); @@ -1696,16 +1704,25 @@ async fn test_peer_stream_error_downgrades_non_snapshot_peer_to_probe() { raft.handle_internal_event(InternalEvent::BecomeLeader).await.unwrap(); let peer_id = 42; - raft.role - .state_mut() - .set_peer_replication_state(peer_id, crate::role_state::PeerReplicationState::Replicate); + { + let crate::RaftRole::Leader(leader) = &mut raft.role else { + panic!("expected Leader role after BecomeLeader"); + }; + leader.set_peer_replication_state( + peer_id, + crate::role_state::PeerReplicationState::Replicate, + ); + } raft.handle_internal_event(InternalEvent::PeerStreamError { peer_id }) .await .unwrap(); + let crate::RaftRole::Leader(leader) = &raft.role else { + panic!("expected Leader role after BecomeLeader"); + }; assert_eq!( - raft.role.state().peer_replication_state(peer_id), + leader.peer_replication_state(peer_id), crate::role_state::PeerReplicationState::Probe, "a bidi stream error for a non-snapshotting peer must still downgrade it to Probe" ); diff --git a/d-engine-core/src/state_machine_handler/snapshot_policy/log_size.rs b/d-engine-core/src/state_machine_handler/snapshot_policy/log_size.rs index 96a466f7..da59406e 100644 --- a/d-engine-core/src/state_machine_handler/snapshot_policy/log_size.rs +++ b/d-engine-core/src/state_machine_handler/snapshot_policy/log_size.rs @@ -4,7 +4,7 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; - +use tracing::error; use tracing::trace; use tracing::warn; @@ -39,12 +39,28 @@ impl SnapshotPolicy for LogSizePolicy { let lag = self.calculate_lag(ctx); let threshold = self.threshold.load(Ordering::Relaxed); - if threshold > 0 && lag >= threshold.saturating_mul(10) { - warn!( - lag, - threshold, - "Log lag exceeds 10x snapshot threshold β€” snapshots may not be keeping up" - ); + metrics::gauge!("core.raft.snapshot.log_lag").set(lag as f64); + + // The in-memory Raft log grows until a snapshot purges it. If snapshot + // creation can't keep up with the write rate this climbs unbounded and + // eventually OOMs the node β€” make it loud well before that. + if threshold > 0 { + if lag >= threshold.saturating_mul(50) { + error!( + lag, + threshold, + "Raft log lag is 50x the snapshot threshold β€” snapshot creation is \ + NOT keeping up with writes; the in-memory log is growing unbounded \ + and will OOM this node. Check snapshot/apply throughput." + ); + } else if lag >= threshold.saturating_mul(10) { + warn!( + lag, + threshold, + "Raft log lag exceeds 10x the snapshot threshold β€” snapshots may \ + not be keeping up" + ); + } } let should_trigger = lag >= threshold; diff --git a/d-engine-core/src/state_machine_handler/worker_test.rs b/d-engine-core/src/state_machine_handler/worker_test.rs index 6c082abe..018e1806 100644 --- a/d-engine-core/src/state_machine_handler/worker_test.rs +++ b/d-engine-core/src/state_machine_handler/worker_test.rs @@ -1337,12 +1337,21 @@ async fn test_local_snapshot_ready_reports_operation_failed_when_superseded_clea let result = response_rx.await.unwrap(); - // Restore permissions before any assertion can panic and skip this β€” otherwise - // the tempdir is left behind, unremovable by the test harness's own cleanup. - let mut perms = std::fs::metadata(&dir_path).unwrap().permissions(); - perms.set_mode(0o700); - std::fs::set_permissions(&dir_path, perms).unwrap(); - std::fs::remove_dir_all(&dir_path).unwrap(); + // The tempdir might already be gone: `OwnedSnapshotDir::drop`'s detached cleanup + // thread (command.rs) races this teardown and, under load, can win β€” that's a + // benign outcome (goal is just "no leftover dir"), not a test failure. + if let Ok(meta) = std::fs::metadata(&dir_path) { + let mut perms = meta.permissions(); + perms.set_mode(0o700); + std::fs::set_permissions(&dir_path, perms).unwrap(); + if let Err(e) = std::fs::remove_dir_all(&dir_path) { + assert_eq!( + e.kind(), + std::io::ErrorKind::NotFound, + "unexpected teardown error: {e}" + ); + } + } assert!( matches!( diff --git a/d-engine-core/src/storage/buffered_raft_log.rs b/d-engine-core/src/storage/buffered_raft_log.rs index 713286e9..16ec557d 100644 --- a/d-engine-core/src/storage/buffered_raft_log.rs +++ b/d-engine-core/src/storage/buffered_raft_log.rs @@ -12,38 +12,52 @@ //! //! ## Write path //! -//! `append_entries` inserts entries into in-memory SkipMap, calls `write_notify.notify_one()`. -//! Multiple concurrent writers coalesce into a single IO thread wakeup. +//! `append_entries` inserts entries into the in-memory SkipMap, then sends a +//! fire-and-forget `IOTask::Persist` on the same channel that carries `Flush` / +//! `ReplaceRange` / `Purge` / `Reset` / `Shutdown`. One ordered wakeup source +//! means the IO thread never races two independent signals for the same pending +//! work (#446). A write burst's redundant `Persist`s are cheap no-ops: each finds +//! `persisted_index` already at `memory_max_index` and persists nothing. //! //! ## IO thread (notify-then-spawn-fsync) //! -//! On wakeup from `write_notify`: -//! 1. **Read** β€” scan SkipMap range `(durable_index, max_index]` -//! 2. **Persist** β€” write range to OS page cache via `persist_entries` -//! 3. **Spawn fsync** β€” dispatch fdatasync to `spawn_blocking` pool via `spawn_fsync`, return immediately -//! 4. **Loop** β€” back to `select!` for next wakeup; prior fsync runs concurrently in pool +//! On `IOTask::Persist` (`run_storage_tasks`) / `Flush` (`run_flush_turn`): +//! 1. **Read** β€” scan the SkipMap range `(persisted_index, memory_max_index]`, +//! where `persisted_index` is the IO thread's own page-cache frontier (not +//! the round-tripping `durable_index`) +//! 2. **Persist** β€” write the range to OS page cache via `persist_entries` (no fsync) +//! 3. **Dispatch fsync** β€” `FsyncCoordinator::submit()` hands the fdatasync to a +//! `spawn_blocking` task and returns immediately +//! 4. **Loop** β€” back to `select!`; the prior fsync runs concurrently in the pool //! -//! `durable_index` is advanced inside the blocking task via `advance_durable_and_notify` -//! (`fetch_max`, AcqRel). Multiple concurrent tasks completing out of order are safe: -//! a late-arriving lower index is a no-op. +//! ## How `durable_index` advances (#446 single-owner) //! -//! ## Fsync triggers +//! The blocking fsync task does **not** write `durable_index`. On completion it +//! calls `notify_fsync_completed(mark)`, sending `InternalEvent::FsyncCompleted(LogId)` to `raft.rs`'s +//! event loop β€” the sole owner of `durable_index`. That loop calls +//! `try_advance_durable_index(mark)`, which rejects the report if +//! `entry_term(mark.index) != Some(mark.term)` and clamps to `memory_max_index` +//! before `fetch_max`. Upstream, `FsyncCoordinator` keeps `pending_max` as a +//! term-first `(term, index)`: a newer term's mark always wins, so a stale +//! pre-truncation submit can never swallow the valid post-truncation one. A +//! `generation` bump (`bump_generation` / `fence_reset`) additionally fences a +//! round whose log was truncated or reset mid-flight. //! -//! All four triggers below funnel through `run_batch_turn` (persist pending -//! entries, drain any queued commands, then dispatch) into the single -//! `FsyncCoordinator::submit()` entry point β€” there is no separate inline path. +//! ## Fsync triggers //! -//! 1. **Notify-driven** (normal): `write_notify` β†’ `run_batch_turn` (no reply) -//! 2. **Explicit** (flush API): `flush()` β†’ `IOTask::Flush(tx)` β†’ `run_batch_turn` with reply sender -//! 3. **Idle timer** (safety net): `idle_flush_interval_ms` elapsed β†’ `persist_pending_range` + `submit()` -//! 4. **Shutdown**: `IOTask::Shutdown` β†’ `run_batch_turn`, then `close()` waits (bounded by -//! `shutdown_timeout_ms`) for the IO thread's runtime to drain any in-flight fsync task +//! 1. **Write-driven** (normal): `append_entries` β†’ `IOTask::Persist` β†’ `run_storage_tasks` +//! 2. **Explicit** (flush API): `flush()` β†’ `IOTask::Flush(tx)` β†’ `run_flush_turn` with a reply sender +//! 3. **Idle timer** (safety net): `idle_flush_interval_ms` elapsed β†’ +//! `persist_pending_range` + `FsyncCoordinator::submit()` directly (not via `run_flush_turn`) +//! 4. **Shutdown**: `IOTask::Shutdown` β†’ `run_flush_turn`, then `close()` waits +//! (bounded by `shutdown_timeout_ms`) for the IO thread's runtime to drain any in-flight fsync //! //! ## Durability contract //! -//! `durable_index` advances only after physical fdatasync in the blocking task. -//! Concurrent fsyncs coalesce at the storage layer: if batch B's fsync covers A's WAL -//! position, A's `flush_wal` returns fast with no extra disk IO β€” storage-layer group commit. +//! `durable_index` advances only after a physical fdatasync completes β€” and only +//! on `raft.rs`'s event loop, after content validation. Concurrent fsyncs +//! coalesce at the storage layer: if batch B's fsync covers A's WAL position, +//! A's `flush_wal` returns fast with no extra disk IO β€” storage-layer group commit. use super::fsync_coordinator::FsyncCoordinator; use crate::Error; @@ -72,7 +86,6 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::time::Duration; -use tokio::sync::Notify; use tokio::sync::mpsc; use tokio::sync::oneshot; use tracing::debug; @@ -218,6 +231,11 @@ pub enum IOTask { /// result (Ok or Err) back to the caller via the oneshot channel. /// Replaces the former `FlushNow` + `WaitDurable` two-message dance. Flush(oneshot::Sender>), + /// Persist the IO thread to persist newly-appended entries. Fire-and-forget, + /// carries no data β€” `append_entries` sends one after each insert. Redundant + /// `Persist`s from writes coalesced while the IO thread was busy are drained + /// and discarded in a single `run_flush_turn`. + Persist, /// Shutdown the IO thread Shutdown, } @@ -258,14 +276,15 @@ where // Raft must not tell a client or a peer a write is safe ahead of this point, // regardless of what's already visible in `entries`. pub(crate) durable_index: AtomicU64, + // The next index to be allocated pub(crate) next_id: AtomicU64, // --- In-memory index --- // O(1) answer to "is this index currently held in memory" β€” lets callers // (e.g. entry_term()) reject an out-of-range index without touching `entries`. - min_index: AtomicU64, // Smallest log index (0 if empty) - max_index: AtomicU64, // Largest log index (0 if empty) + min_index: AtomicU64, // Smallest log index (0 if empty) + memory_max_index: AtomicU64, // Largest log index held in memory (0 if empty) β€” may be ahead of what's persisted/durable // The term of the last entry ever purged (compacted away after a snapshot). // Raft's AppendEntries consistency check needs the term at prev_log_index @@ -273,7 +292,7 @@ where // this, a follower can't tell "purged, but we agree" apart from "conflict". // // Must be published in the same critical section as the entries removal - // and the min_index/max_index advance it corresponds to β€” a reader must + // and the min_index/memory_max_index advance it corresponds to β€” a reader must // never be able to observe the entry gone but this boundary not yet set. last_purged_index: AtomicU64, last_purged_term: AtomicU64, @@ -290,10 +309,9 @@ where term_segments: TermSegments, // --- Flush coordination --- - /// Coalesced write notification. `append_entries` calls `notify_one()` after - /// inserting into the SkipMap. Multiple concurrent writers coalesce into a - /// single IO thread wakeup, eliminating per-write kernel cond_signal overhead. - pub(crate) write_notify: Arc, + /// The IO thread's sole inbound channel: `Persist` / `Flush` / `ReplaceRange` / `Purge` / `Reset` / `Shutdown`. One ordered + /// source so the IO thread never races two independent wakeup signals for + /// the same pending work (#446). pub(crate) command_sender: mpsc::UnboundedSender, // --- P0: LogFlushed event notification --- @@ -329,7 +347,7 @@ where } fn last_entry_id(&self) -> u64 { - self.max_index.load(Ordering::Acquire) + self.memory_max_index.load(Ordering::Acquire) } fn durable_index(&self) -> u64 { @@ -346,6 +364,9 @@ where } } + // #446: this is what election-eligibility comparisons (is_target_log_more_recent) + // read. It must keep reflecting the in-memory tail, never durable_index β€” a node + // with an un-fsynced entry must still be able to reject a less-up-to-date candidate. fn last_log_id(&self) -> Option { let last_index = self.last_entry_id(); if last_index > 0 { @@ -378,7 +399,7 @@ where entry_id: u64, ) -> Option { // Bounds check: skip TermSegments entirely for out-of-range queries. - let max = self.max_index.load(Ordering::Acquire); + let max = self.memory_max_index.load(Ordering::Acquire); let min = self.min_index.load(Ordering::Acquire); if max == 0 || entry_id < min || entry_id > max { // Cold path: check purge boundary so that AppendEntries built with @@ -465,10 +486,17 @@ where } self.insert_to_memory(&entries); - // Signal IO thread to persist. Multiple concurrent notify_one() calls - // while the IO thread is busy coalesce into one wakeup β€” no per-write - // kernel cond_signal. IO thread reads from SkipMap via max_index. - self.write_notify.notify_one(); + + // Persist the IO thread. Fire-and-forget: the entries are already in the + // in-memory log and quorum-visible; the IO thread scans the SkipMap and + // persists + fsyncs off this task. `Persist` rides the same channel as + // `Flush` / `ReplaceRange` / `Shutdown` so the IO thread has one ordered + // wakeup source (#446). RPO=0 is enforced downstream β€” commit quorum + // counts only `durable_index()`, and followers withhold AppendEntries + // ACKs until their own `durable_index` catches up. + self.command_sender + .send(IOTask::Persist) + .map_err(|e| NetworkError::SingalSendFailed(format!("Persist send failed: {e:?}")))?; Ok(()) } @@ -488,22 +516,16 @@ where new_entries: Vec, ) -> Result> { let _timer = ScopedTimer::new("filter_out_conflicts_and_append"); - // prev_log_index == 0 means the leader wants the follower to start from scratch - // (e.g. new follower joining, or follower log fully diverged). Reset and replace. - if prev_log_index == 0 && prev_log_term == 0 { - self.reset().await?; - self.append_entries(new_entries.clone()).await?; - return Ok(new_entries.last().map(|e| LogId { - term: e.term, - index: e.index, - })); - } - // Check log consistency: use entry_term() so purge-boundary entries - // (entries removed from the SkipMap but recorded in last_purged_index/term) - // are still recognised as valid prev_log positions after snapshot install. - if self.entry_term(prev_log_index) != Some(prev_log_term) { - return Ok(self.last_log_id()); + // prev_log_index==0 has no real entry to compare against, not a reset signal + let is_virtual_log_start = prev_log_index == 0 && prev_log_term == 0; + if !is_virtual_log_start { + // Check log consistency: use entry_term() so purge-boundary entries + // (entries removed from the SkipMap but recorded in last_purged_index/term) + // are still recognised as valid prev_log positions after snapshot install. + if self.entry_term(prev_log_index) != Some(prev_log_term) { + return Ok(self.last_log_id()); + } } let last_current_index = self.last_entry_id(); @@ -573,7 +595,7 @@ where if diverge_index <= last_current_index { // Real term conflict: truncate from diverge_index, replace with tail. // Await the done channel so callers can flush() knowing the truncation - // is durable β€” durable_index may exceed max_index after truncation, + // is durable β€” durable_index may exceed memory_max_index after truncation, // which would cause flush() to short-circuit before the replace lands. self.remove_range(diverge_index..=u64::MAX); self.insert_to_memory(tail); @@ -617,11 +639,10 @@ where mut peer_matched_ids: Vec, ) -> Option { let _timer = ScopedTimer::new("calculate_majority_matched_index"); - // Leader's contribution: last_entry_id (in-memory). With MemFirst (Level 2), db.write() - // returns once data reaches OS page cache β€” durable_index advances immediately. - // Followers also ACK after OS page cache write (no fsync wait). Crash safety is - // OS page cache level: process crash is recoverable, power loss is not. - peer_matched_ids.push(self.last_entry_id()); + // RPO=0 (#446): leader's own contribution must be its own durable (fsynced) + // position, not the in-memory tail β€” otherwise a majority-looking commit can + // still lose data on correlated power loss. + peer_matched_ids.push(self.durable_index()); // Sort in descending order peer_matched_ids.sort_unstable_by(|a, b| b.cmp(a)); @@ -657,11 +678,15 @@ where self.purge_prefix(cutoff_index); // Purged entries are backed by the snapshot; treat cutoff as durable. - // fetch_max is monotonic β€” avoids racing fsync_coordinator's concurrent - // advance on the raft-io thread β€” and this fires LogFlushed consistently - // with every other durable_index advancement in this file. - self.advance_durable_and_notify(cutoff_index.index); - + // Already running on the single owner (called from role_state.rs, same + // thread as remove_range) β€” safe to apply directly, no message hop needed. + if let Some(new_durable) = self.try_advance_durable_index(cutoff_index) + && let Some(ref tx) = self.log_flush_tx + { + let _ = tx.send(crate::InternalEvent::LogFlushed { + durable_index: new_durable, + }); + } // Route purge through the IO thread so it never blocks the inbound event loop. // Also writes the purge boundary to META_CF in the RocksDB implementation. let (done_tx, done_rx) = oneshot::channel(); @@ -678,12 +703,35 @@ where Ok(()) } + fn try_advance_durable_index( + &self, + mark: LogId, + ) -> Option { + let prev = self.durable_index.load(Ordering::Acquire); + if mark.index <= prev { + return None; + } + if self.entry_term(mark.index) != Some(mark.term) { + return None; + } + let safe = mark.index.min( + self.memory_max_index + .load(Ordering::Acquire) + .max(self.last_purged_index.load(Ordering::Acquire)), + ); + if safe <= prev { + return None; + } + self.durable_index.fetch_max(safe, Ordering::AcqRel); + Some(safe) + } + async fn flush(&self) -> Result<()> { - let max_index = self.max_index.load(Ordering::Acquire); - if max_index == 0 { + let memory_max_index = self.memory_max_index.load(Ordering::Acquire); + if memory_max_index == 0 { return Ok(()); } - if self.durable_index.load(Ordering::Acquire) >= max_index { + if self.durable_index.load(Ordering::Acquire) >= memory_max_index { return Ok(()); } let (tx, rx) = oneshot::channel(); @@ -753,8 +801,8 @@ where idle_flush_interval_ms, } = persistence_config.flush_policy; debug!( - "Creating BufferedRaftLog with node_id: {}, strategy: {:?}, idle_flush_interval_ms: {:?}, disk_len: {:?}", - node_id, persistence_config.strategy, idle_flush_interval_ms, disk_len + "Creating BufferedRaftLog with node_id: {}, idle_flush_interval_ms: {:?}, disk_len: {:?}", + node_id, idle_flush_interval_ms, disk_len ); let shutdown_timeout_ms = persistence_config.shutdown_timeout_ms; @@ -806,7 +854,7 @@ where // Initialize atomic boundaries let min_index = entries.front().map(|e| *e.key()).unwrap_or(0); - let max_index = entries.back().map(|e| *e.key()).unwrap_or(0); + let memory_max_index = entries.back().map(|e| *e.key()).unwrap_or(0); if disk_len > 0 && loaded_count == 0 { warn!( @@ -835,12 +883,11 @@ where shutdown_timeout_ms, entries: RwLock::new(entries), min_index: AtomicU64::new(min_index), - max_index: AtomicU64::new(max_index), + memory_max_index: AtomicU64::new(memory_max_index), last_purged_index: AtomicU64::new(last_purged_index_val), last_purged_term: AtomicU64::new(last_purged_term_val), durable_index: AtomicU64::new(disk_len), next_id: AtomicU64::new(disk_len + 1), - write_notify: Arc::new(Notify::new()), command_sender: command_sender.clone(), term_first_index, term_last_index, @@ -904,20 +951,22 @@ where arc_self } - /// Notify-driven IO loop. + /// Write-driven IO loop. /// - /// Waits on `write_notify.notified()` for new entries in the SkipMap. - /// Multiple `notify_one()` calls while the IO thread is busy (persisting or - /// fsyncing) coalesce into a single wakeup, reducing kernel cond_signal overhead - /// from one-per-write to one-per-burst. + /// A single `select!` consumes one ordered channel of `IOTask`s and a + /// backstop timer. `Persist` / `ReplaceRange` / `Purge` / `Reset` go to + /// `run_storage_tasks`; `Flush` / `Shutdown` to `run_flush_turn` (which also + /// sweeps the queue for one combined fsync). A write burst's extra `Persist`s + /// are no-ops β€” `persisted_index` is already past `memory_max_index`. /// - /// On each wakeup: - /// 1. Read entries in `(durable_index, max_index]` from SkipMap. + /// On each `Persist` / `Flush`: + /// 1. Read entries in `(persisted_index, memory_max_index]` from the SkipMap. /// 2. persist_entries to OS page cache (no fsync). - /// 3. Drain any pending control commands from the mpsc channel. - /// 4. fsync once β€” advance durable_index, wake WaitDurable callers. + /// 3. Drain any pending control commands from the channel. + /// 4. Hand the range to `FsyncCoordinator` for one concurrent fsync. /// - /// Safety-net timer fires after `idle_flush_interval_ms` of inactivity. + /// Backstop timer ticks every `idle_flush_interval_ms` (fixed period, not + /// reset by writes; skipped under load β€” see the safety-net arm). async fn batch_processor( this: std::sync::Weak, mut receiver: mpsc::UnboundedReceiver, @@ -931,23 +980,59 @@ where tokio::time::interval_at(start, Duration::from_millis(idle_flush_interval_ms)); safety_timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - // Highest index in OS page cache, awaiting fsync. Reset to 0 after each fsync. - let mut pending_max: u64 = 0; + // Highest log index the IO thread has written to the OS page cache β€” + // B-local, the "submitted" watermark sitting between `memory_max_index` + // and `durable_index` + let mut persisted_index: u64 = this.durable_index.load(Ordering::Acquire); loop { tokio::select! { - _ = this.write_notify.notified() => { - if Self::run_batch_turn(&this, &mut receiver, &mut pending_max, Vec::new(), false).await { - break; - } - } cmd = receiver.recv() => { let Some(cmd) = cmd else { break }; let should_break = match cmd { - IOTask::Shutdown => Self::run_batch_turn(&this, &mut receiver, &mut pending_max, Vec::new(), true).await, - IOTask::Flush(reply) => Self::run_batch_turn(&this, &mut receiver, &mut pending_max, vec![reply], false).await, + IOTask::Shutdown => Self::run_flush_turn(&this, &mut receiver, &mut persisted_index,Vec::new(), true).await, + IOTask::Flush(reply) => Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, vec![reply], false).await, cmd => { - if Self::handle_non_write_cmd(cmd, &this, &mut pending_max).await { + let mut extra = Vec::new(); + let mut control: Option = None; + while let Ok(next) = receiver.try_recv() { + match next { + IOTask::Persist => {} // redundant β€” the persist below covers it + f @ (IOTask::Flush(_) | IOTask::Shutdown) => { + + control = Some(f); + break; + } + other => extra.push(other), + } + } + + + + let mut fatal = Self::run_storage_tasks(cmd, &this, &mut persisted_index).await; + if !fatal { + for other in extra { + if Self::run_storage_tasks(other, &this, &mut persisted_index).await { + fatal = true; + break; + } + } + } + if fatal { + break; + } + + let should_break = match control { + Some(IOTask::Shutdown) => { + Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, Vec::new(), true).await + } + Some(IOTask::Flush(reply)) => { + Self::run_flush_turn(&this, &mut receiver, &mut persisted_index, vec![reply], false).await + } + Some(_) => unreachable!(), + None => false, + }; + if should_break { break; } continue; @@ -956,83 +1041,100 @@ where if should_break { break; } } _ = safety_timer.tick() => { - let start = this.durable_index.load(Ordering::Acquire) + 1; - let end = this.max_index.load(Ordering::Acquire); - let _ = Self::persist_pending_range(&this, start, end, &mut pending_max, "safety-net").await; - - if pending_max > 0 { - this.fsync_coordinator.submit(&this, pending_max, vec![]); - pending_max = 0; + let from = this.durable_index.load(Ordering::Acquire) + 1; + let end = this.memory_max_index.load(Ordering::Acquire); + if let Ok(Some(mark)) = + Self::persist_pending_range(&this, from, end, "safety-net").await + { + persisted_index = persisted_index.max(mark.index); + this.fsync_coordinator.submit(&this, mark, vec![]); } } } } } - /// Writes entries in `(from, to]` that haven't reached page cache yet - /// (no fsync). Advances `pending_max` on success; propagates the error - /// as-is on failure β€” whether to notify any waiting `Flush` caller is - /// left to the caller. + /// Persists entries in `(from, to]` that aren't in the OS page cache yet + /// (no fsync). `from` / `to` are only scan bounds β€” the caller passes + /// `persisted_index + 1` and a `memory_max_index` snapshot. + /// + /// The SkipMap range scan returns only entries that still exist. If a + /// concurrent term-conflict truncation removed the top of `(from, to]` + /// between the caller's `memory_max_index` read and this scan, those + /// indices are simply absent and never written. + /// + /// Returns `Some((term, index))` of the last entry written β€” its index may + /// be *below* `to` in that truncation-race case β€” or `None` when the scan + /// found nothing. Callers advance `persisted_index` and submit this mark to + /// the fsync coordinator, so neither points past a real entry. async fn persist_pending_range( this: &Arc, from: u64, to: u64, - pending_max: &mut u64, ctx: &str, - ) -> Result<()> { + ) -> Result> { if this.is_poisoned() { return Err(Error::Fatal("raft log storage is poisoned".to_string())); } - if from > to { - return Ok(()); + return Ok(None); } let entries = this.get_entries_range(from..=to)?; - if entries.is_empty() { - return Ok(()); - } - this.log_store - .persist_entries(entries) - .await - .inspect(|_| { - *pending_max = (*pending_max).max(to); - }) - .inspect_err(|e| { - error!("{ctx} persist_entries failed: {e:?}"); - this.mark_poisoned_and_notify(format!("{ctx}: persist_entries failed: {e:?}")); - }) + let Some(mark) = entries.last().map(|e| LogId { + term: e.term, + index: e.index, + }) else { + return Ok(None); + }; + this.log_store.persist_entries(entries).await.inspect_err(|e| { + error!( + persist_path = ctx, + from, to, "persist_entries failed: {e:?}" + ); + this.mark_poisoned_and_notify(format!("{ctx}: persist_entries failed: {e:?}")); + })?; + Ok(Some(mark)) } - async fn run_batch_turn( + async fn run_flush_turn( this: &Arc, receiver: &mut mpsc::UnboundedReceiver, - pending_max: &mut u64, + persisted_index: &mut u64, mut replies: Vec>>, mut seen_shutdown: bool, ) -> bool { - let start = this.durable_index.load(Ordering::Acquire) + 1; - let end = this.max_index.load(Ordering::Acquire); + // Highest (term, index) this turn persisted. Delivered with `replies` in + // one submit at the end. Any drained ReplaceRange/Persist submits its own + // mark directly; term-first ordering in the coordinator means a stale + // leading mark here loses to a newer-term drained one. + let mut mark = LogId::default(); + + let end = this.memory_max_index.load(Ordering::Acquire); let mut persist_failed = false; - if let Err(e) = Self::persist_pending_range(this, start, end, pending_max, "batch").await { - for reply in replies.drain(..) { - let _ = reply.send(Err(Error::Fatal(format!("persist_entries failed: {e:?}")))); + match Self::persist_pending_range(this, *persisted_index + 1, end, "flush-turn").await { + Ok(Some(m)) => { + *persisted_index = m.index; + mark = m; + } + Ok(None) => {} + Err(e) => { + for reply in replies.drain(..) { + let _ = reply.send(Err(Error::Fatal(format!("persist_entries failed: {e:?}")))); + } + persist_failed = true; } - persist_failed = true; } - // `seen_shutdown` is not a gate here β€” regardless of whether the - // caller already knows shutdown is happening, any commands still - // queued must be drained and replied to. Whether to drain the queue - // and whether to eventually break the loop are separate concerns - // and must not share one flag. + // `seen_shutdown` is not a gate here β€” any queued commands must still be + // drained and replied to. while let Ok(cmd) = receiver.try_recv() { match cmd { - IOTask::Shutdown => { - seen_shutdown = true; - } + IOTask::Shutdown => seen_shutdown = true, IOTask::Flush(reply) => replies.push(reply), + // Coalesced write β€” the unconditional catch-up below covers it. + IOTask::Persist => {} cmd => { - if Self::handle_non_write_cmd(cmd, this, pending_max).await { + if Self::run_storage_tasks(cmd, this, persisted_index).await { for reply in replies { let _ = reply .send(Err(Error::Fatal("fatal IO error, batch aborted".into()))); @@ -1043,40 +1145,69 @@ where } } - if !replies.is_empty() && !persist_failed { - let start = *pending_max + 1; - let end = this.max_index.load(Ordering::Acquire); - let _ = - Self::persist_pending_range(this, start, end, pending_max, "batch catch-up").await; + // Catch-up: a dropped Persist or a drained command may have moved + // memory_max_index past the leading persist. + if !persist_failed { + let end = this.memory_max_index.load(Ordering::Acquire); + if let Ok(Some(m)) = + Self::persist_pending_range(this, *persisted_index + 1, end, "flush-turn catch-up") + .await + { + *persisted_index = m.index; + mark = m; + } } - this.fsync_coordinator.submit(this, *pending_max, replies); - *pending_max = 0; + this.fsync_coordinator.submit(this, mark, replies); + if seen_shutdown { let _ = this.meta_store.flush(); } + + // Observability β€” IO thread, per write burst (not per append). Where the + // log pipeline stands; `rpo_window` = in memory but not yet fsynced. + let mem = this.memory_max_index.load(Ordering::Relaxed); + let dur = this.durable_index.load(Ordering::Relaxed); + metrics::gauge!("core.raft.log.memory_max_index").set(mem as f64); + metrics::gauge!("core.raft.log.durable_index").set(dur as f64); + metrics::gauge!("core.raft.log.rpo_window").set(mem.saturating_sub(dur) as f64); + seen_shutdown } - /// Handle IOTask variants that are NOT `Flush` or `Shutdown`. - /// - /// Callers (`batch_processor`) dispatch `Flush` and `Shutdown` directly in the outer - /// `match` before this function is ever called β€” those two arms are unreachable here. + /// Handles one non-Flush/Shutdown `IOTask`: `Persist` (persist the SkipMap tail + /// then submit it for fsync) or a storage mutation (ReplaceRange/Purge/Reset). + /// Flush/Shutdown never reach here: `batch_processor`'s `select!` routes + /// them to `run_flush_turn`, whose drain loop also handles them inline. /// /// Returns `true` if `batch_processor` must exit immediately (fatal IO error). - async fn handle_non_write_cmd( + async fn run_storage_tasks( cmd: IOTask, this: &Arc, - pending_max: &mut u64, + persisted_index: &mut u64, ) -> bool { match cmd { IOTask::Flush(_) => { - unreachable!( - "Flush must be intercepted in the drain loop before handle_non_write_cmd" - ) + unreachable!("Flush must be intercepted in the drain loop before run_storage_tasks") } IOTask::Shutdown => { - unreachable!("Shutdown is always filtered out before reaching handle_non_write_cmd") + unreachable!("Shutdown is always filtered out before reaching run_storage_tasks") + } + IOTask::Persist => { + // A write landed. Persist the new tail and submit it. No queue + // drain: a burst's redundant `Persist`s find `persisted_index` + // already at `memory_max_index` and no-op here (#446). + let end = this.memory_max_index.load(Ordering::Acquire); + match Self::persist_pending_range(this, *persisted_index + 1, end, "persist").await + { + Ok(Some(mark)) => { + *persisted_index = mark.index; + this.fsync_coordinator.submit(this, mark, Vec::new()); + } + Ok(None) => {} + Err(_) => return true, // poisoned β€” same exit convention as the mutations below + } + false } IOTask::ReplaceRange { truncate_from, @@ -1091,18 +1222,32 @@ where return true; } - let max_idx = new_entries.last().map(|e| e.index).unwrap_or(0); - let result = this.log_store.replace_range(truncate_from, new_entries).await; - if let Err(ref e) = result { - error!("IOTask::ReplaceRange failed (fatal): {e:?}"); - this.mark_poisoned_and_notify(format!("ReplaceRange failed: {e:?}")); - let _ = done.send(result); - return true; // signal batch_processor to exit β€” disk state is corrupted - } - if max_idx > 0 { - *pending_max = (*pending_max).max(max_idx); + // Capture the new tail's term before `new_entries` is moved. + let new_tail_term = new_entries.last().map(|e| e.term).unwrap_or(0); + let new_tail = match this.log_store.replace_range(truncate_from, new_entries).await + { + Ok(new_tail) => new_tail, + Err(e) => { + error!("IOTask::ReplaceRange failed (fatal): {e:?}"); + this.mark_poisoned_and_notify(format!("ReplaceRange failed: {e:?}")); + let _ = done.send(Err(e)); + return true; + } + }; + // New content written β€” submit its (term, index). fdatasync is + // whole-WAL so it also covers any leading persist from this turn. + if new_tail >= truncate_from { + this.fsync_coordinator.submit( + this, + LogId { + term: new_tail_term, + index: new_tail, + }, + vec![], + ); } - let _ = done.send(result); + *persisted_index = new_tail; + let _ = done.send(Ok(())); false } IOTask::Purge { cutoff, done } => { @@ -1119,6 +1264,9 @@ where let _ = done.send(()); return true; // signal batch_processor to exit β€” disk state is corrupted } + // Purged entries were already durable (only applied entries are + // purged) β€” let the persist frontier skip past them (#446). + *persisted_index = (*persisted_index).max(cutoff.index); let _ = done.send(()); false } @@ -1132,9 +1280,13 @@ where error!("IOTask::Reset failed (fatal): {e:?}"); this.mark_poisoned_and_notify(format!("Reset failed: {e:?}")); let _ = done.send(result); - return true; // signal batch_processor to exit β€” disk state is corrupted + + // signal batch_processor to exit β€” disk state is corrupted + return true; } - *pending_max = 0; // disk wiped β€” pending page-cache watermark must be zeroed + + // log wiped β€” next entry to persist is index 1 + *persisted_index = 0; let _ = done.send(result); false } @@ -1152,7 +1304,7 @@ where // Reset boundaries self.min_index.store(0, Ordering::Release); - self.max_index.store(0, Ordering::Release); + self.memory_max_index.store(0, Ordering::Release); // Clear term indexes to ensure consistency after reset self.term_first_index.clear(); @@ -1207,9 +1359,9 @@ where } if let Some(last_entry) = entries.last() { - let mut current_max = self.max_index.load(Ordering::Relaxed); + let mut current_max = self.memory_max_index.load(Ordering::Relaxed); while last_entry.index > current_max { - match self.max_index.compare_exchange_weak( + match self.memory_max_index.compare_exchange_weak( current_max, last_entry.index, Ordering::AcqRel, @@ -1222,18 +1374,14 @@ where } } - /// Advance `durable_index` to `new_durable` (monotonically) and send `LogFlushed`. - pub(super) fn advance_durable_and_notify( + /// Fire-and-forget signal to the single owner (raft.rs's event loop). + /// Called by fsync_coordinator (C) β€” never writes `durable_index` itself. + pub(super) fn notify_fsync_completed( &self, - new_durable: u64, + mark: LogId, ) { - let prev = self.durable_index.fetch_max(new_durable, Ordering::AcqRel); - if new_durable > prev - && let Some(ref tx) = self.log_flush_tx - { - let _ = tx.send(crate::InternalEvent::LogFlushed { - durable_index: new_durable, - }); + if let Some(ref tx) = self.log_flush_tx { + let _ = tx.send(crate::InternalEvent::FsyncCompleted(mark)); } } @@ -1282,7 +1430,10 @@ where let entries = self.entries.write(); let (new_min, new_max) = self.remove_range_locked(&entries, range); self.min_index.store(new_min, Ordering::Release); - self.max_index.store(new_max, Ordering::Release); + self.memory_max_index.store(new_max, Ordering::Release); + + self.durable_index.fetch_min(new_max, Ordering::AcqRel); + self.fsync_coordinator.bump_generation(); // `entries` guard drops here (end of scope) β€” write lock released. } @@ -1365,7 +1516,7 @@ where /// Purge entries at/below `cutoff.index`, publishing `last_purged_index`/ /// `last_purged_term` in the SAME critical section as the entries removal - /// and the min_index/max_index advance. Only place that should ever write + /// and the min_index/memory_max_index advance. Only place that should ever write /// `last_purged_*` β€” a reader must never observe the entries gone but the /// boundary not yet recorded (#442). pub fn purge_prefix( @@ -1376,15 +1527,12 @@ where let (new_min, new_max) = self.remove_range_locked(&entries, 0..=cutoff.index); self.min_index.store(new_min, Ordering::Release); - self.max_index.store(new_max, Ordering::Release); + self.memory_max_index.store(new_max, Ordering::Release); // Write term before index (Release) so readers that load index first // then term (Acquire) always observe a consistent pair. self.last_purged_term.store(cutoff.term, Ordering::Release); self.last_purged_index.store(cutoff.index, Ordering::Release); - - // `entries` guard drops here β€” everything above is now visible together - // to any reader acquiring the read lock or loading these atomics after. } // Update the term index (completely lock-free) @@ -1439,6 +1587,14 @@ where pub fn is_empty(&self) -> bool { self.entries.read().is_empty() } + + #[cfg(test)] + pub(super) fn set_memory_max_index_for_test( + &self, + value: u64, + ) { + self.memory_max_index.store(value, Ordering::Release); + } } impl Drop for BufferedRaftLog @@ -1503,6 +1659,10 @@ mod id_allocation_test; #[path = "buffered_raft_log_test/performance_test.rs"] mod performance_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/durable_index_truncation_clamp_test.rs"] +mod durable_index_truncation_clamp_test; + #[cfg(test)] #[path = "buffered_raft_log_test/pipeline_overlap_test.rs"] mod pipeline_overlap_test; @@ -1519,6 +1679,10 @@ mod raft_properties_test; #[path = "buffered_raft_log_test/remove_range_test.rs"] mod remove_range_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/replace_range_fsync_test.rs"] +mod replace_range_fsync_test; + #[cfg(test)] #[path = "buffered_raft_log_test/shutdown_test.rs"] mod shutdown_test; @@ -1531,6 +1695,18 @@ mod term_index_test; #[path = "buffered_raft_log_test/term_segments_test.rs"] mod term_segments_test; +#[cfg(test)] +#[path = "buffered_raft_log_test/truncation_fsync_fence_test.rs"] +mod truncation_fsync_fence_test; + +#[cfg(test)] +#[path = "buffered_raft_log_test/content_validated_watermark_test.rs"] +mod content_validated_watermark_test; + #[cfg(test)] #[path = "buffered_raft_log_test/worker_test.rs"] mod worker_test; + +#[cfg(test)] +#[path = "buffered_raft_log_test/prev_log_index_zero_idempotency_test.rs"] +mod prev_log_index_zero_idempotency_test; diff --git a/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs index 6790f539..cb12c6d7 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/basic_operations_test.rs @@ -13,7 +13,7 @@ use crate::test_utils::{ BufferedRaftLogTestContext, mock_empty_entries, simulate_delete_command, simulate_insert_command, }; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; /// Test get_entries_range returns correct subset /// @@ -24,7 +24,6 @@ use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; #[tokio::test] async fn test_get_entries_range_returns_correct_subset() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -52,7 +51,6 @@ async fn test_get_entries_range_returns_correct_subset() { #[tokio::test] async fn test_get_entries_range_handles_large_range() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -86,7 +84,6 @@ async fn test_get_entries_range_handles_large_range() { #[tokio::test] async fn test_filter_conflicts_removes_entries_with_different_term() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -133,7 +130,6 @@ async fn test_filter_conflicts_removes_entries_with_different_term() { #[tokio::test] async fn test_filter_conflicts_handles_multiple_scenarios() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -193,7 +189,6 @@ async fn test_filter_conflicts_handles_multiple_scenarios() { #[tokio::test] async fn test_last_entry_returns_highest_index() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -218,7 +213,6 @@ async fn test_last_entry_returns_highest_index() { #[tokio::test] async fn test_last_entry_matches_buffer_length() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -247,7 +241,6 @@ async fn test_last_entry_matches_buffer_length() { #[tokio::test] async fn test_last_entry_with_large_payload_id() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -272,7 +265,6 @@ async fn test_last_entry_with_large_payload_id() { #[tokio::test] async fn test_insert_batch_appends_entries_in_order() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -305,7 +297,6 @@ async fn test_insert_batch_appends_entries_in_order() { #[tokio::test] async fn test_get_entries_range_multiple_bounds() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -360,7 +351,6 @@ async fn test_get_entries_range_multiple_bounds() { #[tokio::test] async fn test_insert_duplicate_commands_as_separate_events() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -395,7 +385,6 @@ async fn test_insert_duplicate_commands_as_separate_events() { #[tokio::test] async fn test_purge_after_insert_maintains_consistency() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -428,7 +417,6 @@ async fn test_purge_after_insert_maintains_consistency() { #[tokio::test] async fn test_purge_logs_removes_entries_up_to_index() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -476,7 +464,6 @@ async fn test_purge_logs_removes_entries_up_to_index() { #[tokio::test] async fn test_concurrent_purge_operations_are_safe() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -515,7 +502,6 @@ async fn test_concurrent_purge_operations_are_safe() { #[tokio::test] async fn test_first_entry_id_after_purge_updates() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -547,7 +533,6 @@ async fn test_first_entry_id_after_purge_updates() { #[tokio::test] async fn test_single_entry_insert_succeeds() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -569,7 +554,6 @@ async fn test_single_entry_insert_succeeds() { #[tokio::test] async fn test_is_empty_returns_true_for_new_log() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -588,7 +572,6 @@ async fn test_is_empty_returns_true_for_new_log() { #[tokio::test] async fn test_is_empty_returns_false_after_append() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -613,7 +596,6 @@ async fn test_is_empty_returns_false_after_append() { #[tokio::test] async fn test_last_log_id_for_empty_log() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -636,7 +618,6 @@ async fn test_last_log_id_for_empty_log() { #[tokio::test] async fn test_last_log_id_after_appends() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -662,7 +643,6 @@ async fn test_last_log_id_after_appends() { #[tokio::test] async fn test_drop_shuts_down_workers_gracefully() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -686,14 +666,12 @@ async fn test_drop_shuts_down_workers_gracefully() { #[tokio::test] async fn test_same_index_and_term_implies_identical_prefix() { let ctx1 = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, "test_log_matching_1", ); let ctx2 = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -726,7 +704,6 @@ async fn test_same_index_and_term_implies_identical_prefix() { #[tokio::test] async fn test_committed_entry_present_in_future_leaders() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -758,7 +735,6 @@ async fn test_committed_entry_present_in_future_leaders() { #[tokio::test] async fn test_append_updates_last_entry() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -784,7 +760,6 @@ async fn test_append_updates_last_entry() { #[tokio::test] async fn test_insert_batch_with_empty_list() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -807,7 +782,6 @@ async fn test_insert_batch_with_empty_list() { #[tokio::test] async fn test_insert_batch_updates_metadata() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs index 178a9328..fad64cec 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_fsync_test.rs @@ -6,11 +6,12 @@ //! advance_durable_and_notify contract //! - **Concurrency**: Reset races, out-of-order completion, crash recovery +use crate::test_utils::drain_and_apply_fsync_completions; use crate::{ - BufferedRaftLog, FlushPolicy, InternalEvent, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, RaftLog, + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, RaftLog, }; use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; @@ -34,16 +35,18 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), ); - let raft_log = raft_log.start(receiver, None); + // A real log_flush_tx is required now: durable_index only advances when + // something drains InternalEvent::FsyncCompleted and calls + // try_advance_durable_index β€” see drain_and_apply_fsync_completions. + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready let pre_write_durable_index = raft_log.durable_index(); @@ -71,12 +74,13 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { "durable_index must not advance before fsync completes" ); - // Release the gate β€” flush() returns, advance_durable_and_notify(1) fires. + // Release the gate β€” flush() returns, notify_fsync_completed(1, 1) fires. flush_gate.send(()).unwrap(); // Pick a polling/backoff strategy instead of a fixed sleep, // to avoid flakiness under CI load. tokio::time::sleep(Duration::from_millis(50)).await; + drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!( raft_log.durable_index(), @@ -85,44 +89,48 @@ async fn test_durable_index_not_advanced_before_fsync_completes() { ); } -/// `calculate_majority_matched_index` uses the in-memory SkipMap (`last_entry_id`), -/// not `durable_index` β€” even when all fsyncs are stalled indefinitely. +/// `calculate_majority_matched_index` uses `durable_index` (fsync-confirmed), not the +/// in-memory `last_entry_id` β€” even when a follower already reports the index, the +/// leader's own contribution must not count toward quorum until it has itself fsynced. /// -/// Stall every flush() call via a MockLogStore barrier, append entries, then verify -/// that majority-matched calculation returns the correct in-memory index. +/// Stall every flush() call via a MockLogStore barrier, append entries, then verify that +/// majority-matched calculation does NOT advance while fsync is stalled, and does advance +/// once fsync completes. /// -/// Regression guard: if majority calculation ever changes to depend on `durable_index`, -/// this test will catch it before it reaches production. +/// Regression guard: RPO=0 (#446) requires the leader's own copy to be durable before it +/// counts toward commit β€” if this ever reverts to using `last_entry_id`, this test will +/// catch it before it reaches production. /// /// Expected: -/// - Append entries so `last_entry_id()` reaches N (e.g. 5) while fsync is -/// permanently stalled β€” `durable_index()` stays at its pre-write value -/// (0) throughout. -/// - Feed `calculate_majority_matched_index` a `match_index` map where enough -/// followers already report N to form a majority. -/// - Assert the returned majority-matched index equals N (matching -/// `last_entry_id()`) β€” NOT 0 (what it would return if it mistakenly used -/// `durable_index()` instead). +/// - Append entries so `last_entry_id()` reaches N (e.g. 2) while fsync is permanently +/// stalled β€” `durable_index()` stays at its pre-write value (0) throughout. +/// - Feed `calculate_majority_matched_index` a `match_index` map where one follower +/// already reports N=2 (majority IF the leader's own un-fsynced entry counted) β€” +/// assert the result is `None` while durable_index is still 0. +/// - Release the gate. Once `durable_index` reaches 2, the same call must return +/// `Some(2)`. #[tokio::test] -async fn test_majority_matched_index_uses_memory_not_durable_index() { +async fn test_majority_matched_index_uses_durable_not_memory() { // Gate closed: the first flush() call will block until we send () on `flush_gate`. let (storage, flush_gate) = MockStorageEngine::not_durable_gated_flush( - "majority_matched_index_uses_memory_not_durable_index".into(), + "majority_matched_index_uses_durable_not_memory".into(), ); let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify should trigger fsync here. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), ); - let raft_log = raft_log.start(receiver, None); + // A real log_flush_tx is required now: durable_index only advances when + // something drains InternalEvent::FsyncCompleted and calls + // try_advance_durable_index β€” see drain_and_apply_fsync_completions. + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready let pre_write_durable_index = raft_log.durable_index(); @@ -160,33 +168,42 @@ async fn test_majority_matched_index_uses_memory_not_durable_index() { assert_eq!( raft_log.last_entry_id(), pre_last_entry_id + size, - "durable_index must not advance before fsync completes" + "in-memory tail should still advance even while fsync is stalled" ); // One follower already matched index 2; the other is still behind at 0 β€” asymmetric - // on purpose. With only ONE follower at 2, the leader's own contribution decides - // whether the majority (2 out of 3 voters) reaches 2. If this ever regresses to use - // `durable_index()` (0, since fsync is still gated) instead of `last_entry_id()` (2), - // the median drops to 0 and the call returns `None` instead of `Some(2)`. + // on purpose. If the leader's own un-fsynced entry counted (the old MemFirst + // behavior), 2 out of 3 voters would reach index 2 β€” but RPO=0 requires the + // leader's own copy to be durable first, so this must return None while fsync + // is still gated. let result = raft_log.calculate_majority_matched_index(1, 1, vec![2, 0]); assert_eq!( - result, - Some(2), - "majority index must use last_entry_id (2), not durable_index (0)" + result, None, + "RPO=0: the leader's own un-fsynced entry must not count toward quorum, even \ + when a follower already reports it" ); - // Release the gate β€” flush() returns, advance_durable_and_notify(1) fires. + // Release the gate β€” flush() returns, notify_fsync_completed(2, 1) fires. flush_gate.send(()).unwrap(); // Pick a polling/backoff strategy instead of a fixed sleep, // to avoid flakiness under CI load. tokio::time::sleep(Duration::from_millis(50)).await; + drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!( raft_log.durable_index(), pre_write_durable_index + size, "durable_index must reach the expected index after fsync completes" ); + + // Now the leader's own contribution is durable (2), so the same call must succeed. + let result_after_fsync = raft_log.calculate_majority_matched_index(1, 1, vec![2, 0]); + assert_eq!( + result_after_fsync, + Some(2), + "once the leader's own entry is durable, majority index must advance to 2" + ); } /// `entry_term()` returns the correct term during high-concurrency writes @@ -212,11 +229,9 @@ async fn test_entry_term_correct_during_concurrent_fsync_delay() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -275,24 +290,24 @@ async fn test_entry_term_correct_during_concurrent_fsync_delay() { // ── Logic correctness ───────────────────────────────────────────────────────── -/// `advance_durable_and_notify` is monotonic: a late-arriving lower index is a no-op. +/// `try_advance_durable_index` is monotonic: a late-arriving lower index is a no-op. /// -/// Directly call `advance_durable_and_notify(150)`, then `advance_durable_and_notify(100)`. +/// Directly call `try_advance_durable_index(150, 1)`, then `try_advance_durable_index(100, 1)`. /// Assert: /// - final `durable_index() == 150` (not 100) -/// - `LogFlushed` event fired exactly once (for 150), not twice +/// - the 150 call returns `Some(150)` (it fired), the 100 call returns `None` (no-op) /// /// Verifies the `fetch_max` invariant that makes out-of-order concurrent fsyncs safe. /// -/// Expected: -/// - After `advance_durable_and_notify(150)`: `durable_index() == 150`. -/// - After the subsequent `advance_durable_and_notify(100)`: `durable_index()` -/// is STILL `150` (unchanged β€” 100 < 150 must be a no-op, not a regression). -/// - The flush-completion notification fires exactly once, carrying 150 β€” the -/// discarded 100 call must not fire a second notification. +/// Test changed from #446/#447's original (which checked an `InternalEvent::LogFlushed` +/// on a channel): `try_advance_durable_index` no longer sends that notification itself β€” +/// the caller (raft.rs's `FsyncCompleted` handler) decides whether to fire +/// `handle_log_flushed`, based on this method's `Option` return value. So "fired +/// exactly once, only for 150" is now asserted directly on the return values instead of +/// on a channel β€” same intent, moved to match where the behavior actually lives now. #[tokio::test] async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { - // Storage engine choice doesn't matter here β€” advance_durable_and_notify is + // Storage engine choice doesn't matter here β€” try_advance_durable_index is // called directly, bypassing the real fsync pipeline entirely. let storage = Arc::new(MockStorageEngine::with_id( "durable_index_monotonic_when_fsyncs_complete_out_of_order".into(), @@ -300,44 +315,53 @@ async fn test_durable_index_monotonic_when_fsyncs_complete_out_of_order() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, ); - let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel::(); - let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready + // try_advance_durable_index() content-validates against entry_term(index) β€” + // needs real entries in memory, not just a raw max_index poke. + let entries: Vec = (1..=150) + .map(|index| Entry { + index, + term: 1, + payload: None, + }) + .collect(); + raft_log.append_entries(entries).await.unwrap(); + // Simulates a fsync task completing with index 150, then a second, older // fsync task (dispatched earlier, finishing later) completing with 100. - raft_log.advance_durable_and_notify(150); - raft_log.advance_durable_and_notify(100); + let result_150 = raft_log.try_advance_durable_index(LogId { + term: 1, + index: 150, + }); + let result_100 = raft_log.try_advance_durable_index(LogId { + term: 1, + index: 100, + }); + assert_eq!( + result_150, + Some(150), + "the 150 call must fire β€” it's the first advance" + ); + assert_eq!( + result_100, None, + "the later, lower 100 call must be a no-op (None), not a regression" + ); assert_eq!( raft_log.durable_index(), 150, "durable_index must reflect the highest index seen (150), not the \ later-arriving lower one (100)" ); - - // Exactly one LogFlushed event must have fired, carrying 150 β€” the - // no-op 100 call must not have sent a second event. - let event = log_flush_rx.try_recv().expect("LogFlushed must fire for the 150 call"); - match event { - InternalEvent::LogFlushed { durable_index } => { - assert_eq!(durable_index, 150, "LogFlushed must carry 150, not 100"); - } - other => panic!("expected InternalEvent::LogFlushed, got {other:?}"), - } - assert!( - log_flush_rx.try_recv().is_err(), - "no second LogFlushed event should have fired for the no-op 100 call" - ); } /// A `flush()` caller receives `Ok(())` only after its batch is physically on disk. @@ -364,11 +388,9 @@ async fn test_flush_caller_blocked_until_fsync_completes() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -438,11 +460,9 @@ async fn test_flush_callers_arriving_during_inflight_fsync_are_coalesced() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -522,11 +542,9 @@ async fn test_shutdown_with_pending_flush_caller_still_receives_ok_reply() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 100, }, Arc::new(storage), @@ -596,11 +614,9 @@ async fn test_shutdown_with_pending_flush_caller_still_receives_err_reply() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 100, }, Arc::new(storage), @@ -677,11 +693,9 @@ async fn test_reset_during_inflight_fsync_does_not_resurrect_stale_durable_index let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -759,16 +773,15 @@ async fn test_post_reset_writes_are_not_discarded_by_stale_fence() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready // Append triggers the automatic round (round 1), which wins the CAS and @@ -819,6 +832,7 @@ async fn test_post_reset_writes_are_not_discarded_by_stale_fence() { result.is_ok(), "Y's flush() must succeed β€” its data was written after reset, not stale" ); + drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!( raft_log.durable_index(), 1, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs index 2b345e79..d3201d2f 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/concurrent_operations_test.rs @@ -3,15 +3,14 @@ use std::time::Duration; use futures::future::join_all; use tokio; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, simulate_insert_command}; -use crate::{FlushPolicy, PersistenceStrategy}; use d_engine_proto::common::{Entry, LogId}; #[tokio::test] async fn test_remove_range_with_concurrent_reads() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -54,7 +53,6 @@ async fn test_remove_range_with_concurrent_reads() { #[tokio::test] async fn test_concurrent_append_and_purge() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -128,7 +126,6 @@ async fn test_get_entries_range_never_returns_torn_result_during_concurrent_purg const ITERATIONS: usize = 500; let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs new file mode 100644 index 00000000..2e55c1fb --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/content_validated_watermark_test.rs @@ -0,0 +1,200 @@ +//! Content-validated `durable_index` advance (#446 single-owner redesign). +//! Tests `try_advance_durable_index(index, term) -> Option`: +//! `Some(new)` only when it actually advanced, `None` when rejected as stale +//! (`entry_term(index) != Some(term)`) or already applied. +//! +//! Why no thread races / timing gates here (unlike `truncation_fsync_fence_test.rs`): +//! `durable_index` has one owner β€” raft.rs's event loop. The fsync-completion +//! report and a truncation are just two sequential calls on that thread, so no +//! interleaving inside a function body is possible. Each test drives one +//! arrival order directly. + +use std::sync::Arc; +use std::time::Duration; + +use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; + +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +async fn new_raft_log() -> Arc> { + let storage = Arc::new(MockStorageEngine::with_id( + "content_validated_watermark_test".into(), + )); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + raft_log.start(receiver, None) +} + +/// Business scenario: follower has entries 1..=100 under term 1. A physical +/// fsync for "up to 100" is still in flight when a new leader (term 2) +/// truncates 81..=100 and replaces it with its own entries. The in-flight +/// fsync's completion β€” a report for (index=100, term=1) β€” arrives after the +/// replacement. Index 100 still exists, but it's term 2 now: the report +/// describes content that's gone. +/// +/// Expected: rejected. `durable_index` must not move to 100. +#[tokio::test] +async fn test_stale_durable_report_rejected_when_term_no_longer_matches() { + let raft_log = new_raft_log().await; + + let term1_entries: Vec = (1..=100).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(term1_entries).await.unwrap(); + + // New leader (term 2) truncates 81..=100 and replaces with its own tail. + let term2_tail: Vec = (81..=100).map(|i| entry(i, 2)).collect(); + raft_log.filter_out_conflicts_and_append(80, 1, term2_tail).await.unwrap(); + + // The stale in-flight fsync's report, generated before the truncation. + let result = raft_log.try_advance_durable_index(LogId { + term: 1, + index: 100, + }); + + assert_eq!( + result, None, + "a durable report for term=1 must be rejected once index 100 belongs to term=2" + ); + assert!( + raft_log.durable_index() < 81, + "durable_index ({}) must not advance into the replaced [81,100] range \ + on a rejected report", + raft_log.durable_index() + ); +} + +/// Sanity check: an unremarkable report (no truncation involved) must still +/// be applied. The new validation must not reject everything. +#[tokio::test] +async fn test_durable_report_accepted_when_term_still_matches() { + let raft_log = new_raft_log().await; + + let entries: Vec = (1..=100).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(entries).await.unwrap(); + + let result = raft_log.try_advance_durable_index(LogId { + term: 1, + index: 100, + }); + + assert_eq!( + result, + Some(100), + "a report matching current log content must be applied" + ); + assert_eq!(raft_log.durable_index(), 100); +} + +/// Same scenario as `test_stale_durable_report_rejected_when_term_no_longer_matches`, +/// but the report arrives BEFORE the truncation instead of after β€” the other +/// possible arrival order. Under single ownership both orders must land on +/// the same final state, because the owner processes one event at a time +/// rather than racing a background write against a live update. +#[tokio::test] +async fn test_durable_report_then_truncation_is_order_independent() { + let raft_log = new_raft_log().await; + + let term1_entries: Vec = (1..=100).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(term1_entries).await.unwrap(); + + // Report arrives first, while the log is still all term 1 β€” legitimately + // applied at this point in time. + let result = raft_log.try_advance_durable_index(LogId { + term: 1, + index: 100, + }); + assert_eq!(result, Some(100)); + assert_eq!(raft_log.durable_index(), 100); + + // Truncation arrives after β€” must still clamp durable_index down, + // exactly as it does today via `remove_range`'s existing fetch_min. + let term2_tail: Vec = (81..=100).map(|i| entry(i, 2)).collect(); + raft_log.filter_out_conflicts_and_append(80, 1, term2_tail).await.unwrap(); + + assert!( + raft_log.durable_index() < 81, + "truncation must clamp durable_index down to 80 regardless of the \ + earlier report having advanced it to 100, durable_index is {}", + raft_log.durable_index() + ); +} + +/// Regression test for the `flush()` short-circuit (`durable_index >= +/// memory_max_index` at `buffered_raft_log.rs:710`). This isn't proving a +/// live bug in the current design (`remove_range` clamps `durable_index` +/// synchronously, so the short-circuit's precondition always holds) β€” it's +/// pinning down that invariant so a future change that defers the clamp +/// (e.g. copying openraft's "don't touch the watermark on truncation, rely +/// on term comparison instead") doesn't silently reopen the RPO=0 violation +/// this whole fix was for: `flush()` returning `Ok(())` before the real, +/// post-truncation tail has actually been fsynced. +/// +/// Scenario: entries 1..=100 durable. New leader truncates 81..=100 (term 2 +/// tail 81..=85 replaces it) β€” `durable_index` clamps to 80, +/// `memory_max_index` becomes 85. Calling `flush()` right after must NOT +/// take the short-circuit (80 < 85) β€” it must dispatch a real physical +/// flush for the new, not-yet-synced tail. +#[tokio::test] +async fn test_flush_does_not_short_circuit_after_truncation_regrows_the_log() { + let (mut ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( + FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + "flush_no_short_circuit_after_truncation", + ); + + ctx.append_entries(1, 100, 1).await; + ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); + assert_eq!( + ctx.raft_log.durable_index(), + 100, + "baseline must be fully durable" + ); + + let flushes_before_truncation = flush_count.load(std::sync::atomic::Ordering::Relaxed); + + // New leader (term 2) truncates 81..=100, replaces with its own tail + // 81..=85 β€” durable_index clamps to 80, memory_max_index becomes 85. + let term2_tail: Vec = (81..=85).map(|i| entry(i, 2)).collect(); + ctx.raft_log.filter_out_conflicts_and_append(80, 1, term2_tail).await.unwrap(); + assert!(ctx.raft_log.durable_index() < 81, "clamp must have fired"); + assert_eq!(ctx.raft_log.last_entry_id(), 85); + + ctx.raft_log.flush().await.unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + ctx.drain_fsync_completions(); + + let flushes_after = flush_count.load(std::sync::atomic::Ordering::Relaxed); + assert!( + flushes_after > flushes_before_truncation, + "flush() must dispatch a real physical flush for the new tail, not \ + short-circuit on a stale-looking durable_index β€” before={flushes_before_truncation}, after={flushes_after}" + ); + assert_eq!( + ctx.raft_log.durable_index(), + 85, + "the new tail must actually become durable, not just claimed so" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs index 838cc386..f8dc28ae 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/drain_fsync_test.rs @@ -20,10 +20,11 @@ use crate::MockMetaStore; use crate::MockStorageEngine; use crate::MockTypeConfig; use crate::PersistenceConfig; -use crate::PersistenceStrategy; use d_engine_proto::common::Entry; use d_engine_proto::common::LogId; use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use tokio::sync::mpsc; use tokio::time::timeout; @@ -39,7 +40,7 @@ use crate::{FlushPolicy, RaftLog}; /// automatically β€” no explicit `flush()` required. #[tokio::test] async fn test_writes_become_durable_via_io_thread() { - let (ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( + let (mut ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -69,6 +70,7 @@ async fn test_writes_become_durable_via_io_thread() { // Give IO thread time to process write_notify wakeup and fsync. sleep(Duration::from_millis(50)).await; + ctx.drain_fsync_completions(); // durable_index must have advanced via IO thread auto-fsync (no explicit flush). assert_eq!( @@ -95,7 +97,7 @@ async fn test_writes_become_durable_via_io_thread() { /// N entries in one call β†’ ≀2 fsyncs (not N), regardless of storage speed. #[tokio::test] async fn test_batch_append_produces_one_flush() { - let (ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( + let (mut ctx, flush_count) = BufferedRaftLogTestContext::new_not_durable( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -113,6 +115,7 @@ async fn test_batch_append_produces_one_flush() { ctx.raft_log.append_entries(entries).await.unwrap(); ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.durable_index(), 100); @@ -135,7 +138,7 @@ async fn test_batch_append_produces_one_flush() { /// `else { pending_max = 0 }` branch is skipped. /// /// ## Original bug (fixed pre-#422) -/// `handle_non_write_cmd(IOTask::Reset)` wiped the on-disk log but did NOT zero +/// `run_storage_tasks(IOTask::Reset)` wiped the on-disk log but did NOT zero /// `pending_max`. On the next `write_notify` wakeup the IO thread would compute: /// ``` /// pending_max = pending_max.max(new_end) // stale 10 wins over new 3 @@ -156,12 +159,10 @@ async fn test_pending_max_zeroed_on_reset_preventing_durable_index_corruption() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify triggers fsync. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -221,7 +222,7 @@ async fn test_pending_max_zeroed_on_reset_preventing_durable_index_corruption() /// flush() call must be durable when flush() returns, regardless of internal batching. #[tokio::test] async fn test_flush_is_strict_durability_barrier() { - let (ctx, _flush_count) = BufferedRaftLogTestContext::new_not_durable( + let (mut ctx, _flush_count) = BufferedRaftLogTestContext::new_not_durable( FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, @@ -240,6 +241,7 @@ async fn test_flush_is_strict_durability_barrier() { .unwrap(); } ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!( ctx.raft_log.durable_index(), 20, @@ -258,6 +260,7 @@ async fn test_flush_is_strict_durability_barrier() { .unwrap(); } ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); assert_eq!( ctx.raft_log.durable_index(), 50, @@ -288,11 +291,9 @@ async fn test_flush_propagates_io_error() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -350,11 +351,9 @@ async fn test_fsync_failure_poisons_and_rejects_writes_after_reset() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -407,11 +406,9 @@ async fn test_replace_range_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -497,11 +494,9 @@ async fn test_purge_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -547,11 +542,9 @@ async fn test_reset_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -596,11 +589,9 @@ async fn test_save_hard_state_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -645,11 +636,9 @@ async fn test_poisoned_rejects_save_hard_state() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -674,7 +663,7 @@ async fn test_poisoned_rejects_save_hard_state() { } // ============================================================================ -// Gap fix: handle_non_write_cmd now checks is_poisoned() before executing +// Gap fix: run_storage_tasks now checks is_poisoned() before executing // ReplaceRange/Purge/Reset, instead of only checking it in run_batch_turn's // drain loop (which missed the direct-dispatch path in batch_processor's // top-level select, and the "just poisoned mid-turn" race). @@ -693,11 +682,9 @@ async fn test_poisoned_skips_replace_range() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -775,11 +762,9 @@ async fn test_poisoned_does_not_skip_reset() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -816,11 +801,9 @@ async fn test_poisoned_skips_purge() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -828,6 +811,9 @@ async fn test_poisoned_skips_purge() { let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); + // advance_durable_and_notify() clamps against max_index β€” simulate a log + // that already has the entry this test purges up to. + raft_log.set_memory_max_index_for_test(1); raft_log.poisoned.store(true, Ordering::SeqCst); let result = raft_log.purge_logs_up_to(LogId { term: 1, index: 1 }).await; @@ -843,13 +829,21 @@ async fn test_poisoned_skips_purge() { /// 2026-07-19 β€” `run_batch_turn`'s drain loop now replies before returning, /// instead of silently dropping the oneshot sender). /// -/// Ordering is made deterministic (not timing-sensitive) by gating the IO -/// thread inside its first `persist_entries()` call. While it's blocked, an +/// Ordering is made deterministic (not timing-sensitive) by gating the base +/// entries' `persist_entries()` call β€” `append_entries()` now calls it +/// synchronously, so the base append is spawned as its own task and blocks +/// there instead of returning immediately. While it's blocked, an /// `IOTask::Flush` is sent directly (guaranteed FIFO-first) followed by a /// conflict-triggering `filter_out_conflicts_and_append` call (sends -/// `IOTask::ReplaceRange` second). Releasing the gate lets `run_batch_turn` -/// drain both in one pass, in that order. -#[tokio::test] +/// `IOTask::ReplaceRange` second). Releasing the gate lets the base append +/// finish and `run_batch_turn` drain both queued commands in one pass, in +/// that order. +/// +/// Needs `flavor = "multi_thread"`: the gate blocks on a synchronous +/// `std::sync::mpsc::Receiver::recv()`, which would otherwise freeze the +/// single default executor thread that the spawned base-append task, the +/// conflict task, and this test body all need to share. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() { let (gate_tx, gate_rx) = std::sync::mpsc::channel::<()>(); let gate_rx = std::sync::Mutex::new(Some(gate_rx)); @@ -888,11 +882,9 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -900,30 +892,34 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); - // Base entries land in memory synchronously; the IO thread wakes and - // immediately blocks inside the gated persist_entries() call, before it - // ever drains the command queue. - raft_log - .append_entries(vec![ - Entry { - index: 1, - term: 1, - payload: None, - }, - Entry { - index: 2, - term: 1, - payload: None, - }, - Entry { - index: 3, - term: 1, - payload: None, - }, - ]) - .await - .unwrap(); - sleep(Duration::from_millis(20)).await; // let the IO thread reach the gate + // Base entries land in memory synchronously (before the gate), then + // append_entries() blocks inside its own gated persist_entries() call β€” + // spawned so the rest of this test can proceed while it's stuck there. + let base_append_task = tokio::spawn({ + let raft_log = raft_log.clone(); + async move { + raft_log + .append_entries(vec![ + Entry { + index: 1, + term: 1, + payload: None, + }, + Entry { + index: 2, + term: 1, + payload: None, + }, + Entry { + index: 3, + term: 1, + payload: None, + }, + ]) + .await + } + }); + sleep(Duration::from_millis(20)).await; // let it reach the gate // Send Flush directly β€” guarantees it's enqueued before the ReplaceRange // sent below, so it's the one already sitting in `replies` when the @@ -955,6 +951,12 @@ async fn test_run_batch_turn_replace_range_failure_replies_err_to_queued_flush() sleep(Duration::from_millis(50)).await; // let the ReplaceRange send land gate_tx.send(()).unwrap(); + timeout(Duration::from_secs(2), base_append_task) + .await + .expect("base append task must not hang") + .expect("base append task must not panic") + .expect("base append must succeed once the gate releases"); + let flush_result = timeout(Duration::from_secs(2), flush_rx) .await .expect("flush reply must not hang"); @@ -989,12 +991,10 @@ async fn test_new_buffered_raft_log_starts_unpoisoned() { let (raft_log, _receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify triggers fsync. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -1021,12 +1021,10 @@ async fn test_poisoned_survives_reset() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, // Safety-net disabled: only write_notify triggers fsync. flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -1045,8 +1043,12 @@ async fn test_poisoned_survives_reset() { /// A `persist_entries()` (page-cache write) failure poisons the log, exactly /// like an fsync failure does β€” these are two independent failure surfaces -/// (see `persist_pending_range` vs `FsyncCoordinator::run_until_caught_up`) -/// and both must reach the same fatal outcome. +/// (`persist_pending_range` vs `FsyncCoordinator::run_until_caught_up`) and +/// both must reach the same fatal outcome. +/// +/// The persist runs on the IO thread off `append_entries`'s task, so the +/// poison lands asynchronously β€” the black-box guarantee is that the *next* +/// write is rejected, not that this one fails. /// /// Without this test, a bug that only wires up ONE of the two poisoning /// paths (e.g. fsync failures poison correctly, but persist_entries @@ -1061,11 +1063,9 @@ async fn test_persist_entries_failure_poisons() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -1073,9 +1073,9 @@ async fn test_persist_entries_failure_poisons() { let raft_log = raft_log.start(receiver, None); std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - // Triggers the IO thread's persist_pending_range call, which hits the - // mock's first (failing) persist_entries() β€” this is the - // persist_pending_range poisoning path, NOT FsyncCoordinator's. + // Notifies the IO thread, which runs persist_pending_range and hits the + // mock's first (failing) persist_entries() β€” the persist_pending_range + // poisoning path, not FsyncCoordinator's. raft_log .append_entries(vec![Entry { index: 1, @@ -1129,11 +1129,9 @@ async fn test_notify_fatal_channel_closed_still_poisons_and_logs() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -1168,3 +1166,289 @@ async fn test_notify_fatal_channel_closed_still_poisons_and_logs() { not a silent no-op β€” see notify_fatal()'s error! call" ); } + +/// Efficiency: the IO thread's persist scan must start from its own page-cache +/// frontier, not from `durable_index`. Since #446 `durable_index` only advances +/// after an `FsyncCompleted` round-trips through raft.rs's event loop; under +/// load it lags far behind what the IO thread has already written. If the scan +/// restarted from `durable_index + 1` on every wakeup, each of N appends would +/// re-scan and re-`persist_entries` the whole not-yet-durable window β€” O(N^2) +/// total work. +/// +/// This test pins `durable_index` at 0 (no `log_flush_tx`, so no +/// `FsyncCompleted` is ever consumed) and appends N entries one at a time. The +/// total number of entries handed to `persist_entries` across all calls must +/// stay ~N, not ~N^2/2. +#[tokio::test] +async fn test_persist_scan_tracks_frontier_not_stuck_durable_index() { + let persisted_total = Arc::new(AtomicU64::new(0)); + let persisted_total_c = persisted_total.clone(); + + let mut log_store = MockLogStore::new(); + log_store.expect_last_index().returning(|| 0); + log_store.expect_persist_entries().returning(move |entries| { + persisted_total_c.fetch_add(entries.len() as u64, Ordering::Relaxed); + Ok(()) + }); + log_store.expect_entry().returning(|_| Ok(None)); + log_store.expect_get_entries().returning(|_| Ok(vec![])); + log_store.expect_purge().returning(|_| Ok(())); + log_store.expect_load_purge_boundary().returning(|| Ok(None)); + log_store.expect_reset().returning(|| Ok(())); + log_store.expect_truncate().returning(|_| Ok(())); + log_store.expect_replace_range().returning(|from, new_entries| { + Ok(new_entries.last().map(|e| e.index).unwrap_or(from.saturating_sub(1))) + }); + log_store.expect_is_write_durable().returning(|| false); + log_store.expect_flush().returning(|| Ok(())); + log_store.expect_flush_async().returning(|| Ok(())); + + let mut meta_store = MockMetaStore::new(); + meta_store.expect_save_hard_state().returning(|_| Ok(())); + meta_store.expect_load_hard_state().returning(|| Ok(None)); + meta_store.expect_flush().returning(|| Ok(())); + meta_store.expect_flush_async().returning(|| Ok(())); + + let storage = Arc::new(MockStorageEngine::from(log_store, meta_store)); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + // No log_flush_tx: FsyncCompleted is never consumed, so durable_index + // stays pinned at 0 for the whole test. + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + const N: u64 = 100; + for i in 1..=N { + raft_log + .append_entries(vec![Entry { + index: i, + term: 1, + payload: None, + }]) + .await + .unwrap(); + // flush() forces the IO thread to persist up to memory_max_index right + // now, so the scan boundary is exercised once per append β€” deterministic, + // no sleeps. + raft_log.flush().await.unwrap(); + } + + assert_eq!( + raft_log.durable_index(), + 0, + "durable_index must stay stuck for this test to be meaningful" + ); + let total = persisted_total.load(Ordering::Relaxed); + assert!( + total < 3 * N, + "persist_entries received {total} entries for {N} appends; a frontier-tracking \ + scan is ~{N}, a durable_index-relative scan would be ~{} (O(N^2))", + N * (N + 1) / 2 + ); +} + +/// Cold start: after a restart, `durable_index` starts at the disk length and +/// the IO thread's persist frontier must start *past* it. The first write's +/// persist scan begins at `durable_index + 1` β€” an already-durable entry on +/// disk must never be handed back to `persist_entries`. +/// +/// Guards the frontier initialization (`= durable_index`, scans use `+ 1`). +#[tokio::test] +async fn test_cold_start_persist_frontier_starts_past_durable_index() { + let persist_calls: Arc>>> = Arc::new(Mutex::new(Vec::new())); + + let mut log_store = MockLogStore::new(); + log_store.expect_last_index().returning(|| 5); // disk already holds 1..=5 + log_store.expect_get_entries().returning(|range| { + Ok(range + .map(|i| Entry { + index: i, + term: 1, + payload: None, + }) + .collect()) + }); + { + let calls = persist_calls.clone(); + log_store.expect_persist_entries().returning(move |entries| { + calls.lock().unwrap().push(entries.iter().map(|e| e.index).collect()); + Ok(()) + }); + } + log_store.expect_entry().returning(|_| Ok(None)); + log_store.expect_purge().returning(|_| Ok(())); + log_store.expect_load_purge_boundary().returning(|| Ok(None)); + log_store.expect_reset().returning(|| Ok(())); + log_store.expect_truncate().returning(|_| Ok(())); + log_store + .expect_replace_range() + .returning(|from, e| Ok(e.last().map(|x| x.index).unwrap_or(from.saturating_sub(1)))); + log_store.expect_is_write_durable().returning(|| false); + log_store.expect_flush().returning(|| Ok(())); + log_store.expect_flush_async().returning(|| Ok(())); + + let mut meta_store = MockMetaStore::new(); + meta_store.expect_save_hard_state().returning(|_| Ok(())); + meta_store.expect_load_hard_state().returning(|| Ok(None)); + meta_store.expect_flush().returning(|| Ok(())); + meta_store.expect_flush_async().returning(|| Ok(())); + + let storage = Arc::new(MockStorageEngine::from(log_store, meta_store)); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + assert_eq!( + raft_log.durable_index(), + 5, + "restart: disk length 5 is treated as durable" + ); + + // First write after restart. Its persist scan must start at 6. + raft_log + .append_entries(vec![Entry { + index: 6, + term: 1, + payload: None, + }]) + .await + .unwrap(); + sleep(Duration::from_millis(50)).await; + + let calls = persist_calls.lock().unwrap().clone(); + assert!(!calls.is_empty(), "entry 6 must have been persisted"); + assert!( + calls.iter().flatten().all(|&idx| idx >= 6), + "cold start: the first persist must scan from durable_index+1 (6), never \ + re-scan already-durable entry 5. Got: {calls:?}" + ); +} + +/// The flush turn's unconditional tail re-scan must persist writes that were +/// coalesced into the turn β€” a write whose `IOTask::Persist` is dropped in the +/// drain loop still becomes durable, because the catch-up re-reads +/// `memory_max_index` and persists everything past the frontier. +/// +/// Deterministic via a per-call persist gate: every `persist_entries` announces +/// its indices and blocks until released. +#[tokio::test(flavor = "multi_thread", worker_threads = 3)] +async fn test_flush_turn_catch_up_persists_writes_coalesced_during_the_turn() { + let (entered_tx, entered_rx) = std::sync::mpsc::channel::>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let release_rx = Mutex::new(release_rx); + + let mut log_store = MockLogStore::new(); + log_store.expect_last_index().returning(|| 0); + log_store.expect_persist_entries().returning(move |entries| { + entered_tx.send(entries.iter().map(|e| e.index).collect()).ok(); + release_rx.lock().unwrap().recv().ok(); + Ok(()) + }); + log_store.expect_entry().returning(|_| Ok(None)); + log_store.expect_get_entries().returning(|_| Ok(vec![])); + log_store.expect_purge().returning(|_| Ok(())); + log_store.expect_load_purge_boundary().returning(|| Ok(None)); + log_store.expect_reset().returning(|| Ok(())); + log_store.expect_truncate().returning(|_| Ok(())); + log_store + .expect_replace_range() + .returning(|from, e| Ok(e.last().map(|x| x.index).unwrap_or(from.saturating_sub(1)))); + log_store.expect_is_write_durable().returning(|| false); + log_store.expect_flush().returning(|| Ok(())); + log_store.expect_flush_async().returning(|| Ok(())); + + let mut meta_store = MockMetaStore::new(); + meta_store.expect_save_hard_state().returning(|_| Ok(())); + meta_store.expect_load_hard_state().returning(|| Ok(None)); + meta_store.expect_flush().returning(|| Ok(())); + meta_store.expect_flush_async().returning(|| Ok(())); + + let storage = Arc::new(MockStorageEngine::from(log_store, meta_store)); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + std::thread::sleep(Duration::from_millis(10)); + + let e = |i: u64| Entry { + index: i, + term: 1, + payload: None, + }; + + // 1..=2 persisted (frontier β†’ 2). + raft_log.append_entries(vec![e(1), e(2)]).await.unwrap(); + assert_eq!(entered_rx.recv().unwrap(), vec![1, 2]); + release_tx.send(()).unwrap(); + + // 3..=4: their Persist is in progress (blocked in persist_entries). + raft_log.append_entries(vec![e(3), e(4)]).await.unwrap(); + assert_eq!(entered_rx.recv().unwrap(), vec![3, 4]); + + // flush() enqueues IOTask::Flush behind the in-progress Persist(3,4). + let flush_task = { + let rl = raft_log.clone(); + tokio::spawn(async move { rl.flush().await }) + }; + tokio::time::sleep(Duration::from_millis(30)).await; + + // 5..=6 land while Persist(3,4) is still blocked β†’ their Persist queues + // behind Flush. + raft_log.append_entries(vec![e(5), e(6)]).await.unwrap(); + release_tx.send(()).unwrap(); // release Persist(3,4) β†’ frontier β†’ 4 + + // IO thread moves to Flush β†’ run_flush_turn. Leading persist covers 5,6. + assert_eq!(entered_rx.recv().unwrap(), vec![5, 6]); + + // 7..=8 land now β€” after run_flush_turn read memory_max for its leading + // persist, before its drain loop. Their Persist queues behind Flush and + // will be dropped in the drain loop. + raft_log.append_entries(vec![e(7), e(8)]).await.unwrap(); + release_tx.send(()).unwrap(); // release leading persist(5,6) β†’ frontier β†’ 6 + + // The drain loop drops Persist(5,6) and Persist(7,8); the unconditional + // tail re-scan then persists 7,8. + assert_eq!( + entered_rx.recv().unwrap(), + vec![7, 8], + "flush turn's tail re-scan must persist 7,8 whose Persist was dropped" + ); + release_tx.send(()).unwrap(); + + flush_task.await.unwrap().unwrap(); + while let Ok(ev) = log_flush_rx.try_recv() { + if let crate::InternalEvent::FsyncCompleted(mark) = ev { + raft_log.try_advance_durable_index(mark); + } + } + assert_eq!( + raft_log.durable_index(), + 8, + "7,8 (coalesced into the flush turn) must be durable via the catch-up" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs index 521338b3..f6602e40 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_test.rs @@ -6,16 +6,12 @@ use tokio::sync::mpsc; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, MockStorageEngine, simulate_insert_command}; -use crate::{ - BufferedRaftLog, FlushPolicy, InternalEvent, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, -}; +use crate::{BufferedRaftLog, FlushPolicy, InternalEvent, MockTypeConfig, PersistenceConfig}; use d_engine_proto::common::{Entry, LogId}; #[tokio::test] async fn test_durable_index_monotonic_under_concurrency() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -43,6 +39,7 @@ async fn test_durable_index_monotonic_under_concurrency() { // Wait for flush to complete tokio::time::sleep(Duration::from_millis(200)).await; + ctx.drain_fsync_completions(); // Verify monotonicity let durable = ctx.raft_log.durable_index(); @@ -55,7 +52,6 @@ async fn test_durable_index_monotonic_under_concurrency() { #[tokio::test] async fn test_durable_index_with_non_contiguous_entries() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -126,11 +122,9 @@ async fn test_purge_does_not_regress_durable_index_already_ahead() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -141,6 +135,7 @@ async fn test_purge_does_not_regress_durable_index_already_ahead() { // Arrange: entries 1..=100, all flushed β€” durable_index reaches 100 and // fires LogFlushed(100). simulate_insert_command(&raft_log, (1..=100).collect(), 1).await; + crate::test_utils::drain_and_apply_fsync_completions(&raft_log, &mut log_flush_rx); assert_eq!(raft_log.durable_index(), 100); // Drain the LogFlushed(100) from the insert+flush above β€” not what this diff --git a/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs new file mode 100644 index 00000000..2df3c6a3 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/durable_index_truncation_clamp_test.rs @@ -0,0 +1,293 @@ +//! `durable_index` must never claim more of the log survived to disk than the +//! log actually holds right now. The danger case: a term-conflict truncation +//! shrinks the log while a persist / fsync for the old, longer log is still in +//! flight β€” the stale in-flight write must not push `durable_index` past the +//! truncation point. Two guards cover this: `try_advance_durable_index`'s term +//! check, and the `FsyncCoordinator` generation fence bumped by `remove_range`. + +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; + +use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; + +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use crate::{ + BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, + PersistenceConfig, +}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// After a drastic truncate-then-regrow, `durable_index` must land exactly on +/// the new tail β€” never above it (would claim durability for discarded +/// entries), never stuck below it (the new tail must actually become durable). +#[tokio::test] +async fn test_durable_index_lands_on_new_tail_after_truncate_and_resync() { + let mut ctx = BufferedRaftLogTestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, // isolate from the safety-net timer + }, + "durable_index_lands_on_new_tail_after_truncate_and_resync", + ); + + // Old leader (term 1) replicates 1..=10. append_entries inserts them into + // memory and notifies the IO thread; nothing is fsync-confirmed until the + // FsyncCompleted events are drained below. + ctx.append_entries(1, 10, 1).await; + assert_eq!(ctx.raft_log.last_entry_id(), 10); + assert_eq!( + ctx.raft_log.durable_index(), + 0, + "no fsync report drained yet" + ); + + // New leader (term 2): index 2 conflicts, so the log is truncated from 2 + // and replaced with a single new entry β€” real log becomes [1, 2]. Slow + // path: remove_range(2..) drops memory_max_index to 1 and clamps + // durable_index down, then the new index 2 is inserted. + ctx.raft_log + .filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]) + .await + .unwrap(); + assert_eq!(ctx.raft_log.last_entry_id(), 2, "log is now [1, 2]"); + + // flush() only returns after its own fsync-completion event is enqueued, + // so draining right here is deterministic β€” no sleep needed. + ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); + + // The stale report for index 10 must be rejected (index 10 no longer + // exists); the report for index 2 must be accepted. + assert!( + ctx.raft_log.durable_index() <= ctx.raft_log.last_entry_id(), + "durable_index ({}) must not exceed last_entry_id ({})", + ctx.raft_log.durable_index(), + ctx.raft_log.last_entry_id() + ); + assert_eq!( + ctx.raft_log.durable_index(), + 2, + "durable_index must reach the true tail (2), not a stale pre-truncation watermark" + ); +} + +/// A persist whose entry set was captured *before* a truncation but finishes +/// *after* it must not let a stale fsync-completion report advance +/// `durable_index` into the range the truncation discarded. +/// +/// Timeline (deterministic via the persist gate): +/// 1. Old leader (term 1) replicates 1..=10. append_entries returns at once; +/// the IO thread starts persisting the range and blocks on the gate β€” its +/// captured entry set is [1..=10]. +/// 2. New leader (term 2): index 2 conflicts. filter_out_conflicts_and_append +/// runs remove_range(2..) synchronously (log is now [1]), inserts the new +/// index 2, and queues IOTask::ReplaceRange β€” which can't run yet, the IO +/// thread is still on the gate. +/// 3. Release the gate: the stale persist writes [1..=10], then the queued +/// ReplaceRange fixes the disk, then a flush() drives one legitimate +/// fsync-completion for index 2. +/// +/// Expected: draining the fsync completions advances `durable_index` to 2 and +/// rejects the stale report for index 10. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_stale_persist_after_truncation_does_not_advance_durable_index() { + let (storage, persist_gate) = + MockStorageEngine::not_durable_gated_persist("stale_persist_after_truncation".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let (log_flush_tx, mut log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + std::thread::sleep(Duration::from_millis(10)); + + // Step 1: replicate 1..=10; the IO thread blocks persisting this range. + let entries: Vec = (1..=10).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(entries).await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + // Step 2: term conflict at index 2 β€” on a task, since its ReplaceRange + // await is stuck behind the gated persist. + let truncate = { + let raft_log = raft_log.clone(); + tokio::spawn(async move { + raft_log.filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]).await + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + raft_log.last_entry_id(), + 2, + "in-memory truncation is synchronous β€” visible without waiting on the IO thread" + ); + + // Step 3: release the stale persist, let ReplaceRange land, then flush. + persist_gate.send(()).expect("IO thread should still be waiting on the gate"); + truncate.await.unwrap().unwrap(); + raft_log.flush().await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + // Drain fsync completions the way raft.rs's event loop would. + while let Ok(event) = log_flush_rx.try_recv() { + if let crate::InternalEvent::FsyncCompleted(mark) = event { + raft_log.try_advance_durable_index(mark); + } + } + + assert!( + raft_log.durable_index() <= raft_log.last_entry_id(), + "durable_index ({}) must never exceed last_entry_id ({}) β€” the stale persist \ + for 1..=10 must not be reported durable after truncation shrank the log to [1, 2]", + raft_log.durable_index(), + raft_log.last_entry_id() + ); + assert_eq!( + raft_log.durable_index(), + 2, + "durable_index must land on the post-truncation tail (2), not the stale 10" + ); +} + +/// `persist_pending_range` must report the highest index it *actually wrote*, +/// not the upper scan bound it was handed. The two differ during a truncation +/// race: the IO thread latched `memory_max_index` = 10 (an old leader had sent +/// 8, 9, 10), then a term-conflict truncation removed everything above 7 before +/// the SkipMap scan ran. Asking to persist `(4, 10]` then writes only 5, 6, 7. +/// +/// Returning the bound (10) would push the caller's `persisted_index` and the +/// fsync target past entries that never reached disk β€” a redundant fdatasync +/// plus a spurious `FsyncCompleted{10}` that the term check then has to reject. +/// Reporting 7 keeps every downstream watermark on real data. +#[tokio::test] +async fn test_persist_pending_range_reports_written_max_not_scan_bound() { + let storage = Arc::new(MockStorageEngine::with_id( + "persist_pending_range_reports_written_max".into(), + )); + // No `.start()` β€” drive `persist_pending_range` directly, no IO thread. + let (raft_log, _receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + let raft_log = Arc::new(raft_log); + + raft_log.append_entries((1..=7).map(|i| entry(i, 1)).collect()).await.unwrap(); + + // Scan bound is 10 (stale latch); the SkipMap holds only 1..=7. + let written = BufferedRaftLog::persist_pending_range(&raft_log, 5, 10, "test").await.unwrap(); + + assert_eq!( + written, + Some(LogId { term: 1, index: 7 }), + "must report the highest index actually written (7), not the scan bound (10)" + ); +} + +/// After a term-conflict truncation, the IO thread's persist frontier must land +/// *past* the new tail β€” not on it. `IOTask::ReplaceRange` already wrote (and +/// fsynced) the new tail via `replace_range`; the next write's persist scan must +/// start at `new_tail + 1`. If the frontier is left *at* `new_tail`, every +/// subsequent write re-scans and re-`persist_entries` that one boundary entry +/// (and re-submits a redundant fsync for it) β€” the exact waste #446 removes. +/// +/// Guards the "highest-persisted" watermark semantics: `ReplaceRange` sets the +/// watermark to `new_tail`, and scans start at `watermark + 1`. +#[tokio::test] +async fn test_persist_frontier_skips_new_tail_after_truncation() { + // Records the index list of every persist_entries() call. + let persist_calls: Arc>>> = Arc::new(Mutex::new(Vec::new())); + + let mut log_store = MockLogStore::new(); + log_store.expect_last_index().returning(|| 0); + { + let calls = persist_calls.clone(); + log_store.expect_persist_entries().returning(move |entries| { + calls.lock().unwrap().push(entries.iter().map(|e| e.index).collect()); + Ok(()) + }); + } + log_store.expect_replace_range().returning(|from, new_entries| { + Ok(new_entries.last().map(|e| e.index).unwrap_or(from.saturating_sub(1))) + }); + log_store.expect_truncate().returning(|_| Ok(())); + log_store.expect_entry().returning(|_| Ok(None)); + log_store.expect_get_entries().returning(|_| Ok(vec![])); + log_store.expect_purge().returning(|_| Ok(())); + log_store.expect_load_purge_boundary().returning(|| Ok(None)); + log_store.expect_reset().returning(|| Ok(())); + log_store.expect_is_write_durable().returning(|| false); + log_store.expect_flush().returning(|| Ok(())); + log_store.expect_flush_async().returning(|| Ok(())); + + let mut meta_store = MockMetaStore::new(); + meta_store.expect_save_hard_state().returning(|_| Ok(())); + meta_store.expect_load_hard_state().returning(|| Ok(None)); + meta_store.expect_flush().returning(|| Ok(())); + meta_store.expect_flush_async().returning(|| Ok(())); + + let storage = Arc::new(MockStorageEngine::from(log_store, meta_store)); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + storage, + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + // Old leader (term 1): entries 1..=10, persisted. + raft_log.append_entries((1..=10).map(|i| entry(i, 1)).collect()).await.unwrap(); + raft_log.flush().await.unwrap(); + + // New leader (term 2): conflict at index 6 β†’ truncate [6..], replace with + // [6, 7] (term 2). `filter_out_conflicts_and_append` awaits the + // `IOTask::ReplaceRange` reply, so the frontier is at new-tail 7 on return. + raft_log + .filter_out_conflicts_and_append(5, 1, vec![entry(6, 2), entry(7, 2)]) + .await + .unwrap(); + + // Only care about persist calls from here on β€” no flush() in between, so the + // next append's `IOTask::Persist` is the first thing to touch the frontier. + persist_calls.lock().unwrap().clear(); + + // Next write extends the log. Its persist scan must start at 8, not 7. + raft_log.append_entries((8..=10).map(|i| entry(i, 2)).collect()).await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + let calls = persist_calls.lock().unwrap().clone(); + let re_persisted_tail = calls.iter().flatten().any(|&idx| idx <= 7); + assert!( + !re_persisted_tail, + "after ReplaceRange set the frontier at new-tail 7, the next persist must \ + start at 8 β€” entry 7 (or below) must not be handed to persist_entries again. \ + Got calls: {calls:?}" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs index 60f00876..15ff8451 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/edge_cases_test.rs @@ -1,14 +1,13 @@ use bytes::Bytes; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; use d_engine_proto::common::{Entry, EntryPayload, LogId}; #[tokio::test] async fn test_empty_log_operations() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -27,7 +26,6 @@ async fn test_empty_log_operations() { #[tokio::test] async fn test_single_entry_operations() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -56,7 +54,6 @@ async fn test_single_entry_operations() { #[tokio::test] async fn test_gap_handling_in_indexes() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -99,7 +96,6 @@ async fn test_gap_handling_in_indexes() { #[tokio::test] async fn test_extreme_boundary_conditions() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs index 44fd0040..d3cbfab7 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/flush_strategy_test.rs @@ -13,7 +13,7 @@ use d_engine_proto::common::{Entry, EntryPayload}; use tokio::time::{Duration, sleep}; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; /// Test MemFirst with threshold=1 persists entries after flush /// @@ -22,8 +22,7 @@ use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; /// - Expected: durable_index == 5 after explicit flush() #[tokio::test] async fn test_mem_first_entries_durable_after_flush() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -33,6 +32,7 @@ async fn test_mem_first_entries_durable_after_flush() { // Act: Append entries then wait for durability ctx.append_entries(1, 5, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Assert: All entries durable after flush assert_eq!( @@ -51,8 +51,7 @@ async fn test_mem_first_entries_durable_after_flush() { /// - Expected: All 1000 entries durable after flush(), no data loss #[tokio::test] async fn test_mem_first_concurrent_writes_durable_after_flush() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -83,6 +82,7 @@ async fn test_mem_first_concurrent_writes_durable_after_flush() { // Wait for all entries to become durable ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Assert: All entries durable after flush assert_eq!( @@ -102,7 +102,6 @@ async fn test_mem_first_concurrent_writes_durable_after_flush() { #[tokio::test] async fn test_mem_first_crash_recovery_restores_flushed_entries() { let original_ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -149,7 +148,6 @@ async fn test_mem_first_crash_recovery_restores_flushed_entries() { #[tokio::test] async fn test_mem_first_buffers_entries_before_flush() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1000, }, @@ -172,8 +170,7 @@ async fn test_mem_first_buffers_entries_before_flush() { /// - Expected: Entries become durable after flush #[tokio::test] async fn test_mem_first_flushes_asynchronously() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -186,6 +183,7 @@ async fn test_mem_first_flushes_asynchronously() { // Act: Explicit flush ctx.raft_log.flush().await.unwrap(); sleep(Duration::from_millis(100)).await; // Allow async flush + ctx.drain_fsync_completions(); // Assert: Entries now durable assert!( @@ -202,7 +200,6 @@ async fn test_mem_first_flushes_asynchronously() { #[tokio::test] async fn test_mem_first_concurrent_buffering() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 5000, }, @@ -242,8 +239,7 @@ async fn test_mem_first_concurrent_buffering() { /// - Expected: Flush triggered at threshold #[tokio::test] async fn test_batched_flushes_at_threshold() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 10000, // High interval to test threshold trigger }, @@ -253,6 +249,7 @@ async fn test_batched_flushes_at_threshold() { // Act: Append exactly threshold entries ctx.append_entries(1, 5, 1).await; sleep(Duration::from_millis(100)).await; // Allow flush + ctx.drain_fsync_completions(); // Assert: Entries should be flushed assert!( @@ -268,8 +265,7 @@ async fn test_batched_flushes_at_threshold() { /// - Expected: Flush triggered by timer #[tokio::test] async fn test_batched_flushes_at_interval() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -279,6 +275,7 @@ async fn test_batched_flushes_at_interval() { // Act: Append few entries and wait for interval ctx.append_entries(1, 2, 1).await; sleep(Duration::from_millis(200)).await; // Wait for interval flush + ctx.drain_fsync_completions(); // Assert: Entries flushed by timer assert!( @@ -295,7 +292,6 @@ async fn test_batched_flushes_at_interval() { #[tokio::test] async fn test_batched_partial_flush_recovery() { let original_ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs index 095db56d..ab405f0c 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/id_allocation_test.rs @@ -11,8 +11,7 @@ use std::sync::Arc; use std::sync::atomic::Ordering; use crate::{ - BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, RaftLog, + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, RaftLog, }; fn setup_memory() -> Arc> { @@ -20,11 +19,9 @@ fn setup_memory() -> Arc> { let (raft_log, _receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs index a9530bf9..62bc61ff 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/performance_test.rs @@ -12,7 +12,7 @@ use tokio::time::Instant; use crate::{ BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, RaftLog, + PersistenceConfig, RaftLog, }; use d_engine_proto::common::{Entry, EntryPayload}; @@ -47,26 +47,18 @@ async fn test_reset_performance_during_active_flush() { let max_reset_duration_ms = FLUSH_DELAY_MS * 3; // 600ms: accounts for IO thread overhead let test_cases = vec![ - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1000, - }, - ), - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1, - }, - ), + FlushPolicy::Batch { + idle_flush_interval_ms: 1000, + }, + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, ]; - for (strategy, flush_policy) in test_cases { + for flush_policy in test_cases { let storage = create_delayed_storage(FLUSH_DELAY_MS); let config = PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }; @@ -100,9 +92,8 @@ async fn test_reset_performance_during_active_flush() { assert!( duration.as_millis() < max_reset_duration_ms as u128, - "Reset took {}ms during active flush ({:?}/{:?})", + "Reset took {}ms during active flush ({:?})", duration.as_millis(), - strategy, flush_policy ); } @@ -124,11 +115,9 @@ async fn test_filter_conflicts_performance_during_flush() { for (idle_flush_interval_ms, max_duration_ms) in test_cases { let storage = create_delayed_storage(FLUSH_DELAY_MS); let config = PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }; @@ -190,21 +179,15 @@ async fn test_fresh_cluster_performance_consistency() { let max_duration_ms = if is_ci { 50 } else { 5 }; let test_cases = vec![ - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1000, - }, - ), - ( - PersistenceStrategy::MemFirst, - FlushPolicy::Batch { - idle_flush_interval_ms: 1, - }, - ), + FlushPolicy::Batch { + idle_flush_interval_ms: 1000, + }, + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, ]; - for (strategy, flush_policy) in test_cases { + for flush_policy in test_cases { let mut log_store = MockLogStore::new(); log_store.expect_is_write_durable().returning(|| true); log_store.expect_flush().return_once(|| Ok(())); @@ -215,9 +198,7 @@ async fn test_fresh_cluster_performance_consistency() { log_store.expect_reset().returning(|| Ok(())); let config = PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }; @@ -235,9 +216,8 @@ async fn test_fresh_cluster_performance_consistency() { assert!( duration.as_millis() < max_duration_ms as u128, - "Fresh cluster reset took {}ms ({:?}/{:?})", + "Fresh cluster reset took {}ms ({:?})", duration.as_millis(), - strategy, flush_policy ); } diff --git a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs index efb54b80..a53e1380 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/pipeline_overlap_test.rs @@ -13,12 +13,11 @@ use d_engine_proto::common::Entry; use crate::test_utils::BufferedRaftLogTestContext; use crate::{ BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, RaftLog, + PersistenceConfig, RaftLog, }; fn ctx(name: &str) -> BufferedRaftLogTestContext { BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, @@ -438,9 +437,9 @@ async fn test_io_task_replace_range_delegates_to_replace_range_not_truncate() { // replace_range() must be called exactly once for one conflict resolution let rr_counter = replace_range_count.clone(); - log_store.expect_replace_range().returning(move |_from, _entries| { + log_store.expect_replace_range().returning(move |_from, new_entries| { rr_counter.fetch_add(1, Ordering::Relaxed); - Ok(()) + Ok(new_entries.last().map(|e| e.index).unwrap_or(0)) }); // truncate() must NOT be called β€” IOTask::ReplaceRange owns the full operation @@ -472,11 +471,9 @@ async fn test_io_task_replace_range_delegates_to_replace_range_not_truncate() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/prev_log_index_zero_idempotency_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/prev_log_index_zero_idempotency_test.rs new file mode 100644 index 00000000..8a277552 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/prev_log_index_zero_idempotency_test.rs @@ -0,0 +1,245 @@ +//! Tests for `filter_out_conflicts_and_append` when `prev_log_index == 0`. +//! +//! Background: `prev_log_index == 0` is Raft's sentinel for "nothing before the start of the +//! log" β€” Rule 2 (term-at-prev-index check) is trivially satisfied because there is no real +//! entry 0 to compare. That does NOT license skipping Rules 3/4: the receiver must still +//! compare incoming entries against whatever it already has, starting at index 1, and only +//! touch the entries that actually conflict (differing term at the same index). A batch that +//! fully matches existing content must be a no-op β€” this is exactly what +//! `pipeline_overlap_test.rs` already proves for `prev_log_index > 0`. +//! +//! The current implementation special-cases `prev_log_index == 0` to unconditionally +//! `reset()` (wipe the whole log, `durable_index` included) before re-appending β€” regardless +//! of whether the incoming entries are a pure duplicate of what's already durably stored. A +//! leader that resends a `prev_log_index=0` probe (no backpressure, a retry, a reconnect) before +//! learning the follower already caught up will repeatedly destroy real, already-durable +//! progress. These tests are RED until `prev_log_index == 0` is folded into the same +//! overlap/conflict comparison used for `prev_log_index > 0`. + +use crate::FlushPolicy; +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; +use d_engine_proto::common::Entry; +use std::time::Duration; + +fn ctx(name: &str) -> BufferedRaftLogTestContext { + BufferedRaftLogTestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, + name, + ) +} + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// A genuinely fresh follower (no prior `append_entries` calls at all) receiving its very +/// first `prev_log_index=0` probe must accept and append normally. +/// +/// # Why this needs its own test, not just implicit coverage +/// The other tests in this file all pre-populate the log before calling +/// `filter_out_conflicts_and_append`, so none of them exercise the case the +/// `is_virtual_log_start` guard exists to protect: without it, `entry_term(0)` returns `None` +/// unconditionally (there is no real entry 0 to look up), so `entry_term(0) != Some(0)` would +/// be true and this β€” the single most basic, legitimate case β€” would be wrongly rejected as a +/// conflict. This test pins that guard directly. +/// +/// # Expected (holds both before and after the fix β€” this is a regression guard for the +/// `is_virtual_log_start` skip, not a RED/GREEN discriminator for the reset() removal) +#[tokio::test] +async fn test_filter_conflicts_zero_prev_on_genuinely_empty_log_appends_all() { + let ctx = ctx("zero_prev_genuinely_empty_log_appends_all"); + assert_eq!( + ctx.raft_log.last_entry_id(), + 0, + "precondition: log must be untouched" + ); + + // Act: the very first AppendEntries this follower ever receives. + let result = ctx + .raft_log + .filter_out_conflicts_and_append( + 0, + 0, + vec![ + entry(1, 1), + entry(2, 1), + entry(3, 1), + entry(4, 1), + entry(5, 1), + ], + ) + .await + .unwrap(); + + assert_eq!(result.unwrap().index, 5); + assert_eq!(ctx.raft_log.last_entry_id(), 5); + for i in 1u64..=5 { + assert_eq!(ctx.raft_log.entry(i).unwrap().unwrap().term, 1); + } +} + +/// A leader resending `prev_log_index=0` with entries the follower already has β€” durably β€” +/// must be a no-op. This is the exact T4/T5 scenario from the #446 investigation: the +/// follower's first response to a `prev_log_index=0` probe is withheld pending its own +/// `durable_index` catching up (RPO=0); if the leader resends the identical probe before that +/// withheld ACK is released, the follower must not throw away the progress it already made. +/// +/// # Expected (RED until fixed) +/// `durable_index()` and `last_entry_id()` stay at 5 β€” the duplicate probe changes nothing. +#[tokio::test] +async fn test_filter_conflicts_zero_prev_duplicate_resend_preserves_durable_index() { + let mut ctx = ctx("zero_prev_duplicate_preserves_durable_index"); + + // Arrange: follower already durably has [1..5], all term=1 β€” simulating a prior + // `prev_log_index=0` probe that succeeded and finished fsyncing. + for i in 1u64..=5 { + ctx.raft_log.append_entries(vec![entry(i, 1)]).await.unwrap(); + } + tokio::time::sleep(Duration::from_millis(50)).await; + ctx.drain_fsync_completions(); + assert_eq!( + ctx.raft_log.durable_index(), + 5, + "precondition: [1..5] must be durable" + ); + + // Act: leader resends the identical prev_log_index=0 probe β€” same entries, same terms. + // This is what happens when the leader's next_index[peer] never advanced (the leader + // hasn't processed a response yet), not a genuinely new peer. + let result = ctx + .raft_log + .filter_out_conflicts_and_append( + 0, + 0, + vec![ + entry(1, 1), + entry(2, 1), + entry(3, 1), + entry(4, 1), + entry(5, 1), + ], + ) + .await + .unwrap(); + + // Assert: no-op β€” durable progress must survive a duplicate zero-prev probe. + assert_eq!(result.unwrap().index, 5); + assert_eq!( + ctx.raft_log.last_entry_id(), + 5, + "last_entry_id must be unchanged" + ); + assert_eq!( + ctx.raft_log.durable_index(), + 5, + "a duplicate prev_log_index=0 resend must not regress durable_index β€” this is what \ + re-arms the withheld-ACK deadlock (RPO=0 withhold never resolves once durable_index \ + is wiped out from under it)" + ); + for i in 1u64..=5 { + assert_eq!( + ctx.raft_log.entry(i).unwrap().unwrap().term, + 1, + "index={i} must not be touched by a duplicate zero-prev probe" + ); + } +} + +/// A `prev_log_index=0` batch that overlaps existing content but also carries genuinely new +/// entries beyond it must append only the new tail β€” mirrors +/// `pipeline_overlap_test::test_filter_conflicts_pipeline_overlap_no_truncation`, anchored at +/// prev=0 instead of prev>0, to prove the same comparison logic applies uniformly regardless +/// of which branch computed `prev_log_index`. +/// +/// # Expected (RED until fixed) +/// [1..5] untouched, [6,7] appended. +#[tokio::test] +async fn test_filter_conflicts_zero_prev_overlap_appends_only_new_tail() { + let ctx = ctx("zero_prev_overlap_appends_only_new_tail"); + + // Arrange: follower has [1..5], term=1 (not necessarily durable yet β€” overlap detection + // must work purely off in-memory content, independent of durability). + for i in 1u64..=5 { + ctx.raft_log.append_entries(vec![entry(i, 1)]).await.unwrap(); + } + assert_eq!(ctx.raft_log.last_entry_id(), 5); + + // Act: leader sends prev_log_index=0 with [1..7] β€” [1..5] match, [6,7] are new. + let new_entries: Vec<_> = (1u64..=7).map(|i| entry(i, 1)).collect(); + let result = ctx.raft_log.filter_out_conflicts_and_append(0, 0, new_entries).await.unwrap(); + + // Assert: existing [1..5] untouched, new tail [6,7] appended. + assert_eq!(result.unwrap().index, 7); + assert_eq!(ctx.raft_log.last_entry_id(), 7); + for i in 1u64..=5 { + assert_eq!( + ctx.raft_log.entry(i).unwrap().unwrap().term, + 1, + "index={i} was already present and must not be truncated" + ); + } + assert_eq!(ctx.raft_log.entry(6).unwrap().unwrap().term, 1); + assert_eq!(ctx.raft_log.entry(7).unwrap().unwrap().term, 1); +} + +/// A genuine conflict at index 1 (different term than what the follower already has) must +/// still truncate and replace β€” proves the fix is a precise reuse of the existing +/// overlap/conflict comparison, not "prev_log_index=0 always becomes a no-op." +/// +/// # Scenario +/// Follower has [1..5] term=1 (stale, from a since-superseded leader). A new leader with no +/// prior knowledge of this follower (or after a purge/snapshot boundary reset) sends +/// prev_log_index=0 with [1..3] all term=2 β€” a real conflict at index=1. +/// +/// # Expected (should already hold both before and after the fix β€” this is the control case) +/// [1..3] replaced with term=2; nothing beyond index=3 survives. +#[tokio::test] +async fn test_filter_conflicts_zero_prev_real_conflict_truncates_and_replaces() { + let ctx = ctx("zero_prev_real_conflict_truncates_and_replaces"); + + // Arrange: follower has [1..5], all term=1. + for i in 1u64..=5 { + ctx.raft_log.append_entries(vec![entry(i, 1)]).await.unwrap(); + } + assert_eq!(ctx.raft_log.last_entry_id(), 5); + + // Act: prev_log_index=0, entries=[1(t2), 2(t2), 3(t2)] β€” conflicts at index=1 immediately. + let result = ctx + .raft_log + .filter_out_conflicts_and_append(0, 0, vec![entry(1, 2), entry(2, 2), entry(3, 2)]) + .await + .unwrap(); + + // Assert: [1..3] replaced with term=2; stale [4,5] from the old leader must not survive. + assert_eq!(result.unwrap().index, 3); + assert_eq!( + ctx.raft_log.last_entry_id(), + 3, + "stale tail beyond the new leader's log must be gone" + ); + for i in 1u64..=3 { + assert_eq!( + ctx.raft_log.entry(i).unwrap().unwrap().term, + 2, + "index={i} must be term=2" + ); + } + assert!( + ctx.raft_log.entry(4).unwrap().is_none(), + "stale index=4 must not survive" + ); + assert!( + ctx.raft_log.entry(5).unwrap().is_none(), + "stale index=5 must not survive" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs index 0429eebc..4d45b7ac 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/quorum_durability_test.rs @@ -1,62 +1,193 @@ //! Quorum Durability Tests //! -//! MemFirst design: leader contributes `last_entry_id` (in-memory) to quorum. -//! IO thread persistence is async and NOT on the commit critical path. +//! RPO=0 (#446): leader contributes `durable_index` (not `last_entry_id`) to quorum β€” +//! commit must not advance past what the leader itself has survived fsync for. //! -//! Follower ACK path: followers ACK immediately after memory write (no `wait_durable`). -//! IO thread fsyncs asynchronously; crash safety is guaranteed by quorum, not per-follower durability. +//! Superseded design (kept here as history, do not resurrect): the old MemFirst model had +//! the leader contribute `last_entry_id` (in-memory) so IO persistence never sat on the +//! commit critical path. That traded away RPO=0 β€” a majority-acked write could still be +//! lost on correlated power loss before fsync. This file's tests now lock in the new +//! behavior instead of the old one. +//! +//! Follower ACK path (tracked separately, not yet landed): followers will ACK only after +//! their own durable_index catches up β€” so a follower's reported match_index is inherently +//! already durable by the time the leader sees it. +//! +//! Election-eligibility comparison must keep reading the in-memory log, never +//! `durable_index` β€” a separate, independent invariant from the durable-quorum change +//! above, but one a majority-count safety argument for #446 depends on. See +//! `test_election_eligibility_reads_memory_log_not_durable_index`. +//! +//! Note: these tests rely on `BufferedRaftLog`'s in-memory layer existing (they force a +//! gap between `last_entry_id` and `durable_index` via a gated mock flush). If that layer +//! is ever removed, this file's setup assumptions need revisiting β€” not a decided plan, +//! just a known dependency to check first. +//! +//! Tests that need a genuine, un-fsynced gap between `last_entry_id` and `durable_index` +//! use `MockStorageEngine::not_durable_gated_flush` β€” a real channel-based gate, not a +//! timing guess. An earlier version of this file relied on a long `idle_flush_interval_ms` +//! and assumed the dedicated `raft-io-*` OS thread just wouldn't get scheduled before the +//! assertions ran; that's a real race (the IO thread is independent of the test's own +//! runtime), and it was intermittently losing under load β€” flaky, not broken logic. Do not +//! reintroduce that pattern here. +use crate::BufferedRaftLog; +use crate::FlushPolicy; +use crate::MockStorageEngine; +use crate::MockTypeConfig; +use crate::PersistenceConfig; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; +use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; +use std::sync::Arc; use std::time::Duration; -/// Flush policy with a far-future safety timer β€” IO thread only fsyncs on WriteNotify. -/// In current_thread test runtime, durable_index stays at 0 immediately after append_entries -/// because the IO thread task has no chance to run until the test yields. -fn no_auto_flush_policy() -> FlushPolicy { - FlushPolicy::Batch { - idle_flush_interval_ms: 999_999, - } +/// Entries `1..=n`, all at `term`, no payload β€” the shape these tests need. +fn entries( + n: u64, + term: u64, +) -> Vec { + (1..=n) + .map(|index| Entry { + index, + term, + payload: None, + }) + .collect() } -// ── Leader quorum uses last_entry_id (in-memory), not durable_index ── +// ── Leader quorum uses durable_index, not last_entry_id (RPO=0) ── -/// MemFirst: leader's quorum contribution is last_entry_id (in-memory), not durable_index. +/// RPO=0: leader's quorum contribution is durable_index (fsync-confirmed), not +/// last_entry_id (in-memory). /// -/// Even when durable_index=0 (IO thread has not flushed), quorum must be satisfied -/// as soon as last_entry_id + follower ACKs form a majority. IO persistence is async -/// and must NOT block commit. +/// Even when a follower has already ACKed an index, the leader must not count its own +/// un-fsynced entry toward quorum β€” otherwise a majority-looking commit can still lose +/// data on correlated power loss (the leader's own copy was never actually durable). /// -/// This test FAILS if calculate_majority_matched_index uses durable_index (the bug -/// introduced by fix #329 which incorrectly put IO thread latency on the commit -/// critical path, causing +617Β΅s avg latency regression in 3-node embedded bench). +/// This test FAILS if calculate_majority_matched_index still uses last_entry_id (the old +/// MemFirst behavior, since revoked). It replaces +/// `test_memfirst_quorum_uses_last_entry_id_not_durable_index`, which asserted the exact +/// opposite of this on purpose β€” that assertion documented a since-revoked design decision. #[tokio::test] -async fn test_memfirst_quorum_uses_last_entry_id_not_durable_index() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, - no_auto_flush_policy(), // durable_index stays 0 β€” IO thread won't run - "test_memfirst_quorum_last_entry_id", +async fn test_quorum_uses_durable_index_not_last_entry_id() { + // Gate closed: the first flush() call blocks until we send () on `flush_gate` β€” fsync + // deterministically never completes until we say so, no timing involved. + let (storage, flush_gate) = + MockStorageEngine::not_durable_gated_flush("test_quorum_durable_index".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready - // Entry written to SkipMap (in memory). IO thread has not flushed yet. - ctx.append_entries(1, 1, 1).await; + raft_log.append_entries(entries(1, 1)).await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; // let it reach the gate - assert_eq!(ctx.raft_log.last_entry_id(), 1); - assert_eq!(ctx.raft_log.durable_index(), 0); // IO thread hasn't run + assert_eq!(raft_log.last_entry_id(), 1); + assert_eq!(raft_log.durable_index(), 0); // gate never released β€” fsync hasn't completed - let result = ctx.raft_log.calculate_majority_matched_index( + let result = raft_log.calculate_majority_matched_index( 1, 0, - vec![1], // one follower acked index=1; together with leader = majority of 3 + vec![1], // one follower reports match=1 (already durable, post-Stage2 semantics) ); - // MemFirst: leader contributes last_entry_id=1. - // quorum = [leader=1, follower=1] β†’ majority of {leader, f1, f2} satisfied β†’ Some(1). + // RPO=0: leader contributes durable_index=0, not last_entry_id=1. + // peer_matched_ids = [follower=1, leader=0], sorted desc = [1,0], median(len/2=1) = 0. + // majority_index=0 is not < commit_index=0, so falls through to the term check on + // entry(0) β€” index 0 is not a real entry (log is 1-indexed) β€” Ok(None) β€” result is None. + assert_eq!( + result, None, + "RPO=0: the leader's own un-fsynced entry must not count toward quorum, even when \ + a follower has already acked it β€” one follower alone isn't majority without the \ + leader's own durable contribution" + ); + + let _ = flush_gate.send(()); // release so the blocked IO thread doesn't linger +} + +/// Election-eligibility comparison (`last_log_id`, consumed by +/// `election_handler::handle_vote_request`) must read the in-memory log, never +/// `durable_index`. A follower with an un-fsynced tail must still be able to correctly +/// reject a candidate whose log is genuinely less up to date β€” voting eligibility and +/// commit-durability are two separate concerns and must not be conflated by sharing the +/// same index source. +#[tokio::test] +async fn test_election_eligibility_reads_memory_log_not_durable_index() { + let (storage, flush_gate) = + MockStorageEngine::not_durable_gated_flush("test_election_eligibility_memory_log".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); + + raft_log.append_entries(entries(10, 1)).await.unwrap(); // entries 1..=10, term=1 + tokio::time::sleep(Duration::from_millis(50)).await; + + assert_eq!(raft_log.durable_index(), 0, "nothing fsynced yet"); + assert_eq!( + raft_log.last_log_id(), + Some(LogId { index: 10, term: 1 }), + "election-eligibility comparison must see the un-fsynced tail, not fall back to \ + durable_index=0 β€” a candidate with a truly-shorter log must still be rejected" + ); + + let _ = flush_gate.send(()); +} + +/// `calculate_majority_matched_index`'s median-based calculation requires an actual +/// majority of `peer_matched_ids` to reach an index before it counts toward commit β€” a +/// minority (here: 2 of 5) reporting a higher index cannot move the result past what the +/// rest of the cluster last confirmed. +/// +/// This is a pure property of the median calculation itself β€” the function has no way to +/// know whether any of its inputs are stale or expired, so this test does not by itself +/// prove anything about stale reports being harmless. It only pins down the arithmetic +/// that a separate, broader safety argument for #446 relies on. +#[tokio::test] +async fn test_majority_matched_index_requires_actual_majority_of_reports() { + let mut ctx = BufferedRaftLogTestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, + "test_majority_requires_actual_majority", + ); + + ctx.append_entries(1, 10, 1).await; + ctx.raft_log.flush().await.unwrap(); // leader's own entries now durable through 10 + ctx.drain_fsync_completions(); + + assert_eq!(ctx.raft_log.durable_index(), 10); + + // 1 of 4 followers reports index 10; the other 3 are still at their last-known + // value, 9. + let result = ctx.raft_log.calculate_majority_matched_index(1, 9, vec![10, 9, 9, 9]); + + // peer_matched_ids after leader's own contribution = [10, 9, 9, 9, 10] + // sorted desc = [10,10,9,9,9], median(len/2=2) = 9 β€” majority stays at 9, entry(9) + // exists with term=1=current_term, so the result is the previously-safe Some(9), not 10. assert_eq!( result, - Some(1), - "MemFirst: quorum must use last_entry_id, not durable_index β€” IO must not block commit" + Some(9), + "a minority (2 of 5) reporting a higher index cannot move majority past what the \ + other 3 nodes last confirmed" ); } @@ -66,8 +197,7 @@ async fn test_memfirst_quorum_uses_last_entry_id_not_durable_index() { /// quorum should proceed normally. #[tokio::test] async fn test_quorum_succeeds_after_leader_flush() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 999_999, // only threshold trigger, no timer }, @@ -79,6 +209,7 @@ async fn test_quorum_succeeds_after_leader_flush() { // Wait for flush to complete tokio::time::sleep(Duration::from_millis(50)).await; + ctx.drain_fsync_completions(); assert_eq!(ctx.raft_log.last_entry_id(), 1); assert_eq!( @@ -103,35 +234,46 @@ async fn test_quorum_succeeds_after_leader_flush() { // ── Bug 2: gap between last_entry_id and durable_index ── -/// Demonstrates that after append_entries with MemFirst + no-auto-flush, -/// last_entry_id and durable_index diverge. +/// Demonstrates that after append_entries with a stalled fsync, last_entry_id and +/// durable_index diverge. /// /// This is the root condition enabling the bug: both values exist, /// but quorum calculation only uses the unsafe one. #[tokio::test] async fn test_last_entry_id_diverges_from_durable_index_with_mem_first() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, - no_auto_flush_policy(), - "test_diverge_mem_first", + let (storage, flush_gate) = + MockStorageEngine::not_durable_gated_flush("test_diverge_mem_first".into()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); - ctx.append_entries(1, 5, 1).await; // entries 1..=5, no flush + raft_log.append_entries(entries(5, 1)).await.unwrap(); // entries 1..=5, no flush + tokio::time::sleep(Duration::from_millis(50)).await; - assert_eq!(ctx.raft_log.last_entry_id(), 5, "memory index should be 5"); + assert_eq!(raft_log.last_entry_id(), 5, "memory index should be 5"); assert_eq!( - ctx.raft_log.durable_index(), + raft_log.durable_index(), 0, "durable_index must remain 0: no flush has run" ); // This gap (5 vs 0) is exactly what the quorum bug exploits. + + let _ = flush_gate.send(()); } /// After explicit flush, durable_index must equal last_entry_id. #[tokio::test] async fn test_durable_index_equals_last_entry_id_after_flush() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -140,6 +282,7 @@ async fn test_durable_index_equals_last_entry_id_after_flush() { ctx.append_entries(1, 5, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); let last = ctx.raft_log.last_entry_id(); let durable = ctx.raft_log.durable_index(); diff --git a/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs index f7acfe7f..bbec2429 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/raft_properties_test.rs @@ -1,12 +1,11 @@ +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, simulate_insert_command}; -use crate::{FlushPolicy, PersistenceStrategy}; use d_engine_proto::common::Entry; #[tokio::test] async fn test_log_matching_property() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -45,8 +44,7 @@ async fn test_log_matching_property() { #[tokio::test] async fn test_leader_completeness_property() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -56,6 +54,7 @@ async fn test_leader_completeness_property() { // Leader writes entries and flushes so durable_index = 10 ctx.append_entries(1, 10, 1).await; ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Simulate majority replication scenario: // Leader has: [1,2,3,4,5,6,7,8,9,10], durable_index=10 @@ -83,8 +82,7 @@ async fn test_leader_completeness_property() { #[tokio::test] async fn test_calculate_majority_matched_index_case0() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -97,6 +95,7 @@ async fn test_calculate_majority_matched_index_case0() { simulate_insert_command(&ctx.raft_log, vec![1], 1).await; simulate_insert_command(&ctx.raft_log, vec![2, 3], 2).await; + ctx.drain_fsync_completions(); assert_eq!( Some(3), @@ -108,7 +107,6 @@ async fn test_calculate_majority_matched_index_case0() { #[tokio::test] async fn test_calculate_majority_matched_index_case1() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -131,8 +129,7 @@ async fn test_calculate_majority_matched_index_case1() { #[tokio::test] async fn test_calculate_majority_matched_index_case2() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -148,6 +145,7 @@ async fn test_calculate_majority_matched_index_case2() { simulate_insert_command(&ctx.raft_log, vec![1], 1).await; simulate_insert_command(&ctx.raft_log, vec![2], 2).await; simulate_insert_command(&ctx.raft_log, vec![3], 3).await; + ctx.drain_fsync_completions(); assert_eq!( Some(3), ctx.raft_log.calculate_majority_matched_index(ct, ci, vec![4, 2]) @@ -157,7 +155,6 @@ async fn test_calculate_majority_matched_index_case2() { #[tokio::test] async fn test_calculate_majority_matched_index_case3() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -180,7 +177,6 @@ async fn test_calculate_majority_matched_index_case3() { #[tokio::test] async fn test_calculate_majority_matched_index_case4() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -203,8 +199,7 @@ async fn test_calculate_majority_matched_index_case4() { #[tokio::test] async fn test_calculate_majority_matched_index_case5() { - let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = BufferedRaftLogTestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -223,6 +218,7 @@ async fn test_calculate_majority_matched_index_case5() { let raft_log_entry_ids: Vec = (1..=raft_log_length).collect(); simulate_insert_command(&ctx.raft_log, raft_log_entry_ids, 1).await; + ctx.drain_fsync_completions(); assert_eq!( Some(peer2_match), diff --git a/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs index 656e2c25..c62dc1eb 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/remove_range_test.rs @@ -1,8 +1,8 @@ use d_engine_proto::common::{Entry, LogId}; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::{BufferedRaftLogTestContext, simulate_insert_command}; -use crate::{FlushPolicy, PersistenceStrategy}; fn entry( index: u64, @@ -18,7 +18,6 @@ fn entry( #[tokio::test] async fn test_remove_middle_range() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -50,7 +49,6 @@ async fn test_remove_middle_range() { #[tokio::test] async fn test_remove_from_start() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -77,7 +75,6 @@ async fn test_remove_from_start() { #[tokio::test] async fn test_remove_to_end() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -105,7 +102,6 @@ async fn test_remove_to_end() { #[tokio::test] async fn test_remove_empty_range() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -126,7 +122,6 @@ async fn test_remove_empty_range() { #[tokio::test] async fn test_remove_entire_log() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -150,7 +145,6 @@ async fn test_remove_entire_log() { #[tokio::test] async fn test_remove_single_entry() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -185,7 +179,6 @@ async fn test_remove_single_entry() { #[tokio::test] async fn test_remove_range_clears_term_indexes_for_removed_entries() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -252,7 +245,6 @@ async fn test_remove_range_clears_term_indexes_for_removed_entries() { #[tokio::test] async fn test_purge_prefix_removes_entries_and_records_boundary_together() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -301,7 +293,6 @@ async fn test_purge_prefix_removes_entries_and_records_boundary_together() { #[tokio::test] async fn test_purge_prefix_multi_term_cutoff_updates_term_indexes_and_boundary() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs new file mode 100644 index 00000000..c6259867 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/replace_range_fsync_test.rs @@ -0,0 +1,89 @@ +//! `IOTask::ReplaceRange` (term-conflict truncation, see +//! `filter_out_conflicts_and_append`'s slow path) writes to the storage engine +//! synchronously and bumps `pending_max`, but is dispatched through the +//! `receiver.recv()` => `cmd => { run_storage_tasks(...) }` arm of the IO +//! thread's select loop β€” a branch that, unlike `run_batch_turn`, never calls +//! `fsync_coordinator.submit()`. If no further `append_entries()` call arrives +//! afterward (which would separately trigger a `run_batch_turn` via +//! `write_notify`), the replaced entries sit "written but never fsync-submitted" +//! indefinitely β€” nothing but the idle-timer safety net would ever flush them. +//! +//! This test pins down that gap: it disables the safety net (a very long +//! `idle_flush_interval_ms`) so only the normal notify-driven path could +//! possibly advance `durable_index`, then proves it never does after a +//! term-conflict truncation with no subsequent append. + +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::FlushPolicy; +use crate::storage::raft_log::RaftLog; +use crate::test_utils::BufferedRaftLogTestContext; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// A term-conflict truncation (`ReplaceRange`) must eventually become durable +/// even if no `append_entries()` call follows it. +/// +/// Today it does not: `ReplaceRange` is handled outside `run_batch_turn`, so +/// nothing submits fsync for it. Only the idle-timer safety net would catch +/// this β€” and this test disables that timer (60s interval, well beyond the +/// test's wait window) to isolate the notify-driven path from the safety net. +/// +/// RED (today): `durable_index()` never reaches `last_entry_id()` after the +/// truncation, because the replaced entries' fsync was never submitted. +#[tokio::test] +async fn test_replace_range_becomes_durable_without_a_following_append() { + let mut ctx = BufferedRaftLogTestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, // effectively disabled for this test's timeframe + }, + "replace_range_becomes_durable_without_a_following_append", + ); + + // Arrange: log [1,2,3] all term=1, explicitly flushed durable. + ctx.append_entries(1, 3, 1).await; + ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); + assert_eq!(ctx.raft_log.durable_index(), 3, "baseline must be durable"); + + // Act: leader (term=2) sends entries that conflict at index=2 and extend + // the log to index=4. filter_out_conflicts_and_append's slow path detects + // the term mismatch at index=2, truncates [2,3], and replaces with + // [2,3,4] (term=2) via IOTask::ReplaceRange β€” with no append_entries() + // call afterward. + let result = ctx + .raft_log + .filter_out_conflicts_and_append(1, 1, vec![entry(2, 2), entry(3, 2), entry(4, 2)]) + .await + .unwrap(); + assert_eq!(result.unwrap().index, 4); + assert_eq!( + ctx.raft_log.last_entry_id(), + 4, + "memory must reflect the replace" + ); + + // Give the IO thread ample time to have submitted fsync, if anything + // besides the (disabled) safety net were going to do it. + tokio::time::sleep(Duration::from_millis(200)).await; + ctx.drain_fsync_completions(); + + // FIXED: ReplaceRange's handler now submits fsync directly instead of + // relying on a following append/notify or the safety net. + assert_eq!( + ctx.raft_log.durable_index(), + 4, + "ReplaceRange must submit fsync itself, without needing a following append" + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs index c14d7c4a..79ebce89 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/shutdown_test.rs @@ -7,7 +7,7 @@ use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; use crate::{ BufferedRaftLog, FlushPolicy, MockLogStore, MockMetaStore, MockStorageEngine, MockTypeConfig, - PersistenceConfig, PersistenceStrategy, + PersistenceConfig, }; use d_engine_proto::common::{Entry, EntryPayload}; @@ -47,11 +47,9 @@ fn test_io_thread_survives_runtime_drop() { let (log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 50, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -84,7 +82,6 @@ fn test_io_thread_survives_runtime_drop() { #[tokio::test] async fn test_shutdown_closes_channel_properly() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -113,7 +110,6 @@ async fn test_shutdown_closes_channel_properly() { #[tokio::test] async fn test_shutdown_awaits_worker_completion() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 5000, }, @@ -159,11 +155,9 @@ async fn test_shutdown_handles_slow_workers() { let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, @@ -203,7 +197,6 @@ async fn test_shutdown_handles_slow_workers() { #[tokio::test] async fn test_shutdown_with_multiple_flushes() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -289,11 +282,9 @@ async fn test_replace_range_failure_propagates_error_and_shuts_down_io_thread() let (raft_log, receiver) = BufferedRaftLog::::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, // no auto-flush }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs index 5e37ef0d..cec818e4 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/term_index_test.rs @@ -8,12 +8,11 @@ use d_engine_proto::common::Entry; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; #[tokio::test] async fn test_first_index_for_term() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -88,7 +87,6 @@ async fn test_first_index_for_term() { #[tokio::test] async fn test_last_index_for_term() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -162,7 +160,6 @@ async fn test_last_index_for_term() { #[tokio::test] async fn test_term_index_functions_with_purged_logs() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -214,7 +211,6 @@ async fn test_term_index_functions_with_purged_logs() { #[tokio::test] async fn test_term_index_sequential_multi_term_insertion() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1000, }, @@ -263,7 +259,6 @@ async fn test_term_index_sequential_multi_term_insertion() { #[tokio::test] async fn test_term_indexes_rebuilt_correctly_after_restart() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -304,7 +299,6 @@ async fn test_term_indexes_rebuilt_correctly_after_restart() { #[tokio::test] async fn test_term_index_performance_large_dataset() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 5000, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs index a385cd5d..5f36a0f6 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/term_segments_test.rs @@ -16,7 +16,7 @@ use d_engine_proto::common::Entry; use crate::storage::buffered_raft_log::TermSegments; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy, RaftLog}; +use crate::{FlushPolicy, RaftLog}; // --------------------------------------------------------------------------- // Helpers @@ -37,7 +37,6 @@ fn entries( fn ctx(name: &str) -> BufferedRaftLogTestContext { BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, diff --git a/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs new file mode 100644 index 00000000..d5affa17 --- /dev/null +++ b/d-engine-core/src/storage/buffered_raft_log_test/truncation_fsync_fence_test.rs @@ -0,0 +1,96 @@ +//! `FsyncCoordinator`'s `generation` fence protects against a physical fsync +//! whose result arrives after the world it was syncing no longer exists β€” but +//! today only `reset()` (full wipe) bumps `generation` via `fence_reset()`. +//! Term-conflict truncation (`filter_out_conflicts_and_append`'s slow path, +//! `remove_range` + `IOTask::ReplaceRange`) does not. +//! +//! Scenario: a follower has 10 entries synchronously written to its storage +//! engine but not yet fsynced β€” a physical fsync for "up to index 10" is +//! already dispatched and running in the background. Before that fsync +//! returns, a new leader tells the follower its log from index=2 onward is +//! wrong; the follower truncates and replaces it, ending up with only +//! entries [1, 2]. The in-flight fsync β€” which has no way to know any of +//! this happened β€” then completes and reports "index 10 is durable" anyway. +//! `durable_index` only moves up (`fetch_max`), so nothing afterward can +//! correct this: `durable_index()` gets stuck claiming durability for +//! entries [3..=10], which don't exist in this follower's log anymore. + +use std::sync::Arc; +use std::time::Duration; + +use d_engine_proto::common::Entry; + +use crate::storage::raft_log::RaftLog; +use crate::{BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig}; + +fn entry( + index: u64, + term: u64, +) -> Entry { + Entry { + index, + term, + payload: None, + } +} + +/// `durable_index()` must never exceed `last_entry_id()` β€” a follower must +/// never claim durability for log entries a truncation has already discarded. +/// +/// RED (today): the stale in-flight fsync (dispatched for index=10, before +/// the truncation) is not fenced, and blindly advances `durable_index` to 10 +/// after the truncation has already shrunk the log to [1, 2]. +#[tokio::test] +async fn test_durable_index_does_not_adopt_a_stale_fsync_after_truncation() { + // Gate closed: the first flush() call β€” for the original 10-entry batch β€” + // blocks here until we release it, letting us deterministically truncate + // the log while that fsync is still "in flight". + let (storage, flush_gate) = MockStorageEngine::not_durable_gated_flush( + "durable_index_does_not_adopt_a_stale_fsync_after_truncation".into(), + ); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let raft_log = raft_log.start(receiver, None); + std::thread::sleep(Duration::from_millis(10)); // ensure IO thread is ready + + // Old leader (term=1) replicates entries 1..=10. append_entries() inserts + // them into memory and notifies the IO thread, which persists the range and + // dispatches a physical fsync for "up to index=10" β€” that fsync is now + // running in the background, blocked on flush_gate. + let entries: Vec = (1..=10).map(|i| entry(i, 1)).collect(); + raft_log.append_entries(entries).await.unwrap(); + + // Give the IO thread + blocking task time to reach the gated flush() call. + tokio::time::sleep(Duration::from_millis(50)).await; + + // New leader (term=2): index=2 conflicts, truncate and replace β€” the + // stale fsync (still blocked on the gate) has no way to observe this. + raft_log.filter_out_conflicts_and_append(1, 1, vec![entry(2, 2)]).await.unwrap(); + assert_eq!( + raft_log.last_entry_id(), + 2, + "log must be truncated and replaced down to [1, 2] before the stale fsync completes" + ); + + // Release the gate β€” the stale fsync (dispatched for index=10, before the + // truncation) now completes. + flush_gate.send(()).unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + assert!( + raft_log.durable_index() <= raft_log.last_entry_id(), + "durable_index ({}) must never exceed last_entry_id ({}) β€” the stale \ + fsync for index=10 must not be adopted after truncation shrank the \ + log to [1, 2]", + raft_log.durable_index(), + raft_log.last_entry_id() + ); +} diff --git a/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs b/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs index c2181802..965e8cf2 100644 --- a/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs +++ b/d-engine-core/src/storage/buffered_raft_log_test/worker_test.rs @@ -1,15 +1,14 @@ use std::time::Duration; +use crate::FlushPolicy; use crate::storage::raft_log::RaftLog; use crate::test_utils::BufferedRaftLogTestContext; -use crate::{FlushPolicy, PersistenceStrategy}; /// Verifies that the flush worker continues operating normally after processing a large number /// of flush tasks β€” the worker does not exit or become unresponsive under sustained load. #[tokio::test] async fn test_flush_worker_sustains_throughput_under_load() { let ctx = BufferedRaftLogTestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 50, }, diff --git a/d-engine-core/src/storage/fsync_coordinator.rs b/d-engine-core/src/storage/fsync_coordinator.rs index 47b74313..21be38f5 100644 --- a/d-engine-core/src/storage/fsync_coordinator.rs +++ b/d-engine-core/src/storage/fsync_coordinator.rs @@ -3,30 +3,34 @@ use crate::Error; use crate::LogStore; use crate::Result; use crate::TypeConfig; +use d_engine_proto::common::LogId; +use parking_lot::Mutex; use std::sync::Arc; -use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use tokio::sync::oneshot; use tracing::error; -/// Tracks whether a fsync task is currently running on the blocking pool. -/// Ensures at most one physical `flush_wal` call is in flight at any time, -/// restoring natural batching: entries that arrive while a fsync is running -/// accumulate in `pending_max`/`pending_replies`, and are picked up by the -/// SAME task once it finishes its current round β€” rather than spawning a -/// new competing task per `write_notify` wakeup. +/// Schedules physical fsync calls β€” batches concurrent requests into one +/// flush() at a time. The final content check is `BufferedRaftLog:: +/// try_advance_durable_index`; this only orders the pending mark term-first. pub(super) struct FsyncCoordinator { inflight: AtomicBool, - pending_max: AtomicU64, + /// Highest `(term, index)` awaiting fsync. Term-first: a newer term's mark + /// wins over an older term's higher index, so a stale pre-truncation submit + /// cannot swallow the valid post-truncation one. + pending_max: Mutex, pending_replies: Mutex>>>, - generation: AtomicU64, // Bumped on every reset; fences out stale in-flight fsync results. + + /// Bumped on truncation/reset. A round whose start predates the bump errs + /// its queued flush() replies instead of reporting a superseded result. + generation: AtomicU64, } impl FsyncCoordinator { pub(super) fn new() -> Self { Self { inflight: AtomicBool::new(false), - pending_max: AtomicU64::new(0), + pending_max: Mutex::new(LogId::default()), pending_replies: Mutex::new(Vec::new()), generation: AtomicU64::new(0), } @@ -39,14 +43,17 @@ impl FsyncCoordinator { pub(super) fn submit( self: &Arc, this: &Arc>, - max_index: u64, + mark: LogId, replies: Vec>>, ) { - if max_index > 0 { - self.pending_max.fetch_max(max_index, Ordering::AcqRel); + if mark.index > 0 { + let mut p = self.pending_max.lock(); + if (mark.term, mark.index) > (p.term, p.index) { + *p = mark; + } } if !replies.is_empty() { - self.pending_replies.lock().unwrap().extend(replies); + self.pending_replies.lock().extend(replies); } if self @@ -54,8 +61,10 @@ impl FsyncCoordinator { .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) .is_err() { + metrics::counter!("core.raft.fsync.coalesced_submit").increment(1); return; // Already running β€” it will pick up what we just recorded. } + metrics::counter!("core.raft.fsync.fresh_round").increment(1); metrics::gauge!("core.raft.fsync.inflight").set(1.0); @@ -73,8 +82,8 @@ impl FsyncCoordinator { loop { let gen_at_start = self.generation.load(Ordering::Acquire); - let max_index = self.pending_max.swap(0, Ordering::AcqRel); - let replies = std::mem::take(&mut *self.pending_replies.lock().unwrap()); + let mark = std::mem::take(&mut *self.pending_max.lock()); + let replies = std::mem::take(&mut *self.pending_replies.lock()); if this.is_poisoned() { for reply in replies { @@ -85,13 +94,12 @@ impl FsyncCoordinator { return; } - if max_index == 0 && replies.is_empty() { + if mark.index == 0 && replies.is_empty() { self.inflight.store(false, Ordering::Release); metrics::gauge!("core.raft.fsync.inflight").set(0.0); // Re-check: something may have slipped in between the swap // above and clearing `inflight`. If so, re-arm. - if (self.pending_max.load(Ordering::Acquire) > 0 - || !self.pending_replies.lock().unwrap().is_empty()) + if (self.pending_max.lock().index > 0 || !self.pending_replies.lock().is_empty()) && self .inflight .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -103,9 +111,9 @@ impl FsyncCoordinator { return; } - if max_index > 0 { + if mark.index > 0 { let batch_size = - max_index.saturating_sub(this.durable_index.load(Ordering::Acquire)); + mark.index.saturating_sub(this.durable_index.load(Ordering::Acquire)); metrics::histogram!("core.raft.fsync.batch_entries").record(batch_size as f64); } @@ -122,8 +130,7 @@ impl FsyncCoordinator { r }; - // Fence check: if a reset happened while this batch was in flight, - // its result is for data that no longer exists β€” discard. + // Skip replying if this round is already known stale. if self.generation.load(Ordering::Acquire) != gen_at_start { for reply in replies { let _ = reply.send(Err(crate::Error::Fatal( @@ -134,7 +141,7 @@ impl FsyncCoordinator { } match &result { - Ok(()) => this.advance_durable_and_notify(max_index), + Ok(()) => this.notify_fsync_completed(mark), Err(e) => { // One fsync failure = fatal, no threshold, no retry-and-hope. // Durability state is now unknown, this node @@ -142,7 +149,7 @@ impl FsyncCoordinator { this.mark_poisoned_and_notify(format!("fsync failed: {e:?}")); // mirrors advance_durable_and_notify's pattern error!( "WAL fsync failed at index {}: {:?} β€” node entering fatal state", - max_index, e + mark.index, e ); } } @@ -162,14 +169,18 @@ impl FsyncCoordinator { /// flush round β€” that queued data was submitted before reset and /// must not be silently adopted by the next round. pub(super) fn fence_reset(&self) { - self.generation.fetch_add(1, Ordering::AcqRel); - self.pending_max.store(0, Ordering::Release); - let stale = std::mem::take(&mut *self.pending_replies.lock().unwrap()); + *self.pending_max.lock() = LogId::default(); + let stale = std::mem::take(&mut *self.pending_replies.lock()); for reply in stale { let _ = reply.send(Err(Error::Fatal( "stale fsync generation, superseded by reset".into(), ))); } + self.bump_generation(); + } + + pub(super) fn bump_generation(&self) { + self.generation.fetch_add(1, Ordering::AcqRel); } } diff --git a/d-engine-core/src/storage/fsync_coordinator_test.rs b/d-engine-core/src/storage/fsync_coordinator_test.rs index 76cf0bf7..94787e4d 100644 --- a/d-engine-core/src/storage/fsync_coordinator_test.rs +++ b/d-engine-core/src/storage/fsync_coordinator_test.rs @@ -14,14 +14,18 @@ use super::*; use crate::FlushPolicy; +use crate::InternalEvent; use crate::MockLogStore; use crate::MockMetaStore; use crate::MockStorageEngine; use crate::MockTypeConfig; use crate::PersistenceConfig; -use crate::PersistenceStrategy; +use crate::RaftLog; // try_advance_durable_index is a RaftLog trait method use crate::Result; +use d_engine_proto::common::Entry; +use d_engine_proto::common::LogId; use std::sync::Arc; +use tokio::sync::mpsc; /// Build a `BufferedRaftLog` for direct `FsyncCoordinator` method calls β€” /// never `.start()`-ed, no IO thread, no channel plumbing. Only `log_store`/ @@ -31,11 +35,9 @@ fn minimal_raft_log(storage: MockStorageEngine) -> Arc::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 60_000, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, Arc::new(storage), @@ -59,12 +61,12 @@ fn test_new_initializes_empty_state() { "inflight must start false" ); assert_eq!( - coord.pending_max.load(Ordering::Acquire), + coord.pending_max.lock().index, 0, "pending_max must start at 0" ); assert!( - coord.pending_replies.lock().unwrap().is_empty(), + coord.pending_replies.lock().is_empty(), "pending_replies must start empty" ); assert_eq!( @@ -87,12 +89,12 @@ fn test_fence_reset_zeroes_pending_max() { let coord = FsyncCoordinator::new(); // Directly seed pending_max β€” no need to go through submit() (which would // spawn a real background task and race with fence_reset() below). - coord.pending_max.store(10, Ordering::Release); + *coord.pending_max.lock() = LogId { term: 1, index: 10 }; coord.fence_reset(); assert_eq!( - coord.pending_max.load(Ordering::Acquire), + coord.pending_max.lock().index, 0, "fence_reset() must zero pending_max" ); @@ -113,12 +115,12 @@ fn test_fence_reset_drains_pending_replies_with_err() { let (tx1, mut rx1) = oneshot::channel(); let (tx2, mut rx2) = oneshot::channel(); - coord.pending_replies.lock().unwrap().extend([tx1, tx2]); + coord.pending_replies.lock().extend([tx1, tx2]); coord.fence_reset(); assert!( - coord.pending_replies.lock().unwrap().is_empty(), + coord.pending_replies.lock().is_empty(), "pending_replies must be empty after fence_reset()" ); assert!( @@ -178,13 +180,9 @@ fn test_fence_reset_is_safe_with_nothing_pending() { 1, "generation must still increment even with nothing pending" ); - assert_eq!( - coord.pending_max.load(Ordering::Acquire), - 0, - "pending_max must stay 0" - ); + assert_eq!(coord.pending_max.lock().index, 0, "pending_max must stay 0"); assert!( - coord.pending_replies.lock().unwrap().is_empty(), + coord.pending_replies.lock().is_empty(), "pending_replies must stay empty" ); } @@ -207,11 +205,18 @@ fn test_submit_pending_max_uses_fetch_max_not_last_write() { // deterministic, no real concurrency needed to verify fetch_max order. coord.inflight.store(true, Ordering::Release); - coord.submit(&raft_log, 100, vec![]); - coord.submit(&raft_log, 50, vec![]); + coord.submit( + &raft_log, + LogId { + term: 1, + index: 100, + }, + vec![], + ); + coord.submit(&raft_log, LogId { term: 1, index: 50 }, vec![]); assert_eq!( - coord.pending_max.load(Ordering::Acquire), + coord.pending_max.lock().index, 100, "pending_max must stay at the high-water mark (100), not regress to \ a later, smaller submit() value (50)" @@ -234,7 +239,7 @@ async fn test_submit_first_call_sets_inflight_true() { let coord = Arc::new(FsyncCoordinator::new()); let raft_log = minimal_raft_log(storage); - coord.submit(&raft_log, 1, vec![]); + coord.submit(&raft_log, LogId { term: 1, index: 1 }, vec![]); assert!( coord.inflight.load(Ordering::Acquire), @@ -269,15 +274,15 @@ fn test_submit_second_call_does_not_spawn_second_task_while_inflight() { coord.inflight.store(true, Ordering::Release); let (tx, mut rx) = oneshot::channel::>(); - coord.submit(&raft_log, 1, vec![tx]); + coord.submit(&raft_log, LogId { term: 1, index: 1 }, vec![tx]); assert_eq!( - coord.pending_max.load(Ordering::Acquire), + coord.pending_max.lock().index, 1, "submit() must still record pending_max even though it lost the CAS" ); assert_eq!( - coord.pending_replies.lock().unwrap().len(), + coord.pending_replies.lock().len(), 1, "submit() must still queue the reply even though it lost the CAS" ); @@ -340,13 +345,45 @@ fn test_run_until_caught_up_advances_durable_index_on_success() { "run_until_caught_up_advances_durable_index_on_success".into(), ); let coord = FsyncCoordinator::new(); - let raft_log = minimal_raft_log(storage); + // run_until_caught_up() now only *reports* completion (notify_fsync_completed) β€” + // durable_index only moves once something (normally raft.rs) drains that + // report and calls try_advance_durable_index, which content-validates + // against entry_term(index). Needs a real backing entry (not just + // set_memory_max_index_for_test's raw atomic poke) and a registered + // log_flush_tx to receive the report β€” minimal_raft_log() has neither, so + // this test builds its own instead of using it. + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + raft_log.entries.write().insert( + 5, + Entry { + index: 5, + term: 1, + payload: None, + }, + ); + raft_log.set_memory_max_index_for_test(5); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(5, Ordering::Release); + *coord.pending_max.lock() = LogId { term: 1, index: 5 }; coord.run_until_caught_up(&raft_log); + // Stand in for raft.rs's InternalEvent::FsyncCompleted handler. + while let Ok(InternalEvent::FsyncCompleted(mark)) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(mark); + } + assert_eq!( raft_log.durable_index.load(Ordering::Acquire), 5, @@ -370,7 +407,7 @@ fn test_run_until_caught_up_does_not_advance_durable_index_on_flush_failure() { let pre = raft_log.durable_index.load(Ordering::Acquire); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(5, Ordering::Release); + *coord.pending_max.lock() = LogId { term: 1, index: 5 }; coord.run_until_caught_up(&raft_log); @@ -397,8 +434,8 @@ fn test_run_until_caught_up_sends_err_to_replies_on_flush_failure() { let (tx, mut rx) = oneshot::channel::>(); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(5, Ordering::Release); - coord.pending_replies.lock().unwrap().push(tx); + *coord.pending_max.lock() = LogId { term: 1, index: 5 }; + coord.pending_replies.lock().push(tx); coord.run_until_caught_up(&raft_log); @@ -443,8 +480,8 @@ fn test_run_until_caught_up_discards_stale_generation_result_without_advancing() let (tx, mut rx) = oneshot::channel::>(); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(5, Ordering::Release); - coord.pending_replies.lock().unwrap().push(tx); + *coord.pending_max.lock() = LogId { term: 1, index: 5 }; + coord.pending_replies.lock().push(tx); coord.run_until_caught_up(&raft_log); @@ -461,6 +498,80 @@ fn test_run_until_caught_up_discards_stale_generation_result_without_advancing() ); } +/// The other half of the fence: when `generation` at completion still +/// matches `generation` at the round's start (nothing fenced it while the +/// physical flush was running), the result must be accepted normally β€” +/// `durable_index` advances and queued replies resolve to `Ok`. +/// +/// Bumps `generation` twice before the round starts (so this isn't just +/// "stays at the default 0"), proving it's the *match*, not the specific +/// value, that matters. +/// +/// Expected: +/// - `durable_index()` advances to the round's `max_index`. +/// - The queued reply resolves to `Ok(())`. +#[test] +fn test_run_until_caught_up_accepts_result_when_generation_unchanged() { + let (storage, _flush_call_count) = MockStorageEngine::not_durable( + "run_until_caught_up_accepts_result_when_generation_unchanged".into(), + ); + let coord = FsyncCoordinator::new(); + // See test_run_until_caught_up_advances_durable_index_on_success for why + // this doesn't use minimal_raft_log(): needs a real backing entry for + // try_advance_durable_index's content check, and a registered + // log_flush_tx to receive the FsyncCompleted report. + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + raft_log.entries.write().insert( + 5, + Entry { + index: 5, + term: 1, + payload: None, + }, + ); + raft_log.set_memory_max_index_for_test(5); + + // Two unrelated fences happened earlier β€” generation is 2, not 0 β€” before + // this round is even recorded as in flight. + coord.fence_reset(); + coord.fence_reset(); + assert_eq!(coord.generation.load(Ordering::Acquire), 2); + + let (tx, mut rx) = oneshot::channel::>(); + coord.inflight.store(true, Ordering::Release); + *coord.pending_max.lock() = LogId { term: 1, index: 5 }; + coord.pending_replies.lock().push(tx); + + // Nothing fences this round while it runs β€” generation stays at 2. + coord.run_until_caught_up(&raft_log); + + // Stand in for raft.rs's InternalEvent::FsyncCompleted handler. + while let Ok(InternalEvent::FsyncCompleted(mark)) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(mark); + } + + assert_eq!( + raft_log.durable_index.load(Ordering::Acquire), + 5, + "a matching generation must let the result advance durable_index normally" + ); + assert!( + rx.try_recv().expect("reply must have been answered").is_ok(), + "a matching generation must resolve queued replies as Ok, not Err" + ); +} + /// Multiple `submit()` calls made while a round is in flight are coalesced /// into a single subsequent physical `flush()` call by the same task β€” not /// one physical flush per `submit()` call. @@ -476,14 +587,16 @@ fn test_run_until_caught_up_coalesces_queued_submits_into_one_flush() { ); let coord = FsyncCoordinator::new(); let raft_log = minimal_raft_log(storage); + // advance_durable_and_notify() clamps against max_index β€” matches pending_max below. + raft_log.set_memory_max_index_for_test(10); // Simulate two submit() calls that both lost the CAS while a round was // in flight β€” both just accumulated into the same pending state. let (tx1, mut rx1) = oneshot::channel::>(); let (tx2, mut rx2) = oneshot::channel::>(); coord.inflight.store(true, Ordering::Release); - coord.pending_max.store(10, Ordering::Release); - coord.pending_replies.lock().unwrap().extend([tx1, tx2]); + *coord.pending_max.lock() = LogId { term: 1, index: 10 }; + coord.pending_replies.lock().extend([tx1, tx2]); coord.run_until_caught_up(&raft_log); @@ -501,3 +614,82 @@ fn test_run_until_caught_up_coalesces_queued_submits_into_one_flush() { "both queued replies must resolve to Ok" ); } + +/// Two `submit()` calls race into the same pending round: a stale +/// pre-truncation persist carrying `(term 1, index 10)` (its captured entries +/// were all term 1, and index 10 has since been truncated away) and the valid +/// post-truncation `ReplaceRange` mark `(term 2, index 2)`. +/// +/// Term-first ordering must keep `(term 2, index 2)` β€” a newer term wins over +/// an older term's higher index. Before this fix `pending_max` was a bare +/// `fetch_max` on the index: `10` swallowed `2`, the round reported the stale +/// `10`, content-validation rejected it, and `durable_index` never reached 2. +/// +/// Deterministic, IO-thread-free repro of +/// `test_stale_persist_after_truncation_does_not_advance_durable_index`. +#[test] +fn test_submit_term_first_keeps_valid_mark_over_stale_higher_index() { + let (storage, _flush_call_count) = MockStorageEngine::not_durable( + "submit_term_first_keeps_valid_mark_over_stale_higher_index".into(), + ); + let coord = Arc::new(FsyncCoordinator::new()); + let (raft_log, receiver) = BufferedRaftLog::::new( + 1, + PersistenceConfig { + flush_policy: FlushPolicy::Batch { + idle_flush_interval_ms: 60_000, + }, + shutdown_timeout_ms: 5000, + }, + Arc::new(storage), + ); + let (log_flush_tx, mut log_flush_rx) = mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); + + // Post-truncation log: entry 1 (term 1) survived; entry 2 is the new + // leader's replacement (term 2). Index 10 is gone. + { + let e = raft_log.entries.write(); + e.insert( + 1, + Entry { + index: 1, + term: 1, + payload: None, + }, + ); + e.insert( + 2, + Entry { + index: 2, + term: 2, + payload: None, + }, + ); + } + raft_log.set_memory_max_index_for_test(2); + + // Pretend a round is in flight so submit() only records state. + coord.inflight.store(true, Ordering::Release); + coord.submit(&raft_log, LogId { term: 1, index: 10 }, vec![]); // stale + coord.submit(&raft_log, LogId { term: 2, index: 2 }, vec![]); // valid + + assert_eq!( + *coord.pending_max.lock(), + LogId { term: 2, index: 2 }, + "term-first: the newer-term mark must win over the stale higher index" + ); + + coord.run_until_caught_up(&raft_log); + + while let Ok(InternalEvent::FsyncCompleted(mark)) = log_flush_rx.try_recv() { + raft_log.try_advance_durable_index(mark); + } + + assert_eq!( + raft_log.durable_index.load(Ordering::Acquire), + 2, + "durable_index must reach the valid post-truncation tail (2), not stall \ + because a stale batch top swallowed it" + ); +} diff --git a/d-engine-core/src/storage/raft_log.rs b/d-engine-core/src/storage/raft_log.rs index b21778bc..05e76866 100644 --- a/d-engine-core/src/storage/raft_log.rs +++ b/d-engine-core/src/storage/raft_log.rs @@ -77,6 +77,17 @@ pub trait RaftLog: Send + Sync + 'static { /// - DiskFirst: equals `last_entry_id()` (every append blocks until durable). fn durable_index(&self) -> u64; + /// Content-validated durable-watermark advance. `index`/`term` describe + /// what a completed fsync claims is now safe β€” rejected (`None`) if + /// `entry_term(index) != Some(term)`, meaning the log content at that + /// index has changed (truncated + replaced) since fsync started on it. + /// `Some(new_value)` only when it actually advanced β€” callers use this + /// to decide whether to fire `handle_log_flushed`. + fn try_advance_durable_index( + &self, + mark: LogId, + ) -> Option; + /// Returns the LogId (term + index) of the last entry. /// /// # Returns @@ -209,14 +220,14 @@ pub trait RaftLog: Send + Sync + 'static { /// - Persist entries to durable storage BEFORE updating in-memory state /// - Call fsync/flush before returning Ok(()) /// - Ensures entries survive crashes immediately - /// - Example: BufferedRaftLog with PersistenceStrategy::DiskFirst + /// - Example: a store that fsyncs before returning /// /// 2. **Memory-First (Performance-optimized, Acceptable for Followers)**: /// - Update in-memory state first /// - Enqueue entries for asynchronous durability /// - MUST guarantee eventual durability via background flush /// - MUST call flush() before acknowledging commits - /// - Example: BufferedRaftLog with PersistenceStrategy::MemFirst + /// - Example: a store that fsyncs asynchronously /// - WARNING: Leader MUST wait_durable() before responding to AppendEntries RPCs /// /// # Safety Invariants @@ -228,10 +239,10 @@ pub trait RaftLog: Send + Sync + 'static { /// - MUST update term indexes (first/last_index_for_term) atomically /// /// # Raft Protocol Integration - /// - Leaders using MemFirst MUST call wait_durable(index) before: + /// - Leaders using async fsync MUST call wait_durable(index) before: /// * Responding success to AppendEntries RPC /// * Advancing commit index - /// - Followers can use MemFirst safely because leader durability guarantees safety + /// - Followers can use async fsync safely because leader durability guarantees safety /// /// # Failure Semantics /// - On error, implementer MAY roll back partial writes @@ -252,7 +263,7 @@ pub trait RaftLog: Send + Sync + 'static { /// /// # Usage Pattern /// ```rust,ignore - /// // Leader with MemFirst strategy + /// // Leader with async fsync /// raft_log.append_entries(new_entries).await?; /// raft_log.wait_durable(max_index).await?; // MUST wait before RPC response /// respond_to_client(Ok(())); @@ -261,7 +272,7 @@ pub trait RaftLog: Send + Sync + 'static { /// # Safety Invariants /// - MUST NOT return until flush() for this index completes successfully /// - If implementation doesn't support async durability, return Ok(()) immediately - /// - Critical for MemFirst strategy correctness + /// - Critical for async-fsync correctness async fn wait_durable( &self, index: u64, diff --git a/d-engine-core/src/storage/storage_engine.rs b/d-engine-core/src/storage/storage_engine.rs index 29a61830..5ece2a1f 100644 --- a/d-engine-core/src/storage/storage_engine.rs +++ b/d-engine-core/src/storage/storage_engine.rs @@ -79,16 +79,19 @@ pub trait LogStore: Send + Sync + 'static { /// /// Default implementation calls `truncate` then `persist_entries` sequentially /// (non-atomic). Override with a single WriteBatch for true crash atomicity. + /// Returns the highest log index on disk after the operation: the last of + /// `new_entries`, or `from_index - 1` when `new_entries` is empty. async fn replace_range( &self, from_index: u64, new_entries: Vec, - ) -> Result<(), Error> { + ) -> Result { + let new_last = new_entries.last().map(|e| e.index).unwrap_or(from_index.saturating_sub(1)); self.truncate(from_index).await?; if !new_entries.is_empty() { self.persist_entries(new_entries).await?; } - Ok(()) + Ok(new_last) } /// Whether a single `persist_entries` call is crash-safe without an explicit `flush()`. diff --git a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs index 9bcfaa07..bad5ac2f 100644 --- a/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs +++ b/d-engine-core/src/test_utils/buffered_raft_log_test_helpers.rs @@ -12,23 +12,21 @@ use bytes::Bytes; use d_engine_proto::common::{Entry, EntryPayload}; use crate::{ - BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, - PersistenceStrategy, RaftLog, + BufferedRaftLog, FlushPolicy, MockStorageEngine, MockTypeConfig, PersistenceConfig, RaftLog, }; /// Test context for BufferedRaftLog tests pub struct BufferedRaftLogTestContext { pub raft_log: Arc>, pub storage: Arc, - pub strategy: PersistenceStrategy, pub flush_policy: FlushPolicy, pub instance_id: String, + log_flush_rx: tokio::sync::mpsc::UnboundedReceiver, } impl BufferedRaftLogTestContext { /// Create a new test context with specified strategy and flush policy pub fn new( - strategy: PersistenceStrategy, flush_policy: FlushPolicy, instance_id: &str, ) -> Self { @@ -37,14 +35,13 @@ impl BufferedRaftLogTestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); // Small delay to ensure processor is ready std::thread::sleep(std::time::Duration::from_millis(10)); @@ -52,12 +49,23 @@ impl BufferedRaftLogTestContext { Self { raft_log, storage, - strategy, flush_policy, instance_id: instance_id.to_string(), + log_flush_rx, } } + /// Stands in for `raft.rs`'s `InternalEvent::FsyncCompleted` handler, + /// which isn't running in these `BufferedRaftLog`-only unit tests. Call + /// after any operation that should make `durable_index` advance + /// (`append_entries`, `flush`, truncation + resync, ...) and before + /// asserting on `durable_index()` β€” see + /// `drain_and_apply_fsync_completions` for why this is necessary since + /// #446/#447. + pub fn drain_fsync_completions(&mut self) { + drain_and_apply_fsync_completions(&self.raft_log, &mut self.log_flush_rx); + } + /// Helper to append a batch of entries with specified range and term pub async fn append_entries( &self, @@ -92,22 +100,21 @@ impl BufferedRaftLogTestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(std::time::Duration::from_millis(10)); let ctx = Self { raft_log, storage, - strategy: PersistenceStrategy::MemFirst, flush_policy, instance_id: instance_id.to_string(), + log_flush_rx, }; (ctx, flush_count) } @@ -120,14 +127,13 @@ impl BufferedRaftLogTestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); // Small delay to ensure processor is ready std::thread::sleep(std::time::Duration::from_millis(10)); @@ -135,9 +141,9 @@ impl BufferedRaftLogTestContext { Self { raft_log, storage, - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), instance_id: self.instance_id.clone(), + log_flush_rx, } } } @@ -237,3 +243,24 @@ pub async fn simulate_delete_command( raft_log.insert_batch(entries).await.unwrap(); raft_log.flush().await.unwrap(); } + +/// Stands in for `raft.rs`'s `InternalEvent::FsyncCompleted` handler, which +/// `BufferedRaftLog`-only unit tests don't have running. Since #446/#447, +/// `durable_index` only advances when something calls +/// `try_advance_durable_index(index, term)` in response to that event β€” +/// `FsyncCoordinator`/`IOTask::ReplaceRange`'s `notify_fsync_completed` only +/// *sends* the event, it never writes `durable_index` itself. A test that +/// registers a `log_flush_tx` and wants to see `durable_index()` actually +/// advance must drain that channel through this helper β€” otherwise the +/// event sits unread and `durable_index()` never moves, no matter how long +/// you sleep. +pub fn drain_and_apply_fsync_completions( + raft_log: &Arc>, + log_flush_rx: &mut tokio::sync::mpsc::UnboundedReceiver, +) { + while let Ok(event) = log_flush_rx.try_recv() { + if let crate::InternalEvent::FsyncCompleted(mark) = event { + raft_log.try_advance_durable_index(mark); + } + } +} diff --git a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs index b21f91da..83a3d4f9 100644 --- a/d-engine-core/src/test_utils/mock/mock_storage_engine.rs +++ b/d-engine-core/src/test_utils/mock/mock_storage_engine.rs @@ -263,7 +263,7 @@ impl MockStorageEngine { } else { data.insert(last_key, new_last_index.to_be_bytes().to_vec()); } - Ok(()) + Ok(new_last_index) }); } @@ -322,7 +322,7 @@ impl MockStorageEngine { /// After a failed fsync, `batch_processor` logs the error and does NOT zero /// `pending_max` (the success branch `else { pending_max = 0 }` is not taken). /// This is the deterministic pre-condition needed to exercise the bug where - /// `handle_non_write_cmd(IOTask::Reset)` forgets to zero `pending_max`. + /// `run_storage_tasks(IOTask::Reset)` forgets to zero `pending_max`. pub fn not_durable_first_flush_fails(id: String) -> Self { let mut mock_log_store = MockLogStore::new(); let mut mock_meta_store = MockMetaStore::new(); @@ -393,7 +393,7 @@ impl MockStorageEngine { /// Create a MockStorageEngine where `replace_range()` always fails, /// simulating a fatal storage error during conflict-resolution - /// (truncate + write). `handle_non_write_cmd`'s `IOTask::ReplaceRange` + /// (truncate + write). `run_storage_tasks`'s `IOTask::ReplaceRange` /// arm treats this as unrecoverable β€” disk state is now uncertain. pub fn not_durable_replace_range_fails(id: String) -> Self { let mut mock_log_store = MockLogStore::new(); @@ -635,6 +635,57 @@ impl MockStorageEngine { (engine, tx) } + /// Create a MockStorageEngine where the first `persist_entries()` call blocks + /// until the returned sender fires. `flush()`/`is_write_durable()` are left at + /// their always-succeeds default (`configure_durable`) β€” this gate is only + /// about the write-to-storage-engine step, not fsync. + /// + /// Use this to freeze the IO thread mid-persist so a concurrent truncation + /// can be driven deterministically β€” see + /// `durable_index_truncation_clamp_test.rs`. + pub fn not_durable_gated_persist(id: String) -> (Self, std::sync::mpsc::Sender<()>) { + let (tx, rx) = std::sync::mpsc::channel::<()>(); + let rx = Mutex::new(Some(rx)); + + let mut mock_log_store = MockLogStore::new(); + let mut mock_meta_store = MockMetaStore::new(); + + Self::configure_mocks(&mut mock_log_store, &mut mock_meta_store, &id); + // persist_entries is gated below instead of via configure_persist_entries_success. + Self::configure_replace_range_success(&mut mock_log_store, &id); + Self::configure_purge_success(&mut mock_log_store); + Self::configure_reset_success(&mut mock_log_store, &id); + Self::configure_save_hard_state_success(&mut mock_meta_store, &id); + Self::configure_durable(&mut mock_log_store); + + let instance_id_ref = id.clone(); + mock_log_store.expect_persist_entries().returning(move |entries| { + // Only the first call blocks β€” take() leaves None for subsequent calls. + if let Some(gate) = rx.lock().unwrap().take() { + let _ = gate.recv(); // blocks until the test sends () + } + let mut data = MOCK_STORAGE_DATA.lock().unwrap(); + for entry in &entries { + let key = format!("{instance_id_ref}_entry_{}", entry.index); + let value = bincode::serialize(entry).unwrap(); + data.insert(key, value); + } + if let Some(last_entry) = entries.last() { + let key = format!("{instance_id_ref}_last_index"); + data.insert(key, last_entry.index.to_be_bytes().to_vec()); + } + Ok(()) + }); + + let engine = Self { + log_store: Arc::new(mock_log_store), + meta_store: Arc::new(mock_meta_store), + instance_id: id, + }; + + (engine, tx) + } + /// Configure `is_write_durable=true` and no-op flush (durable mock). fn configure_durable(log_store: &mut MockLogStore) { log_store.expect_is_write_durable().returning(|| true); diff --git a/d-engine-core/src/utils/scoped_timer.rs b/d-engine-core/src/utils/scoped_timer.rs index e4c7d9a5..dbfa2f5a 100644 --- a/d-engine-core/src/utils/scoped_timer.rs +++ b/d-engine-core/src/utils/scoped_timer.rs @@ -19,5 +19,7 @@ impl Drop for ScopedTimer { fn drop(&mut self) { let elapsed = self.start.elapsed(); trace!(target: "timing", "[TIMING] {} took {} ms", self.name, elapsed.as_millis()); + metrics::histogram!("core.timing.scoped_duration_ms", "phase" => self.name) + .record(elapsed.as_secs_f64() * 1_000.0); } } diff --git a/d-engine-core/src/watch/mod.rs b/d-engine-core/src/watch/mod.rs index 0998f2c0..32fcc4ba 100644 --- a/d-engine-core/src/watch/mod.rs +++ b/d-engine-core/src/watch/mod.rs @@ -87,6 +87,7 @@ //! watcher_buffer_size: 256, //! enable_metrics: true, //! max_watcher_count: 5000, +//! heartbeat_interval_ms: 30_000, //! }; //! ``` //! diff --git a/d-engine-proto/go/go.mod b/d-engine-proto/go/go.mod index 68f93120..b5dd4aa2 100644 --- a/d-engine-proto/go/go.mod +++ b/d-engine-proto/go/go.mod @@ -3,13 +3,13 @@ module github.com/deventlab/d-engine/proto go 1.25.0 require ( - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.11 ) require ( - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect ) diff --git a/d-engine-proto/go/go.sum b/d-engine-proto/go/go.sum index adb3ad1c..2d3ad8d1 100644 --- a/d-engine-proto/go/go.sum +++ b/d-engine-proto/go/go.sum @@ -12,27 +12,27 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/d-engine-server/src/api/embedded_test/embedded_env_test.rs b/d-engine-server/src/api/embedded_test/embedded_env_test.rs index 67173209..8c43d96e 100644 --- a/d-engine-server/src/api/embedded_test/embedded_env_test.rs +++ b/d-engine-server/src/api/embedded_test/embedded_env_test.rs @@ -31,15 +31,29 @@ mod start_data_dir_tests { } /// Opening an existing data directory is idempotent (data is preserved). + /// + /// Uses `start_with` + a timeout-only config so the client's write deadline + /// is not the 50ms `general_raft_timeout_duration_in_ms` default: since + /// #446 a `put` ack waits for a physical fdatasync, which under a loaded + /// test suite (many parallel RocksDB instances) can exceed 50ms. The + /// config's `data_dir` is still ignored β€” the explicit arg wins β€” so this + /// keeps testing exactly the reopen/data-preservation path. #[tokio::test] #[serial] async fn test_start_existing_directory_is_idempotent() { let temp_dir = tempfile::tempdir().expect("tempdir"); let data_dir = temp_dir.path().join("db"); + let config_path = temp_dir.path().join("d-engine.toml"); + std::fs::write( + &config_path, + "[raft]\ngeneral_raft_timeout_duration_in_ms = 3000\n", + ) + .expect("write config"); // First start: write a key { - let engine = EmbeddedEngine::start(&data_dir).await.expect("first start"); + let engine = + EmbeddedEngine::start_with(&data_dir, &config_path).await.expect("first start"); engine.wait_ready(std::time::Duration::from_secs(5)).await.expect("ready"); engine.client().put(b"k".to_vec(), b"v".to_vec()).await.expect("put"); tokio::time::sleep(std::time::Duration::from_millis(50)).await; @@ -48,7 +62,8 @@ mod start_data_dir_tests { // Second start: data must still be there { - let engine = EmbeddedEngine::start(&data_dir).await.expect("second start"); + let engine = + EmbeddedEngine::start_with(&data_dir, &config_path).await.expect("second start"); engine.wait_ready(std::time::Duration::from_secs(5)).await.expect("ready"); let val = engine.client().get_linearizable(b"k".to_vec()).await.expect("get"); assert_eq!(val.as_deref(), Some(b"v".as_ref()), "data must persist"); diff --git a/d-engine-server/src/network/grpc/grpc_raft_service.rs b/d-engine-server/src/network/grpc/grpc_raft_service.rs index 15940d48..5b66126c 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service.rs @@ -6,7 +6,6 @@ use crate::Node; use crate::proto_convert; use d_engine_core::InboundEvent; use d_engine_core::MaybeCloneOneshot; -use d_engine_core::MaybeCloneOneshotReceiver; use d_engine_core::RaftOneshot; use d_engine_core::TypeConfig; #[cfg(feature = "watch")] @@ -136,12 +135,24 @@ where Pin> + Send>>; /// Processes a persistent bidirectional AppendEntries stream from the cluster leader. + /// #446: responses are forwarded as soon as each one is ready, not in strict arrival + /// order β€” leader-side match_index/next_index updates are already designed to + /// tolerate out-of-order pipeline responses (`leader_state.rs`, "only advance, + /// never retreat"), so nothing downstream needs strict ordering. Strict FIFO would + /// let one response still waiting on this node's own durable_index (RPO=0) block + /// every later, already-ready response on the same connection β€” including + /// unrelated ones like heartbeats. /// - /// Decouples request ingestion from response emission: - /// - recv task: reads batches from the stream, dispatches each as a `InboundEvent::AppendEntries` - /// (non-blocking between batches) - /// - forwarder task: drains ordered response handles sequentially; ordering is guaranteed - /// by the Raft single-threaded event loop + /// Single task, bounded concurrency: reads a new request only while fewer than + /// `max_pending_append_responses` requests are still in flight, so a stalled fsync + /// bounds memory/task growth instead of growing without limit. + /// + /// Known tradeoff, not an oversight: this is a single task, so a slow/stuck + /// network write (`out_tx.send().await` blocking because the peer isn't reading) + /// also delays reading new requests AND processing the shutdown signal, until the + /// write unblocks or the connection dies. Accepted deliberately β€” if the peer + /// isn't reading responses, there's no useful work to do by reading more requests + /// either; this is legitimate backpressure, not a bug. async fn stream_append_entries( &self, request: tonic::Request>, @@ -157,30 +168,31 @@ where let mut in_stream = request.into_inner(); let event_tx = self.event_tx.clone(); - let ordered_channel_capacity = self.node_config.raft.ordered_channel_capacity; + let max_pending = self.node_config.raft.max_pending_append_responses; let mut shutdown = self.shutdown_signal.clone(); + let node_id = self.node_id; - // Output: ordered ACKs sent back to the leader over the bidi stream - let (out_tx, out_rx) = mpsc::channel::>(128); - - // Ordered queue: response oneshot receivers in FIFO arrival order - let (ordered_tx, mut ordered_rx) = mpsc::channel::< - MaybeCloneOneshotReceiver>, - >(ordered_channel_capacity); + // Output: ACKs sent back to the leader over the bidi stream, in completion order. + // Capacity matches max_pending β€” completed responses can never outnumber + // in-flight requests, so there's no separate number to reason about here. + let (out_tx, out_rx) = mpsc::channel::>(max_pending); - // Recv task: read batches, dispatch to Raft loop without waiting for each ACK. - // Selects on shutdown signal so the task exits immediately on node stop, rather - // than waiting for the next message from the leader. This unblocks serve_with_shutdown - // and allows Arc (and Arc) to be released promptly after stop(). + // Single task: read requests, dispatch to the Raft loop, and forward whichever + // response becomes ready first β€” bounded by `max_pending` in-flight responses. tokio::spawn(async move { use futures::StreamExt; + use futures::stream::FuturesUnordered; + + let mut pending = FuturesUnordered::new(); + let mut inbound_open = true; + loop { tokio::select! { biased; _ = shutdown.changed() => { break; } - result = in_stream.next() => { + result = in_stream.next(), if inbound_open && pending.len() < max_pending => { match result { Some(Ok(req)) => { let (resp_tx, resp_rx) = MaybeCloneOneshot::new(); @@ -188,32 +200,51 @@ where debug!("[stream_append_entries|recv] event_tx closed"); break; } - if ordered_tx.send(resp_rx).await.is_err() { - break; - } + pending.push(async move { + match resp_rx.await { + Ok(Ok(resp)) => Ok(resp), + Ok(Err(status)) => Err(status), + Err(_) => Err(Status::internal("Response channel closed")), + } + }); } Some(Err(e)) => { // Debug: expected when the peer goes away (crash/restart/shutdown), self-heals. debug!("[stream_append_entries|recv] stream error: {:?}", e); - break; + inbound_open = false; } - None => break, + None => inbound_open = false, + } + } + Some(result) = pending.next(), if !pending.is_empty() => { + // Observability only β€” behavior doesn't change, the send still + // runs to completion normally. If the peer isn't reading (network + // stall, dead connection with the TCP timeout not yet fired), this + // surfaces it instead of silently blocking with zero signal. + let mut send_fut = std::pin::pin!(out_tx.send(result)); + let mut stuck_logged = false; + let closed = loop { + tokio::select! { + res = &mut send_fut => break res.is_err(), + _ = tokio::time::sleep(Duration::from_secs(5)), if !stuck_logged => { + stuck_logged = true; + error!( + node_id, + "stream_append_entries forwarder stuck sending a \ + response for >5s β€” peer may not be reading \ + (network stall or dead connection)" + ); + metrics::counter!( + "server.grpc.stream_append_entries.forwarder_stuck" + ) + .increment(1); + } + } + }; + if closed { + break; } } - } - } - }); - - // Forwarder task: drain ordered queue sequentially (FIFO guaranteed by Raft loop) - tokio::spawn(async move { - while let Some(resp_rx) = ordered_rx.recv().await { - let result = match resp_rx.await { - Ok(Ok(resp)) => Ok(resp), - Ok(Err(status)) => Err(status), - Err(_) => Err(Status::internal("Response channel closed")), - }; - if out_tx.send(result).await.is_err() { - break; } } }); diff --git a/d-engine-server/src/network/grpc/grpc_raft_service_test.rs b/d-engine-server/src/network/grpc/grpc_raft_service_test.rs index 97d5583f..32f8de17 100644 --- a/d-engine-server/src/network/grpc/grpc_raft_service_test.rs +++ b/d-engine-server/src/network/grpc/grpc_raft_service_test.rs @@ -3,12 +3,15 @@ use std::time::Duration; use crate::ApplyResult; use d_engine_core::AppendResponseWithUpdates; use d_engine_core::InternalEvent; +use d_engine_core::MaybeCloneOneshot; +use d_engine_core::MaybeCloneOneshotReceiver; use d_engine_core::MockElectionCore; use d_engine_core::MockMembership; use d_engine_core::MockRaftLog; use d_engine_core::MockReplicationCore; use d_engine_core::MockTypeConfig; use d_engine_core::RaftNodeConfig; +use d_engine_core::RaftOneshot; use d_engine_core::convert::safe_kv_bytes; use d_engine_proto::client::ClientReadRequest; use d_engine_proto::client::ClientWriteRequest; @@ -605,3 +608,175 @@ async fn test_handle_client_scan_not_leader_carries_leader_hint_in_metadata() { Some("http://127.0.0.1:9082") ); } + +/// Historical record, not a regression guard: this is the strict-FIFO forwarder +/// pattern `stream_append_entries` used *before* #446 (one withheld response blocked +/// every later response on the same connection). It reconstructs the old primitives +/// rather than calling production code, because that code no longer exists β€” +/// `stream_append_entries` was rewritten to a bounded, order-tolerant forwarder (see +/// `test_stream_append_entries_does_not_block_ready_response_behind_pending_one` for +/// the real, current behavior). Kept only so a future reader can see what the old +/// failure mode looked like; do not treat this as coverage of current code. +#[tokio::test] +async fn test_ordered_forwarder_head_of_line_blocking() { + let (out_tx, mut out_rx) = mpsc::channel::>(128); + let (ordered_tx, mut ordered_rx) = mpsc::channel::< + MaybeCloneOneshotReceiver>, + >(128); + + // Mirrors grpc_raft_service.rs's forwarder loop. + tokio::spawn(async move { + while let Some(resp_rx) = ordered_rx.recv().await { + let result = match resp_rx.await { + Ok(Ok(resp)) => Ok(resp), + Ok(Err(status)) => Err(status), + Err(_) => Err(tonic::Status::internal("Response channel closed")), + }; + if out_tx.send(result).await.is_err() { + break; + } + } + }); + + // First item: never resolved β€” stands in for a response withheld pending durable_index. + let (_stuck_tx, stuck_rx) = MaybeCloneOneshot::new(); + ordered_tx.send(stuck_rx).await.unwrap(); + + // Second item: already resolved β€” stands in for an unrelated, ready-to-send response + // (e.g. a heartbeat) that arrived right after. + let (ready_tx, ready_rx) = MaybeCloneOneshot::new(); + ordered_tx.send(ready_rx).await.unwrap(); + ready_tx.send(Ok(AppendEntriesResponse::success(1, 1, None))).unwrap(); + + // The second, already-ready response must not be observable yet β€” it's stuck + // behind the first, unresolved one in strict FIFO order. + let blocked = time::timeout(Duration::from_millis(50), out_rx.recv()).await; + assert!( + blocked.is_err(), + "an already-ready response was blocked behind an earlier unresolved one β€” \ + confirms the forwarder is strict FIFO" + ); +} + +/// #446: `stream_append_entries` must not let a response still withheld (durable_index +/// hasn't caught up to what it claims) block a later, unrelated response that's already +/// answerable. Drives the real production method end-to-end β€” not a reconstruction β€” +/// via a synthetic 2-item input stream. +/// +/// request 1 claims index 10 while `durable_index()` is fixed at 5 β€” withheld, +/// queued in `pending_append_acks`, never released in this test. +/// request 2 claims index 3, which is `<= durable_index` β€” answerable immediately. +/// +/// If the forwarder is still strict FIFO, the first item out of the response stream +/// would have to be request 1's (never arrives) β€” this test would time out. If it's +/// the new bounded/order-tolerant forwarder, request 2's response (identifiable by its +/// distinct `last_match.term` marker) comes out first. +#[tokio::test] +async fn test_stream_append_entries_does_not_block_ready_response_behind_pending_one() { + tokio::time::pause(); + let settings = RaftNodeConfig::new().expect("Should succeed to init RaftNodeConfig."); + let mut settings = settings.validate().expect("Validate RaftNodeConfig successfully"); + settings.raft.general_raft_timeout_duration_in_ms = 200; + settings.raft.batching.max_batch_size = 1; + + let mut membership = MockMembership::::new(); + membership.expect_voters().returning(Vec::new); + membership.expect_members().returning(Vec::new); + membership.expect_replication_peers().returning(Vec::new); + membership.expect_get_peers_id_with_condition().returning(|_| vec![]); + + let mut raft_log = MockRaftLog::new(); + raft_log.expect_last_entry_id().returning(|| 0); + raft_log.expect_flush().returning(|| Ok(())); + raft_log.expect_load_hard_state().returning(|| Ok(None)); + raft_log.expect_save_hard_state().returning(|_| Ok(())); + raft_log.expect_last_log_id().returning(|| None); + // Fixed durable frontier: only request 2's claimed index (3) clears it. + raft_log.expect_durable_index().returning(|| 5); + + let call_count = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let call_count_clone = call_count.clone(); + let mut replication_handler = MockReplicationCore::::new(); + replication_handler + .expect_check_append_entries_request_is_legal() + .returning(|my_term, _, _| AppendEntriesResponse::success(1, my_term, None)); + replication_handler.expect_handle_append_entries().returning(move |_, _, _| { + let is_first = call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0; + let (claimed_index, term_marker) = if is_first { (10, 111) } else { (3, 222) }; + Ok(AppendResponseWithUpdates { + response: AppendEntriesResponse::success( + 1, + term_marker, + Some(LogId { + term: term_marker, + index: claimed_index, + }), + ), + commit_index_update: None, + }) + }); + + let (_graceful_tx, graceful_rx) = watch::channel(()); + let builder = MockBuilder::new(graceful_rx); + let node = builder + .with_raft_log(raft_log) + .with_membership(membership) + .with_replication_handler(replication_handler) + .with_node_config(settings) + .build_node(); + node.set_rpc_ready(true); + + let raft_lock = node.raft_core.clone(); + let _raft_handle = tokio::spawn(async move { + let mut raft = raft_lock.lock().await; + let _ = time::timeout(Duration::from_secs(5), raft.run()).await; + }); + + tokio::time::advance(Duration::from_millis(2)).await; + tokio::time::sleep(Duration::from_millis(2)).await; + + // request 1: prev_log_index=0. request 2: prev_log_index=99 β€” deliberately different + // from request 1's, so merge_append_entries (which only merges contiguous requests) + // can never combine them into a single handle_append_entries call. + let req1 = AppendEntriesRequest { + term: 1, + leader_id: 1, + prev_log_index: 0, + prev_log_term: 0, + entries: vec![], + leader_commit_index: 0, + }; + let req2 = AppendEntriesRequest { + prev_log_index: 99, + ..req1.clone() + }; + let stream = crate::test_utils::create_test_snapshot_stream(vec![req1, req2]); + + let response = node + .stream_append_entries(Request::new(stream)) + .await + .expect("stream_append_entries must accept the request"); + use futures::StreamExt; + let mut out_stream = response.into_inner(); + + let first_out = time::timeout(Duration::from_secs(2), out_stream.next()) + .await + .expect( + "the response for request 2 (already durable) must arrive without waiting for \ + request 1 (withheld) β€” if this times out, the forwarder is still strict FIFO", + ) + .expect("stream must yield an item") + .expect("must be Ok, not a transport error"); + + let last_match = match first_out.result { + Some(d_engine_proto::server::replication::append_entries_response::Result::Success( + success, + )) => success.last_match.expect("success response must carry last_match"), + other => panic!("expected a success response, got {other:?}"), + }; + assert_eq!( + last_match.term, 222, + "the first response observed must be request 2's (marker term=222) β€” request 1 \ + (marker term=111) is still withheld and must not be observed yet, nor block this one" + ); +} diff --git a/d-engine-server/src/node/builder.rs b/d-engine-server/src/node/builder.rs index 7d66360c..8c1c2ac2 100644 --- a/d-engine-server/src/node/builder.rs +++ b/d-engine-server/src/node/builder.rs @@ -367,9 +367,31 @@ where GrpcTransport::new_with_channels(node_id, peer_failure_tx, peer_success_tx) }); - let snapshot_policy = self.snapshot_policy.take().unwrap_or(LogSizePolicy::new( - node_config.raft.snapshot.max_log_entries_before_snapshot, - )); + let max_log_entries = node_config.raft.snapshot.max_log_entries_before_snapshot; + + // Startup memory-budget line β€” visible on stdout regardless of log setup. + { + const EST_LOG_ENTRY_BYTES: u64 = 512; // small/medium KV write + proto + SkipMap node overhead + let peak_mb = max_log_entries.saturating_mul(EST_LOG_ENTRY_BYTES) / (1024 * 1024); + tracing::info!( + node_id, + max_log_entries, + est_peak_ram_mb = peak_mb, + "Raft log memory: peak ~{peak_mb} MB in memory between snapshots \ + ({max_log_entries} entries Γ— ~512 B/entry est.), purged after each snapshot" + ); + if peak_mb > 100 { + tracing::warn!( + node_id, + est_peak_ram_mb = peak_mb, + "in-memory Raft log budget > 100 MB β€” lower \ + raft.snapshot.max_log_entries_before_snapshot if RAM-constrained" + ); + } + } + + let snapshot_policy = + self.snapshot_policy.take().unwrap_or(LogSizePolicy::new(max_log_entries)); let shutdown_signal = self.shutdown_signal.clone(); diff --git a/d-engine-server/src/node/builder_test.rs b/d-engine-server/src/node/builder_test.rs index 6c3f85b5..eb675479 100644 --- a/d-engine-server/src/node/builder_test.rs +++ b/d-engine-server/src/node/builder_test.rs @@ -7,7 +7,6 @@ use d_engine_core::LogStore; use d_engine_core::MockStateMachine; use d_engine_core::MockStorageEngine; use d_engine_core::PersistenceConfig; -use d_engine_core::PersistenceStrategy; use d_engine_core::RaftNodeConfig; use d_engine_core::StateMachine; use d_engine_core::StorageEngine; @@ -58,11 +57,9 @@ async fn test_set_raft_log_replaces_default() { BufferedRaftLog::>::new( id, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 1000, shutdown_timeout_ms: 5000, }, mock_storage_engine.clone(), diff --git a/d-engine-server/src/storage/adaptors/file/file_storage_engine.rs b/d-engine-server/src/storage/adaptors/file/file_storage_engine.rs index 4aec4eba..db63c514 100644 --- a/d-engine-server/src/storage/adaptors/file/file_storage_engine.rs +++ b/d-engine-server/src/storage/adaptors/file/file_storage_engine.rs @@ -350,7 +350,7 @@ impl LogStore for FileLogStore { &self, from_index: u64, new_entries: Vec, - ) -> Result<(), Error> { + ) -> Result { let encoded: Vec> = new_entries.iter().map(|e| e.encode_to_vec()).collect(); let new_last = { @@ -378,7 +378,7 @@ impl LogStore for FileLogStore { }; self.last_index.store(new_last, Ordering::SeqCst); - Ok(()) + Ok(new_last) } fn is_write_durable(&self) -> bool { diff --git a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_storage_engine.rs b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_storage_engine.rs index 624c901e..82f4ff5b 100644 --- a/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_storage_engine.rs +++ b/d-engine-server/src/storage/adaptors/rocksdb/rocksdb_storage_engine.rs @@ -383,7 +383,7 @@ impl LogStore for RocksDBLogStore { &self, from_index: u64, new_entries: Vec, - ) -> Result<(), Error> { + ) -> Result { let cf = self .db .cf_handle(LOG_CF) @@ -404,7 +404,7 @@ impl LogStore for RocksDBLogStore { self.db.write(&batch).map_err(|e| StorageError::DbError(e.to_string()))?; self.last_index.store(new_last_index, Ordering::SeqCst); - Ok(()) + Ok(new_last_index) } fn is_write_durable(&self) -> bool { diff --git a/d-engine-server/src/test_utils/integration/mod.rs b/d-engine-server/src/test_utils/integration/mod.rs index 838f4c9c..e0208b75 100644 --- a/d-engine-server/src/test_utils/integration/mod.rs +++ b/d-engine-server/src/test_utils/integration/mod.rs @@ -49,7 +49,6 @@ use d_engine_core::FlushPolicy; use d_engine_core::LogSizePolicy; use d_engine_core::MockStateMachine; use d_engine_core::PersistenceConfig; -use d_engine_core::PersistenceStrategy; use d_engine_core::RaftLog; use d_engine_core::RaftNodeConfig; use d_engine_core::ReplicationHandler; @@ -181,11 +180,9 @@ pub fn setup_raft_components( let (buffered_raft_log, receiver) = BufferedRaftLog::new( id, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage_engine.clone(), diff --git a/d-engine-server/tests/common/mod.rs b/d-engine-server/tests/common/mod.rs index 2d10210a..7dfd1b17 100644 --- a/d-engine-server/tests/common/mod.rs +++ b/d-engine-server/tests/common/mod.rs @@ -13,7 +13,6 @@ use d_engine_core::config::BackoffPolicy; use d_engine_core::config::ElectionConfig; use d_engine_core::config::FlushPolicy; use d_engine_core::config::PersistenceConfig; -use d_engine_core::config::PersistenceStrategy; use d_engine_core::config::RaftConfig; use d_engine_core::config::RaftNodeConfig; use d_engine_core::config::SnapshotConfig; @@ -124,7 +123,6 @@ pub async fn create_node_config( ] [raft.persistence] - strategy = "MemFirst" flush_policy = {{ Batch = {{ threshold = 100, idle_flush_interval_ms = 1 }} }} [raft.election] @@ -174,7 +172,6 @@ pub async fn create_node_config_with_role( ] [raft.persistence] - strategy = "MemFirst" flush_policy = {{ Batch = {{ threshold = 1, idle_flush_interval_ms = 1 }} }} [raft.election] @@ -218,7 +215,6 @@ pub fn node_config(cluster_toml: &str) -> RaftNodeConfig { ..Default::default() }, persistence: PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs b/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs index b54a1080..666f1146 100644 --- a/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs +++ b/d-engine-server/tests/snapshot_and_recovery/snapshot_transfer_does_not_block_apply_embedded.rs @@ -91,9 +91,20 @@ async fn test_snapshot_transfer_does_not_block_apply() -> Result<(), Box RETAINED_LOGS=8 (checked below), no node's log can cross a - // purge boundary before these writes land, so nothing earlier in the buffer can - // satisfy the match below β€” no extra gate is needed to make `since` safe. - // // This used to gate `since` on a `wait_for_snapshot` directory scan for the leader's // `.gz` file first, on the theory that "file exists" proves the snapshot is built. // It doesn't: `compress_directory` (default_state_machine_handler.rs) calls @@ -196,7 +202,6 @@ push_queue_size = 1 // made this test flake under CI load: RocksDB checkpoint export + tar/gzip // compression + metadata persist is genuinely sequential disk+CPU work that slows // down under contention. - let since = logs.lock().unwrap().len(); // The retained-log purge boundary β€” the actual signal this test needs, not just "a // snapshot file exists" (see comment above for why those differ). Emitted by @@ -208,14 +213,29 @@ push_queue_size = 1 // SNAPSHOT_THRESHOLD=64 > RETAINED_LOGS=8, the earliest possible snapshot on this // cluster already has last_included.index >= 64, so purge_upto_index is always > 0 // by the time this log line can appear at all. + // 60 x 500ms = 30s, not 15s: this test lives in the `multi-node-cluster-local` + // nextest group (throttled but not serialized, see .config/nextest.toml), and the + // log line polled below shares a process-global Mutex> with every other + // concurrently-running test in this binary (see log_capture.rs). Under full-suite + // load, RocksDB checkpoint export + tar/gzip (genuinely sequential CPU+disk work, + // see comment above) plus that shared-mutex contention can push real completion past + // 15s even though nothing is actually wrong β€” same root cause already documented in + // stress_test.rs's 30s bound. let mut purged = false; - for _ in 0..30 { + for _ in 0..60 { if logs_contain_globally_since(&logs, since, "purge_upto_index=") { purged = true; break; } tokio::time::sleep(Duration::from_millis(500)).await; } + if !purged { + eprintln!("=== DEBUG: captured logs since baseline writes ==="); + for line in logs.lock().unwrap()[since..].iter() { + eprintln!("{line}"); + } + eprintln!("=== END DEBUG ==="); + } assert!( purged, "Leader never logged a completed log purge β€” node 4 joining now would prove \ diff --git a/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs b/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs index de85eebd..9130f08f 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/crash_recovery_test.rs @@ -8,9 +8,7 @@ use std::sync::Arc; use std::time::Duration; use bytes::Bytes; -use d_engine_core::{ - BufferedRaftLog, FlushPolicy, PersistenceConfig, PersistenceStrategy, RaftLog, -}; +use d_engine_core::{BufferedRaftLog, FlushPolicy, PersistenceConfig, RaftLog}; use d_engine_proto::common::{Entry, EntryPayload}; use d_engine_server::{FileStateMachine, FileStorageEngine, node::RaftTypeConfig}; use tokio::time::sleep; @@ -21,7 +19,6 @@ use super::TestContext; async fn test_crash_recovery() { // Create and populate storage let original_ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -64,8 +61,7 @@ async fn test_crash_recovery() { #[tokio::test] async fn test_crash_recovery_with_multiple_entries() { // Create and populate storage - let original_ctx = TestContext::new( - PersistenceStrategy::MemFirst, + let mut original_ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -89,6 +85,7 @@ async fn test_crash_recovery_with_multiple_entries() { // Ensure all entries are persisted for DiskFirst strategy original_ctx.raft_log.flush().await.unwrap(); + original_ctx.drain_fsync_completions(); // Verify all entries are in memory and durable assert_eq!(original_ctx.raft_log.durable_index(), 5); @@ -128,11 +125,9 @@ async fn test_partial_flush_with_graceful_shutdown() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -165,11 +160,9 @@ async fn test_partial_flush_with_graceful_shutdown() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -210,11 +203,9 @@ async fn test_partial_flush_after_crash() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 100, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -259,11 +250,9 @@ async fn test_partial_flush_after_crash() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -299,21 +288,18 @@ async fn test_recovery_under_different_scenarios() { // drain cycle, so all 100 entries are always durable after explicit flush(). let scenarios = vec![ ( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, 100usize, ), ( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 10, }, 100, ), ( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1000, }, @@ -321,9 +307,9 @@ async fn test_recovery_under_different_scenarios() { ), ]; - for (strategy, flush_policy, expected_recovery) in scenarios { - let instance_id = format!("recovery_test_{strategy:?}_{flush_policy:?}"); - let original_ctx = TestContext::new(strategy.clone(), flush_policy.clone(), &instance_id); + for (flush_policy, expected_recovery) in scenarios { + let instance_id = format!("recovery_test_{flush_policy:?}"); + let original_ctx = TestContext::new(flush_policy.clone(), &instance_id); // Add test data for i in 1..=100 { @@ -351,7 +337,7 @@ async fn test_recovery_under_different_scenarios() { assert_eq!( recovered_ctx.raft_log.len(), expected_recovery, - "Recovery mismatch for strategy {strategy:?} policy {flush_policy:?}" + "Recovery mismatch for policy {flush_policy:?}" ); recovered_ctx.close().await; } @@ -363,7 +349,6 @@ async fn test_memfirst_crash_recovery_durability() { let recovered_path = { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 10000, }, @@ -390,11 +375,9 @@ async fn test_memfirst_crash_recovery_durability() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -424,11 +407,9 @@ async fn test_diskfirst_crash_recovery_durability() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, @@ -458,11 +439,9 @@ async fn test_diskfirst_crash_recovery_durability() { BufferedRaftLog::>::new( 1, PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms: 1, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage, diff --git a/d-engine-server/tests/storage_buffered_raft_log/mod.rs b/d-engine-server/tests/storage_buffered_raft_log/mod.rs index ad11cddb..f8701b18 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/mod.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/mod.rs @@ -15,15 +15,14 @@ use std::sync::Arc; use std::time::Duration; use bytes::Bytes; -use d_engine_core::{ - BufferedRaftLog, FlushPolicy, PersistenceConfig, PersistenceStrategy, RaftLog, alias::ROF, -}; +use d_engine_core::{BufferedRaftLog, FlushPolicy, PersistenceConfig, RaftLog, alias::ROF}; use d_engine_proto::common::{Entry, EntryPayload}; use d_engine_server::{FileStateMachine, FileStorageEngine, node::RaftTypeConfig}; use tempfile::tempdir; mod crash_recovery_test; mod performance_test; +mod quorum_crash_recovery_test; mod storage_integration_test; mod stress_test; @@ -32,15 +31,14 @@ pub struct TestContext { pub raft_log: Arc>>, pub storage: Arc, pub _temp_dir: Option, - pub strategy: PersistenceStrategy, pub flush_policy: FlushPolicy, pub path: String, + log_flush_rx: tokio::sync::mpsc::UnboundedReceiver, } impl TestContext { /// Create new test context with FileStorageEngine pub fn new( - strategy: PersistenceStrategy, flush_policy: FlushPolicy, instance_id: &str, ) -> Self { @@ -51,14 +49,13 @@ impl TestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: strategy.clone(), flush_policy: flush_policy.clone(), - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); // Small delay to ensure processor is ready std::thread::sleep(Duration::from_millis(10)); @@ -67,9 +64,25 @@ impl TestContext { path: path.to_str().unwrap().to_string(), raft_log, storage, - strategy, flush_policy, _temp_dir: Some(temp_dir), + log_flush_rx, + } + } + + /// Stands in for `raft.rs`'s `InternalEvent::FsyncCompleted` handler, + /// which isn't running in these `BufferedRaftLog`-only integration + /// tests. Since #446/#447, `durable_index` only advances when something + /// drains that event and calls `try_advance_durable_index` β€” call this + /// after any operation that should make `durable_index` advance and + /// before asserting on it. Not needed after `recover_from_crash()`: the + /// recovered context's `durable_index` is derived directly from on-disk + /// state at construction, not from this event. + pub fn drain_fsync_completions(&mut self) { + while let Ok(event) = self.log_flush_rx.try_recv() { + if let d_engine_core::InternalEvent::FsyncCompleted(mark) = event { + self.raft_log.try_advance_durable_index(mark); + } } } @@ -92,24 +105,23 @@ impl TestContext { let (raft_log, receiver) = BufferedRaftLog::new( 1, PersistenceConfig { - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }, storage.clone(), ); - let raft_log = raft_log.start(receiver, None); + let (log_flush_tx, log_flush_rx) = tokio::sync::mpsc::unbounded_channel(); + let raft_log = raft_log.start(receiver, Some(log_flush_tx)); std::thread::sleep(Duration::from_millis(10)); Self { raft_log, storage, - strategy: self.strategy.clone(), flush_policy: self.flush_policy.clone(), _temp_dir: Some(temp_dir), path: self.path.clone(), + log_flush_rx, } } diff --git a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs index 0368f99a..c007c105 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/performance_test.rs @@ -6,9 +6,7 @@ use super::TestContext; use bytes::Bytes; -use d_engine_core::{ - BufferedRaftLog, FlushPolicy, PersistenceConfig, PersistenceStrategy, RaftLog, -}; +use d_engine_core::{BufferedRaftLog, FlushPolicy, PersistenceConfig, RaftLog}; use d_engine_proto::common::{Entry, EntryPayload}; use d_engine_server::{FileStateMachine, FileStorageEngine, node::RaftTypeConfig}; use std::collections::HashMap; @@ -33,11 +31,9 @@ mod filter_out_conflicts_and_append_performance_tests { for (idle_flush_interval_ms, max_duration_ms) in test_cases { // Create MemFirst storage with batch policy let config = PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }; @@ -85,10 +81,17 @@ mod filter_out_conflicts_and_append_performance_tests { "Duration {duration}ms exceeds max {max_duration_ms}ms for {idle_flush_interval_ms}ms interval" ); - // Verify correctness - assert!(log.entry(500).unwrap().is_none()); + // Verify correctness: index=501 term=1 already exists (populated above), so this + // prev_log_index=0 resend is a pure duplicate β€” must be a no-op, not a reset. + // See 446-expert-q-probe-backpressure-fix-8020.md. + assert_eq!( + log.last_entry_id(), + 1000, + "duplicate resend must not touch the log" + ); + assert!(log.entry(500).unwrap().is_some()); assert!(log.entry(501).unwrap().is_some()); - assert!(log.entry(502).unwrap().is_none()); + assert!(log.entry(502).unwrap().is_some()); } } @@ -104,11 +107,9 @@ mod filter_out_conflicts_and_append_performance_tests { for (idle_flush_interval_ms, max_duration_ms) in test_cases { // Create MemFirst storage with batch policy let config = PersistenceConfig { - strategy: PersistenceStrategy::MemFirst, flush_policy: FlushPolicy::Batch { idle_flush_interval_ms, }, - max_buffered_entries: 10000, shutdown_timeout_ms: 5000, }; @@ -171,7 +172,6 @@ mod filter_out_conflicts_and_append_performance_tests { async fn test_last_entry_id_performance() { // Set up test context let test_context = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 360_000, }, @@ -225,16 +225,26 @@ async fn test_performance_benchmarks() { // Adjust test parameters according to the environment let operations = if is_ci { // CI environment uses a more relaxed threshold + // + // append_entries now round-trips through a dedicated IO thread via + // oneshot (see #444: leader's own write must reach the storage + // engine before counting toward quorum) β€” this is an intentional + // correctness/speed tradeoff, not a regression. The old threshold + // (500) predates that fix. New floor leaves ~2x headroom below the + // observed ~318-328 ops/sec on a modern dev machine, keeping the + // 2:1 local:CI ratio from before. [ - ("append_entries", 500, 500.0), + ("append_entries", 500, 100.0), ("get_entries_range", 2500, 25000.0), ("entry_lookup", 5000, 100000.0), ("term_queries", 4000, 25000.0), ] } else { // Local environment uses a stricter threshold + // + // See CI-branch comment above β€” same #444 rationale. [ - ("append_entries", 1000, 1000.0), + ("append_entries", 1000, 200.0), ("get_entries_range", 5000, 50000.0), ("entry_lookup", 10000, 200000.0), ("term_queries", 8000, 50000.0), @@ -242,7 +252,6 @@ async fn test_performance_benchmarks() { }; let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -332,7 +341,6 @@ async fn test_read_performance_under_concurrent_write_load() { }; let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, diff --git a/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs b/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs new file mode 100644 index 00000000..ea60cdc1 --- /dev/null +++ b/d-engine-server/tests/storage_buffered_raft_log/quorum_crash_recovery_test.rs @@ -0,0 +1,110 @@ +//! Quorum + real-disk crash recovery integration test (#446 gap 4). +//! +//! Composes two pieces that are each already covered in isolation elsewhere, but never +//! together: `calculate_majority_matched_index` (RPO=0 quorum arithmetic, unit-tested +//! against a gated mock in `buffered_raft_log_test/quorum_durability_test.rs`) and real +//! `FileStorageEngine` crash/reopen (unit-tested without any quorum math in +//! `crash_recovery_test.rs`). This file proves they actually compose: an index that the +//! quorum calculation says is safe to acknowledge to the client is still there after a +//! real crash + reopen from the same on-disk path. +//! +//! Followers are represented as reported match_index values, same as in +//! `quorum_durability_test.rs` β€” this file's job is the leader-side real-disk durability +//! boundary, not follower ACK withholding (covered by follower_state_test.rs / +//! learner_state_test.rs). +//! +//! Deliberately NOT attempted here, and now CONFIRMED impossible with this engine's +//! architecture (not just a flakiness risk β€” an actual dead end, verified by building and +//! deadlocking it): proving that an entry which never reached quorum-durable is genuinely +//! absent from a real crash + reopen. `BufferedRaftLog::append_entries` +//! (`d-engine-core/src/storage/buffered_raft_log.rs:467-500`) is documented and +//! implemented to block the caller until `persist_entries()` returns β€” "still blocks the +//! caller until truly persisted" β€” and `FileLogStore::persist_entries` +//! (`d-engine-server/src/storage/adaptors/file/file_storage_engine.rs:254`) already writes +//! the entry to the real OS-visible file as an unconditional part of its body, before it +//! can return. So by the time `append_entries().await` ever resolves at all, the entry is +//! already on the file β€” there is no window where it's "acknowledged as appended" yet +//! "recoverably absent." A gate on `persist_entries()` was built and tried here; it did +//! not create the intended window, it just deadlocked `append_entries()` forever (the +//! call this file's other test depends on to make progress at all). Reverted. +//! What IS real and already correctly tested (see below): the gap between +//! `last_entry_id` and `durable_index` β€” `persist_entries()` writes the bytes, but a +//! *separate* `flush()` call (`sync_all()`) is what advances `durable_index`, and that one +//! genuinely runs later/independently. What's NOT reachable by a same-process test is +//! observing that an un-`sync_all`'d write doesn't survive β€” on the same OS instance, +//! `write()` alone (which `persist_entries` already does) is enough for a freshly-opened +//! handle to see the bytes, real crash or not. Proving the un-fsynced case would need an +//! actual power-loss simulation (dropped page cache / real reboot) β€” which prior sessions +//! already found to be a poor fit for this class of bug (see mempalace notes on the +//! Jepsen/lazyfs work for #444). + +use super::TestContext; +use d_engine_core::FlushPolicy; +use d_engine_core::RaftLog; + +#[tokio::test] +async fn test_quorum_acknowledged_index_survives_real_crash_and_reopen() { + let mut ctx = TestContext::new( + FlushPolicy::Batch { + idle_flush_interval_ms: 1, + }, + "test_quorum_ack_survives_crash", + ); + + // First 5 entries, explicitly flushed: genuinely durable, deterministic. + ctx.append_entries(1, 5, 1).await; + ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); + assert_eq!(ctx.raft_log.durable_index(), 5); + + // 3-node cluster: both followers already report match_index=5 (post-Stage2 + // semantics β€” a follower only reports a match_index once its own durable_index + // reaches it). This is the index the leader would actually acknowledge to the + // client. + let commit = ctx.raft_log.calculate_majority_matched_index(1, 0, vec![5, 5]); + assert_eq!( + commit, + Some(5), + "index 5 is durable on the leader and acked by both followers" + ); + + // A follower report of 10 must not move commit past what the leader itself has + // fsynced β€” restates the Stage1 invariant as this test's own setup precondition + // rather than assuming it silently. + let would_be_wrong = ctx.raft_log.calculate_majority_matched_index(1, 0, vec![10, 5]); + assert_eq!( + would_be_wrong, + Some(5), + "leader's own un-fsynced tail must not leak into the client-visible commit index" + ); + + // Second batch, also explicitly flushed, so the whole log is durable before the + // simulated crash β€” keeps this test's crash/recovery assertions exact, not bounded. + ctx.append_entries(6, 5, 1).await; + ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); + assert_eq!(ctx.raft_log.durable_index(), 10); + + let recovered = ctx.recover_from_crash(); + ctx.close().await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + // The index that was actually acknowledged to the client (5) β€” and everything else + // that was durably flushed (up to 10) β€” survives the real crash + reopen. + assert_eq!(recovered.raft_log.durable_index(), 10); + for i in 1..=10 { + assert!( + recovered.raft_log.entry(i).unwrap().is_some(), + "entry {i} must survive real crash + reopen" + ); + } + + // Re-running the same quorum calculation against the recovered log reaches the same + // conclusion β€” the leader's durability contribution to quorum is stable across a + // real restart, not just in the pre-crash in-memory view. + let commit_after_recovery = + recovered.raft_log.calculate_majority_matched_index(1, 0, vec![5, 5]); + assert_eq!(commit_after_recovery, Some(5)); + + recovered.close().await; +} diff --git a/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs b/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs index 8cd4ab85..acf79449 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/storage_integration_test.rs @@ -3,7 +3,7 @@ //! These tests verify BufferedRaftLog integration with FileStorageEngine //! at the storage layer, including compaction and storage-specific operations. -use d_engine_core::{FlushPolicy, PersistenceStrategy, RaftLog}; +use d_engine_core::{FlushPolicy, RaftLog}; use d_engine_proto::common::LogId; use super::TestContext; @@ -13,8 +13,7 @@ use super::TestContext; #[tokio::test] async fn test_log_compaction() { - let ctx = TestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -24,6 +23,7 @@ async fn test_log_compaction() { // With MemFirst, entries are buffered and flushed asynchronously. // Wait for all entries to become durable before checking durable_index. ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Compact first 50 entries ctx.raft_log.purge_logs_up_to(LogId { index: 50, term: 1 }).await.unwrap(); diff --git a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs index f89ae55a..7968af39 100644 --- a/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs +++ b/d-engine-server/tests/storage_buffered_raft_log/stress_test.rs @@ -6,7 +6,7 @@ use std::time::Duration; use bytes::Bytes; -use d_engine_core::{FlushPolicy, LogStore, PersistenceStrategy, RaftLog, StorageEngine}; +use d_engine_core::{FlushPolicy, LogStore, RaftLog, StorageEngine}; use d_engine_proto::common::{Entry, EntryPayload}; use futures::future::join_all; use tokio::time::Instant; @@ -22,8 +22,7 @@ use super::TestContext; #[tokio::test] async fn test_high_concurrency() { - let ctx = TestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -53,6 +52,7 @@ async fn test_high_concurrency() { // With MemFirst, entries are buffered; wait for all to be durable before asserting. ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Verify all entries persisted assert_eq!(ctx.raft_log.durable_index(), 1000); @@ -63,7 +63,6 @@ async fn test_high_concurrency() { #[traced_test] async fn test_high_concurrency_mixed_operations() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 100, }, @@ -123,8 +122,15 @@ async fn test_high_concurrency_mixed_operations() { // Verify data integrity assert_eq!(ctx.raft_log.len(), 10000); + // append_entries() now round-trips through a dedicated IO thread via + // oneshot (see #444: leader's own write must reach the storage engine + // before counting toward quorum) β€” an intentional correctness/speed + // tradeoff, not a regression. The old 10s bound predates that fix; + // observed wall-clock for this test's 10k concurrent writes is now + // 13-18s depending on machine load. New bound leaves real headroom + // above that range rather than chasing the exact number. assert!( - duration < Duration::from_secs(10), + duration < Duration::from_secs(30), "Operations took too long: {duration:?}" ); } @@ -135,7 +141,6 @@ mod mem_first_tests { #[tokio::test] async fn test_basic_write_before_persist() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -150,8 +155,7 @@ mod mem_first_tests { #[tokio::test] async fn test_async_persistence() { - let ctx = TestContext::new( - PersistenceStrategy::MemFirst, + let mut ctx = TestContext::new( FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -161,6 +165,7 @@ mod mem_first_tests { // Trigger flush ctx.raft_log.flush().await.unwrap(); + ctx.drain_fsync_completions(); // Verify persistence assert_eq!(ctx.raft_log.durable_index(), 100); @@ -170,7 +175,6 @@ mod mem_first_tests { #[tokio::test] async fn test_power_loss_data_loss() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -188,7 +192,6 @@ mod mem_first_tests { #[tokio::test] async fn test_high_concurrency_memory_only() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, @@ -225,7 +228,6 @@ mod mem_first_tests { #[tokio::test] async fn test_term_index_correctness_under_load() { let ctx = TestContext::new( - PersistenceStrategy::MemFirst, FlushPolicy::Batch { idle_flush_interval_ms: 1, }, diff --git a/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs b/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs index b5edc3e4..051bd020 100644 --- a/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs +++ b/d-engine-server/tests/watch_and_subscriptions/watch_membership_embedded.rs @@ -125,8 +125,6 @@ learner_check_throttle_ms = 100 election_timeout_min = 300 election_timeout_max = 3000 -[raft.persistence] -strategy = "MemFirst" [retry.election] max_retries = 5 diff --git a/d-engine/src/docs/examples/three-nodes-standalone.md b/d-engine/src/docs/examples/three-nodes-standalone.md index 96cbe01a..b5ab1764 100644 --- a/d-engine/src/docs/examples/three-nodes-standalone.md +++ b/d-engine/src/docs/examples/three-nodes-standalone.md @@ -59,9 +59,7 @@ default_policy = "LeaseRead" lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" # Only strategy in v0.2.4+ (DiskFirst removed) flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -max_buffered_entries = 10000 ``` **Key differences from single-node expansion:** @@ -123,7 +121,7 @@ All performance reports in `/benches/standalone-bench/reports` use this exact co **Raft settings:** -- Persistence: `MemFirst` (only strategy in v0.2.4+) with 1000ms idle flush interval +- Persistence: batched fsync (Level 3, fdatasync) with 1000ms idle flush interval - Read consistency: `LeaseRead` (500ms lease duration) - Replication: Batched append entries (5000 threshold, 0ms delay) - Network: Tuned for high throughput (see `config/n1.toml` for details) diff --git a/d-engine/src/docs/performance/throughput-optimization-guide.md b/d-engine/src/docs/performance/throughput-optimization-guide.md index 03bc4efc..1fe3341e 100644 --- a/d-engine/src/docs/performance/throughput-optimization-guide.md +++ b/d-engine/src/docs/performance/throughput-optimization-guide.md @@ -23,18 +23,6 @@ pub(crate) enum ConnectionType { ## Persistence Strategy & Throughput/Latency Trade-offs -`MemFirst` is the only persistence strategy in v0.2.4+. It batches writes to OS page cache and flushes with fsync asynchronously β€” committing data to disk before notifying Raft. - -### Strategy Configuration - -```toml -[raft.persistence] -strategy = "MemFirst" -flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -``` - -### `MemFirst` Strategy - - **Write Path**: Entries are written to OS page cache via `db.write()` / `file.write()`; the IO thread batches them and calls fsync (`flush_wal(true)` / `sync_all()`) before advancing `durable_index`. Raft only counts an entry toward quorum after fsync completes. @@ -54,8 +42,7 @@ flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } - Lower values reduce the unflushed batch window but increase IO pressure. - Default `1000` ms is suitable for most workloads. -> **Note**: `DiskFirst` strategy was removed in v0.2.4. `MemFirst` replaced it with batched -> fsync β€” multiple writes share one fsync call, reducing IO overhead while still providing +> **Note**: Writes are batched into a single fsync, reducing IO overhead while still providing > disk-level durability for all client-acknowledged (committed) writes. ## Batching Configuration @@ -179,7 +166,7 @@ tonic::transport::Server::builder() ``` -Inbound message size is the one setting that *is* per-service rather than +Inbound message size is the one setting that _is_ per-service rather than transport-wide, so it's applied on each `XxxServiceServer` individually: ```rust,ignore @@ -197,7 +184,7 @@ RaftReplicationServiceServer::from_arc(node.clone()) | p99.9 Latency | 14015 Β΅s | 11279 Β΅s | -19.5% | > **Key improvement**: 15% reduction in tail latency - critical for consensus stability -> **Note**: These metrics show the impact of connection pooling optimization. These results can be further improved by tuning the PersistenceStrategy for your specific workload. +> **Note**: These metrics show the impact of connection pooling optimization. These results can be further improved by tuning `FlushPolicy` for your specific workload. > > For absolute performance benchmarks, see [v0.2.4 Performance Report](https://github.com/deventlab/d-engine/tree/main/benches/reports/v0.2.4/bench_report_v0.2.4.md) @@ -230,7 +217,7 @@ RaftReplicationServiceServer::from_arc(node.clone()) ``` -5. **Monitor Flush Lag**: When using `MemFirst`, monitor the difference between `last_log_index` and `durable_index`. A growing gap indicates the disk is not keeping up with writes, increasing potential data loss. +5. **Monitor Flush Lag**: Monitor the difference between `last_log_index` and `durable_index`. Raft only counts an entry toward quorum and acknowledges it to the client after fsync β€” so a growing gap does not put acknowledged writes at risk. It does mean client-facing write latency is growing, and (if the gap keeps growing) the amount of work an unflushed batch would need to redo on restart is growing too. ## Anti-Patterns to Avoid @@ -244,11 +231,9 @@ get_peer_channel(peer_id, ConnectionType::Control).await?; client.request_vote(...) // DON'T: Set idle_flush_interval_ms too low β€” defeats batching. -[strategy = "MemFirst"] flush_policy = { Batch = { idle_flush_interval_ms = 1 } } // Near-synchronous; low throughput // DO: Use a generous idle interval to amortize disk I/O cost. -[strategy = "MemFirst"] flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } ``` @@ -261,8 +246,8 @@ flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } Control: Low latency ↔ Data: High throughput ↔ Bulk: Bandwidth 3. **Improves fault containment** Connection issues affect only one operation type -4. **Decouples Performance from Durability** - `MemFirst` with tunable `idle_flush_interval_ms` lets you balance write throughput against flush frequency. +4. **Decouples Performance from Ack Latency** + Client-acknowledged writes are always fsync-durable β€” that's not tunable. `idle_flush_interval_ms` lets you balance write throughput against how long a client waits for that fsync. ## Reference Deployment Configurations @@ -277,7 +262,6 @@ Adjust values based on snapshot size, log append rate, and cluster size. ```toml [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [network.control] @@ -306,7 +290,6 @@ max_concurrent_streams = 128 ```toml [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [network.control] @@ -325,7 +308,7 @@ max_concurrent_streams = 256 ``` -**Tip**: For public cloud, moderate concurrency and 32MB bulk windows ensure stable snapshot streaming without affecting heartbeats. The batch policy is tuned for high throughput with a reasonable data loss window. +**Tip**: For public cloud, moderate concurrency and 32MB bulk windows ensure stable snapshot streaming without affecting heartbeats. The batch policy is tuned for high throughput; acknowledged writes are never at risk regardless of the interval, only ack latency and unflushed-batch replay time on restart scale with it. ### 3. 5-Node High-Durability Cluster (Production) @@ -335,8 +318,7 @@ max_concurrent_streams = 256 ```toml [raft.persistence] -strategy = "MemFirst" -flush_policy = { Batch = { idle_flush_interval_ms = 100 } } # More frequent flush for durability +flush_policy = { Batch = { idle_flush_interval_ms = 100 } } # More frequent flush, shorter ack latency [network.control] connection_window_size = 4_194_304 # 4MB @@ -354,7 +336,7 @@ max_concurrent_streams = 512 ``` -**Tip**: For higher write persistence within a process lifecycle, lower `idle_flush_interval_ms` (e.g., 100ms). Note: `MemFirst` is not power-loss safe regardless of flush interval. +**Tip**: Lower `idle_flush_interval_ms` (e.g., 100ms) shortens client-facing write latency and shrinks the unflushed-batch window an IO thread has to redo on restart. Acknowledged writes are power-loss safe regardless of this setting β€” Raft only counts an entry toward quorum, and acknowledges it to the client, after fsync completes. ## Network Environment Tuning Recommendations @@ -362,12 +344,12 @@ These parameters are primarilyΒ **network-dependent**, not CPU/memory dependent. Adjust them based on latency, packet loss, and connection stability. -| **Environment** | **tcp_keepalive_in_secs** | **http2_keep_alive_interval_in_secs** | **http2_keep_alive_timeout_in_secs** | **Notes** | -| -------------------------------- | ------------------------- | ------------------------------------- | ------------------------------------ | ------------------------------------------------------- | -| **Local / In-Cluster (LAN)** | 60 | 10 | 5 | Low latency & stable; defaults are fine | -| **Cross-Region / Stable WAN** | 60 | 15 | 8 | Slightly longer keep-alive to avoid false disconnects | -| **Public Cloud / Moderate Loss** | 60 | 20 | 10 | Higher interval & timeout for lossy links | -| **High Latency / Unstable WAN** | 120 | 30 | 15 | Longer timeouts prevent spurious drops | +| **Environment** | **tcp_keepalive_in_secs** | **http2_keep_alive_interval_in_secs** | **http2_keep_alive_timeout_in_secs** | **Notes** | +| -------------------------------- | ------------------------- | ------------------------------------- | ------------------------------------ | ----------------------------------------------------- | +| **Local / In-Cluster (LAN)** | 60 | 10 | 5 | Low latency & stable; defaults are fine | +| **Cross-Region / Stable WAN** | 60 | 15 | 8 | Slightly longer keep-alive to avoid false disconnects | +| **Public Cloud / Moderate Loss** | 60 | 20 | 10 | Higher interval & timeout for lossy links | +| **High Latency / Unstable WAN** | 120 | 30 | 15 | Longer timeouts prevent spurious drops | **Guidelines:** diff --git a/d-engine/src/docs/server_guide/customize-storage-engine.md b/d-engine/src/docs/server_guide/customize-storage-engine.md index 0874acb0..ef9b088b 100644 --- a/d-engine/src/docs/server_guide/customize-storage-engine.md +++ b/d-engine/src/docs/server_guide/customize-storage-engine.md @@ -91,7 +91,7 @@ impl StorageEngine for CustomStorageEngine { - **Consistency**: Maintain exactly-once semantics for log entries - **Performance**: Target >100k ops/sec for log persistence. Do not call `fsync` inside `persist_entries()`β€”the framework batches entries and calls `flush()` once per batch - (`MemFirst + FlushPolicy::Batch`), which amortises the `fsync` cost across many entries. + (`FlushPolicy::Batch`), which amortises the `fsync` cost across many entries. - **Resource Management**: Clean up resources in `Drop` implementation ## 3. StorageEngine API Reference diff --git a/examples/quick-start-standalone/go.mod b/examples/quick-start-standalone/go.mod index 79b1ef9c..9865cf61 100644 --- a/examples/quick-start-standalone/go.mod +++ b/examples/quick-start-standalone/go.mod @@ -6,13 +6,13 @@ replace github.com/deventlab/d-engine/proto => ../../d-engine-proto/go require ( github.com/deventlab/d-engine/proto v0.0.0 - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.83.2 ) require ( - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/examples/quick-start-standalone/go.sum b/examples/quick-start-standalone/go.sum index adb3ad1c..2d3ad8d1 100644 --- a/examples/quick-start-standalone/go.sum +++ b/examples/quick-start-standalone/go.sum @@ -12,27 +12,27 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/examples/single-node-expansion/Makefile b/examples/single-node-expansion/Makefile index a31b2775..a00218c4 100644 --- a/examples/single-node-expansion/Makefile +++ b/examples/single-node-expansion/Makefile @@ -8,12 +8,35 @@ # =============================== LOG_LEVEL ?= debug + +# On macOS with Homebrew: auto-detect compression lib paths to skip bundled C++ +# compilation of RocksDB dependencies, which fails under macOS 26 + Xcode 26 +# (Clang 16 lacks __builtin_ctzg/__builtin_clzg from LLVM 18+ SDK headers). +# brew --prefix resolves correctly on both Apple Silicon (/opt/homebrew) and +# Intel Mac (/usr/local). Silently no-ops when brew or a lib is absent. +SNAPPY_PREFIX := $(shell brew --prefix snappy 2>/dev/null) +LZ4_PREFIX := $(shell brew --prefix lz4 2>/dev/null) +ZSTD_PREFIX := $(shell brew --prefix zstd 2>/dev/null) +BREW_ROCKSDB_ENV := + +ifneq ($(SNAPPY_PREFIX),) +ifneq ($(wildcard $(SNAPPY_PREFIX)/lib),) + BREW_ROCKSDB_ENV += SNAPPY_LIB_DIR=$(SNAPPY_PREFIX)/lib +endif +endif +ifneq ($(LZ4_PREFIX),) + BREW_ROCKSDB_ENV += LZ4_LIB_DIR=$(LZ4_PREFIX)/lib +endif +ifneq ($(ZSTD_PREFIX),) + BREW_ROCKSDB_ENV += ZSTD_LIB_DIR=$(ZSTD_PREFIX)/lib +endif + # =============================== # Build Targets # =============================== build: @echo "Building release binary..." - cargo build --release --jobs 4 + $(BREW_ROCKSDB_ENV) cargo build --release --jobs 4 # =============================== # Single Node Bootstrap diff --git a/examples/single-node-expansion/config/n1.toml b/examples/single-node-expansion/config/n1.toml index bdb78715..1d961cb3 100644 --- a/examples/single-node-expansion/config/n1.toml +++ b/examples/single-node-expansion/config/n1.toml @@ -14,6 +14,8 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 +cmd_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -23,27 +25,49 @@ election_timeout_max = 2000 default_policy = "LeaseRead" lease_duration_ms = 500 +[raft.read_actor] +channel_capacity = 10240 +max_drain = 2000 + +[raft.batching] +# Maximum number of commands to accumulate in a single batch during drain operations +max_batch_size = 200 + +[raft.metrics] +enable_backpressure = false +enable_batch = false + +[raft.backpressure] +max_pending_writes = 1000 +max_pending_reads = 500 + + [raft.persistence] -strategy = "MemFirst" -flush_policy = { Batch = { idle_flush_interval_ms = 20 } } -max_buffered_entries = 10000 +flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } [raft.snapshot] -enable = false -max_log_entries_before_snapshot = 10000 -retained_log_entries = 3 +enable = true +max_log_entries_before_snapshot = 5000 +retained_log_entries = 100 +cleanup_retain_count = 100 -# == Network Control Plane == +# == TTL Lease Configuration == +[raft.state_machine.lease] +cleanup_interval_ms = 1000 +max_cleanup_duration_ms = 1 + +# == Network Control Plane (voting, heartbeat, etc.) == [network.control] connection_window_size = 4_194_304 stream_window_size = 2_097_152 tcp_keepalive_in_secs = 60 -http2_keep_alive_interval_in_secs = 15 -http2_keep_alive_timeout_in_secs = 10 +http2_keep_alive_interval_in_secs = 15 # Slightly increase to reduce frequent keep-alives +http2_keep_alive_timeout_in_secs = 10 # Increase timeout +# New performance tuning parameters -# == Network Data Plane == +# == Network Data Plane (append_entries, etc.) == [network.data] connect_timeout_in_ms = 100 request_timeout_in_ms = 300 @@ -51,13 +75,16 @@ connection_window_size = 8_388_608 stream_window_size = 4_194_304 tcp_keepalive_in_secs = 60 -http2_keep_alive_interval_in_secs = 15 +http2_keep_alive_interval_in_secs = 15 # Same as control plane http2_keep_alive_timeout_in_secs = 10 - +# New data plane optimizations # == Server Transport (single listener serving every RPC type) == [network.server] concurrency_limit_per_connection = 100 # Increased for higher concurrent replication load max_concurrent_streams = 4096 # Increased to reduce stream creation overhead max_pending_accept_reset_streams = 2000 # Higher pending stream limit for Rapid Reset mitigation + +[storage] +unified_db = false diff --git a/examples/single-node-expansion/config/n2.toml b/examples/single-node-expansion/config/n2.toml index 5e5356c3..0e4bb985 100644 --- a/examples/single-node-expansion/config/n2.toml +++ b/examples/single-node-expansion/config/n2.toml @@ -28,9 +28,7 @@ lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 20 } } -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/single-node-expansion/config/n3.toml b/examples/single-node-expansion/config/n3.toml index 75a585e3..32dc4c3b 100644 --- a/examples/single-node-expansion/config/n3.toml +++ b/examples/single-node-expansion/config/n3.toml @@ -30,9 +30,7 @@ lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 20 } } -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/sled-cluster/config/n1.toml b/examples/sled-cluster/config/n1.toml index a5b391e8..5902a6ad 100644 --- a/examples/sled-cluster/config/n1.toml +++ b/examples/sled-cluster/config/n1.toml @@ -16,8 +16,6 @@ batch_size = 5000 [raft.persistence] -strategy = "MemFirst" -# strategy = "DiskFirst" flush_policy = { Batch = { idle_flush_interval_ms = 100 } } [raft.snapshot] diff --git a/examples/sled-cluster/config/n2.toml b/examples/sled-cluster/config/n2.toml index 87a5f29a..c7a89b70 100644 --- a/examples/sled-cluster/config/n2.toml +++ b/examples/sled-cluster/config/n2.toml @@ -16,8 +16,6 @@ batch_size = 5000 [raft.persistence] -strategy = "MemFirst" -# strategy = "DiskFirst" flush_policy = { Batch = { idle_flush_interval_ms = 100 } } [raft.snapshot] diff --git a/examples/sled-cluster/config/n3.toml b/examples/sled-cluster/config/n3.toml index 5fe5b628..c099227a 100644 --- a/examples/sled-cluster/config/n3.toml +++ b/examples/sled-cluster/config/n3.toml @@ -16,8 +16,6 @@ batch_size = 5000 [raft.persistence] -strategy = "MemFirst" -# strategy = "DiskFirst" flush_policy = { Batch = { idle_flush_interval_ms = 100 } } diff --git a/examples/sled-cluster/src/sled_storage_engine.rs b/examples/sled-cluster/src/sled_storage_engine.rs index 03948a64..1ecec903 100644 --- a/examples/sled-cluster/src/sled_storage_engine.rs +++ b/examples/sled-cluster/src/sled_storage_engine.rs @@ -147,7 +147,8 @@ impl LogStore for SledLogStore { &self, from_index: u64, new_entries: Vec, - ) -> Result<()> { + ) -> Result { + let new_last = new_entries.last().map(|e| e.index).unwrap_or(from_index.saturating_sub(1)); let mut batch = sled::Batch::default(); // collect and remove all keys >= from_index @@ -164,7 +165,7 @@ impl LogStore for SledLogStore { } self.tree.apply_batch(batch).map_err(|e| StorageError::DbError(e.to_string()))?; - Ok(()) + Ok(new_last) } fn is_write_durable(&self) -> bool { diff --git a/examples/three-nodes-embedded/README.md b/examples/three-nodes-embedded/README.md index 25255a59..b6f2dda2 100644 --- a/examples/three-nodes-embedded/README.md +++ b/examples/three-nodes-embedded/README.md @@ -84,7 +84,6 @@ default_policy = "LeaseRead" lease_duration_ms = 500 [raft.persistence] -strategy = "MemFirst" flush_policy = { Batch = { threshold = 100, interval_ms = 20 } } ``` diff --git a/examples/three-nodes-standalone/Makefile b/examples/three-nodes-standalone/Makefile index 81a14311..7f23d971 100644 --- a/examples/three-nodes-standalone/Makefile +++ b/examples/three-nodes-standalone/Makefile @@ -3,7 +3,8 @@ .PHONY: build start-cluster clean clean-log-db help \ start-node1 start-node2 start-node3 \ perf-node1 perf-node2 perf-node3 perf-cluster \ - tokio-console-node1 tokio-console-node2 tokio-console-node3 tokio-console-cluster + tokio-console-node1 tokio-console-node2 tokio-console-node3 tokio-console-cluster \ + start-ram-cluster ramdisk-create clean-ram-log-db ramdisk-release .DEFAULT_GOAL := help # =============================== @@ -43,10 +44,16 @@ build: # =============================== # Cluster Management (Normal Mode) # =============================== +# Overridable so start-ram-cluster can point these at /Volumes/RAMDiskN +# without duplicating the node targets. +DB_PATH_1 ?= ./db/1 +DB_PATH_2 ?= ./db/2 +DB_PATH_3 ?= ./db/3 + start-node1: @echo "πŸš€ Starting Node 1..." @CONFIG_PATH=config/n1 \ - DB_PATH="./db/1" \ + DB_PATH="$(DB_PATH_1)" \ LOG_DIR="./logs/1" \ METRICS_PORT=8081 \ RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ @@ -56,7 +63,7 @@ start-node1: start-node2: @echo "πŸš€ Starting Node 2..." @CONFIG_PATH=config/n2 \ - DB_PATH="./db/2" \ + DB_PATH="$(DB_PATH_2)" \ LOG_DIR="./logs/2" \ METRICS_PORT=8082 \ RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ @@ -65,7 +72,7 @@ start-node2: start-node3: @echo "πŸš€ Starting Node 3..." @CONFIG_PATH=config/n3 \ - DB_PATH="./db/3" \ + DB_PATH="$(DB_PATH_3)" \ LOG_DIR="./logs/3" \ METRICS_PORT=8083 \ RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ @@ -76,6 +83,40 @@ start-cluster: @echo "Starting 3-node cluster in parallel..." $(MAKE) -j3 start-node1 start-node2 start-node3 +# =============================== +# RAM Disk (macOS only, optional) +# =============================== +# Isolates each node's storage on its own independent RAM disk volume β€” use +# this to strip physical-disk latency out of a benchmark (e.g. to study +# fsync/scheduling behavior in isolation), not as a general fix for +# unrelated flakiness. See tickets/milestones/v0.2.5/446-perf-batching-measurement-2026-09-13.md. +RAMDISK_SIZE_MB ?= 1024 +RAMDISK_SECTORS := $(shell echo $$(( $(RAMDISK_SIZE_MB) * 2048 ))) +RAMDISK_VOLUMES := RAMDisk1 RAMDisk2 RAMDisk3 + +# One command: wipe stale node data, ensure the 3 RAM disks exist, start the +# cluster on them instead of ./db. +start-ram-cluster: clean-ram-log-db ramdisk-create + @echo "Starting 3-node cluster on RAM disk..." + $(MAKE) -j3 start-node1 start-node2 start-node3 \ + DB_PATH_1=/Volumes/RAMDisk1/n1 DB_PATH_2=/Volumes/RAMDisk2/n2 DB_PATH_3=/Volumes/RAMDisk3/n3 + +# Idempotent β€” mounts any of RAMDisk1/2/3 that aren't already present. +ramdisk-create: + @[ "$$(uname)" = "Darwin" ] || { echo "RAM disk targets need macOS."; exit 1; } + @for v in $(RAMDISK_VOLUMES); do \ + [ -d "/Volumes/$$v" ] || diskutil erasevolume HFS+ $$v `hdiutil attach -nomount ram://$(RAMDISK_SECTORS)` >/dev/null; \ + done + +# Wipes node data off the RAM disks; keeps the volumes mounted so the next +# start-ram-cluster doesn't pay to recreate them. +clean-ram-log-db: + @for v in $(RAMDISK_VOLUMES); do rm -rf /Volumes/$$v/*; done + +# Unmounts the RAM disks entirely, releasing the memory back to the OS. +ramdisk-release: + @for v in $(RAMDISK_VOLUMES); do diskutil eject /Volumes/$$v 2>/dev/null || true; done + # =============================== # Performance Profiling with Samply @@ -184,6 +225,10 @@ help: @echo " perf-node1..3 - Run individual nodes under samply profiler" @echo " perf-cluster - Run full 3-node cluster under samply profiling" @echo " tokio-console-cluster - Run full 3-node cluster under tokio console monitoring" + @echo " start-ram-cluster - (macOS) Clean+create RAM disks, start cluster on them" + @echo " ramdisk-create - (macOS) Create RAMDisk1/2/3 if not already mounted" + @echo " clean-ram-log-db - Wipe node data off the RAM disks (keeps them mounted)" + @echo " ramdisk-release - Unmount RAMDisk1/2/3, freeing the memory" @echo " clean - Remove build artifacts, logs, and profiles" @echo " clean-log-db - Remove only logs and database files" @echo " help - Show this help message" diff --git a/examples/three-nodes-standalone/config/n1.toml b/examples/three-nodes-standalone/config/n1.toml index a040e1ae..431f4e09 100644 --- a/examples/three-nodes-standalone/config/n1.toml +++ b/examples/three-nodes-standalone/config/n1.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,15 +39,10 @@ max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) -max_buffered_entries = 10000 [raft.snapshot] -enable = true +enable = false max_log_entries_before_snapshot = 5000 retained_log_entries = 100 cleanup_retain_count = 100 diff --git a/examples/three-nodes-standalone/config/n2.toml b/examples/three-nodes-standalone/config/n2.toml index 20e816f2..0d100281 100644 --- a/examples/three-nodes-standalone/config/n2.toml +++ b/examples/three-nodes-standalone/config/n2.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,15 +39,10 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) -max_buffered_entries = 10000 [raft.snapshot] -enable = true +enable = false max_log_entries_before_snapshot = 5000 retained_log_entries = 100 cleanup_retain_count = 100 diff --git a/examples/three-nodes-standalone/config/n3.toml b/examples/three-nodes-standalone/config/n3.toml index 0a29fe2d..5e3d80d2 100644 --- a/examples/three-nodes-standalone/config/n3.toml +++ b/examples/three-nodes-standalone/config/n3.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,15 +39,10 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) -max_buffered_entries = 10000 [raft.snapshot] -enable = true +enable = false max_log_entries_before_snapshot = 5000 retained_log_entries = 100 cleanup_retain_count = 100 diff --git a/examples/three-nodes-standalone/docker/Dockerfile b/examples/three-nodes-standalone/docker/Dockerfile index df52fc4b..e971495e 100644 --- a/examples/three-nodes-standalone/docker/Dockerfile +++ b/examples/three-nodes-standalone/docker/Dockerfile @@ -55,6 +55,8 @@ RUN apt-get update && \ iptables \ iproute2 \ libc6 \ + libfuse3-3 \ + fuse3 \ tzdata && \ rm -rf /var/lib/apt/lists/* && \ mkdir -p /var/run/sshd && \ @@ -85,4 +87,4 @@ COPY examples/three-nodes-standalone/docker/monitoring/promtail/config.yml /etc/ WORKDIR /app -CMD ["sh", "-c", "/usr/sbin/sshd -D & CONFIG_PATH=$CONFIG_PATH LOG_DIR=$LOG_DIR METRICS_PORT=$METRICS_PORT RUST_LOG=demo=$LOG_LEVEL,d_engine=$LOG_LEVEL,hyper=warn,sled=warn demo & promtail --config.file=/etc/promtail/config.yml > /app/logs/promtail.log 2>&1"] +CMD ["sh", "-c", "/usr/sbin/sshd -D & CONFIG_PATH=$CONFIG_PATH LOG_DIR=$LOG_DIR METRICS_PORT=$METRICS_PORT DB_PATH=/app/db/$ID RUST_LOG=demo=$LOG_LEVEL,d_engine=$LOG_LEVEL,hyper=warn,sled=warn demo & promtail --config.file=/etc/promtail/config.yml > /app/logs/promtail.log 2>&1"] diff --git a/examples/three-nodes-standalone/docker/config/n1.toml b/examples/three-nodes-standalone/docker/config/n1.toml index efa8e941..a5b007dc 100644 --- a/examples/three-nodes-standalone/docker/config/n1.toml +++ b/examples/three-nodes-standalone/docker/config/n1.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,12 +39,7 @@ max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/three-nodes-standalone/docker/config/n2.toml b/examples/three-nodes-standalone/docker/config/n2.toml index 78f66899..06c54f50 100644 --- a/examples/three-nodes-standalone/docker/config/n2.toml +++ b/examples/three-nodes-standalone/docker/config/n2.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,12 +39,7 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/three-nodes-standalone/docker/config/n3.toml b/examples/three-nodes-standalone/docker/config/n3.toml index 674109a2..2cfe4ad9 100644 --- a/examples/three-nodes-standalone/docker/config/n3.toml +++ b/examples/three-nodes-standalone/docker/config/n3.toml @@ -11,7 +11,7 @@ initial_cluster = [ [raft] general_raft_timeout_duration_in_ms = 100 cmd_channel_capacity = 1024 -ordered_channel_capacity = 1024 +max_pending_append_responses = 1024 [raft.election] election_timeout_min = 1000 @@ -39,12 +39,7 @@ max_pending_writes = 1000 max_pending_reads = 500 [raft.persistence] -# strategy = "DiskFirst" -strategy = "MemFirst" flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } -# Maximum number of log entries to buffer in memory -# when using async persistence strategies (MemFirst/Batched) -max_buffered_entries = 10000 [raft.snapshot] enable = false diff --git a/examples/three-nodes-standalone/src/main.rs b/examples/three-nodes-standalone/src/main.rs index 4a0fd403..c61dc79e 100644 --- a/examples/three-nodes-standalone/src/main.rs +++ b/examples/three-nodes-standalone/src/main.rs @@ -58,8 +58,11 @@ async fn main() { let (graceful_tx, graceful_rx) = watch::channel(()); // Start the server (wait for its initialization to complete) - let server_handler = - tokio::spawn(start_dengine_server(data_dir, config_path, graceful_rx.clone())); + let server_handler = tokio::spawn(start_dengine_server( + data_dir, + config_path, + graceful_rx.clone(), + )); // Wait for the server to initialize (adjust the waiting time according to the actual logic) tokio::time::sleep(Duration::from_secs(1)).await;