From 5e7b3dd946d19a82299a325c9c7bb93a3ceb2c19 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 20 Oct 2026 19:53:38 +0000 Subject: [PATCH] Let a MySQL node wait out another node's bootstrap instead of refusing MySQL commits DDL on its own, so between one node creating dw_server_schema and recording its rust-initializing marker, a second node's read-only preflight saw an empty marker table and refused with unsupported_rust_schema. A node killed in that window left a database no node could start on. Found when three nodes started together in the multi-node harness. Preflight now accepts an empty marker table when it is the only table, so the node waits for the bootstrap lock; under the lock a missing marker is recorded before the migration runs. The kill qualification gains a marker-table phase in which a concurrent start must wait for the lock and then complete. Co-Authored-By: Claude Opus 5.5 --- docs/rust-server-port.md | 21 ++++++++++++++++++++ rust/src/runtime/mysql.rs | 40 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/docs/rust-server-port.md b/docs/rust-server-port.md index ac964afa9..f5981264a 100644 --- a/docs/rust-server-port.md +++ b/docs/rust-server-port.md @@ -142,6 +142,27 @@ support is not evidence of multi-node safety. Newest checkpoints first. Each entry is exact to the source, tree and runs it names, including unsuccessful observations. +October 20, concurrent MySQL and MariaDB bootstrap: + +- **Found by.** A multi-node smoke run started three nodes together on an + empty MySQL database. One node exited with `409 unsupported_rust_schema`. +- **Cause.** MySQL commits DDL on its own. Between one node creating + `dw_server_schema` and recording its `rust-initializing` marker, another + node's read-only preflight, which runs before the bootstrap lock, saw an + empty marker table and refused. A node killed in the same window left a + database that no node could start on. PostgreSQL bootstraps inside one + locked transaction and SQLite in one file transaction, so neither has the + window. +- **Source.** Preflight accepts an empty marker table when it is the only + table, and the node then waits for the bootstrap lock. Under the lock, a + missing marker is recorded before the migration runs. +- **Checks.** The kill qualification has a `marker-table` phase. It kills a + bootstrapping child right after it creates the marker table and holds the + bootstrap lock. A concurrent start must still be waiting after one second, + then complete once the child is killed. Without the preflight change, it + refused at once. All 15 ignored MySQL qualification tests passed on MySQL + 8.0 and MariaDB 10.11 locally. + October 19, before #478, #476, #494 and #460 merged: - **Open native PRs.** [#478](https://github.com/durable-workflow/server/pull/478) diff --git a/rust/src/runtime/mysql.rs b/rust/src/runtime/mysql.rs index a4a76dbf5..96175f6cb 100644 --- a/rust/src/runtime/mysql.rs +++ b/rust/src/runtime/mysql.rs @@ -448,6 +448,11 @@ async fn preflight(connection: &mut MySqlConnection, flavor: Flavor) -> Result<( let engines: Vec = sqlx::query_scalar("SELECT engine FROM dw_server_schema") .fetch_all(&mut *connection) .await?; + // Another node is between creating the marker table and recording its + // marker (DDL commits on its own). Its bootstrap lock decides the rest. + if engines.is_empty() && tables == ["dw_server_schema"] { + return Ok(()); + } if engines == ["rust-initializing"] { verify_initializing(connection, flavor).await?; Ok(()) @@ -518,6 +523,12 @@ impl MySqlStorage { preflight(&mut connection,flavor).await?; if scope(&mut connection).await?.is_empty() { connection.execute(MARKER_SQL).await?; + } + // A node killed between creating the marker table and recording its + // marker leaves the table empty. Preflight accepted that state only + // when nothing else exists yet. + let marked:i64=sqlx::query_scalar("SELECT COUNT(*) FROM dw_server_schema").fetch_one(&mut connection).await?; + if marked==0 { sqlx::query("INSERT INTO dw_server_schema(engine,version,bootstrap_checksum) VALUES ('rust-initializing',?,?)").bind(VERSION).bind(flavor.checksum()).execute(&mut connection).await?; } let engine:String=sqlx::query_scalar("SELECT engine FROM dw_server_schema").fetch_one(&mut connection).await?; @@ -1010,7 +1021,12 @@ mod tests { .unwrap(); assert_eq!(lock, Some(1)); let flavor = Flavor::detect(&mut connection).await.unwrap(); - initializing(&mut connection, flavor).await; + if phase == "marker-table" { + // Killed after the marker table is created, before its marker. + connection.execute(MARKER_SQL).await.unwrap(); + } else { + initializing(&mut connection, flavor).await; + } if phase == "dirty-partial" { dirty(&mut connection, flavor).await; partial(&mut connection, flavor).await; @@ -1034,7 +1050,7 @@ mod tests { #[ignore = "requires disposable MySQL/MariaDB; run qualification_ explicitly"] async fn qualification_real_kill_resumes_only_recognized_initialization() { use std::os::unix::process::ExitStatusExt; - for phase in ["marker", "dirty-partial", "before-marker"] { + for phase in ["marker-table", "marker", "dirty-partial", "before-marker"] { let db = Database::new().await; let directory = tempfile::tempdir().unwrap(); let receipt = directory.path().join("boundary"); @@ -1063,6 +1079,26 @@ mod tests { panic!("child did not reach {phase}"); } assert_eq!(std::fs::read_to_string(receipt).unwrap(), phase); + if phase == "marker-table" { + // A node starting now sees only the empty marker table. It must + // wait for the holder's bootstrap lock rather than refuse. + let open = MySqlStorage::open(db.options.clone()); + tokio::pin!(open); + tokio::select! { + result = &mut open => panic!( + "a concurrent start finished while the bootstrap lock was held (ok: {})", + result.is_ok() + ), + _ = tokio::time::sleep(Duration::from_secs(1)) => {} + } + child.kill().unwrap(); + assert_eq!(child.wait().unwrap().signal(), Some(9)); + let storage = open.await.unwrap(); + storage.close().await; + MySqlStorage::check(db.options.clone()).await.unwrap(); + db.remove().await; + continue; + } child.kill().unwrap(); assert_eq!(child.wait().unwrap().signal(), Some(9)); let mut connection = db.connection().await;